guppy_basecaller: env/lib/python3.7/site-packages/cwltool/process.py annotate

annotate env/lib/python3.7/site-packages/cwltool/process.py @ 2:6af9afd405e9 draft

"planemo upload commit 0a63dd5f4d38a1f6944587f52a8cd79874177fc1"

author	shellac
date	Thu, 14 May 2020 14:56:58 -0400
parents	26e78fe6e8c4
children

rev	line source
0 26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1 from __future__ import absolute_import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	2
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	3 import abc
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	4 import copy
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	5 import errno
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	6 import functools
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	7 import hashlib
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	8 import json
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	9 import logging
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	10 import os
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	11 import shutil
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	12 import stat
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	13 import tempfile
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	14 import textwrap
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	15 import uuid
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	16
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	17 from io import open
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	18 from typing import (Any, Callable, Dict, Generator, Iterator, List,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	19 Mapping, MutableMapping, MutableSequence, Optional, Set, Tuple,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	20 Type, Union, cast)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	21
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	22 from pkg_resources import resource_stream
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	23 from rdflib import Graph # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	24 from ruamel.yaml.comments import CommentedMap, CommentedSeq
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	25 from six import PY3, iteritems, itervalues, string_types, with_metaclass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	26 from six.moves import urllib
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	27 from future.utils import raise_from
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	28 from typing_extensions import (TYPE_CHECKING, # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	29 Text)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	30 from schema_salad import schema, validate
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	31 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	32 from schema_salad.sourceline import SourceLine, strip_dup_lineno
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	33
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	34 from . import expression
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	35 from .builder import Builder, HasReqsHints
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	36 from .context import LoadingContext # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	37 from .context import RuntimeContext, getdefault
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	38 from .errors import UnsupportedRequirement, WorkflowException
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	39 from .loghandler import _logger
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	40 from .mutation import MutationManager # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	41 from .pathmapper import (PathMapper, adjustDirObjs, ensure_writable,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	42 get_listing, normalizeFilesDirs, visit_class,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	43 MapperEnt)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	44 from .secrets import SecretStore # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	45 from .software_requirements import ( # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	46 DependenciesConfiguration)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	47 from .stdfsaccess import StdFsAccess
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	48 from .utils import (DEFAULT_TMP_PREFIX, aslist, cmp_like_py2,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	49 copytree_with_merge, onWindows, random_outdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	50 from .validate_js import validate_js_expressions
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	51 from .update import INTERNAL_VERSION
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	52
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	53 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	54 from os import scandir # type: ignore
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	55 except ImportError:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	56 from scandir import scandir # type: ignore
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	57 if TYPE_CHECKING:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	58 from .provenance import ProvenanceProfile # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	59
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	60 if PY3:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	61 from collections.abc import Iterable # only works on python 3.3+
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	62 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	63 from collections import Iterable # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	64
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	65 class LogAsDebugFilter(logging.Filter):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	66 def __init__(self, name, parent): # type: (Text, logging.Logger) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	67 """Initialize."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	68 name = str(name)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	69 super(LogAsDebugFilter, self).__init__(name)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	70 self.parent = parent
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	71
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	72 def filter(self, record): # type: (logging.LogRecord) -> bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	73 return self.parent.isEnabledFor(logging.DEBUG)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	74
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	75
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	76 _logger_validation_warnings = logging.getLogger("cwltool.validation_warnings")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	77 _logger_validation_warnings.setLevel(_logger.getEffectiveLevel())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	78 _logger_validation_warnings.addFilter(LogAsDebugFilter("cwltool.validation_warnings", _logger))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	79
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	80 supportedProcessRequirements = ["DockerRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	81 "SchemaDefRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	82 "EnvVarRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	83 "ScatterFeatureRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	84 "SubworkflowFeatureRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	85 "MultipleInputFeatureRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	86 "InlineJavascriptRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	87 "ShellCommandRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	88 "StepInputExpressionRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	89 "ResourceRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	90 "InitialWorkDirRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	91 "ToolTimeLimit",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	92 "WorkReuse",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	93 "NetworkAccess",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	94 "InplaceUpdateRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	95 "LoadListingRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	96 "http://commonwl.org/cwltool#TimeLimit",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	97 "http://commonwl.org/cwltool#WorkReuse",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	98 "http://commonwl.org/cwltool#NetworkAccess",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	99 "http://commonwl.org/cwltool#LoadListingRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	100 "http://commonwl.org/cwltool#InplaceUpdateRequirement"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	101
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	102 cwl_files = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	103 "Workflow.yml",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	104 "CommandLineTool.yml",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	105 "CommonWorkflowLanguage.yml",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	106 "Process.yml",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	107 "concepts.md",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	108 "contrib.md",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	109 "intro.md",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	110 "invocation.md")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	111
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	112 salad_files = ('metaschema.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	113 'metaschema_base.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	114 'salad.md',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	115 'field_name.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	116 'import_include.md',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	117 'link_res.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	118 'ident_res.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	119 'vocab_res.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	120 'vocab_res.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	121 'field_name_schema.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	122 'field_name_src.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	123 'field_name_proc.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	124 'ident_res_schema.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	125 'ident_res_src.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	126 'ident_res_proc.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	127 'link_res_schema.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	128 'link_res_src.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	129 'link_res_proc.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	130 'vocab_res_schema.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	131 'vocab_res_src.yml',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	132 'vocab_res_proc.yml')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	133
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	134 SCHEMA_CACHE = {} # type: Dict[Text, Tuple[Loader, Union[schema.Names, schema.SchemaParseException], Dict[Text, Any], Loader]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	135 SCHEMA_FILE = None # type: Optional[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	136 SCHEMA_DIR = None # type: Optional[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	137 SCHEMA_ANY = None # type: Optional[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	138
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	139 custom_schemas = {} # type: Dict[Text, Tuple[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	140
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	141 def use_standard_schema(version):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	142 # type: (Text) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	143 if version in custom_schemas:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	144 del custom_schemas[version]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	145 if version in SCHEMA_CACHE:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	146 del SCHEMA_CACHE[version]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	147
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	148 def use_custom_schema(version, name, text):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	149 # type: (Text, Text, Union[Text, bytes]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	150 if isinstance(text, bytes):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	151 text2 = text.decode()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	152 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	153 text2 = text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	154 custom_schemas[version] = (name, text2)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	155 if version in SCHEMA_CACHE:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	156 del SCHEMA_CACHE[version]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	157
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	158 def get_schema(version):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	159 # type: (Text) -> Tuple[Loader, Union[schema.Names, schema.SchemaParseException], Dict[Text,Any], Loader]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	160
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	161 if version in SCHEMA_CACHE:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	162 return SCHEMA_CACHE[version]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	163
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	164 cache = {} # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	165 version = version.split("#")[-1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	166 if '.dev' in version:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	167 version = ".".join(version.split(".")[:-1])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	168 for f in cwl_files:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	169 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	170 res = resource_stream(__name__, 'schemas/%s/%s' % (version, f))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	171 cache["https://w3id.org/cwl/" + f] = res.read()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	172 res.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	173 except IOError:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	174 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	175
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	176 for f in salad_files:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	177 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	178 res = resource_stream(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	179 __name__, 'schemas/{}/salad/schema_salad/metaschema/{}'.format(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	180 version, f))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	181 cache["https://w3id.org/cwl/salad/schema_salad/metaschema/"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	182 + f] = res.read()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	183 res.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	184 except IOError:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	185 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	186
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	187 if version in custom_schemas:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	188 cache[custom_schemas[version][0]] = custom_schemas[version][1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	189 SCHEMA_CACHE[version] = schema.load_schema(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	190 custom_schemas[version][0], cache=cache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	191 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	192 SCHEMA_CACHE[version] = schema.load_schema(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	193 "https://w3id.org/cwl/CommonWorkflowLanguage.yml", cache=cache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	194
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	195 return SCHEMA_CACHE[version]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	196
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	197
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	198 def shortname(inputid):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	199 # type: (Text) -> Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	200 d = urllib.parse.urlparse(inputid)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	201 if d.fragment:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	202 return d.fragment.split(u"/")[-1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	203 return d.path.split(u"/")[-1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	204
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	205
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	206 def checkRequirements(rec, supported_process_requirements):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	207 # type: (Any, Iterable[Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	208 if isinstance(rec, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	209 if "requirements" in rec:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	210 for i, entry in enumerate(rec["requirements"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	211 with SourceLine(rec["requirements"], i, UnsupportedRequirement):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	212 if entry["class"] not in supported_process_requirements:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	213 raise UnsupportedRequirement(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	214 u"Unsupported requirement {}".format(entry["class"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	215 for key in rec:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	216 checkRequirements(rec[key], supported_process_requirements)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	217 if isinstance(rec, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	218 for entry in rec:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	219 checkRequirements(entry, supported_process_requirements)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	220
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	221
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	222 def stage_files(pathmapper, # type: PathMapper
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	223 stage_func=None, # type: Optional[Callable[..., Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	224 ignore_writable=False, # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	225 symlink=True, # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	226 secret_store=None, # type: Optional[SecretStore]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	227 fix_conflicts=False # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	228 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	229 """Link or copy files to their targets. Create them as needed."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	230
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	231 targets = {} # type: Dict[Text, MapperEnt]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	232 for key, entry in pathmapper.items():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	233 if not 'File' in entry.type:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	234 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	235 if entry.target not in targets:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	236 targets[entry.target] = entry
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	237 elif targets[entry.target].resolved != entry.resolved:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	238 if fix_conflicts:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	239 tgt = entry.target
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	240 i = 2
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	241 tgt = "%s_%s" % (tgt, i)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	242 while tgt in targets:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	243 i += 1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	244 tgt = "%s_%s" % (tgt, i)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	245 targets[tgt] = pathmapper.update(key, entry.resolved, tgt, entry.type, entry.staged)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	246 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	247 raise WorkflowException("File staging conflict, trying to stage both %s and %s to the same target %s" % (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	248 targets[entry.target].resolved, entry.resolved, entry.target))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	249
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	250 for key, entry in pathmapper.items():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	251 if not entry.staged:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	252 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	253 if not os.path.exists(os.path.dirname(entry.target)):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	254 os.makedirs(os.path.dirname(entry.target))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	255 if entry.type in ("File", "Directory") and os.path.exists(entry.resolved):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	256 if symlink: # Use symlink func if allowed
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	257 if onWindows():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	258 if entry.type == "File":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	259 shutil.copy(entry.resolved, entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	260 elif entry.type == "Directory":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	261 if os.path.exists(entry.target) \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	262 and os.path.isdir(entry.target):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	263 shutil.rmtree(entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	264 copytree_with_merge(entry.resolved, entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	265 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	266 os.symlink(entry.resolved, entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	267 elif stage_func is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	268 stage_func(entry.resolved, entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	269 elif entry.type == "Directory" and not os.path.exists(entry.target) \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	270 and entry.resolved.startswith("_:"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	271 os.makedirs(entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	272 elif entry.type == "WritableFile" and not ignore_writable:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	273 shutil.copy(entry.resolved, entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	274 ensure_writable(entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	275 elif entry.type == "WritableDirectory" and not ignore_writable:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	276 if entry.resolved.startswith("_:"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	277 os.makedirs(entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	278 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	279 shutil.copytree(entry.resolved, entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	280 ensure_writable(entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	281 elif entry.type == "CreateFile" or entry.type == "CreateWritableFile":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	282 with open(entry.target, "wb") as new:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	283 if secret_store is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	284 new.write(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	285 secret_store.retrieve(entry.resolved).encode("utf-8"))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	286 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	287 new.write(entry.resolved.encode("utf-8"))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	288 if entry.type == "CreateFile":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	289 os.chmod(entry.target, stat.S_IRUSR) # Read only
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	290 else: # it is a "CreateWritableFile"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	291 ensure_writable(entry.target)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	292 pathmapper.update(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	293 key, entry.target, entry.target, entry.type, entry.staged)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	294
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	295
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	296 def relocateOutputs(outputObj, # type: Union[Dict[Text, Any], List[Dict[Text, Any]]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	297 destination_path, # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	298 source_directories, # type: Set[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	299 action, # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	300 fs_access, # type: StdFsAccess
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	301 compute_checksum=True, # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	302 path_mapper=PathMapper # type: Type[PathMapper]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	303 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	304 # type: (...) -> Union[Dict[Text, Any], List[Dict[Text, Any]]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	305 adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	306
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	307 if action not in ("move", "copy"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	308 return outputObj
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	309
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	310 def _collectDirEntries(obj):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	311 # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> Iterator[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	312 if isinstance(obj, dict):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	313 if obj.get("class") in ("File", "Directory"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	314 yield obj
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	315 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	316 for sub_obj in obj.values():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	317 for dir_entry in _collectDirEntries(sub_obj):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	318 yield dir_entry
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	319 elif isinstance(obj, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	320 for sub_obj in obj:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	321 for dir_entry in _collectDirEntries(sub_obj):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	322 yield dir_entry
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	323
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	324 def _relocate(src, dst): # type: (Text, Text) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	325 if src == dst:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	326 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	327
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	328 # If the source is not contained in source_directories we're not allowed to delete it
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	329 src = fs_access.realpath(src)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	330 src_can_deleted = any(os.path.commonprefix([p, src]) == p for p in source_directories)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	331
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	332 _action = "move" if action == "move" and src_can_deleted else "copy"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	333
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	334 if _action == "move":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	335 _logger.debug("Moving %s to %s", src, dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	336 if fs_access.isdir(src) and fs_access.isdir(dst):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	337 # merge directories
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	338 for dir_entry in scandir(src):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	339 _relocate(dir_entry.path, fs_access.join(dst, dir_entry.name))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	340 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	341 shutil.move(src, dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	342
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	343 elif _action == "copy":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	344 _logger.debug("Copying %s to %s", src, dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	345 if fs_access.isdir(src):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	346 if os.path.isdir(dst):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	347 shutil.rmtree(dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	348 elif os.path.isfile(dst):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	349 os.unlink(dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	350 shutil.copytree(src, dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	351 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	352 shutil.copy2(src, dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	353
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	354 def _realpath(ob): # type: (Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	355 if ob["location"].startswith("file:"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	356 ob["location"] = file_uri(os.path.realpath(uri_file_path(ob["location"])))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	357 if ob["location"].startswith("/"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	358 ob["location"] = os.path.realpath(ob["location"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	359
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	360 outfiles = list(_collectDirEntries(outputObj))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	361 visit_class(outfiles, ("File", "Directory"), _realpath)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	362 pm = path_mapper(outfiles, "", destination_path, separateDirs=False)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	363 stage_files(pm, stage_func=_relocate, symlink=False, fix_conflicts=True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	364
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	365 def _check_adjust(a_file): # type: (Dict[Text, Text]) -> Dict[Text, Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	366 a_file["location"] = file_uri(pm.mapper(a_file["location"])[1])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	367 if "contents" in a_file:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	368 del a_file["contents"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	369 return a_file
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	370
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	371 visit_class(outputObj, ("File", "Directory"), _check_adjust)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	372
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	373 if compute_checksum:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	374 visit_class(outputObj, ("File",), functools.partial(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	375 compute_checksums, fs_access))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	376 return outputObj
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	377
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	378
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	379 def cleanIntermediate(output_dirs): # type: (Iterable[Text]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	380 for a in output_dirs:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	381 if os.path.exists(a):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	382 _logger.debug(u"Removing intermediate output directory %s", a)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	383 shutil.rmtree(a, True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	384
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	385 def add_sizes(fsaccess, obj): # type: (StdFsAccess, Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	386 if 'location' in obj:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	387 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	388 if "size" not in obj:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	389 obj["size"] = fsaccess.size(obj["location"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	390 except OSError:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	391 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	392 elif 'contents' in obj:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	393 obj["size"] = len(obj['contents'])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	394 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	395 return # best effort
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	396
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	397 def fill_in_defaults(inputs, # type: List[Dict[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	398 job, # type: Dict[Text, expression.JSON]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	399 fsaccess # type: StdFsAccess
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	400 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	401 for e, inp in enumerate(inputs):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	402 with SourceLine(inputs, e, WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	403 fieldname = shortname(inp[u"id"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	404 if job.get(fieldname) is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	405 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	406 elif job.get(fieldname) is None and u"default" in inp:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	407 job[fieldname] = copy.deepcopy(inp[u"default"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	408 elif job.get(fieldname) is None and u"null" in aslist(inp[u"type"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	409 job[fieldname] = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	410 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	411 raise WorkflowException("Missing required input parameter '%s'" % shortname(inp["id"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	412
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	413
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	414 def avroize_type(field_type, name_prefix=""):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	415 # type: (Union[List[Dict[Text, Any]], Dict[Text, Any]], Text) -> Any
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	416 """Add missing information to a type so that CWL types are valid."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	417 if isinstance(field_type, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	418 for field in field_type:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	419 avroize_type(field, name_prefix)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	420 elif isinstance(field_type, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	421 if field_type["type"] in ("enum", "record"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	422 if "name" not in field_type:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	423 field_type["name"] = name_prefix + Text(uuid.uuid4())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	424 if field_type["type"] == "record":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	425 avroize_type(field_type["fields"], name_prefix)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	426 if field_type["type"] == "array":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	427 avroize_type(field_type["items"], name_prefix)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	428 if isinstance(field_type["type"], MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	429 for ctype in field_type["type"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	430 avroize_type(ctype, name_prefix)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	431 return field_type
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	432
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	433 def get_overrides(overrides, toolid): # type: (List[Dict[Text, Any]], Text) -> Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	434 req = {} # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	435 if not isinstance(overrides, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	436 raise validate.ValidationException("Expected overrides to be a list, but was %s" % type(overrides))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	437 for ov in overrides:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	438 if ov["overrideTarget"] == toolid:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	439 req.update(ov)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	440 return req
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	441
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	442
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	443 _VAR_SPOOL_ERROR = textwrap.dedent(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	444 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	445 Non-portable reference to /var/spool/cwl detected: '{}'.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	446 To fix, replace /var/spool/cwl with $(runtime.outdir) or add
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	447 DockerRequirement to the 'requirements' section and declare
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	448 'dockerOutputDirectory: /var/spool/cwl'.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	449 """)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	450
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	451
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	452 def var_spool_cwl_detector(obj, # type: Union[MutableMapping[Text, Text], List[Dict[Text, Any]], Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	453 item=None, # type: Optional[Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	454 obj_key=None, # type: Optional[Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	455 ): # type: (...)->bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	456 """Detect any textual reference to /var/spool/cwl."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	457 r = False
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	458 if isinstance(obj, string_types):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	459 if "var/spool/cwl" in obj and obj_key != "dockerOutputDirectory":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	460 _logger.warning(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	461 SourceLine(item=item, key=obj_key, raise_type=Text).makeError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	462 _VAR_SPOOL_ERROR.format(obj)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	463 r = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	464 elif isinstance(obj, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	465 for mkey, mvalue in iteritems(obj):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	466 r = var_spool_cwl_detector(mvalue, obj, mkey) or r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	467 elif isinstance(obj, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	468 for lkey, lvalue in enumerate(obj):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	469 r = var_spool_cwl_detector(lvalue, obj, lkey) or r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	470 return r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	471
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	472 def eval_resource(builder, resource_req): # type: (Builder, Text) -> Any
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	473 if expression.needs_parsing(resource_req):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	474 return builder.do_eval(resource_req)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	475 return resource_req
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	476
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	477
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	478 # Threshold where the "too many files" warning kicks in
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	479 FILE_COUNT_WARNING = 5000
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	480
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	481 class Process(with_metaclass(abc.ABCMeta, HasReqsHints)):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	482 def __init__(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	483 toolpath_object, # type: MutableMapping[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	484 loadingContext # type: LoadingContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	485 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	486 """Build a Process object from the provided dictionary."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	487 self.metadata = getdefault(loadingContext.metadata, {}) # type: Dict[Text,Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	488 self.provenance_object = None # type: Optional[ProvenanceProfile]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	489 self.parent_wf = None # type: Optional[ProvenanceProfile]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	490 global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY # pylint: disable=global-statement
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	491 if SCHEMA_FILE is None or SCHEMA_ANY is None or SCHEMA_DIR is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	492 get_schema("v1.0")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	493 SCHEMA_ANY = cast(Dict[Text, Any],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	494 SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/salad#Any"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	495 SCHEMA_FILE = cast(Dict[Text, Any],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	496 SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#File"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	497 SCHEMA_DIR = cast(Dict[Text, Any],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	498 SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#Directory"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	499
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	500 self.names = schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	501 Loader({}))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	502 self.tool = toolpath_object
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	503 self.requirements = copy.deepcopy(getdefault(loadingContext.requirements, []))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	504 self.requirements.extend(self.tool.get("requirements", []))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	505 if "id" not in self.tool:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	506 self.tool["id"] = "_:" + Text(uuid.uuid4())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	507 self.requirements.extend(get_overrides(getdefault(loadingContext.overrides_list, []),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	508 self.tool["id"]).get("requirements", []))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	509 self.hints = copy.deepcopy(getdefault(loadingContext.hints, []))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	510 self.hints.extend(self.tool.get("hints", []))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	511 # Versions of requirements and hints which aren't mutated.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	512 self.original_requirements = copy.deepcopy(self.requirements)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	513 self.original_hints = copy.deepcopy(self.hints)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	514 self.doc_loader = loadingContext.loader
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	515 self.doc_schema = loadingContext.avsc_names
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	516
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	517 self.formatgraph = None # type: Optional[Graph]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	518 if self.doc_loader is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	519 self.formatgraph = self.doc_loader.graph
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	520
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	521 checkRequirements(self.tool, supportedProcessRequirements)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	522 self.validate_hints(loadingContext.avsc_names, self.tool.get("hints", []),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	523 strict=getdefault(loadingContext.strict, False))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	524
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	525 self.schemaDefs = {} # type: Dict[Text,Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	526
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	527 sd, _ = self.get_requirement("SchemaDefRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	528
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	529 if sd is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	530 sdtypes = avroize_type(sd["types"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	531 av = schema.make_valid_avro(sdtypes, {t["name"]: t for t in sdtypes}, set())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	532 for i in av:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	533 self.schemaDefs[i["name"]] = i # type: ignore
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	534 schema.make_avsc_object(schema.convert_to_dict(av), self.names)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	535
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	536 # Build record schema from inputs
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	537 self.inputs_record_schema = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	538 "name": "input_record_schema", "type": "record",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	539 "fields": []} # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	540 self.outputs_record_schema = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	541 "name": "outputs_record_schema", "type": "record",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	542 "fields": []} # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	543
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	544 for key in ("inputs", "outputs"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	545 for i in self.tool[key]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	546 c = copy.deepcopy(i)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	547 c["name"] = shortname(c["id"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	548 del c["id"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	549
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	550 if "type" not in c:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	551 raise validate.ValidationException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	552 u"Missing 'type' in parameter '{}'".format(c["name"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	553
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	554 if "default" in c and "null" not in aslist(c["type"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	555 nullable = ["null"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	556 nullable.extend(aslist(c["type"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	557 c["type"] = nullable
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	558 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	559 c["type"] = c["type"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	560 c["type"] = avroize_type(c["type"], c["name"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	561 if key == "inputs":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	562 self.inputs_record_schema["fields"].append(c)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	563 elif key == "outputs":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	564 self.outputs_record_schema["fields"].append(c)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	565
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	566 with SourceLine(toolpath_object, "inputs", validate.ValidationException):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	567 self.inputs_record_schema = cast(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	568 Dict[Text, Any], schema.make_valid_avro(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	569 self.inputs_record_schema, {}, set()))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	570 schema.make_avsc_object(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	571 schema.convert_to_dict(self.inputs_record_schema), self.names)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	572 with SourceLine(toolpath_object, "outputs", validate.ValidationException):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	573 self.outputs_record_schema = cast(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	574 Dict[Text, Any],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	575 schema.make_valid_avro(self.outputs_record_schema, {}, set()))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	576 schema.make_avsc_object(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	577 schema.convert_to_dict(self.outputs_record_schema), self.names)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	578
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	579 if toolpath_object.get("class") is not None \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	580 and not getdefault(loadingContext.disable_js_validation, False):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	581 if loadingContext.js_hint_options_file is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	582 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	583 with open(loadingContext.js_hint_options_file) as options_file:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	584 validate_js_options = json.load(options_file)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	585 except (OSError, ValueError) as err:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	586 _logger.error(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	587 "Failed to read options file %s",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	588 loadingContext.js_hint_options_file)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	589 raise
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	590 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	591 validate_js_options = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	592 if self.doc_schema is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	593 validate_js_expressions(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	594 cast(CommentedMap, toolpath_object),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	595 self.doc_schema.names[toolpath_object["class"]],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	596 validate_js_options)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	597
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	598 dockerReq, is_req = self.get_requirement("DockerRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	599
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	600 if dockerReq is not None and "dockerOutputDirectory" in dockerReq\
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	601 and is_req is not None and not is_req:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	602 _logger.warning(SourceLine(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	603 item=dockerReq, raise_type=Text).makeError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	604 "When 'dockerOutputDirectory' is declared, DockerRequirement "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	605 "should go in the 'requirements' section, not 'hints'."""))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	606
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	607 if dockerReq is not None and is_req is not None\
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	608 and dockerReq.get("dockerOutputDirectory") == "/var/spool/cwl":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	609 if is_req:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	610 # In this specific case, it is legal to have /var/spool/cwl, so skip the check.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	611 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	612 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	613 # Must be a requirement
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	614 var_spool_cwl_detector(self.tool)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	615 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	616 var_spool_cwl_detector(self.tool)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	617
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	618 def _init_job(self, joborder, runtime_context):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	619 # type: (Mapping[Text, Text], RuntimeContext) -> Builder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	620
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	621 if self.metadata.get("cwlVersion") != INTERNAL_VERSION:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	622 raise WorkflowException("Process object loaded with version '%s', must update to '%s' in order to execute." % (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	623 self.metadata.get("cwlVersion"), INTERNAL_VERSION))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	624
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	625 job = cast(Dict[Text, expression.JSON], copy.deepcopy(joborder))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	626
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	627 make_fs_access = getdefault(runtime_context.make_fs_access, StdFsAccess)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	628 fs_access = make_fs_access(runtime_context.basedir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	629
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	630 load_listing_req, _ = self.get_requirement(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	631 "LoadListingRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	632
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	633 if load_listing_req is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	634 load_listing = load_listing_req.get("loadListing")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	635 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	636 load_listing = "no_listing"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	637
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	638 # Validate job order
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	639 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	640 fill_in_defaults(self.tool[u"inputs"], job, fs_access)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	641
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	642 normalizeFilesDirs(job)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	643 schema = self.names.get_name("input_record_schema", "")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	644 if schema is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	645 raise WorkflowException("Missing input record schema: "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	646 "{}".format(self.names))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	647 validate.validate_ex(schema, job, strict=False,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	648 logger=_logger_validation_warnings)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	649
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	650 if load_listing and load_listing != "no_listing":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	651 get_listing(fs_access, job, recursive=(load_listing == "deep_listing"))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	652
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	653 visit_class(job, ("File",), functools.partial(add_sizes, fs_access))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	654
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	655 if load_listing == "deep_listing":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	656 for i, inparm in enumerate(self.tool["inputs"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	657 k = shortname(inparm["id"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	658 if k not in job:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	659 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	660 v = job[k]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	661 dircount = [0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	662
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	663 def inc(d): # type: (List[int]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	664 d[0] += 1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	665 visit_class(v, ("Directory",), lambda x: inc(dircount))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	666 if dircount[0] == 0:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	667 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	668 filecount = [0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	669 visit_class(v, ("File",), lambda x: inc(filecount))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	670 if filecount[0] > FILE_COUNT_WARNING:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	671 # Long lines in this message are okay, will be reflowed based on terminal columns.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	672 _logger.warning(strip_dup_lineno(SourceLine(self.tool["inputs"], i, Text).makeError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	673 """Recursive directory listing has resulted in a large number of File objects (%s) passed to the input parameter '%s'. This may negatively affect workflow performance and memory use.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	674
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	675 If this is a problem, use the hint 'cwltool:LoadListingRequirement' with "shallow_listing" or "no_listing" to change the directory listing behavior:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	676
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	677 $namespaces:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	678 cwltool: "http://commonwl.org/cwltool#"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	679 hints:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	680 cwltool:LoadListingRequirement:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	681 loadListing: shallow_listing
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	682
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	683 """ % (filecount[0], k))))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	684
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	685 except (validate.ValidationException, WorkflowException) as err:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	686 raise_from(WorkflowException("Invalid job input record:\n" + Text(err)), err)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	687
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	688 files = [] # type: List[Dict[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	689 bindings = CommentedSeq()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	690 tmpdir = u""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	691 stagedir = u""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	692
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	693 docker_req, _ = self.get_requirement("DockerRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	694 default_docker = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	695
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	696 if docker_req is None and runtime_context.default_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	697 default_docker = runtime_context.default_container
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	698
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	699 if (docker_req or default_docker) and runtime_context.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	700 if docker_req is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	701 # Check if docker output directory is absolute
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	702 if docker_req.get("dockerOutputDirectory") and \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	703 docker_req.get("dockerOutputDirectory").startswith('/'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	704 outdir = docker_req.get("dockerOutputDirectory")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	705 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	706 outdir = docker_req.get("dockerOutputDirectory") or \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	707 runtime_context.docker_outdir or random_outdir()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	708 elif default_docker is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	709 outdir = runtime_context.docker_outdir or random_outdir()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	710 tmpdir = runtime_context.docker_tmpdir or "/tmp" # nosec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	711 stagedir = runtime_context.docker_stagedir or "/var/lib/cwl"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	712 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	713 outdir = fs_access.realpath(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	714 runtime_context.outdir or tempfile.mkdtemp(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	715 prefix=getdefault(runtime_context.tmp_outdir_prefix,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	716 DEFAULT_TMP_PREFIX)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	717 if self.tool[u"class"] != 'Workflow':
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	718 tmpdir = fs_access.realpath(runtime_context.tmpdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	719 or tempfile.mkdtemp())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	720 stagedir = fs_access.realpath(runtime_context.stagedir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	721 or tempfile.mkdtemp())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	722
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	723 builder = Builder(job,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	724 files,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	725 bindings,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	726 self.schemaDefs,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	727 self.names,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	728 self.requirements,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	729 self.hints,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	730 {},
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	731 runtime_context.mutation_manager,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	732 self.formatgraph,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	733 make_fs_access,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	734 fs_access,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	735 runtime_context.job_script_provider,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	736 runtime_context.eval_timeout,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	737 runtime_context.debug,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	738 runtime_context.js_console,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	739 runtime_context.force_docker_pull,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	740 load_listing,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	741 outdir,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	742 tmpdir,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	743 stagedir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	744
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	745 bindings.extend(builder.bind_input(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	746 self.inputs_record_schema, job,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	747 discover_secondaryFiles=getdefault(runtime_context.toplevel, False)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	748
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	749 if self.tool.get("baseCommand"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	750 for index, command in enumerate(aslist(self.tool["baseCommand"])):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	751 bindings.append({
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	752 "position": [-1000000, index],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	753 "datum": command
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	754 })
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	755
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	756 if self.tool.get("arguments"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	757 for i, arg in enumerate(self.tool["arguments"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	758 lc = self.tool["arguments"].lc.data[i]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	759 filename = self.tool["arguments"].lc.filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	760 bindings.lc.add_kv_line_col(len(bindings), lc)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	761 if isinstance(arg, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	762 arg = copy.deepcopy(arg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	763 if arg.get("position"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	764 position = arg.get("position")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	765 if isinstance(position, str): # no need to test the
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	766 # CWLVersion as the v1.0
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	767 # schema only allows ints
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	768 position = builder.do_eval(position)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	769 if position is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	770 position = 0
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	771 arg["position"] = [position, i]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	772 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	773 arg["position"] = [0, i]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	774 bindings.append(arg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	775 elif ("$(" in arg) or ("${" in arg):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	776 cm = CommentedMap((
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	777 ("position", [0, i]),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	778 ("valueFrom", arg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	779 ))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	780 cm.lc.add_kv_line_col("valueFrom", lc)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	781 cm.lc.filename = filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	782 bindings.append(cm)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	783 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	784 cm = CommentedMap((
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	785 ("position", [0, i]),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	786 ("datum", arg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	787 ))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	788 cm.lc.add_kv_line_col("datum", lc)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	789 cm.lc.filename = filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	790 bindings.append(cm)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	791
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	792 # use python2 like sorting of heterogeneous lists
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	793 # (containing str and int types),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	794 if PY3:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	795 key = functools.cmp_to_key(cmp_like_py2)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	796 else: # PY2
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	797 key = lambda d: d["position"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	798
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	799 # This awkward construction replaces the contents of
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	800 # "bindings" in place (because Builder expects it to be
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	801 # mutated in place, sigh, I'm sorry) with its contents sorted,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	802 # supporting different versions of Python and ruamel.yaml with
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	803 # different behaviors/bugs in CommentedSeq.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	804 bindings_copy = copy.deepcopy(bindings)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	805 del bindings[:]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	806 bindings.extend(sorted(bindings_copy, key=key))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	807
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	808 if self.tool[u"class"] != 'Workflow':
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	809 builder.resources = self.evalResources(builder, runtime_context)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	810 return builder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	811
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	812 def evalResources(self, builder, runtimeContext):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	813 # type: (Builder, RuntimeContext) -> Dict[str, int]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	814 resourceReq, _ = self.get_requirement("ResourceRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	815 if resourceReq is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	816 resourceReq = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	817 cwl_version = self.metadata.get(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	818 "http://commonwl.org/cwltool#original_cwlVersion", None)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	819 if cwl_version == "v1.0":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	820 ram = 1024
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	821 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	822 ram = 256
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	823 request = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	824 "coresMin": 1,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	825 "coresMax": 1,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	826 "ramMin": ram,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	827 "ramMax": ram,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	828 "tmpdirMin": 1024,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	829 "tmpdirMax": 1024,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	830 "outdirMin": 1024,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	831 "outdirMax": 1024
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	832 } # type: Dict[str, int]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	833 for a in ("cores", "ram", "tmpdir", "outdir"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	834 mn = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	835 mx = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	836 if resourceReq.get(a + "Min"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	837 mn = eval_resource(builder, resourceReq[a + "Min"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	838 if resourceReq.get(a + "Max"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	839 mx = eval_resource(builder, resourceReq[a + "Max"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	840 if mn is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	841 mn = mx
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	842 elif mx is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	843 mx = mn
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	844
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	845 if mn is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	846 request[a + "Min"] = cast(int, mn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	847 request[a + "Max"] = cast(int, mx)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	848
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	849 if runtimeContext.select_resources is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	850 return runtimeContext.select_resources(request, runtimeContext)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	851 return {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	852 "cores": request["coresMin"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	853 "ram": request["ramMin"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	854 "tmpdirSize": request["tmpdirMin"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	855 "outdirSize": request["outdirMin"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	856 }
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	857
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	858 def validate_hints(self, avsc_names, hints, strict):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	859 # type: (Any, List[Dict[Text, Any]], bool) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	860 for i, r in enumerate(hints):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	861 sl = SourceLine(hints, i, validate.ValidationException)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	862 with sl:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	863 if avsc_names.get_name(r["class"], "") is not None and self.doc_loader is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	864 plain_hint = dict((key, r[key]) for key in r if key not in
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	865 self.doc_loader.identifiers) # strip identifiers
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	866 validate.validate_ex(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	867 avsc_names.get_name(plain_hint["class"], ""),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	868 plain_hint, strict=strict)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	869 elif r["class"] in ("NetworkAccess", "LoadListingRequirement"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	870 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	871 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	872 _logger.info(Text(sl.makeError(u"Unknown hint %s" % (r["class"]))))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	873
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	874 def visit(self, op): # type: (Callable[[MutableMapping[Text, Any]], None]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	875 op(self.tool)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	876
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	877 @abc.abstractmethod
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	878 def job(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	879 job_order, # type: Mapping[Text, Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	880 output_callbacks, # type: Callable[[Any, Any], Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	881 runtimeContext # type: RuntimeContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	882 ): # type: (...) -> Generator[Any, None, None]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	883 # FIXME: Declare base type for what Generator yields
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	884 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	885
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	886
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	887 _names = set() # type: Set[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	888
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	889
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	890 def uniquename(stem, names=None): # type: (Text, Optional[Set[Text]]) -> Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	891 global _names
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	892 if names is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	893 names = _names
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	894 c = 1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	895 u = stem
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	896 while u in names:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	897 c += 1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	898 u = u"%s_%s" % (stem, c)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	899 names.add(u)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	900 return u
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	901
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	902
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	903 def nestdir(base, deps):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	904 # type: (Text, Dict[Text, Any]) -> Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	905 dirname = os.path.dirname(base) + "/"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	906 subid = deps["location"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	907 if subid.startswith(dirname):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	908 s2 = subid[len(dirname):]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	909 sp = s2.split('/')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	910 sp.pop()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	911 while sp:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	912 nx = sp.pop()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	913 deps = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	914 "class": "Directory",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	915 "basename": nx,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	916 "listing": [deps]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	917 }
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	918 return deps
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	919
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	920
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	921 def mergedirs(listing):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	922 # type: (List[Dict[Text, Any]]) -> List[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	923 r = [] # type: List[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	924 ents = {} # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	925 collided = set() # type: Set[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	926 for e in listing:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	927 if e["basename"] not in ents:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	928 ents[e["basename"]] = e
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	929 elif e["class"] == "Directory":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	930 if e.get("listing"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	931 ents[e["basename"]].setdefault("listing", []).extend(e["listing"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	932 if ents[e["basename"]]["location"].startswith("_:"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	933 ents[e["basename"]]["location"] = e["location"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	934 elif e["location"] != ents[e["basename"]]["location"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	935 # same basename, different location, collision,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	936 # rename both.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	937 collided.add(e["basename"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	938 e2 = ents[e["basename"]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	939
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	940 e["basename"] = urllib.parse.quote(e["location"], safe="")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	941 e2["basename"] = urllib.parse.quote(e2["location"], safe="")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	942
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	943 e["nameroot"], e["nameext"] = os.path.splitext(e["basename"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	944 e2["nameroot"], e2["nameext"] = os.path.splitext(e2["basename"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	945
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	946 ents[e["basename"]] = e
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	947 ents[e2["basename"]] = e2
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	948 for c in collided:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	949 del ents[c]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	950 for e in itervalues(ents):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	951 if e["class"] == "Directory" and "listing" in e:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	952 e["listing"] = mergedirs(e["listing"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	953 r.extend(itervalues(ents))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	954 return r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	955
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	956
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	957 CWL_IANA = "https://www.iana.org/assignments/media-types/application/cwl"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	958
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	959 def scandeps(base, # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	960 doc, # type: Any
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	961 reffields, # type: Set[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	962 urlfields, # type: Set[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	963 loadref, # type: Callable[[Text, Text], Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	964 urljoin=urllib.parse.urljoin, # type: Callable[[Text, Text], Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	965 nestdirs=True # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	966 ): # type: (...) -> List[Dict[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	967 r = [] # type: List[Dict[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	968 if isinstance(doc, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	969 if "id" in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	970 if doc["id"].startswith("file://"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	971 df, _ = urllib.parse.urldefrag(doc["id"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	972 if base != df:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	973 r.append({
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	974 "class": "File",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	975 "location": df,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	976 "format": CWL_IANA
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	977 })
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	978 base = df
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	979
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	980 if doc.get("class") in ("File", "Directory") and "location" in urlfields:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	981 u = doc.get("location", doc.get("path"))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	982 if u and not u.startswith("_:"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	983 deps = {"class": doc["class"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	984 "location": urljoin(base, u)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	985 } # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	986 if "basename" in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	987 deps["basename"] = doc["basename"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	988 if doc["class"] == "Directory" and "listing" in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	989 deps["listing"] = doc["listing"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	990 if doc["class"] == "File" and "secondaryFiles" in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	991 deps["secondaryFiles"] = doc["secondaryFiles"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	992 if nestdirs:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	993 deps = nestdir(base, deps)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	994 r.append(deps)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	995 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	996 if doc["class"] == "Directory" and "listing" in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	997 r.extend(scandeps(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	998 base, doc["listing"], reffields, urlfields, loadref,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	999 urljoin=urljoin, nestdirs=nestdirs))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1000 elif doc["class"] == "File" and "secondaryFiles" in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1001 r.extend(scandeps(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1002 base, doc["secondaryFiles"], reffields, urlfields,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1003 loadref, urljoin=urljoin, nestdirs=nestdirs))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1004
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1005 for k, v in iteritems(doc):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1006 if k in reffields:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1007 for u in aslist(v):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1008 if isinstance(u, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1009 r.extend(scandeps(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1010 base, u, reffields, urlfields, loadref,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1011 urljoin=urljoin, nestdirs=nestdirs))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1012 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1013 subid = urljoin(base, u)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1014 basedf, _ = urllib.parse.urldefrag(base)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1015 subiddf, _ = urllib.parse.urldefrag(subid)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1016 if basedf == subiddf:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1017 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1018 sub = loadref(base, u)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1019 deps = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1020 "class": "File",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1021 "location": subid,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1022 "format": CWL_IANA
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1023 }
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1024 sf = scandeps(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1025 subid, sub, reffields, urlfields, loadref,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1026 urljoin=urljoin, nestdirs=nestdirs)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1027 if sf:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1028 deps["secondaryFiles"] = sf
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1029 if nestdirs:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1030 deps = nestdir(base, deps)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1031 r.append(deps)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1032 elif k in urlfields and k != "location":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1033 for u in aslist(v):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1034 deps = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1035 "class": "File",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1036 "location": urljoin(base, u)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1037 }
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1038 if nestdirs:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1039 deps = nestdir(base, deps)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1040 r.append(deps)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1041 elif k not in ("listing", "secondaryFiles"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1042 r.extend(scandeps(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1043 base, v, reffields, urlfields, loadref, urljoin=urljoin,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1044 nestdirs=nestdirs))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1045 elif isinstance(doc, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1046 for d in doc:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1047 r.extend(scandeps(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1048 base, d, reffields, urlfields, loadref, urljoin=urljoin,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1049 nestdirs=nestdirs))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1050
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1051 if r:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1052 normalizeFilesDirs(r)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1053 r = mergedirs(r)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1054
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1055 return r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1056
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1057
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1058 def compute_checksums(fs_access, fileobj): # type: (StdFsAccess, Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1059 if "checksum" not in fileobj:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1060 checksum = hashlib.sha1() # nosec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1061 with fs_access.open(fileobj["location"], "rb") as f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1062 contents = f.read(1024 * 1024)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1063 while contents != b"":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1064 checksum.update(contents)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1065 contents = f.read(1024 * 1024)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1066 fileobj["checksum"] = "sha1$%s" % checksum.hexdigest()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1067 fileobj["size"] = fs_access.size(fileobj["location"])

Mercurial > repos > shellac > guppy_basecaller

annotate env/lib/python3.7/site-packages/cwltool/process.py @ 2:6af9afd405e9 draft