guppy_basecaller: env/lib/python3.7/site-packages/cwltool/command_line

annotate env/lib/python3.7/site-packages/cwltool/command_line_tool.py @ 4:79f47841a781 draft

"planemo upload commit 2a0fe2cc28b09e101d37293e53e82f61762262ec"

author	shellac
date	Thu, 14 May 2020 16:47:39 -0400
parents	26e78fe6e8c4
children

rev	line source
0 26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1 """Implementation of CommandLineTool."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	2 from __future__ import absolute_import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	3
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	4 import copy
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	5 import hashlib
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	6 import json
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	7 import locale
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	8 import logging
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	9 import os
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	10 import re
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	11 import shutil
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	12 import tempfile
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	13 import threading
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	14 from functools import cmp_to_key, partial
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	15 from typing import (Any, Callable, Dict, Generator, IO, List, Mapping,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	16 MutableMapping, MutableSequence, Optional, Set, Union, cast)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	17
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	18 from typing_extensions import Text, Type, TYPE_CHECKING # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	19 # move to a regular typing import when Python 3.3-3.6 is no longer supported
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	20
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	21 import shellescape
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	22 from schema_salad import validate
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	23 from schema_salad.avro.schema import Schema
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	24 from schema_salad.ref_resolver import file_uri, uri_file_path
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	25 from schema_salad.sourceline import SourceLine
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	26 from six import string_types
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	27 from future.utils import raise_from
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	28
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	29 from six.moves import map, urllib
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	30 from typing_extensions import (TYPE_CHECKING, # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	31 Text, Type)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	32 # move to a regular typing import when Python 3.3-3.6 is no longer supported
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	33
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	34 from .builder import (Builder, content_limit_respected_read_bytes, # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	35 substitute)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	36 from .context import LoadingContext # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	37 from .context import RuntimeContext, getdefault
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	38 from .docker import DockerCommandLineJob
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	39 from .errors import WorkflowException
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	40 from .flatten import flatten
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	41 from .job import CommandLineJob, JobBase # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	42 from .loghandler import _logger
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	43 from .mutation import MutationManager # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	44 from .pathmapper import (PathMapper, adjustDirObjs, adjustFileObjs,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	45 get_listing, trim_listing, visit_class)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	46 from .process import (Process, UnsupportedRequirement,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	47 _logger_validation_warnings, compute_checksums,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	48 normalizeFilesDirs, shortname, uniquename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	49 from .singularity import SingularityCommandLineJob
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	50 from .software_requirements import ( # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	51 DependenciesConfiguration)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	52 from .stdfsaccess import StdFsAccess # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	53 from .utils import (aslist, convert_pathsep_to_unix,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	54 docker_windows_path_adjust, json_dumps, onWindows,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	55 random_outdir, windows_default_container_id,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	56 shared_file_lock, upgrade_lock)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	57 if TYPE_CHECKING:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	58 from .provenance import ProvenanceProfile # pylint: disable=unused-import
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	59
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	60 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	61 ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*") # Accept anything
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	62 ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	63 DEFAULT_CONTAINER_MSG = """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	64 We are on Microsoft Windows and not all components of this CWL description have a
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	65 container specified. This means that these steps will be executed in the default container,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	66 which is %s.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	67
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	68 Note, this could affect portability if this CWL description relies on non-POSIX features
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	69 or commands in this container. For best results add the following to your CWL
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	70 description's hints section:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	71
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	72 hints:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	73 DockerRequirement:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	74 dockerPull: %s
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	75 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	76
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	77
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	78 class ExpressionTool(Process):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	79 class ExpressionJob(object):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	80 """Job for ExpressionTools."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	81
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	82 def __init__(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	83 builder, # type: Builder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	84 script, # type: Dict[Text, Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	85 output_callback, # type: Callable[[Any, Any], Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	86 requirements, # type: List[Dict[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	87 hints, # type: List[Dict[Text, Text]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	88 outdir=None, # type: Optional[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	89 tmpdir=None, # type: Optional[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	90 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	91 """Initializet this ExpressionJob."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	92 self.builder = builder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	93 self.requirements = requirements
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	94 self.hints = hints
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	95 self.collect_outputs = None # type: Optional[Callable[[Any], Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	96 self.output_callback = output_callback
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	97 self.outdir = outdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	98 self.tmpdir = tmpdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	99 self.script = script
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	100 self.prov_obj = None # type: Optional[ProvenanceProfile]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	101
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	102 def run(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	103 runtimeContext, # type: RuntimeContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	104 tmpdir_lock=None # type: Optional[threading.Lock]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	105 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	106 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	107 normalizeFilesDirs(self.builder.job)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	108 ev = self.builder.do_eval(self.script)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	109 normalizeFilesDirs(ev)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	110 self.output_callback(ev, "success")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	111 except Exception as err:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	112 _logger.warning(u"Failed to evaluate expression:\n%s",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	113 Text(err), exc_info=runtimeContext.debug)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	114 self.output_callback({}, "permanentFail")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	115
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	116 def job(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	117 job_order, # type: Mapping[Text, Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	118 output_callbacks, # type: Callable[[Any, Any], Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	119 runtimeContext # type: RuntimeContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	120 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	121 # type: (...) -> Generator[ExpressionTool.ExpressionJob, None, None]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	122 builder = self._init_job(job_order, runtimeContext)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	123
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	124 job = ExpressionTool.ExpressionJob(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	125 builder, self.tool["expression"], output_callbacks,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	126 self.requirements, self.hints)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	127 job.prov_obj = runtimeContext.prov_obj
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	128 yield job
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	129
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	130
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	131 def remove_path(f): # type: (Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	132 if "path" in f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	133 del f["path"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	134
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	135
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	136 def revmap_file(builder, outdir, f):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	137 # type: (Builder, Text, Dict[Text, Any]) -> Union[Dict[Text, Any], None]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	138 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	139 Remap a file from internal path to external path.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	140
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	141 For Docker, this maps from the path inside tho container to the path
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	142 outside the container. Recognizes files in the pathmapper or remaps
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	143 internal output directories to the external directory.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	144 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	145 split = urllib.parse.urlsplit(outdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	146 if not split.scheme:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	147 outdir = file_uri(str(outdir))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	148
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	149 # builder.outdir is the inner (container/compute node) output directory
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	150 # outdir is the outer (host/storage system) output directory
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	151
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	152 if "location" in f and "path" not in f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	153 if f["location"].startswith("file://"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	154 f["path"] = convert_pathsep_to_unix(uri_file_path(f["location"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	155 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	156 return f
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	157
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	158 if "path" in f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	159 path = f["path"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	160 uripath = file_uri(path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	161 del f["path"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	162
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	163 if "basename" not in f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	164 f["basename"] = os.path.basename(path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	165
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	166 if not builder.pathmapper:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	167 raise ValueError("Do not call revmap_file using a builder that doesn't have a pathmapper.")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	168 revmap_f = builder.pathmapper.reversemap(path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	169
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	170 if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	171 f["location"] = revmap_f[1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	172 elif uripath == outdir or uripath.startswith(outdir+os.sep) or uripath.startswith(outdir+'/'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	173 f["location"] = file_uri(path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	174 elif path == builder.outdir or path.startswith(builder.outdir+os.sep) or path.startswith(builder.outdir+'/'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	175 f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir) + 1:])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	176 elif not os.path.isabs(path):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	177 f["location"] = builder.fs_access.join(outdir, path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	178 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	179 raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	180 u"file pass through." % (path, builder.outdir))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	181 return f
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	182
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	183 raise WorkflowException(u"Output File object is missing both 'location' "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	184 "and 'path' fields: %s" % f)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	185
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	186
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	187 class CallbackJob(object):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	188 def __init__(self, job, output_callback, cachebuilder, jobcache):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	189 # type: (CommandLineTool, Callable[[Any, Any], Any], Builder, Text) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	190 """Initialize this CallbackJob."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	191 self.job = job
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	192 self.output_callback = output_callback
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	193 self.cachebuilder = cachebuilder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	194 self.outdir = jobcache
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	195 self.prov_obj = None # type: Optional[ProvenanceProfile]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	196
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	197 def run(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	198 runtimeContext, # type: RuntimeContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	199 tmpdir_lock=None # type: Optional[threading.Lock]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	200 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	201 self.output_callback(self.job.collect_output_ports(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	202 self.job.tool["outputs"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	203 self.cachebuilder,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	204 self.outdir,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	205 getdefault(runtimeContext.compute_checksum, True)), "success")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	206
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	207
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	208 def check_adjust(builder, file_o):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	209 # type: (Builder, Dict[Text, Any]) -> Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	210 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	211 Map files to assigned path inside a container.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	212
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	213 We need to also explicitly walk over input, as implicit reassignment
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	214 doesn't reach everything in builder.bindings
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	215 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	216 if not builder.pathmapper:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	217 raise ValueError("Do not call check_adjust using a builder that doesn't have a pathmapper.")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	218 file_o["path"] = docker_windows_path_adjust(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	219 builder.pathmapper.mapper(file_o["location"])[1])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	220 dn, bn = os.path.split(file_o["path"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	221 if file_o.get("dirname") != dn:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	222 file_o["dirname"] = Text(dn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	223 if file_o.get("basename") != bn:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	224 file_o["basename"] = Text(bn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	225 if file_o["class"] == "File":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	226 nr, ne = os.path.splitext(file_o["basename"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	227 if file_o.get("nameroot") != nr:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	228 file_o["nameroot"] = Text(nr)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	229 if file_o.get("nameext") != ne:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	230 file_o["nameext"] = Text(ne)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	231 if not ACCEPTLIST_RE.match(file_o["basename"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	232 raise WorkflowException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	233 "Invalid filename: '{}' contains illegal characters".format(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	234 file_o["basename"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	235 return file_o
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	236
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	237 def check_valid_locations(fs_access, ob): # type: (StdFsAccess, Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	238 if ob["location"].startswith("_:"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	239 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	240 if ob["class"] == "File" and not fs_access.isfile(ob["location"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	241 raise validate.ValidationException("Does not exist or is not a File: '%s'" % ob["location"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	242 if ob["class"] == "Directory" and not fs_access.isdir(ob["location"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	243 raise validate.ValidationException("Does not exist or is not a Directory: '%s'" % ob["location"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	244
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	245
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	246 OutputPorts = Dict[Text, Union[None, Text, List[Union[Dict[Text, Any], Text]], Dict[Text, Any]]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	247
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	248 class CommandLineTool(Process):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	249 def __init__(self, toolpath_object, loadingContext):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	250 # type: (MutableMapping[Text, Any], LoadingContext) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	251 """Initialize this CommandLineTool."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	252 super(CommandLineTool, self).__init__(toolpath_object, loadingContext)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	253 self.prov_obj = loadingContext.prov_obj
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	254
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	255 def make_job_runner(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	256 runtimeContext # type: RuntimeContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	257 ): # type: (...) -> Type[JobBase]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	258 dockerReq, _ = self.get_requirement("DockerRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	259 if not dockerReq and runtimeContext.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	260 if runtimeContext.find_default_container is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	261 default_container = runtimeContext.find_default_container(self)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	262 if default_container is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	263 self.requirements.insert(0, {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	264 "class": "DockerRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	265 "dockerPull": default_container
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	266 })
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	267 dockerReq = self.requirements[0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	268 if default_container == windows_default_container_id \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	269 and runtimeContext.use_container and onWindows():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	270 _logger.warning(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	271 DEFAULT_CONTAINER_MSG, windows_default_container_id,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	272 windows_default_container_id)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	273
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	274 if dockerReq is not None and runtimeContext.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	275 if runtimeContext.singularity:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	276 return SingularityCommandLineJob
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	277 return DockerCommandLineJob
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	278 for t in reversed(self.requirements):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	279 if t["class"] == "DockerRequirement":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	280 raise UnsupportedRequirement(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	281 "--no-container, but this CommandLineTool has "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	282 "DockerRequirement under 'requirements'.")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	283 return CommandLineJob
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	284
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	285 def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	286 # type: (List[Any], Text, RuntimeContext, bool) -> PathMapper
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	287 return PathMapper(reffiles, runtimeContext.basedir, stagedir, separateDirs)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	288
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	289 def updatePathmap(self, outdir, pathmap, fn):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	290 # type: (Text, PathMapper, Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	291 if "location" in fn and fn["location"] in pathmap:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	292 pathmap.update(fn["location"], pathmap.mapper(fn["location"]).resolved,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	293 os.path.join(outdir, fn["basename"]),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	294 ("Writable" if fn.get("writable") else "") + fn["class"], False)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	295 for sf in fn.get("secondaryFiles", []):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	296 self.updatePathmap(outdir, pathmap, sf)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	297 for ls in fn.get("listing", []):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	298 self.updatePathmap(os.path.join(outdir, fn["basename"]), pathmap, ls)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	299
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	300 def job(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	301 job_order, # type: Mapping[Text, Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	302 output_callbacks, # type: Callable[[Any, Any], Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	303 runtimeContext # type: RuntimeContext
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	304 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	305 # type: (...) -> Generator[Union[JobBase, CallbackJob], None, None]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	306
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	307 workReuse, _ = self.get_requirement("WorkReuse")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	308 enableReuse = workReuse.get("enableReuse", True) if workReuse else True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	309
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	310 jobname = uniquename(runtimeContext.name or shortname(self.tool.get("id", "job")))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	311 if runtimeContext.cachedir and enableReuse:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	312 cachecontext = runtimeContext.copy()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	313 cachecontext.outdir = "/out"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	314 cachecontext.tmpdir = "/tmp" # nosec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	315 cachecontext.stagedir = "/stage"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	316 cachebuilder = self._init_job(job_order, cachecontext)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	317 cachebuilder.pathmapper = PathMapper(cachebuilder.files,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	318 runtimeContext.basedir,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	319 cachebuilder.stagedir,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	320 separateDirs=False)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	321 _check_adjust = partial(check_adjust, cachebuilder)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	322 visit_class([cachebuilder.files, cachebuilder.bindings],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	323 ("File", "Directory"), _check_adjust)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	324
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	325 cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	326 docker_req, _ = self.get_requirement("DockerRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	327 if docker_req is not None and runtimeContext.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	328 dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	329 elif runtimeContext.default_container is not None and runtimeContext.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	330 dockerimg = runtimeContext.default_container
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	331 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	332 dockerimg = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	333
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	334 if dockerimg is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	335 cmdline = ["docker", "run", dockerimg] + cmdline
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	336 # not really run using docker, just for hashing purposes
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	337 keydict = {u"cmdline": cmdline} # type: Dict[Text, Union[Dict[Text, Any], List[Any]]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	338
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	339 for shortcut in ["stdin", "stdout", "stderr"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	340 if shortcut in self.tool:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	341 keydict[shortcut] = self.tool[shortcut]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	342
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	343 for location, fobj in cachebuilder.pathmapper.items():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	344 if fobj.type == "File":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	345 checksum = next(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	346 (e['checksum'] for e in cachebuilder.files
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	347 if 'location' in e and e['location'] == location
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	348 and 'checksum' in e
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	349 and e['checksum'] != 'sha1$hash'), None)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	350 fobj_stat = os.stat(fobj.resolved)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	351 if checksum is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	352 keydict[fobj.resolved] = [fobj_stat.st_size, checksum]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	353 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	354 keydict[fobj.resolved] = [fobj_stat.st_size,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	355 int(fobj_stat.st_mtime * 1000)]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	356
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	357 interesting = {"DockerRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	358 "EnvVarRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	359 "InitialWorkDirRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	360 "ShellCommandRequirement",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	361 "NetworkAccess"}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	362 for rh in (self.original_requirements, self.original_hints):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	363 for r in reversed(rh):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	364 if r["class"] in interesting and r["class"] not in keydict:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	365 keydict[r["class"]] = r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	366
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	367 keydictstr = json_dumps(keydict, separators=(',', ':'),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	368 sort_keys=True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	369 cachekey = hashlib.md5( # nosec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	370 keydictstr.encode('utf-8')).hexdigest()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	371
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	372 _logger.debug("[job %s] keydictstr is %s -> %s", jobname,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	373 keydictstr, cachekey)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	374
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	375 jobcache = os.path.join(runtimeContext.cachedir, cachekey)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	376
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	377 # Create a lockfile to manage cache status.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	378 jobcachepending = "{}.status".format(jobcache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	379 jobcachelock = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	380 jobstatus = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	381
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	382 # Opens the file for read/write, or creates an empty file.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	383 jobcachelock = open(jobcachepending, "a+")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	384
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	385 # get the shared lock to ensure no other process is trying
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	386 # to write to this cache
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	387 shared_file_lock(jobcachelock)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	388 jobcachelock.seek(0)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	389 jobstatus = jobcachelock.read()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	390
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	391 if os.path.isdir(jobcache) and jobstatus == "success":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	392 if docker_req and runtimeContext.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	393 cachebuilder.outdir = runtimeContext.docker_outdir or random_outdir()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	394 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	395 cachebuilder.outdir = jobcache
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	396
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	397 _logger.info("[job %s] Using cached output in %s", jobname, jobcache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	398 yield CallbackJob(self, output_callbacks, cachebuilder, jobcache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	399 # we're done with the cache so release lock
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	400 jobcachelock.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	401 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	402 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	403 _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	404
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	405 # turn shared lock into an exclusive lock since we'll
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	406 # be writing the cache directory
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	407 upgrade_lock(jobcachelock)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	408
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	409 shutil.rmtree(jobcache, True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	410 os.makedirs(jobcache)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	411 runtimeContext = runtimeContext.copy()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	412 runtimeContext.outdir = jobcache
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	413
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	414 def update_status_output_callback(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	415 output_callbacks, # type: Callable[[List[Dict[Text, Any]], Text], None]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	416 jobcachelock, # type: IO[Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	417 outputs, # type: List[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	418 processStatus # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	419 ): # type: (...) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	420 # save status to the lockfile then release the lock
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	421 jobcachelock.seek(0)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	422 jobcachelock.truncate()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	423 jobcachelock.write(processStatus)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	424 jobcachelock.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	425 output_callbacks(outputs, processStatus)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	426
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	427 output_callbacks = partial(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	428 update_status_output_callback, output_callbacks, jobcachelock)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	429
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	430 builder = self._init_job(job_order, runtimeContext)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	431
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	432 reffiles = copy.deepcopy(builder.files)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	433
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	434 j = self.make_job_runner(runtimeContext)(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	435 builder, builder.job, self.make_path_mapper, self.requirements,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	436 self.hints, jobname)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	437 j.prov_obj = self.prov_obj
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	438
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	439 j.successCodes = self.tool.get("successCodes", [])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	440 j.temporaryFailCodes = self.tool.get("temporaryFailCodes", [])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	441 j.permanentFailCodes = self.tool.get("permanentFailCodes", [])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	442
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	443 debug = _logger.isEnabledFor(logging.DEBUG)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	444
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	445 if debug:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	446 _logger.debug(u"[job %s] initializing from %s%s",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	447 j.name,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	448 self.tool.get("id", ""),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	449 u" as part of %s" % runtimeContext.part_of
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	450 if runtimeContext.part_of else "")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	451 _logger.debug(u"[job %s] %s", j.name, json_dumps(builder.job,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	452 indent=4))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	453
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	454 builder.pathmapper = self.make_path_mapper(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	455 reffiles, builder.stagedir, runtimeContext, True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	456 builder.requirements = j.requirements
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	457
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	458 _check_adjust = partial(check_adjust, builder)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	459
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	460 visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	461
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	462 initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	463 if initialWorkdir is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	464 ls = [] # type: List[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	465 if isinstance(initialWorkdir["listing"], string_types):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	466 ls = builder.do_eval(initialWorkdir["listing"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	467 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	468 for t in initialWorkdir["listing"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	469 if isinstance(t, Mapping) and "entry" in t:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	470 entry_exp = builder.do_eval(t["entry"], strip_whitespace=False)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	471 for entry in aslist(entry_exp):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	472 et = {u"entry": entry}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	473 if "entryname" in t:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	474 et["entryname"] = builder.do_eval(t["entryname"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	475 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	476 et["entryname"] = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	477 et["writable"] = t.get("writable", False)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	478 if et[u"entry"] is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	479 ls.append(et)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	480 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	481 initwd_item = builder.do_eval(t)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	482 if not initwd_item:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	483 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	484 if isinstance(initwd_item, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	485 ls.extend(initwd_item)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	486 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	487 ls.append(initwd_item)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	488 for i, t in enumerate(ls):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	489 if "entry" in t:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	490 if isinstance(t["entry"], string_types):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	491 ls[i] = {
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	492 "class": "File",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	493 "basename": t["entryname"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	494 "contents": t["entry"],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	495 "writable": t.get("writable")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	496 }
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	497 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	498 if t.get("entryname") or t.get("writable"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	499 t = copy.deepcopy(t)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	500 if t.get("entryname"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	501 t["entry"]["basename"] = t["entryname"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	502 t["entry"]["writable"] = t.get("writable")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	503 ls[i] = t["entry"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	504 j.generatefiles["listing"] = ls
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	505 for l in ls:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	506 self.updatePathmap(builder.outdir, builder.pathmapper, l)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	507 visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	508
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	509 if debug:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	510 _logger.debug(u"[job %s] path mappings is %s", j.name,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	511 json_dumps({p: builder.pathmapper.mapper(p)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	512 for p in builder.pathmapper.files()},
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	513 indent=4))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	514
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	515 if self.tool.get("stdin"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	516 with SourceLine(self.tool, "stdin", validate.ValidationException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	517 j.stdin = builder.do_eval(self.tool["stdin"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	518 if j.stdin:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	519 reffiles.append({"class": "File", "path": j.stdin})
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	520
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	521 if self.tool.get("stderr"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	522 with SourceLine(self.tool, "stderr", validate.ValidationException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	523 j.stderr = builder.do_eval(self.tool["stderr"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	524 if j.stderr:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	525 if os.path.isabs(j.stderr) or ".." in j.stderr:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	526 raise validate.ValidationException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	527 "stderr must be a relative path, got '%s'" % j.stderr)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	528
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	529 if self.tool.get("stdout"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	530 with SourceLine(self.tool, "stdout", validate.ValidationException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	531 j.stdout = builder.do_eval(self.tool["stdout"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	532 if j.stdout:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	533 if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	534 raise validate.ValidationException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	535 "stdout must be a relative path, got '%s'" % j.stdout)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	536
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	537 if debug:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	538 _logger.debug(u"[job %s] command line bindings is %s", j.name,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	539 json_dumps(builder.bindings, indent=4))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	540 dockerReq, _ = self.get_requirement("DockerRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	541 if dockerReq is not None and runtimeContext.use_container:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	542 out_dir, out_prefix = os.path.split(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	543 runtimeContext.tmp_outdir_prefix)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	544 j.outdir = runtimeContext.outdir or \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	545 tempfile.mkdtemp(prefix=out_prefix, dir=out_dir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	546 tmpdir_dir, tmpdir_prefix = os.path.split(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	547 runtimeContext.tmpdir_prefix)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	548 j.tmpdir = runtimeContext.tmpdir or \
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	549 tempfile.mkdtemp(prefix=tmpdir_prefix, dir=tmpdir_dir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	550 j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix, dir=tmpdir_dir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	551 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	552 j.outdir = builder.outdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	553 j.tmpdir = builder.tmpdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	554 j.stagedir = builder.stagedir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	555
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	556 inplaceUpdateReq, _ = self.get_requirement("InplaceUpdateRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	557 if inplaceUpdateReq is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	558 j.inplace_update = inplaceUpdateReq["inplaceUpdate"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	559 normalizeFilesDirs(j.generatefiles)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	560
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	561 readers = {} # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	562 muts = set() # type: Set[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	563
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	564 if builder.mutation_manager is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	565 def register_mut(f): # type: (Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	566 mm = cast(MutationManager, builder.mutation_manager)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	567 muts.add(f["location"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	568 mm.register_mutation(j.name, f)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	569
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	570 def register_reader(f): # type: (Dict[Text, Any]) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	571 mm = cast(MutationManager, builder.mutation_manager)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	572 if f["location"] not in muts:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	573 mm.register_reader(j.name, f)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	574 readers[f["location"]] = copy.deepcopy(f)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	575
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	576 for li in j.generatefiles["listing"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	577 li = cast(Dict[Text, Any], li)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	578 if li.get("writable") and j.inplace_update:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	579 adjustFileObjs(li, register_mut)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	580 adjustDirObjs(li, register_mut)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	581 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	582 adjustFileObjs(li, register_reader)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	583 adjustDirObjs(li, register_reader)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	584
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	585 adjustFileObjs(builder.files, register_reader)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	586 adjustFileObjs(builder.bindings, register_reader)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	587 adjustDirObjs(builder.files, register_reader)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	588 adjustDirObjs(builder.bindings, register_reader)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	589
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	590 timelimit, _ = self.get_requirement("ToolTimeLimit")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	591 if timelimit is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	592 with SourceLine(timelimit, "timelimit", validate.ValidationException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	593 j.timelimit = builder.do_eval(timelimit["timelimit"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	594 if not isinstance(j.timelimit, int) or j.timelimit < 0:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	595 raise Exception("timelimit must be an integer >= 0, got: %s" % j.timelimit)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	596
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	597 networkaccess, _ = self.get_requirement("NetworkAccess")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	598 if networkaccess is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	599 with SourceLine(networkaccess, "networkAccess", validate.ValidationException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	600 j.networkaccess = builder.do_eval(networkaccess["networkAccess"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	601 if not isinstance(j.networkaccess, bool):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	602 raise Exception("networkAccess must be a boolean, got: %s" % j.networkaccess)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	603
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	604 j.environment = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	605 evr, _ = self.get_requirement("EnvVarRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	606 if evr is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	607 for t in evr["envDef"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	608 j.environment[t["envName"]] = builder.do_eval(t["envValue"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	609
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	610 shellcmd, _ = self.get_requirement("ShellCommandRequirement")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	611 if shellcmd is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	612 cmd = [] # type: List[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	613 for b in builder.bindings:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	614 arg = builder.generate_arg(b)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	615 if b.get("shellQuote", True):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	616 arg = [shellescape.quote(a) for a in aslist(arg)]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	617 cmd.extend(aslist(arg))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	618 j.command_line = ["/bin/sh", "-c", " ".join(cmd)]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	619 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	620 j.command_line = flatten(list(map(builder.generate_arg, builder.bindings)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	621
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	622 j.pathmapper = builder.pathmapper
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	623 j.collect_outputs = partial(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	624 self.collect_output_ports, self.tool["outputs"], builder,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	625 compute_checksum=getdefault(runtimeContext.compute_checksum, True),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	626 jobname=jobname,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	627 readers=readers)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	628 j.output_callback = output_callbacks
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	629
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	630 yield j
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	631
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	632 def collect_output_ports(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	633 ports, # type: Set[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	634 builder, # type: Builder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	635 outdir, # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	636 rcode, # type: int
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	637 compute_checksum=True, # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	638 jobname="", # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	639 readers=None # type: Optional[Dict[Text, Any]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	640 ): # type: (...) -> OutputPorts
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	641 ret = {} # type: OutputPorts
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	642 debug = _logger.isEnabledFor(logging.DEBUG)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	643 cwl_version = self.metadata.get(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	644 "http://commonwl.org/cwltool#original_cwlVersion", None)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	645 if cwl_version != "v1.0":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	646 builder.resources["exitCode"] = rcode
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	647 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	648 fs_access = builder.make_fs_access(outdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	649 custom_output = fs_access.join(outdir, "cwl.output.json")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	650 if fs_access.exists(custom_output):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	651 with fs_access.open(custom_output, "r") as f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	652 ret = json.load(f)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	653 if debug:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	654 _logger.debug(u"Raw output from %s: %s", custom_output,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	655 json_dumps(ret, indent=4))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	656 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	657 for i, port in enumerate(ports):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	658 class ParameterOutputWorkflowException(WorkflowException):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	659 def __init__(self, msg, kwargs): # type: (Text, Any) -> None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	660 super(ParameterOutputWorkflowException, self).__init__(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	661 u"Error collecting output for parameter '%s':\n%s"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	662 % (shortname(port["id"]), msg), kwargs)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	663 with SourceLine(ports, i, ParameterOutputWorkflowException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	664 fragment = shortname(port["id"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	665 ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	666 compute_checksum=compute_checksum)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	667 if ret:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	668 revmap = partial(revmap_file, builder, outdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	669 adjustDirObjs(ret, trim_listing)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	670 visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	671 visit_class(ret, ("File", "Directory"), remove_path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	672 normalizeFilesDirs(ret)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	673 visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	674
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	675 if compute_checksum:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	676 adjustFileObjs(ret, partial(compute_checksums, fs_access))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	677 expected_schema = cast(Schema, self.names.get_name(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	678 "outputs_record_schema", ""))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	679 validate.validate_ex(expected_schema, ret,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	680 strict=False, logger=_logger_validation_warnings)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	681 if ret is not None and builder.mutation_manager is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	682 adjustFileObjs(ret, builder.mutation_manager.set_generation)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	683 return ret if ret is not None else {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	684 except validate.ValidationException as e:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	685 raise_from(WorkflowException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	686 "Error validating output record. " + Text(e) + "\n in "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	687 + json_dumps(ret, indent=4)), e)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	688 finally:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	689 if builder.mutation_manager and readers:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	690 for r in readers.values():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	691 builder.mutation_manager.release_reader(jobname, r)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	692
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	693 def collect_output(self,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	694 schema, # type: Dict[Text, Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	695 builder, # type: Builder
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	696 outdir, # type: Text
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	697 fs_access, # type: StdFsAccess
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	698 compute_checksum=True # type: bool
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	699 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	700 # type: (...) -> Optional[Union[Dict[Text, Any], List[Union[Dict[Text, Any], Text]]]]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	701 r = [] # type: List[Any]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	702 empty_and_optional = False
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	703 debug = _logger.isEnabledFor(logging.DEBUG)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	704 if "outputBinding" in schema:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	705 binding = schema["outputBinding"]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	706 globpatterns = [] # type: List[Text]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	707
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	708 revmap = partial(revmap_file, builder, outdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	709
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	710 if "glob" in binding:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	711 with SourceLine(binding, "glob", WorkflowException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	712 for gb in aslist(binding["glob"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	713 gb = builder.do_eval(gb)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	714 if gb:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	715 globpatterns.extend(aslist(gb))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	716
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	717 for gb in globpatterns:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	718 if gb.startswith(builder.outdir):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	719 gb = gb[len(builder.outdir) + 1:]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	720 elif gb == ".":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	721 gb = outdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	722 elif gb.startswith("/"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	723 raise WorkflowException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	724 "glob patterns must not start with '/'")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	725 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	726 prefix = fs_access.glob(outdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	727 r.extend([{"location": g,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	728 "path": fs_access.join(builder.outdir,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	729 g[len(prefix[0])+1:]),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	730 "basename": os.path.basename(g),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	731 "nameroot": os.path.splitext(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	732 os.path.basename(g))[0],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	733 "nameext": os.path.splitext(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	734 os.path.basename(g))[1],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	735 "class": "File" if fs_access.isfile(g)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	736 else "Directory"}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	737 for g in sorted(fs_access.glob(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	738 fs_access.join(outdir, gb)),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	739 key=cmp_to_key(cast(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	740 Callable[[Text, Text],
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	741 int], locale.strcoll)))])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	742 except (OSError, IOError) as e:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	743 _logger.warning(Text(e))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	744 except Exception:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	745 _logger.error("Unexpected error from fs_access", exc_info=True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	746 raise
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	747
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	748 for files in r:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	749 rfile = files.copy()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	750 revmap(rfile)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	751 if files["class"] == "Directory":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	752 ll = schema.get("loadListing") or builder.loadListing
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	753 if ll and ll != "no_listing":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	754 get_listing(fs_access, files, (ll == "deep_listing"))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	755 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	756 if binding.get("loadContents"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	757 with fs_access.open(rfile["location"], "rb") as f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	758 files["contents"] = content_limit_respected_read_bytes(f).decode("utf-8")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	759 if compute_checksum:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	760 with fs_access.open(rfile["location"], "rb") as f:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	761 checksum = hashlib.sha1() # nosec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	762 contents = f.read(1024 * 1024)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	763 while contents != b"":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	764 checksum.update(contents)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	765 contents = f.read(1024 * 1024)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	766 files["checksum"] = "sha1$%s" % checksum.hexdigest()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	767 files["size"] = fs_access.size(rfile["location"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	768
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	769 optional = False
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	770 single = False
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	771 if isinstance(schema["type"], MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	772 if "null" in schema["type"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	773 optional = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	774 if "File" in schema["type"] or "Directory" in schema["type"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	775 single = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	776 elif schema["type"] == "File" or schema["type"] == "Directory":
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	777 single = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	778
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	779 if "outputEval" in binding:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	780 with SourceLine(binding, "outputEval", WorkflowException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	781 r = builder.do_eval(binding["outputEval"], context=r)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	782
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	783 if single:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	784 if not r and not optional:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	785 with SourceLine(binding, "glob", WorkflowException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	786 raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	787 elif not r and optional:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	788 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	789 elif isinstance(r, MutableSequence):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	790 if len(r) > 1:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	791 raise WorkflowException("Multiple matches for output item that is a single file.")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	792 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	793 r = r[0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	794
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	795 if "secondaryFiles" in schema:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	796 with SourceLine(schema, "secondaryFiles", WorkflowException, debug):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	797 for primary in aslist(r):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	798 if isinstance(primary, MutableMapping):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	799 primary.setdefault("secondaryFiles", [])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	800 pathprefix = primary["path"][0:primary["path"].rindex("/")+1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	801 for sf in aslist(schema["secondaryFiles"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	802 if 'required' in sf:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	803 sf_required = builder.do_eval(sf['required'], context=primary)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	804 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	805 sf_required = False
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	806
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	807 if "$(" in sf["pattern"] or "${" in sf["pattern"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	808 sfpath = builder.do_eval(sf["pattern"], context=primary)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	809 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	810 sfpath = substitute(primary["basename"], sf["pattern"])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	811
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	812 for sfitem in aslist(sfpath):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	813 if not sfitem:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	814 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	815 if isinstance(sfitem, string_types):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	816 sfitem = {"path": pathprefix+sfitem}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	817 if not fs_access.exists(sfitem['path']) and sf_required:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	818 raise WorkflowException(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	819 "Missing required secondary file '%s'" % (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	820 sfitem["path"]))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	821 if "path" in sfitem and "location" not in sfitem:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	822 revmap(sfitem)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	823 if fs_access.isfile(sfitem["location"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	824 sfitem["class"] = "File"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	825 primary["secondaryFiles"].append(sfitem)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	826 elif fs_access.isdir(sfitem["location"]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	827 sfitem["class"] = "Directory"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	828 primary["secondaryFiles"].append(sfitem)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	829
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	830 if "format" in schema:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	831 for primary in aslist(r):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	832 primary["format"] = builder.do_eval(schema["format"], context=primary)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	833
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	834 # Ensure files point to local references outside of the run environment
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	835 adjustFileObjs(r, revmap)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	836
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	837 if not r and optional:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	838 # Don't convert zero or empty string to None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	839 if r in [0, '']:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	840 return r
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	841 # For [] or None, return None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	842 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	843 return None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	844
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	845 if (not empty_and_optional and isinstance(schema["type"], MutableMapping)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	846 and schema["type"]["type"] == "record"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	847 out = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	848 for f in schema["type"]["fields"]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	849 out[shortname(f["name"])] = self.collect_output( # type: ignore
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	850 f, builder, outdir, fs_access,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	851 compute_checksum=compute_checksum)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	852 return out
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	853 return r

Mercurial > repos > shellac > guppy_basecaller

annotate env/lib/python3.7/site-packages/cwltool/command_line_tool.py @ 4:79f47841a781 draft