Mercurial > repos > guerler > hhblits

annotate lib/python3.8/site-packages/pip/_internal/req/req_file.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
1 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
2 Requirements file parsing
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
3 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
4
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
5 # The following comment should be removed at some point in the future.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
6 # mypy: strict-optional=False
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
7
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
8 from __future__ import absolute_import
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
9
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
10 import optparse
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
11 import os
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
12 import re
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
13 import shlex
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
14 import sys
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
15
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
16 from pip._vendor.six.moves import filterfalse
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
17 from pip._vendor.six.moves.urllib import parse as urllib_parse
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
18
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
19 from pip._internal.cli import cmdoptions
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
20 from pip._internal.exceptions import (
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
21 InstallationError,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
22 RequirementsFileParseError,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
23 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
24 from pip._internal.models.search_scope import SearchScope
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
25 from pip._internal.req.constructors import (
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
26 install_req_from_editable,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
27 install_req_from_line,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
28 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
29 from pip._internal.utils.encoding import auto_decode
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
30 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
31 from pip._internal.utils.urls import get_url_scheme
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
32
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
33 if MYPY_CHECK_RUNNING:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
34 from optparse import Values
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
35 from typing import (
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
36 Any, Callable, Iterator, List, NoReturn, Optional, Text, Tuple,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
37 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
38
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
39 from pip._internal.req import InstallRequirement
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
40 from pip._internal.cache import WheelCache
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
41 from pip._internal.index.package_finder import PackageFinder
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
42 from pip._internal.network.session import PipSession
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
43
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
44 ReqFileLines = Iterator[Tuple[int, Text]]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
45
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
46 LineParser = Callable[[Text], Tuple[str, Values]]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
47
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
48
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
49 __all__ = ['parse_requirements']
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
50
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
51 SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
52 COMMENT_RE = re.compile(r'(^|\s+)#.*$')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
53
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
54 # Matches environment variable-style values in '${MY_VARIABLE_1}' with the
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
55 # variable name consisting of only uppercase letters, digits or the '_'
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
56 # (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
57 # 2013 Edition.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
58 ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
59
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
60 SUPPORTED_OPTIONS = [
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
61 cmdoptions.index_url,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
62 cmdoptions.extra_index_url,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
63 cmdoptions.no_index,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
64 cmdoptions.constraints,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
65 cmdoptions.requirements,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
66 cmdoptions.editable,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
67 cmdoptions.find_links,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
68 cmdoptions.no_binary,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
69 cmdoptions.only_binary,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
70 cmdoptions.require_hashes,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
71 cmdoptions.pre,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
72 cmdoptions.trusted_host,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
73 cmdoptions.always_unzip, # Deprecated
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
74 ] # type: List[Callable[..., optparse.Option]]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
75
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
76 # options to be passed to requirements
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
77 SUPPORTED_OPTIONS_REQ = [
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
78 cmdoptions.install_options,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
79 cmdoptions.global_options,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
80 cmdoptions.hash,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
81 ] # type: List[Callable[..., optparse.Option]]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
82
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
83 # the 'dest' string values
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
84 SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
85
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
86
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
87 class ParsedLine(object):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
88 def __init__(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
89 self,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
90 filename, # type: str
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
91 lineno, # type: int
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
92 comes_from, # type: str
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
93 args, # type: str
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
94 opts, # type: Values
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
95 constraint, # type: bool
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
96 ):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
97 # type: (...) -> None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
98 self.filename = filename
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
99 self.lineno = lineno
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
100 self.comes_from = comes_from
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
101 self.args = args
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
102 self.opts = opts
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
103 self.constraint = constraint
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
104
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
105
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
106 def parse_requirements(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
107 filename, # type: str
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
108 session, # type: PipSession
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
109 finder=None, # type: Optional[PackageFinder]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
110 comes_from=None, # type: Optional[str]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
111 options=None, # type: Optional[optparse.Values]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
112 constraint=False, # type: bool
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
113 wheel_cache=None, # type: Optional[WheelCache]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
114 use_pep517=None # type: Optional[bool]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
115 ):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
116 # type: (...) -> Iterator[InstallRequirement]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
117 """Parse a requirements file and yield InstallRequirement instances.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
118
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
119 :param filename: Path or url of requirements file.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
120 :param session: PipSession instance.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
121 :param finder: Instance of pip.index.PackageFinder.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
122 :param comes_from: Origin description of requirements.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
123 :param options: cli options.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
124 :param constraint: If true, parsing a constraint file rather than
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
125 requirements file.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
126 :param wheel_cache: Instance of pip.wheel.WheelCache
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
127 :param use_pep517: Value of the --use-pep517 option.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
128 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
129 skip_requirements_regex = (
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
130 options.skip_requirements_regex if options else None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
131 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
132 line_parser = get_line_parser(finder)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
133 parser = RequirementsFileParser(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
134 session, line_parser, comes_from, skip_requirements_regex
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
135 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
136
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
137 for parsed_line in parser.parse(filename, constraint):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
138 req = handle_line(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
139 parsed_line, finder, options, session, wheel_cache, use_pep517
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
140 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
141 if req is not None:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
142 yield req
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
143
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
144
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
145 def preprocess(content, skip_requirements_regex):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
146 # type: (Text, Optional[str]) -> ReqFileLines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
147 """Split, filter, and join lines, and return a line iterator
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
148
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
149 :param content: the content of the requirements file
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
150 :param options: cli options
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
151 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
152 lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
153 lines_enum = join_lines(lines_enum)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
154 lines_enum = ignore_comments(lines_enum)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
155 if skip_requirements_regex:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
156 lines_enum = skip_regex(lines_enum, skip_requirements_regex)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
157 lines_enum = expand_env_variables(lines_enum)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
158 return lines_enum
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
159
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
160
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
161 def handle_line(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
162 line, # type: ParsedLine
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
163 finder=None, # type: Optional[PackageFinder]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
164 options=None, # type: Optional[optparse.Values]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
165 session=None, # type: Optional[PipSession]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
166 wheel_cache=None, # type: Optional[WheelCache]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
167 use_pep517=None, # type: Optional[bool]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
168 ):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
169 # type: (...) -> Optional[InstallRequirement]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
170 """Handle a single parsed requirements line; This can result in
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
171 creating/yielding requirements, or updating the finder.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
172
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
173 For lines that contain requirements, the only options that have an effect
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
174 are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
175 requirement. Other options from SUPPORTED_OPTIONS may be present, but are
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
176 ignored.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
177
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
178 For lines that do not contain requirements, the only options that have an
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
179 effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
180 be present, but are ignored. These lines may contain multiple options
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
181 (although our docs imply only one is supported), and all our parsed and
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
182 affect the finder.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
183 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
184
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
185 # preserve for the nested code path
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
186 line_comes_from = '%s %s (line %s)' % (
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
187 '-c' if line.constraint else '-r', line.filename, line.lineno,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
188 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
189
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
190 # return a line requirement
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
191 if line.args:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
192 isolated = options.isolated_mode if options else False
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
193 if options:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
194 cmdoptions.check_install_build_global(options, line.opts)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
195 # get the options that apply to requirements
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
196 req_options = {}
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
197 for dest in SUPPORTED_OPTIONS_REQ_DEST:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
198 if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
199 req_options[dest] = line.opts.__dict__[dest]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
200 line_source = 'line {} of {}'.format(line.lineno, line.filename)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
201 return install_req_from_line(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
202 line.args,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
203 comes_from=line_comes_from,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
204 use_pep517=use_pep517,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
205 isolated=isolated,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
206 options=req_options,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
207 wheel_cache=wheel_cache,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
208 constraint=line.constraint,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
209 line_source=line_source,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
210 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
211
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
212 # return an editable requirement
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
213 elif line.opts.editables:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
214 isolated = options.isolated_mode if options else False
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
215 return install_req_from_editable(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
216 line.opts.editables[0], comes_from=line_comes_from,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
217 use_pep517=use_pep517,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
218 constraint=line.constraint, isolated=isolated,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
219 wheel_cache=wheel_cache
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
220 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
221
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
222 # percolate hash-checking option upward
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
223 elif line.opts.require_hashes:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
224 options.require_hashes = line.opts.require_hashes
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
225
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
226 # set finder options
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
227 elif finder:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
228 find_links = finder.find_links
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
229 index_urls = finder.index_urls
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
230 if line.opts.index_url:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
231 index_urls = [line.opts.index_url]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
232 if line.opts.no_index is True:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
233 index_urls = []
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
234 if line.opts.extra_index_urls:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
235 index_urls.extend(line.opts.extra_index_urls)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
236 if line.opts.find_links:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
237 # FIXME: it would be nice to keep track of the source
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
238 # of the find_links: support a find-links local path
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
239 # relative to a requirements file.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
240 value = line.opts.find_links[0]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
241 req_dir = os.path.dirname(os.path.abspath(line.filename))
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
242 relative_to_reqs_file = os.path.join(req_dir, value)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
243 if os.path.exists(relative_to_reqs_file):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
244 value = relative_to_reqs_file
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
245 find_links.append(value)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
246
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
247 search_scope = SearchScope(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
248 find_links=find_links,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
249 index_urls=index_urls,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
250 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
251 finder.search_scope = search_scope
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
252
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
253 if line.opts.pre:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
254 finder.set_allow_all_prereleases()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
255
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
256 if session:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
257 for host in line.opts.trusted_hosts or []:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
258 source = 'line {} of {}'.format(line.lineno, line.filename)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
259 session.add_trusted_host(host, source=source)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
260
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
261 return None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
262
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
263
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
264 class RequirementsFileParser(object):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
265 def __init__(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
266 self,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
267 session, # type: PipSession
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
268 line_parser, # type: LineParser
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
269 comes_from, # type: str
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
270 skip_requirements_regex, # type: Optional[str]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
271 ):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
272 # type: (...) -> None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
273 self._session = session
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
274 self._line_parser = line_parser
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
275 self._comes_from = comes_from
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
276 self._skip_requirements_regex = skip_requirements_regex
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
277
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
278 def parse(self, filename, constraint):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
279 # type: (str, bool) -> Iterator[ParsedLine]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
280 """Parse a given file, yielding parsed lines.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
281 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
282 for line in self._parse_and_recurse(filename, constraint):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
283 yield line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
284
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
285 def _parse_and_recurse(self, filename, constraint):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
286 # type: (str, bool) -> Iterator[ParsedLine]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
287 for line in self._parse_file(filename, constraint):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
288 if (
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
289 not line.args and
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
290 not line.opts.editables and
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
291 (line.opts.requirements or line.opts.constraints)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
292 ):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
293 # parse a nested requirements file
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
294 if line.opts.requirements:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
295 req_path = line.opts.requirements[0]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
296 nested_constraint = False
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
297 else:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
298 req_path = line.opts.constraints[0]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
299 nested_constraint = True
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
300
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
301 # original file is over http
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
302 if SCHEME_RE.search(filename):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
303 # do a url join so relative paths work
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
304 req_path = urllib_parse.urljoin(filename, req_path)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
305 # original file and nested file are paths
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
306 elif not SCHEME_RE.search(req_path):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
307 # do a join so relative paths work
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
308 req_path = os.path.join(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
309 os.path.dirname(filename), req_path,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
310 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
311
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
312 for inner_line in self._parse_and_recurse(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
313 req_path, nested_constraint,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
314 ):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
315 yield inner_line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
316 else:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
317 yield line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
318
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
319 def _parse_file(self, filename, constraint):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
320 # type: (str, bool) -> Iterator[ParsedLine]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
321 _, content = get_file_content(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
322 filename, self._session, comes_from=self._comes_from
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
323 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
324
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
325 lines_enum = preprocess(content, self._skip_requirements_regex)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
326
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
327 for line_number, line in lines_enum:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
328 try:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
329 args_str, opts = self._line_parser(line)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
330 except OptionParsingError as e:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
331 # add offending line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
332 msg = 'Invalid requirement: %s\n%s' % (line, e.msg)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
333 raise RequirementsFileParseError(msg)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
334
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
335 yield ParsedLine(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
336 filename,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
337 line_number,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
338 self._comes_from,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
339 args_str,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
340 opts,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
341 constraint,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
342 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
343
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
344
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
345 def get_line_parser(finder):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
346 # type: (Optional[PackageFinder]) -> LineParser
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
347 def parse_line(line):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
348 # type: (Text) -> Tuple[str, Values]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
349 # Build new parser for each line since it accumulates appendable
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
350 # options.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
351 parser = build_parser()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
352 defaults = parser.get_default_values()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
353 defaults.index_url = None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
354 if finder:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
355 defaults.format_control = finder.format_control
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
356
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
357 args_str, options_str = break_args_options(line)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
358 # Prior to 2.7.3, shlex cannot deal with unicode entries
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
359 if sys.version_info < (2, 7, 3):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
360 # https://github.com/python/mypy/issues/1174
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
361 options_str = options_str.encode('utf8') # type: ignore
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
362
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
363 # https://github.com/python/mypy/issues/1174
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
364 opts, _ = parser.parse_args(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
365 shlex.split(options_str), defaults) # type: ignore
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
366
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
367 return args_str, opts
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
368
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
369 return parse_line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
370
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
371
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
372 def break_args_options(line):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
373 # type: (Text) -> Tuple[str, Text]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
374 """Break up the line into an args and options string. We only want to shlex
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
375 (and then optparse) the options, not the args. args can contain markers
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
376 which are corrupted by shlex.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
377 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
378 tokens = line.split(' ')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
379 args = []
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
380 options = tokens[:]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
381 for token in tokens:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
382 if token.startswith('-') or token.startswith('--'):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
383 break
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
384 else:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
385 args.append(token)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
386 options.pop(0)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
387 return ' '.join(args), ' '.join(options) # type: ignore
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
388
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
389
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
390 class OptionParsingError(Exception):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
391 def __init__(self, msg):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
392 # type: (str) -> None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
393 self.msg = msg
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
394
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
395
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
396 def build_parser():
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
397 # type: () -> optparse.OptionParser
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
398 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
399 Return a parser for parsing requirement lines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
400 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
401 parser = optparse.OptionParser(add_help_option=False)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
402
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
403 option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
404 for option_factory in option_factories:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
405 option = option_factory()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
406 parser.add_option(option)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
407
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
408 # By default optparse sys.exits on parsing errors. We want to wrap
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
409 # that in our own exception.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
410 def parser_exit(self, msg):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
411 # type: (Any, str) -> NoReturn
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
412 raise OptionParsingError(msg)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
413 # NOTE: mypy disallows assigning to a method
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
414 # https://github.com/python/mypy/issues/2427
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
415 parser.exit = parser_exit # type: ignore
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
416
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
417 return parser
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
418
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
419
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
420 def join_lines(lines_enum):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
421 # type: (ReqFileLines) -> ReqFileLines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
422 """Joins a line ending in '\' with the previous line (except when following
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
423 comments). The joined line takes on the index of the first line.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
424 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
425 primary_line_number = None
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
426 new_line = [] # type: List[Text]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
427 for line_number, line in lines_enum:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
428 if not line.endswith('\\') or COMMENT_RE.match(line):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
429 if COMMENT_RE.match(line):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
430 # this ensures comments are always matched later
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
431 line = ' ' + line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
432 if new_line:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
433 new_line.append(line)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
434 yield primary_line_number, ''.join(new_line)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
435 new_line = []
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
436 else:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
437 yield line_number, line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
438 else:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
439 if not new_line:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
440 primary_line_number = line_number
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
441 new_line.append(line.strip('\\'))
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
442
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
443 # last line contains \
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
444 if new_line:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
445 yield primary_line_number, ''.join(new_line)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
446
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
447 # TODO: handle space after '\'.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
448
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
449
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
450 def ignore_comments(lines_enum):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
451 # type: (ReqFileLines) -> ReqFileLines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
452 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
453 Strips comments and filter empty lines.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
454 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
455 for line_number, line in lines_enum:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
456 line = COMMENT_RE.sub('', line)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
457 line = line.strip()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
458 if line:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
459 yield line_number, line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
460
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
461
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
462 def skip_regex(lines_enum, pattern):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
463 # type: (ReqFileLines, str) -> ReqFileLines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
464 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
465 Skip lines that match the provided pattern
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
466
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
467 Note: the regex pattern is only built once
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
468 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
469 matcher = re.compile(pattern)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
470 lines_enum = filterfalse(lambda e: matcher.search(e[1]), lines_enum)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
471 return lines_enum
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
472
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
473
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
474 def expand_env_variables(lines_enum):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
475 # type: (ReqFileLines) -> ReqFileLines
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
476 """Replace all environment variables that can be retrieved via `os.getenv`.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
477
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
478 The only allowed format for environment variables defined in the
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
479 requirement file is `${MY_VARIABLE_1}` to ensure two things:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
480
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
481 1. Strings that contain a `$` aren't accidentally (partially) expanded.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
482 2. Ensure consistency across platforms for requirement files.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
483
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
484 These points are the result of a discussion on the `github pull
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
485 request #3514 <https://github.com/pypa/pip/pull/3514>`_.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
486
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
487 Valid characters in variable names follow the `POSIX standard
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
488 <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
489 to uppercase letter, digits and the `_` (underscore).
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
490 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
491 for line_number, line in lines_enum:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
492 for env_var, var_name in ENV_VAR_RE.findall(line):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
493 value = os.getenv(var_name)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
494 if not value:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
495 continue
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
496
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
497 line = line.replace(env_var, value)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
498
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
499 yield line_number, line
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
500
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
501
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
502 def get_file_content(url, session, comes_from=None):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
503 # type: (str, PipSession, Optional[str]) -> Tuple[str, Text]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
504 """Gets the content of a file; it may be a filename, file: URL, or
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
505 http: URL. Returns (location, content). Content is unicode.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
506 Respects # -*- coding: declarations on the retrieved files.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
507
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
508 :param url: File path or url.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
509 :param session: PipSession instance.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
510 :param comes_from: Origin description of requirements.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
511 """
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
512 scheme = get_url_scheme(url)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
513
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
514 if scheme in ['http', 'https']:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
515 # FIXME: catch some errors
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
516 resp = session.get(url)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
517 resp.raise_for_status()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
518 return resp.url, resp.text
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
519
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
520 elif scheme == 'file':
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
521 if comes_from and comes_from.startswith('http'):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
522 raise InstallationError(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
523 'Requirements file %s references URL %s, which is local'
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
524 % (comes_from, url))
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
525
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
526 path = url.split(':', 1)[1]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
527 path = path.replace('\\', '/')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
528 match = _url_slash_drive_re.match(path)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
529 if match:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
530 path = match.group(1) + ':' + path.split('|', 1)[1]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
531 path = urllib_parse.unquote(path)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
532 if path.startswith('/'):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
533 path = '/' + path.lstrip('/')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
534 url = path
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
535
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
536 try:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
537 with open(url, 'rb') as f:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
538 content = auto_decode(f.read())
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
539 except IOError as exc:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
540 raise InstallationError(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
541 'Could not open requirements file: %s' % str(exc)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
542 )
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
543 return url, content
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
544
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
545
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
546 _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)