Mercurial > repos > guerler > hhblits

comparison lib/python3.8/site-packages/pip/_internal/req/req_file.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:9e54283cc701
1 """
2 Requirements file parsing
3 """
4
5 # The following comment should be removed at some point in the future.
6 # mypy: strict-optional=False
7
8 from __future__ import absolute_import
9
10 import optparse
11 import os
12 import re
13 import shlex
14 import sys
15
16 from pip._vendor.six.moves import filterfalse
17 from pip._vendor.six.moves.urllib import parse as urllib_parse
18
19 from pip._internal.cli import cmdoptions
20 from pip._internal.exceptions import (
21 InstallationError,
22 RequirementsFileParseError,
23 )
24 from pip._internal.models.search_scope import SearchScope
25 from pip._internal.req.constructors import (
26 install_req_from_editable,
27 install_req_from_line,
28 )
29 from pip._internal.utils.encoding import auto_decode
30 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
31 from pip._internal.utils.urls import get_url_scheme
32
33 if MYPY_CHECK_RUNNING:
34 from optparse import Values
35 from typing import (
36 Any, Callable, Iterator, List, NoReturn, Optional, Text, Tuple,
37 )
38
39 from pip._internal.req import InstallRequirement
40 from pip._internal.cache import WheelCache
41 from pip._internal.index.package_finder import PackageFinder
42 from pip._internal.network.session import PipSession
43
44 ReqFileLines = Iterator[Tuple[int, Text]]
45
46 LineParser = Callable[[Text], Tuple[str, Values]]
47
48
49 __all__ = ['parse_requirements']
50
51 SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
52 COMMENT_RE = re.compile(r'(^|\s+)#.*$')
53
54 # Matches environment variable-style values in '${MY_VARIABLE_1}' with the
55 # variable name consisting of only uppercase letters, digits or the '_'
56 # (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
57 # 2013 Edition.
58 ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
59
60 SUPPORTED_OPTIONS = [
61 cmdoptions.index_url,
62 cmdoptions.extra_index_url,
63 cmdoptions.no_index,
64 cmdoptions.constraints,
65 cmdoptions.requirements,
66 cmdoptions.editable,
67 cmdoptions.find_links,
68 cmdoptions.no_binary,
69 cmdoptions.only_binary,
70 cmdoptions.require_hashes,
71 cmdoptions.pre,
72 cmdoptions.trusted_host,
73 cmdoptions.always_unzip, # Deprecated
74 ] # type: List[Callable[..., optparse.Option]]
75
76 # options to be passed to requirements
77 SUPPORTED_OPTIONS_REQ = [
78 cmdoptions.install_options,
79 cmdoptions.global_options,
80 cmdoptions.hash,
81 ] # type: List[Callable[..., optparse.Option]]
82
83 # the 'dest' string values
84 SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]
85
86
87 class ParsedLine(object):
88 def __init__(
89 self,
90 filename, # type: str
91 lineno, # type: int
92 comes_from, # type: str
93 args, # type: str
94 opts, # type: Values
95 constraint, # type: bool
96 ):
97 # type: (...) -> None
98 self.filename = filename
99 self.lineno = lineno
100 self.comes_from = comes_from
101 self.args = args
102 self.opts = opts
103 self.constraint = constraint
104
105
106 def parse_requirements(
107 filename, # type: str
108 session, # type: PipSession
109 finder=None, # type: Optional[PackageFinder]
110 comes_from=None, # type: Optional[str]
111 options=None, # type: Optional[optparse.Values]
112 constraint=False, # type: bool
113 wheel_cache=None, # type: Optional[WheelCache]
114 use_pep517=None # type: Optional[bool]
115 ):
116 # type: (...) -> Iterator[InstallRequirement]
117 """Parse a requirements file and yield InstallRequirement instances.
118
119 :param filename: Path or url of requirements file.
120 :param session: PipSession instance.
121 :param finder: Instance of pip.index.PackageFinder.
122 :param comes_from: Origin description of requirements.
123 :param options: cli options.
124 :param constraint: If true, parsing a constraint file rather than
125 requirements file.
126 :param wheel_cache: Instance of pip.wheel.WheelCache
127 :param use_pep517: Value of the --use-pep517 option.
128 """
129 skip_requirements_regex = (
130 options.skip_requirements_regex if options else None
131 )
132 line_parser = get_line_parser(finder)
133 parser = RequirementsFileParser(
134 session, line_parser, comes_from, skip_requirements_regex
135 )
136
137 for parsed_line in parser.parse(filename, constraint):
138 req = handle_line(
139 parsed_line, finder, options, session, wheel_cache, use_pep517
140 )
141 if req is not None:
142 yield req
143
144
145 def preprocess(content, skip_requirements_regex):
146 # type: (Text, Optional[str]) -> ReqFileLines
147 """Split, filter, and join lines, and return a line iterator
148
149 :param content: the content of the requirements file
150 :param options: cli options
151 """
152 lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines
153 lines_enum = join_lines(lines_enum)
154 lines_enum = ignore_comments(lines_enum)
155 if skip_requirements_regex:
156 lines_enum = skip_regex(lines_enum, skip_requirements_regex)
157 lines_enum = expand_env_variables(lines_enum)
158 return lines_enum
159
160
161 def handle_line(
162 line, # type: ParsedLine
163 finder=None, # type: Optional[PackageFinder]
164 options=None, # type: Optional[optparse.Values]
165 session=None, # type: Optional[PipSession]
166 wheel_cache=None, # type: Optional[WheelCache]
167 use_pep517=None, # type: Optional[bool]
168 ):
169 # type: (...) -> Optional[InstallRequirement]
170 """Handle a single parsed requirements line; This can result in
171 creating/yielding requirements, or updating the finder.
172
173 For lines that contain requirements, the only options that have an effect
174 are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
175 requirement. Other options from SUPPORTED_OPTIONS may be present, but are
176 ignored.
177
178 For lines that do not contain requirements, the only options that have an
179 effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
180 be present, but are ignored. These lines may contain multiple options
181 (although our docs imply only one is supported), and all our parsed and
182 affect the finder.
183 """
184
185 # preserve for the nested code path
186 line_comes_from = '%s %s (line %s)' % (
187 '-c' if line.constraint else '-r', line.filename, line.lineno,
188 )
189
190 # return a line requirement
191 if line.args:
192 isolated = options.isolated_mode if options else False
193 if options:
194 cmdoptions.check_install_build_global(options, line.opts)
195 # get the options that apply to requirements
196 req_options = {}
197 for dest in SUPPORTED_OPTIONS_REQ_DEST:
198 if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
199 req_options[dest] = line.opts.__dict__[dest]
200 line_source = 'line {} of {}'.format(line.lineno, line.filename)
201 return install_req_from_line(
202 line.args,
203 comes_from=line_comes_from,
204 use_pep517=use_pep517,
205 isolated=isolated,
206 options=req_options,
207 wheel_cache=wheel_cache,
208 constraint=line.constraint,
209 line_source=line_source,
210 )
211
212 # return an editable requirement
213 elif line.opts.editables:
214 isolated = options.isolated_mode if options else False
215 return install_req_from_editable(
216 line.opts.editables[0], comes_from=line_comes_from,
217 use_pep517=use_pep517,
218 constraint=line.constraint, isolated=isolated,
219 wheel_cache=wheel_cache
220 )
221
222 # percolate hash-checking option upward
223 elif line.opts.require_hashes:
224 options.require_hashes = line.opts.require_hashes
225
226 # set finder options
227 elif finder:
228 find_links = finder.find_links
229 index_urls = finder.index_urls
230 if line.opts.index_url:
231 index_urls = [line.opts.index_url]
232 if line.opts.no_index is True:
233 index_urls = []
234 if line.opts.extra_index_urls:
235 index_urls.extend(line.opts.extra_index_urls)
236 if line.opts.find_links:
237 # FIXME: it would be nice to keep track of the source
238 # of the find_links: support a find-links local path
239 # relative to a requirements file.
240 value = line.opts.find_links[0]
241 req_dir = os.path.dirname(os.path.abspath(line.filename))
242 relative_to_reqs_file = os.path.join(req_dir, value)
243 if os.path.exists(relative_to_reqs_file):
244 value = relative_to_reqs_file
245 find_links.append(value)
246
247 search_scope = SearchScope(
248 find_links=find_links,
249 index_urls=index_urls,
250 )
251 finder.search_scope = search_scope
252
253 if line.opts.pre:
254 finder.set_allow_all_prereleases()
255
256 if session:
257 for host in line.opts.trusted_hosts or []:
258 source = 'line {} of {}'.format(line.lineno, line.filename)
259 session.add_trusted_host(host, source=source)
260
261 return None
262
263
264 class RequirementsFileParser(object):
265 def __init__(
266 self,
267 session, # type: PipSession
268 line_parser, # type: LineParser
269 comes_from, # type: str
270 skip_requirements_regex, # type: Optional[str]
271 ):
272 # type: (...) -> None
273 self._session = session
274 self._line_parser = line_parser
275 self._comes_from = comes_from
276 self._skip_requirements_regex = skip_requirements_regex
277
278 def parse(self, filename, constraint):
279 # type: (str, bool) -> Iterator[ParsedLine]
280 """Parse a given file, yielding parsed lines.
281 """
282 for line in self._parse_and_recurse(filename, constraint):
283 yield line
284
285 def _parse_and_recurse(self, filename, constraint):
286 # type: (str, bool) -> Iterator[ParsedLine]
287 for line in self._parse_file(filename, constraint):
288 if (
289 not line.args and
290 not line.opts.editables and
291 (line.opts.requirements or line.opts.constraints)
292 ):
293 # parse a nested requirements file
294 if line.opts.requirements:
295 req_path = line.opts.requirements[0]
296 nested_constraint = False
297 else:
298 req_path = line.opts.constraints[0]
299 nested_constraint = True
300
301 # original file is over http
302 if SCHEME_RE.search(filename):
303 # do a url join so relative paths work
304 req_path = urllib_parse.urljoin(filename, req_path)
305 # original file and nested file are paths
306 elif not SCHEME_RE.search(req_path):
307 # do a join so relative paths work
308 req_path = os.path.join(
309 os.path.dirname(filename), req_path,
310 )
311
312 for inner_line in self._parse_and_recurse(
313 req_path, nested_constraint,
314 ):
315 yield inner_line
316 else:
317 yield line
318
319 def _parse_file(self, filename, constraint):
320 # type: (str, bool) -> Iterator[ParsedLine]
321 _, content = get_file_content(
322 filename, self._session, comes_from=self._comes_from
323 )
324
325 lines_enum = preprocess(content, self._skip_requirements_regex)
326
327 for line_number, line in lines_enum:
328 try:
329 args_str, opts = self._line_parser(line)
330 except OptionParsingError as e:
331 # add offending line
332 msg = 'Invalid requirement: %s\n%s' % (line, e.msg)
333 raise RequirementsFileParseError(msg)
334
335 yield ParsedLine(
336 filename,
337 line_number,
338 self._comes_from,
339 args_str,
340 opts,
341 constraint,
342 )
343
344
345 def get_line_parser(finder):
346 # type: (Optional[PackageFinder]) -> LineParser
347 def parse_line(line):
348 # type: (Text) -> Tuple[str, Values]
349 # Build new parser for each line since it accumulates appendable
350 # options.
351 parser = build_parser()
352 defaults = parser.get_default_values()
353 defaults.index_url = None
354 if finder:
355 defaults.format_control = finder.format_control
356
357 args_str, options_str = break_args_options(line)
358 # Prior to 2.7.3, shlex cannot deal with unicode entries
359 if sys.version_info < (2, 7, 3):
360 # https://github.com/python/mypy/issues/1174
361 options_str = options_str.encode('utf8') # type: ignore
362
363 # https://github.com/python/mypy/issues/1174
364 opts, _ = parser.parse_args(
365 shlex.split(options_str), defaults) # type: ignore
366
367 return args_str, opts
368
369 return parse_line
370
371
372 def break_args_options(line):
373 # type: (Text) -> Tuple[str, Text]
374 """Break up the line into an args and options string. We only want to shlex
375 (and then optparse) the options, not the args. args can contain markers
376 which are corrupted by shlex.
377 """
378 tokens = line.split(' ')
379 args = []
380 options = tokens[:]
381 for token in tokens:
382 if token.startswith('-') or token.startswith('--'):
383 break
384 else:
385 args.append(token)
386 options.pop(0)
387 return ' '.join(args), ' '.join(options) # type: ignore
388
389
390 class OptionParsingError(Exception):
391 def __init__(self, msg):
392 # type: (str) -> None
393 self.msg = msg
394
395
396 def build_parser():
397 # type: () -> optparse.OptionParser
398 """
399 Return a parser for parsing requirement lines
400 """
401 parser = optparse.OptionParser(add_help_option=False)
402
403 option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
404 for option_factory in option_factories:
405 option = option_factory()
406 parser.add_option(option)
407
408 # By default optparse sys.exits on parsing errors. We want to wrap
409 # that in our own exception.
410 def parser_exit(self, msg):
411 # type: (Any, str) -> NoReturn
412 raise OptionParsingError(msg)
413 # NOTE: mypy disallows assigning to a method
414 # https://github.com/python/mypy/issues/2427
415 parser.exit = parser_exit # type: ignore
416
417 return parser
418
419
420 def join_lines(lines_enum):
421 # type: (ReqFileLines) -> ReqFileLines
422 """Joins a line ending in '\' with the previous line (except when following
423 comments). The joined line takes on the index of the first line.
424 """
425 primary_line_number = None
426 new_line = [] # type: List[Text]
427 for line_number, line in lines_enum:
428 if not line.endswith('\\') or COMMENT_RE.match(line):
429 if COMMENT_RE.match(line):
430 # this ensures comments are always matched later
431 line = ' ' + line
432 if new_line:
433 new_line.append(line)
434 yield primary_line_number, ''.join(new_line)
435 new_line = []
436 else:
437 yield line_number, line
438 else:
439 if not new_line:
440 primary_line_number = line_number
441 new_line.append(line.strip('\\'))
442
443 # last line contains \
444 if new_line:
445 yield primary_line_number, ''.join(new_line)
446
447 # TODO: handle space after '\'.
448
449
450 def ignore_comments(lines_enum):
451 # type: (ReqFileLines) -> ReqFileLines
452 """
453 Strips comments and filter empty lines.
454 """
455 for line_number, line in lines_enum:
456 line = COMMENT_RE.sub('', line)
457 line = line.strip()
458 if line:
459 yield line_number, line
460
461
462 def skip_regex(lines_enum, pattern):
463 # type: (ReqFileLines, str) -> ReqFileLines
464 """
465 Skip lines that match the provided pattern
466
467 Note: the regex pattern is only built once
468 """
469 matcher = re.compile(pattern)
470 lines_enum = filterfalse(lambda e: matcher.search(e[1]), lines_enum)
471 return lines_enum
472
473
474 def expand_env_variables(lines_enum):
475 # type: (ReqFileLines) -> ReqFileLines
476 """Replace all environment variables that can be retrieved via `os.getenv`.
477
478 The only allowed format for environment variables defined in the
479 requirement file is `${MY_VARIABLE_1}` to ensure two things:
480
481 1. Strings that contain a `$` aren't accidentally (partially) expanded.
482 2. Ensure consistency across platforms for requirement files.
483
484 These points are the result of a discussion on the `github pull
485 request #3514 <https://github.com/pypa/pip/pull/3514>`_.
486
487 Valid characters in variable names follow the `POSIX standard
488 <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
489 to uppercase letter, digits and the `_` (underscore).
490 """
491 for line_number, line in lines_enum:
492 for env_var, var_name in ENV_VAR_RE.findall(line):
493 value = os.getenv(var_name)
494 if not value:
495 continue
496
497 line = line.replace(env_var, value)
498
499 yield line_number, line
500
501
502 def get_file_content(url, session, comes_from=None):
503 # type: (str, PipSession, Optional[str]) -> Tuple[str, Text]
504 """Gets the content of a file; it may be a filename, file: URL, or
505 http: URL. Returns (location, content). Content is unicode.
506 Respects # -*- coding: declarations on the retrieved files.
507
508 :param url: File path or url.
509 :param session: PipSession instance.
510 :param comes_from: Origin description of requirements.
511 """
512 scheme = get_url_scheme(url)
513
514 if scheme in ['http', 'https']:
515 # FIXME: catch some errors
516 resp = session.get(url)
517 resp.raise_for_status()
518 return resp.url, resp.text
519
520 elif scheme == 'file':
521 if comes_from and comes_from.startswith('http'):
522 raise InstallationError(
523 'Requirements file %s references URL %s, which is local'
524 % (comes_from, url))
525
526 path = url.split(':', 1)[1]
527 path = path.replace('\\', '/')
528 match = _url_slash_drive_re.match(path)
529 if match:
530 path = match.group(1) + ':' + path.split('|', 1)[1]
531 path = urllib_parse.unquote(path)
532 if path.startswith('/'):
533 path = '/' + path.lstrip('/')
534 url = path
535
536 try:
537 with open(url, 'rb') as f:
538 content = auto_decode(f.read())
539 except IOError as exc:
540 raise InstallationError(
541 'Could not open requirements file: %s' % str(exc)
542 )
543 return url, content
544
545
546 _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)