Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/pip/_internal/utils/misc.py @ 1:56ad4e20f292 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:32:28 -0400
parents
children
comparison
equal deleted inserted replaced
0:d30785e31577 1:56ad4e20f292
1 from __future__ import absolute_import
2
3 import contextlib
4 import errno
5 import getpass
6 import io
7 # we have a submodule named 'logging' which would shadow this if we used the
8 # regular name:
9 import logging as std_logging
10 import os
11 import posixpath
12 import re
13 import shutil
14 import stat
15 import subprocess
16 import sys
17 import tarfile
18 import zipfile
19 from collections import deque
20
21 from pip._vendor import pkg_resources
22 # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
23 # why we ignore the type on this import.
24 from pip._vendor.retrying import retry # type: ignore
25 from pip._vendor.six import PY2, text_type
26 from pip._vendor.six.moves import input, shlex_quote
27 from pip._vendor.six.moves.urllib import parse as urllib_parse
28 from pip._vendor.six.moves.urllib import request as urllib_request
29 from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
30
31 from pip import __version__
32 from pip._internal.exceptions import CommandError, InstallationError
33 from pip._internal.locations import site_packages, user_site
34 from pip._internal.utils.compat import (
35 WINDOWS, console_to_str, expanduser, stdlib_pkgs, str_to_display,
36 )
37 from pip._internal.utils.marker_files import write_delete_marker_file
38 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
39 from pip._internal.utils.virtualenv import (
40 running_under_virtualenv, virtualenv_no_global,
41 )
42
43 if PY2:
44 from io import BytesIO as StringIO
45 else:
46 from io import StringIO
47
48 if MYPY_CHECK_RUNNING:
49 from typing import (
50 Any, AnyStr, Container, Iterable, List, Mapping, Match, Optional, Text,
51 Tuple, Union, cast,
52 )
53 from pip._vendor.pkg_resources import Distribution
54 from pip._internal.models.link import Link
55 from pip._internal.utils.ui import SpinnerInterface
56
57 VersionInfo = Tuple[int, int, int]
58 else:
59 # typing's cast() is needed at runtime, but we don't want to import typing.
60 # Thus, we use a dummy no-op version, which we tell mypy to ignore.
61 def cast(type_, value): # type: ignore
62 return value
63
64
65 __all__ = ['rmtree', 'display_path', 'backup_dir',
66 'ask', 'splitext',
67 'format_size', 'is_installable_dir',
68 'is_svn_page', 'file_contents',
69 'split_leading_dir', 'has_leading_dir',
70 'normalize_path',
71 'renames', 'get_prog',
72 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess',
73 'captured_stdout', 'ensure_dir',
74 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', 'WHEEL_EXTENSION',
75 'get_installed_version', 'remove_auth_from_url']
76
77
78 logger = std_logging.getLogger(__name__)
79 subprocess_logger = std_logging.getLogger('pip.subprocessor')
80
81 LOG_DIVIDER = '----------------------------------------'
82
83 WHEEL_EXTENSION = '.whl'
84 BZ2_EXTENSIONS = ('.tar.bz2', '.tbz')
85 XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma')
86 ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION)
87 TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar')
88 ARCHIVE_EXTENSIONS = (
89 ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS)
90 SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS
91
92 try:
93 import bz2 # noqa
94 SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS
95 except ImportError:
96 logger.debug('bz2 module is not available')
97
98 try:
99 # Only for Python 3.3+
100 import lzma # noqa
101 SUPPORTED_EXTENSIONS += XZ_EXTENSIONS
102 except ImportError:
103 logger.debug('lzma module is not available')
104
105
106 def get_pip_version():
107 # type: () -> str
108 pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..")
109 pip_pkg_dir = os.path.abspath(pip_pkg_dir)
110
111 return (
112 'pip {} from {} (python {})'.format(
113 __version__, pip_pkg_dir, sys.version[:3],
114 )
115 )
116
117
118 def normalize_version_info(py_version_info):
119 # type: (Tuple[int, ...]) -> Tuple[int, int, int]
120 """
121 Convert a tuple of ints representing a Python version to one of length
122 three.
123
124 :param py_version_info: a tuple of ints representing a Python version,
125 or None to specify no version. The tuple can have any length.
126
127 :return: a tuple of length three if `py_version_info` is non-None.
128 Otherwise, return `py_version_info` unchanged (i.e. None).
129 """
130 if len(py_version_info) < 3:
131 py_version_info += (3 - len(py_version_info)) * (0,)
132 elif len(py_version_info) > 3:
133 py_version_info = py_version_info[:3]
134
135 return cast('VersionInfo', py_version_info)
136
137
138 def ensure_dir(path):
139 # type: (AnyStr) -> None
140 """os.path.makedirs without EEXIST."""
141 try:
142 os.makedirs(path)
143 except OSError as e:
144 if e.errno != errno.EEXIST:
145 raise
146
147
148 def get_prog():
149 # type: () -> str
150 try:
151 prog = os.path.basename(sys.argv[0])
152 if prog in ('__main__.py', '-c'):
153 return "%s -m pip" % sys.executable
154 else:
155 return prog
156 except (AttributeError, TypeError, IndexError):
157 pass
158 return 'pip'
159
160
161 # Retry every half second for up to 3 seconds
162 @retry(stop_max_delay=3000, wait_fixed=500)
163 def rmtree(dir, ignore_errors=False):
164 # type: (str, bool) -> None
165 shutil.rmtree(dir, ignore_errors=ignore_errors,
166 onerror=rmtree_errorhandler)
167
168
169 def rmtree_errorhandler(func, path, exc_info):
170 """On Windows, the files in .svn are read-only, so when rmtree() tries to
171 remove them, an exception is thrown. We catch that here, remove the
172 read-only attribute, and hopefully continue without problems."""
173 # if file type currently read only
174 if os.stat(path).st_mode & stat.S_IREAD:
175 # convert to read/write
176 os.chmod(path, stat.S_IWRITE)
177 # use the original function to repeat the operation
178 func(path)
179 return
180 else:
181 raise
182
183
184 def path_to_display(path):
185 # type: (Optional[Union[str, Text]]) -> Optional[Text]
186 """
187 Convert a bytes (or text) path to text (unicode in Python 2) for display
188 and logging purposes.
189
190 This function should never error out. Also, this function is mainly needed
191 for Python 2 since in Python 3 str paths are already text.
192 """
193 if path is None:
194 return None
195 if isinstance(path, text_type):
196 return path
197 # Otherwise, path is a bytes object (str in Python 2).
198 try:
199 display_path = path.decode(sys.getfilesystemencoding(), 'strict')
200 except UnicodeDecodeError:
201 # Include the full bytes to make troubleshooting easier, even though
202 # it may not be very human readable.
203 if PY2:
204 # Convert the bytes to a readable str representation using
205 # repr(), and then convert the str to unicode.
206 # Also, we add the prefix "b" to the repr() return value both
207 # to make the Python 2 output look like the Python 3 output, and
208 # to signal to the user that this is a bytes representation.
209 display_path = str_to_display('b{!r}'.format(path))
210 else:
211 # Silence the "F821 undefined name 'ascii'" flake8 error since
212 # in Python 3 ascii() is a built-in.
213 display_path = ascii(path) # noqa: F821
214
215 return display_path
216
217
218 def display_path(path):
219 # type: (Union[str, Text]) -> str
220 """Gives the display value for a given path, making it relative to cwd
221 if possible."""
222 path = os.path.normcase(os.path.abspath(path))
223 if sys.version_info[0] == 2:
224 path = path.decode(sys.getfilesystemencoding(), 'replace')
225 path = path.encode(sys.getdefaultencoding(), 'replace')
226 if path.startswith(os.getcwd() + os.path.sep):
227 path = '.' + path[len(os.getcwd()):]
228 return path
229
230
231 def backup_dir(dir, ext='.bak'):
232 # type: (str, str) -> str
233 """Figure out the name of a directory to back up the given dir to
234 (adding .bak, .bak2, etc)"""
235 n = 1
236 extension = ext
237 while os.path.exists(dir + extension):
238 n += 1
239 extension = ext + str(n)
240 return dir + extension
241
242
243 def ask_path_exists(message, options):
244 # type: (str, Iterable[str]) -> str
245 for action in os.environ.get('PIP_EXISTS_ACTION', '').split():
246 if action in options:
247 return action
248 return ask(message, options)
249
250
251 def _check_no_input(message):
252 # type: (str) -> None
253 """Raise an error if no input is allowed."""
254 if os.environ.get('PIP_NO_INPUT'):
255 raise Exception(
256 'No input was expected ($PIP_NO_INPUT set); question: %s' %
257 message
258 )
259
260
261 def ask(message, options):
262 # type: (str, Iterable[str]) -> str
263 """Ask the message interactively, with the given possible responses"""
264 while 1:
265 _check_no_input(message)
266 response = input(message)
267 response = response.strip().lower()
268 if response not in options:
269 print(
270 'Your response (%r) was not one of the expected responses: '
271 '%s' % (response, ', '.join(options))
272 )
273 else:
274 return response
275
276
277 def ask_input(message):
278 # type: (str) -> str
279 """Ask for input interactively."""
280 _check_no_input(message)
281 return input(message)
282
283
284 def ask_password(message):
285 # type: (str) -> str
286 """Ask for a password interactively."""
287 _check_no_input(message)
288 return getpass.getpass(message)
289
290
291 def format_size(bytes):
292 # type: (float) -> str
293 if bytes > 1000 * 1000:
294 return '%.1fMB' % (bytes / 1000.0 / 1000)
295 elif bytes > 10 * 1000:
296 return '%ikB' % (bytes / 1000)
297 elif bytes > 1000:
298 return '%.1fkB' % (bytes / 1000.0)
299 else:
300 return '%ibytes' % bytes
301
302
303 def is_installable_dir(path):
304 # type: (str) -> bool
305 """Is path is a directory containing setup.py or pyproject.toml?
306 """
307 if not os.path.isdir(path):
308 return False
309 setup_py = os.path.join(path, 'setup.py')
310 if os.path.isfile(setup_py):
311 return True
312 pyproject_toml = os.path.join(path, 'pyproject.toml')
313 if os.path.isfile(pyproject_toml):
314 return True
315 return False
316
317
318 def is_svn_page(html):
319 # type: (Union[str, Text]) -> Optional[Match[Union[str, Text]]]
320 """
321 Returns true if the page appears to be the index page of an svn repository
322 """
323 return (re.search(r'<title>[^<]*Revision \d+:', html) and
324 re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I))
325
326
327 def file_contents(filename):
328 # type: (str) -> Text
329 with open(filename, 'rb') as fp:
330 return fp.read().decode('utf-8')
331
332
333 def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
334 """Yield pieces of data from a file-like object until EOF."""
335 while True:
336 chunk = file.read(size)
337 if not chunk:
338 break
339 yield chunk
340
341
342 def split_leading_dir(path):
343 # type: (Union[str, Text]) -> List[Union[str, Text]]
344 path = path.lstrip('/').lstrip('\\')
345 if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or
346 '\\' not in path):
347 return path.split('/', 1)
348 elif '\\' in path:
349 return path.split('\\', 1)
350 else:
351 return [path, '']
352
353
354 def has_leading_dir(paths):
355 # type: (Iterable[Union[str, Text]]) -> bool
356 """Returns true if all the paths have the same leading path name
357 (i.e., everything is in one subdirectory in an archive)"""
358 common_prefix = None
359 for path in paths:
360 prefix, rest = split_leading_dir(path)
361 if not prefix:
362 return False
363 elif common_prefix is None:
364 common_prefix = prefix
365 elif prefix != common_prefix:
366 return False
367 return True
368
369
370 def normalize_path(path, resolve_symlinks=True):
371 # type: (str, bool) -> str
372 """
373 Convert a path to its canonical, case-normalized, absolute version.
374
375 """
376 path = expanduser(path)
377 if resolve_symlinks:
378 path = os.path.realpath(path)
379 else:
380 path = os.path.abspath(path)
381 return os.path.normcase(path)
382
383
384 def splitext(path):
385 # type: (str) -> Tuple[str, str]
386 """Like os.path.splitext, but take off .tar too"""
387 base, ext = posixpath.splitext(path)
388 if base.lower().endswith('.tar'):
389 ext = base[-4:] + ext
390 base = base[:-4]
391 return base, ext
392
393
394 def renames(old, new):
395 # type: (str, str) -> None
396 """Like os.renames(), but handles renaming across devices."""
397 # Implementation borrowed from os.renames().
398 head, tail = os.path.split(new)
399 if head and tail and not os.path.exists(head):
400 os.makedirs(head)
401
402 shutil.move(old, new)
403
404 head, tail = os.path.split(old)
405 if head and tail:
406 try:
407 os.removedirs(head)
408 except OSError:
409 pass
410
411
412 def is_local(path):
413 # type: (str) -> bool
414 """
415 Return True if path is within sys.prefix, if we're running in a virtualenv.
416
417 If we're not in a virtualenv, all paths are considered "local."
418
419 """
420 if not running_under_virtualenv():
421 return True
422 return normalize_path(path).startswith(normalize_path(sys.prefix))
423
424
425 def dist_is_local(dist):
426 # type: (Distribution) -> bool
427 """
428 Return True if given Distribution object is installed locally
429 (i.e. within current virtualenv).
430
431 Always True if we're not in a virtualenv.
432
433 """
434 return is_local(dist_location(dist))
435
436
437 def dist_in_usersite(dist):
438 # type: (Distribution) -> bool
439 """
440 Return True if given Distribution is installed in user site.
441 """
442 norm_path = normalize_path(dist_location(dist))
443 return norm_path.startswith(normalize_path(user_site))
444
445
446 def dist_in_site_packages(dist):
447 # type: (Distribution) -> bool
448 """
449 Return True if given Distribution is installed in
450 sysconfig.get_python_lib().
451 """
452 return normalize_path(
453 dist_location(dist)
454 ).startswith(normalize_path(site_packages))
455
456
457 def dist_is_editable(dist):
458 # type: (Distribution) -> bool
459 """
460 Return True if given Distribution is an editable install.
461 """
462 for path_item in sys.path:
463 egg_link = os.path.join(path_item, dist.project_name + '.egg-link')
464 if os.path.isfile(egg_link):
465 return True
466 return False
467
468
469 def get_installed_distributions(
470 local_only=True, # type: bool
471 skip=stdlib_pkgs, # type: Container[str]
472 include_editables=True, # type: bool
473 editables_only=False, # type: bool
474 user_only=False, # type: bool
475 paths=None # type: Optional[List[str]]
476 ):
477 # type: (...) -> List[Distribution]
478 """
479 Return a list of installed Distribution objects.
480
481 If ``local_only`` is True (default), only return installations
482 local to the current virtualenv, if in a virtualenv.
483
484 ``skip`` argument is an iterable of lower-case project names to
485 ignore; defaults to stdlib_pkgs
486
487 If ``include_editables`` is False, don't report editables.
488
489 If ``editables_only`` is True , only report editables.
490
491 If ``user_only`` is True , only report installations in the user
492 site directory.
493
494 If ``paths`` is set, only report the distributions present at the
495 specified list of locations.
496 """
497 if paths:
498 working_set = pkg_resources.WorkingSet(paths)
499 else:
500 working_set = pkg_resources.working_set
501
502 if local_only:
503 local_test = dist_is_local
504 else:
505 def local_test(d):
506 return True
507
508 if include_editables:
509 def editable_test(d):
510 return True
511 else:
512 def editable_test(d):
513 return not dist_is_editable(d)
514
515 if editables_only:
516 def editables_only_test(d):
517 return dist_is_editable(d)
518 else:
519 def editables_only_test(d):
520 return True
521
522 if user_only:
523 user_test = dist_in_usersite
524 else:
525 def user_test(d):
526 return True
527
528 # because of pkg_resources vendoring, mypy cannot find stub in typeshed
529 return [d for d in working_set # type: ignore
530 if local_test(d) and
531 d.key not in skip and
532 editable_test(d) and
533 editables_only_test(d) and
534 user_test(d)
535 ]
536
537
538 def egg_link_path(dist):
539 # type: (Distribution) -> Optional[str]
540 """
541 Return the path for the .egg-link file if it exists, otherwise, None.
542
543 There's 3 scenarios:
544 1) not in a virtualenv
545 try to find in site.USER_SITE, then site_packages
546 2) in a no-global virtualenv
547 try to find in site_packages
548 3) in a yes-global virtualenv
549 try to find in site_packages, then site.USER_SITE
550 (don't look in global location)
551
552 For #1 and #3, there could be odd cases, where there's an egg-link in 2
553 locations.
554
555 This method will just return the first one found.
556 """
557 sites = []
558 if running_under_virtualenv():
559 if virtualenv_no_global():
560 sites.append(site_packages)
561 else:
562 sites.append(site_packages)
563 if user_site:
564 sites.append(user_site)
565 else:
566 if user_site:
567 sites.append(user_site)
568 sites.append(site_packages)
569
570 for site in sites:
571 egglink = os.path.join(site, dist.project_name) + '.egg-link'
572 if os.path.isfile(egglink):
573 return egglink
574 return None
575
576
577 def dist_location(dist):
578 # type: (Distribution) -> str
579 """
580 Get the site-packages location of this distribution. Generally
581 this is dist.location, except in the case of develop-installed
582 packages, where dist.location is the source code location, and we
583 want to know where the egg-link file is.
584
585 """
586 egg_link = egg_link_path(dist)
587 if egg_link:
588 return egg_link
589 return dist.location
590
591
592 def current_umask():
593 """Get the current umask which involves having to set it temporarily."""
594 mask = os.umask(0)
595 os.umask(mask)
596 return mask
597
598
599 def unzip_file(filename, location, flatten=True):
600 # type: (str, str, bool) -> None
601 """
602 Unzip the file (with path `filename`) to the destination `location`. All
603 files are written based on system defaults and umask (i.e. permissions are
604 not preserved), except that regular file members with any execute
605 permissions (user, group, or world) have "chmod +x" applied after being
606 written. Note that for windows, any execute changes using os.chmod are
607 no-ops per the python docs.
608 """
609 ensure_dir(location)
610 zipfp = open(filename, 'rb')
611 try:
612 zip = zipfile.ZipFile(zipfp, allowZip64=True)
613 leading = has_leading_dir(zip.namelist()) and flatten
614 for info in zip.infolist():
615 name = info.filename
616 fn = name
617 if leading:
618 fn = split_leading_dir(name)[1]
619 fn = os.path.join(location, fn)
620 dir = os.path.dirname(fn)
621 if fn.endswith('/') or fn.endswith('\\'):
622 # A directory
623 ensure_dir(fn)
624 else:
625 ensure_dir(dir)
626 # Don't use read() to avoid allocating an arbitrarily large
627 # chunk of memory for the file's content
628 fp = zip.open(name)
629 try:
630 with open(fn, 'wb') as destfp:
631 shutil.copyfileobj(fp, destfp)
632 finally:
633 fp.close()
634 mode = info.external_attr >> 16
635 # if mode and regular file and any execute permissions for
636 # user/group/world?
637 if mode and stat.S_ISREG(mode) and mode & 0o111:
638 # make dest file have execute for user/group/world
639 # (chmod +x) no-op on windows per python docs
640 os.chmod(fn, (0o777 - current_umask() | 0o111))
641 finally:
642 zipfp.close()
643
644
645 def untar_file(filename, location):
646 # type: (str, str) -> None
647 """
648 Untar the file (with path `filename`) to the destination `location`.
649 All files are written based on system defaults and umask (i.e. permissions
650 are not preserved), except that regular file members with any execute
651 permissions (user, group, or world) have "chmod +x" applied after being
652 written. Note that for windows, any execute changes using os.chmod are
653 no-ops per the python docs.
654 """
655 ensure_dir(location)
656 if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
657 mode = 'r:gz'
658 elif filename.lower().endswith(BZ2_EXTENSIONS):
659 mode = 'r:bz2'
660 elif filename.lower().endswith(XZ_EXTENSIONS):
661 mode = 'r:xz'
662 elif filename.lower().endswith('.tar'):
663 mode = 'r'
664 else:
665 logger.warning(
666 'Cannot determine compression type for file %s', filename,
667 )
668 mode = 'r:*'
669 tar = tarfile.open(filename, mode)
670 try:
671 leading = has_leading_dir([
672 member.name for member in tar.getmembers()
673 ])
674 for member in tar.getmembers():
675 fn = member.name
676 if leading:
677 # https://github.com/python/mypy/issues/1174
678 fn = split_leading_dir(fn)[1] # type: ignore
679 path = os.path.join(location, fn)
680 if member.isdir():
681 ensure_dir(path)
682 elif member.issym():
683 try:
684 # https://github.com/python/typeshed/issues/2673
685 tar._extract_member(member, path) # type: ignore
686 except Exception as exc:
687 # Some corrupt tar files seem to produce this
688 # (specifically bad symlinks)
689 logger.warning(
690 'In the tar file %s the member %s is invalid: %s',
691 filename, member.name, exc,
692 )
693 continue
694 else:
695 try:
696 fp = tar.extractfile(member)
697 except (KeyError, AttributeError) as exc:
698 # Some corrupt tar files seem to produce this
699 # (specifically bad symlinks)
700 logger.warning(
701 'In the tar file %s the member %s is invalid: %s',
702 filename, member.name, exc,
703 )
704 continue
705 ensure_dir(os.path.dirname(path))
706 with open(path, 'wb') as destfp:
707 shutil.copyfileobj(fp, destfp)
708 fp.close()
709 # Update the timestamp (useful for cython compiled files)
710 # https://github.com/python/typeshed/issues/2673
711 tar.utime(member, path) # type: ignore
712 # member have any execute permissions for user/group/world?
713 if member.mode & 0o111:
714 # make dest file have execute for user/group/world
715 # no-op on windows per python docs
716 os.chmod(path, (0o777 - current_umask() | 0o111))
717 finally:
718 tar.close()
719
720
721 def unpack_file(
722 filename, # type: str
723 location, # type: str
724 content_type, # type: Optional[str]
725 link # type: Optional[Link]
726 ):
727 # type: (...) -> None
728 filename = os.path.realpath(filename)
729 if (content_type == 'application/zip' or
730 filename.lower().endswith(ZIP_EXTENSIONS) or
731 zipfile.is_zipfile(filename)):
732 unzip_file(
733 filename,
734 location,
735 flatten=not filename.endswith('.whl')
736 )
737 elif (content_type == 'application/x-gzip' or
738 tarfile.is_tarfile(filename) or
739 filename.lower().endswith(
740 TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)):
741 untar_file(filename, location)
742 elif (content_type and content_type.startswith('text/html') and
743 is_svn_page(file_contents(filename))):
744 # We don't really care about this
745 from pip._internal.vcs.subversion import Subversion
746 url = 'svn+' + link.url
747 Subversion().unpack(location, url=url)
748 else:
749 # FIXME: handle?
750 # FIXME: magic signatures?
751 logger.critical(
752 'Cannot unpack file %s (downloaded from %s, content-type: %s); '
753 'cannot detect archive format',
754 filename, location, content_type,
755 )
756 raise InstallationError(
757 'Cannot determine archive format of %s' % location
758 )
759
760
761 def format_command_args(args):
762 # type: (List[str]) -> str
763 """
764 Format command arguments for display.
765 """
766 return ' '.join(shlex_quote(arg) for arg in args)
767
768
769 def make_subprocess_output_error(
770 cmd_args, # type: List[str]
771 cwd, # type: Optional[str]
772 lines, # type: List[Text]
773 exit_status, # type: int
774 ):
775 # type: (...) -> Text
776 """
777 Create and return the error message to use to log a subprocess error
778 with command output.
779
780 :param lines: A list of lines, each ending with a newline.
781 """
782 command = format_command_args(cmd_args)
783 # Convert `command` and `cwd` to text (unicode in Python 2) so we can use
784 # them as arguments in the unicode format string below. This avoids
785 # "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2
786 # if either contains a non-ascii character.
787 command_display = str_to_display(command, desc='command bytes')
788 cwd_display = path_to_display(cwd)
789
790 # We know the joined output value ends in a newline.
791 output = ''.join(lines)
792 msg = (
793 # Use a unicode string to avoid "UnicodeEncodeError: 'ascii'
794 # codec can't encode character ..." in Python 2 when a format
795 # argument (e.g. `output`) has a non-ascii character.
796 u'Command errored out with exit status {exit_status}:\n'
797 ' command: {command_display}\n'
798 ' cwd: {cwd_display}\n'
799 'Complete output ({line_count} lines):\n{output}{divider}'
800 ).format(
801 exit_status=exit_status,
802 command_display=command_display,
803 cwd_display=cwd_display,
804 line_count=len(lines),
805 output=output,
806 divider=LOG_DIVIDER,
807 )
808 return msg
809
810
811 def call_subprocess(
812 cmd, # type: List[str]
813 show_stdout=False, # type: bool
814 cwd=None, # type: Optional[str]
815 on_returncode='raise', # type: str
816 extra_ok_returncodes=None, # type: Optional[Iterable[int]]
817 command_desc=None, # type: Optional[str]
818 extra_environ=None, # type: Optional[Mapping[str, Any]]
819 unset_environ=None, # type: Optional[Iterable[str]]
820 spinner=None # type: Optional[SpinnerInterface]
821 ):
822 # type: (...) -> Text
823 """
824 Args:
825 show_stdout: if true, use INFO to log the subprocess's stderr and
826 stdout streams. Otherwise, use DEBUG. Defaults to False.
827 extra_ok_returncodes: an iterable of integer return codes that are
828 acceptable, in addition to 0. Defaults to None, which means [].
829 unset_environ: an iterable of environment variable names to unset
830 prior to calling subprocess.Popen().
831 """
832 if extra_ok_returncodes is None:
833 extra_ok_returncodes = []
834 if unset_environ is None:
835 unset_environ = []
836 # Most places in pip use show_stdout=False. What this means is--
837 #
838 # - We connect the child's output (combined stderr and stdout) to a
839 # single pipe, which we read.
840 # - We log this output to stderr at DEBUG level as it is received.
841 # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't
842 # requested), then we show a spinner so the user can still see the
843 # subprocess is in progress.
844 # - If the subprocess exits with an error, we log the output to stderr
845 # at ERROR level if it hasn't already been displayed to the console
846 # (e.g. if --verbose logging wasn't enabled). This way we don't log
847 # the output to the console twice.
848 #
849 # If show_stdout=True, then the above is still done, but with DEBUG
850 # replaced by INFO.
851 if show_stdout:
852 # Then log the subprocess output at INFO level.
853 log_subprocess = subprocess_logger.info
854 used_level = std_logging.INFO
855 else:
856 # Then log the subprocess output using DEBUG. This also ensures
857 # it will be logged to the log file (aka user_log), if enabled.
858 log_subprocess = subprocess_logger.debug
859 used_level = std_logging.DEBUG
860
861 # Whether the subprocess will be visible in the console.
862 showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level
863
864 # Only use the spinner if we're not showing the subprocess output
865 # and we have a spinner.
866 use_spinner = not showing_subprocess and spinner is not None
867
868 if command_desc is None:
869 command_desc = format_command_args(cmd)
870
871 log_subprocess("Running command %s", command_desc)
872 env = os.environ.copy()
873 if extra_environ:
874 env.update(extra_environ)
875 for name in unset_environ:
876 env.pop(name, None)
877 try:
878 proc = subprocess.Popen(
879 cmd, stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
880 stdout=subprocess.PIPE, cwd=cwd, env=env,
881 )
882 proc.stdin.close()
883 except Exception as exc:
884 subprocess_logger.critical(
885 "Error %s while executing command %s", exc, command_desc,
886 )
887 raise
888 all_output = []
889 while True:
890 # The "line" value is a unicode string in Python 2.
891 line = console_to_str(proc.stdout.readline())
892 if not line:
893 break
894 line = line.rstrip()
895 all_output.append(line + '\n')
896
897 # Show the line immediately.
898 log_subprocess(line)
899 # Update the spinner.
900 if use_spinner:
901 spinner.spin()
902 try:
903 proc.wait()
904 finally:
905 if proc.stdout:
906 proc.stdout.close()
907 proc_had_error = (
908 proc.returncode and proc.returncode not in extra_ok_returncodes
909 )
910 if use_spinner:
911 if proc_had_error:
912 spinner.finish("error")
913 else:
914 spinner.finish("done")
915 if proc_had_error:
916 if on_returncode == 'raise':
917 if not showing_subprocess:
918 # Then the subprocess streams haven't been logged to the
919 # console yet.
920 msg = make_subprocess_output_error(
921 cmd_args=cmd,
922 cwd=cwd,
923 lines=all_output,
924 exit_status=proc.returncode,
925 )
926 subprocess_logger.error(msg)
927 exc_msg = (
928 'Command errored out with exit status {}: {} '
929 'Check the logs for full command output.'
930 ).format(proc.returncode, command_desc)
931 raise InstallationError(exc_msg)
932 elif on_returncode == 'warn':
933 subprocess_logger.warning(
934 'Command "%s" had error code %s in %s',
935 command_desc, proc.returncode, cwd,
936 )
937 elif on_returncode == 'ignore':
938 pass
939 else:
940 raise ValueError('Invalid value: on_returncode=%s' %
941 repr(on_returncode))
942 return ''.join(all_output)
943
944
945 def _make_build_dir(build_dir):
946 os.makedirs(build_dir)
947 write_delete_marker_file(build_dir)
948
949
950 class FakeFile(object):
951 """Wrap a list of lines in an object with readline() to make
952 ConfigParser happy."""
953 def __init__(self, lines):
954 self._gen = (l for l in lines)
955
956 def readline(self):
957 try:
958 try:
959 return next(self._gen)
960 except NameError:
961 return self._gen.next()
962 except StopIteration:
963 return ''
964
965 def __iter__(self):
966 return self._gen
967
968
969 class StreamWrapper(StringIO):
970
971 @classmethod
972 def from_stream(cls, orig_stream):
973 cls.orig_stream = orig_stream
974 return cls()
975
976 # compileall.compile_dir() needs stdout.encoding to print to stdout
977 @property
978 def encoding(self):
979 return self.orig_stream.encoding
980
981
982 @contextlib.contextmanager
983 def captured_output(stream_name):
984 """Return a context manager used by captured_stdout/stdin/stderr
985 that temporarily replaces the sys stream *stream_name* with a StringIO.
986
987 Taken from Lib/support/__init__.py in the CPython repo.
988 """
989 orig_stdout = getattr(sys, stream_name)
990 setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout))
991 try:
992 yield getattr(sys, stream_name)
993 finally:
994 setattr(sys, stream_name, orig_stdout)
995
996
997 def captured_stdout():
998 """Capture the output of sys.stdout:
999
1000 with captured_stdout() as stdout:
1001 print('hello')
1002 self.assertEqual(stdout.getvalue(), 'hello\n')
1003
1004 Taken from Lib/support/__init__.py in the CPython repo.
1005 """
1006 return captured_output('stdout')
1007
1008
1009 def captured_stderr():
1010 """
1011 See captured_stdout().
1012 """
1013 return captured_output('stderr')
1014
1015
1016 class cached_property(object):
1017 """A property that is only computed once per instance and then replaces
1018 itself with an ordinary attribute. Deleting the attribute resets the
1019 property.
1020
1021 Source: https://github.com/bottlepy/bottle/blob/0.11.5/bottle.py#L175
1022 """
1023
1024 def __init__(self, func):
1025 self.__doc__ = getattr(func, '__doc__')
1026 self.func = func
1027
1028 def __get__(self, obj, cls):
1029 if obj is None:
1030 # We're being accessed from the class itself, not from an object
1031 return self
1032 value = obj.__dict__[self.func.__name__] = self.func(obj)
1033 return value
1034
1035
1036 def get_installed_version(dist_name, working_set=None):
1037 """Get the installed version of dist_name avoiding pkg_resources cache"""
1038 # Create a requirement that we'll look for inside of setuptools.
1039 req = pkg_resources.Requirement.parse(dist_name)
1040
1041 if working_set is None:
1042 # We want to avoid having this cached, so we need to construct a new
1043 # working set each time.
1044 working_set = pkg_resources.WorkingSet()
1045
1046 # Get the installed distribution from our working set
1047 dist = working_set.find(req)
1048
1049 # Check to see if we got an installed distribution or not, if we did
1050 # we want to return it's version.
1051 return dist.version if dist else None
1052
1053
1054 def consume(iterator):
1055 """Consume an iterable at C speed."""
1056 deque(iterator, maxlen=0)
1057
1058
1059 # Simulates an enum
1060 def enum(*sequential, **named):
1061 enums = dict(zip(sequential, range(len(sequential))), **named)
1062 reverse = {value: key for key, value in enums.items()}
1063 enums['reverse_mapping'] = reverse
1064 return type('Enum', (), enums)
1065
1066
1067 def path_to_url(path):
1068 # type: (Union[str, Text]) -> str
1069 """
1070 Convert a path to a file: URL. The path will be made absolute and have
1071 quoted path parts.
1072 """
1073 path = os.path.normpath(os.path.abspath(path))
1074 url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
1075 return url
1076
1077
1078 def split_auth_from_netloc(netloc):
1079 """
1080 Parse out and remove the auth information from a netloc.
1081
1082 Returns: (netloc, (username, password)).
1083 """
1084 if '@' not in netloc:
1085 return netloc, (None, None)
1086
1087 # Split from the right because that's how urllib.parse.urlsplit()
1088 # behaves if more than one @ is present (which can be checked using
1089 # the password attribute of urlsplit()'s return value).
1090 auth, netloc = netloc.rsplit('@', 1)
1091 if ':' in auth:
1092 # Split from the left because that's how urllib.parse.urlsplit()
1093 # behaves if more than one : is present (which again can be checked
1094 # using the password attribute of the return value)
1095 user_pass = auth.split(':', 1)
1096 else:
1097 user_pass = auth, None
1098
1099 user_pass = tuple(
1100 None if x is None else urllib_unquote(x) for x in user_pass
1101 )
1102
1103 return netloc, user_pass
1104
1105
1106 def redact_netloc(netloc):
1107 # type: (str) -> str
1108 """
1109 Replace the password in a netloc with "****", if it exists.
1110
1111 For example, "user:pass@example.com" returns "user:****@example.com".
1112 """
1113 netloc, (user, password) = split_auth_from_netloc(netloc)
1114 if user is None:
1115 return netloc
1116 password = '' if password is None else ':****'
1117 return '{user}{password}@{netloc}'.format(user=urllib_parse.quote(user),
1118 password=password,
1119 netloc=netloc)
1120
1121
1122 def _transform_url(url, transform_netloc):
1123 """Transform and replace netloc in a url.
1124
1125 transform_netloc is a function taking the netloc and returning a
1126 tuple. The first element of this tuple is the new netloc. The
1127 entire tuple is returned.
1128
1129 Returns a tuple containing the transformed url as item 0 and the
1130 original tuple returned by transform_netloc as item 1.
1131 """
1132 purl = urllib_parse.urlsplit(url)
1133 netloc_tuple = transform_netloc(purl.netloc)
1134 # stripped url
1135 url_pieces = (
1136 purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment
1137 )
1138 surl = urllib_parse.urlunsplit(url_pieces)
1139 return surl, netloc_tuple
1140
1141
1142 def _get_netloc(netloc):
1143 return split_auth_from_netloc(netloc)
1144
1145
1146 def _redact_netloc(netloc):
1147 return (redact_netloc(netloc),)
1148
1149
1150 def split_auth_netloc_from_url(url):
1151 # type: (str) -> Tuple[str, str, Tuple[str, str]]
1152 """
1153 Parse a url into separate netloc, auth, and url with no auth.
1154
1155 Returns: (url_without_auth, netloc, (username, password))
1156 """
1157 url_without_auth, (netloc, auth) = _transform_url(url, _get_netloc)
1158 return url_without_auth, netloc, auth
1159
1160
1161 def remove_auth_from_url(url):
1162 # type: (str) -> str
1163 """Return a copy of url with 'username:password@' removed."""
1164 # username/pass params are passed to subversion through flags
1165 # and are not recognized in the url.
1166 return _transform_url(url, _get_netloc)[0]
1167
1168
1169 def redact_password_from_url(url):
1170 # type: (str) -> str
1171 """Replace the password in a given url with ****."""
1172 return _transform_url(url, _redact_netloc)[0]
1173
1174
1175 def protect_pip_from_modification_on_windows(modifying_pip):
1176 """Protection of pip.exe from modification on Windows
1177
1178 On Windows, any operation modifying pip should be run as:
1179 python -m pip ...
1180 """
1181 pip_names = [
1182 "pip.exe",
1183 "pip{}.exe".format(sys.version_info[0]),
1184 "pip{}.{}.exe".format(*sys.version_info[:2])
1185 ]
1186
1187 # See https://github.com/pypa/pip/issues/1299 for more discussion
1188 should_show_use_python_msg = (
1189 modifying_pip and
1190 WINDOWS and
1191 os.path.basename(sys.argv[0]) in pip_names
1192 )
1193
1194 if should_show_use_python_msg:
1195 new_command = [
1196 sys.executable, "-m", "pip"
1197 ] + sys.argv[1:]
1198 raise CommandError(
1199 'To modify pip, please run the following command:\n{}'
1200 .format(" ".join(new_command))
1201 )