Mercurial > repos > guerler > hhblits

comparison lib/python3.8/site-packages/pip/_internal/utils/encoding.py @ 1:64071f2a4cf0 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Deleted selected files
author guerler
date Mon, 27 Jul 2020 03:55:49 -0400
parents 9e54283cc701
children
comparison
equal deleted inserted replaced
0:9e54283cc701 1:64071f2a4cf0
1 # The following comment should be removed at some point in the future.
2 # mypy: strict-optional=False
3
4 import codecs
5 import locale
6 import re
7 import sys
8
9 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
10
11 if MYPY_CHECK_RUNNING:
12 from typing import List, Tuple, Text
13
14 BOMS = [
15 (codecs.BOM_UTF8, 'utf-8'),
16 (codecs.BOM_UTF16, 'utf-16'),
17 (codecs.BOM_UTF16_BE, 'utf-16-be'),
18 (codecs.BOM_UTF16_LE, 'utf-16-le'),
19 (codecs.BOM_UTF32, 'utf-32'),
20 (codecs.BOM_UTF32_BE, 'utf-32-be'),
21 (codecs.BOM_UTF32_LE, 'utf-32-le'),
22 ] # type: List[Tuple[bytes, Text]]
23
24 ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)')
25
26
27 def auto_decode(data):
28 # type: (bytes) -> Text
29 """Check a bytes string for a BOM to correctly detect the encoding
30
31 Fallback to locale.getpreferredencoding(False) like open() on Python3"""
32 for bom, encoding in BOMS:
33 if data.startswith(bom):
34 return data[len(bom):].decode(encoding)
35 # Lets check the first two lines as in PEP263
36 for line in data.split(b'\n')[:2]:
37 if line[0:1] == b'#' and ENCODING_RE.search(line):
38 encoding = ENCODING_RE.search(line).groups()[0].decode('ascii')
39 return data.decode(encoding)
40 return data.decode(
41 locale.getpreferredencoding(False) or sys.getdefaultencoding(),
42 )