Mercurial > repos > guerler > hhblits

annotate lib/python3.8/site-packages/pip/_internal/utils/encoding.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400 (2020-07-27)
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
1 # The following comment should be removed at some point in the future.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
2 # mypy: strict-optional=False
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
3
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
4 import codecs
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
5 import locale
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
6 import re
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
7 import sys
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
8
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
9 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
10
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
11 if MYPY_CHECK_RUNNING:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
12 from typing import List, Tuple, Text
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
13
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
14 BOMS = [
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
15 (codecs.BOM_UTF8, 'utf-8'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
16 (codecs.BOM_UTF16, 'utf-16'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
17 (codecs.BOM_UTF16_BE, 'utf-16-be'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
18 (codecs.BOM_UTF16_LE, 'utf-16-le'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
19 (codecs.BOM_UTF32, 'utf-32'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
20 (codecs.BOM_UTF32_BE, 'utf-32-be'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
21 (codecs.BOM_UTF32_LE, 'utf-32-le'),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
22 ] # type: List[Tuple[bytes, Text]]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
23
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
24 ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
25
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
26
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
27 def auto_decode(data):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
28 # type: (bytes) -> Text
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
29 """Check a bytes string for a BOM to correctly detect the encoding
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
30
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
31 Fallback to locale.getpreferredencoding(False) like open() on Python3"""
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
32 for bom, encoding in BOMS:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
33 if data.startswith(bom):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
34 return data[len(bom):].decode(encoding)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
35 # Lets check the first two lines as in PEP263
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
36 for line in data.split(b'\n')[:2]:
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
37 if line[0:1] == b'#' and ENCODING_RE.search(line):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
38 encoding = ENCODING_RE.search(line).groups()[0].decode('ascii')
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
39 return data.decode(encoding)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
40 return data.decode(
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
41 locale.getpreferredencoding(False) or sys.getdefaultencoding(),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
42 )