diff lib/python3.8/site-packages/pip/_internal/utils/encoding.py @ 0:9e54283cc701 draft

"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/python3.8/site-packages/pip/_internal/utils/encoding.py	Mon Jul 27 03:47:31 2020 -0400
@@ -0,0 +1,42 @@
+# The following comment should be removed at some point in the future.
+# mypy: strict-optional=False
+
+import codecs
+import locale
+import re
+import sys
+
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import List, Tuple, Text
+
+BOMS = [
+    (codecs.BOM_UTF8, 'utf-8'),
+    (codecs.BOM_UTF16, 'utf-16'),
+    (codecs.BOM_UTF16_BE, 'utf-16-be'),
+    (codecs.BOM_UTF16_LE, 'utf-16-le'),
+    (codecs.BOM_UTF32, 'utf-32'),
+    (codecs.BOM_UTF32_BE, 'utf-32-be'),
+    (codecs.BOM_UTF32_LE, 'utf-32-le'),
+]  # type: List[Tuple[bytes, Text]]
+
+ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)')
+
+
+def auto_decode(data):
+    # type: (bytes) -> Text
+    """Check a bytes string for a BOM to correctly detect the encoding
+
+    Fallback to locale.getpreferredencoding(False) like open() on Python3"""
+    for bom, encoding in BOMS:
+        if data.startswith(bom):
+            return data[len(bom):].decode(encoding)
+    # Lets check the first two lines as in PEP263
+    for line in data.split(b'\n')[:2]:
+        if line[0:1] == b'#' and ENCODING_RE.search(line):
+            encoding = ENCODING_RE.search(line).groups()[0].decode('ascii')
+            return data.decode(encoding)
+    return data.decode(
+        locale.getpreferredencoding(False) or sys.getdefaultencoding(),
+    )