Mercurial > repos > guerler > hhblits
annotate lib/python3.8/site-packages/setuptools/package_index.py @ 0:9e54283cc701 draft
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author | guerler |
---|---|
date | Mon, 27 Jul 2020 03:47:31 -0400 (2020-07-27) |
parents | |
children |
rev | line source |
---|---|
0
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1 """PyPI and direct package downloading""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
2 import sys |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
3 import os |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
4 import re |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
5 import shutil |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
6 import socket |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
7 import base64 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
8 import hashlib |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
9 import itertools |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
10 import warnings |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
11 from functools import wraps |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
12 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
13 from setuptools.extern import six |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
14 from setuptools.extern.six.moves import urllib, http_client, configparser, map |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
15 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
16 import setuptools |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
17 from pkg_resources import ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
18 CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
19 Environment, find_distributions, safe_name, safe_version, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
20 to_filename, Requirement, DEVELOP_DIST, EGG_DIST, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
21 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
22 from setuptools import ssl_support |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
23 from distutils import log |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
24 from distutils.errors import DistutilsError |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
25 from fnmatch import translate |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
26 from setuptools.py27compat import get_all_headers |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
27 from setuptools.py33compat import unescape |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
28 from setuptools.wheel import Wheel |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
29 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
30 __metaclass__ = type |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
31 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
32 EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
33 HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
34 PYPI_MD5 = re.compile( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
35 r'<a href="([^"#]+)">([^<]+)</a>\n\s+\(<a (?:title="MD5 hash"\n\s+)' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
36 r'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\)' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
37 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
38 URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
39 EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
40 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
41 __all__ = [ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
42 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst', |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
43 'interpret_distro_name', |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
44 ] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
45 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
46 _SOCKET_TIMEOUT = 15 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
47 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
48 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
49 user_agent = _tmpl.format( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
50 py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
51 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
52 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
53 def parse_requirement_arg(spec): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
54 try: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
55 return Requirement.parse(spec) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
56 except ValueError: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
57 raise DistutilsError( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
58 "Not a URL, existing file, or requirement spec: %r" % (spec,) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
59 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
60 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
61 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
62 def parse_bdist_wininst(name): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
63 """Return (base,pyversion) or (None,None) for possible .exe name""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
64 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
65 lower = name.lower() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
66 base, py_ver, plat = None, None, None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
67 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
68 if lower.endswith('.exe'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
69 if lower.endswith('.win32.exe'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
70 base = name[:-10] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
71 plat = 'win32' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
72 elif lower.startswith('.win32-py', -16): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
73 py_ver = name[-7:-4] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
74 base = name[:-16] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
75 plat = 'win32' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
76 elif lower.endswith('.win-amd64.exe'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
77 base = name[:-14] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
78 plat = 'win-amd64' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
79 elif lower.startswith('.win-amd64-py', -20): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
80 py_ver = name[-7:-4] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
81 base = name[:-20] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
82 plat = 'win-amd64' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
83 return base, py_ver, plat |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
84 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
85 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
86 def egg_info_for_url(url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
87 parts = urllib.parse.urlparse(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
88 scheme, server, path, parameters, query, fragment = parts |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
89 base = urllib.parse.unquote(path.split('/')[-1]) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
90 if server == 'sourceforge.net' and base == 'download': # XXX Yuck |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
91 base = urllib.parse.unquote(path.split('/')[-2]) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
92 if '#' in base: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
93 base, fragment = base.split('#', 1) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
94 return base, fragment |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
95 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
96 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
97 def distros_for_url(url, metadata=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
98 """Yield egg or source distribution objects that might be found at a URL""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
99 base, fragment = egg_info_for_url(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
100 for dist in distros_for_location(url, base, metadata): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
101 yield dist |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
102 if fragment: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
103 match = EGG_FRAGMENT.match(fragment) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
104 if match: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
105 for dist in interpret_distro_name( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
106 url, match.group(1), metadata, precedence=CHECKOUT_DIST |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
107 ): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
108 yield dist |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
109 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
110 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
111 def distros_for_location(location, basename, metadata=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
112 """Yield egg or source distribution objects based on basename""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
113 if basename.endswith('.egg.zip'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
114 basename = basename[:-4] # strip the .zip |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
115 if basename.endswith('.egg') and '-' in basename: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
116 # only one, unambiguous interpretation |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
117 return [Distribution.from_location(location, basename, metadata)] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
118 if basename.endswith('.whl') and '-' in basename: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
119 wheel = Wheel(basename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
120 if not wheel.is_compatible(): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
121 return [] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
122 return [Distribution( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
123 location=location, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
124 project_name=wheel.project_name, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
125 version=wheel.version, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
126 # Increase priority over eggs. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
127 precedence=EGG_DIST + 1, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
128 )] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
129 if basename.endswith('.exe'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
130 win_base, py_ver, platform = parse_bdist_wininst(basename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
131 if win_base is not None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
132 return interpret_distro_name( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
133 location, win_base, metadata, py_ver, BINARY_DIST, platform |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
134 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
135 # Try source distro extensions (.zip, .tgz, etc.) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
136 # |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
137 for ext in EXTENSIONS: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
138 if basename.endswith(ext): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
139 basename = basename[:-len(ext)] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
140 return interpret_distro_name(location, basename, metadata) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
141 return [] # no extension matched |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
142 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
143 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
144 def distros_for_filename(filename, metadata=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
145 """Yield possible egg or source distribution objects based on a filename""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
146 return distros_for_location( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
147 normalize_path(filename), os.path.basename(filename), metadata |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
148 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
149 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
150 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
151 def interpret_distro_name( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
152 location, basename, metadata, py_version=None, precedence=SOURCE_DIST, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
153 platform=None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
154 ): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
155 """Generate alternative interpretations of a source distro name |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
156 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
157 Note: if `location` is a filesystem filename, you should call |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
158 ``pkg_resources.normalize_path()`` on it before passing it to this |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
159 routine! |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
160 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
161 # Generate alternative interpretations of a source distro name |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
162 # Because some packages are ambiguous as to name/versions split |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
163 # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
164 # So, we generate each possible interepretation (e.g. "adns, python-1.1.0" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
165 # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
166 # the spurious interpretations should be ignored, because in the event |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
167 # there's also an "adns" package, the spurious "python-1.1.0" version will |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
168 # compare lower than any numeric version number, and is therefore unlikely |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
169 # to match a request for it. It's still a potential problem, though, and |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
170 # in the long run PyPI and the distutils should go for "safe" names and |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
171 # versions in distribution archive names (sdist and bdist). |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
172 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
173 parts = basename.split('-') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
174 if not py_version and any(re.match(r'py\d\.\d$', p) for p in parts[2:]): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
175 # it is a bdist_dumb, not an sdist -- bail out |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
176 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
177 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
178 for p in range(1, len(parts) + 1): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
179 yield Distribution( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
180 location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
181 py_version=py_version, precedence=precedence, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
182 platform=platform |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
183 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
184 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
185 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
186 # From Python 2.7 docs |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
187 def unique_everseen(iterable, key=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
188 "List unique elements, preserving order. Remember all elements ever seen." |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
189 # unique_everseen('AAAABBBCCDAABBB') --> A B C D |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
190 # unique_everseen('ABBCcAD', str.lower) --> A B C D |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
191 seen = set() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
192 seen_add = seen.add |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
193 if key is None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
194 for element in six.moves.filterfalse(seen.__contains__, iterable): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
195 seen_add(element) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
196 yield element |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
197 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
198 for element in iterable: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
199 k = key(element) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
200 if k not in seen: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
201 seen_add(k) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
202 yield element |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
203 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
204 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
205 def unique_values(func): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
206 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
207 Wrap a function returning an iterable such that the resulting iterable |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
208 only ever yields unique items. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
209 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
210 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
211 @wraps(func) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
212 def wrapper(*args, **kwargs): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
213 return unique_everseen(func(*args, **kwargs)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
214 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
215 return wrapper |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
216 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
217 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
218 REL = re.compile(r"""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
219 # this line is here to fix emacs' cruddy broken syntax highlighting |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
220 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
221 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
222 @unique_values |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
223 def find_external_links(url, page): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
224 """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
225 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
226 for match in REL.finditer(page): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
227 tag, rel = match.groups() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
228 rels = set(map(str.strip, rel.lower().split(','))) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
229 if 'homepage' in rels or 'download' in rels: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
230 for match in HREF.finditer(tag): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
231 yield urllib.parse.urljoin(url, htmldecode(match.group(1))) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
232 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
233 for tag in ("<th>Home Page", "<th>Download URL"): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
234 pos = page.find(tag) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
235 if pos != -1: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
236 match = HREF.search(page, pos) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
237 if match: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
238 yield urllib.parse.urljoin(url, htmldecode(match.group(1))) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
239 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
240 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
241 class ContentChecker: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
242 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
243 A null content checker that defines the interface for checking content |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
244 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
245 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
246 def feed(self, block): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
247 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
248 Feed a block of data to the hash. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
249 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
250 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
251 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
252 def is_valid(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
253 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
254 Check the hash. Return False if validation fails. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
255 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
256 return True |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
257 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
258 def report(self, reporter, template): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
259 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
260 Call reporter with information about the checker (hash name) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
261 substituted into the template. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
262 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
263 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
264 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
265 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
266 class HashChecker(ContentChecker): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
267 pattern = re.compile( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
268 r'(?P<hash_name>sha1|sha224|sha384|sha256|sha512|md5)=' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
269 r'(?P<expected>[a-f0-9]+)' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
270 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
271 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
272 def __init__(self, hash_name, expected): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
273 self.hash_name = hash_name |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
274 self.hash = hashlib.new(hash_name) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
275 self.expected = expected |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
276 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
277 @classmethod |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
278 def from_url(cls, url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
279 "Construct a (possibly null) ContentChecker from a URL" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
280 fragment = urllib.parse.urlparse(url)[-1] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
281 if not fragment: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
282 return ContentChecker() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
283 match = cls.pattern.search(fragment) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
284 if not match: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
285 return ContentChecker() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
286 return cls(**match.groupdict()) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
287 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
288 def feed(self, block): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
289 self.hash.update(block) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
290 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
291 def is_valid(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
292 return self.hash.hexdigest() == self.expected |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
293 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
294 def report(self, reporter, template): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
295 msg = template % self.hash_name |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
296 return reporter(msg) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
297 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
298 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
299 class PackageIndex(Environment): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
300 """A distribution index that scans web pages for download URLs""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
301 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
302 def __init__( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
303 self, index_url="https://pypi.org/simple/", hosts=('*',), |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
304 ca_bundle=None, verify_ssl=True, *args, **kw |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
305 ): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
306 Environment.__init__(self, *args, **kw) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
307 self.index_url = index_url + "/" [:not index_url.endswith('/')] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
308 self.scanned_urls = {} |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
309 self.fetched_urls = {} |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
310 self.package_pages = {} |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
311 self.allows = re.compile('|'.join(map(translate, hosts))).match |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
312 self.to_scan = [] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
313 use_ssl = ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
314 verify_ssl |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
315 and ssl_support.is_available |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
316 and (ca_bundle or ssl_support.find_ca_bundle()) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
317 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
318 if use_ssl: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
319 self.opener = ssl_support.opener_for(ca_bundle) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
320 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
321 self.opener = urllib.request.urlopen |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
322 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
323 def process_url(self, url, retrieve=False): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
324 """Evaluate a URL as a possible download, and maybe retrieve it""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
325 if url in self.scanned_urls and not retrieve: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
326 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
327 self.scanned_urls[url] = True |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
328 if not URL_SCHEME(url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
329 self.process_filename(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
330 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
331 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
332 dists = list(distros_for_url(url)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
333 if dists: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
334 if not self.url_ok(url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
335 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
336 self.debug("Found link: %s", url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
337 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
338 if dists or not retrieve or url in self.fetched_urls: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
339 list(map(self.add, dists)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
340 return # don't need the actual page |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
341 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
342 if not self.url_ok(url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
343 self.fetched_urls[url] = True |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
344 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
345 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
346 self.info("Reading %s", url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
347 self.fetched_urls[url] = True # prevent multiple fetch attempts |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
348 tmpl = "Download error on %s: %%s -- Some packages may not be found!" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
349 f = self.open_url(url, tmpl % url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
350 if f is None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
351 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
352 if isinstance(f, urllib.error.HTTPError) and f.code == 401: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
353 self.info("Authentication error: %s" % f.msg) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
354 self.fetched_urls[f.url] = True |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
355 if 'html' not in f.headers.get('content-type', '').lower(): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
356 f.close() # not html, we can't process it |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
357 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
358 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
359 base = f.url # handle redirects |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
360 page = f.read() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
361 if not isinstance(page, str): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
362 # In Python 3 and got bytes but want str. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
363 if isinstance(f, urllib.error.HTTPError): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
364 # Errors have no charset, assume latin1: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
365 charset = 'latin-1' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
366 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
367 charset = f.headers.get_param('charset') or 'latin-1' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
368 page = page.decode(charset, "ignore") |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
369 f.close() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
370 for match in HREF.finditer(page): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
371 link = urllib.parse.urljoin(base, htmldecode(match.group(1))) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
372 self.process_url(link) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
373 if url.startswith(self.index_url) and getattr(f, 'code', None) != 404: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
374 page = self.process_index(url, page) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
375 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
376 def process_filename(self, fn, nested=False): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
377 # process filenames or directories |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
378 if not os.path.exists(fn): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
379 self.warn("Not found: %s", fn) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
380 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
381 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
382 if os.path.isdir(fn) and not nested: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
383 path = os.path.realpath(fn) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
384 for item in os.listdir(path): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
385 self.process_filename(os.path.join(path, item), True) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
386 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
387 dists = distros_for_filename(fn) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
388 if dists: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
389 self.debug("Found: %s", fn) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
390 list(map(self.add, dists)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
391 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
392 def url_ok(self, url, fatal=False): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
393 s = URL_SCHEME(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
394 is_file = s and s.group(1).lower() == 'file' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
395 if is_file or self.allows(urllib.parse.urlparse(url)[1]): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
396 return True |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
397 msg = ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
398 "\nNote: Bypassing %s (disallowed host; see " |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
399 "http://bit.ly/2hrImnY for details).\n") |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
400 if fatal: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
401 raise DistutilsError(msg % url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
402 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
403 self.warn(msg, url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
404 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
405 def scan_egg_links(self, search_path): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
406 dirs = filter(os.path.isdir, search_path) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
407 egg_links = ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
408 (path, entry) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
409 for path in dirs |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
410 for entry in os.listdir(path) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
411 if entry.endswith('.egg-link') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
412 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
413 list(itertools.starmap(self.scan_egg_link, egg_links)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
414 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
415 def scan_egg_link(self, path, entry): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
416 with open(os.path.join(path, entry)) as raw_lines: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
417 # filter non-empty lines |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
418 lines = list(filter(None, map(str.strip, raw_lines))) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
419 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
420 if len(lines) != 2: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
421 # format is not recognized; punt |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
422 return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
423 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
424 egg_path, setup_path = lines |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
425 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
426 for dist in find_distributions(os.path.join(path, egg_path)): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
427 dist.location = os.path.join(path, *lines) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
428 dist.precedence = SOURCE_DIST |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
429 self.add(dist) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
430 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
431 def process_index(self, url, page): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
432 """Process the contents of a PyPI page""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
433 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
434 def scan(link): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
435 # Process a URL to see if it's for a package page |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
436 if link.startswith(self.index_url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
437 parts = list(map( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
438 urllib.parse.unquote, link[len(self.index_url):].split('/') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
439 )) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
440 if len(parts) == 2 and '#' not in parts[1]: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
441 # it's a package page, sanitize and index it |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
442 pkg = safe_name(parts[0]) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
443 ver = safe_version(parts[1]) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
444 self.package_pages.setdefault(pkg.lower(), {})[link] = True |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
445 return to_filename(pkg), to_filename(ver) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
446 return None, None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
447 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
448 # process an index page into the package-page index |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
449 for match in HREF.finditer(page): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
450 try: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
451 scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
452 except ValueError: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
453 pass |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
454 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
455 pkg, ver = scan(url) # ensure this page is in the page index |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
456 if pkg: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
457 # process individual package page |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
458 for new_url in find_external_links(url, page): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
459 # Process the found URL |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
460 base, frag = egg_info_for_url(new_url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
461 if base.endswith('.py') and not frag: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
462 if ver: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
463 new_url += '#egg=%s-%s' % (pkg, ver) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
464 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
465 self.need_version_info(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
466 self.scan_url(new_url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
467 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
468 return PYPI_MD5.sub( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
469 lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
470 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
471 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
472 return "" # no sense double-scanning non-package pages |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
473 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
474 def need_version_info(self, url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
475 self.scan_all( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
476 "Page at %s links to .py file(s) without version info; an index " |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
477 "scan is required.", url |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
478 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
479 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
480 def scan_all(self, msg=None, *args): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
481 if self.index_url not in self.fetched_urls: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
482 if msg: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
483 self.warn(msg, *args) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
484 self.info( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
485 "Scanning index of all packages (this may take a while)" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
486 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
487 self.scan_url(self.index_url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
488 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
489 def find_packages(self, requirement): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
490 self.scan_url(self.index_url + requirement.unsafe_name + '/') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
491 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
492 if not self.package_pages.get(requirement.key): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
493 # Fall back to safe version of the name |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
494 self.scan_url(self.index_url + requirement.project_name + '/') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
495 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
496 if not self.package_pages.get(requirement.key): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
497 # We couldn't find the target package, so search the index page too |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
498 self.not_found_in_index(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
499 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
500 for url in list(self.package_pages.get(requirement.key, ())): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
501 # scan each page that might be related to the desired package |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
502 self.scan_url(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
503 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
504 def obtain(self, requirement, installer=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
505 self.prescan() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
506 self.find_packages(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
507 for dist in self[requirement.key]: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
508 if dist in requirement: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
509 return dist |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
510 self.debug("%s does not match %s", requirement, dist) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
511 return super(PackageIndex, self).obtain(requirement, installer) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
512 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
513 def check_hash(self, checker, filename, tfp): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
514 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
515 checker is a ContentChecker |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
516 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
517 checker.report( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
518 self.debug, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
519 "Validating %%s checksum for %s" % filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
520 if not checker.is_valid(): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
521 tfp.close() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
522 os.unlink(filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
523 raise DistutilsError( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
524 "%s validation failed for %s; " |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
525 "possible download problem?" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
526 % (checker.hash.name, os.path.basename(filename)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
527 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
528 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
529 def add_find_links(self, urls): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
530 """Add `urls` to the list that will be prescanned for searches""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
531 for url in urls: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
532 if ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
533 self.to_scan is None # if we have already "gone online" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
534 or not URL_SCHEME(url) # or it's a local file/directory |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
535 or url.startswith('file:') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
536 or list(distros_for_url(url)) # or a direct package link |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
537 ): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
538 # then go ahead and process it now |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
539 self.scan_url(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
540 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
541 # otherwise, defer retrieval till later |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
542 self.to_scan.append(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
543 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
544 def prescan(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
545 """Scan urls scheduled for prescanning (e.g. --find-links)""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
546 if self.to_scan: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
547 list(map(self.scan_url, self.to_scan)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
548 self.to_scan = None # from now on, go ahead and process immediately |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
549 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
550 def not_found_in_index(self, requirement): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
551 if self[requirement.key]: # we've seen at least one distro |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
552 meth, msg = self.info, "Couldn't retrieve index page for %r" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
553 else: # no distros seen for this name, might be misspelled |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
554 meth, msg = ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
555 self.warn, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
556 "Couldn't find index page for %r (maybe misspelled?)") |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
557 meth(msg, requirement.unsafe_name) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
558 self.scan_all() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
559 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
560 def download(self, spec, tmpdir): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
561 """Locate and/or download `spec` to `tmpdir`, returning a local path |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
562 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
563 `spec` may be a ``Requirement`` object, or a string containing a URL, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
564 an existing local filename, or a project/version requirement spec |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
565 (i.e. the string form of a ``Requirement`` object). If it is the URL |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
566 of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
567 that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
568 automatically created alongside the downloaded file. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
569 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
570 If `spec` is a ``Requirement`` object or a string containing a |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
571 project/version requirement spec, this method returns the location of |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
572 a matching distribution (possibly after downloading it to `tmpdir`). |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
573 If `spec` is a locally existing file or directory name, it is simply |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
574 returned unchanged. If `spec` is a URL, it is downloaded to a subpath |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
575 of `tmpdir`, and the local filename is returned. Various errors may be |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
576 raised if a problem occurs during downloading. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
577 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
578 if not isinstance(spec, Requirement): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
579 scheme = URL_SCHEME(spec) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
580 if scheme: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
581 # It's a url, download it to tmpdir |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
582 found = self._download_url(scheme.group(1), spec, tmpdir) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
583 base, fragment = egg_info_for_url(spec) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
584 if base.endswith('.py'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
585 found = self.gen_setup(found, fragment, tmpdir) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
586 return found |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
587 elif os.path.exists(spec): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
588 # Existing file or directory, just return it |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
589 return spec |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
590 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
591 spec = parse_requirement_arg(spec) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
592 return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
593 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
594 def fetch_distribution( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
595 self, requirement, tmpdir, force_scan=False, source=False, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
596 develop_ok=False, local_index=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
597 """Obtain a distribution suitable for fulfilling `requirement` |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
598 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
599 `requirement` must be a ``pkg_resources.Requirement`` instance. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
600 If necessary, or if the `force_scan` flag is set, the requirement is |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
601 searched for in the (online) package index as well as the locally |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
602 installed packages. If a distribution matching `requirement` is found, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
603 the returned distribution's ``location`` is the value you would have |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
604 gotten from calling the ``download()`` method with the matching |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
605 distribution's URL or filename. If no matching distribution is found, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
606 ``None`` is returned. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
607 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
608 If the `source` flag is set, only source distributions and source |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
609 checkout links will be considered. Unless the `develop_ok` flag is |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
610 set, development and system eggs (i.e., those using the ``.egg-info`` |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
611 format) will be ignored. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
612 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
613 # process a Requirement |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
614 self.info("Searching for %s", requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
615 skipped = {} |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
616 dist = None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
617 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
618 def find(req, env=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
619 if env is None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
620 env = self |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
621 # Find a matching distribution; may be called more than once |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
622 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
623 for dist in env[req.key]: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
624 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
625 if dist.precedence == DEVELOP_DIST and not develop_ok: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
626 if dist not in skipped: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
627 self.warn( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
628 "Skipping development or system egg: %s", dist, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
629 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
630 skipped[dist] = 1 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
631 continue |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
632 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
633 test = ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
634 dist in req |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
635 and (dist.precedence <= SOURCE_DIST or not source) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
636 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
637 if test: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
638 loc = self.download(dist.location, tmpdir) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
639 dist.download_location = loc |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
640 if os.path.exists(dist.download_location): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
641 return dist |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
642 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
643 if force_scan: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
644 self.prescan() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
645 self.find_packages(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
646 dist = find(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
647 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
648 if not dist and local_index is not None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
649 dist = find(requirement, local_index) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
650 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
651 if dist is None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
652 if self.to_scan is not None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
653 self.prescan() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
654 dist = find(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
655 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
656 if dist is None and not force_scan: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
657 self.find_packages(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
658 dist = find(requirement) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
659 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
660 if dist is None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
661 self.warn( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
662 "No local packages or working download links found for %s%s", |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
663 (source and "a source distribution of " or ""), |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
664 requirement, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
665 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
666 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
667 self.info("Best match: %s", dist) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
668 return dist.clone(location=dist.download_location) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
669 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
670 def fetch(self, requirement, tmpdir, force_scan=False, source=False): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
671 """Obtain a file suitable for fulfilling `requirement` |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
672 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
673 DEPRECATED; use the ``fetch_distribution()`` method now instead. For |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
674 backward compatibility, this routine is identical but returns the |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
675 ``location`` of the downloaded distribution instead of a distribution |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
676 object. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
677 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
678 dist = self.fetch_distribution(requirement, tmpdir, force_scan, source) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
679 if dist is not None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
680 return dist.location |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
681 return None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
682 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
683 def gen_setup(self, filename, fragment, tmpdir): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
684 match = EGG_FRAGMENT.match(fragment) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
685 dists = match and [ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
686 d for d in |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
687 interpret_distro_name(filename, match.group(1), None) if d.version |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
688 ] or [] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
689 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
690 if len(dists) == 1: # unambiguous ``#egg`` fragment |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
691 basename = os.path.basename(filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
692 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
693 # Make sure the file has been downloaded to the temp dir. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
694 if os.path.dirname(filename) != tmpdir: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
695 dst = os.path.join(tmpdir, basename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
696 from setuptools.command.easy_install import samefile |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
697 if not samefile(filename, dst): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
698 shutil.copy2(filename, dst) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
699 filename = dst |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
700 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
701 with open(os.path.join(tmpdir, 'setup.py'), 'w') as file: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
702 file.write( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
703 "from setuptools import setup\n" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
704 "setup(name=%r, version=%r, py_modules=[%r])\n" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
705 % ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
706 dists[0].project_name, dists[0].version, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
707 os.path.splitext(basename)[0] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
708 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
709 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
710 return filename |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
711 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
712 elif match: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
713 raise DistutilsError( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
714 "Can't unambiguously interpret project/version identifier %r; " |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
715 "any dashes in the name or version should be escaped using " |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
716 "underscores. %r" % (fragment, dists) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
717 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
718 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
719 raise DistutilsError( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
720 "Can't process plain .py files without an '#egg=name-version'" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
721 " suffix to enable automatic setup script generation." |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
722 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
723 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
724 dl_blocksize = 8192 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
725 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
726 def _download_to(self, url, filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
727 self.info("Downloading %s", url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
728 # Download the file |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
729 fp = None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
730 try: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
731 checker = HashChecker.from_url(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
732 fp = self.open_url(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
733 if isinstance(fp, urllib.error.HTTPError): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
734 raise DistutilsError( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
735 "Can't download %s: %s %s" % (url, fp.code, fp.msg) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
736 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
737 headers = fp.info() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
738 blocknum = 0 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
739 bs = self.dl_blocksize |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
740 size = -1 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
741 if "content-length" in headers: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
742 # Some servers return multiple Content-Length headers :( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
743 sizes = get_all_headers(headers, 'Content-Length') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
744 size = max(map(int, sizes)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
745 self.reporthook(url, filename, blocknum, bs, size) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
746 with open(filename, 'wb') as tfp: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
747 while True: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
748 block = fp.read(bs) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
749 if block: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
750 checker.feed(block) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
751 tfp.write(block) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
752 blocknum += 1 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
753 self.reporthook(url, filename, blocknum, bs, size) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
754 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
755 break |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
756 self.check_hash(checker, filename, tfp) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
757 return headers |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
758 finally: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
759 if fp: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
760 fp.close() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
761 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
762 def reporthook(self, url, filename, blocknum, blksize, size): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
763 pass # no-op |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
764 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
765 def open_url(self, url, warning=None): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
766 if url.startswith('file:'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
767 return local_open(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
768 try: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
769 return open_with_auth(url, self.opener) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
770 except (ValueError, http_client.InvalidURL) as v: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
771 msg = ' '.join([str(arg) for arg in v.args]) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
772 if warning: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
773 self.warn(warning, msg) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
774 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
775 raise DistutilsError('%s %s' % (url, msg)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
776 except urllib.error.HTTPError as v: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
777 return v |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
778 except urllib.error.URLError as v: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
779 if warning: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
780 self.warn(warning, v.reason) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
781 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
782 raise DistutilsError("Download error for %s: %s" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
783 % (url, v.reason)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
784 except http_client.BadStatusLine as v: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
785 if warning: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
786 self.warn(warning, v.line) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
787 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
788 raise DistutilsError( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
789 '%s returned a bad status line. The server might be ' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
790 'down, %s' % |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
791 (url, v.line) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
792 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
793 except (http_client.HTTPException, socket.error) as v: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
794 if warning: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
795 self.warn(warning, v) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
796 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
797 raise DistutilsError("Download error for %s: %s" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
798 % (url, v)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
799 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
800 def _download_url(self, scheme, url, tmpdir): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
801 # Determine download filename |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
802 # |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
803 name, fragment = egg_info_for_url(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
804 if name: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
805 while '..' in name: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
806 name = name.replace('..', '.').replace('\\', '_') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
807 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
808 name = "__downloaded__" # default if URL has no path contents |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
809 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
810 if name.endswith('.egg.zip'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
811 name = name[:-4] # strip the extra .zip before download |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
812 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
813 filename = os.path.join(tmpdir, name) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
814 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
815 # Download the file |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
816 # |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
817 if scheme == 'svn' or scheme.startswith('svn+'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
818 return self._download_svn(url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
819 elif scheme == 'git' or scheme.startswith('git+'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
820 return self._download_git(url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
821 elif scheme.startswith('hg+'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
822 return self._download_hg(url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
823 elif scheme == 'file': |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
824 return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
825 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
826 self.url_ok(url, True) # raises error if not allowed |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
827 return self._attempt_download(url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
828 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
829 def scan_url(self, url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
830 self.process_url(url, True) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
831 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
832 def _attempt_download(self, url, filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
833 headers = self._download_to(url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
834 if 'html' in headers.get('content-type', '').lower(): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
835 return self._download_html(url, headers, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
836 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
837 return filename |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
838 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
839 def _download_html(self, url, headers, filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
840 file = open(filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
841 for line in file: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
842 if line.strip(): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
843 # Check for a subversion index page |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
844 if re.search(r'<title>([^- ]+ - )?Revision \d+:', line): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
845 # it's a subversion index page: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
846 file.close() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
847 os.unlink(filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
848 return self._download_svn(url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
849 break # not an index page |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
850 file.close() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
851 os.unlink(filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
852 raise DistutilsError("Unexpected HTML page found at " + url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
853 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
854 def _download_svn(self, url, filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
855 warnings.warn("SVN download support is deprecated", UserWarning) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
856 url = url.split('#', 1)[0] # remove any fragment for svn's sake |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
857 creds = '' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
858 if url.lower().startswith('svn:') and '@' in url: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
859 scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
860 if not netloc and path.startswith('//') and '/' in path[2:]: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
861 netloc, path = path[2:].split('/', 1) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
862 auth, host = _splituser(netloc) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
863 if auth: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
864 if ':' in auth: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
865 user, pw = auth.split(':', 1) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
866 creds = " --username=%s --password=%s" % (user, pw) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
867 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
868 creds = " --username=" + auth |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
869 netloc = host |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
870 parts = scheme, netloc, url, p, q, f |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
871 url = urllib.parse.urlunparse(parts) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
872 self.info("Doing subversion checkout from %s to %s", url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
873 os.system("svn checkout%s -q %s %s" % (creds, url, filename)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
874 return filename |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
875 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
876 @staticmethod |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
877 def _vcs_split_rev_from_url(url, pop_prefix=False): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
878 scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
879 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
880 scheme = scheme.split('+', 1)[-1] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
881 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
882 # Some fragment identification fails |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
883 path = path.split('#', 1)[0] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
884 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
885 rev = None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
886 if '@' in path: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
887 path, rev = path.rsplit('@', 1) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
888 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
889 # Also, discard fragment |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
890 url = urllib.parse.urlunsplit((scheme, netloc, path, query, '')) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
891 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
892 return url, rev |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
893 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
894 def _download_git(self, url, filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
895 filename = filename.split('#', 1)[0] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
896 url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
897 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
898 self.info("Doing git clone from %s to %s", url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
899 os.system("git clone --quiet %s %s" % (url, filename)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
900 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
901 if rev is not None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
902 self.info("Checking out %s", rev) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
903 os.system("git -C %s checkout --quiet %s" % ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
904 filename, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
905 rev, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
906 )) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
907 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
908 return filename |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
909 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
910 def _download_hg(self, url, filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
911 filename = filename.split('#', 1)[0] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
912 url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
913 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
914 self.info("Doing hg clone from %s to %s", url, filename) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
915 os.system("hg clone --quiet %s %s" % (url, filename)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
916 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
917 if rev is not None: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
918 self.info("Updating to %s", rev) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
919 os.system("hg --cwd %s up -C -r %s -q" % ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
920 filename, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
921 rev, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
922 )) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
923 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
924 return filename |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
925 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
926 def debug(self, msg, *args): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
927 log.debug(msg, *args) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
928 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
929 def info(self, msg, *args): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
930 log.info(msg, *args) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
931 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
932 def warn(self, msg, *args): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
933 log.warn(msg, *args) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
934 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
935 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
936 # This pattern matches a character entity reference (a decimal numeric |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
937 # references, a hexadecimal numeric reference, or a named reference). |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
938 entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
939 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
940 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
941 def decode_entity(match): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
942 what = match.group(0) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
943 return unescape(what) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
944 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
945 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
946 def htmldecode(text): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
947 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
948 Decode HTML entities in the given text. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
949 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
950 >>> htmldecode( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
951 ... 'https://../package_name-0.1.2.tar.gz' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
952 ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
953 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
954 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
955 return entity_sub(decode_entity, text) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
956 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
957 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
958 def socket_timeout(timeout=15): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
959 def _socket_timeout(func): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
960 def _socket_timeout(*args, **kwargs): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
961 old_timeout = socket.getdefaulttimeout() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
962 socket.setdefaulttimeout(timeout) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
963 try: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
964 return func(*args, **kwargs) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
965 finally: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
966 socket.setdefaulttimeout(old_timeout) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
967 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
968 return _socket_timeout |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
969 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
970 return _socket_timeout |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
971 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
972 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
973 def _encode_auth(auth): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
974 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
975 A function compatible with Python 2.3-3.3 that will encode |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
976 auth from a URL suitable for an HTTP header. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
977 >>> str(_encode_auth('username%3Apassword')) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
978 'dXNlcm5hbWU6cGFzc3dvcmQ=' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
979 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
980 Long auth strings should not cause a newline to be inserted. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
981 >>> long_auth = 'username:' + 'password'*10 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
982 >>> chr(10) in str(_encode_auth(long_auth)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
983 False |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
984 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
985 auth_s = urllib.parse.unquote(auth) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
986 # convert to bytes |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
987 auth_bytes = auth_s.encode() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
988 encoded_bytes = base64.b64encode(auth_bytes) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
989 # convert back to a string |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
990 encoded = encoded_bytes.decode() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
991 # strip the trailing carriage return |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
992 return encoded.replace('\n', '') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
993 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
994 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
995 class Credential: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
996 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
997 A username/password pair. Use like a namedtuple. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
998 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
999 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1000 def __init__(self, username, password): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1001 self.username = username |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1002 self.password = password |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1003 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1004 def __iter__(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1005 yield self.username |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1006 yield self.password |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1007 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1008 def __str__(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1009 return '%(username)s:%(password)s' % vars(self) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1010 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1011 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1012 class PyPIConfig(configparser.RawConfigParser): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1013 def __init__(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1014 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1015 Load from ~/.pypirc |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1016 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1017 defaults = dict.fromkeys(['username', 'password', 'repository'], '') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1018 configparser.RawConfigParser.__init__(self, defaults) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1019 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1020 rc = os.path.join(os.path.expanduser('~'), '.pypirc') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1021 if os.path.exists(rc): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1022 self.read(rc) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1023 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1024 @property |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1025 def creds_by_repository(self): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1026 sections_with_repositories = [ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1027 section for section in self.sections() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1028 if self.get(section, 'repository').strip() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1029 ] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1030 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1031 return dict(map(self._get_repo_cred, sections_with_repositories)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1032 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1033 def _get_repo_cred(self, section): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1034 repo = self.get(section, 'repository').strip() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1035 return repo, Credential( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1036 self.get(section, 'username').strip(), |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1037 self.get(section, 'password').strip(), |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1038 ) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1039 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1040 def find_credential(self, url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1041 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1042 If the URL indicated appears to be a repository defined in this |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1043 config, return the credential for that repository. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1044 """ |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1045 for repository, cred in self.creds_by_repository.items(): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1046 if url.startswith(repository): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1047 return cred |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1048 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1049 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1050 def open_with_auth(url, opener=urllib.request.urlopen): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1051 """Open a urllib2 request, handling HTTP authentication""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1052 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1053 parsed = urllib.parse.urlparse(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1054 scheme, netloc, path, params, query, frag = parsed |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1055 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1056 # Double scheme does not raise on Mac OS X as revealed by a |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1057 # failing test. We would expect "nonnumeric port". Refs #20. |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1058 if netloc.endswith(':'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1059 raise http_client.InvalidURL("nonnumeric port: ''") |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1060 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1061 if scheme in ('http', 'https'): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1062 auth, address = _splituser(netloc) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1063 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1064 auth = None |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1065 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1066 if not auth: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1067 cred = PyPIConfig().find_credential(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1068 if cred: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1069 auth = str(cred) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1070 info = cred.username, url |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1071 log.info('Authenticating as %s for %s (from .pypirc)', *info) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1072 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1073 if auth: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1074 auth = "Basic " + _encode_auth(auth) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1075 parts = scheme, address, path, params, query, frag |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1076 new_url = urllib.parse.urlunparse(parts) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1077 request = urllib.request.Request(new_url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1078 request.add_header("Authorization", auth) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1079 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1080 request = urllib.request.Request(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1081 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1082 request.add_header('User-Agent', user_agent) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1083 fp = opener(request) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1084 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1085 if auth: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1086 # Put authentication info back into request URL if same host, |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1087 # so that links found on the page will work |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1088 s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1089 if s2 == scheme and h2 == address: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1090 parts = s2, netloc, path2, param2, query2, frag2 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1091 fp.url = urllib.parse.urlunparse(parts) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1092 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1093 return fp |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1094 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1095 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1096 # copy of urllib.parse._splituser from Python 3.8 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1097 def _splituser(host): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1098 """splituser('user[:passwd]@host[:port]') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1099 --> 'user[:passwd]', 'host[:port]'.""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1100 user, delim, host = host.rpartition('@') |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1101 return (user if delim else None), host |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1102 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1103 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1104 # adding a timeout to avoid freezing package_index |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1105 open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1106 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1107 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1108 def fix_sf_url(url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1109 return url # backward compatibility |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1110 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1111 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1112 def local_open(url): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1113 """Read a local path, with special support for directories""" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1114 scheme, server, path, param, query, frag = urllib.parse.urlparse(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1115 filename = urllib.request.url2pathname(path) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1116 if os.path.isfile(filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1117 return urllib.request.urlopen(url) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1118 elif path.endswith('/') and os.path.isdir(filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1119 files = [] |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1120 for f in os.listdir(filename): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1121 filepath = os.path.join(filename, f) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1122 if f == 'index.html': |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1123 with open(filepath, 'r') as fp: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1124 body = fp.read() |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1125 break |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1126 elif os.path.isdir(filepath): |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1127 f += '/' |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1128 files.append('<a href="{name}">{name}</a>'.format(name=f)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1129 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1130 tmpl = ( |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1131 "<html><head><title>{url}</title>" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1132 "</head><body>{files}</body></html>") |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1133 body = tmpl.format(url=url, files='\n'.join(files)) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1134 status, message = 200, "OK" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1135 else: |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1136 status, message, body = 404, "Path not found", "Not found" |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1137 |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1138 headers = {'content-type': 'text/html'} |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1139 body_stream = six.StringIO(body) |
9e54283cc701
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff
changeset
|
1140 return urllib.error.HTTPError(url, status, message, headers, body_stream) |