Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/setuptools/package_index.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 """PyPI and direct package downloading"""
2 import sys
3 import os
4 import re
5 import shutil
6 import socket
7 import base64
8 import hashlib
9 import itertools
10 import warnings
11 from functools import wraps
12
13 from setuptools.extern import six
14 from setuptools.extern.six.moves import urllib, http_client, configparser, map
15
16 import setuptools
17 from pkg_resources import (
18 CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
19 Environment, find_distributions, safe_name, safe_version,
20 to_filename, Requirement, DEVELOP_DIST, EGG_DIST,
21 )
22 from setuptools import ssl_support
23 from distutils import log
24 from distutils.errors import DistutilsError
25 from fnmatch import translate
26 from setuptools.py27compat import get_all_headers
27 from setuptools.py33compat import unescape
28 from setuptools.wheel import Wheel
29
30 __metaclass__ = type
31
32 EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')
33 HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
34 PYPI_MD5 = re.compile(
35 r'<a href="([^"#]+)">([^<]+)</a>\n\s+\(<a (?:title="MD5 hash"\n\s+)'
36 r'href="[^?]+\?:action=show_md5&amp;digest=([0-9a-f]{32})">md5</a>\)'
37 )
38 URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match
39 EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
40
41 __all__ = [
42 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
43 'interpret_distro_name',
44 ]
45
46 _SOCKET_TIMEOUT = 15
47
48 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}"
49 user_agent = _tmpl.format(py_major=sys.version[:3], setuptools=setuptools)
50
51
52 def parse_requirement_arg(spec):
53 try:
54 return Requirement.parse(spec)
55 except ValueError:
56 raise DistutilsError(
57 "Not a URL, existing file, or requirement spec: %r" % (spec,)
58 )
59
60
61 def parse_bdist_wininst(name):
62 """Return (base,pyversion) or (None,None) for possible .exe name"""
63
64 lower = name.lower()
65 base, py_ver, plat = None, None, None
66
67 if lower.endswith('.exe'):
68 if lower.endswith('.win32.exe'):
69 base = name[:-10]
70 plat = 'win32'
71 elif lower.startswith('.win32-py', -16):
72 py_ver = name[-7:-4]
73 base = name[:-16]
74 plat = 'win32'
75 elif lower.endswith('.win-amd64.exe'):
76 base = name[:-14]
77 plat = 'win-amd64'
78 elif lower.startswith('.win-amd64-py', -20):
79 py_ver = name[-7:-4]
80 base = name[:-20]
81 plat = 'win-amd64'
82 return base, py_ver, plat
83
84
85 def egg_info_for_url(url):
86 parts = urllib.parse.urlparse(url)
87 scheme, server, path, parameters, query, fragment = parts
88 base = urllib.parse.unquote(path.split('/')[-1])
89 if server == 'sourceforge.net' and base == 'download': # XXX Yuck
90 base = urllib.parse.unquote(path.split('/')[-2])
91 if '#' in base:
92 base, fragment = base.split('#', 1)
93 return base, fragment
94
95
96 def distros_for_url(url, metadata=None):
97 """Yield egg or source distribution objects that might be found at a URL"""
98 base, fragment = egg_info_for_url(url)
99 for dist in distros_for_location(url, base, metadata):
100 yield dist
101 if fragment:
102 match = EGG_FRAGMENT.match(fragment)
103 if match:
104 for dist in interpret_distro_name(
105 url, match.group(1), metadata, precedence=CHECKOUT_DIST
106 ):
107 yield dist
108
109
110 def distros_for_location(location, basename, metadata=None):
111 """Yield egg or source distribution objects based on basename"""
112 if basename.endswith('.egg.zip'):
113 basename = basename[:-4] # strip the .zip
114 if basename.endswith('.egg') and '-' in basename:
115 # only one, unambiguous interpretation
116 return [Distribution.from_location(location, basename, metadata)]
117 if basename.endswith('.whl') and '-' in basename:
118 wheel = Wheel(basename)
119 if not wheel.is_compatible():
120 return []
121 return [Distribution(
122 location=location,
123 project_name=wheel.project_name,
124 version=wheel.version,
125 # Increase priority over eggs.
126 precedence=EGG_DIST + 1,
127 )]
128 if basename.endswith('.exe'):
129 win_base, py_ver, platform = parse_bdist_wininst(basename)
130 if win_base is not None:
131 return interpret_distro_name(
132 location, win_base, metadata, py_ver, BINARY_DIST, platform
133 )
134 # Try source distro extensions (.zip, .tgz, etc.)
135 #
136 for ext in EXTENSIONS:
137 if basename.endswith(ext):
138 basename = basename[:-len(ext)]
139 return interpret_distro_name(location, basename, metadata)
140 return [] # no extension matched
141
142
143 def distros_for_filename(filename, metadata=None):
144 """Yield possible egg or source distribution objects based on a filename"""
145 return distros_for_location(
146 normalize_path(filename), os.path.basename(filename), metadata
147 )
148
149
150 def interpret_distro_name(
151 location, basename, metadata, py_version=None, precedence=SOURCE_DIST,
152 platform=None
153 ):
154 """Generate alternative interpretations of a source distro name
155
156 Note: if `location` is a filesystem filename, you should call
157 ``pkg_resources.normalize_path()`` on it before passing it to this
158 routine!
159 """
160 # Generate alternative interpretations of a source distro name
161 # Because some packages are ambiguous as to name/versions split
162 # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
163 # So, we generate each possible interepretation (e.g. "adns, python-1.1.0"
164 # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice,
165 # the spurious interpretations should be ignored, because in the event
166 # there's also an "adns" package, the spurious "python-1.1.0" version will
167 # compare lower than any numeric version number, and is therefore unlikely
168 # to match a request for it. It's still a potential problem, though, and
169 # in the long run PyPI and the distutils should go for "safe" names and
170 # versions in distribution archive names (sdist and bdist).
171
172 parts = basename.split('-')
173 if not py_version and any(re.match(r'py\d\.\d$', p) for p in parts[2:]):
174 # it is a bdist_dumb, not an sdist -- bail out
175 return
176
177 for p in range(1, len(parts) + 1):
178 yield Distribution(
179 location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
180 py_version=py_version, precedence=precedence,
181 platform=platform
182 )
183
184
185 # From Python 2.7 docs
186 def unique_everseen(iterable, key=None):
187 "List unique elements, preserving order. Remember all elements ever seen."
188 # unique_everseen('AAAABBBCCDAABBB') --> A B C D
189 # unique_everseen('ABBCcAD', str.lower) --> A B C D
190 seen = set()
191 seen_add = seen.add
192 if key is None:
193 for element in six.moves.filterfalse(seen.__contains__, iterable):
194 seen_add(element)
195 yield element
196 else:
197 for element in iterable:
198 k = key(element)
199 if k not in seen:
200 seen_add(k)
201 yield element
202
203
204 def unique_values(func):
205 """
206 Wrap a function returning an iterable such that the resulting iterable
207 only ever yields unique items.
208 """
209
210 @wraps(func)
211 def wrapper(*args, **kwargs):
212 return unique_everseen(func(*args, **kwargs))
213
214 return wrapper
215
216
217 REL = re.compile(r"""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I)
218 # this line is here to fix emacs' cruddy broken syntax highlighting
219
220
221 @unique_values
222 def find_external_links(url, page):
223 """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""
224
225 for match in REL.finditer(page):
226 tag, rel = match.groups()
227 rels = set(map(str.strip, rel.lower().split(',')))
228 if 'homepage' in rels or 'download' in rels:
229 for match in HREF.finditer(tag):
230 yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
231
232 for tag in ("<th>Home Page", "<th>Download URL"):
233 pos = page.find(tag)
234 if pos != -1:
235 match = HREF.search(page, pos)
236 if match:
237 yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
238
239
240 class ContentChecker:
241 """
242 A null content checker that defines the interface for checking content
243 """
244
245 def feed(self, block):
246 """
247 Feed a block of data to the hash.
248 """
249 return
250
251 def is_valid(self):
252 """
253 Check the hash. Return False if validation fails.
254 """
255 return True
256
257 def report(self, reporter, template):
258 """
259 Call reporter with information about the checker (hash name)
260 substituted into the template.
261 """
262 return
263
264
265 class HashChecker(ContentChecker):
266 pattern = re.compile(
267 r'(?P<hash_name>sha1|sha224|sha384|sha256|sha512|md5)='
268 r'(?P<expected>[a-f0-9]+)'
269 )
270
271 def __init__(self, hash_name, expected):
272 self.hash_name = hash_name
273 self.hash = hashlib.new(hash_name)
274 self.expected = expected
275
276 @classmethod
277 def from_url(cls, url):
278 "Construct a (possibly null) ContentChecker from a URL"
279 fragment = urllib.parse.urlparse(url)[-1]
280 if not fragment:
281 return ContentChecker()
282 match = cls.pattern.search(fragment)
283 if not match:
284 return ContentChecker()
285 return cls(**match.groupdict())
286
287 def feed(self, block):
288 self.hash.update(block)
289
290 def is_valid(self):
291 return self.hash.hexdigest() == self.expected
292
293 def report(self, reporter, template):
294 msg = template % self.hash_name
295 return reporter(msg)
296
297
298 class PackageIndex(Environment):
299 """A distribution index that scans web pages for download URLs"""
300
301 def __init__(
302 self, index_url="https://pypi.org/simple/", hosts=('*',),
303 ca_bundle=None, verify_ssl=True, *args, **kw
304 ):
305 Environment.__init__(self, *args, **kw)
306 self.index_url = index_url + "/" [:not index_url.endswith('/')]
307 self.scanned_urls = {}
308 self.fetched_urls = {}
309 self.package_pages = {}
310 self.allows = re.compile('|'.join(map(translate, hosts))).match
311 self.to_scan = []
312 use_ssl = (
313 verify_ssl
314 and ssl_support.is_available
315 and (ca_bundle or ssl_support.find_ca_bundle())
316 )
317 if use_ssl:
318 self.opener = ssl_support.opener_for(ca_bundle)
319 else:
320 self.opener = urllib.request.urlopen
321
322 def process_url(self, url, retrieve=False):
323 """Evaluate a URL as a possible download, and maybe retrieve it"""
324 if url in self.scanned_urls and not retrieve:
325 return
326 self.scanned_urls[url] = True
327 if not URL_SCHEME(url):
328 self.process_filename(url)
329 return
330 else:
331 dists = list(distros_for_url(url))
332 if dists:
333 if not self.url_ok(url):
334 return
335 self.debug("Found link: %s", url)
336
337 if dists or not retrieve or url in self.fetched_urls:
338 list(map(self.add, dists))
339 return # don't need the actual page
340
341 if not self.url_ok(url):
342 self.fetched_urls[url] = True
343 return
344
345 self.info("Reading %s", url)
346 self.fetched_urls[url] = True # prevent multiple fetch attempts
347 tmpl = "Download error on %s: %%s -- Some packages may not be found!"
348 f = self.open_url(url, tmpl % url)
349 if f is None:
350 return
351 self.fetched_urls[f.url] = True
352 if 'html' not in f.headers.get('content-type', '').lower():
353 f.close() # not html, we can't process it
354 return
355
356 base = f.url # handle redirects
357 page = f.read()
358 if not isinstance(page, str):
359 # In Python 3 and got bytes but want str.
360 if isinstance(f, urllib.error.HTTPError):
361 # Errors have no charset, assume latin1:
362 charset = 'latin-1'
363 else:
364 charset = f.headers.get_param('charset') or 'latin-1'
365 page = page.decode(charset, "ignore")
366 f.close()
367 for match in HREF.finditer(page):
368 link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
369 self.process_url(link)
370 if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
371 page = self.process_index(url, page)
372
373 def process_filename(self, fn, nested=False):
374 # process filenames or directories
375 if not os.path.exists(fn):
376 self.warn("Not found: %s", fn)
377 return
378
379 if os.path.isdir(fn) and not nested:
380 path = os.path.realpath(fn)
381 for item in os.listdir(path):
382 self.process_filename(os.path.join(path, item), True)
383
384 dists = distros_for_filename(fn)
385 if dists:
386 self.debug("Found: %s", fn)
387 list(map(self.add, dists))
388
389 def url_ok(self, url, fatal=False):
390 s = URL_SCHEME(url)
391 is_file = s and s.group(1).lower() == 'file'
392 if is_file or self.allows(urllib.parse.urlparse(url)[1]):
393 return True
394 msg = (
395 "\nNote: Bypassing %s (disallowed host; see "
396 "http://bit.ly/2hrImnY for details).\n")
397 if fatal:
398 raise DistutilsError(msg % url)
399 else:
400 self.warn(msg, url)
401
402 def scan_egg_links(self, search_path):
403 dirs = filter(os.path.isdir, search_path)
404 egg_links = (
405 (path, entry)
406 for path in dirs
407 for entry in os.listdir(path)
408 if entry.endswith('.egg-link')
409 )
410 list(itertools.starmap(self.scan_egg_link, egg_links))
411
412 def scan_egg_link(self, path, entry):
413 with open(os.path.join(path, entry)) as raw_lines:
414 # filter non-empty lines
415 lines = list(filter(None, map(str.strip, raw_lines)))
416
417 if len(lines) != 2:
418 # format is not recognized; punt
419 return
420
421 egg_path, setup_path = lines
422
423 for dist in find_distributions(os.path.join(path, egg_path)):
424 dist.location = os.path.join(path, *lines)
425 dist.precedence = SOURCE_DIST
426 self.add(dist)
427
428 def process_index(self, url, page):
429 """Process the contents of a PyPI page"""
430
431 def scan(link):
432 # Process a URL to see if it's for a package page
433 if link.startswith(self.index_url):
434 parts = list(map(
435 urllib.parse.unquote, link[len(self.index_url):].split('/')
436 ))
437 if len(parts) == 2 and '#' not in parts[1]:
438 # it's a package page, sanitize and index it
439 pkg = safe_name(parts[0])
440 ver = safe_version(parts[1])
441 self.package_pages.setdefault(pkg.lower(), {})[link] = True
442 return to_filename(pkg), to_filename(ver)
443 return None, None
444
445 # process an index page into the package-page index
446 for match in HREF.finditer(page):
447 try:
448 scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))
449 except ValueError:
450 pass
451
452 pkg, ver = scan(url) # ensure this page is in the page index
453 if pkg:
454 # process individual package page
455 for new_url in find_external_links(url, page):
456 # Process the found URL
457 base, frag = egg_info_for_url(new_url)
458 if base.endswith('.py') and not frag:
459 if ver:
460 new_url += '#egg=%s-%s' % (pkg, ver)
461 else:
462 self.need_version_info(url)
463 self.scan_url(new_url)
464
465 return PYPI_MD5.sub(
466 lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page
467 )
468 else:
469 return "" # no sense double-scanning non-package pages
470
471 def need_version_info(self, url):
472 self.scan_all(
473 "Page at %s links to .py file(s) without version info; an index "
474 "scan is required.", url
475 )
476
477 def scan_all(self, msg=None, *args):
478 if self.index_url not in self.fetched_urls:
479 if msg:
480 self.warn(msg, *args)
481 self.info(
482 "Scanning index of all packages (this may take a while)"
483 )
484 self.scan_url(self.index_url)
485
486 def find_packages(self, requirement):
487 self.scan_url(self.index_url + requirement.unsafe_name + '/')
488
489 if not self.package_pages.get(requirement.key):
490 # Fall back to safe version of the name
491 self.scan_url(self.index_url + requirement.project_name + '/')
492
493 if not self.package_pages.get(requirement.key):
494 # We couldn't find the target package, so search the index page too
495 self.not_found_in_index(requirement)
496
497 for url in list(self.package_pages.get(requirement.key, ())):
498 # scan each page that might be related to the desired package
499 self.scan_url(url)
500
501 def obtain(self, requirement, installer=None):
502 self.prescan()
503 self.find_packages(requirement)
504 for dist in self[requirement.key]:
505 if dist in requirement:
506 return dist
507 self.debug("%s does not match %s", requirement, dist)
508 return super(PackageIndex, self).obtain(requirement, installer)
509
510 def check_hash(self, checker, filename, tfp):
511 """
512 checker is a ContentChecker
513 """
514 checker.report(
515 self.debug,
516 "Validating %%s checksum for %s" % filename)
517 if not checker.is_valid():
518 tfp.close()
519 os.unlink(filename)
520 raise DistutilsError(
521 "%s validation failed for %s; "
522 "possible download problem?"
523 % (checker.hash.name, os.path.basename(filename))
524 )
525
526 def add_find_links(self, urls):
527 """Add `urls` to the list that will be prescanned for searches"""
528 for url in urls:
529 if (
530 self.to_scan is None # if we have already "gone online"
531 or not URL_SCHEME(url) # or it's a local file/directory
532 or url.startswith('file:')
533 or list(distros_for_url(url)) # or a direct package link
534 ):
535 # then go ahead and process it now
536 self.scan_url(url)
537 else:
538 # otherwise, defer retrieval till later
539 self.to_scan.append(url)
540
541 def prescan(self):
542 """Scan urls scheduled for prescanning (e.g. --find-links)"""
543 if self.to_scan:
544 list(map(self.scan_url, self.to_scan))
545 self.to_scan = None # from now on, go ahead and process immediately
546
547 def not_found_in_index(self, requirement):
548 if self[requirement.key]: # we've seen at least one distro
549 meth, msg = self.info, "Couldn't retrieve index page for %r"
550 else: # no distros seen for this name, might be misspelled
551 meth, msg = (
552 self.warn,
553 "Couldn't find index page for %r (maybe misspelled?)")
554 meth(msg, requirement.unsafe_name)
555 self.scan_all()
556
557 def download(self, spec, tmpdir):
558 """Locate and/or download `spec` to `tmpdir`, returning a local path
559
560 `spec` may be a ``Requirement`` object, or a string containing a URL,
561 an existing local filename, or a project/version requirement spec
562 (i.e. the string form of a ``Requirement`` object). If it is the URL
563 of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
564 that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
565 automatically created alongside the downloaded file.
566
567 If `spec` is a ``Requirement`` object or a string containing a
568 project/version requirement spec, this method returns the location of
569 a matching distribution (possibly after downloading it to `tmpdir`).
570 If `spec` is a locally existing file or directory name, it is simply
571 returned unchanged. If `spec` is a URL, it is downloaded to a subpath
572 of `tmpdir`, and the local filename is returned. Various errors may be
573 raised if a problem occurs during downloading.
574 """
575 if not isinstance(spec, Requirement):
576 scheme = URL_SCHEME(spec)
577 if scheme:
578 # It's a url, download it to tmpdir
579 found = self._download_url(scheme.group(1), spec, tmpdir)
580 base, fragment = egg_info_for_url(spec)
581 if base.endswith('.py'):
582 found = self.gen_setup(found, fragment, tmpdir)
583 return found
584 elif os.path.exists(spec):
585 # Existing file or directory, just return it
586 return spec
587 else:
588 spec = parse_requirement_arg(spec)
589 return getattr(self.fetch_distribution(spec, tmpdir), 'location', None)
590
591 def fetch_distribution(
592 self, requirement, tmpdir, force_scan=False, source=False,
593 develop_ok=False, local_index=None):
594 """Obtain a distribution suitable for fulfilling `requirement`
595
596 `requirement` must be a ``pkg_resources.Requirement`` instance.
597 If necessary, or if the `force_scan` flag is set, the requirement is
598 searched for in the (online) package index as well as the locally
599 installed packages. If a distribution matching `requirement` is found,
600 the returned distribution's ``location`` is the value you would have
601 gotten from calling the ``download()`` method with the matching
602 distribution's URL or filename. If no matching distribution is found,
603 ``None`` is returned.
604
605 If the `source` flag is set, only source distributions and source
606 checkout links will be considered. Unless the `develop_ok` flag is
607 set, development and system eggs (i.e., those using the ``.egg-info``
608 format) will be ignored.
609 """
610 # process a Requirement
611 self.info("Searching for %s", requirement)
612 skipped = {}
613 dist = None
614
615 def find(req, env=None):
616 if env is None:
617 env = self
618 # Find a matching distribution; may be called more than once
619
620 for dist in env[req.key]:
621
622 if dist.precedence == DEVELOP_DIST and not develop_ok:
623 if dist not in skipped:
624 self.warn(
625 "Skipping development or system egg: %s", dist,
626 )
627 skipped[dist] = 1
628 continue
629
630 test = (
631 dist in req
632 and (dist.precedence <= SOURCE_DIST or not source)
633 )
634 if test:
635 loc = self.download(dist.location, tmpdir)
636 dist.download_location = loc
637 if os.path.exists(dist.download_location):
638 return dist
639
640 if force_scan:
641 self.prescan()
642 self.find_packages(requirement)
643 dist = find(requirement)
644
645 if not dist and local_index is not None:
646 dist = find(requirement, local_index)
647
648 if dist is None:
649 if self.to_scan is not None:
650 self.prescan()
651 dist = find(requirement)
652
653 if dist is None and not force_scan:
654 self.find_packages(requirement)
655 dist = find(requirement)
656
657 if dist is None:
658 self.warn(
659 "No local packages or working download links found for %s%s",
660 (source and "a source distribution of " or ""),
661 requirement,
662 )
663 else:
664 self.info("Best match: %s", dist)
665 return dist.clone(location=dist.download_location)
666
667 def fetch(self, requirement, tmpdir, force_scan=False, source=False):
668 """Obtain a file suitable for fulfilling `requirement`
669
670 DEPRECATED; use the ``fetch_distribution()`` method now instead. For
671 backward compatibility, this routine is identical but returns the
672 ``location`` of the downloaded distribution instead of a distribution
673 object.
674 """
675 dist = self.fetch_distribution(requirement, tmpdir, force_scan, source)
676 if dist is not None:
677 return dist.location
678 return None
679
680 def gen_setup(self, filename, fragment, tmpdir):
681 match = EGG_FRAGMENT.match(fragment)
682 dists = match and [
683 d for d in
684 interpret_distro_name(filename, match.group(1), None) if d.version
685 ] or []
686
687 if len(dists) == 1: # unambiguous ``#egg`` fragment
688 basename = os.path.basename(filename)
689
690 # Make sure the file has been downloaded to the temp dir.
691 if os.path.dirname(filename) != tmpdir:
692 dst = os.path.join(tmpdir, basename)
693 from setuptools.command.easy_install import samefile
694 if not samefile(filename, dst):
695 shutil.copy2(filename, dst)
696 filename = dst
697
698 with open(os.path.join(tmpdir, 'setup.py'), 'w') as file:
699 file.write(
700 "from setuptools import setup\n"
701 "setup(name=%r, version=%r, py_modules=[%r])\n"
702 % (
703 dists[0].project_name, dists[0].version,
704 os.path.splitext(basename)[0]
705 )
706 )
707 return filename
708
709 elif match:
710 raise DistutilsError(
711 "Can't unambiguously interpret project/version identifier %r; "
712 "any dashes in the name or version should be escaped using "
713 "underscores. %r" % (fragment, dists)
714 )
715 else:
716 raise DistutilsError(
717 "Can't process plain .py files without an '#egg=name-version'"
718 " suffix to enable automatic setup script generation."
719 )
720
721 dl_blocksize = 8192
722
723 def _download_to(self, url, filename):
724 self.info("Downloading %s", url)
725 # Download the file
726 fp = None
727 try:
728 checker = HashChecker.from_url(url)
729 fp = self.open_url(url)
730 if isinstance(fp, urllib.error.HTTPError):
731 raise DistutilsError(
732 "Can't download %s: %s %s" % (url, fp.code, fp.msg)
733 )
734 headers = fp.info()
735 blocknum = 0
736 bs = self.dl_blocksize
737 size = -1
738 if "content-length" in headers:
739 # Some servers return multiple Content-Length headers :(
740 sizes = get_all_headers(headers, 'Content-Length')
741 size = max(map(int, sizes))
742 self.reporthook(url, filename, blocknum, bs, size)
743 with open(filename, 'wb') as tfp:
744 while True:
745 block = fp.read(bs)
746 if block:
747 checker.feed(block)
748 tfp.write(block)
749 blocknum += 1
750 self.reporthook(url, filename, blocknum, bs, size)
751 else:
752 break
753 self.check_hash(checker, filename, tfp)
754 return headers
755 finally:
756 if fp:
757 fp.close()
758
759 def reporthook(self, url, filename, blocknum, blksize, size):
760 pass # no-op
761
762 def open_url(self, url, warning=None):
763 if url.startswith('file:'):
764 return local_open(url)
765 try:
766 return open_with_auth(url, self.opener)
767 except (ValueError, http_client.InvalidURL) as v:
768 msg = ' '.join([str(arg) for arg in v.args])
769 if warning:
770 self.warn(warning, msg)
771 else:
772 raise DistutilsError('%s %s' % (url, msg))
773 except urllib.error.HTTPError as v:
774 return v
775 except urllib.error.URLError as v:
776 if warning:
777 self.warn(warning, v.reason)
778 else:
779 raise DistutilsError("Download error for %s: %s"
780 % (url, v.reason))
781 except http_client.BadStatusLine as v:
782 if warning:
783 self.warn(warning, v.line)
784 else:
785 raise DistutilsError(
786 '%s returned a bad status line. The server might be '
787 'down, %s' %
788 (url, v.line)
789 )
790 except (http_client.HTTPException, socket.error) as v:
791 if warning:
792 self.warn(warning, v)
793 else:
794 raise DistutilsError("Download error for %s: %s"
795 % (url, v))
796
797 def _download_url(self, scheme, url, tmpdir):
798 # Determine download filename
799 #
800 name, fragment = egg_info_for_url(url)
801 if name:
802 while '..' in name:
803 name = name.replace('..', '.').replace('\\', '_')
804 else:
805 name = "__downloaded__" # default if URL has no path contents
806
807 if name.endswith('.egg.zip'):
808 name = name[:-4] # strip the extra .zip before download
809
810 filename = os.path.join(tmpdir, name)
811
812 # Download the file
813 #
814 if scheme == 'svn' or scheme.startswith('svn+'):
815 return self._download_svn(url, filename)
816 elif scheme == 'git' or scheme.startswith('git+'):
817 return self._download_git(url, filename)
818 elif scheme.startswith('hg+'):
819 return self._download_hg(url, filename)
820 elif scheme == 'file':
821 return urllib.request.url2pathname(urllib.parse.urlparse(url)[2])
822 else:
823 self.url_ok(url, True) # raises error if not allowed
824 return self._attempt_download(url, filename)
825
826 def scan_url(self, url):
827 self.process_url(url, True)
828
829 def _attempt_download(self, url, filename):
830 headers = self._download_to(url, filename)
831 if 'html' in headers.get('content-type', '').lower():
832 return self._download_html(url, headers, filename)
833 else:
834 return filename
835
836 def _download_html(self, url, headers, filename):
837 file = open(filename)
838 for line in file:
839 if line.strip():
840 # Check for a subversion index page
841 if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
842 # it's a subversion index page:
843 file.close()
844 os.unlink(filename)
845 return self._download_svn(url, filename)
846 break # not an index page
847 file.close()
848 os.unlink(filename)
849 raise DistutilsError("Unexpected HTML page found at " + url)
850
851 def _download_svn(self, url, filename):
852 warnings.warn("SVN download support is deprecated", UserWarning)
853 url = url.split('#', 1)[0] # remove any fragment for svn's sake
854 creds = ''
855 if url.lower().startswith('svn:') and '@' in url:
856 scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
857 if not netloc and path.startswith('//') and '/' in path[2:]:
858 netloc, path = path[2:].split('/', 1)
859 auth, host = _splituser(netloc)
860 if auth:
861 if ':' in auth:
862 user, pw = auth.split(':', 1)
863 creds = " --username=%s --password=%s" % (user, pw)
864 else:
865 creds = " --username=" + auth
866 netloc = host
867 parts = scheme, netloc, url, p, q, f
868 url = urllib.parse.urlunparse(parts)
869 self.info("Doing subversion checkout from %s to %s", url, filename)
870 os.system("svn checkout%s -q %s %s" % (creds, url, filename))
871 return filename
872
873 @staticmethod
874 def _vcs_split_rev_from_url(url, pop_prefix=False):
875 scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
876
877 scheme = scheme.split('+', 1)[-1]
878
879 # Some fragment identification fails
880 path = path.split('#', 1)[0]
881
882 rev = None
883 if '@' in path:
884 path, rev = path.rsplit('@', 1)
885
886 # Also, discard fragment
887 url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))
888
889 return url, rev
890
891 def _download_git(self, url, filename):
892 filename = filename.split('#', 1)[0]
893 url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
894
895 self.info("Doing git clone from %s to %s", url, filename)
896 os.system("git clone --quiet %s %s" % (url, filename))
897
898 if rev is not None:
899 self.info("Checking out %s", rev)
900 os.system("git -C %s checkout --quiet %s" % (
901 filename,
902 rev,
903 ))
904
905 return filename
906
907 def _download_hg(self, url, filename):
908 filename = filename.split('#', 1)[0]
909 url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
910
911 self.info("Doing hg clone from %s to %s", url, filename)
912 os.system("hg clone --quiet %s %s" % (url, filename))
913
914 if rev is not None:
915 self.info("Updating to %s", rev)
916 os.system("hg --cwd %s up -C -r %s -q" % (
917 filename,
918 rev,
919 ))
920
921 return filename
922
923 def debug(self, msg, *args):
924 log.debug(msg, *args)
925
926 def info(self, msg, *args):
927 log.info(msg, *args)
928
929 def warn(self, msg, *args):
930 log.warn(msg, *args)
931
932
933 # This pattern matches a character entity reference (a decimal numeric
934 # references, a hexadecimal numeric reference, or a named reference).
935 entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub
936
937
938 def decode_entity(match):
939 what = match.group(0)
940 return unescape(what)
941
942
943 def htmldecode(text):
944 """
945 Decode HTML entities in the given text.
946
947 >>> htmldecode(
948 ... 'https://../package_name-0.1.2.tar.gz'
949 ... '?tokena=A&amp;tokenb=B">package_name-0.1.2.tar.gz')
950 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
951 """
952 return entity_sub(decode_entity, text)
953
954
955 def socket_timeout(timeout=15):
956 def _socket_timeout(func):
957 def _socket_timeout(*args, **kwargs):
958 old_timeout = socket.getdefaulttimeout()
959 socket.setdefaulttimeout(timeout)
960 try:
961 return func(*args, **kwargs)
962 finally:
963 socket.setdefaulttimeout(old_timeout)
964
965 return _socket_timeout
966
967 return _socket_timeout
968
969
970 def _encode_auth(auth):
971 """
972 A function compatible with Python 2.3-3.3 that will encode
973 auth from a URL suitable for an HTTP header.
974 >>> str(_encode_auth('username%3Apassword'))
975 'dXNlcm5hbWU6cGFzc3dvcmQ='
976
977 Long auth strings should not cause a newline to be inserted.
978 >>> long_auth = 'username:' + 'password'*10
979 >>> chr(10) in str(_encode_auth(long_auth))
980 False
981 """
982 auth_s = urllib.parse.unquote(auth)
983 # convert to bytes
984 auth_bytes = auth_s.encode()
985 encoded_bytes = base64.b64encode(auth_bytes)
986 # convert back to a string
987 encoded = encoded_bytes.decode()
988 # strip the trailing carriage return
989 return encoded.replace('\n', '')
990
991
992 class Credential:
993 """
994 A username/password pair. Use like a namedtuple.
995 """
996
997 def __init__(self, username, password):
998 self.username = username
999 self.password = password
1000
1001 def __iter__(self):
1002 yield self.username
1003 yield self.password
1004
1005 def __str__(self):
1006 return '%(username)s:%(password)s' % vars(self)
1007
1008
1009 class PyPIConfig(configparser.RawConfigParser):
1010 def __init__(self):
1011 """
1012 Load from ~/.pypirc
1013 """
1014 defaults = dict.fromkeys(['username', 'password', 'repository'], '')
1015 configparser.RawConfigParser.__init__(self, defaults)
1016
1017 rc = os.path.join(os.path.expanduser('~'), '.pypirc')
1018 if os.path.exists(rc):
1019 self.read(rc)
1020
1021 @property
1022 def creds_by_repository(self):
1023 sections_with_repositories = [
1024 section for section in self.sections()
1025 if self.get(section, 'repository').strip()
1026 ]
1027
1028 return dict(map(self._get_repo_cred, sections_with_repositories))
1029
1030 def _get_repo_cred(self, section):
1031 repo = self.get(section, 'repository').strip()
1032 return repo, Credential(
1033 self.get(section, 'username').strip(),
1034 self.get(section, 'password').strip(),
1035 )
1036
1037 def find_credential(self, url):
1038 """
1039 If the URL indicated appears to be a repository defined in this
1040 config, return the credential for that repository.
1041 """
1042 for repository, cred in self.creds_by_repository.items():
1043 if url.startswith(repository):
1044 return cred
1045
1046
1047 def open_with_auth(url, opener=urllib.request.urlopen):
1048 """Open a urllib2 request, handling HTTP authentication"""
1049
1050 parsed = urllib.parse.urlparse(url)
1051 scheme, netloc, path, params, query, frag = parsed
1052
1053 # Double scheme does not raise on Mac OS X as revealed by a
1054 # failing test. We would expect "nonnumeric port". Refs #20.
1055 if netloc.endswith(':'):
1056 raise http_client.InvalidURL("nonnumeric port: ''")
1057
1058 if scheme in ('http', 'https'):
1059 auth, address = _splituser(netloc)
1060 else:
1061 auth = None
1062
1063 if not auth:
1064 cred = PyPIConfig().find_credential(url)
1065 if cred:
1066 auth = str(cred)
1067 info = cred.username, url
1068 log.info('Authenticating as %s for %s (from .pypirc)', *info)
1069
1070 if auth:
1071 auth = "Basic " + _encode_auth(auth)
1072 parts = scheme, address, path, params, query, frag
1073 new_url = urllib.parse.urlunparse(parts)
1074 request = urllib.request.Request(new_url)
1075 request.add_header("Authorization", auth)
1076 else:
1077 request = urllib.request.Request(url)
1078
1079 request.add_header('User-Agent', user_agent)
1080 fp = opener(request)
1081
1082 if auth:
1083 # Put authentication info back into request URL if same host,
1084 # so that links found on the page will work
1085 s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url)
1086 if s2 == scheme and h2 == address:
1087 parts = s2, netloc, path2, param2, query2, frag2
1088 fp.url = urllib.parse.urlunparse(parts)
1089
1090 return fp
1091
1092
1093 # copy of urllib.parse._splituser from Python 3.8
1094 def _splituser(host):
1095 """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
1096 user, delim, host = host.rpartition('@')
1097 return (user if delim else None), host
1098
1099
1100 # adding a timeout to avoid freezing package_index
1101 open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)
1102
1103
1104 def fix_sf_url(url):
1105 return url # backward compatibility
1106
1107
1108 def local_open(url):
1109 """Read a local path, with special support for directories"""
1110 scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
1111 filename = urllib.request.url2pathname(path)
1112 if os.path.isfile(filename):
1113 return urllib.request.urlopen(url)
1114 elif path.endswith('/') and os.path.isdir(filename):
1115 files = []
1116 for f in os.listdir(filename):
1117 filepath = os.path.join(filename, f)
1118 if f == 'index.html':
1119 with open(filepath, 'r') as fp:
1120 body = fp.read()
1121 break
1122 elif os.path.isdir(filepath):
1123 f += '/'
1124 files.append('<a href="{name}">{name}</a>'.format(name=f))
1125 else:
1126 tmpl = (
1127 "<html><head><title>{url}</title>"
1128 "</head><body>{files}</body></html>")
1129 body = tmpl.format(url=url, files='\n'.join(files))
1130 status, message = 200, "OK"
1131 else:
1132 status, message, body = 404, "Path not found", "Not found"
1133
1134 headers = {'content-type': 'text/html'}
1135 body_stream = six.StringIO(body)
1136 return urllib.error.HTTPError(url, status, message, headers, body_stream)