springsuite: planemo/lib/python3.7/site-packages/distlib/locators.py annotate

annotate planemo/lib/python3.7/site-packages/distlib/locators.py @ 1:56ad4e20f292 draft

"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"

author	guerler
date	Fri, 31 Jul 2020 00:32:28 -0400
parents	d30785e31577
children

rev	line source
0 d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1 # -- coding: utf-8 --
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	2 #
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	3 # Copyright (C) 2012-2015 Vinay Sajip.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	4 # Licensed to the Python Software Foundation under a contributor agreement.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	5 # See LICENSE.txt and CONTRIBUTORS.txt.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	6 #
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	7
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	8 import gzip
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	9 from io import BytesIO
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	10 import json
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	11 import logging
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	12 import os
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	13 import posixpath
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	14 import re
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	15 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	16 import threading
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	17 except ImportError: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	18 import dummy_threading as threading
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	19 import zlib
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	20
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	21 from . import DistlibException
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	22 from .compat import (urljoin, urlparse, urlunparse, url2pathname, pathname2url,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	23 queue, quote, unescape, string_types, build_opener,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	24 HTTPRedirectHandler as BaseRedirectHandler, text_type,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	25 Request, HTTPError, URLError)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	26 from .database import Distribution, DistributionPath, make_dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	27 from .metadata import Metadata, MetadataInvalidError
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	28 from .util import (cached_property, parse_credentials, ensure_slash,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	29 split_filename, get_project_data, parse_requirement,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	30 parse_name_and_version, ServerProxy, normalize_name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	31 from .version import get_scheme, UnsupportedVersionError
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	32 from .wheel import Wheel, is_compatible
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	33
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	34 logger = logging.getLogger(__name__)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	35
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	36 HASHER_HASH = re.compile(r'^(\w+)=([a-f0-9]+)')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	37 CHARSET = re.compile(r';\scharset\s=\s(.)\s*$', re.I)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	38 HTML_CONTENT_TYPE = re.compile('text/html\|application/x(ht)?ml')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	39 DEFAULT_INDEX = 'https://pypi.org/pypi'
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	40
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	41 def get_all_distribution_names(url=None):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	42 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	43 Return all distribution names known by an index.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	44 :param url: The URL of the index.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	45 :return: A list of all known distribution names.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	46 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	47 if url is None:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	48 url = DEFAULT_INDEX
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	49 client = ServerProxy(url, timeout=3.0)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	50 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	51 return client.list_packages()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	52 finally:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	53 client('close')()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	54
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	55 class RedirectHandler(BaseRedirectHandler):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	56 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	57 A class to work around a bug in some Python 3.2.x releases.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	58 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	59 # There's a bug in the base version for some 3.2.x
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	60 # (e.g. 3.2.2 on Ubuntu Oneiric). If a Location header
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	61 # returns e.g. /abc, it bails because it says the scheme ''
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	62 # is bogus, when actually it should use the request's
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	63 # URL for the scheme. See Python issue #13696.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	64 def http_error_302(self, req, fp, code, msg, headers):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	65 # Some servers (incorrectly) return multiple Location headers
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	66 # (so probably same goes for URI). Use first header.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	67 newurl = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	68 for key in ('location', 'uri'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	69 if key in headers:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	70 newurl = headers[key]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	71 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	72 if newurl is None: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	73 return
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	74 urlparts = urlparse(newurl)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	75 if urlparts.scheme == '':
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	76 newurl = urljoin(req.get_full_url(), newurl)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	77 if hasattr(headers, 'replace_header'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	78 headers.replace_header(key, newurl)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	79 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	80 headers[key] = newurl
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	81 return BaseRedirectHandler.http_error_302(self, req, fp, code, msg,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	82 headers)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	83
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	84 http_error_301 = http_error_303 = http_error_307 = http_error_302
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	85
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	86 class Locator(object):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	87 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	88 A base class for locators - things that locate distributions.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	89 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	90 source_extensions = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	91 binary_extensions = ('.egg', '.exe', '.whl')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	92 excluded_extensions = ('.pdf',)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	93
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	94 # A list of tags indicating which wheels you want to match. The default
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	95 # value of None matches against the tags compatible with the running
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	96 # Python. If you want to match other values, set wheel_tags on a locator
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	97 # instance to a list of tuples (pyver, abi, arch) which you want to match.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	98 wheel_tags = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	99
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	100 downloadable_extensions = source_extensions + ('.whl',)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	101
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	102 def __init__(self, scheme='default'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	103 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	104 Initialise an instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	105 :param scheme: Because locators look for most recent versions, they
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	106 need to know the version scheme to use. This specifies
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	107 the current PEP-recommended scheme - use ``'legacy'``
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	108 if you need to support existing distributions on PyPI.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	109 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	110 self._cache = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	111 self.scheme = scheme
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	112 # Because of bugs in some of the handlers on some of the platforms,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	113 # we use our own opener rather than just using urlopen.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	114 self.opener = build_opener(RedirectHandler())
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	115 # If get_project() is called from locate(), the matcher instance
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	116 # is set from the requirement passed to locate(). See issue #18 for
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	117 # why this can be useful to know.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	118 self.matcher = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	119 self.errors = queue.Queue()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	120
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	121 def get_errors(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	122 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	123 Return any errors which have occurred.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	124 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	125 result = []
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	126 while not self.errors.empty(): # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	127 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	128 e = self.errors.get(False)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	129 result.append(e)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	130 except self.errors.Empty:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	131 continue
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	132 self.errors.task_done()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	133 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	134
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	135 def clear_errors(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	136 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	137 Clear any errors which may have been logged.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	138 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	139 # Just get the errors and throw them away
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	140 self.get_errors()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	141
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	142 def clear_cache(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	143 self._cache.clear()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	144
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	145 def _get_scheme(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	146 return self._scheme
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	147
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	148 def _set_scheme(self, value):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	149 self._scheme = value
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	150
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	151 scheme = property(_get_scheme, _set_scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	152
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	153 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	154 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	155 For a given project, get a dictionary mapping available versions to Distribution
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	156 instances.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	157
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	158 This should be implemented in subclasses.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	159
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	160 If called from a locate() request, self.matcher will be set to a
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	161 matcher for the requirement to satisfy, otherwise it will be None.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	162 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	163 raise NotImplementedError('Please implement in the subclass')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	164
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	165 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	166 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	167 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	168 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	169 raise NotImplementedError('Please implement in the subclass')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	170
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	171 def get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	172 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	173 For a given project, get a dictionary mapping available versions to Distribution
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	174 instances.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	175
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	176 This calls _get_project to do all the work, and just implements a caching layer on top.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	177 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	178 if self._cache is None: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	179 result = self._get_project(name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	180 elif name in self._cache:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	181 result = self._cache[name]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	182 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	183 self.clear_errors()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	184 result = self._get_project(name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	185 self._cache[name] = result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	186 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	187
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	188 def score_url(self, url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	189 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	190 Give an url a score which can be used to choose preferred URLs
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	191 for a given project release.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	192 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	193 t = urlparse(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	194 basename = posixpath.basename(t.path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	195 compatible = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	196 is_wheel = basename.endswith('.whl')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	197 is_downloadable = basename.endswith(self.downloadable_extensions)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	198 if is_wheel:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	199 compatible = is_compatible(Wheel(basename), self.wheel_tags)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	200 return (t.scheme == 'https', 'pypi.org' in t.netloc,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	201 is_downloadable, is_wheel, compatible, basename)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	202
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	203 def prefer_url(self, url1, url2):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	204 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	205 Choose one of two URLs where both are candidates for distribution
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	206 archives for the same version of a distribution (for example,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	207 .tar.gz vs. zip).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	208
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	209 The current implementation favours https:// URLs over http://, archives
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	210 from PyPI over those from other locations, wheel compatibility (if a
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	211 wheel) and then the archive name.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	212 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	213 result = url2
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	214 if url1:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	215 s1 = self.score_url(url1)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	216 s2 = self.score_url(url2)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	217 if s1 > s2:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	218 result = url1
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	219 if result != url2:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	220 logger.debug('Not replacing %r with %r', url1, url2)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	221 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	222 logger.debug('Replacing %r with %r', url1, url2)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	223 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	224
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	225 def split_filename(self, filename, project_name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	226 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	227 Attempt to split a filename in project name, version and Python version.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	228 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	229 return split_filename(filename, project_name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	230
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	231 def convert_url_to_download_info(self, url, project_name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	232 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	233 See if a URL is a candidate for a download URL for a project (the URL
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	234 has typically been scraped from an HTML page).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	235
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	236 If it is, a dictionary is returned with keys "name", "version",
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	237 "filename" and "url"; otherwise, None is returned.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	238 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	239 def same_project(name1, name2):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	240 return normalize_name(name1) == normalize_name(name2)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	241
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	242 result = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	243 scheme, netloc, path, params, query, frag = urlparse(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	244 if frag.lower().startswith('egg='): # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	245 logger.debug('%s: version hint in fragment: %r',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	246 project_name, frag)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	247 m = HASHER_HASH.match(frag)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	248 if m:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	249 algo, digest = m.groups()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	250 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	251 algo, digest = None, None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	252 origpath = path
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	253 if path and path[-1] == '/': # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	254 path = path[:-1]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	255 if path.endswith('.whl'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	256 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	257 wheel = Wheel(path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	258 if not is_compatible(wheel, self.wheel_tags):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	259 logger.debug('Wheel not compatible: %s', path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	260 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	261 if project_name is None:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	262 include = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	263 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	264 include = same_project(wheel.name, project_name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	265 if include:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	266 result = {
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	267 'name': wheel.name,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	268 'version': wheel.version,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	269 'filename': wheel.filename,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	270 'url': urlunparse((scheme, netloc, origpath,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	271 params, query, '')),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	272 'python-version': ', '.join(
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	273 ['.'.join(list(v[2:])) for v in wheel.pyver]),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	274 }
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	275 except Exception as e: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	276 logger.warning('invalid path for wheel: %s', path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	277 elif not path.endswith(self.downloadable_extensions): # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	278 logger.debug('Not downloadable: %s', path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	279 else: # downloadable extension
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	280 path = filename = posixpath.basename(path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	281 for ext in self.downloadable_extensions:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	282 if path.endswith(ext):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	283 path = path[:-len(ext)]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	284 t = self.split_filename(path, project_name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	285 if not t: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	286 logger.debug('No match for project/version: %s', path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	287 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	288 name, version, pyver = t
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	289 if not project_name or same_project(project_name, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	290 result = {
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	291 'name': name,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	292 'version': version,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	293 'filename': filename,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	294 'url': urlunparse((scheme, netloc, origpath,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	295 params, query, '')),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	296 #'packagetype': 'sdist',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	297 }
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	298 if pyver: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	299 result['python-version'] = pyver
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	300 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	301 if result and algo:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	302 result['%s_digest' % algo] = digest
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	303 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	304
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	305 def _get_digest(self, info):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	306 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	307 Get a digest from a dictionary by looking at a "digests" dictionary
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	308 or keys of the form 'algo_digest'.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	309
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	310 Returns a 2-tuple (algo, digest) if found, else None. Currently
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	311 looks only for SHA256, then MD5.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	312 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	313 result = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	314 if 'digests' in info:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	315 digests = info['digests']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	316 for algo in ('sha256', 'md5'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	317 if algo in digests:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	318 result = (algo, digests[algo])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	319 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	320 if not result:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	321 for algo in ('sha256', 'md5'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	322 key = '%s_digest' % algo
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	323 if key in info:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	324 result = (algo, info[key])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	325 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	326 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	327
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	328 def _update_version_data(self, result, info):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	329 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	330 Update a result dictionary (the final result from _get_project) with a
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	331 dictionary for a specific version, which typically holds information
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	332 gleaned from a filename or URL for an archive for the distribution.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	333 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	334 name = info.pop('name')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	335 version = info.pop('version')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	336 if version in result:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	337 dist = result[version]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	338 md = dist.metadata
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	339 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	340 dist = make_dist(name, version, scheme=self.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	341 md = dist.metadata
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	342 dist.digest = digest = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	343 url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	344 result['digests'][url] = digest
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	345 if md.source_url != info['url']:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	346 md.source_url = self.prefer_url(md.source_url, url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	347 result['urls'].setdefault(version, set()).add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	348 dist.locator = self
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	349 result[version] = dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	350
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	351 def locate(self, requirement, prereleases=False):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	352 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	353 Find the most recent distribution which matches the given
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	354 requirement.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	355
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	356 :param requirement: A requirement of the form 'foo (1.0)' or perhaps
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	357 'foo (>= 1.0, < 2.0, != 1.3)'
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	358 :param prereleases: If ``True``, allow pre-release versions
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	359 to be located. Otherwise, pre-release versions
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	360 are not returned.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	361 :return: A :class:`Distribution` instance, or ``None`` if no such
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	362 distribution could be located.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	363 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	364 result = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	365 r = parse_requirement(requirement)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	366 if r is None: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	367 raise DistlibException('Not a valid requirement: %r' % requirement)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	368 scheme = get_scheme(self.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	369 self.matcher = matcher = scheme.matcher(r.requirement)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	370 logger.debug('matcher: %s (%s)', matcher, type(matcher).__name__)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	371 versions = self.get_project(r.name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	372 if len(versions) > 2: # urls and digests keys are present
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	373 # sometimes, versions are invalid
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	374 slist = []
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	375 vcls = matcher.version_class
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	376 for k in versions:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	377 if k in ('urls', 'digests'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	378 continue
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	379 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	380 if not matcher.match(k):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	381 logger.debug('%s did not match %r', matcher, k)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	382 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	383 if prereleases or not vcls(k).is_prerelease:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	384 slist.append(k)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	385 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	386 logger.debug('skipping pre-release '
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	387 'version %s of %s', k, matcher.name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	388 except Exception: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	389 logger.warning('error matching %s with %r', matcher, k)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	390 pass # slist.append(k)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	391 if len(slist) > 1:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	392 slist = sorted(slist, key=scheme.key)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	393 if slist:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	394 logger.debug('sorted list: %s', slist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	395 version = slist[-1]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	396 result = versions[version]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	397 if result:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	398 if r.extras:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	399 result.extras = r.extras
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	400 result.download_urls = versions.get('urls', {}).get(version, set())
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	401 d = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	402 sd = versions.get('digests', {})
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	403 for url in result.download_urls:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	404 if url in sd: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	405 d[url] = sd[url]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	406 result.digests = d
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	407 self.matcher = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	408 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	409
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	410
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	411 class PyPIRPCLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	412 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	413 This locator uses XML-RPC to locate distributions. It therefore
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	414 cannot be used with simple mirrors (that only mirror file content).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	415 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	416 def __init__(self, url, **kwargs):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	417 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	418 Initialise an instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	419
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	420 :param url: The URL to use for XML-RPC.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	421 :param kwargs: Passed to the superclass constructor.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	422 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	423 super(PyPIRPCLocator, self).__init__(**kwargs)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	424 self.base_url = url
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	425 self.client = ServerProxy(url, timeout=3.0)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	426
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	427 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	428 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	429 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	430 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	431 return set(self.client.list_packages())
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	432
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	433 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	434 result = {'urls': {}, 'digests': {}}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	435 versions = self.client.package_releases(name, True)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	436 for v in versions:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	437 urls = self.client.release_urls(name, v)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	438 data = self.client.release_data(name, v)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	439 metadata = Metadata(scheme=self.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	440 metadata.name = data['name']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	441 metadata.version = data['version']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	442 metadata.license = data.get('license')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	443 metadata.keywords = data.get('keywords', [])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	444 metadata.summary = data.get('summary')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	445 dist = Distribution(metadata)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	446 if urls:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	447 info = urls[0]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	448 metadata.source_url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	449 dist.digest = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	450 dist.locator = self
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	451 result[v] = dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	452 for info in urls:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	453 url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	454 digest = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	455 result['urls'].setdefault(v, set()).add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	456 result['digests'][url] = digest
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	457 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	458
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	459 class PyPIJSONLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	460 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	461 This locator uses PyPI's JSON interface. It's very limited in functionality
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	462 and probably not worth using.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	463 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	464 def __init__(self, url, **kwargs):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	465 super(PyPIJSONLocator, self).__init__(**kwargs)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	466 self.base_url = ensure_slash(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	467
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	468 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	469 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	470 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	471 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	472 raise NotImplementedError('Not available from this locator')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	473
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	474 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	475 result = {'urls': {}, 'digests': {}}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	476 url = urljoin(self.base_url, '%s/json' % quote(name))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	477 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	478 resp = self.opener.open(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	479 data = resp.read().decode() # for now
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	480 d = json.loads(data)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	481 md = Metadata(scheme=self.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	482 data = d['info']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	483 md.name = data['name']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	484 md.version = data['version']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	485 md.license = data.get('license')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	486 md.keywords = data.get('keywords', [])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	487 md.summary = data.get('summary')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	488 dist = Distribution(md)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	489 dist.locator = self
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	490 urls = d['urls']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	491 result[md.version] = dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	492 for info in d['urls']:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	493 url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	494 dist.download_urls.add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	495 dist.digests[url] = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	496 result['urls'].setdefault(md.version, set()).add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	497 result['digests'][url] = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	498 # Now get other releases
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	499 for version, infos in d['releases'].items():
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	500 if version == md.version:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	501 continue # already done
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	502 omd = Metadata(scheme=self.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	503 omd.name = md.name
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	504 omd.version = version
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	505 odist = Distribution(omd)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	506 odist.locator = self
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	507 result[version] = odist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	508 for info in infos:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	509 url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	510 odist.download_urls.add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	511 odist.digests[url] = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	512 result['urls'].setdefault(version, set()).add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	513 result['digests'][url] = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	514 # for info in urls:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	515 # md.source_url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	516 # dist.digest = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	517 # dist.locator = self
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	518 # for info in urls:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	519 # url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	520 # result['urls'].setdefault(md.version, set()).add(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	521 # result['digests'][url] = self._get_digest(info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	522 except Exception as e:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	523 self.errors.put(text_type(e))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	524 logger.exception('JSON fetch failed: %s', e)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	525 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	526
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	527
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	528 class Page(object):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	529 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	530 This class represents a scraped HTML page.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	531 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	532 # The following slightly hairy-looking regex just looks for the contents of
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	533 # an anchor link, which has an attribute "href" either immediately preceded
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	534 # or immediately followed by a "rel" attribute. The attribute values can be
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	535 # declared with double quotes, single quotes or no quotes - which leads to
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	536 # the length of the expression.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	537 _href = re.compile("""
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	538 (rel\\s=\\s(?:"(?P<rel1>[^"])"\|'(?P<rel2>[^'])'\|(?P<rel3>[^>\\s\n]*))\\s+)?
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	539 href\\s=\\s(?:"(?P<url1>[^"])"\|'(?P<url2>[^'])'\|(?P<url3>[^>\\s\n]*))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	540 (\\s+rel\\s=\\s(?:"(?P<rel4>[^"])"\|'(?P<rel5>[^'])'\|(?P<rel6>[^>\\s\n]*)))?
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	541 """, re.I \| re.S \| re.X)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	542 _base = re.compile(r"""<base\s+href\s=\s['"]?([^'">]+)""", re.I \| re.S)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	543
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	544 def __init__(self, data, url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	545 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	546 Initialise an instance with the Unicode page contents and the URL they
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	547 came from.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	548 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	549 self.data = data
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	550 self.base_url = self.url = url
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	551 m = self._base.search(self.data)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	552 if m:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	553 self.base_url = m.group(1)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	554
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	555 _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\\|-]', re.I)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	556
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	557 @cached_property
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	558 def links(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	559 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	560 Return the URLs of all the links on a page together with information
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	561 about their "rel" attribute, for determining which ones to treat as
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	562 downloads and which ones to queue for further scraping.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	563 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	564 def clean(url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	565 "Tidy up an URL."
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	566 scheme, netloc, path, params, query, frag = urlparse(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	567 return urlunparse((scheme, netloc, quote(path),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	568 params, query, frag))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	569
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	570 result = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	571 for match in self._href.finditer(self.data):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	572 d = match.groupdict('')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	573 rel = (d['rel1'] or d['rel2'] or d['rel3'] or
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	574 d['rel4'] or d['rel5'] or d['rel6'])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	575 url = d['url1'] or d['url2'] or d['url3']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	576 url = urljoin(self.base_url, url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	577 url = unescape(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	578 url = self._clean_re.sub(lambda m: '%%%2x' % ord(m.group(0)), url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	579 result.add((url, rel))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	580 # We sort the result, hoping to bring the most recent versions
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	581 # to the front
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	582 result = sorted(result, key=lambda t: t[0], reverse=True)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	583 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	584
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	585
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	586 class SimpleScrapingLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	587 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	588 A locator which scrapes HTML pages to locate downloads for a distribution.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	589 This runs multiple threads to do the I/O; performance is at least as good
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	590 as pip's PackageFinder, which works in an analogous fashion.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	591 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	592
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	593 # These are used to deal with various Content-Encoding schemes.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	594 decoders = {
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	595 'deflate': zlib.decompress,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	596 'gzip': lambda b: gzip.GzipFile(fileobj=BytesIO(d)).read(),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	597 'none': lambda b: b,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	598 }
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	599
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	600 def __init__(self, url, timeout=None, num_workers=10, **kwargs):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	601 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	602 Initialise an instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	603 :param url: The root URL to use for scraping.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	604 :param timeout: The timeout, in seconds, to be applied to requests.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	605 This defaults to ``None`` (no timeout specified).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	606 :param num_workers: The number of worker threads you want to do I/O,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	607 This defaults to 10.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	608 :param kwargs: Passed to the superclass.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	609 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	610 super(SimpleScrapingLocator, self).__init__(**kwargs)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	611 self.base_url = ensure_slash(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	612 self.timeout = timeout
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	613 self._page_cache = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	614 self._seen = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	615 self._to_fetch = queue.Queue()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	616 self._bad_hosts = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	617 self.skip_externals = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	618 self.num_workers = num_workers
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	619 self._lock = threading.RLock()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	620 # See issue #45: we need to be resilient when the locator is used
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	621 # in a thread, e.g. with concurrent.futures. We can't use self._lock
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	622 # as it is for coordinating our internal threads - the ones created
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	623 # in _prepare_threads.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	624 self._gplock = threading.RLock()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	625 self.platform_check = False # See issue #112
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	626
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	627 def _prepare_threads(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	628 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	629 Threads are created only when get_project is called, and terminate
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	630 before it returns. They are there primarily to parallelise I/O (i.e.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	631 fetching web pages).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	632 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	633 self._threads = []
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	634 for i in range(self.num_workers):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	635 t = threading.Thread(target=self._fetch)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	636 t.setDaemon(True)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	637 t.start()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	638 self._threads.append(t)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	639
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	640 def _wait_threads(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	641 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	642 Tell all the threads to terminate (by sending a sentinel value) and
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	643 wait for them to do so.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	644 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	645 # Note that you need two loops, since you can't say which
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	646 # thread will get each sentinel
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	647 for t in self._threads:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	648 self._to_fetch.put(None) # sentinel
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	649 for t in self._threads:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	650 t.join()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	651 self._threads = []
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	652
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	653 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	654 result = {'urls': {}, 'digests': {}}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	655 with self._gplock:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	656 self.result = result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	657 self.project_name = name
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	658 url = urljoin(self.base_url, '%s/' % quote(name))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	659 self._seen.clear()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	660 self._page_cache.clear()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	661 self._prepare_threads()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	662 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	663 logger.debug('Queueing %s', url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	664 self._to_fetch.put(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	665 self._to_fetch.join()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	666 finally:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	667 self._wait_threads()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	668 del self.result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	669 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	670
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	671 platform_dependent = re.compile(r'\b(linux_(i\d86\|x86_64\|arm\w+)\|'
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	672 r'win(32\|_amd64)\|macosx_?\d+)\b', re.I)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	673
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	674 def _is_platform_dependent(self, url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	675 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	676 Does an URL refer to a platform-specific download?
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	677 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	678 return self.platform_dependent.search(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	679
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	680 def _process_download(self, url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	681 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	682 See if an URL is a suitable download for a project.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	683
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	684 If it is, register information in the result dictionary (for
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	685 _get_project) about the specific version it's for.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	686
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	687 Note that the return value isn't actually used other than as a boolean
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	688 value.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	689 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	690 if self.platform_check and self._is_platform_dependent(url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	691 info = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	692 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	693 info = self.convert_url_to_download_info(url, self.project_name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	694 logger.debug('process_download: %s -> %s', url, info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	695 if info:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	696 with self._lock: # needed because self.result is shared
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	697 self._update_version_data(self.result, info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	698 return info
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	699
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	700 def _should_queue(self, link, referrer, rel):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	701 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	702 Determine whether a link URL from a referring page and with a
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	703 particular "rel" attribute should be queued for scraping.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	704 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	705 scheme, netloc, path, _, _, _ = urlparse(link)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	706 if path.endswith(self.source_extensions + self.binary_extensions +
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	707 self.excluded_extensions):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	708 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	709 elif self.skip_externals and not link.startswith(self.base_url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	710 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	711 elif not referrer.startswith(self.base_url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	712 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	713 elif rel not in ('homepage', 'download'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	714 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	715 elif scheme not in ('http', 'https', 'ftp'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	716 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	717 elif self._is_platform_dependent(link):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	718 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	719 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	720 host = netloc.split(':', 1)[0]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	721 if host.lower() == 'localhost':
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	722 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	723 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	724 result = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	725 logger.debug('should_queue: %s (%s) from %s -> %s', link, rel,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	726 referrer, result)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	727 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	728
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	729 def _fetch(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	730 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	731 Get a URL to fetch from the work queue, get the HTML page, examine its
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	732 links for download candidates and candidates for further scraping.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	733
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	734 This is a handy method to run in a thread.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	735 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	736 while True:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	737 url = self._to_fetch.get()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	738 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	739 if url:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	740 page = self.get_page(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	741 if page is None: # e.g. after an error
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	742 continue
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	743 for link, rel in page.links:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	744 if link not in self._seen:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	745 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	746 self._seen.add(link)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	747 if (not self._process_download(link) and
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	748 self._should_queue(link, url, rel)):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	749 logger.debug('Queueing %s from %s', link, url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	750 self._to_fetch.put(link)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	751 except MetadataInvalidError: # e.g. invalid versions
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	752 pass
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	753 except Exception as e: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	754 self.errors.put(text_type(e))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	755 finally:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	756 # always do this, to avoid hangs :-)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	757 self._to_fetch.task_done()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	758 if not url:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	759 #logger.debug('Sentinel seen, quitting.')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	760 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	761
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	762 def get_page(self, url):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	763 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	764 Get the HTML for an URL, possibly from an in-memory cache.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	765
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	766 XXX TODO Note: this cache is never actually cleared. It's assumed that
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	767 the data won't get stale over the lifetime of a locator instance (not
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	768 necessarily true for the default_locator).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	769 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	770 # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	771 scheme, netloc, path, _, _, _ = urlparse(url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	772 if scheme == 'file' and os.path.isdir(url2pathname(path)):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	773 url = urljoin(ensure_slash(url), 'index.html')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	774
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	775 if url in self._page_cache:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	776 result = self._page_cache[url]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	777 logger.debug('Returning %s from cache: %s', url, result)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	778 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	779 host = netloc.split(':', 1)[0]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	780 result = None
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	781 if host in self._bad_hosts:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	782 logger.debug('Skipping %s due to bad host %s', url, host)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	783 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	784 req = Request(url, headers={'Accept-encoding': 'identity'})
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	785 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	786 logger.debug('Fetching %s', url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	787 resp = self.opener.open(req, timeout=self.timeout)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	788 logger.debug('Fetched %s', url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	789 headers = resp.info()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	790 content_type = headers.get('Content-Type', '')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	791 if HTML_CONTENT_TYPE.match(content_type):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	792 final_url = resp.geturl()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	793 data = resp.read()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	794 encoding = headers.get('Content-Encoding')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	795 if encoding:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	796 decoder = self.decoders[encoding] # fail if not found
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	797 data = decoder(data)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	798 encoding = 'utf-8'
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	799 m = CHARSET.search(content_type)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	800 if m:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	801 encoding = m.group(1)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	802 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	803 data = data.decode(encoding)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	804 except UnicodeError: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	805 data = data.decode('latin-1') # fallback
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	806 result = Page(data, final_url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	807 self._page_cache[final_url] = result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	808 except HTTPError as e:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	809 if e.code != 404:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	810 logger.exception('Fetch failed: %s: %s', url, e)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	811 except URLError as e: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	812 logger.exception('Fetch failed: %s: %s', url, e)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	813 with self._lock:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	814 self._bad_hosts.add(host)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	815 except Exception as e: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	816 logger.exception('Fetch failed: %s: %s', url, e)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	817 finally:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	818 self._page_cache[url] = result # even if None (failure)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	819 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	820
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	821 _distname_re = re.compile('<a href=[^>]*>([^<]+)<')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	822
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	823 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	824 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	825 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	826 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	827 result = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	828 page = self.get_page(self.base_url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	829 if not page:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	830 raise DistlibException('Unable to get %s' % self.base_url)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	831 for match in self._distname_re.finditer(page.data):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	832 result.add(match.group(1))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	833 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	834
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	835 class DirectoryLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	836 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	837 This class locates distributions in a directory tree.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	838 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	839
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	840 def __init__(self, path, **kwargs):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	841 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	842 Initialise an instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	843 :param path: The root of the directory tree to search.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	844 :param kwargs: Passed to the superclass constructor,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	845 except for:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	846 * recursive - if True (the default), subdirectories are
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	847 recursed into. If False, only the top-level directory
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	848 is searched,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	849 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	850 self.recursive = kwargs.pop('recursive', True)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	851 super(DirectoryLocator, self).__init__(**kwargs)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	852 path = os.path.abspath(path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	853 if not os.path.isdir(path): # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	854 raise DistlibException('Not a directory: %r' % path)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	855 self.base_dir = path
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	856
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	857 def should_include(self, filename, parent):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	858 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	859 Should a filename be considered as a candidate for a distribution
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	860 archive? As well as the filename, the directory which contains it
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	861 is provided, though not used by the current implementation.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	862 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	863 return filename.endswith(self.downloadable_extensions)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	864
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	865 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	866 result = {'urls': {}, 'digests': {}}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	867 for root, dirs, files in os.walk(self.base_dir):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	868 for fn in files:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	869 if self.should_include(fn, root):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	870 fn = os.path.join(root, fn)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	871 url = urlunparse(('file', '',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	872 pathname2url(os.path.abspath(fn)),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	873 '', '', ''))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	874 info = self.convert_url_to_download_info(url, name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	875 if info:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	876 self._update_version_data(result, info)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	877 if not self.recursive:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	878 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	879 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	880
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	881 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	882 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	883 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	884 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	885 result = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	886 for root, dirs, files in os.walk(self.base_dir):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	887 for fn in files:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	888 if self.should_include(fn, root):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	889 fn = os.path.join(root, fn)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	890 url = urlunparse(('file', '',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	891 pathname2url(os.path.abspath(fn)),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	892 '', '', ''))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	893 info = self.convert_url_to_download_info(url, None)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	894 if info:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	895 result.add(info['name'])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	896 if not self.recursive:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	897 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	898 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	899
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	900 class JSONLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	901 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	902 This locator uses special extended metadata (not available on PyPI) and is
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	903 the basis of performant dependency resolution in distlib. Other locators
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	904 require archive downloads before dependencies can be determined! As you
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	905 might imagine, that can be slow.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	906 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	907 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	908 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	909 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	910 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	911 raise NotImplementedError('Not available from this locator')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	912
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	913 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	914 result = {'urls': {}, 'digests': {}}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	915 data = get_project_data(name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	916 if data:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	917 for info in data.get('files', []):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	918 if info['ptype'] != 'sdist' or info['pyversion'] != 'source':
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	919 continue
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	920 # We don't store summary in project metadata as it makes
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	921 # the data bigger for no benefit during dependency
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	922 # resolution
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	923 dist = make_dist(data['name'], info['version'],
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	924 summary=data.get('summary',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	925 'Placeholder for summary'),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	926 scheme=self.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	927 md = dist.metadata
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	928 md.source_url = info['url']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	929 # TODO SHA256 digest
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	930 if 'digest' in info and info['digest']:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	931 dist.digest = ('md5', info['digest'])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	932 md.dependencies = info.get('requirements', {})
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	933 dist.exports = info.get('exports', {})
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	934 result[dist.version] = dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	935 result['urls'].setdefault(dist.version, set()).add(info['url'])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	936 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	937
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	938 class DistPathLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	939 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	940 This locator finds installed distributions in a path. It can be useful for
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	941 adding to an :class:`AggregatingLocator`.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	942 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	943 def __init__(self, distpath, **kwargs):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	944 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	945 Initialise an instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	946
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	947 :param distpath: A :class:`DistributionPath` instance to search.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	948 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	949 super(DistPathLocator, self).__init__(**kwargs)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	950 assert isinstance(distpath, DistributionPath)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	951 self.distpath = distpath
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	952
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	953 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	954 dist = self.distpath.get_distribution(name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	955 if dist is None:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	956 result = {'urls': {}, 'digests': {}}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	957 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	958 result = {
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	959 dist.version: dist,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	960 'urls': {dist.version: set([dist.source_url])},
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	961 'digests': {dist.version: set([None])}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	962 }
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	963 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	964
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	965
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	966 class AggregatingLocator(Locator):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	967 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	968 This class allows you to chain and/or merge a list of locators.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	969 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	970 def __init__(self, locators, *kwargs):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	971 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	972 Initialise an instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	973
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	974 :param locators: The list of locators to search.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	975 :param kwargs: Passed to the superclass constructor,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	976 except for:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	977 * merge - if False (the default), the first successful
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	978 search from any of the locators is returned. If True,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	979 the results from all locators are merged (this can be
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	980 slow).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	981 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	982 self.merge = kwargs.pop('merge', False)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	983 self.locators = locators
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	984 super(AggregatingLocator, self).__init__(**kwargs)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	985
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	986 def clear_cache(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	987 super(AggregatingLocator, self).clear_cache()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	988 for locator in self.locators:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	989 locator.clear_cache()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	990
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	991 def _set_scheme(self, value):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	992 self._scheme = value
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	993 for locator in self.locators:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	994 locator.scheme = value
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	995
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	996 scheme = property(Locator.scheme.fget, _set_scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	997
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	998 def _get_project(self, name):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	999 result = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1000 for locator in self.locators:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1001 d = locator.get_project(name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1002 if d:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1003 if self.merge:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1004 files = result.get('urls', {})
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1005 digests = result.get('digests', {})
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1006 # next line could overwrite result['urls'], result['digests']
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1007 result.update(d)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1008 df = result.get('urls')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1009 if files and df:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1010 for k, v in files.items():
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1011 if k in df:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1012 df[k] \|= v
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1013 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1014 df[k] = v
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1015 dd = result.get('digests')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1016 if digests and dd:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1017 dd.update(digests)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1018 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1019 # See issue #18. If any dists are found and we're looking
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1020 # for specific constraints, we only return something if
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1021 # a match is found. For example, if a DirectoryLocator
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1022 # returns just foo (1.0) while we're looking for
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1023 # foo (>= 2.0), we'll pretend there was nothing there so
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1024 # that subsequent locators can be queried. Otherwise we
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1025 # would just return foo (1.0) which would then lead to a
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1026 # failure to find foo (>= 2.0), because other locators
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1027 # weren't searched. Note that this only matters when
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1028 # merge=False.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1029 if self.matcher is None:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1030 found = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1031 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1032 found = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1033 for k in d:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1034 if self.matcher.match(k):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1035 found = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1036 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1037 if found:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1038 result = d
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1039 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1040 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1041
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1042 def get_distribution_names(self):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1043 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1044 Return all the distribution names known to this locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1045 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1046 result = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1047 for locator in self.locators:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1048 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1049 result \|= locator.get_distribution_names()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1050 except NotImplementedError:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1051 pass
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1052 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1053
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1054
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1055 # We use a legacy scheme simply because most of the dists on PyPI use legacy
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1056 # versions which don't conform to PEP 426 / PEP 440.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1057 default_locator = AggregatingLocator(
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1058 JSONLocator(),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1059 SimpleScrapingLocator('https://pypi.org/simple/',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1060 timeout=3.0),
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1061 scheme='legacy')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1062
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1063 locate = default_locator.locate
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1064
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1065 NAME_VERSION_RE = re.compile(r'(?P<name>[\w-]+)\s*'
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1066 r'$\s(==\s)?(?P<ver>[^)]+)$$')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1067
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1068 class DependencyFinder(object):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1069 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1070 Locate dependencies for distributions.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1071 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1072
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1073 def __init__(self, locator=None):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1074 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1075 Initialise an instance, using the specified locator
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1076 to locate distributions.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1077 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1078 self.locator = locator or default_locator
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1079 self.scheme = get_scheme(self.locator.scheme)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1080
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1081 def add_distribution(self, dist):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1082 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1083 Add a distribution to the finder. This will update internal information
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1084 about who provides what.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1085 :param dist: The distribution to add.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1086 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1087 logger.debug('adding distribution %s', dist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1088 name = dist.key
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1089 self.dists_by_name[name] = dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1090 self.dists[(name, dist.version)] = dist
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1091 for p in dist.provides:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1092 name, version = parse_name_and_version(p)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1093 logger.debug('Add to provided: %s, %s, %s', name, version, dist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1094 self.provided.setdefault(name, set()).add((version, dist))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1095
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1096 def remove_distribution(self, dist):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1097 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1098 Remove a distribution from the finder. This will update internal
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1099 information about who provides what.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1100 :param dist: The distribution to remove.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1101 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1102 logger.debug('removing distribution %s', dist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1103 name = dist.key
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1104 del self.dists_by_name[name]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1105 del self.dists[(name, dist.version)]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1106 for p in dist.provides:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1107 name, version = parse_name_and_version(p)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1108 logger.debug('Remove from provided: %s, %s, %s', name, version, dist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1109 s = self.provided[name]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1110 s.remove((version, dist))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1111 if not s:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1112 del self.provided[name]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1113
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1114 def get_matcher(self, reqt):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1115 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1116 Get a version matcher for a requirement.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1117 :param reqt: The requirement
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1118 :type reqt: str
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1119 :return: A version matcher (an instance of
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1120 :class:`distlib.version.Matcher`).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1121 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1122 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1123 matcher = self.scheme.matcher(reqt)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1124 except UnsupportedVersionError: # pragma: no cover
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1125 # XXX compat-mode if cannot read the version
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1126 name = reqt.split()[0]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1127 matcher = self.scheme.matcher(name)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1128 return matcher
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1129
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1130 def find_providers(self, reqt):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1131 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1132 Find the distributions which can fulfill a requirement.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1133
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1134 :param reqt: The requirement.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1135 :type reqt: str
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1136 :return: A set of distribution which can fulfill the requirement.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1137 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1138 matcher = self.get_matcher(reqt)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1139 name = matcher.key # case-insensitive
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1140 result = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1141 provided = self.provided
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1142 if name in provided:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1143 for version, provider in provided[name]:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1144 try:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1145 match = matcher.match(version)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1146 except UnsupportedVersionError:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1147 match = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1148
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1149 if match:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1150 result.add(provider)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1151 break
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1152 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1153
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1154 def try_to_replace(self, provider, other, problems):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1155 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1156 Attempt to replace one provider with another. This is typically used
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1157 when resolving dependencies from multiple sources, e.g. A requires
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1158 (B >= 1.0) while C requires (B >= 1.1).
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1159
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1160 For successful replacement, ``provider`` must meet all the requirements
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1161 which ``other`` fulfills.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1162
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1163 :param provider: The provider we are trying to replace with.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1164 :param other: The provider we're trying to replace.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1165 :param problems: If False is returned, this will contain what
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1166 problems prevented replacement. This is currently
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1167 a tuple of the literal string 'cantreplace',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1168 ``provider``, ``other`` and the set of requirements
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1169 that ``provider`` couldn't fulfill.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1170 :return: True if we can replace ``other`` with ``provider``, else
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1171 False.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1172 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1173 rlist = self.reqts[other]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1174 unmatched = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1175 for s in rlist:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1176 matcher = self.get_matcher(s)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1177 if not matcher.match(provider.version):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1178 unmatched.add(s)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1179 if unmatched:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1180 # can't replace other with provider
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1181 problems.add(('cantreplace', provider, other,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1182 frozenset(unmatched)))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1183 result = False
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1184 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1185 # can replace other with provider
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1186 self.remove_distribution(other)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1187 del self.reqts[other]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1188 for s in rlist:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1189 self.reqts.setdefault(provider, set()).add(s)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1190 self.add_distribution(provider)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1191 result = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1192 return result
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1193
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1194 def find(self, requirement, meta_extras=None, prereleases=False):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1195 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1196 Find a distribution and all distributions it depends on.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1197
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1198 :param requirement: The requirement specifying the distribution to
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1199 find, or a Distribution instance.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1200 :param meta_extras: A list of meta extras such as :test:, :build: and
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1201 so on.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1202 :param prereleases: If ``True``, allow pre-release versions to be
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1203 returned - otherwise, don't return prereleases
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1204 unless they're all that's available.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1205
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1206 Return a set of :class:`Distribution` instances and a set of
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1207 problems.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1208
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1209 The distributions returned should be such that they have the
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1210 :attr:`required` attribute set to ``True`` if they were
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1211 from the ``requirement`` passed to ``find()``, and they have the
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1212 :attr:`build_time_dependency` attribute set to ``True`` unless they
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1213 are post-installation dependencies of the ``requirement``.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1214
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1215 The problems should be a tuple consisting of the string
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1216 ``'unsatisfied'`` and the requirement which couldn't be satisfied
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1217 by any distribution known to the locator.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1218 """
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1219
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1220 self.provided = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1221 self.dists = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1222 self.dists_by_name = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1223 self.reqts = {}
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1224
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1225 meta_extras = set(meta_extras or [])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1226 if ':*:' in meta_extras:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1227 meta_extras.remove(':*:')
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1228 # :meta: and :run: are implicitly included
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1229 meta_extras \|= set([':test:', ':build:', ':dev:'])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1230
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1231 if isinstance(requirement, Distribution):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1232 dist = odist = requirement
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1233 logger.debug('passed %s as requirement', odist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1234 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1235 dist = odist = self.locator.locate(requirement,
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1236 prereleases=prereleases)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1237 if dist is None:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1238 raise DistlibException('Unable to locate %r' % requirement)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1239 logger.debug('located %s', odist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1240 dist.requested = True
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1241 problems = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1242 todo = set([dist])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1243 install_dists = set([odist])
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1244 while todo:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1245 dist = todo.pop()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1246 name = dist.key # case-insensitive
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1247 if name not in self.dists_by_name:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1248 self.add_distribution(dist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1249 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1250 #import pdb; pdb.set_trace()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1251 other = self.dists_by_name[name]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1252 if other != dist:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1253 self.try_to_replace(dist, other, problems)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1254
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1255 ireqts = dist.run_requires \| dist.meta_requires
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1256 sreqts = dist.build_requires
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1257 ereqts = set()
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1258 if meta_extras and dist in install_dists:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1259 for key in ('test', 'build', 'dev'):
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1260 e = ':%s:' % key
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1261 if e in meta_extras:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1262 ereqts \|= getattr(dist, '%s_requires' % key)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1263 all_reqts = ireqts \| sreqts \| ereqts
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1264 for r in all_reqts:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1265 providers = self.find_providers(r)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1266 if not providers:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1267 logger.debug('No providers found for %r', r)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1268 provider = self.locator.locate(r, prereleases=prereleases)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1269 # If no provider is found and we didn't consider
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1270 # prereleases, consider them now.
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1271 if provider is None and not prereleases:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1272 provider = self.locator.locate(r, prereleases=True)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1273 if provider is None:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1274 logger.debug('Cannot satisfy %r', r)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1275 problems.add(('unsatisfied', r))
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1276 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1277 n, v = provider.key, provider.version
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1278 if (n, v) not in self.dists:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1279 todo.add(provider)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1280 providers.add(provider)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1281 if r in ireqts and dist in install_dists:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1282 install_dists.add(provider)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1283 logger.debug('Adding %s to install_dists',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1284 provider.name_and_version)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1285 for p in providers:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1286 name = p.key
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1287 if name not in self.dists_by_name:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1288 self.reqts.setdefault(p, set()).add(r)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1289 else:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1290 other = self.dists_by_name[name]
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1291 if other != p:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1292 # see if other can be replaced by p
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1293 self.try_to_replace(p, other, problems)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1294
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1295 dists = set(self.dists.values())
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1296 for dist in dists:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1297 dist.build_time_dependency = dist not in install_dists
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1298 if dist.build_time_dependency:
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1299 logger.debug('%s is a build-time dependency only.',
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1300 dist.name_and_version)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1301 logger.debug('find done for %s', odist)
d30785e31577 "planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1" guerler parents: diff changeset	1302 return dists, problems

Mercurial > repos > guerler > springsuite

annotate planemo/lib/python3.7/site-packages/distlib/locators.py @ 1:56ad4e20f292 draft