guppy_basecaller: env/lib/python3.7/site-packages/setuptools/package

annotate env/lib/python3.7/site-packages/setuptools/package_index.py @ 3:758bc20232e8 draft

"planemo upload commit 2a0fe2cc28b09e101d37293e53e82f61762262ec"

author	shellac
date	Thu, 14 May 2020 16:20:52 -0400
parents	26e78fe6e8c4
children

rev	line source
0 26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1 """PyPI and direct package downloading"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	2 import sys
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	3 import os
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	4 import re
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	5 import shutil
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	6 import socket
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	7 import base64
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	8 import hashlib
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	9 import itertools
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	10 import warnings
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	11 from functools import wraps
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	12
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	13 from setuptools.extern import six
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	14 from setuptools.extern.six.moves import urllib, http_client, configparser, map
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	15
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	16 import setuptools
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	17 from pkg_resources import (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	18 CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	19 Environment, find_distributions, safe_name, safe_version,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	20 to_filename, Requirement, DEVELOP_DIST, EGG_DIST,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	21 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	22 from setuptools import ssl_support
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	23 from distutils import log
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	24 from distutils.errors import DistutilsError
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	25 from fnmatch import translate
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	26 from setuptools.py27compat import get_all_headers
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	27 from setuptools.py33compat import unescape
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	28 from setuptools.wheel import Wheel
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	29
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	30 __metaclass__ = type
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	31
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	32 EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	33 HREF = re.compile(r"""href\s=\s['"]?([^'"> ]+)""", re.I)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	34 PYPI_MD5 = re.compile(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	35 r'<a href="([^"#]+)">([^<]+)</a>\n\s+\(<a (?:title="MD5 hash"\n\s+)'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	36 r'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\)'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	37 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	38 URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	39 EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	40
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	41 __all__ = [
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	42 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	43 'interpret_distro_name',
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	44 ]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	45
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	46 _SOCKET_TIMEOUT = 15
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	47
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	48 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	49 user_agent = _tmpl.format(py_major=sys.version[:3], setuptools=setuptools)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	50
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	51
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	52 def parse_requirement_arg(spec):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	53 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	54 return Requirement.parse(spec)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	55 except ValueError:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	56 raise DistutilsError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	57 "Not a URL, existing file, or requirement spec: %r" % (spec,)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	58 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	59
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	60
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	61 def parse_bdist_wininst(name):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	62 """Return (base,pyversion) or (None,None) for possible .exe name"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	63
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	64 lower = name.lower()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	65 base, py_ver, plat = None, None, None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	66
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	67 if lower.endswith('.exe'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	68 if lower.endswith('.win32.exe'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	69 base = name[:-10]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	70 plat = 'win32'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	71 elif lower.startswith('.win32-py', -16):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	72 py_ver = name[-7:-4]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	73 base = name[:-16]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	74 plat = 'win32'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	75 elif lower.endswith('.win-amd64.exe'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	76 base = name[:-14]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	77 plat = 'win-amd64'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	78 elif lower.startswith('.win-amd64-py', -20):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	79 py_ver = name[-7:-4]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	80 base = name[:-20]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	81 plat = 'win-amd64'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	82 return base, py_ver, plat
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	83
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	84
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	85 def egg_info_for_url(url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	86 parts = urllib.parse.urlparse(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	87 scheme, server, path, parameters, query, fragment = parts
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	88 base = urllib.parse.unquote(path.split('/')[-1])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	89 if server == 'sourceforge.net' and base == 'download': # XXX Yuck
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	90 base = urllib.parse.unquote(path.split('/')[-2])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	91 if '#' in base:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	92 base, fragment = base.split('#', 1)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	93 return base, fragment
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	94
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	95
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	96 def distros_for_url(url, metadata=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	97 """Yield egg or source distribution objects that might be found at a URL"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	98 base, fragment = egg_info_for_url(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	99 for dist in distros_for_location(url, base, metadata):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	100 yield dist
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	101 if fragment:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	102 match = EGG_FRAGMENT.match(fragment)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	103 if match:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	104 for dist in interpret_distro_name(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	105 url, match.group(1), metadata, precedence=CHECKOUT_DIST
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	106 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	107 yield dist
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	108
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	109
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	110 def distros_for_location(location, basename, metadata=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	111 """Yield egg or source distribution objects based on basename"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	112 if basename.endswith('.egg.zip'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	113 basename = basename[:-4] # strip the .zip
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	114 if basename.endswith('.egg') and '-' in basename:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	115 # only one, unambiguous interpretation
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	116 return [Distribution.from_location(location, basename, metadata)]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	117 if basename.endswith('.whl') and '-' in basename:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	118 wheel = Wheel(basename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	119 if not wheel.is_compatible():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	120 return []
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	121 return [Distribution(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	122 location=location,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	123 project_name=wheel.project_name,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	124 version=wheel.version,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	125 # Increase priority over eggs.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	126 precedence=EGG_DIST + 1,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	127 )]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	128 if basename.endswith('.exe'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	129 win_base, py_ver, platform = parse_bdist_wininst(basename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	130 if win_base is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	131 return interpret_distro_name(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	132 location, win_base, metadata, py_ver, BINARY_DIST, platform
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	133 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	134 # Try source distro extensions (.zip, .tgz, etc.)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	135 #
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	136 for ext in EXTENSIONS:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	137 if basename.endswith(ext):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	138 basename = basename[:-len(ext)]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	139 return interpret_distro_name(location, basename, metadata)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	140 return [] # no extension matched
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	141
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	142
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	143 def distros_for_filename(filename, metadata=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	144 """Yield possible egg or source distribution objects based on a filename"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	145 return distros_for_location(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	146 normalize_path(filename), os.path.basename(filename), metadata
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	147 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	148
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	149
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	150 def interpret_distro_name(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	151 location, basename, metadata, py_version=None, precedence=SOURCE_DIST,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	152 platform=None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	153 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	154 """Generate alternative interpretations of a source distro name
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	155
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	156 Note: if `location` is a filesystem filename, you should call
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	157 ``pkg_resources.normalize_path()`` on it before passing it to this
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	158 routine!
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	159 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	160 # Generate alternative interpretations of a source distro name
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	161 # Because some packages are ambiguous as to name/versions split
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	162 # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	163 # So, we generate each possible interepretation (e.g. "adns, python-1.1.0"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	164 # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	165 # the spurious interpretations should be ignored, because in the event
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	166 # there's also an "adns" package, the spurious "python-1.1.0" version will
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	167 # compare lower than any numeric version number, and is therefore unlikely
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	168 # to match a request for it. It's still a potential problem, though, and
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	169 # in the long run PyPI and the distutils should go for "safe" names and
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	170 # versions in distribution archive names (sdist and bdist).
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	171
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	172 parts = basename.split('-')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	173 if not py_version and any(re.match(r'py\d\.\d$', p) for p in parts[2:]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	174 # it is a bdist_dumb, not an sdist -- bail out
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	175 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	176
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	177 for p in range(1, len(parts) + 1):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	178 yield Distribution(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	179 location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	180 py_version=py_version, precedence=precedence,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	181 platform=platform
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	182 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	183
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	184
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	185 # From Python 2.7 docs
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	186 def unique_everseen(iterable, key=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	187 "List unique elements, preserving order. Remember all elements ever seen."
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	188 # unique_everseen('AAAABBBCCDAABBB') --> A B C D
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	189 # unique_everseen('ABBCcAD', str.lower) --> A B C D
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	190 seen = set()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	191 seen_add = seen.add
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	192 if key is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	193 for element in six.moves.filterfalse(seen.__contains__, iterable):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	194 seen_add(element)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	195 yield element
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	196 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	197 for element in iterable:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	198 k = key(element)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	199 if k not in seen:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	200 seen_add(k)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	201 yield element
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	202
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	203
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	204 def unique_values(func):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	205 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	206 Wrap a function returning an iterable such that the resulting iterable
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	207 only ever yields unique items.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	208 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	209
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	210 @wraps(func)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	211 def wrapper(args, *kwargs):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	212 return unique_everseen(func(args, *kwargs))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	213
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	214 return wrapper
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	215
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	216
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	217 REL = re.compile(r"""<([^>]\srel\s=\s['"]?([^'">]+)[^>])>""", re.I)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	218 # this line is here to fix emacs' cruddy broken syntax highlighting
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	219
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	220
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	221 @unique_values
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	222 def find_external_links(url, page):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	223 """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	224
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	225 for match in REL.finditer(page):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	226 tag, rel = match.groups()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	227 rels = set(map(str.strip, rel.lower().split(',')))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	228 if 'homepage' in rels or 'download' in rels:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	229 for match in HREF.finditer(tag):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	230 yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	231
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	232 for tag in ("<th>Home Page", "<th>Download URL"):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	233 pos = page.find(tag)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	234 if pos != -1:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	235 match = HREF.search(page, pos)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	236 if match:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	237 yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	238
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	239
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	240 class ContentChecker:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	241 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	242 A null content checker that defines the interface for checking content
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	243 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	244
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	245 def feed(self, block):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	246 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	247 Feed a block of data to the hash.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	248 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	249 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	250
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	251 def is_valid(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	252 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	253 Check the hash. Return False if validation fails.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	254 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	255 return True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	256
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	257 def report(self, reporter, template):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	258 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	259 Call reporter with information about the checker (hash name)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	260 substituted into the template.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	261 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	262 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	263
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	264
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	265 class HashChecker(ContentChecker):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	266 pattern = re.compile(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	267 r'(?P<hash_name>sha1\|sha224\|sha384\|sha256\|sha512\|md5)='
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	268 r'(?P<expected>[a-f0-9]+)'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	269 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	270
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	271 def __init__(self, hash_name, expected):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	272 self.hash_name = hash_name
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	273 self.hash = hashlib.new(hash_name)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	274 self.expected = expected
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	275
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	276 @classmethod
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	277 def from_url(cls, url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	278 "Construct a (possibly null) ContentChecker from a URL"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	279 fragment = urllib.parse.urlparse(url)[-1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	280 if not fragment:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	281 return ContentChecker()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	282 match = cls.pattern.search(fragment)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	283 if not match:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	284 return ContentChecker()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	285 return cls(**match.groupdict())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	286
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	287 def feed(self, block):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	288 self.hash.update(block)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	289
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	290 def is_valid(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	291 return self.hash.hexdigest() == self.expected
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	292
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	293 def report(self, reporter, template):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	294 msg = template % self.hash_name
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	295 return reporter(msg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	296
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	297
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	298 class PackageIndex(Environment):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	299 """A distribution index that scans web pages for download URLs"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	300
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	301 def __init__(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	302 self, index_url="https://pypi.org/simple/", hosts=('*',),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	303 ca_bundle=None, verify_ssl=True, args, *kw
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	304 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	305 Environment.__init__(self, args, *kw)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	306 self.index_url = index_url + "/" [:not index_url.endswith('/')]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	307 self.scanned_urls = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	308 self.fetched_urls = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	309 self.package_pages = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	310 self.allows = re.compile('\|'.join(map(translate, hosts))).match
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	311 self.to_scan = []
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	312 use_ssl = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	313 verify_ssl
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	314 and ssl_support.is_available
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	315 and (ca_bundle or ssl_support.find_ca_bundle())
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	316 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	317 if use_ssl:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	318 self.opener = ssl_support.opener_for(ca_bundle)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	319 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	320 self.opener = urllib.request.urlopen
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	321
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	322 def process_url(self, url, retrieve=False):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	323 """Evaluate a URL as a possible download, and maybe retrieve it"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	324 if url in self.scanned_urls and not retrieve:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	325 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	326 self.scanned_urls[url] = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	327 if not URL_SCHEME(url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	328 self.process_filename(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	329 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	330 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	331 dists = list(distros_for_url(url))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	332 if dists:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	333 if not self.url_ok(url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	334 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	335 self.debug("Found link: %s", url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	336
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	337 if dists or not retrieve or url in self.fetched_urls:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	338 list(map(self.add, dists))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	339 return # don't need the actual page
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	340
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	341 if not self.url_ok(url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	342 self.fetched_urls[url] = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	343 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	344
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	345 self.info("Reading %s", url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	346 self.fetched_urls[url] = True # prevent multiple fetch attempts
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	347 tmpl = "Download error on %s: %%s -- Some packages may not be found!"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	348 f = self.open_url(url, tmpl % url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	349 if f is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	350 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	351 self.fetched_urls[f.url] = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	352 if 'html' not in f.headers.get('content-type', '').lower():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	353 f.close() # not html, we can't process it
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	354 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	355
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	356 base = f.url # handle redirects
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	357 page = f.read()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	358 if not isinstance(page, str):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	359 # In Python 3 and got bytes but want str.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	360 if isinstance(f, urllib.error.HTTPError):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	361 # Errors have no charset, assume latin1:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	362 charset = 'latin-1'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	363 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	364 charset = f.headers.get_param('charset') or 'latin-1'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	365 page = page.decode(charset, "ignore")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	366 f.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	367 for match in HREF.finditer(page):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	368 link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	369 self.process_url(link)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	370 if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	371 page = self.process_index(url, page)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	372
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	373 def process_filename(self, fn, nested=False):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	374 # process filenames or directories
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	375 if not os.path.exists(fn):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	376 self.warn("Not found: %s", fn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	377 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	378
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	379 if os.path.isdir(fn) and not nested:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	380 path = os.path.realpath(fn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	381 for item in os.listdir(path):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	382 self.process_filename(os.path.join(path, item), True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	383
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	384 dists = distros_for_filename(fn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	385 if dists:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	386 self.debug("Found: %s", fn)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	387 list(map(self.add, dists))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	388
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	389 def url_ok(self, url, fatal=False):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	390 s = URL_SCHEME(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	391 is_file = s and s.group(1).lower() == 'file'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	392 if is_file or self.allows(urllib.parse.urlparse(url)[1]):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	393 return True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	394 msg = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	395 "\nNote: Bypassing %s (disallowed host; see "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	396 "http://bit.ly/2hrImnY for details).\n")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	397 if fatal:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	398 raise DistutilsError(msg % url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	399 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	400 self.warn(msg, url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	401
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	402 def scan_egg_links(self, search_path):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	403 dirs = filter(os.path.isdir, search_path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	404 egg_links = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	405 (path, entry)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	406 for path in dirs
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	407 for entry in os.listdir(path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	408 if entry.endswith('.egg-link')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	409 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	410 list(itertools.starmap(self.scan_egg_link, egg_links))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	411
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	412 def scan_egg_link(self, path, entry):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	413 with open(os.path.join(path, entry)) as raw_lines:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	414 # filter non-empty lines
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	415 lines = list(filter(None, map(str.strip, raw_lines)))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	416
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	417 if len(lines) != 2:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	418 # format is not recognized; punt
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	419 return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	420
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	421 egg_path, setup_path = lines
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	422
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	423 for dist in find_distributions(os.path.join(path, egg_path)):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	424 dist.location = os.path.join(path, *lines)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	425 dist.precedence = SOURCE_DIST
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	426 self.add(dist)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	427
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	428 def process_index(self, url, page):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	429 """Process the contents of a PyPI page"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	430
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	431 def scan(link):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	432 # Process a URL to see if it's for a package page
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	433 if link.startswith(self.index_url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	434 parts = list(map(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	435 urllib.parse.unquote, link[len(self.index_url):].split('/')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	436 ))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	437 if len(parts) == 2 and '#' not in parts[1]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	438 # it's a package page, sanitize and index it
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	439 pkg = safe_name(parts[0])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	440 ver = safe_version(parts[1])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	441 self.package_pages.setdefault(pkg.lower(), {})[link] = True
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	442 return to_filename(pkg), to_filename(ver)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	443 return None, None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	444
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	445 # process an index page into the package-page index
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	446 for match in HREF.finditer(page):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	447 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	448 scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	449 except ValueError:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	450 pass
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	451
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	452 pkg, ver = scan(url) # ensure this page is in the page index
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	453 if pkg:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	454 # process individual package page
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	455 for new_url in find_external_links(url, page):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	456 # Process the found URL
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	457 base, frag = egg_info_for_url(new_url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	458 if base.endswith('.py') and not frag:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	459 if ver:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	460 new_url += '#egg=%s-%s' % (pkg, ver)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	461 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	462 self.need_version_info(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	463 self.scan_url(new_url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	464
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	465 return PYPI_MD5.sub(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	466 lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	467 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	468 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	469 return "" # no sense double-scanning non-package pages
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	470
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	471 def need_version_info(self, url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	472 self.scan_all(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	473 "Page at %s links to .py file(s) without version info; an index "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	474 "scan is required.", url
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	475 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	476
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	477 def scan_all(self, msg=None, *args):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	478 if self.index_url not in self.fetched_urls:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	479 if msg:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	480 self.warn(msg, *args)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	481 self.info(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	482 "Scanning index of all packages (this may take a while)"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	483 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	484 self.scan_url(self.index_url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	485
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	486 def find_packages(self, requirement):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	487 self.scan_url(self.index_url + requirement.unsafe_name + '/')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	488
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	489 if not self.package_pages.get(requirement.key):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	490 # Fall back to safe version of the name
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	491 self.scan_url(self.index_url + requirement.project_name + '/')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	492
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	493 if not self.package_pages.get(requirement.key):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	494 # We couldn't find the target package, so search the index page too
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	495 self.not_found_in_index(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	496
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	497 for url in list(self.package_pages.get(requirement.key, ())):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	498 # scan each page that might be related to the desired package
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	499 self.scan_url(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	500
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	501 def obtain(self, requirement, installer=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	502 self.prescan()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	503 self.find_packages(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	504 for dist in self[requirement.key]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	505 if dist in requirement:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	506 return dist
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	507 self.debug("%s does not match %s", requirement, dist)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	508 return super(PackageIndex, self).obtain(requirement, installer)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	509
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	510 def check_hash(self, checker, filename, tfp):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	511 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	512 checker is a ContentChecker
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	513 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	514 checker.report(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	515 self.debug,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	516 "Validating %%s checksum for %s" % filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	517 if not checker.is_valid():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	518 tfp.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	519 os.unlink(filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	520 raise DistutilsError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	521 "%s validation failed for %s; "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	522 "possible download problem?"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	523 % (checker.hash.name, os.path.basename(filename))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	524 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	525
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	526 def add_find_links(self, urls):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	527 """Add `urls` to the list that will be prescanned for searches"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	528 for url in urls:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	529 if (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	530 self.to_scan is None # if we have already "gone online"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	531 or not URL_SCHEME(url) # or it's a local file/directory
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	532 or url.startswith('file:')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	533 or list(distros_for_url(url)) # or a direct package link
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	534 ):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	535 # then go ahead and process it now
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	536 self.scan_url(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	537 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	538 # otherwise, defer retrieval till later
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	539 self.to_scan.append(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	540
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	541 def prescan(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	542 """Scan urls scheduled for prescanning (e.g. --find-links)"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	543 if self.to_scan:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	544 list(map(self.scan_url, self.to_scan))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	545 self.to_scan = None # from now on, go ahead and process immediately
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	546
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	547 def not_found_in_index(self, requirement):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	548 if self[requirement.key]: # we've seen at least one distro
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	549 meth, msg = self.info, "Couldn't retrieve index page for %r"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	550 else: # no distros seen for this name, might be misspelled
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	551 meth, msg = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	552 self.warn,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	553 "Couldn't find index page for %r (maybe misspelled?)")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	554 meth(msg, requirement.unsafe_name)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	555 self.scan_all()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	556
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	557 def download(self, spec, tmpdir):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	558 """Locate and/or download `spec` to `tmpdir`, returning a local path
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	559
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	560 `spec` may be a ``Requirement`` object, or a string containing a URL,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	561 an existing local filename, or a project/version requirement spec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	562 (i.e. the string form of a ``Requirement`` object). If it is the URL
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	563 of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	564 that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	565 automatically created alongside the downloaded file.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	566
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	567 If `spec` is a ``Requirement`` object or a string containing a
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	568 project/version requirement spec, this method returns the location of
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	569 a matching distribution (possibly after downloading it to `tmpdir`).
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	570 If `spec` is a locally existing file or directory name, it is simply
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	571 returned unchanged. If `spec` is a URL, it is downloaded to a subpath
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	572 of `tmpdir`, and the local filename is returned. Various errors may be
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	573 raised if a problem occurs during downloading.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	574 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	575 if not isinstance(spec, Requirement):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	576 scheme = URL_SCHEME(spec)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	577 if scheme:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	578 # It's a url, download it to tmpdir
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	579 found = self._download_url(scheme.group(1), spec, tmpdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	580 base, fragment = egg_info_for_url(spec)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	581 if base.endswith('.py'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	582 found = self.gen_setup(found, fragment, tmpdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	583 return found
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	584 elif os.path.exists(spec):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	585 # Existing file or directory, just return it
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	586 return spec
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	587 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	588 spec = parse_requirement_arg(spec)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	589 return getattr(self.fetch_distribution(spec, tmpdir), 'location', None)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	590
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	591 def fetch_distribution(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	592 self, requirement, tmpdir, force_scan=False, source=False,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	593 develop_ok=False, local_index=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	594 """Obtain a distribution suitable for fulfilling `requirement`
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	595
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	596 `requirement` must be a ``pkg_resources.Requirement`` instance.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	597 If necessary, or if the `force_scan` flag is set, the requirement is
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	598 searched for in the (online) package index as well as the locally
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	599 installed packages. If a distribution matching `requirement` is found,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	600 the returned distribution's ``location`` is the value you would have
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	601 gotten from calling the ``download()`` method with the matching
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	602 distribution's URL or filename. If no matching distribution is found,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	603 ``None`` is returned.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	604
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	605 If the `source` flag is set, only source distributions and source
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	606 checkout links will be considered. Unless the `develop_ok` flag is
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	607 set, development and system eggs (i.e., those using the ``.egg-info``
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	608 format) will be ignored.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	609 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	610 # process a Requirement
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	611 self.info("Searching for %s", requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	612 skipped = {}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	613 dist = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	614
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	615 def find(req, env=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	616 if env is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	617 env = self
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	618 # Find a matching distribution; may be called more than once
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	619
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	620 for dist in env[req.key]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	621
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	622 if dist.precedence == DEVELOP_DIST and not develop_ok:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	623 if dist not in skipped:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	624 self.warn(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	625 "Skipping development or system egg: %s", dist,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	626 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	627 skipped[dist] = 1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	628 continue
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	629
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	630 test = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	631 dist in req
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	632 and (dist.precedence <= SOURCE_DIST or not source)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	633 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	634 if test:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	635 loc = self.download(dist.location, tmpdir)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	636 dist.download_location = loc
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	637 if os.path.exists(dist.download_location):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	638 return dist
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	639
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	640 if force_scan:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	641 self.prescan()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	642 self.find_packages(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	643 dist = find(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	644
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	645 if not dist and local_index is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	646 dist = find(requirement, local_index)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	647
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	648 if dist is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	649 if self.to_scan is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	650 self.prescan()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	651 dist = find(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	652
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	653 if dist is None and not force_scan:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	654 self.find_packages(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	655 dist = find(requirement)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	656
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	657 if dist is None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	658 self.warn(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	659 "No local packages or working download links found for %s%s",
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	660 (source and "a source distribution of " or ""),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	661 requirement,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	662 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	663 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	664 self.info("Best match: %s", dist)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	665 return dist.clone(location=dist.download_location)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	666
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	667 def fetch(self, requirement, tmpdir, force_scan=False, source=False):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	668 """Obtain a file suitable for fulfilling `requirement`
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	669
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	670 DEPRECATED; use the ``fetch_distribution()`` method now instead. For
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	671 backward compatibility, this routine is identical but returns the
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	672 ``location`` of the downloaded distribution instead of a distribution
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	673 object.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	674 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	675 dist = self.fetch_distribution(requirement, tmpdir, force_scan, source)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	676 if dist is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	677 return dist.location
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	678 return None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	679
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	680 def gen_setup(self, filename, fragment, tmpdir):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	681 match = EGG_FRAGMENT.match(fragment)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	682 dists = match and [
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	683 d for d in
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	684 interpret_distro_name(filename, match.group(1), None) if d.version
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	685 ] or []
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	686
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	687 if len(dists) == 1: # unambiguous ``#egg`` fragment
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	688 basename = os.path.basename(filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	689
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	690 # Make sure the file has been downloaded to the temp dir.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	691 if os.path.dirname(filename) != tmpdir:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	692 dst = os.path.join(tmpdir, basename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	693 from setuptools.command.easy_install import samefile
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	694 if not samefile(filename, dst):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	695 shutil.copy2(filename, dst)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	696 filename = dst
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	697
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	698 with open(os.path.join(tmpdir, 'setup.py'), 'w') as file:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	699 file.write(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	700 "from setuptools import setup\n"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	701 "setup(name=%r, version=%r, py_modules=[%r])\n"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	702 % (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	703 dists[0].project_name, dists[0].version,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	704 os.path.splitext(basename)[0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	705 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	706 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	707 return filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	708
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	709 elif match:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	710 raise DistutilsError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	711 "Can't unambiguously interpret project/version identifier %r; "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	712 "any dashes in the name or version should be escaped using "
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	713 "underscores. %r" % (fragment, dists)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	714 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	715 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	716 raise DistutilsError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	717 "Can't process plain .py files without an '#egg=name-version'"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	718 " suffix to enable automatic setup script generation."
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	719 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	720
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	721 dl_blocksize = 8192
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	722
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	723 def _download_to(self, url, filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	724 self.info("Downloading %s", url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	725 # Download the file
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	726 fp = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	727 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	728 checker = HashChecker.from_url(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	729 fp = self.open_url(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	730 if isinstance(fp, urllib.error.HTTPError):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	731 raise DistutilsError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	732 "Can't download %s: %s %s" % (url, fp.code, fp.msg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	733 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	734 headers = fp.info()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	735 blocknum = 0
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	736 bs = self.dl_blocksize
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	737 size = -1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	738 if "content-length" in headers:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	739 # Some servers return multiple Content-Length headers :(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	740 sizes = get_all_headers(headers, 'Content-Length')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	741 size = max(map(int, sizes))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	742 self.reporthook(url, filename, blocknum, bs, size)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	743 with open(filename, 'wb') as tfp:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	744 while True:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	745 block = fp.read(bs)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	746 if block:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	747 checker.feed(block)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	748 tfp.write(block)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	749 blocknum += 1
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	750 self.reporthook(url, filename, blocknum, bs, size)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	751 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	752 break
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	753 self.check_hash(checker, filename, tfp)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	754 return headers
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	755 finally:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	756 if fp:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	757 fp.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	758
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	759 def reporthook(self, url, filename, blocknum, blksize, size):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	760 pass # no-op
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	761
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	762 def open_url(self, url, warning=None):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	763 if url.startswith('file:'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	764 return local_open(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	765 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	766 return open_with_auth(url, self.opener)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	767 except (ValueError, http_client.InvalidURL) as v:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	768 msg = ' '.join([str(arg) for arg in v.args])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	769 if warning:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	770 self.warn(warning, msg)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	771 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	772 raise DistutilsError('%s %s' % (url, msg))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	773 except urllib.error.HTTPError as v:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	774 return v
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	775 except urllib.error.URLError as v:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	776 if warning:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	777 self.warn(warning, v.reason)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	778 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	779 raise DistutilsError("Download error for %s: %s"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	780 % (url, v.reason))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	781 except http_client.BadStatusLine as v:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	782 if warning:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	783 self.warn(warning, v.line)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	784 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	785 raise DistutilsError(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	786 '%s returned a bad status line. The server might be '
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	787 'down, %s' %
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	788 (url, v.line)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	789 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	790 except (http_client.HTTPException, socket.error) as v:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	791 if warning:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	792 self.warn(warning, v)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	793 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	794 raise DistutilsError("Download error for %s: %s"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	795 % (url, v))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	796
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	797 def _download_url(self, scheme, url, tmpdir):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	798 # Determine download filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	799 #
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	800 name, fragment = egg_info_for_url(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	801 if name:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	802 while '..' in name:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	803 name = name.replace('..', '.').replace('\\', '_')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	804 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	805 name = "__downloaded__" # default if URL has no path contents
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	806
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	807 if name.endswith('.egg.zip'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	808 name = name[:-4] # strip the extra .zip before download
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	809
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	810 filename = os.path.join(tmpdir, name)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	811
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	812 # Download the file
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	813 #
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	814 if scheme == 'svn' or scheme.startswith('svn+'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	815 return self._download_svn(url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	816 elif scheme == 'git' or scheme.startswith('git+'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	817 return self._download_git(url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	818 elif scheme.startswith('hg+'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	819 return self._download_hg(url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	820 elif scheme == 'file':
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	821 return urllib.request.url2pathname(urllib.parse.urlparse(url)[2])
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	822 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	823 self.url_ok(url, True) # raises error if not allowed
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	824 return self._attempt_download(url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	825
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	826 def scan_url(self, url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	827 self.process_url(url, True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	828
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	829 def _attempt_download(self, url, filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	830 headers = self._download_to(url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	831 if 'html' in headers.get('content-type', '').lower():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	832 return self._download_html(url, headers, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	833 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	834 return filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	835
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	836 def _download_html(self, url, headers, filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	837 file = open(filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	838 for line in file:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	839 if line.strip():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	840 # Check for a subversion index page
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	841 if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	842 # it's a subversion index page:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	843 file.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	844 os.unlink(filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	845 return self._download_svn(url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	846 break # not an index page
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	847 file.close()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	848 os.unlink(filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	849 raise DistutilsError("Unexpected HTML page found at " + url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	850
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	851 def _download_svn(self, url, filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	852 warnings.warn("SVN download support is deprecated", UserWarning)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	853 url = url.split('#', 1)[0] # remove any fragment for svn's sake
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	854 creds = ''
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	855 if url.lower().startswith('svn:') and '@' in url:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	856 scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	857 if not netloc and path.startswith('//') and '/' in path[2:]:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	858 netloc, path = path[2:].split('/', 1)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	859 auth, host = _splituser(netloc)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	860 if auth:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	861 if ':' in auth:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	862 user, pw = auth.split(':', 1)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	863 creds = " --username=%s --password=%s" % (user, pw)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	864 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	865 creds = " --username=" + auth
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	866 netloc = host
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	867 parts = scheme, netloc, url, p, q, f
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	868 url = urllib.parse.urlunparse(parts)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	869 self.info("Doing subversion checkout from %s to %s", url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	870 os.system("svn checkout%s -q %s %s" % (creds, url, filename))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	871 return filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	872
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	873 @staticmethod
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	874 def _vcs_split_rev_from_url(url, pop_prefix=False):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	875 scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	876
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	877 scheme = scheme.split('+', 1)[-1]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	878
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	879 # Some fragment identification fails
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	880 path = path.split('#', 1)[0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	881
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	882 rev = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	883 if '@' in path:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	884 path, rev = path.rsplit('@', 1)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	885
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	886 # Also, discard fragment
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	887 url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	888
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	889 return url, rev
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	890
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	891 def _download_git(self, url, filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	892 filename = filename.split('#', 1)[0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	893 url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	894
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	895 self.info("Doing git clone from %s to %s", url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	896 os.system("git clone --quiet %s %s" % (url, filename))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	897
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	898 if rev is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	899 self.info("Checking out %s", rev)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	900 os.system("git -C %s checkout --quiet %s" % (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	901 filename,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	902 rev,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	903 ))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	904
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	905 return filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	906
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	907 def _download_hg(self, url, filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	908 filename = filename.split('#', 1)[0]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	909 url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	910
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	911 self.info("Doing hg clone from %s to %s", url, filename)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	912 os.system("hg clone --quiet %s %s" % (url, filename))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	913
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	914 if rev is not None:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	915 self.info("Updating to %s", rev)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	916 os.system("hg --cwd %s up -C -r %s -q" % (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	917 filename,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	918 rev,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	919 ))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	920
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	921 return filename
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	922
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	923 def debug(self, msg, *args):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	924 log.debug(msg, *args)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	925
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	926 def info(self, msg, *args):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	927 log.info(msg, *args)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	928
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	929 def warn(self, msg, *args):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	930 log.warn(msg, *args)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	931
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	932
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	933 # This pattern matches a character entity reference (a decimal numeric
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	934 # references, a hexadecimal numeric reference, or a named reference).
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	935 entity_sub = re.compile(r'&(#(\d+\|x[\da-fA-F]+)\|[\w.:-]+);?').sub
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	936
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	937
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	938 def decode_entity(match):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	939 what = match.group(0)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	940 return unescape(what)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	941
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	942
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	943 def htmldecode(text):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	944 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	945 Decode HTML entities in the given text.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	946
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	947 >>> htmldecode(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	948 ... 'https://../package_name-0.1.2.tar.gz'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	949 ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	950 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	951 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	952 return entity_sub(decode_entity, text)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	953
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	954
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	955 def socket_timeout(timeout=15):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	956 def _socket_timeout(func):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	957 def _socket_timeout(args, *kwargs):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	958 old_timeout = socket.getdefaulttimeout()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	959 socket.setdefaulttimeout(timeout)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	960 try:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	961 return func(args, *kwargs)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	962 finally:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	963 socket.setdefaulttimeout(old_timeout)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	964
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	965 return _socket_timeout
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	966
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	967 return _socket_timeout
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	968
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	969
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	970 def _encode_auth(auth):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	971 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	972 A function compatible with Python 2.3-3.3 that will encode
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	973 auth from a URL suitable for an HTTP header.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	974 >>> str(_encode_auth('username%3Apassword'))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	975 'dXNlcm5hbWU6cGFzc3dvcmQ='
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	976
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	977 Long auth strings should not cause a newline to be inserted.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	978 >>> long_auth = 'username:' + 'password'*10
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	979 >>> chr(10) in str(_encode_auth(long_auth))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	980 False
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	981 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	982 auth_s = urllib.parse.unquote(auth)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	983 # convert to bytes
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	984 auth_bytes = auth_s.encode()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	985 encoded_bytes = base64.b64encode(auth_bytes)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	986 # convert back to a string
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	987 encoded = encoded_bytes.decode()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	988 # strip the trailing carriage return
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	989 return encoded.replace('\n', '')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	990
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	991
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	992 class Credential:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	993 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	994 A username/password pair. Use like a namedtuple.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	995 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	996
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	997 def __init__(self, username, password):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	998 self.username = username
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	999 self.password = password
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1000
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1001 def __iter__(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1002 yield self.username
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1003 yield self.password
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1004
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1005 def __str__(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1006 return '%(username)s:%(password)s' % vars(self)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1007
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1008
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1009 class PyPIConfig(configparser.RawConfigParser):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1010 def __init__(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1011 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1012 Load from ~/.pypirc
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1013 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1014 defaults = dict.fromkeys(['username', 'password', 'repository'], '')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1015 configparser.RawConfigParser.__init__(self, defaults)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1016
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1017 rc = os.path.join(os.path.expanduser('~'), '.pypirc')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1018 if os.path.exists(rc):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1019 self.read(rc)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1020
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1021 @property
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1022 def creds_by_repository(self):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1023 sections_with_repositories = [
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1024 section for section in self.sections()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1025 if self.get(section, 'repository').strip()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1026 ]
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1027
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1028 return dict(map(self._get_repo_cred, sections_with_repositories))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1029
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1030 def _get_repo_cred(self, section):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1031 repo = self.get(section, 'repository').strip()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1032 return repo, Credential(
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1033 self.get(section, 'username').strip(),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1034 self.get(section, 'password').strip(),
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1035 )
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1036
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1037 def find_credential(self, url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1038 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1039 If the URL indicated appears to be a repository defined in this
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1040 config, return the credential for that repository.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1041 """
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1042 for repository, cred in self.creds_by_repository.items():
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1043 if url.startswith(repository):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1044 return cred
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1045
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1046
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1047 def open_with_auth(url, opener=urllib.request.urlopen):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1048 """Open a urllib2 request, handling HTTP authentication"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1049
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1050 parsed = urllib.parse.urlparse(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1051 scheme, netloc, path, params, query, frag = parsed
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1052
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1053 # Double scheme does not raise on Mac OS X as revealed by a
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1054 # failing test. We would expect "nonnumeric port". Refs #20.
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1055 if netloc.endswith(':'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1056 raise http_client.InvalidURL("nonnumeric port: ''")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1057
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1058 if scheme in ('http', 'https'):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1059 auth, address = _splituser(netloc)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1060 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1061 auth = None
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1062
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1063 if not auth:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1064 cred = PyPIConfig().find_credential(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1065 if cred:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1066 auth = str(cred)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1067 info = cred.username, url
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1068 log.info('Authenticating as %s for %s (from .pypirc)', *info)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1069
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1070 if auth:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1071 auth = "Basic " + _encode_auth(auth)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1072 parts = scheme, address, path, params, query, frag
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1073 new_url = urllib.parse.urlunparse(parts)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1074 request = urllib.request.Request(new_url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1075 request.add_header("Authorization", auth)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1076 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1077 request = urllib.request.Request(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1078
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1079 request.add_header('User-Agent', user_agent)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1080 fp = opener(request)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1081
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1082 if auth:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1083 # Put authentication info back into request URL if same host,
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1084 # so that links found on the page will work
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1085 s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1086 if s2 == scheme and h2 == address:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1087 parts = s2, netloc, path2, param2, query2, frag2
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1088 fp.url = urllib.parse.urlunparse(parts)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1089
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1090 return fp
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1091
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1092
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1093 # copy of urllib.parse._splituser from Python 3.8
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1094 def _splituser(host):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1095 """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1096 user, delim, host = host.rpartition('@')
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1097 return (user if delim else None), host
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1098
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1099
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1100 # adding a timeout to avoid freezing package_index
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1101 open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1102
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1103
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1104 def fix_sf_url(url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1105 return url # backward compatibility
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1106
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1107
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1108 def local_open(url):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1109 """Read a local path, with special support for directories"""
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1110 scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1111 filename = urllib.request.url2pathname(path)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1112 if os.path.isfile(filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1113 return urllib.request.urlopen(url)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1114 elif path.endswith('/') and os.path.isdir(filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1115 files = []
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1116 for f in os.listdir(filename):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1117 filepath = os.path.join(filename, f)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1118 if f == 'index.html':
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1119 with open(filepath, 'r') as fp:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1120 body = fp.read()
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1121 break
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1122 elif os.path.isdir(filepath):
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1123 f += '/'
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1124 files.append('<a href="{name}">{name}</a>'.format(name=f))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1125 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1126 tmpl = (
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1127 "<html><head><title>{url}</title>"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1128 "</head><body>{files}</body></html>")
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1129 body = tmpl.format(url=url, files='\n'.join(files))
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1130 status, message = 200, "OK"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1131 else:
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1132 status, message, body = 404, "Path not found", "Not found"
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1133
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1134 headers = {'content-type': 'text/html'}
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1135 body_stream = six.StringIO(body)
26e78fe6e8c4 "planemo upload commit c699937486c35866861690329de38ec1a5d9f783" shellac parents: diff changeset	1136 return urllib.error.HTTPError(url, status, message, headers, body_stream)

Mercurial > repos > shellac > guppy_basecaller

annotate env/lib/python3.7/site-packages/setuptools/package_index.py @ 3:758bc20232e8 draft