Mercurial > repos > guerler > hhblits

comparison lib/python3.8/site-packages/pip/_vendor/urllib3/poolmanager.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:9e54283cc701
1 from __future__ import absolute_import
2 import collections
3 import functools
4 import logging
5
6 from ._collections import RecentlyUsedContainer
7 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
8 from .connectionpool import port_by_scheme
9 from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown
10 from .packages import six
11 from .packages.six.moves.urllib.parse import urljoin
12 from .request import RequestMethods
13 from .util.url import parse_url
14 from .util.retry import Retry
15
16
17 __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
18
19
20 log = logging.getLogger(__name__)
21
22 SSL_KEYWORDS = (
23 "key_file",
24 "cert_file",
25 "cert_reqs",
26 "ca_certs",
27 "ssl_version",
28 "ca_cert_dir",
29 "ssl_context",
30 "key_password",
31 )
32
33 # All known keyword arguments that could be provided to the pool manager, its
34 # pools, or the underlying connections. This is used to construct a pool key.
35 _key_fields = (
36 "key_scheme", # str
37 "key_host", # str
38 "key_port", # int
39 "key_timeout", # int or float or Timeout
40 "key_retries", # int or Retry
41 "key_strict", # bool
42 "key_block", # bool
43 "key_source_address", # str
44 "key_key_file", # str
45 "key_key_password", # str
46 "key_cert_file", # str
47 "key_cert_reqs", # str
48 "key_ca_certs", # str
49 "key_ssl_version", # str
50 "key_ca_cert_dir", # str
51 "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
52 "key_maxsize", # int
53 "key_headers", # dict
54 "key__proxy", # parsed proxy url
55 "key__proxy_headers", # dict
56 "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
57 "key__socks_options", # dict
58 "key_assert_hostname", # bool or string
59 "key_assert_fingerprint", # str
60 "key_server_hostname", # str
61 )
62
63 #: The namedtuple class used to construct keys for the connection pool.
64 #: All custom key schemes should include the fields in this key at a minimum.
65 PoolKey = collections.namedtuple("PoolKey", _key_fields)
66
67
68 def _default_key_normalizer(key_class, request_context):
69 """
70 Create a pool key out of a request context dictionary.
71
72 According to RFC 3986, both the scheme and host are case-insensitive.
73 Therefore, this function normalizes both before constructing the pool
74 key for an HTTPS request. If you wish to change this behaviour, provide
75 alternate callables to ``key_fn_by_scheme``.
76
77 :param key_class:
78 The class to use when constructing the key. This should be a namedtuple
79 with the ``scheme`` and ``host`` keys at a minimum.
80 :type key_class: namedtuple
81 :param request_context:
82 A dictionary-like object that contain the context for a request.
83 :type request_context: dict
84
85 :return: A namedtuple that can be used as a connection pool key.
86 :rtype: PoolKey
87 """
88 # Since we mutate the dictionary, make a copy first
89 context = request_context.copy()
90 context["scheme"] = context["scheme"].lower()
91 context["host"] = context["host"].lower()
92
93 # These are both dictionaries and need to be transformed into frozensets
94 for key in ("headers", "_proxy_headers", "_socks_options"):
95 if key in context and context[key] is not None:
96 context[key] = frozenset(context[key].items())
97
98 # The socket_options key may be a list and needs to be transformed into a
99 # tuple.
100 socket_opts = context.get("socket_options")
101 if socket_opts is not None:
102 context["socket_options"] = tuple(socket_opts)
103
104 # Map the kwargs to the names in the namedtuple - this is necessary since
105 # namedtuples can't have fields starting with '_'.
106 for key in list(context.keys()):
107 context["key_" + key] = context.pop(key)
108
109 # Default to ``None`` for keys missing from the context
110 for field in key_class._fields:
111 if field not in context:
112 context[field] = None
113
114 return key_class(**context)
115
116
117 #: A dictionary that maps a scheme to a callable that creates a pool key.
118 #: This can be used to alter the way pool keys are constructed, if desired.
119 #: Each PoolManager makes a copy of this dictionary so they can be configured
120 #: globally here, or individually on the instance.
121 key_fn_by_scheme = {
122 "http": functools.partial(_default_key_normalizer, PoolKey),
123 "https": functools.partial(_default_key_normalizer, PoolKey),
124 }
125
126 pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
127
128
129 class PoolManager(RequestMethods):
130 """
131 Allows for arbitrary requests while transparently keeping track of
132 necessary connection pools for you.
133
134 :param num_pools:
135 Number of connection pools to cache before discarding the least
136 recently used pool.
137
138 :param headers:
139 Headers to include with all requests, unless other headers are given
140 explicitly.
141
142 :param \\**connection_pool_kw:
143 Additional parameters are used to create fresh
144 :class:`urllib3.connectionpool.ConnectionPool` instances.
145
146 Example::
147
148 >>> manager = PoolManager(num_pools=2)
149 >>> r = manager.request('GET', 'http://google.com/')
150 >>> r = manager.request('GET', 'http://google.com/mail')
151 >>> r = manager.request('GET', 'http://yahoo.com/')
152 >>> len(manager.pools)
153 2
154
155 """
156
157 proxy = None
158
159 def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
160 RequestMethods.__init__(self, headers)
161 self.connection_pool_kw = connection_pool_kw
162 self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close())
163
164 # Locally set the pool classes and keys so other PoolManagers can
165 # override them.
166 self.pool_classes_by_scheme = pool_classes_by_scheme
167 self.key_fn_by_scheme = key_fn_by_scheme.copy()
168
169 def __enter__(self):
170 return self
171
172 def __exit__(self, exc_type, exc_val, exc_tb):
173 self.clear()
174 # Return False to re-raise any potential exceptions
175 return False
176
177 def _new_pool(self, scheme, host, port, request_context=None):
178 """
179 Create a new :class:`ConnectionPool` based on host, port, scheme, and
180 any additional pool keyword arguments.
181
182 If ``request_context`` is provided, it is provided as keyword arguments
183 to the pool class used. This method is used to actually create the
184 connection pools handed out by :meth:`connection_from_url` and
185 companion methods. It is intended to be overridden for customization.
186 """
187 pool_cls = self.pool_classes_by_scheme[scheme]
188 if request_context is None:
189 request_context = self.connection_pool_kw.copy()
190
191 # Although the context has everything necessary to create the pool,
192 # this function has historically only used the scheme, host, and port
193 # in the positional args. When an API change is acceptable these can
194 # be removed.
195 for key in ("scheme", "host", "port"):
196 request_context.pop(key, None)
197
198 if scheme == "http":
199 for kw in SSL_KEYWORDS:
200 request_context.pop(kw, None)
201
202 return pool_cls(host, port, **request_context)
203
204 def clear(self):
205 """
206 Empty our store of pools and direct them all to close.
207
208 This will not affect in-flight connections, but they will not be
209 re-used after completion.
210 """
211 self.pools.clear()
212
213 def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
214 """
215 Get a :class:`ConnectionPool` based on the host, port, and scheme.
216
217 If ``port`` isn't given, it will be derived from the ``scheme`` using
218 ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
219 provided, it is merged with the instance's ``connection_pool_kw``
220 variable and used to create the new connection pool, if one is
221 needed.
222 """
223
224 if not host:
225 raise LocationValueError("No host specified.")
226
227 request_context = self._merge_pool_kwargs(pool_kwargs)
228 request_context["scheme"] = scheme or "http"
229 if not port:
230 port = port_by_scheme.get(request_context["scheme"].lower(), 80)
231 request_context["port"] = port
232 request_context["host"] = host
233
234 return self.connection_from_context(request_context)
235
236 def connection_from_context(self, request_context):
237 """
238 Get a :class:`ConnectionPool` based on the request context.
239
240 ``request_context`` must at least contain the ``scheme`` key and its
241 value must be a key in ``key_fn_by_scheme`` instance variable.
242 """
243 scheme = request_context["scheme"].lower()
244 pool_key_constructor = self.key_fn_by_scheme[scheme]
245 pool_key = pool_key_constructor(request_context)
246
247 return self.connection_from_pool_key(pool_key, request_context=request_context)
248
249 def connection_from_pool_key(self, pool_key, request_context=None):
250 """
251 Get a :class:`ConnectionPool` based on the provided pool key.
252
253 ``pool_key`` should be a namedtuple that only contains immutable
254 objects. At a minimum it must have the ``scheme``, ``host``, and
255 ``port`` fields.
256 """
257 with self.pools.lock:
258 # If the scheme, host, or port doesn't match existing open
259 # connections, open a new ConnectionPool.
260 pool = self.pools.get(pool_key)
261 if pool:
262 return pool
263
264 # Make a fresh ConnectionPool of the desired type
265 scheme = request_context["scheme"]
266 host = request_context["host"]
267 port = request_context["port"]
268 pool = self._new_pool(scheme, host, port, request_context=request_context)
269 self.pools[pool_key] = pool
270
271 return pool
272
273 def connection_from_url(self, url, pool_kwargs=None):
274 """
275 Similar to :func:`urllib3.connectionpool.connection_from_url`.
276
277 If ``pool_kwargs`` is not provided and a new pool needs to be
278 constructed, ``self.connection_pool_kw`` is used to initialize
279 the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
280 is provided, it is used instead. Note that if a new pool does not
281 need to be created for the request, the provided ``pool_kwargs`` are
282 not used.
283 """
284 u = parse_url(url)
285 return self.connection_from_host(
286 u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
287 )
288
289 def _merge_pool_kwargs(self, override):
290 """
291 Merge a dictionary of override values for self.connection_pool_kw.
292
293 This does not modify self.connection_pool_kw and returns a new dict.
294 Any keys in the override dictionary with a value of ``None`` are
295 removed from the merged dictionary.
296 """
297 base_pool_kwargs = self.connection_pool_kw.copy()
298 if override:
299 for key, value in override.items():
300 if value is None:
301 try:
302 del base_pool_kwargs[key]
303 except KeyError:
304 pass
305 else:
306 base_pool_kwargs[key] = value
307 return base_pool_kwargs
308
309 def urlopen(self, method, url, redirect=True, **kw):
310 """
311 Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
312 with custom cross-host redirect logic and only sends the request-uri
313 portion of the ``url``.
314
315 The given ``url`` parameter must be absolute, such that an appropriate
316 :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
317 """
318 u = parse_url(url)
319 conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
320
321 kw["assert_same_host"] = False
322 kw["redirect"] = False
323
324 if "headers" not in kw:
325 kw["headers"] = self.headers.copy()
326
327 if self.proxy is not None and u.scheme == "http":
328 response = conn.urlopen(method, url, **kw)
329 else:
330 response = conn.urlopen(method, u.request_uri, **kw)
331
332 redirect_location = redirect and response.get_redirect_location()
333 if not redirect_location:
334 return response
335
336 # Support relative URLs for redirecting.
337 redirect_location = urljoin(url, redirect_location)
338
339 # RFC 7231, Section 6.4.4
340 if response.status == 303:
341 method = "GET"
342
343 retries = kw.get("retries")
344 if not isinstance(retries, Retry):
345 retries = Retry.from_int(retries, redirect=redirect)
346
347 # Strip headers marked as unsafe to forward to the redirected location.
348 # Check remove_headers_on_redirect to avoid a potential network call within
349 # conn.is_same_host() which may use socket.gethostbyname() in the future.
350 if retries.remove_headers_on_redirect and not conn.is_same_host(
351 redirect_location
352 ):
353 headers = list(six.iterkeys(kw["headers"]))
354 for header in headers:
355 if header.lower() in retries.remove_headers_on_redirect:
356 kw["headers"].pop(header, None)
357
358 try:
359 retries = retries.increment(method, url, response=response, _pool=conn)
360 except MaxRetryError:
361 if retries.raise_on_redirect:
362 raise
363 return response
364
365 kw["retries"] = retries
366 kw["redirect"] = redirect
367
368 log.info("Redirecting %s -> %s", url, redirect_location)
369 return self.urlopen(method, redirect_location, **kw)
370
371
372 class ProxyManager(PoolManager):
373 """
374 Behaves just like :class:`PoolManager`, but sends all requests through
375 the defined proxy, using the CONNECT method for HTTPS URLs.
376
377 :param proxy_url:
378 The URL of the proxy to be used.
379
380 :param proxy_headers:
381 A dictionary containing headers that will be sent to the proxy. In case
382 of HTTP they are being sent with each request, while in the
383 HTTPS/CONNECT case they are sent only once. Could be used for proxy
384 authentication.
385
386 Example:
387 >>> proxy = urllib3.ProxyManager('http://localhost:3128/')
388 >>> r1 = proxy.request('GET', 'http://google.com/')
389 >>> r2 = proxy.request('GET', 'http://httpbin.org/')
390 >>> len(proxy.pools)
391 1
392 >>> r3 = proxy.request('GET', 'https://httpbin.org/')
393 >>> r4 = proxy.request('GET', 'https://twitter.com/')
394 >>> len(proxy.pools)
395 3
396
397 """
398
399 def __init__(
400 self,
401 proxy_url,
402 num_pools=10,
403 headers=None,
404 proxy_headers=None,
405 **connection_pool_kw
406 ):
407
408 if isinstance(proxy_url, HTTPConnectionPool):
409 proxy_url = "%s://%s:%i" % (
410 proxy_url.scheme,
411 proxy_url.host,
412 proxy_url.port,
413 )
414 proxy = parse_url(proxy_url)
415 if not proxy.port:
416 port = port_by_scheme.get(proxy.scheme, 80)
417 proxy = proxy._replace(port=port)
418
419 if proxy.scheme not in ("http", "https"):
420 raise ProxySchemeUnknown(proxy.scheme)
421
422 self.proxy = proxy
423 self.proxy_headers = proxy_headers or {}
424
425 connection_pool_kw["_proxy"] = self.proxy
426 connection_pool_kw["_proxy_headers"] = self.proxy_headers
427
428 super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
429
430 def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
431 if scheme == "https":
432 return super(ProxyManager, self).connection_from_host(
433 host, port, scheme, pool_kwargs=pool_kwargs
434 )
435
436 return super(ProxyManager, self).connection_from_host(
437 self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
438 )
439
440 def _set_proxy_headers(self, url, headers=None):
441 """
442 Sets headers needed by proxies: specifically, the Accept and Host
443 headers. Only sets headers not provided by the user.
444 """
445 headers_ = {"Accept": "*/*"}
446
447 netloc = parse_url(url).netloc
448 if netloc:
449 headers_["Host"] = netloc
450
451 if headers:
452 headers_.update(headers)
453 return headers_
454
455 def urlopen(self, method, url, redirect=True, **kw):
456 "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
457 u = parse_url(url)
458
459 if u.scheme == "http":
460 # For proxied HTTPS requests, httplib sets the necessary headers
461 # on the CONNECT to the proxy. For HTTP, we'll definitely
462 # need to set 'Host' at the very least.
463 headers = kw.get("headers", self.headers)
464 kw["headers"] = self._set_proxy_headers(url, headers)
465
466 return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
467
468
469 def proxy_from_url(url, **kw):
470 return ProxyManager(proxy_url=url, **kw)