Mercurial > repos > guerler > hhblits

comparison lib/python3.8/site-packages/pip/_vendor/urllib3/connectionpool.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:9e54283cc701
1 from __future__ import absolute_import
2 import errno
3 import logging
4 import sys
5 import warnings
6
7 from socket import error as SocketError, timeout as SocketTimeout
8 import socket
9
10
11 from .exceptions import (
12 ClosedPoolError,
13 ProtocolError,
14 EmptyPoolError,
15 HeaderParsingError,
16 HostChangedError,
17 LocationValueError,
18 MaxRetryError,
19 ProxyError,
20 ReadTimeoutError,
21 SSLError,
22 TimeoutError,
23 InsecureRequestWarning,
24 NewConnectionError,
25 )
26 from .packages.ssl_match_hostname import CertificateError
27 from .packages import six
28 from .packages.six.moves import queue
29 from .connection import (
30 port_by_scheme,
31 DummyConnection,
32 HTTPConnection,
33 HTTPSConnection,
34 VerifiedHTTPSConnection,
35 HTTPException,
36 BaseSSLError,
37 )
38 from .request import RequestMethods
39 from .response import HTTPResponse
40
41 from .util.connection import is_connection_dropped
42 from .util.request import set_file_position
43 from .util.response import assert_header_parsing
44 from .util.retry import Retry
45 from .util.timeout import Timeout
46 from .util.url import (
47 get_host,
48 parse_url,
49 Url,
50 _normalize_host as normalize_host,
51 _encode_target,
52 )
53 from .util.queue import LifoQueue
54
55
56 xrange = six.moves.xrange
57
58 log = logging.getLogger(__name__)
59
60 _Default = object()
61
62
63 # Pool objects
64 class ConnectionPool(object):
65 """
66 Base class for all connection pools, such as
67 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
68 """
69
70 scheme = None
71 QueueCls = LifoQueue
72
73 def __init__(self, host, port=None):
74 if not host:
75 raise LocationValueError("No host specified.")
76
77 self.host = _normalize_host(host, scheme=self.scheme)
78 self._proxy_host = host.lower()
79 self.port = port
80
81 def __str__(self):
82 return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)
83
84 def __enter__(self):
85 return self
86
87 def __exit__(self, exc_type, exc_val, exc_tb):
88 self.close()
89 # Return False to re-raise any potential exceptions
90 return False
91
92 def close(self):
93 """
94 Close all pooled connections and disable the pool.
95 """
96 pass
97
98
99 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
100 _blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
101
102
103 class HTTPConnectionPool(ConnectionPool, RequestMethods):
104 """
105 Thread-safe connection pool for one host.
106
107 :param host:
108 Host used for this HTTP Connection (e.g. "localhost"), passed into
109 :class:`httplib.HTTPConnection`.
110
111 :param port:
112 Port used for this HTTP Connection (None is equivalent to 80), passed
113 into :class:`httplib.HTTPConnection`.
114
115 :param strict:
116 Causes BadStatusLine to be raised if the status line can't be parsed
117 as a valid HTTP/1.0 or 1.1 status line, passed into
118 :class:`httplib.HTTPConnection`.
119
120 .. note::
121 Only works in Python 2. This parameter is ignored in Python 3.
122
123 :param timeout:
124 Socket timeout in seconds for each individual connection. This can
125 be a float or integer, which sets the timeout for the HTTP request,
126 or an instance of :class:`urllib3.util.Timeout` which gives you more
127 fine-grained control over request timeouts. After the constructor has
128 been parsed, this is always a `urllib3.util.Timeout` object.
129
130 :param maxsize:
131 Number of connections to save that can be reused. More than 1 is useful
132 in multithreaded situations. If ``block`` is set to False, more
133 connections will be created but they will not be saved once they've
134 been used.
135
136 :param block:
137 If set to True, no more than ``maxsize`` connections will be used at
138 a time. When no free connections are available, the call will block
139 until a connection has been released. This is a useful side effect for
140 particular multithreaded situations where one does not want to use more
141 than maxsize connections per host to prevent flooding.
142
143 :param headers:
144 Headers to include with all requests, unless other headers are given
145 explicitly.
146
147 :param retries:
148 Retry configuration to use by default with requests in this pool.
149
150 :param _proxy:
151 Parsed proxy URL, should not be used directly, instead, see
152 :class:`urllib3.connectionpool.ProxyManager`"
153
154 :param _proxy_headers:
155 A dictionary with proxy headers, should not be used directly,
156 instead, see :class:`urllib3.connectionpool.ProxyManager`"
157
158 :param \\**conn_kw:
159 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
160 :class:`urllib3.connection.HTTPSConnection` instances.
161 """
162
163 scheme = "http"
164 ConnectionCls = HTTPConnection
165 ResponseCls = HTTPResponse
166
167 def __init__(
168 self,
169 host,
170 port=None,
171 strict=False,
172 timeout=Timeout.DEFAULT_TIMEOUT,
173 maxsize=1,
174 block=False,
175 headers=None,
176 retries=None,
177 _proxy=None,
178 _proxy_headers=None,
179 **conn_kw
180 ):
181 ConnectionPool.__init__(self, host, port)
182 RequestMethods.__init__(self, headers)
183
184 self.strict = strict
185
186 if not isinstance(timeout, Timeout):
187 timeout = Timeout.from_float(timeout)
188
189 if retries is None:
190 retries = Retry.DEFAULT
191
192 self.timeout = timeout
193 self.retries = retries
194
195 self.pool = self.QueueCls(maxsize)
196 self.block = block
197
198 self.proxy = _proxy
199 self.proxy_headers = _proxy_headers or {}
200
201 # Fill the queue up so that doing get() on it will block properly
202 for _ in xrange(maxsize):
203 self.pool.put(None)
204
205 # These are mostly for testing and debugging purposes.
206 self.num_connections = 0
207 self.num_requests = 0
208 self.conn_kw = conn_kw
209
210 if self.proxy:
211 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
212 # We cannot know if the user has added default socket options, so we cannot replace the
213 # list.
214 self.conn_kw.setdefault("socket_options", [])
215
216 def _new_conn(self):
217 """
218 Return a fresh :class:`HTTPConnection`.
219 """
220 self.num_connections += 1
221 log.debug(
222 "Starting new HTTP connection (%d): %s:%s",
223 self.num_connections,
224 self.host,
225 self.port or "80",
226 )
227
228 conn = self.ConnectionCls(
229 host=self.host,
230 port=self.port,
231 timeout=self.timeout.connect_timeout,
232 strict=self.strict,
233 **self.conn_kw
234 )
235 return conn
236
237 def _get_conn(self, timeout=None):
238 """
239 Get a connection. Will return a pooled connection if one is available.
240
241 If no connections are available and :prop:`.block` is ``False``, then a
242 fresh connection is returned.
243
244 :param timeout:
245 Seconds to wait before giving up and raising
246 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
247 :prop:`.block` is ``True``.
248 """
249 conn = None
250 try:
251 conn = self.pool.get(block=self.block, timeout=timeout)
252
253 except AttributeError: # self.pool is None
254 raise ClosedPoolError(self, "Pool is closed.")
255
256 except queue.Empty:
257 if self.block:
258 raise EmptyPoolError(
259 self,
260 "Pool reached maximum size and no more connections are allowed.",
261 )
262 pass # Oh well, we'll create a new connection then
263
264 # If this is a persistent connection, check if it got disconnected
265 if conn and is_connection_dropped(conn):
266 log.debug("Resetting dropped connection: %s", self.host)
267 conn.close()
268 if getattr(conn, "auto_open", 1) == 0:
269 # This is a proxied connection that has been mutated by
270 # httplib._tunnel() and cannot be reused (since it would
271 # attempt to bypass the proxy)
272 conn = None
273
274 return conn or self._new_conn()
275
276 def _put_conn(self, conn):
277 """
278 Put a connection back into the pool.
279
280 :param conn:
281 Connection object for the current host and port as returned by
282 :meth:`._new_conn` or :meth:`._get_conn`.
283
284 If the pool is already full, the connection is closed and discarded
285 because we exceeded maxsize. If connections are discarded frequently,
286 then maxsize should be increased.
287
288 If the pool is closed, then the connection will be closed and discarded.
289 """
290 try:
291 self.pool.put(conn, block=False)
292 return # Everything is dandy, done.
293 except AttributeError:
294 # self.pool is None.
295 pass
296 except queue.Full:
297 # This should never happen if self.block == True
298 log.warning("Connection pool is full, discarding connection: %s", self.host)
299
300 # Connection never got put back into the pool, close it.
301 if conn:
302 conn.close()
303
304 def _validate_conn(self, conn):
305 """
306 Called right before a request is made, after the socket is created.
307 """
308 pass
309
310 def _prepare_proxy(self, conn):
311 # Nothing to do for HTTP connections.
312 pass
313
314 def _get_timeout(self, timeout):
315 """ Helper that always returns a :class:`urllib3.util.Timeout` """
316 if timeout is _Default:
317 return self.timeout.clone()
318
319 if isinstance(timeout, Timeout):
320 return timeout.clone()
321 else:
322 # User passed us an int/float. This is for backwards compatibility,
323 # can be removed later
324 return Timeout.from_float(timeout)
325
326 def _raise_timeout(self, err, url, timeout_value):
327 """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
328
329 if isinstance(err, SocketTimeout):
330 raise ReadTimeoutError(
331 self, url, "Read timed out. (read timeout=%s)" % timeout_value
332 )
333
334 # See the above comment about EAGAIN in Python 3. In Python 2 we have
335 # to specifically catch it and throw the timeout error
336 if hasattr(err, "errno") and err.errno in _blocking_errnos:
337 raise ReadTimeoutError(
338 self, url, "Read timed out. (read timeout=%s)" % timeout_value
339 )
340
341 # Catch possible read timeouts thrown as SSL errors. If not the
342 # case, rethrow the original. We need to do this because of:
343 # http://bugs.python.org/issue10272
344 if "timed out" in str(err) or "did not complete (read)" in str(
345 err
346 ): # Python < 2.7.4
347 raise ReadTimeoutError(
348 self, url, "Read timed out. (read timeout=%s)" % timeout_value
349 )
350
351 def _make_request(
352 self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
353 ):
354 """
355 Perform a request on a given urllib connection object taken from our
356 pool.
357
358 :param conn:
359 a connection from one of our connection pools
360
361 :param timeout:
362 Socket timeout in seconds for the request. This can be a
363 float or integer, which will set the same timeout value for
364 the socket connect and the socket read, or an instance of
365 :class:`urllib3.util.Timeout`, which gives you more fine-grained
366 control over your timeouts.
367 """
368 self.num_requests += 1
369
370 timeout_obj = self._get_timeout(timeout)
371 timeout_obj.start_connect()
372 conn.timeout = timeout_obj.connect_timeout
373
374 # Trigger any extra validation we need to do.
375 try:
376 self._validate_conn(conn)
377 except (SocketTimeout, BaseSSLError) as e:
378 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
379 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
380 raise
381
382 # conn.request() calls httplib.*.request, not the method in
383 # urllib3.request. It also calls makefile (recv) on the socket.
384 if chunked:
385 conn.request_chunked(method, url, **httplib_request_kw)
386 else:
387 conn.request(method, url, **httplib_request_kw)
388
389 # Reset the timeout for the recv() on the socket
390 read_timeout = timeout_obj.read_timeout
391
392 # App Engine doesn't have a sock attr
393 if getattr(conn, "sock", None):
394 # In Python 3 socket.py will catch EAGAIN and return None when you
395 # try and read into the file pointer created by http.client, which
396 # instead raises a BadStatusLine exception. Instead of catching
397 # the exception and assuming all BadStatusLine exceptions are read
398 # timeouts, check for a zero timeout before making the request.
399 if read_timeout == 0:
400 raise ReadTimeoutError(
401 self, url, "Read timed out. (read timeout=%s)" % read_timeout
402 )
403 if read_timeout is Timeout.DEFAULT_TIMEOUT:
404 conn.sock.settimeout(socket.getdefaulttimeout())
405 else: # None or a value
406 conn.sock.settimeout(read_timeout)
407
408 # Receive the response from the server
409 try:
410 try:
411 # Python 2.7, use buffering of HTTP responses
412 httplib_response = conn.getresponse(buffering=True)
413 except TypeError:
414 # Python 3
415 try:
416 httplib_response = conn.getresponse()
417 except BaseException as e:
418 # Remove the TypeError from the exception chain in
419 # Python 3 (including for exceptions like SystemExit).
420 # Otherwise it looks like a bug in the code.
421 six.raise_from(e, None)
422 except (SocketTimeout, BaseSSLError, SocketError) as e:
423 self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
424 raise
425
426 # AppEngine doesn't have a version attr.
427 http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
428 log.debug(
429 '%s://%s:%s "%s %s %s" %s %s',
430 self.scheme,
431 self.host,
432 self.port,
433 method,
434 url,
435 http_version,
436 httplib_response.status,
437 httplib_response.length,
438 )
439
440 try:
441 assert_header_parsing(httplib_response.msg)
442 except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3
443 log.warning(
444 "Failed to parse headers (url=%s): %s",
445 self._absolute_url(url),
446 hpe,
447 exc_info=True,
448 )
449
450 return httplib_response
451
452 def _absolute_url(self, path):
453 return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url
454
455 def close(self):
456 """
457 Close all pooled connections and disable the pool.
458 """
459 if self.pool is None:
460 return
461 # Disable access to the pool
462 old_pool, self.pool = self.pool, None
463
464 try:
465 while True:
466 conn = old_pool.get(block=False)
467 if conn:
468 conn.close()
469
470 except queue.Empty:
471 pass # Done.
472
473 def is_same_host(self, url):
474 """
475 Check if the given ``url`` is a member of the same host as this
476 connection pool.
477 """
478 if url.startswith("/"):
479 return True
480
481 # TODO: Add optional support for socket.gethostbyname checking.
482 scheme, host, port = get_host(url)
483 if host is not None:
484 host = _normalize_host(host, scheme=scheme)
485
486 # Use explicit default port for comparison when none is given
487 if self.port and not port:
488 port = port_by_scheme.get(scheme)
489 elif not self.port and port == port_by_scheme.get(scheme):
490 port = None
491
492 return (scheme, host, port) == (self.scheme, self.host, self.port)
493
494 def urlopen(
495 self,
496 method,
497 url,
498 body=None,
499 headers=None,
500 retries=None,
501 redirect=True,
502 assert_same_host=True,
503 timeout=_Default,
504 pool_timeout=None,
505 release_conn=None,
506 chunked=False,
507 body_pos=None,
508 **response_kw
509 ):
510 """
511 Get a connection from the pool and perform an HTTP request. This is the
512 lowest level call for making a request, so you'll need to specify all
513 the raw details.
514
515 .. note::
516
517 More commonly, it's appropriate to use a convenience method provided
518 by :class:`.RequestMethods`, such as :meth:`request`.
519
520 .. note::
521
522 `release_conn` will only behave as expected if
523 `preload_content=False` because we want to make
524 `preload_content=False` the default behaviour someday soon without
525 breaking backwards compatibility.
526
527 :param method:
528 HTTP request method (such as GET, POST, PUT, etc.)
529
530 :param body:
531 Data to send in the request body (useful for creating
532 POST requests, see HTTPConnectionPool.post_url for
533 more convenience).
534
535 :param headers:
536 Dictionary of custom headers to send, such as User-Agent,
537 If-None-Match, etc. If None, pool headers are used. If provided,
538 these headers completely replace any pool-specific headers.
539
540 :param retries:
541 Configure the number of retries to allow before raising a
542 :class:`~urllib3.exceptions.MaxRetryError` exception.
543
544 Pass ``None`` to retry until you receive a response. Pass a
545 :class:`~urllib3.util.retry.Retry` object for fine-grained control
546 over different types of retries.
547 Pass an integer number to retry connection errors that many times,
548 but no other types of errors. Pass zero to never retry.
549
550 If ``False``, then retries are disabled and any exception is raised
551 immediately. Also, instead of raising a MaxRetryError on redirects,
552 the redirect response will be returned.
553
554 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
555
556 :param redirect:
557 If True, automatically handle redirects (status codes 301, 302,
558 303, 307, 308). Each redirect counts as a retry. Disabling retries
559 will disable redirect, too.
560
561 :param assert_same_host:
562 If ``True``, will make sure that the host of the pool requests is
563 consistent else will raise HostChangedError. When False, you can
564 use the pool on an HTTP proxy and request foreign hosts.
565
566 :param timeout:
567 If specified, overrides the default timeout for this one
568 request. It may be a float (in seconds) or an instance of
569 :class:`urllib3.util.Timeout`.
570
571 :param pool_timeout:
572 If set and the pool is set to block=True, then this method will
573 block for ``pool_timeout`` seconds and raise EmptyPoolError if no
574 connection is available within the time period.
575
576 :param release_conn:
577 If False, then the urlopen call will not release the connection
578 back into the pool once a response is received (but will release if
579 you read the entire contents of the response such as when
580 `preload_content=True`). This is useful if you're not preloading
581 the response's content immediately. You will need to call
582 ``r.release_conn()`` on the response ``r`` to return the connection
583 back into the pool. If None, it takes the value of
584 ``response_kw.get('preload_content', True)``.
585
586 :param chunked:
587 If True, urllib3 will send the body using chunked transfer
588 encoding. Otherwise, urllib3 will send the body using the standard
589 content-length form. Defaults to False.
590
591 :param int body_pos:
592 Position to seek to in file-like body in the event of a retry or
593 redirect. Typically this won't need to be set because urllib3 will
594 auto-populate the value when needed.
595
596 :param \\**response_kw:
597 Additional parameters are passed to
598 :meth:`urllib3.response.HTTPResponse.from_httplib`
599 """
600 if headers is None:
601 headers = self.headers
602
603 if not isinstance(retries, Retry):
604 retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
605
606 if release_conn is None:
607 release_conn = response_kw.get("preload_content", True)
608
609 # Check host
610 if assert_same_host and not self.is_same_host(url):
611 raise HostChangedError(self, url, retries)
612
613 # Ensure that the URL we're connecting to is properly encoded
614 if url.startswith("/"):
615 url = six.ensure_str(_encode_target(url))
616 else:
617 url = six.ensure_str(parse_url(url).url)
618
619 conn = None
620
621 # Track whether `conn` needs to be released before
622 # returning/raising/recursing. Update this variable if necessary, and
623 # leave `release_conn` constant throughout the function. That way, if
624 # the function recurses, the original value of `release_conn` will be
625 # passed down into the recursive call, and its value will be respected.
626 #
627 # See issue #651 [1] for details.
628 #
629 # [1] <https://github.com/urllib3/urllib3/issues/651>
630 release_this_conn = release_conn
631
632 # Merge the proxy headers. Only do this in HTTP. We have to copy the
633 # headers dict so we can safely change it without those changes being
634 # reflected in anyone else's copy.
635 if self.scheme == "http":
636 headers = headers.copy()
637 headers.update(self.proxy_headers)
638
639 # Must keep the exception bound to a separate variable or else Python 3
640 # complains about UnboundLocalError.
641 err = None
642
643 # Keep track of whether we cleanly exited the except block. This
644 # ensures we do proper cleanup in finally.
645 clean_exit = False
646
647 # Rewind body position, if needed. Record current position
648 # for future rewinds in the event of a redirect/retry.
649 body_pos = set_file_position(body, body_pos)
650
651 try:
652 # Request a connection from the queue.
653 timeout_obj = self._get_timeout(timeout)
654 conn = self._get_conn(timeout=pool_timeout)
655
656 conn.timeout = timeout_obj.connect_timeout
657
658 is_new_proxy_conn = self.proxy is not None and not getattr(
659 conn, "sock", None
660 )
661 if is_new_proxy_conn:
662 self._prepare_proxy(conn)
663
664 # Make the request on the httplib connection object.
665 httplib_response = self._make_request(
666 conn,
667 method,
668 url,
669 timeout=timeout_obj,
670 body=body,
671 headers=headers,
672 chunked=chunked,
673 )
674
675 # If we're going to release the connection in ``finally:``, then
676 # the response doesn't need to know about the connection. Otherwise
677 # it will also try to release it and we'll have a double-release
678 # mess.
679 response_conn = conn if not release_conn else None
680
681 # Pass method to Response for length checking
682 response_kw["request_method"] = method
683
684 # Import httplib's response into our own wrapper object
685 response = self.ResponseCls.from_httplib(
686 httplib_response,
687 pool=self,
688 connection=response_conn,
689 retries=retries,
690 **response_kw
691 )
692
693 # Everything went great!
694 clean_exit = True
695
696 except queue.Empty:
697 # Timed out by queue.
698 raise EmptyPoolError(self, "No pool connections are available.")
699
700 except (
701 TimeoutError,
702 HTTPException,
703 SocketError,
704 ProtocolError,
705 BaseSSLError,
706 SSLError,
707 CertificateError,
708 ) as e:
709 # Discard the connection for these exceptions. It will be
710 # replaced during the next _get_conn() call.
711 clean_exit = False
712 if isinstance(e, (BaseSSLError, CertificateError)):
713 e = SSLError(e)
714 elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
715 e = ProxyError("Cannot connect to proxy.", e)
716 elif isinstance(e, (SocketError, HTTPException)):
717 e = ProtocolError("Connection aborted.", e)
718
719 retries = retries.increment(
720 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
721 )
722 retries.sleep()
723
724 # Keep track of the error for the retry warning.
725 err = e
726
727 finally:
728 if not clean_exit:
729 # We hit some kind of exception, handled or otherwise. We need
730 # to throw the connection away unless explicitly told not to.
731 # Close the connection, set the variable to None, and make sure
732 # we put the None back in the pool to avoid leaking it.
733 conn = conn and conn.close()
734 release_this_conn = True
735
736 if release_this_conn:
737 # Put the connection back to be reused. If the connection is
738 # expired then it will be None, which will get replaced with a
739 # fresh connection during _get_conn.
740 self._put_conn(conn)
741
742 if not conn:
743 # Try again
744 log.warning(
745 "Retrying (%r) after connection broken by '%r': %s", retries, err, url
746 )
747 return self.urlopen(
748 method,
749 url,
750 body,
751 headers,
752 retries,
753 redirect,
754 assert_same_host,
755 timeout=timeout,
756 pool_timeout=pool_timeout,
757 release_conn=release_conn,
758 chunked=chunked,
759 body_pos=body_pos,
760 **response_kw
761 )
762
763 def drain_and_release_conn(response):
764 try:
765 # discard any remaining response body, the connection will be
766 # released back to the pool once the entire response is read
767 response.read()
768 except (
769 TimeoutError,
770 HTTPException,
771 SocketError,
772 ProtocolError,
773 BaseSSLError,
774 SSLError,
775 ):
776 pass
777
778 # Handle redirect?
779 redirect_location = redirect and response.get_redirect_location()
780 if redirect_location:
781 if response.status == 303:
782 method = "GET"
783
784 try:
785 retries = retries.increment(method, url, response=response, _pool=self)
786 except MaxRetryError:
787 if retries.raise_on_redirect:
788 # Drain and release the connection for this response, since
789 # we're not returning it to be released manually.
790 drain_and_release_conn(response)
791 raise
792 return response
793
794 # drain and return the connection to the pool before recursing
795 drain_and_release_conn(response)
796
797 retries.sleep_for_retry(response)
798 log.debug("Redirecting %s -> %s", url, redirect_location)
799 return self.urlopen(
800 method,
801 redirect_location,
802 body,
803 headers,
804 retries=retries,
805 redirect=redirect,
806 assert_same_host=assert_same_host,
807 timeout=timeout,
808 pool_timeout=pool_timeout,
809 release_conn=release_conn,
810 chunked=chunked,
811 body_pos=body_pos,
812 **response_kw
813 )
814
815 # Check if we should retry the HTTP response.
816 has_retry_after = bool(response.getheader("Retry-After"))
817 if retries.is_retry(method, response.status, has_retry_after):
818 try:
819 retries = retries.increment(method, url, response=response, _pool=self)
820 except MaxRetryError:
821 if retries.raise_on_status:
822 # Drain and release the connection for this response, since
823 # we're not returning it to be released manually.
824 drain_and_release_conn(response)
825 raise
826 return response
827
828 # drain and return the connection to the pool before recursing
829 drain_and_release_conn(response)
830
831 retries.sleep(response)
832 log.debug("Retry: %s", url)
833 return self.urlopen(
834 method,
835 url,
836 body,
837 headers,
838 retries=retries,
839 redirect=redirect,
840 assert_same_host=assert_same_host,
841 timeout=timeout,
842 pool_timeout=pool_timeout,
843 release_conn=release_conn,
844 chunked=chunked,
845 body_pos=body_pos,
846 **response_kw
847 )
848
849 return response
850
851
852 class HTTPSConnectionPool(HTTPConnectionPool):
853 """
854 Same as :class:`.HTTPConnectionPool`, but HTTPS.
855
856 When Python is compiled with the :mod:`ssl` module, then
857 :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
858 instead of :class:`.HTTPSConnection`.
859
860 :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
861 ``assert_hostname`` and ``host`` in this order to verify connections.
862 If ``assert_hostname`` is False, no verification is done.
863
864 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
865 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
866 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
867 the connection socket into an SSL socket.
868 """
869
870 scheme = "https"
871 ConnectionCls = HTTPSConnection
872
873 def __init__(
874 self,
875 host,
876 port=None,
877 strict=False,
878 timeout=Timeout.DEFAULT_TIMEOUT,
879 maxsize=1,
880 block=False,
881 headers=None,
882 retries=None,
883 _proxy=None,
884 _proxy_headers=None,
885 key_file=None,
886 cert_file=None,
887 cert_reqs=None,
888 key_password=None,
889 ca_certs=None,
890 ssl_version=None,
891 assert_hostname=None,
892 assert_fingerprint=None,
893 ca_cert_dir=None,
894 **conn_kw
895 ):
896
897 HTTPConnectionPool.__init__(
898 self,
899 host,
900 port,
901 strict,
902 timeout,
903 maxsize,
904 block,
905 headers,
906 retries,
907 _proxy,
908 _proxy_headers,
909 **conn_kw
910 )
911
912 self.key_file = key_file
913 self.cert_file = cert_file
914 self.cert_reqs = cert_reqs
915 self.key_password = key_password
916 self.ca_certs = ca_certs
917 self.ca_cert_dir = ca_cert_dir
918 self.ssl_version = ssl_version
919 self.assert_hostname = assert_hostname
920 self.assert_fingerprint = assert_fingerprint
921
922 def _prepare_conn(self, conn):
923 """
924 Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
925 and establish the tunnel if proxy is used.
926 """
927
928 if isinstance(conn, VerifiedHTTPSConnection):
929 conn.set_cert(
930 key_file=self.key_file,
931 key_password=self.key_password,
932 cert_file=self.cert_file,
933 cert_reqs=self.cert_reqs,
934 ca_certs=self.ca_certs,
935 ca_cert_dir=self.ca_cert_dir,
936 assert_hostname=self.assert_hostname,
937 assert_fingerprint=self.assert_fingerprint,
938 )
939 conn.ssl_version = self.ssl_version
940 return conn
941
942 def _prepare_proxy(self, conn):
943 """
944 Establish tunnel connection early, because otherwise httplib
945 would improperly set Host: header to proxy's IP:port.
946 """
947 conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
948 conn.connect()
949
950 def _new_conn(self):
951 """
952 Return a fresh :class:`httplib.HTTPSConnection`.
953 """
954 self.num_connections += 1
955 log.debug(
956 "Starting new HTTPS connection (%d): %s:%s",
957 self.num_connections,
958 self.host,
959 self.port or "443",
960 )
961
962 if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
963 raise SSLError(
964 "Can't connect to HTTPS URL because the SSL module is not available."
965 )
966
967 actual_host = self.host
968 actual_port = self.port
969 if self.proxy is not None:
970 actual_host = self.proxy.host
971 actual_port = self.proxy.port
972
973 conn = self.ConnectionCls(
974 host=actual_host,
975 port=actual_port,
976 timeout=self.timeout.connect_timeout,
977 strict=self.strict,
978 cert_file=self.cert_file,
979 key_file=self.key_file,
980 key_password=self.key_password,
981 **self.conn_kw
982 )
983
984 return self._prepare_conn(conn)
985
986 def _validate_conn(self, conn):
987 """
988 Called right before a request is made, after the socket is created.
989 """
990 super(HTTPSConnectionPool, self)._validate_conn(conn)
991
992 # Force connect early to allow us to validate the connection.
993 if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
994 conn.connect()
995
996 if not conn.is_verified:
997 warnings.warn(
998 (
999 "Unverified HTTPS request is being made. "
1000 "Adding certificate verification is strongly advised. See: "
1001 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1002 "#ssl-warnings"
1003 ),
1004 InsecureRequestWarning,
1005 )
1006
1007
1008 def connection_from_url(url, **kw):
1009 """
1010 Given a url, return an :class:`.ConnectionPool` instance of its host.
1011
1012 This is a shortcut for not having to parse out the scheme, host, and port
1013 of the url before creating an :class:`.ConnectionPool` instance.
1014
1015 :param url:
1016 Absolute URL string that must include the scheme. Port is optional.
1017
1018 :param \\**kw:
1019 Passes additional parameters to the constructor of the appropriate
1020 :class:`.ConnectionPool`. Useful for specifying things like
1021 timeout, maxsize, headers, etc.
1022
1023 Example::
1024
1025 >>> conn = connection_from_url('http://google.com/')
1026 >>> r = conn.request('GET', '/')
1027 """
1028 scheme, host, port = get_host(url)
1029 port = port or port_by_scheme.get(scheme, 80)
1030 if scheme == "https":
1031 return HTTPSConnectionPool(host, port=port, **kw)
1032 else:
1033 return HTTPConnectionPool(host, port=port, **kw)
1034
1035
1036 def _normalize_host(host, scheme):
1037 """
1038 Normalize hosts for comparisons and use with sockets.
1039 """
1040
1041 host = normalize_host(host, scheme)
1042
1043 # httplib doesn't like it when we include brackets in IPv6 addresses
1044 # Specifically, if we include brackets but also pass the port then
1045 # httplib crazily doubles up the square brackets on the Host header.
1046 # Instead, we need to make sure we never pass ``None`` as the port.
1047 # However, for backward compatibility reasons we can't actually
1048 # *assert* that. See http://bugs.python.org/issue28539
1049 if host.startswith("[") and host.endswith("]"):
1050 host = host[1:-1]
1051 return host