Mercurial > repos > shellac > sam_consensus_v3

comparison env/lib/python3.9/site-packages/urllib3/util/retry.py @ 0:4f3585e2f14b draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author shellac
date Mon, 22 Mar 2021 18:12:50 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:4f3585e2f14b
1 from __future__ import absolute_import
2
3 import email
4 import logging
5 import re
6 import time
7 import warnings
8 from collections import namedtuple
9 from itertools import takewhile
10
11 from ..exceptions import (
12 ConnectTimeoutError,
13 InvalidHeader,
14 MaxRetryError,
15 ProtocolError,
16 ProxyError,
17 ReadTimeoutError,
18 ResponseError,
19 )
20 from ..packages import six
21
22 log = logging.getLogger(__name__)
23
24
25 # Data structure for representing the metadata of requests that result in a retry.
26 RequestHistory = namedtuple(
27 "RequestHistory", ["method", "url", "error", "status", "redirect_location"]
28 )
29
30
31 # TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
32 _Default = object()
33
34
35 class _RetryMeta(type):
36 @property
37 def DEFAULT_METHOD_WHITELIST(cls):
38 warnings.warn(
39 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
40 "will be removed in v2.0. Use 'Retry.DEFAULT_METHODS_ALLOWED' instead",
41 DeprecationWarning,
42 )
43 return cls.DEFAULT_ALLOWED_METHODS
44
45 @DEFAULT_METHOD_WHITELIST.setter
46 def DEFAULT_METHOD_WHITELIST(cls, value):
47 warnings.warn(
48 "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
49 "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
50 DeprecationWarning,
51 )
52 cls.DEFAULT_ALLOWED_METHODS = value
53
54 @property
55 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
56 warnings.warn(
57 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
58 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
59 DeprecationWarning,
60 )
61 return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
62
63 @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
64 def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
65 warnings.warn(
66 "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
67 "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
68 DeprecationWarning,
69 )
70 cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
71
72
73 @six.add_metaclass(_RetryMeta)
74 class Retry(object):
75 """Retry configuration.
76
77 Each retry attempt will create a new Retry object with updated values, so
78 they can be safely reused.
79
80 Retries can be defined as a default for a pool::
81
82 retries = Retry(connect=5, read=2, redirect=5)
83 http = PoolManager(retries=retries)
84 response = http.request('GET', 'http://example.com/')
85
86 Or per-request (which overrides the default for the pool)::
87
88 response = http.request('GET', 'http://example.com/', retries=Retry(10))
89
90 Retries can be disabled by passing ``False``::
91
92 response = http.request('GET', 'http://example.com/', retries=False)
93
94 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
95 retries are disabled, in which case the causing exception will be raised.
96
97 :param int total:
98 Total number of retries to allow. Takes precedence over other counts.
99
100 Set to ``None`` to remove this constraint and fall back on other
101 counts.
102
103 Set to ``0`` to fail on the first retry.
104
105 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
106
107 :param int connect:
108 How many connection-related errors to retry on.
109
110 These are errors raised before the request is sent to the remote server,
111 which we assume has not triggered the server to process the request.
112
113 Set to ``0`` to fail on the first retry of this type.
114
115 :param int read:
116 How many times to retry on read errors.
117
118 These errors are raised after the request was sent to the server, so the
119 request may have side-effects.
120
121 Set to ``0`` to fail on the first retry of this type.
122
123 :param int redirect:
124 How many redirects to perform. Limit this to avoid infinite redirect
125 loops.
126
127 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
128 308.
129
130 Set to ``0`` to fail on the first retry of this type.
131
132 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
133
134 :param int status:
135 How many times to retry on bad status codes.
136
137 These are retries made on responses, where status code matches
138 ``status_forcelist``.
139
140 Set to ``0`` to fail on the first retry of this type.
141
142 :param int other:
143 How many times to retry on other errors.
144
145 Other errors are errors that are not connect, read, redirect or status errors.
146 These errors might be raised after the request was sent to the server, so the
147 request might have side-effects.
148
149 Set to ``0`` to fail on the first retry of this type.
150
151 If ``total`` is not set, it's a good idea to set this to 0 to account
152 for unexpected edge cases and avoid infinite retry loops.
153
154 :param iterable allowed_methods:
155 Set of uppercased HTTP method verbs that we should retry on.
156
157 By default, we only retry on methods which are considered to be
158 idempotent (multiple requests with the same parameters end with the
159 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
160
161 Set to a ``False`` value to retry on any verb.
162
163 .. warning::
164
165 Previously this parameter was named ``method_whitelist``, that
166 usage is deprecated in v1.26.0 and will be removed in v2.0.
167
168 :param iterable status_forcelist:
169 A set of integer HTTP status codes that we should force a retry on.
170 A retry is initiated if the request method is in ``allowed_methods``
171 and the response status code is in ``status_forcelist``.
172
173 By default, this is disabled with ``None``.
174
175 :param float backoff_factor:
176 A backoff factor to apply between attempts after the second try
177 (most errors are resolved immediately by a second try without a
178 delay). urllib3 will sleep for::
179
180 {backoff factor} * (2 ** ({number of total retries} - 1))
181
182 seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
183 for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
184 than :attr:`Retry.BACKOFF_MAX`.
185
186 By default, backoff is disabled (set to 0).
187
188 :param bool raise_on_redirect: Whether, if the number of redirects is
189 exhausted, to raise a MaxRetryError, or to return a response with a
190 response code in the 3xx range.
191
192 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
193 whether we should raise an exception, or return a response,
194 if status falls in ``status_forcelist`` range and retries have
195 been exhausted.
196
197 :param tuple history: The history of the request encountered during
198 each call to :meth:`~Retry.increment`. The list is in the order
199 the requests occurred. Each list item is of class :class:`RequestHistory`.
200
201 :param bool respect_retry_after_header:
202 Whether to respect Retry-After header on status codes defined as
203 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
204
205 :param iterable remove_headers_on_redirect:
206 Sequence of headers to remove from the request when a response
207 indicating a redirect is returned before firing off the redirected
208 request.
209 """
210
211 #: Default methods to be used for ``allowed_methods``
212 DEFAULT_ALLOWED_METHODS = frozenset(
213 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
214 )
215
216 #: Default status codes to be used for ``status_forcelist``
217 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
218
219 #: Default headers to be used for ``remove_headers_on_redirect``
220 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
221
222 #: Maximum backoff time.
223 BACKOFF_MAX = 120
224
225 def __init__(
226 self,
227 total=10,
228 connect=None,
229 read=None,
230 redirect=None,
231 status=None,
232 other=None,
233 allowed_methods=_Default,
234 status_forcelist=None,
235 backoff_factor=0,
236 raise_on_redirect=True,
237 raise_on_status=True,
238 history=None,
239 respect_retry_after_header=True,
240 remove_headers_on_redirect=_Default,
241 # TODO: Deprecated, remove in v2.0
242 method_whitelist=_Default,
243 ):
244
245 if method_whitelist is not _Default:
246 if allowed_methods is not _Default:
247 raise ValueError(
248 "Using both 'allowed_methods' and "
249 "'method_whitelist' together is not allowed. "
250 "Instead only use 'allowed_methods'"
251 )
252 warnings.warn(
253 "Using 'method_whitelist' with Retry is deprecated and "
254 "will be removed in v2.0. Use 'allowed_methods' instead",
255 DeprecationWarning,
256 stacklevel=2,
257 )
258 allowed_methods = method_whitelist
259 if allowed_methods is _Default:
260 allowed_methods = self.DEFAULT_ALLOWED_METHODS
261 if remove_headers_on_redirect is _Default:
262 remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
263
264 self.total = total
265 self.connect = connect
266 self.read = read
267 self.status = status
268 self.other = other
269
270 if redirect is False or total is False:
271 redirect = 0
272 raise_on_redirect = False
273
274 self.redirect = redirect
275 self.status_forcelist = status_forcelist or set()
276 self.allowed_methods = allowed_methods
277 self.backoff_factor = backoff_factor
278 self.raise_on_redirect = raise_on_redirect
279 self.raise_on_status = raise_on_status
280 self.history = history or tuple()
281 self.respect_retry_after_header = respect_retry_after_header
282 self.remove_headers_on_redirect = frozenset(
283 [h.lower() for h in remove_headers_on_redirect]
284 )
285
286 def new(self, **kw):
287 params = dict(
288 total=self.total,
289 connect=self.connect,
290 read=self.read,
291 redirect=self.redirect,
292 status=self.status,
293 other=self.other,
294 status_forcelist=self.status_forcelist,
295 backoff_factor=self.backoff_factor,
296 raise_on_redirect=self.raise_on_redirect,
297 raise_on_status=self.raise_on_status,
298 history=self.history,
299 remove_headers_on_redirect=self.remove_headers_on_redirect,
300 respect_retry_after_header=self.respect_retry_after_header,
301 )
302
303 # TODO: If already given in **kw we use what's given to us
304 # If not given we need to figure out what to pass. We decide
305 # based on whether our class has the 'method_whitelist' property
306 # and if so we pass the deprecated 'method_whitelist' otherwise
307 # we use 'allowed_methods'. Remove in v2.0
308 if "method_whitelist" not in kw and "allowed_methods" not in kw:
309 if "method_whitelist" in self.__dict__:
310 warnings.warn(
311 "Using 'method_whitelist' with Retry is deprecated and "
312 "will be removed in v2.0. Use 'allowed_methods' instead",
313 DeprecationWarning,
314 )
315 params["method_whitelist"] = self.allowed_methods
316 else:
317 params["allowed_methods"] = self.allowed_methods
318
319 params.update(kw)
320 return type(self)(**params)
321
322 @classmethod
323 def from_int(cls, retries, redirect=True, default=None):
324 """ Backwards-compatibility for the old retries format."""
325 if retries is None:
326 retries = default if default is not None else cls.DEFAULT
327
328 if isinstance(retries, Retry):
329 return retries
330
331 redirect = bool(redirect) and None
332 new_retries = cls(retries, redirect=redirect)
333 log.debug("Converted retries value: %r -> %r", retries, new_retries)
334 return new_retries
335
336 def get_backoff_time(self):
337 """Formula for computing the current backoff
338
339 :rtype: float
340 """
341 # We want to consider only the last consecutive errors sequence (Ignore redirects).
342 consecutive_errors_len = len(
343 list(
344 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
345 )
346 )
347 if consecutive_errors_len <= 1:
348 return 0
349
350 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
351 return min(self.BACKOFF_MAX, backoff_value)
352
353 def parse_retry_after(self, retry_after):
354 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
355 if re.match(r"^\s*[0-9]+\s*$", retry_after):
356 seconds = int(retry_after)
357 else:
358 retry_date_tuple = email.utils.parsedate_tz(retry_after)
359 if retry_date_tuple is None:
360 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
361 if retry_date_tuple[9] is None: # Python 2
362 # Assume UTC if no timezone was specified
363 # On Python2.7, parsedate_tz returns None for a timezone offset
364 # instead of 0 if no timezone is given, where mktime_tz treats
365 # a None timezone offset as local time.
366 retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
367
368 retry_date = email.utils.mktime_tz(retry_date_tuple)
369 seconds = retry_date - time.time()
370
371 if seconds < 0:
372 seconds = 0
373
374 return seconds
375
376 def get_retry_after(self, response):
377 """ Get the value of Retry-After in seconds. """
378
379 retry_after = response.getheader("Retry-After")
380
381 if retry_after is None:
382 return None
383
384 return self.parse_retry_after(retry_after)
385
386 def sleep_for_retry(self, response=None):
387 retry_after = self.get_retry_after(response)
388 if retry_after:
389 time.sleep(retry_after)
390 return True
391
392 return False
393
394 def _sleep_backoff(self):
395 backoff = self.get_backoff_time()
396 if backoff <= 0:
397 return
398 time.sleep(backoff)
399
400 def sleep(self, response=None):
401 """Sleep between retry attempts.
402
403 This method will respect a server's ``Retry-After`` response header
404 and sleep the duration of the time requested. If that is not present, it
405 will use an exponential backoff. By default, the backoff factor is 0 and
406 this method will return immediately.
407 """
408
409 if self.respect_retry_after_header and response:
410 slept = self.sleep_for_retry(response)
411 if slept:
412 return
413
414 self._sleep_backoff()
415
416 def _is_connection_error(self, err):
417 """Errors when we're fairly sure that the server did not receive the
418 request, so it should be safe to retry.
419 """
420 if isinstance(err, ProxyError):
421 err = err.original_error
422 return isinstance(err, ConnectTimeoutError)
423
424 def _is_read_error(self, err):
425 """Errors that occur after the request has been started, so we should
426 assume that the server began processing it.
427 """
428 return isinstance(err, (ReadTimeoutError, ProtocolError))
429
430 def _is_method_retryable(self, method):
431 """Checks if a given HTTP method should be retried upon, depending if
432 it is included in the allowed_methods
433 """
434 # TODO: For now favor if the Retry implementation sets its own method_whitelist
435 # property outside of our constructor to avoid breaking custom implementations.
436 if "method_whitelist" in self.__dict__:
437 warnings.warn(
438 "Using 'method_whitelist' with Retry is deprecated and "
439 "will be removed in v2.0. Use 'allowed_methods' instead",
440 DeprecationWarning,
441 )
442 allowed_methods = self.method_whitelist
443 else:
444 allowed_methods = self.allowed_methods
445
446 if allowed_methods and method.upper() not in allowed_methods:
447 return False
448 return True
449
450 def is_retry(self, method, status_code, has_retry_after=False):
451 """Is this method/status code retryable? (Based on allowlists and control
452 variables such as the number of total retries to allow, whether to
453 respect the Retry-After header, whether this header is present, and
454 whether the returned status code is on the list of status codes to
455 be retried upon on the presence of the aforementioned header)
456 """
457 if not self._is_method_retryable(method):
458 return False
459
460 if self.status_forcelist and status_code in self.status_forcelist:
461 return True
462
463 return (
464 self.total
465 and self.respect_retry_after_header
466 and has_retry_after
467 and (status_code in self.RETRY_AFTER_STATUS_CODES)
468 )
469
470 def is_exhausted(self):
471 """ Are we out of retries? """
472 retry_counts = (
473 self.total,
474 self.connect,
475 self.read,
476 self.redirect,
477 self.status,
478 self.other,
479 )
480 retry_counts = list(filter(None, retry_counts))
481 if not retry_counts:
482 return False
483
484 return min(retry_counts) < 0
485
486 def increment(
487 self,
488 method=None,
489 url=None,
490 response=None,
491 error=None,
492 _pool=None,
493 _stacktrace=None,
494 ):
495 """Return a new Retry object with incremented retry counters.
496
497 :param response: A response object, or None, if the server did not
498 return a response.
499 :type response: :class:`~urllib3.response.HTTPResponse`
500 :param Exception error: An error encountered during the request, or
501 None if the response was received successfully.
502
503 :return: A new ``Retry`` object.
504 """
505 if self.total is False and error:
506 # Disabled, indicate to re-raise the error.
507 raise six.reraise(type(error), error, _stacktrace)
508
509 total = self.total
510 if total is not None:
511 total -= 1
512
513 connect = self.connect
514 read = self.read
515 redirect = self.redirect
516 status_count = self.status
517 other = self.other
518 cause = "unknown"
519 status = None
520 redirect_location = None
521
522 if error and self._is_connection_error(error):
523 # Connect retry?
524 if connect is False:
525 raise six.reraise(type(error), error, _stacktrace)
526 elif connect is not None:
527 connect -= 1
528
529 elif error and self._is_read_error(error):
530 # Read retry?
531 if read is False or not self._is_method_retryable(method):
532 raise six.reraise(type(error), error, _stacktrace)
533 elif read is not None:
534 read -= 1
535
536 elif error:
537 # Other retry?
538 if other is not None:
539 other -= 1
540
541 elif response and response.get_redirect_location():
542 # Redirect retry?
543 if redirect is not None:
544 redirect -= 1
545 cause = "too many redirects"
546 redirect_location = response.get_redirect_location()
547 status = response.status
548
549 else:
550 # Incrementing because of a server error like a 500 in
551 # status_forcelist and the given method is in the allowed_methods
552 cause = ResponseError.GENERIC_ERROR
553 if response and response.status:
554 if status_count is not None:
555 status_count -= 1
556 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
557 status = response.status
558
559 history = self.history + (
560 RequestHistory(method, url, error, status, redirect_location),
561 )
562
563 new_retry = self.new(
564 total=total,
565 connect=connect,
566 read=read,
567 redirect=redirect,
568 status=status_count,
569 other=other,
570 history=history,
571 )
572
573 if new_retry.is_exhausted():
574 raise MaxRetryError(_pool, url, error or ResponseError(cause))
575
576 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
577
578 return new_retry
579
580 def __repr__(self):
581 return (
582 "{cls.__name__}(total={self.total}, connect={self.connect}, "
583 "read={self.read}, redirect={self.redirect}, status={self.status})"
584 ).format(cls=type(self), self=self)
585
586 def __getattr__(self, item):
587 if item == "method_whitelist":
588 # TODO: Remove this deprecated alias in v2.0
589 warnings.warn(
590 "Using 'method_whitelist' with Retry is deprecated and "
591 "will be removed in v2.0. Use 'allowed_methods' instead",
592 DeprecationWarning,
593 )
594 return self.allowed_methods
595 try:
596 return getattr(super(Retry, self), item)
597 except AttributeError:
598 return getattr(Retry, item)
599
600
601 # For backwards compatibility (equivalent to pre-v1.9):
602 Retry.DEFAULT = Retry(3)