Mercurial > repos > shellac > guppy_basecaller
diff env/lib/python3.7/site-packages/boto/connection.py @ 5:9b1c78e6ba9c draft default tip
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author | shellac |
---|---|
date | Mon, 01 Jun 2020 08:59:25 -0400 |
parents | 79f47841a781 |
children |
line wrap: on
line diff
--- a/env/lib/python3.7/site-packages/boto/connection.py Thu May 14 16:47:39 2020 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1227 +0,0 @@ -# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. -# Copyright (c) 2010 Google -# Copyright (c) 2008 rPath, Inc. -# Copyright (c) 2009 The Echo Nest Corporation -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# Copyright (c) 2011, Nexenta Systems Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# -# Parts of this code were copied or derived from sample code supplied by AWS. -# The following notice applies to that code. -# -# This software code is made available "AS IS" without warranties of any -# kind. You may copy, display, modify and redistribute the software -# code either by itself or as incorporated into your code; provided that -# you do not remove any proprietary notices. Your use of this software -# code is at your own risk and you waive any claim against Amazon -# Digital Services, Inc. or its affiliates with respect to your use of -# this software code. (c) 2006 Amazon Digital Services, Inc. or its -# affiliates. - -""" -Handles basic connections to AWS -""" -from datetime import datetime -import errno -import os -import random -import re -import socket -import sys -import time -import xml.sax -import copy - -from boto import auth -from boto import auth_handler -import boto -import boto.utils -import boto.handler -import boto.cacerts - -from boto import config, UserAgent -from boto.compat import six, http_client, urlparse, quote, encodebytes -from boto.exception import AWSConnectionError -from boto.exception import BotoClientError -from boto.exception import BotoServerError -from boto.exception import PleaseRetryException -from boto.provider import Provider -from boto.resultset import ResultSet - -HAVE_HTTPS_CONNECTION = False -try: - import ssl - from boto import https_connection - # Google App Engine runs on Python 2.5 so doesn't have ssl.SSLError. - if hasattr(ssl, 'SSLError'): - HAVE_HTTPS_CONNECTION = True -except ImportError: - pass - -try: - import threading -except ImportError: - import dummy_threading as threading - -ON_APP_ENGINE = all(key in os.environ for key in ( - 'USER_IS_ADMIN', 'CURRENT_VERSION_ID', 'APPLICATION_ID')) - -PORTS_BY_SECURITY = {True: 443, - False: 80} - -DEFAULT_CA_CERTS_FILE = os.path.join(os.path.dirname(os.path.abspath(boto.cacerts.__file__)), "cacerts.txt") - - -class HostConnectionPool(object): - - """ - A pool of connections for one remote (host,port,is_secure). - - When connections are added to the pool, they are put into a - pending queue. The _mexe method returns connections to the pool - before the response body has been read, so they connections aren't - ready to send another request yet. They stay in the pending queue - until they are ready for another request, at which point they are - returned to the pool of ready connections. - - The pool of ready connections is an ordered list of - (connection,time) pairs, where the time is the time the connection - was returned from _mexe. After a certain period of time, - connections are considered stale, and discarded rather than being - reused. This saves having to wait for the connection to time out - if AWS has decided to close it on the other end because of - inactivity. - - Thread Safety: - - This class is used only from ConnectionPool while it's mutex - is held. - """ - - def __init__(self): - self.queue = [] - - def size(self): - """ - Returns the number of connections in the pool for this host. - Some of the connections may still be in use, and may not be - ready to be returned by get(). - """ - return len(self.queue) - - def put(self, conn): - """ - Adds a connection to the pool, along with the time it was - added. - """ - self.queue.append((conn, time.time())) - - def get(self): - """ - Returns the next connection in this pool that is ready to be - reused. Returns None if there aren't any. - """ - # Discard ready connections that are too old. - self.clean() - - # Return the first connection that is ready, and remove it - # from the queue. Connections that aren't ready are returned - # to the end of the queue with an updated time, on the - # assumption that somebody is actively reading the response. - for _ in range(len(self.queue)): - (conn, _) = self.queue.pop(0) - if self._conn_ready(conn): - return conn - else: - self.put(conn) - return None - - def _conn_ready(self, conn): - """ - There is a nice state diagram at the top of http_client.py. It - indicates that once the response headers have been read (which - _mexe does before adding the connection to the pool), a - response is attached to the connection, and it stays there - until it's done reading. This isn't entirely true: even after - the client is done reading, the response may be closed, but - not removed from the connection yet. - - This is ugly, reading a private instance variable, but the - state we care about isn't available in any public methods. - """ - if ON_APP_ENGINE: - # Google AppEngine implementation of HTTPConnection doesn't contain - # _HTTPConnection__response attribute. Moreover, it's not possible - # to determine if given connection is ready. Reusing connections - # simply doesn't make sense with App Engine urlfetch service. - return False - else: - response = getattr(conn, '_HTTPConnection__response', None) - return (response is None) or response.isclosed() - - def clean(self): - """ - Get rid of stale connections. - """ - # Note that we do not close the connection here -- somebody - # may still be reading from it. - while len(self.queue) > 0 and self._pair_stale(self.queue[0]): - self.queue.pop(0) - - def _pair_stale(self, pair): - """ - Returns true of the (connection,time) pair is too old to be - used. - """ - (_conn, return_time) = pair - now = time.time() - return return_time + ConnectionPool.STALE_DURATION < now - - -class ConnectionPool(object): - - """ - A connection pool that expires connections after a fixed period of - time. This saves time spent waiting for a connection that AWS has - timed out on the other end. - - This class is thread-safe. - """ - - # - # The amout of time between calls to clean. - # - - CLEAN_INTERVAL = 5.0 - - # - # How long before a connection becomes "stale" and won't be reused - # again. The intention is that this time is less that the timeout - # period that AWS uses, so we'll never try to reuse a connection - # and find that AWS is timing it out. - # - # Experimentation in July 2011 shows that AWS starts timing things - # out after three minutes. The 60 seconds here is conservative so - # we should never hit that 3-minute timout. - # - - STALE_DURATION = 60.0 - - def __init__(self): - # Mapping from (host,port,is_secure) to HostConnectionPool. - # If a pool becomes empty, it is removed. - self.host_to_pool = {} - # The last time the pool was cleaned. - self.last_clean_time = 0.0 - self.mutex = threading.Lock() - ConnectionPool.STALE_DURATION = \ - config.getfloat('Boto', 'connection_stale_duration', - ConnectionPool.STALE_DURATION) - - def __getstate__(self): - pickled_dict = copy.copy(self.__dict__) - pickled_dict['host_to_pool'] = {} - del pickled_dict['mutex'] - return pickled_dict - - def __setstate__(self, dct): - self.__init__() - - def size(self): - """ - Returns the number of connections in the pool. - """ - return sum(pool.size() for pool in self.host_to_pool.values()) - - def get_http_connection(self, host, port, is_secure): - """ - Gets a connection from the pool for the named host. Returns - None if there is no connection that can be reused. It's the caller's - responsibility to call close() on the connection when it's no longer - needed. - """ - self.clean() - with self.mutex: - key = (host, port, is_secure) - if key not in self.host_to_pool: - return None - return self.host_to_pool[key].get() - - def put_http_connection(self, host, port, is_secure, conn): - """ - Adds a connection to the pool of connections that can be - reused for the named host. - """ - with self.mutex: - key = (host, port, is_secure) - if key not in self.host_to_pool: - self.host_to_pool[key] = HostConnectionPool() - self.host_to_pool[key].put(conn) - - def clean(self): - """ - Clean up the stale connections in all of the pools, and then - get rid of empty pools. Pools clean themselves every time a - connection is fetched; this cleaning takes care of pools that - aren't being used any more, so nothing is being gotten from - them. - """ - with self.mutex: - now = time.time() - if self.last_clean_time + self.CLEAN_INTERVAL < now: - to_remove = [] - for (host, pool) in self.host_to_pool.items(): - pool.clean() - if pool.size() == 0: - to_remove.append(host) - for host in to_remove: - del self.host_to_pool[host] - self.last_clean_time = now - - -class HTTPRequest(object): - - def __init__(self, method, protocol, host, port, path, auth_path, - params, headers, body): - """Represents an HTTP request. - - :type method: string - :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. - - :type protocol: string - :param protocol: The http protocol used, 'http' or 'https'. - - :type host: string - :param host: Host to which the request is addressed. eg. abc.com - - :type port: int - :param port: port on which the request is being sent. Zero means unset, - in which case default port will be chosen. - - :type path: string - :param path: URL path that is being accessed. - - :type auth_path: string - :param path: The part of the URL path used when creating the - authentication string. - - :type params: dict - :param params: HTTP url query parameters, with key as name of - the param, and value as value of param. - - :type headers: dict - :param headers: HTTP headers, with key as name of the header and value - as value of header. - - :type body: string - :param body: Body of the HTTP request. If not present, will be None or - empty string (''). - """ - self.method = method - self.protocol = protocol - self.host = host - self.port = port - self.path = path - if auth_path is None: - auth_path = path - self.auth_path = auth_path - self.params = params - # chunked Transfer-Encoding should act only on PUT request. - if headers and 'Transfer-Encoding' in headers and \ - headers['Transfer-Encoding'] == 'chunked' and \ - self.method != 'PUT': - self.headers = headers.copy() - del self.headers['Transfer-Encoding'] - else: - self.headers = headers - self.body = body - - def __str__(self): - return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) ' - 'params(%s) headers(%s) body(%s)') % (self.method, - self.protocol, self.host, self.port, self.path, self.params, - self.headers, self.body)) - - def authorize(self, connection, **kwargs): - if not getattr(self, '_headers_quoted', False): - for key in self.headers: - val = self.headers[key] - if isinstance(val, six.text_type): - safe = '!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~ ' - self.headers[key] = quote(val.encode('utf-8'), safe) - setattr(self, '_headers_quoted', True) - - self.headers['User-Agent'] = UserAgent - - connection._auth_handler.add_auth(self, **kwargs) - - # I'm not sure if this is still needed, now that add_auth is - # setting the content-length for POST requests. - if 'Content-Length' not in self.headers: - if 'Transfer-Encoding' not in self.headers or \ - self.headers['Transfer-Encoding'] != 'chunked': - self.headers['Content-Length'] = str(len(self.body)) - - -class HTTPResponse(http_client.HTTPResponse): - - def __init__(self, *args, **kwargs): - http_client.HTTPResponse.__init__(self, *args, **kwargs) - self._cached_response = '' - - def read(self, amt=None): - """Read the response. - - This method does not have the same behavior as - http_client.HTTPResponse.read. Instead, if this method is called with - no ``amt`` arg, then the response body will be cached. Subsequent - calls to ``read()`` with no args **will return the cached response**. - - """ - if amt is None: - # The reason for doing this is that many places in boto call - # response.read() and except to get the response body that they - # can then process. To make sure this always works as they expect - # we're caching the response so that multiple calls to read() - # will return the full body. Note that this behavior only - # happens if the amt arg is not specified. - if not self._cached_response: - self._cached_response = http_client.HTTPResponse.read(self) - return self._cached_response - else: - return http_client.HTTPResponse.read(self, amt) - - -class AWSAuthConnection(object): - def __init__(self, host, aws_access_key_id=None, - aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, path='/', - provider='aws', security_token=None, - suppress_consec_slashes=True, - validate_certs=True, profile_name=None): - """ - :type host: str - :param host: The host to make the connection to - - :keyword str aws_access_key_id: Your AWS Access Key ID (provided by - Amazon). If none is specified, the value in your - ``AWS_ACCESS_KEY_ID`` environmental variable is used. - :keyword str aws_secret_access_key: Your AWS Secret Access Key - (provided by Amazon). If none is specified, the value in your - ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. - :keyword str security_token: The security token associated with - temporary credentials issued by STS. Optional unless using - temporary credentials. If none is specified, the environment - variable ``AWS_SECURITY_TOKEN`` is used if defined. - - :type is_secure: boolean - :param is_secure: Whether the connection is over SSL - - :type https_connection_factory: list or tuple - :param https_connection_factory: A pair of an HTTP connection - factory and the exceptions to catch. The factory should have - a similar interface to L{http_client.HTTPSConnection}. - - :param str proxy: Address/hostname for a proxy server - - :type proxy_port: int - :param proxy_port: The port to use when connecting over a proxy - - :type proxy_user: str - :param proxy_user: The username to connect with on the proxy - - :type proxy_pass: str - :param proxy_pass: The password to use when connection over a proxy. - - :type port: int - :param port: The port to use to connect - - :type suppress_consec_slashes: bool - :param suppress_consec_slashes: If provided, controls whether - consecutive slashes will be suppressed in key paths. - - :type validate_certs: bool - :param validate_certs: Controls whether SSL certificates - will be validated or not. Defaults to True. - - :type profile_name: str - :param profile_name: Override usual Credentials section in config - file to use a named set of keys instead. - """ - self.suppress_consec_slashes = suppress_consec_slashes - self.num_retries = 6 - # Override passed-in is_secure setting if value was defined in config. - if config.has_option('Boto', 'is_secure'): - is_secure = config.getboolean('Boto', 'is_secure') - self.is_secure = is_secure - # Whether or not to validate server certificates. - # The default is now to validate certificates. This can be - # overridden in the boto config file are by passing an - # explicit validate_certs parameter to the class constructor. - self.https_validate_certificates = config.getbool( - 'Boto', 'https_validate_certificates', - validate_certs) - if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: - raise BotoClientError( - "SSL server certificate validation is enabled in boto " - "configuration, but Python dependencies required to " - "support this feature are not available. Certificate " - "validation is only supported when running under Python " - "2.6 or later.") - certs_file = config.get_value( - 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) - if certs_file == 'system': - certs_file = None - self.ca_certificates_file = certs_file - if port: - self.port = port - else: - self.port = PORTS_BY_SECURITY[is_secure] - - self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) - # define exceptions from http_client that we want to catch and retry - self.http_exceptions = (http_client.HTTPException, socket.error, - socket.gaierror, http_client.BadStatusLine) - # define subclasses of the above that are not retryable. - self.http_unretryable_exceptions = [] - if HAVE_HTTPS_CONNECTION: - self.http_unretryable_exceptions.append( - https_connection.InvalidCertificateException) - - # define values in socket exceptions we don't want to catch - self.socket_exception_values = (errno.EINTR,) - if https_connection_factory is not None: - self.https_connection_factory = https_connection_factory[0] - self.http_exceptions += https_connection_factory[1] - else: - self.https_connection_factory = None - if (is_secure): - self.protocol = 'https' - else: - self.protocol = 'http' - self.host = host - self.path = path - # if the value passed in for debug - if not isinstance(debug, six.integer_types): - debug = 0 - self.debug = config.getint('Boto', 'debug', debug) - self.host_header = None - - # Timeout used to tell http_client how long to wait for socket timeouts. - # Default is to leave timeout unchanged, which will in turn result in - # the socket's default global timeout being used. To specify a - # timeout, set http_socket_timeout in Boto config. Regardless, - # timeouts will only be applied if Python is 2.6 or greater. - self.http_connection_kwargs = {} - if (sys.version_info[0], sys.version_info[1]) >= (2, 6): - # If timeout isn't defined in boto config file, use 70 second - # default as recommended by - # http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html - self.http_connection_kwargs['timeout'] = config.getint( - 'Boto', 'http_socket_timeout', 70) - - if isinstance(provider, Provider): - # Allow overriding Provider - self.provider = provider - else: - self._provider_type = provider - self.provider = Provider(self._provider_type, - aws_access_key_id, - aws_secret_access_key, - security_token, - profile_name) - - # Allow config file to override default host, port, and host header. - if self.provider.host: - self.host = self.provider.host - if self.provider.port: - self.port = self.provider.port - if self.provider.host_header: - self.host_header = self.provider.host_header - - self._pool = ConnectionPool() - self._connection = (self.host, self.port, self.is_secure) - self._last_rs = None - self._auth_handler = auth.get_auth_handler( - host, config, self.provider, self._required_auth_capability()) - if getattr(self, 'AuthServiceName', None) is not None: - self.auth_service_name = self.AuthServiceName - self.request_hook = None - - def __repr__(self): - return '%s:%s' % (self.__class__.__name__, self.host) - - def _required_auth_capability(self): - return [] - - def _get_auth_service_name(self): - return getattr(self._auth_handler, 'service_name') - - # For Sigv4, the auth_service_name/auth_region_name properties allow - # the service_name/region_name to be explicitly set instead of being - # derived from the endpoint url. - def _set_auth_service_name(self, value): - self._auth_handler.service_name = value - auth_service_name = property(_get_auth_service_name, _set_auth_service_name) - - def _get_auth_region_name(self): - return getattr(self._auth_handler, 'region_name') - - def _set_auth_region_name(self, value): - self._auth_handler.region_name = value - auth_region_name = property(_get_auth_region_name, _set_auth_region_name) - - def connection(self): - return self.get_http_connection(*self._connection) - connection = property(connection) - - def aws_access_key_id(self): - return self.provider.access_key - aws_access_key_id = property(aws_access_key_id) - gs_access_key_id = aws_access_key_id - access_key = aws_access_key_id - - def aws_secret_access_key(self): - return self.provider.secret_key - aws_secret_access_key = property(aws_secret_access_key) - gs_secret_access_key = aws_secret_access_key - secret_key = aws_secret_access_key - - def profile_name(self): - return self.provider.profile_name - profile_name = property(profile_name) - - def get_path(self, path='/'): - # The default behavior is to suppress consecutive slashes for reasons - # discussed at - # https://groups.google.com/forum/#!topic/boto-dev/-ft0XPUy0y8 - # You can override that behavior with the suppress_consec_slashes param. - if not self.suppress_consec_slashes: - return self.path + re.sub('^(/*)/', "\\1", path) - pos = path.find('?') - if pos >= 0: - params = path[pos:] - path = path[:pos] - else: - params = None - if path[-1] == '/': - need_trailing = True - else: - need_trailing = False - path_elements = self.path.split('/') - path_elements.extend(path.split('/')) - path_elements = [p for p in path_elements if p] - path = '/' + '/'.join(path_elements) - if path[-1] != '/' and need_trailing: - path += '/' - if params: - path = path + params - return path - - def server_name(self, port=None): - if not port: - port = self.port - if port == 80: - signature_host = self.host - else: - # This unfortunate little hack can be attributed to - # a difference in the 2.6 version of http_client. In old - # versions, it would append ":443" to the hostname sent - # in the Host header and so we needed to make sure we - # did the same when calculating the V2 signature. In 2.6 - # (and higher!) - # it no longer does that. Hence, this kludge. - if ((ON_APP_ENGINE and sys.version[:3] == '2.5') or - sys.version[:3] in ('2.6', '2.7')) and port == 443: - signature_host = self.host - else: - signature_host = '%s:%d' % (self.host, port) - return signature_host - - def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): - self.proxy = proxy - self.proxy_port = proxy_port - self.proxy_user = proxy_user - self.proxy_pass = proxy_pass - if 'http_proxy' in os.environ and not self.proxy: - pattern = re.compile( - '(?:http://)?' - '(?:(?P<user>[\w\-\.]+):(?P<pass>.*)@)?' - '(?P<host>[\w\-\.]+)' - '(?::(?P<port>\d+))?' - ) - match = pattern.match(os.environ['http_proxy']) - if match: - self.proxy = match.group('host') - self.proxy_port = match.group('port') - self.proxy_user = match.group('user') - self.proxy_pass = match.group('pass') - else: - if not self.proxy: - self.proxy = config.get_value('Boto', 'proxy', None) - if not self.proxy_port: - self.proxy_port = config.get_value('Boto', 'proxy_port', None) - if not self.proxy_user: - self.proxy_user = config.get_value('Boto', 'proxy_user', None) - if not self.proxy_pass: - self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) - - if not self.proxy_port and self.proxy: - print("http_proxy environment variable does not specify " - "a port, using default") - self.proxy_port = self.port - - self.no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '') - self.use_proxy = (self.proxy is not None) - - def get_http_connection(self, host, port, is_secure): - conn = self._pool.get_http_connection(host, port, is_secure) - if conn is not None: - return conn - else: - return self.new_http_connection(host, port, is_secure) - - def skip_proxy(self, host): - if not self.no_proxy: - return False - - if self.no_proxy == "*": - return True - - hostonly = host - hostonly = host.split(':')[0] - - for name in self.no_proxy.split(','): - if name and (hostonly.endswith(name) or host.endswith(name)): - return True - - return False - - def new_http_connection(self, host, port, is_secure): - if host is None: - host = self.server_name() - - # Make sure the host is really just the host, not including - # the port number - host = boto.utils.parse_host(host) - - http_connection_kwargs = self.http_connection_kwargs.copy() - - # Connection factories below expect a port keyword argument - http_connection_kwargs['port'] = port - - # Override host with proxy settings if needed - if self.use_proxy and not is_secure and \ - not self.skip_proxy(host): - host = self.proxy - http_connection_kwargs['port'] = int(self.proxy_port) - - if is_secure: - boto.log.debug( - 'establishing HTTPS connection: host=%s, kwargs=%s', - host, http_connection_kwargs) - if self.use_proxy and not self.skip_proxy(host): - connection = self.proxy_ssl(host, is_secure and 443 or 80) - elif self.https_connection_factory: - connection = self.https_connection_factory(host) - elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION: - connection = https_connection.CertValidatingHTTPSConnection( - host, ca_certs=self.ca_certificates_file, - **http_connection_kwargs) - else: - connection = http_client.HTTPSConnection( - host, **http_connection_kwargs) - else: - boto.log.debug('establishing HTTP connection: kwargs=%s' % - http_connection_kwargs) - if self.https_connection_factory: - # even though the factory says https, this is too handy - # to not be able to allow overriding for http also. - connection = self.https_connection_factory( - host, **http_connection_kwargs) - else: - connection = http_client.HTTPConnection( - host, **http_connection_kwargs) - if self.debug > 1: - connection.set_debuglevel(self.debug) - # self.connection must be maintained for backwards-compatibility - # however, it must be dynamically pulled from the connection pool - # set a private variable which will enable that - if host.split(':')[0] == self.host and is_secure == self.is_secure: - self._connection = (host, port, is_secure) - # Set the response class of the http connection to use our custom - # class. - connection.response_class = HTTPResponse - return connection - - def put_http_connection(self, host, port, is_secure, connection): - self._pool.put_http_connection(host, port, is_secure, connection) - - def proxy_ssl(self, host=None, port=None): - if host and port: - host = '%s:%d' % (host, port) - else: - host = '%s:%d' % (self.host, self.port) - # Seems properly to use timeout for connect too - timeout = self.http_connection_kwargs.get("timeout") - if timeout is not None: - sock = socket.create_connection((self.proxy, - int(self.proxy_port)), timeout) - else: - sock = socket.create_connection((self.proxy, int(self.proxy_port))) - boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host) - sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) - sock.sendall("User-Agent: %s\r\n" % UserAgent) - if self.proxy_user and self.proxy_pass: - for k, v in self.get_proxy_auth_header().items(): - sock.sendall("%s: %s\r\n" % (k, v)) - # See discussion about this config option at - # https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc - if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False): - sock.sendall("\r\n") - else: - sock.sendall("\r\n") - resp = http_client.HTTPResponse(sock, strict=True, debuglevel=self.debug) - resp.begin() - - if resp.status != 200: - # Fake a socket error, use a code that make it obvious it hasn't - # been generated by the socket library - raise socket.error(-71, - "Error talking to HTTP proxy %s:%s: %s (%s)" % - (self.proxy, self.proxy_port, - resp.status, resp.reason)) - - # We can safely close the response, it duped the original socket - resp.close() - - h = http_client.HTTPConnection(host) - - if self.https_validate_certificates and HAVE_HTTPS_CONNECTION: - msg = "wrapping ssl socket for proxied connection; " - if self.ca_certificates_file: - msg += "CA certificate file=%s" % self.ca_certificates_file - else: - msg += "using system provided SSL certs" - boto.log.debug(msg) - key_file = self.http_connection_kwargs.get('key_file', None) - cert_file = self.http_connection_kwargs.get('cert_file', None) - sslSock = ssl.wrap_socket(sock, keyfile=key_file, - certfile=cert_file, - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=self.ca_certificates_file) - cert = sslSock.getpeercert() - hostname = self.host.split(':', 0)[0] - if not https_connection.ValidateCertificateHostname(cert, hostname): - raise https_connection.InvalidCertificateException( - hostname, cert, 'hostname mismatch') - else: - # Fallback for old Python without ssl.wrap_socket - if hasattr(http_client, 'ssl'): - sslSock = http_client.ssl.SSLSocket(sock) - else: - sslSock = socket.ssl(sock, None, None) - sslSock = http_client.FakeSocket(sock, sslSock) - - # This is a bit unclean - h.sock = sslSock - return h - - def prefix_proxy_to_path(self, path, host=None): - path = self.protocol + '://' + (host or self.server_name()) + path - return path - - def get_proxy_auth_header(self): - auth = encodebytes(self.proxy_user + ':' + self.proxy_pass) - return {'Proxy-Authorization': 'Basic %s' % auth} - - # For passing proxy information to other connection libraries, e.g. cloudsearch2 - def get_proxy_url_with_auth(self): - if not self.use_proxy: - return None - - if self.proxy_user or self.proxy_pass: - if self.proxy_pass: - login_info = '%s:%s@' % (self.proxy_user, self.proxy_pass) - else: - login_info = '%s@' % self.proxy_user - else: - login_info = '' - - return 'http://%s%s:%s' % (login_info, self.proxy, str(self.proxy_port or self.port)) - - def set_host_header(self, request): - try: - request.headers['Host'] = \ - self._auth_handler.host_header(self.host, request) - except AttributeError: - request.headers['Host'] = self.host.split(':', 1)[0] - - def set_request_hook(self, hook): - self.request_hook = hook - - def _mexe(self, request, sender=None, override_num_retries=None, - retry_handler=None): - """ - mexe - Multi-execute inside a loop, retrying multiple times to handle - transient Internet errors by simply trying again. - Also handles redirects. - - This code was inspired by the S3Utils classes posted to the boto-users - Google group by Larry Bates. Thanks! - - """ - boto.log.debug('Method: %s' % request.method) - boto.log.debug('Path: %s' % request.path) - boto.log.debug('Data: %s' % request.body) - boto.log.debug('Headers: %s' % request.headers) - boto.log.debug('Host: %s' % request.host) - boto.log.debug('Port: %s' % request.port) - boto.log.debug('Params: %s' % request.params) - response = None - body = None - ex = None - if override_num_retries is None: - num_retries = config.getint('Boto', 'num_retries', self.num_retries) - else: - num_retries = override_num_retries - i = 0 - connection = self.get_http_connection(request.host, request.port, - self.is_secure) - - # Convert body to bytes if needed - if not isinstance(request.body, bytes) and hasattr(request.body, - 'encode'): - request.body = request.body.encode('utf-8') - - while i <= num_retries: - # Use binary exponential backoff to desynchronize client requests. - next_sleep = min(random.random() * (2 ** i), - boto.config.get('Boto', 'max_retry_delay', 60)) - try: - # we now re-sign each request before it is retried - boto.log.debug('Token: %s' % self.provider.security_token) - request.authorize(connection=self) - # Only force header for non-s3 connections, because s3 uses - # an older signing method + bucket resource URLs that include - # the port info. All others should be now be up to date and - # not include the port. - if 's3' not in self._required_auth_capability(): - if not getattr(self, 'anon', False): - if not request.headers.get('Host'): - self.set_host_header(request) - boto.log.debug('Final headers: %s' % request.headers) - request.start_time = datetime.now() - if callable(sender): - response = sender(connection, request.method, request.path, - request.body, request.headers) - else: - connection.request(request.method, request.path, - request.body, request.headers) - response = connection.getresponse() - boto.log.debug('Response headers: %s' % response.getheaders()) - location = response.getheader('location') - # -- gross hack -- - # http_client gets confused with chunked responses to HEAD requests - # so I have to fake it out - if request.method == 'HEAD' and getattr(response, - 'chunked', False): - response.chunked = 0 - if callable(retry_handler): - status = retry_handler(response, i, next_sleep) - if status: - msg, i, next_sleep = status - if msg: - boto.log.debug(msg) - time.sleep(next_sleep) - continue - if response.status in [500, 502, 503, 504]: - msg = 'Received %d response. ' % response.status - msg += 'Retrying in %3.1f seconds' % next_sleep - boto.log.debug(msg) - body = response.read() - if isinstance(body, bytes): - body = body.decode('utf-8') - elif response.status < 300 or response.status >= 400 or \ - not location: - # don't return connection to the pool if response contains - # Connection:close header, because the connection has been - # closed and default reconnect behavior may do something - # different than new_http_connection. Also, it's probably - # less efficient to try to reuse a closed connection. - conn_header_value = response.getheader('connection') - if conn_header_value == 'close': - connection.close() - else: - self.put_http_connection(request.host, request.port, - self.is_secure, connection) - if self.request_hook is not None: - self.request_hook.handle_request_data(request, response) - return response - else: - scheme, request.host, request.path, \ - params, query, fragment = urlparse(location) - if query: - request.path += '?' + query - # urlparse can return both host and port in netloc, so if - # that's the case we need to split them up properly - if ':' in request.host: - request.host, request.port = request.host.split(':', 1) - msg = 'Redirecting: %s' % scheme + '://' - msg += request.host + request.path - boto.log.debug(msg) - connection = self.get_http_connection(request.host, - request.port, - scheme == 'https') - response = None - continue - except PleaseRetryException as e: - boto.log.debug('encountered a retry exception: %s' % e) - connection = self.new_http_connection(request.host, request.port, - self.is_secure) - response = e.response - ex = e - except self.http_exceptions as e: - for unretryable in self.http_unretryable_exceptions: - if isinstance(e, unretryable): - boto.log.debug( - 'encountered unretryable %s exception, re-raising' % - e.__class__.__name__) - raise - boto.log.debug('encountered %s exception, reconnecting' % - e.__class__.__name__) - connection = self.new_http_connection(request.host, request.port, - self.is_secure) - ex = e - time.sleep(next_sleep) - i += 1 - # If we made it here, it's because we have exhausted our retries - # and stil haven't succeeded. So, if we have a response object, - # use it to raise an exception. - # Otherwise, raise the exception that must have already happened. - if self.request_hook is not None: - self.request_hook.handle_request_data(request, response, error=True) - if response: - raise BotoServerError(response.status, response.reason, body) - elif ex: - raise ex - else: - msg = 'Please report this exception as a Boto Issue!' - raise BotoClientError(msg) - - def build_base_http_request(self, method, path, auth_path, - params=None, headers=None, data='', host=None): - path = self.get_path(path) - if auth_path is not None: - auth_path = self.get_path(auth_path) - if params is None: - params = {} - else: - params = params.copy() - if headers is None: - headers = {} - else: - headers = headers.copy() - if self.host_header and not boto.utils.find_matching_headers('host', headers): - headers['host'] = self.host_header - host = host or self.host - if self.use_proxy and not self.skip_proxy(host): - if not auth_path: - auth_path = path - path = self.prefix_proxy_to_path(path, host) - if self.proxy_user and self.proxy_pass and not self.is_secure: - # If is_secure, we don't have to set the proxy authentication - # header here, we did that in the CONNECT to the proxy. - headers.update(self.get_proxy_auth_header()) - return HTTPRequest(method, self.protocol, host, self.port, - path, auth_path, params, headers, data) - - def make_request(self, method, path, headers=None, data='', host=None, - auth_path=None, sender=None, override_num_retries=None, - params=None, retry_handler=None): - """Makes a request to the server, with stock multiple-retry logic.""" - if params is None: - params = {} - http_request = self.build_base_http_request(method, path, auth_path, - params, headers, data, host) - return self._mexe(http_request, sender, override_num_retries, - retry_handler=retry_handler) - - def close(self): - """(Optional) Close any open HTTP connections. This is non-destructive, - and making a new request will open a connection again.""" - - boto.log.debug('closing all HTTP connections') - self._connection = None # compat field - - -class AWSQueryConnection(AWSAuthConnection): - - APIVersion = '' - ResponseError = BotoServerError - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, host=None, debug=0, - https_connection_factory=None, path='/', security_token=None, - validate_certs=True, profile_name=None, provider='aws'): - super(AWSQueryConnection, self).__init__( - host, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, - proxy_port, proxy_user, proxy_pass, - debug, https_connection_factory, path, - security_token=security_token, - validate_certs=validate_certs, - profile_name=profile_name, - provider=provider) - - def _required_auth_capability(self): - return [] - - def get_utf8_value(self, value): - return boto.utils.get_utf8_value(value) - - def make_request(self, action, params=None, path='/', verb='GET'): - http_request = self.build_base_http_request(verb, path, None, - params, {}, '', - self.host) - if action: - http_request.params['Action'] = action - if self.APIVersion: - http_request.params['Version'] = self.APIVersion - return self._mexe(http_request) - - def build_list_params(self, params, items, label): - if isinstance(items, six.string_types): - items = [items] - for i in range(1, len(items) + 1): - params['%s.%d' % (label, i)] = items[i - 1] - - def build_complex_list_params(self, params, items, label, names): - """Serialize a list of structures. - - For example:: - - items = [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')] - label = 'ParamName.member' - names = ('One', 'Two', 'Three') - self.build_complex_list_params(params, items, label, names) - - would result in the params dict being updated with these params:: - - ParamName.member.1.One = foo - ParamName.member.1.Two = bar - ParamName.member.1.Three = baz - - ParamName.member.2.One = foo2 - ParamName.member.2.Two = bar2 - ParamName.member.2.Three = baz2 - - :type params: dict - :param params: The params dict. The complex list params - will be added to this dict. - - :type items: list of tuples - :param items: The list to serialize. - - :type label: string - :param label: The prefix to apply to the parameter. - - :type names: tuple of strings - :param names: The names associated with each tuple element. - - """ - for i, item in enumerate(items, 1): - current_prefix = '%s.%s' % (label, i) - for key, value in zip(names, item): - full_key = '%s.%s' % (current_prefix, key) - params[full_key] = value - - # generics - - def get_list(self, action, params, markers, path='/', - parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if not body: - boto.log.error('Null body %s' % body) - raise self.ResponseError(response.status, response.reason, body) - elif response.status == 200: - rs = ResultSet(markers) - h = boto.handler.XmlHandler(rs, parent) - if isinstance(body, six.text_type): - body = body.encode('utf-8') - xml.sax.parseString(body, h) - return rs - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_object(self, action, params, cls, path='/', - parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if not body: - boto.log.error('Null body %s' % body) - raise self.ResponseError(response.status, response.reason, body) - elif response.status == 200: - obj = cls(parent) - h = boto.handler.XmlHandler(obj, parent) - if isinstance(body, six.text_type): - body = body.encode('utf-8') - xml.sax.parseString(body, h) - return obj - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_status(self, action, params, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if not body: - boto.log.error('Null body %s' % body) - raise self.ResponseError(response.status, response.reason, body) - elif response.status == 200: - rs = ResultSet() - h = boto.handler.XmlHandler(rs, parent) - xml.sax.parseString(body, h) - return rs.status - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body)