Mercurial > repos > shellac > guppy_basecaller
diff env/lib/python3.7/site-packages/boto/connection.py @ 0:26e78fe6e8c4 draft
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author | shellac |
---|---|
date | Sat, 02 May 2020 07:14:21 -0400 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/env/lib/python3.7/site-packages/boto/connection.py Sat May 02 07:14:21 2020 -0400 @@ -0,0 +1,1227 @@ +# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. +# Copyright (c) 2010 Google +# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2009 The Echo Nest Corporation +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# Copyright (c) 2011, Nexenta Systems Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Handles basic connections to AWS +""" +from datetime import datetime +import errno +import os +import random +import re +import socket +import sys +import time +import xml.sax +import copy + +from boto import auth +from boto import auth_handler +import boto +import boto.utils +import boto.handler +import boto.cacerts + +from boto import config, UserAgent +from boto.compat import six, http_client, urlparse, quote, encodebytes +from boto.exception import AWSConnectionError +from boto.exception import BotoClientError +from boto.exception import BotoServerError +from boto.exception import PleaseRetryException +from boto.provider import Provider +from boto.resultset import ResultSet + +HAVE_HTTPS_CONNECTION = False +try: + import ssl + from boto import https_connection + # Google App Engine runs on Python 2.5 so doesn't have ssl.SSLError. + if hasattr(ssl, 'SSLError'): + HAVE_HTTPS_CONNECTION = True +except ImportError: + pass + +try: + import threading +except ImportError: + import dummy_threading as threading + +ON_APP_ENGINE = all(key in os.environ for key in ( + 'USER_IS_ADMIN', 'CURRENT_VERSION_ID', 'APPLICATION_ID')) + +PORTS_BY_SECURITY = {True: 443, + False: 80} + +DEFAULT_CA_CERTS_FILE = os.path.join(os.path.dirname(os.path.abspath(boto.cacerts.__file__)), "cacerts.txt") + + +class HostConnectionPool(object): + + """ + A pool of connections for one remote (host,port,is_secure). + + When connections are added to the pool, they are put into a + pending queue. The _mexe method returns connections to the pool + before the response body has been read, so they connections aren't + ready to send another request yet. They stay in the pending queue + until they are ready for another request, at which point they are + returned to the pool of ready connections. + + The pool of ready connections is an ordered list of + (connection,time) pairs, where the time is the time the connection + was returned from _mexe. After a certain period of time, + connections are considered stale, and discarded rather than being + reused. This saves having to wait for the connection to time out + if AWS has decided to close it on the other end because of + inactivity. + + Thread Safety: + + This class is used only from ConnectionPool while it's mutex + is held. + """ + + def __init__(self): + self.queue = [] + + def size(self): + """ + Returns the number of connections in the pool for this host. + Some of the connections may still be in use, and may not be + ready to be returned by get(). + """ + return len(self.queue) + + def put(self, conn): + """ + Adds a connection to the pool, along with the time it was + added. + """ + self.queue.append((conn, time.time())) + + def get(self): + """ + Returns the next connection in this pool that is ready to be + reused. Returns None if there aren't any. + """ + # Discard ready connections that are too old. + self.clean() + + # Return the first connection that is ready, and remove it + # from the queue. Connections that aren't ready are returned + # to the end of the queue with an updated time, on the + # assumption that somebody is actively reading the response. + for _ in range(len(self.queue)): + (conn, _) = self.queue.pop(0) + if self._conn_ready(conn): + return conn + else: + self.put(conn) + return None + + def _conn_ready(self, conn): + """ + There is a nice state diagram at the top of http_client.py. It + indicates that once the response headers have been read (which + _mexe does before adding the connection to the pool), a + response is attached to the connection, and it stays there + until it's done reading. This isn't entirely true: even after + the client is done reading, the response may be closed, but + not removed from the connection yet. + + This is ugly, reading a private instance variable, but the + state we care about isn't available in any public methods. + """ + if ON_APP_ENGINE: + # Google AppEngine implementation of HTTPConnection doesn't contain + # _HTTPConnection__response attribute. Moreover, it's not possible + # to determine if given connection is ready. Reusing connections + # simply doesn't make sense with App Engine urlfetch service. + return False + else: + response = getattr(conn, '_HTTPConnection__response', None) + return (response is None) or response.isclosed() + + def clean(self): + """ + Get rid of stale connections. + """ + # Note that we do not close the connection here -- somebody + # may still be reading from it. + while len(self.queue) > 0 and self._pair_stale(self.queue[0]): + self.queue.pop(0) + + def _pair_stale(self, pair): + """ + Returns true of the (connection,time) pair is too old to be + used. + """ + (_conn, return_time) = pair + now = time.time() + return return_time + ConnectionPool.STALE_DURATION < now + + +class ConnectionPool(object): + + """ + A connection pool that expires connections after a fixed period of + time. This saves time spent waiting for a connection that AWS has + timed out on the other end. + + This class is thread-safe. + """ + + # + # The amout of time between calls to clean. + # + + CLEAN_INTERVAL = 5.0 + + # + # How long before a connection becomes "stale" and won't be reused + # again. The intention is that this time is less that the timeout + # period that AWS uses, so we'll never try to reuse a connection + # and find that AWS is timing it out. + # + # Experimentation in July 2011 shows that AWS starts timing things + # out after three minutes. The 60 seconds here is conservative so + # we should never hit that 3-minute timout. + # + + STALE_DURATION = 60.0 + + def __init__(self): + # Mapping from (host,port,is_secure) to HostConnectionPool. + # If a pool becomes empty, it is removed. + self.host_to_pool = {} + # The last time the pool was cleaned. + self.last_clean_time = 0.0 + self.mutex = threading.Lock() + ConnectionPool.STALE_DURATION = \ + config.getfloat('Boto', 'connection_stale_duration', + ConnectionPool.STALE_DURATION) + + def __getstate__(self): + pickled_dict = copy.copy(self.__dict__) + pickled_dict['host_to_pool'] = {} + del pickled_dict['mutex'] + return pickled_dict + + def __setstate__(self, dct): + self.__init__() + + def size(self): + """ + Returns the number of connections in the pool. + """ + return sum(pool.size() for pool in self.host_to_pool.values()) + + def get_http_connection(self, host, port, is_secure): + """ + Gets a connection from the pool for the named host. Returns + None if there is no connection that can be reused. It's the caller's + responsibility to call close() on the connection when it's no longer + needed. + """ + self.clean() + with self.mutex: + key = (host, port, is_secure) + if key not in self.host_to_pool: + return None + return self.host_to_pool[key].get() + + def put_http_connection(self, host, port, is_secure, conn): + """ + Adds a connection to the pool of connections that can be + reused for the named host. + """ + with self.mutex: + key = (host, port, is_secure) + if key not in self.host_to_pool: + self.host_to_pool[key] = HostConnectionPool() + self.host_to_pool[key].put(conn) + + def clean(self): + """ + Clean up the stale connections in all of the pools, and then + get rid of empty pools. Pools clean themselves every time a + connection is fetched; this cleaning takes care of pools that + aren't being used any more, so nothing is being gotten from + them. + """ + with self.mutex: + now = time.time() + if self.last_clean_time + self.CLEAN_INTERVAL < now: + to_remove = [] + for (host, pool) in self.host_to_pool.items(): + pool.clean() + if pool.size() == 0: + to_remove.append(host) + for host in to_remove: + del self.host_to_pool[host] + self.last_clean_time = now + + +class HTTPRequest(object): + + def __init__(self, method, protocol, host, port, path, auth_path, + params, headers, body): + """Represents an HTTP request. + + :type method: string + :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. + + :type protocol: string + :param protocol: The http protocol used, 'http' or 'https'. + + :type host: string + :param host: Host to which the request is addressed. eg. abc.com + + :type port: int + :param port: port on which the request is being sent. Zero means unset, + in which case default port will be chosen. + + :type path: string + :param path: URL path that is being accessed. + + :type auth_path: string + :param path: The part of the URL path used when creating the + authentication string. + + :type params: dict + :param params: HTTP url query parameters, with key as name of + the param, and value as value of param. + + :type headers: dict + :param headers: HTTP headers, with key as name of the header and value + as value of header. + + :type body: string + :param body: Body of the HTTP request. If not present, will be None or + empty string (''). + """ + self.method = method + self.protocol = protocol + self.host = host + self.port = port + self.path = path + if auth_path is None: + auth_path = path + self.auth_path = auth_path + self.params = params + # chunked Transfer-Encoding should act only on PUT request. + if headers and 'Transfer-Encoding' in headers and \ + headers['Transfer-Encoding'] == 'chunked' and \ + self.method != 'PUT': + self.headers = headers.copy() + del self.headers['Transfer-Encoding'] + else: + self.headers = headers + self.body = body + + def __str__(self): + return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) ' + 'params(%s) headers(%s) body(%s)') % (self.method, + self.protocol, self.host, self.port, self.path, self.params, + self.headers, self.body)) + + def authorize(self, connection, **kwargs): + if not getattr(self, '_headers_quoted', False): + for key in self.headers: + val = self.headers[key] + if isinstance(val, six.text_type): + safe = '!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~ ' + self.headers[key] = quote(val.encode('utf-8'), safe) + setattr(self, '_headers_quoted', True) + + self.headers['User-Agent'] = UserAgent + + connection._auth_handler.add_auth(self, **kwargs) + + # I'm not sure if this is still needed, now that add_auth is + # setting the content-length for POST requests. + if 'Content-Length' not in self.headers: + if 'Transfer-Encoding' not in self.headers or \ + self.headers['Transfer-Encoding'] != 'chunked': + self.headers['Content-Length'] = str(len(self.body)) + + +class HTTPResponse(http_client.HTTPResponse): + + def __init__(self, *args, **kwargs): + http_client.HTTPResponse.__init__(self, *args, **kwargs) + self._cached_response = '' + + def read(self, amt=None): + """Read the response. + + This method does not have the same behavior as + http_client.HTTPResponse.read. Instead, if this method is called with + no ``amt`` arg, then the response body will be cached. Subsequent + calls to ``read()`` with no args **will return the cached response**. + + """ + if amt is None: + # The reason for doing this is that many places in boto call + # response.read() and except to get the response body that they + # can then process. To make sure this always works as they expect + # we're caching the response so that multiple calls to read() + # will return the full body. Note that this behavior only + # happens if the amt arg is not specified. + if not self._cached_response: + self._cached_response = http_client.HTTPResponse.read(self) + return self._cached_response + else: + return http_client.HTTPResponse.read(self, amt) + + +class AWSAuthConnection(object): + def __init__(self, host, aws_access_key_id=None, + aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, path='/', + provider='aws', security_token=None, + suppress_consec_slashes=True, + validate_certs=True, profile_name=None): + """ + :type host: str + :param host: The host to make the connection to + + :keyword str aws_access_key_id: Your AWS Access Key ID (provided by + Amazon). If none is specified, the value in your + ``AWS_ACCESS_KEY_ID`` environmental variable is used. + :keyword str aws_secret_access_key: Your AWS Secret Access Key + (provided by Amazon). If none is specified, the value in your + ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. + :keyword str security_token: The security token associated with + temporary credentials issued by STS. Optional unless using + temporary credentials. If none is specified, the environment + variable ``AWS_SECURITY_TOKEN`` is used if defined. + + :type is_secure: boolean + :param is_secure: Whether the connection is over SSL + + :type https_connection_factory: list or tuple + :param https_connection_factory: A pair of an HTTP connection + factory and the exceptions to catch. The factory should have + a similar interface to L{http_client.HTTPSConnection}. + + :param str proxy: Address/hostname for a proxy server + + :type proxy_port: int + :param proxy_port: The port to use when connecting over a proxy + + :type proxy_user: str + :param proxy_user: The username to connect with on the proxy + + :type proxy_pass: str + :param proxy_pass: The password to use when connection over a proxy. + + :type port: int + :param port: The port to use to connect + + :type suppress_consec_slashes: bool + :param suppress_consec_slashes: If provided, controls whether + consecutive slashes will be suppressed in key paths. + + :type validate_certs: bool + :param validate_certs: Controls whether SSL certificates + will be validated or not. Defaults to True. + + :type profile_name: str + :param profile_name: Override usual Credentials section in config + file to use a named set of keys instead. + """ + self.suppress_consec_slashes = suppress_consec_slashes + self.num_retries = 6 + # Override passed-in is_secure setting if value was defined in config. + if config.has_option('Boto', 'is_secure'): + is_secure = config.getboolean('Boto', 'is_secure') + self.is_secure = is_secure + # Whether or not to validate server certificates. + # The default is now to validate certificates. This can be + # overridden in the boto config file are by passing an + # explicit validate_certs parameter to the class constructor. + self.https_validate_certificates = config.getbool( + 'Boto', 'https_validate_certificates', + validate_certs) + if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: + raise BotoClientError( + "SSL server certificate validation is enabled in boto " + "configuration, but Python dependencies required to " + "support this feature are not available. Certificate " + "validation is only supported when running under Python " + "2.6 or later.") + certs_file = config.get_value( + 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) + if certs_file == 'system': + certs_file = None + self.ca_certificates_file = certs_file + if port: + self.port = port + else: + self.port = PORTS_BY_SECURITY[is_secure] + + self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) + # define exceptions from http_client that we want to catch and retry + self.http_exceptions = (http_client.HTTPException, socket.error, + socket.gaierror, http_client.BadStatusLine) + # define subclasses of the above that are not retryable. + self.http_unretryable_exceptions = [] + if HAVE_HTTPS_CONNECTION: + self.http_unretryable_exceptions.append( + https_connection.InvalidCertificateException) + + # define values in socket exceptions we don't want to catch + self.socket_exception_values = (errno.EINTR,) + if https_connection_factory is not None: + self.https_connection_factory = https_connection_factory[0] + self.http_exceptions += https_connection_factory[1] + else: + self.https_connection_factory = None + if (is_secure): + self.protocol = 'https' + else: + self.protocol = 'http' + self.host = host + self.path = path + # if the value passed in for debug + if not isinstance(debug, six.integer_types): + debug = 0 + self.debug = config.getint('Boto', 'debug', debug) + self.host_header = None + + # Timeout used to tell http_client how long to wait for socket timeouts. + # Default is to leave timeout unchanged, which will in turn result in + # the socket's default global timeout being used. To specify a + # timeout, set http_socket_timeout in Boto config. Regardless, + # timeouts will only be applied if Python is 2.6 or greater. + self.http_connection_kwargs = {} + if (sys.version_info[0], sys.version_info[1]) >= (2, 6): + # If timeout isn't defined in boto config file, use 70 second + # default as recommended by + # http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html + self.http_connection_kwargs['timeout'] = config.getint( + 'Boto', 'http_socket_timeout', 70) + + if isinstance(provider, Provider): + # Allow overriding Provider + self.provider = provider + else: + self._provider_type = provider + self.provider = Provider(self._provider_type, + aws_access_key_id, + aws_secret_access_key, + security_token, + profile_name) + + # Allow config file to override default host, port, and host header. + if self.provider.host: + self.host = self.provider.host + if self.provider.port: + self.port = self.provider.port + if self.provider.host_header: + self.host_header = self.provider.host_header + + self._pool = ConnectionPool() + self._connection = (self.host, self.port, self.is_secure) + self._last_rs = None + self._auth_handler = auth.get_auth_handler( + host, config, self.provider, self._required_auth_capability()) + if getattr(self, 'AuthServiceName', None) is not None: + self.auth_service_name = self.AuthServiceName + self.request_hook = None + + def __repr__(self): + return '%s:%s' % (self.__class__.__name__, self.host) + + def _required_auth_capability(self): + return [] + + def _get_auth_service_name(self): + return getattr(self._auth_handler, 'service_name') + + # For Sigv4, the auth_service_name/auth_region_name properties allow + # the service_name/region_name to be explicitly set instead of being + # derived from the endpoint url. + def _set_auth_service_name(self, value): + self._auth_handler.service_name = value + auth_service_name = property(_get_auth_service_name, _set_auth_service_name) + + def _get_auth_region_name(self): + return getattr(self._auth_handler, 'region_name') + + def _set_auth_region_name(self, value): + self._auth_handler.region_name = value + auth_region_name = property(_get_auth_region_name, _set_auth_region_name) + + def connection(self): + return self.get_http_connection(*self._connection) + connection = property(connection) + + def aws_access_key_id(self): + return self.provider.access_key + aws_access_key_id = property(aws_access_key_id) + gs_access_key_id = aws_access_key_id + access_key = aws_access_key_id + + def aws_secret_access_key(self): + return self.provider.secret_key + aws_secret_access_key = property(aws_secret_access_key) + gs_secret_access_key = aws_secret_access_key + secret_key = aws_secret_access_key + + def profile_name(self): + return self.provider.profile_name + profile_name = property(profile_name) + + def get_path(self, path='/'): + # The default behavior is to suppress consecutive slashes for reasons + # discussed at + # https://groups.google.com/forum/#!topic/boto-dev/-ft0XPUy0y8 + # You can override that behavior with the suppress_consec_slashes param. + if not self.suppress_consec_slashes: + return self.path + re.sub('^(/*)/', "\\1", path) + pos = path.find('?') + if pos >= 0: + params = path[pos:] + path = path[:pos] + else: + params = None + if path[-1] == '/': + need_trailing = True + else: + need_trailing = False + path_elements = self.path.split('/') + path_elements.extend(path.split('/')) + path_elements = [p for p in path_elements if p] + path = '/' + '/'.join(path_elements) + if path[-1] != '/' and need_trailing: + path += '/' + if params: + path = path + params + return path + + def server_name(self, port=None): + if not port: + port = self.port + if port == 80: + signature_host = self.host + else: + # This unfortunate little hack can be attributed to + # a difference in the 2.6 version of http_client. In old + # versions, it would append ":443" to the hostname sent + # in the Host header and so we needed to make sure we + # did the same when calculating the V2 signature. In 2.6 + # (and higher!) + # it no longer does that. Hence, this kludge. + if ((ON_APP_ENGINE and sys.version[:3] == '2.5') or + sys.version[:3] in ('2.6', '2.7')) and port == 443: + signature_host = self.host + else: + signature_host = '%s:%d' % (self.host, port) + return signature_host + + def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): + self.proxy = proxy + self.proxy_port = proxy_port + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass + if 'http_proxy' in os.environ and not self.proxy: + pattern = re.compile( + '(?:http://)?' + '(?:(?P<user>[\w\-\.]+):(?P<pass>.*)@)?' + '(?P<host>[\w\-\.]+)' + '(?::(?P<port>\d+))?' + ) + match = pattern.match(os.environ['http_proxy']) + if match: + self.proxy = match.group('host') + self.proxy_port = match.group('port') + self.proxy_user = match.group('user') + self.proxy_pass = match.group('pass') + else: + if not self.proxy: + self.proxy = config.get_value('Boto', 'proxy', None) + if not self.proxy_port: + self.proxy_port = config.get_value('Boto', 'proxy_port', None) + if not self.proxy_user: + self.proxy_user = config.get_value('Boto', 'proxy_user', None) + if not self.proxy_pass: + self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) + + if not self.proxy_port and self.proxy: + print("http_proxy environment variable does not specify " + "a port, using default") + self.proxy_port = self.port + + self.no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '') + self.use_proxy = (self.proxy is not None) + + def get_http_connection(self, host, port, is_secure): + conn = self._pool.get_http_connection(host, port, is_secure) + if conn is not None: + return conn + else: + return self.new_http_connection(host, port, is_secure) + + def skip_proxy(self, host): + if not self.no_proxy: + return False + + if self.no_proxy == "*": + return True + + hostonly = host + hostonly = host.split(':')[0] + + for name in self.no_proxy.split(','): + if name and (hostonly.endswith(name) or host.endswith(name)): + return True + + return False + + def new_http_connection(self, host, port, is_secure): + if host is None: + host = self.server_name() + + # Make sure the host is really just the host, not including + # the port number + host = boto.utils.parse_host(host) + + http_connection_kwargs = self.http_connection_kwargs.copy() + + # Connection factories below expect a port keyword argument + http_connection_kwargs['port'] = port + + # Override host with proxy settings if needed + if self.use_proxy and not is_secure and \ + not self.skip_proxy(host): + host = self.proxy + http_connection_kwargs['port'] = int(self.proxy_port) + + if is_secure: + boto.log.debug( + 'establishing HTTPS connection: host=%s, kwargs=%s', + host, http_connection_kwargs) + if self.use_proxy and not self.skip_proxy(host): + connection = self.proxy_ssl(host, is_secure and 443 or 80) + elif self.https_connection_factory: + connection = self.https_connection_factory(host) + elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION: + connection = https_connection.CertValidatingHTTPSConnection( + host, ca_certs=self.ca_certificates_file, + **http_connection_kwargs) + else: + connection = http_client.HTTPSConnection( + host, **http_connection_kwargs) + else: + boto.log.debug('establishing HTTP connection: kwargs=%s' % + http_connection_kwargs) + if self.https_connection_factory: + # even though the factory says https, this is too handy + # to not be able to allow overriding for http also. + connection = self.https_connection_factory( + host, **http_connection_kwargs) + else: + connection = http_client.HTTPConnection( + host, **http_connection_kwargs) + if self.debug > 1: + connection.set_debuglevel(self.debug) + # self.connection must be maintained for backwards-compatibility + # however, it must be dynamically pulled from the connection pool + # set a private variable which will enable that + if host.split(':')[0] == self.host and is_secure == self.is_secure: + self._connection = (host, port, is_secure) + # Set the response class of the http connection to use our custom + # class. + connection.response_class = HTTPResponse + return connection + + def put_http_connection(self, host, port, is_secure, connection): + self._pool.put_http_connection(host, port, is_secure, connection) + + def proxy_ssl(self, host=None, port=None): + if host and port: + host = '%s:%d' % (host, port) + else: + host = '%s:%d' % (self.host, self.port) + # Seems properly to use timeout for connect too + timeout = self.http_connection_kwargs.get("timeout") + if timeout is not None: + sock = socket.create_connection((self.proxy, + int(self.proxy_port)), timeout) + else: + sock = socket.create_connection((self.proxy, int(self.proxy_port))) + boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host) + sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) + sock.sendall("User-Agent: %s\r\n" % UserAgent) + if self.proxy_user and self.proxy_pass: + for k, v in self.get_proxy_auth_header().items(): + sock.sendall("%s: %s\r\n" % (k, v)) + # See discussion about this config option at + # https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc + if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False): + sock.sendall("\r\n") + else: + sock.sendall("\r\n") + resp = http_client.HTTPResponse(sock, strict=True, debuglevel=self.debug) + resp.begin() + + if resp.status != 200: + # Fake a socket error, use a code that make it obvious it hasn't + # been generated by the socket library + raise socket.error(-71, + "Error talking to HTTP proxy %s:%s: %s (%s)" % + (self.proxy, self.proxy_port, + resp.status, resp.reason)) + + # We can safely close the response, it duped the original socket + resp.close() + + h = http_client.HTTPConnection(host) + + if self.https_validate_certificates and HAVE_HTTPS_CONNECTION: + msg = "wrapping ssl socket for proxied connection; " + if self.ca_certificates_file: + msg += "CA certificate file=%s" % self.ca_certificates_file + else: + msg += "using system provided SSL certs" + boto.log.debug(msg) + key_file = self.http_connection_kwargs.get('key_file', None) + cert_file = self.http_connection_kwargs.get('cert_file', None) + sslSock = ssl.wrap_socket(sock, keyfile=key_file, + certfile=cert_file, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=self.ca_certificates_file) + cert = sslSock.getpeercert() + hostname = self.host.split(':', 0)[0] + if not https_connection.ValidateCertificateHostname(cert, hostname): + raise https_connection.InvalidCertificateException( + hostname, cert, 'hostname mismatch') + else: + # Fallback for old Python without ssl.wrap_socket + if hasattr(http_client, 'ssl'): + sslSock = http_client.ssl.SSLSocket(sock) + else: + sslSock = socket.ssl(sock, None, None) + sslSock = http_client.FakeSocket(sock, sslSock) + + # This is a bit unclean + h.sock = sslSock + return h + + def prefix_proxy_to_path(self, path, host=None): + path = self.protocol + '://' + (host or self.server_name()) + path + return path + + def get_proxy_auth_header(self): + auth = encodebytes(self.proxy_user + ':' + self.proxy_pass) + return {'Proxy-Authorization': 'Basic %s' % auth} + + # For passing proxy information to other connection libraries, e.g. cloudsearch2 + def get_proxy_url_with_auth(self): + if not self.use_proxy: + return None + + if self.proxy_user or self.proxy_pass: + if self.proxy_pass: + login_info = '%s:%s@' % (self.proxy_user, self.proxy_pass) + else: + login_info = '%s@' % self.proxy_user + else: + login_info = '' + + return 'http://%s%s:%s' % (login_info, self.proxy, str(self.proxy_port or self.port)) + + def set_host_header(self, request): + try: + request.headers['Host'] = \ + self._auth_handler.host_header(self.host, request) + except AttributeError: + request.headers['Host'] = self.host.split(':', 1)[0] + + def set_request_hook(self, hook): + self.request_hook = hook + + def _mexe(self, request, sender=None, override_num_retries=None, + retry_handler=None): + """ + mexe - Multi-execute inside a loop, retrying multiple times to handle + transient Internet errors by simply trying again. + Also handles redirects. + + This code was inspired by the S3Utils classes posted to the boto-users + Google group by Larry Bates. Thanks! + + """ + boto.log.debug('Method: %s' % request.method) + boto.log.debug('Path: %s' % request.path) + boto.log.debug('Data: %s' % request.body) + boto.log.debug('Headers: %s' % request.headers) + boto.log.debug('Host: %s' % request.host) + boto.log.debug('Port: %s' % request.port) + boto.log.debug('Params: %s' % request.params) + response = None + body = None + ex = None + if override_num_retries is None: + num_retries = config.getint('Boto', 'num_retries', self.num_retries) + else: + num_retries = override_num_retries + i = 0 + connection = self.get_http_connection(request.host, request.port, + self.is_secure) + + # Convert body to bytes if needed + if not isinstance(request.body, bytes) and hasattr(request.body, + 'encode'): + request.body = request.body.encode('utf-8') + + while i <= num_retries: + # Use binary exponential backoff to desynchronize client requests. + next_sleep = min(random.random() * (2 ** i), + boto.config.get('Boto', 'max_retry_delay', 60)) + try: + # we now re-sign each request before it is retried + boto.log.debug('Token: %s' % self.provider.security_token) + request.authorize(connection=self) + # Only force header for non-s3 connections, because s3 uses + # an older signing method + bucket resource URLs that include + # the port info. All others should be now be up to date and + # not include the port. + if 's3' not in self._required_auth_capability(): + if not getattr(self, 'anon', False): + if not request.headers.get('Host'): + self.set_host_header(request) + boto.log.debug('Final headers: %s' % request.headers) + request.start_time = datetime.now() + if callable(sender): + response = sender(connection, request.method, request.path, + request.body, request.headers) + else: + connection.request(request.method, request.path, + request.body, request.headers) + response = connection.getresponse() + boto.log.debug('Response headers: %s' % response.getheaders()) + location = response.getheader('location') + # -- gross hack -- + # http_client gets confused with chunked responses to HEAD requests + # so I have to fake it out + if request.method == 'HEAD' and getattr(response, + 'chunked', False): + response.chunked = 0 + if callable(retry_handler): + status = retry_handler(response, i, next_sleep) + if status: + msg, i, next_sleep = status + if msg: + boto.log.debug(msg) + time.sleep(next_sleep) + continue + if response.status in [500, 502, 503, 504]: + msg = 'Received %d response. ' % response.status + msg += 'Retrying in %3.1f seconds' % next_sleep + boto.log.debug(msg) + body = response.read() + if isinstance(body, bytes): + body = body.decode('utf-8') + elif response.status < 300 or response.status >= 400 or \ + not location: + # don't return connection to the pool if response contains + # Connection:close header, because the connection has been + # closed and default reconnect behavior may do something + # different than new_http_connection. Also, it's probably + # less efficient to try to reuse a closed connection. + conn_header_value = response.getheader('connection') + if conn_header_value == 'close': + connection.close() + else: + self.put_http_connection(request.host, request.port, + self.is_secure, connection) + if self.request_hook is not None: + self.request_hook.handle_request_data(request, response) + return response + else: + scheme, request.host, request.path, \ + params, query, fragment = urlparse(location) + if query: + request.path += '?' + query + # urlparse can return both host and port in netloc, so if + # that's the case we need to split them up properly + if ':' in request.host: + request.host, request.port = request.host.split(':', 1) + msg = 'Redirecting: %s' % scheme + '://' + msg += request.host + request.path + boto.log.debug(msg) + connection = self.get_http_connection(request.host, + request.port, + scheme == 'https') + response = None + continue + except PleaseRetryException as e: + boto.log.debug('encountered a retry exception: %s' % e) + connection = self.new_http_connection(request.host, request.port, + self.is_secure) + response = e.response + ex = e + except self.http_exceptions as e: + for unretryable in self.http_unretryable_exceptions: + if isinstance(e, unretryable): + boto.log.debug( + 'encountered unretryable %s exception, re-raising' % + e.__class__.__name__) + raise + boto.log.debug('encountered %s exception, reconnecting' % + e.__class__.__name__) + connection = self.new_http_connection(request.host, request.port, + self.is_secure) + ex = e + time.sleep(next_sleep) + i += 1 + # If we made it here, it's because we have exhausted our retries + # and stil haven't succeeded. So, if we have a response object, + # use it to raise an exception. + # Otherwise, raise the exception that must have already happened. + if self.request_hook is not None: + self.request_hook.handle_request_data(request, response, error=True) + if response: + raise BotoServerError(response.status, response.reason, body) + elif ex: + raise ex + else: + msg = 'Please report this exception as a Boto Issue!' + raise BotoClientError(msg) + + def build_base_http_request(self, method, path, auth_path, + params=None, headers=None, data='', host=None): + path = self.get_path(path) + if auth_path is not None: + auth_path = self.get_path(auth_path) + if params is None: + params = {} + else: + params = params.copy() + if headers is None: + headers = {} + else: + headers = headers.copy() + if self.host_header and not boto.utils.find_matching_headers('host', headers): + headers['host'] = self.host_header + host = host or self.host + if self.use_proxy and not self.skip_proxy(host): + if not auth_path: + auth_path = path + path = self.prefix_proxy_to_path(path, host) + if self.proxy_user and self.proxy_pass and not self.is_secure: + # If is_secure, we don't have to set the proxy authentication + # header here, we did that in the CONNECT to the proxy. + headers.update(self.get_proxy_auth_header()) + return HTTPRequest(method, self.protocol, host, self.port, + path, auth_path, params, headers, data) + + def make_request(self, method, path, headers=None, data='', host=None, + auth_path=None, sender=None, override_num_retries=None, + params=None, retry_handler=None): + """Makes a request to the server, with stock multiple-retry logic.""" + if params is None: + params = {} + http_request = self.build_base_http_request(method, path, auth_path, + params, headers, data, host) + return self._mexe(http_request, sender, override_num_retries, + retry_handler=retry_handler) + + def close(self): + """(Optional) Close any open HTTP connections. This is non-destructive, + and making a new request will open a connection again.""" + + boto.log.debug('closing all HTTP connections') + self._connection = None # compat field + + +class AWSQueryConnection(AWSAuthConnection): + + APIVersion = '' + ResponseError = BotoServerError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=None, debug=0, + https_connection_factory=None, path='/', security_token=None, + validate_certs=True, profile_name=None, provider='aws'): + super(AWSQueryConnection, self).__init__( + host, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, + proxy_port, proxy_user, proxy_pass, + debug, https_connection_factory, path, + security_token=security_token, + validate_certs=validate_certs, + profile_name=profile_name, + provider=provider) + + def _required_auth_capability(self): + return [] + + def get_utf8_value(self, value): + return boto.utils.get_utf8_value(value) + + def make_request(self, action, params=None, path='/', verb='GET'): + http_request = self.build_base_http_request(verb, path, None, + params, {}, '', + self.host) + if action: + http_request.params['Action'] = action + if self.APIVersion: + http_request.params['Version'] = self.APIVersion + return self._mexe(http_request) + + def build_list_params(self, params, items, label): + if isinstance(items, six.string_types): + items = [items] + for i in range(1, len(items) + 1): + params['%s.%d' % (label, i)] = items[i - 1] + + def build_complex_list_params(self, params, items, label, names): + """Serialize a list of structures. + + For example:: + + items = [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')] + label = 'ParamName.member' + names = ('One', 'Two', 'Three') + self.build_complex_list_params(params, items, label, names) + + would result in the params dict being updated with these params:: + + ParamName.member.1.One = foo + ParamName.member.1.Two = bar + ParamName.member.1.Three = baz + + ParamName.member.2.One = foo2 + ParamName.member.2.Two = bar2 + ParamName.member.2.Three = baz2 + + :type params: dict + :param params: The params dict. The complex list params + will be added to this dict. + + :type items: list of tuples + :param items: The list to serialize. + + :type label: string + :param label: The prefix to apply to the parameter. + + :type names: tuple of strings + :param names: The names associated with each tuple element. + + """ + for i, item in enumerate(items, 1): + current_prefix = '%s.%s' % (label, i) + for key, value in zip(names, item): + full_key = '%s.%s' % (current_prefix, key) + params[full_key] = value + + # generics + + def get_list(self, action, params, markers, path='/', + parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if not body: + boto.log.error('Null body %s' % body) + raise self.ResponseError(response.status, response.reason, body) + elif response.status == 200: + rs = ResultSet(markers) + h = boto.handler.XmlHandler(rs, parent) + if isinstance(body, six.text_type): + body = body.encode('utf-8') + xml.sax.parseString(body, h) + return rs + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_object(self, action, params, cls, path='/', + parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if not body: + boto.log.error('Null body %s' % body) + raise self.ResponseError(response.status, response.reason, body) + elif response.status == 200: + obj = cls(parent) + h = boto.handler.XmlHandler(obj, parent) + if isinstance(body, six.text_type): + body = body.encode('utf-8') + xml.sax.parseString(body, h) + return obj + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_status(self, action, params, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if not body: + boto.log.error('Null body %s' % body) + raise self.ResponseError(response.status, response.reason, body) + elif response.status == 200: + rs = ResultSet() + h = boto.handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs.status + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body)