diff env/lib/python3.7/site-packages/boto/provider.py @ 0:26e78fe6e8c4 draft

"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/env/lib/python3.7/site-packages/boto/provider.py	Sat May 02 07:14:21 2020 -0400
@@ -0,0 +1,484 @@
+# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
+# Copyright 2010 Google Inc.
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+# Copyright (c) 2011, Nexenta Systems Inc.
+# All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+"""
+This class encapsulates the provider-specific header differences.
+"""
+
+import os
+from boto.compat import six
+from datetime import datetime
+
+import boto
+from boto import config
+from boto.compat import expanduser
+from boto.pyami.config import Config
+from boto.exception import InvalidInstanceMetadataError
+from boto.gs.acl import ACL
+from boto.gs.acl import CannedACLStrings as CannedGSACLStrings
+from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings
+from boto.s3.acl import Policy
+
+
+HEADER_PREFIX_KEY = 'header_prefix'
+METADATA_PREFIX_KEY = 'metadata_prefix'
+
+AWS_HEADER_PREFIX = 'x-amz-'
+GOOG_HEADER_PREFIX = 'x-goog-'
+
+ACL_HEADER_KEY = 'acl-header'
+AUTH_HEADER_KEY = 'auth-header'
+COPY_SOURCE_HEADER_KEY = 'copy-source-header'
+COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header'
+COPY_SOURCE_RANGE_HEADER_KEY = 'copy-source-range-header'
+DELETE_MARKER_HEADER_KEY = 'delete-marker-header'
+DATE_HEADER_KEY = 'date-header'
+METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header'
+RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header'
+SECURITY_TOKEN_HEADER_KEY = 'security-token-header'
+STORAGE_CLASS_HEADER_KEY = 'storage-class'
+MFA_HEADER_KEY = 'mfa-header'
+SERVER_SIDE_ENCRYPTION_KEY = 'server-side-encryption-header'
+VERSION_ID_HEADER_KEY = 'version-id-header'
+RESTORE_HEADER_KEY = 'restore-header'
+
+STORAGE_COPY_ERROR = 'StorageCopyError'
+STORAGE_CREATE_ERROR = 'StorageCreateError'
+STORAGE_DATA_ERROR = 'StorageDataError'
+STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError'
+STORAGE_RESPONSE_ERROR = 'StorageResponseError'
+NO_CREDENTIALS_PROVIDED = object()
+
+
+class ProfileNotFoundError(ValueError):
+    pass
+
+
+class Provider(object):
+
+    CredentialMap = {
+        'aws':    ('aws_access_key_id', 'aws_secret_access_key',
+                   'aws_security_token', 'aws_profile'),
+        'google': ('gs_access_key_id',  'gs_secret_access_key',
+                   None, None),
+    }
+
+    AclClassMap = {
+        'aws':    Policy,
+        'google': ACL
+    }
+
+    CannedAclsMap = {
+        'aws':    CannedS3ACLStrings,
+        'google': CannedGSACLStrings
+    }
+
+    HostKeyMap = {
+        'aws':    's3',
+        'google': 'gs'
+    }
+
+    ChunkedTransferSupport = {
+        'aws':    False,
+        'google': True
+    }
+
+    MetadataServiceSupport = {
+        'aws': True,
+        'google': False
+    }
+
+    # If you update this map please make sure to put "None" for the
+    # right-hand-side for any headers that don't apply to a provider, rather
+    # than simply leaving that header out (which would cause KeyErrors).
+    HeaderInfoMap = {
+        'aws': {
+            HEADER_PREFIX_KEY: AWS_HEADER_PREFIX,
+            METADATA_PREFIX_KEY: AWS_HEADER_PREFIX + 'meta-',
+            ACL_HEADER_KEY: AWS_HEADER_PREFIX + 'acl',
+            AUTH_HEADER_KEY: 'AWS',
+            COPY_SOURCE_HEADER_KEY: AWS_HEADER_PREFIX + 'copy-source',
+            COPY_SOURCE_VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX +
+                                                'copy-source-version-id',
+            COPY_SOURCE_RANGE_HEADER_KEY: AWS_HEADER_PREFIX +
+                                           'copy-source-range',
+            DATE_HEADER_KEY: AWS_HEADER_PREFIX + 'date',
+            DELETE_MARKER_HEADER_KEY: AWS_HEADER_PREFIX + 'delete-marker',
+            METADATA_DIRECTIVE_HEADER_KEY: AWS_HEADER_PREFIX +
+                                            'metadata-directive',
+            RESUMABLE_UPLOAD_HEADER_KEY: None,
+            SECURITY_TOKEN_HEADER_KEY: AWS_HEADER_PREFIX + 'security-token',
+            SERVER_SIDE_ENCRYPTION_KEY: AWS_HEADER_PREFIX +
+                                         'server-side-encryption',
+            VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 'version-id',
+            STORAGE_CLASS_HEADER_KEY: AWS_HEADER_PREFIX + 'storage-class',
+            MFA_HEADER_KEY: AWS_HEADER_PREFIX + 'mfa',
+            RESTORE_HEADER_KEY: AWS_HEADER_PREFIX + 'restore',
+        },
+        'google': {
+            HEADER_PREFIX_KEY: GOOG_HEADER_PREFIX,
+            METADATA_PREFIX_KEY: GOOG_HEADER_PREFIX + 'meta-',
+            ACL_HEADER_KEY: GOOG_HEADER_PREFIX + 'acl',
+            AUTH_HEADER_KEY: 'GOOG1',
+            COPY_SOURCE_HEADER_KEY: GOOG_HEADER_PREFIX + 'copy-source',
+            COPY_SOURCE_VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX +
+                                                'copy-source-version-id',
+            COPY_SOURCE_RANGE_HEADER_KEY: None,
+            DATE_HEADER_KEY: GOOG_HEADER_PREFIX + 'date',
+            DELETE_MARKER_HEADER_KEY: GOOG_HEADER_PREFIX + 'delete-marker',
+            METADATA_DIRECTIVE_HEADER_KEY: GOOG_HEADER_PREFIX  +
+                                            'metadata-directive',
+            RESUMABLE_UPLOAD_HEADER_KEY: GOOG_HEADER_PREFIX + 'resumable',
+            SECURITY_TOKEN_HEADER_KEY: GOOG_HEADER_PREFIX + 'security-token',
+            SERVER_SIDE_ENCRYPTION_KEY: None,
+            # Note that this version header is not to be confused with
+            # the Google Cloud Storage 'x-goog-api-version' header.
+            VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 'version-id',
+            STORAGE_CLASS_HEADER_KEY: GOOG_HEADER_PREFIX + 'storage-class',
+            MFA_HEADER_KEY: None,
+            RESTORE_HEADER_KEY: None,
+        }
+    }
+
+    ErrorMap = {
+        'aws': {
+            STORAGE_COPY_ERROR: boto.exception.S3CopyError,
+            STORAGE_CREATE_ERROR: boto.exception.S3CreateError,
+            STORAGE_DATA_ERROR: boto.exception.S3DataError,
+            STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError,
+            STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError,
+        },
+        'google': {
+            STORAGE_COPY_ERROR: boto.exception.GSCopyError,
+            STORAGE_CREATE_ERROR: boto.exception.GSCreateError,
+            STORAGE_DATA_ERROR: boto.exception.GSDataError,
+            STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError,
+            STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError,
+        }
+    }
+
+    def __init__(self, name, access_key=None, secret_key=None,
+                 security_token=None, profile_name=None):
+        self.host = None
+        self.port = None
+        self.host_header = None
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.security_token = security_token
+        self.profile_name = profile_name
+        self.name = name
+        self.acl_class = self.AclClassMap[self.name]
+        self.canned_acls = self.CannedAclsMap[self.name]
+        self._credential_expiry_time = None
+
+        # Load shared credentials file if it exists
+        shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
+        self.shared_credentials = Config(do_load=False)
+        if os.path.isfile(shared_path):
+            self.shared_credentials.load_from_path(shared_path)
+
+        self.get_credentials(access_key, secret_key, security_token, profile_name)
+        self.configure_headers()
+        self.configure_errors()
+
+        # Allow config file to override default host and port.
+        host_opt_name = '%s_host' % self.HostKeyMap[self.name]
+        if config.has_option('Credentials', host_opt_name):
+            self.host = config.get('Credentials', host_opt_name)
+        port_opt_name = '%s_port' % self.HostKeyMap[self.name]
+        if config.has_option('Credentials', port_opt_name):
+            self.port = config.getint('Credentials', port_opt_name)
+        host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
+        if config.has_option('Credentials', host_header_opt_name):
+            self.host_header = config.get('Credentials', host_header_opt_name)
+
+    def get_access_key(self):
+        if self._credentials_need_refresh():
+            self._populate_keys_from_metadata_server()
+        return self._access_key
+
+    def set_access_key(self, value):
+        self._access_key = value
+
+    access_key = property(get_access_key, set_access_key)
+
+    def get_secret_key(self):
+        if self._credentials_need_refresh():
+            self._populate_keys_from_metadata_server()
+        return self._secret_key
+
+    def set_secret_key(self, value):
+        self._secret_key = value
+
+    secret_key = property(get_secret_key, set_secret_key)
+
+    def get_security_token(self):
+        if self._credentials_need_refresh():
+            self._populate_keys_from_metadata_server()
+        return self._security_token
+
+    def set_security_token(self, value):
+        self._security_token = value
+
+    security_token = property(get_security_token, set_security_token)
+
+    def _credentials_need_refresh(self):
+        if self._credential_expiry_time is None:
+            return False
+        else:
+            # The credentials should be refreshed if they're going to expire
+            # in less than 5 minutes.
+            delta = self._credential_expiry_time - datetime.utcnow()
+            # python2.6 does not have timedelta.total_seconds() so we have
+            # to calculate this ourselves.  This is straight from the
+            # datetime docs.
+            seconds_left = (
+                (delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
+                 * 10 ** 6) / 10 ** 6)
+            if seconds_left < (5 * 60):
+                boto.log.debug("Credentials need to be refreshed.")
+                return True
+            else:
+                return False
+
+    def get_credentials(self, access_key=None, secret_key=None,
+                        security_token=None, profile_name=None):
+        access_key_name, secret_key_name, security_token_name, \
+            profile_name_name = self.CredentialMap[self.name]
+
+        # Load profile from shared environment variable if it was not
+        # already passed in and the environment variable exists
+        if profile_name is None and profile_name_name is not None and \
+           profile_name_name.upper() in os.environ:
+            profile_name = os.environ[profile_name_name.upper()]
+
+        shared = self.shared_credentials
+
+        if access_key is not None:
+            self.access_key = access_key
+            boto.log.debug("Using access key provided by client.")
+        elif access_key_name.upper() in os.environ:
+            self.access_key = os.environ[access_key_name.upper()]
+            boto.log.debug("Using access key found in environment variable.")
+        elif profile_name is not None:
+            if shared.has_option(profile_name, access_key_name):
+                self.access_key = shared.get(profile_name, access_key_name)
+                boto.log.debug("Using access key found in shared credential "
+                               "file for profile %s." % profile_name)
+            elif config.has_option("profile %s" % profile_name,
+                                   access_key_name):
+                self.access_key = config.get("profile %s" % profile_name,
+                                             access_key_name)
+                boto.log.debug("Using access key found in config file: "
+                               "profile %s." % profile_name)
+            else:
+                raise ProfileNotFoundError('Profile "%s" not found!' %
+                                           profile_name)
+        elif shared.has_option('default', access_key_name):
+            self.access_key = shared.get('default', access_key_name)
+            boto.log.debug("Using access key found in shared credential file.")
+        elif config.has_option('Credentials', access_key_name):
+            self.access_key = config.get('Credentials', access_key_name)
+            boto.log.debug("Using access key found in config file.")
+
+        if secret_key is not None:
+            self.secret_key = secret_key
+            boto.log.debug("Using secret key provided by client.")
+        elif secret_key_name.upper() in os.environ:
+            self.secret_key = os.environ[secret_key_name.upper()]
+            boto.log.debug("Using secret key found in environment variable.")
+        elif profile_name is not None:
+            if shared.has_option(profile_name, secret_key_name):
+                self.secret_key = shared.get(profile_name, secret_key_name)
+                boto.log.debug("Using secret key found in shared credential "
+                               "file for profile %s." % profile_name)
+            elif config.has_option("profile %s" % profile_name, secret_key_name):
+                self.secret_key = config.get("profile %s" % profile_name,
+                                             secret_key_name)
+                boto.log.debug("Using secret key found in config file: "
+                               "profile %s." % profile_name)
+            else:
+                raise ProfileNotFoundError('Profile "%s" not found!' %
+                                           profile_name)
+        elif shared.has_option('default', secret_key_name):
+            self.secret_key = shared.get('default', secret_key_name)
+            boto.log.debug("Using secret key found in shared credential file.")
+        elif config.has_option('Credentials', secret_key_name):
+            self.secret_key = config.get('Credentials', secret_key_name)
+            boto.log.debug("Using secret key found in config file.")
+        elif config.has_option('Credentials', 'keyring'):
+            keyring_name = config.get('Credentials', 'keyring')
+            try:
+                import keyring
+            except ImportError:
+                boto.log.error("The keyring module could not be imported. "
+                               "For keyring support, install the keyring "
+                               "module.")
+                raise
+            self.secret_key = keyring.get_password(
+                keyring_name, self.access_key)
+            boto.log.debug("Using secret key found in keyring.")
+
+        if security_token is not None:
+            self.security_token = security_token
+            boto.log.debug("Using security token provided by client.")
+        elif ((security_token_name is not None) and
+              (access_key is None) and (secret_key is None)):
+            # Only provide a token from the environment/config if the
+            # caller did not specify a key and secret.  Otherwise an
+            # environment/config token could be paired with a
+            # different set of credentials provided by the caller
+            if security_token_name.upper() in os.environ:
+                self.security_token = os.environ[security_token_name.upper()]
+                boto.log.debug("Using security token found in environment"
+                               " variable.")
+            elif shared.has_option(profile_name or 'default',
+                                   security_token_name):
+                self.security_token = shared.get(profile_name or 'default',
+                                                 security_token_name)
+                boto.log.debug("Using security token found in shared "
+                               "credential file.")
+            elif profile_name is not None:
+                if config.has_option("profile %s" % profile_name,
+                                     security_token_name):
+                    boto.log.debug("config has option")
+                    self.security_token = config.get("profile %s" % profile_name,
+                                                     security_token_name)
+                    boto.log.debug("Using security token found in config file: "
+                                   "profile %s." % profile_name)
+            elif config.has_option('Credentials', security_token_name):
+                self.security_token = config.get('Credentials',
+                                                 security_token_name)
+                boto.log.debug("Using security token found in config file.")
+
+        if ((self._access_key is None or self._secret_key is None) and
+                self.MetadataServiceSupport[self.name]):
+            self._populate_keys_from_metadata_server()
+        self._secret_key = self._convert_key_to_str(self._secret_key)
+
+    def _populate_keys_from_metadata_server(self):
+        # get_instance_metadata is imported here because of a circular
+        # dependency.
+        boto.log.debug("Retrieving credentials from metadata server.")
+        from boto.utils import get_instance_metadata
+        timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
+        attempts = config.getint('Boto', 'metadata_service_num_attempts', 1)
+        # The num_retries arg is actually the total number of attempts made,
+        # so the config options is named *_num_attempts to make this more
+        # clear to users.
+        metadata = get_instance_metadata(
+            timeout=timeout, num_retries=attempts,
+            data='meta-data/iam/security-credentials/')
+        if metadata:
+            creds = self._get_credentials_from_metadata(metadata)
+            self._access_key = creds[0]
+            self._secret_key = creds[1]
+            self._security_token = creds[2]
+            expires_at = creds[3]
+            # I'm assuming there's only one role on the instance profile.
+            self._credential_expiry_time = datetime.strptime(
+                expires_at, "%Y-%m-%dT%H:%M:%SZ")
+            boto.log.debug("Retrieved credentials will expire in %s at: %s",
+                           self._credential_expiry_time - datetime.now(),
+                           expires_at)
+
+    def _get_credentials_from_metadata(self, metadata):
+        # Given metadata, return a tuple of (access, secret, token, expiration)
+        # On errors, an InvalidInstanceMetadataError will be raised.
+        # The "metadata" is a lazy loaded dictionary means that it's possible
+        # to still encounter errors as we traverse through the metadata dict.
+        # We try to be careful and raise helpful error messages when this
+        # happens.
+        creds = list(metadata.values())[0]
+        if not isinstance(creds, dict):
+            # We want to special case a specific error condition which is
+            # where get_instance_metadata() returns an empty string on
+            # error conditions.
+            if creds == '':
+                msg = 'an empty string'
+            else:
+                msg = 'type: %s' % creds
+            raise InvalidInstanceMetadataError("Expected a dict type of "
+                                               "credentials instead received "
+                                               "%s" % (msg))
+        try:
+            access_key = creds['AccessKeyId']
+            secret_key = self._convert_key_to_str(creds['SecretAccessKey'])
+            security_token = creds['Token']
+            expires_at = creds['Expiration']
+        except KeyError as e:
+            raise InvalidInstanceMetadataError(
+                "Credentials from instance metadata missing "
+                "required key: %s" % e)
+        return access_key, secret_key, security_token, expires_at
+
+    def _convert_key_to_str(self, key):
+        if isinstance(key, six.text_type):
+            # the secret key must be bytes and not unicode to work
+            #  properly with hmac.new (see http://bugs.python.org/issue5285)
+            return str(key)
+        return key
+
+    def configure_headers(self):
+        header_info_map = self.HeaderInfoMap[self.name]
+        self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY]
+        self.header_prefix = header_info_map[HEADER_PREFIX_KEY]
+        self.acl_header = header_info_map[ACL_HEADER_KEY]
+        self.auth_header = header_info_map[AUTH_HEADER_KEY]
+        self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY]
+        self.copy_source_version_id = header_info_map[
+            COPY_SOURCE_VERSION_ID_HEADER_KEY]
+        self.copy_source_range_header = header_info_map[
+            COPY_SOURCE_RANGE_HEADER_KEY]
+        self.date_header = header_info_map[DATE_HEADER_KEY]
+        self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY]
+        self.metadata_directive_header = (
+            header_info_map[METADATA_DIRECTIVE_HEADER_KEY])
+        self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY]
+        self.resumable_upload_header = (
+            header_info_map[RESUMABLE_UPLOAD_HEADER_KEY])
+        self.server_side_encryption_header = header_info_map[SERVER_SIDE_ENCRYPTION_KEY]
+        self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY]
+        self.version_id = header_info_map[VERSION_ID_HEADER_KEY]
+        self.mfa_header = header_info_map[MFA_HEADER_KEY]
+        self.restore_header = header_info_map[RESTORE_HEADER_KEY]
+
+    def configure_errors(self):
+        error_map = self.ErrorMap[self.name]
+        self.storage_copy_error = error_map[STORAGE_COPY_ERROR]
+        self.storage_create_error = error_map[STORAGE_CREATE_ERROR]
+        self.storage_data_error = error_map[STORAGE_DATA_ERROR]
+        self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR]
+        self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR]
+
+    def get_provider_name(self):
+        return self.HostKeyMap[self.name]
+
+    def supports_chunked_transfer(self):
+        return self.ChunkedTransferSupport[self.name]
+
+
+# Static utility method for getting default Provider.
+def get_default():
+    return Provider('aws')