Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/boto/provider.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
2 # Copyright 2010 Google Inc.
3 # Copyright (c) 2010, Eucalyptus Systems, Inc.
4 # Copyright (c) 2011, Nexenta Systems Inc.
5 # All rights reserved.
6 #
7 # Permission is hereby granted, free of charge, to any person obtaining a
8 # copy of this software and associated documentation files (the
9 # "Software"), to deal in the Software without restriction, including
10 # without limitation the rights to use, copy, modify, merge, publish, dis-
11 # tribute, sublicense, and/or sell copies of the Software, and to permit
12 # persons to whom the Software is furnished to do so, subject to the fol-
13 # lowing conditions:
14 #
15 # The above copyright notice and this permission notice shall be included
16 # in all copies or substantial portions of the Software.
17 #
18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
20 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
21 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
22 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 # IN THE SOFTWARE.
25 """
26 This class encapsulates the provider-specific header differences.
27 """
28
29 import os
30 from boto.compat import six
31 from datetime import datetime
32
33 import boto
34 from boto import config
35 from boto.compat import expanduser
36 from boto.pyami.config import Config
37 from boto.exception import InvalidInstanceMetadataError
38 from boto.gs.acl import ACL
39 from boto.gs.acl import CannedACLStrings as CannedGSACLStrings
40 from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings
41 from boto.s3.acl import Policy
42
43
44 HEADER_PREFIX_KEY = 'header_prefix'
45 METADATA_PREFIX_KEY = 'metadata_prefix'
46
47 AWS_HEADER_PREFIX = 'x-amz-'
48 GOOG_HEADER_PREFIX = 'x-goog-'
49
50 ACL_HEADER_KEY = 'acl-header'
51 AUTH_HEADER_KEY = 'auth-header'
52 COPY_SOURCE_HEADER_KEY = 'copy-source-header'
53 COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header'
54 COPY_SOURCE_RANGE_HEADER_KEY = 'copy-source-range-header'
55 DELETE_MARKER_HEADER_KEY = 'delete-marker-header'
56 DATE_HEADER_KEY = 'date-header'
57 METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header'
58 RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header'
59 SECURITY_TOKEN_HEADER_KEY = 'security-token-header'
60 STORAGE_CLASS_HEADER_KEY = 'storage-class'
61 MFA_HEADER_KEY = 'mfa-header'
62 SERVER_SIDE_ENCRYPTION_KEY = 'server-side-encryption-header'
63 VERSION_ID_HEADER_KEY = 'version-id-header'
64 RESTORE_HEADER_KEY = 'restore-header'
65
66 STORAGE_COPY_ERROR = 'StorageCopyError'
67 STORAGE_CREATE_ERROR = 'StorageCreateError'
68 STORAGE_DATA_ERROR = 'StorageDataError'
69 STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError'
70 STORAGE_RESPONSE_ERROR = 'StorageResponseError'
71 NO_CREDENTIALS_PROVIDED = object()
72
73
74 class ProfileNotFoundError(ValueError):
75 pass
76
77
78 class Provider(object):
79
80 CredentialMap = {
81 'aws': ('aws_access_key_id', 'aws_secret_access_key',
82 'aws_security_token', 'aws_profile'),
83 'google': ('gs_access_key_id', 'gs_secret_access_key',
84 None, None),
85 }
86
87 AclClassMap = {
88 'aws': Policy,
89 'google': ACL
90 }
91
92 CannedAclsMap = {
93 'aws': CannedS3ACLStrings,
94 'google': CannedGSACLStrings
95 }
96
97 HostKeyMap = {
98 'aws': 's3',
99 'google': 'gs'
100 }
101
102 ChunkedTransferSupport = {
103 'aws': False,
104 'google': True
105 }
106
107 MetadataServiceSupport = {
108 'aws': True,
109 'google': False
110 }
111
112 # If you update this map please make sure to put "None" for the
113 # right-hand-side for any headers that don't apply to a provider, rather
114 # than simply leaving that header out (which would cause KeyErrors).
115 HeaderInfoMap = {
116 'aws': {
117 HEADER_PREFIX_KEY: AWS_HEADER_PREFIX,
118 METADATA_PREFIX_KEY: AWS_HEADER_PREFIX + 'meta-',
119 ACL_HEADER_KEY: AWS_HEADER_PREFIX + 'acl',
120 AUTH_HEADER_KEY: 'AWS',
121 COPY_SOURCE_HEADER_KEY: AWS_HEADER_PREFIX + 'copy-source',
122 COPY_SOURCE_VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX +
123 'copy-source-version-id',
124 COPY_SOURCE_RANGE_HEADER_KEY: AWS_HEADER_PREFIX +
125 'copy-source-range',
126 DATE_HEADER_KEY: AWS_HEADER_PREFIX + 'date',
127 DELETE_MARKER_HEADER_KEY: AWS_HEADER_PREFIX + 'delete-marker',
128 METADATA_DIRECTIVE_HEADER_KEY: AWS_HEADER_PREFIX +
129 'metadata-directive',
130 RESUMABLE_UPLOAD_HEADER_KEY: None,
131 SECURITY_TOKEN_HEADER_KEY: AWS_HEADER_PREFIX + 'security-token',
132 SERVER_SIDE_ENCRYPTION_KEY: AWS_HEADER_PREFIX +
133 'server-side-encryption',
134 VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 'version-id',
135 STORAGE_CLASS_HEADER_KEY: AWS_HEADER_PREFIX + 'storage-class',
136 MFA_HEADER_KEY: AWS_HEADER_PREFIX + 'mfa',
137 RESTORE_HEADER_KEY: AWS_HEADER_PREFIX + 'restore',
138 },
139 'google': {
140 HEADER_PREFIX_KEY: GOOG_HEADER_PREFIX,
141 METADATA_PREFIX_KEY: GOOG_HEADER_PREFIX + 'meta-',
142 ACL_HEADER_KEY: GOOG_HEADER_PREFIX + 'acl',
143 AUTH_HEADER_KEY: 'GOOG1',
144 COPY_SOURCE_HEADER_KEY: GOOG_HEADER_PREFIX + 'copy-source',
145 COPY_SOURCE_VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX +
146 'copy-source-version-id',
147 COPY_SOURCE_RANGE_HEADER_KEY: None,
148 DATE_HEADER_KEY: GOOG_HEADER_PREFIX + 'date',
149 DELETE_MARKER_HEADER_KEY: GOOG_HEADER_PREFIX + 'delete-marker',
150 METADATA_DIRECTIVE_HEADER_KEY: GOOG_HEADER_PREFIX +
151 'metadata-directive',
152 RESUMABLE_UPLOAD_HEADER_KEY: GOOG_HEADER_PREFIX + 'resumable',
153 SECURITY_TOKEN_HEADER_KEY: GOOG_HEADER_PREFIX + 'security-token',
154 SERVER_SIDE_ENCRYPTION_KEY: None,
155 # Note that this version header is not to be confused with
156 # the Google Cloud Storage 'x-goog-api-version' header.
157 VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 'version-id',
158 STORAGE_CLASS_HEADER_KEY: GOOG_HEADER_PREFIX + 'storage-class',
159 MFA_HEADER_KEY: None,
160 RESTORE_HEADER_KEY: None,
161 }
162 }
163
164 ErrorMap = {
165 'aws': {
166 STORAGE_COPY_ERROR: boto.exception.S3CopyError,
167 STORAGE_CREATE_ERROR: boto.exception.S3CreateError,
168 STORAGE_DATA_ERROR: boto.exception.S3DataError,
169 STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError,
170 STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError,
171 },
172 'google': {
173 STORAGE_COPY_ERROR: boto.exception.GSCopyError,
174 STORAGE_CREATE_ERROR: boto.exception.GSCreateError,
175 STORAGE_DATA_ERROR: boto.exception.GSDataError,
176 STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError,
177 STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError,
178 }
179 }
180
181 def __init__(self, name, access_key=None, secret_key=None,
182 security_token=None, profile_name=None):
183 self.host = None
184 self.port = None
185 self.host_header = None
186 self.access_key = access_key
187 self.secret_key = secret_key
188 self.security_token = security_token
189 self.profile_name = profile_name
190 self.name = name
191 self.acl_class = self.AclClassMap[self.name]
192 self.canned_acls = self.CannedAclsMap[self.name]
193 self._credential_expiry_time = None
194
195 # Load shared credentials file if it exists
196 shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
197 self.shared_credentials = Config(do_load=False)
198 if os.path.isfile(shared_path):
199 self.shared_credentials.load_from_path(shared_path)
200
201 self.get_credentials(access_key, secret_key, security_token, profile_name)
202 self.configure_headers()
203 self.configure_errors()
204
205 # Allow config file to override default host and port.
206 host_opt_name = '%s_host' % self.HostKeyMap[self.name]
207 if config.has_option('Credentials', host_opt_name):
208 self.host = config.get('Credentials', host_opt_name)
209 port_opt_name = '%s_port' % self.HostKeyMap[self.name]
210 if config.has_option('Credentials', port_opt_name):
211 self.port = config.getint('Credentials', port_opt_name)
212 host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
213 if config.has_option('Credentials', host_header_opt_name):
214 self.host_header = config.get('Credentials', host_header_opt_name)
215
216 def get_access_key(self):
217 if self._credentials_need_refresh():
218 self._populate_keys_from_metadata_server()
219 return self._access_key
220
221 def set_access_key(self, value):
222 self._access_key = value
223
224 access_key = property(get_access_key, set_access_key)
225
226 def get_secret_key(self):
227 if self._credentials_need_refresh():
228 self._populate_keys_from_metadata_server()
229 return self._secret_key
230
231 def set_secret_key(self, value):
232 self._secret_key = value
233
234 secret_key = property(get_secret_key, set_secret_key)
235
236 def get_security_token(self):
237 if self._credentials_need_refresh():
238 self._populate_keys_from_metadata_server()
239 return self._security_token
240
241 def set_security_token(self, value):
242 self._security_token = value
243
244 security_token = property(get_security_token, set_security_token)
245
246 def _credentials_need_refresh(self):
247 if self._credential_expiry_time is None:
248 return False
249 else:
250 # The credentials should be refreshed if they're going to expire
251 # in less than 5 minutes.
252 delta = self._credential_expiry_time - datetime.utcnow()
253 # python2.6 does not have timedelta.total_seconds() so we have
254 # to calculate this ourselves. This is straight from the
255 # datetime docs.
256 seconds_left = (
257 (delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
258 * 10 ** 6) / 10 ** 6)
259 if seconds_left < (5 * 60):
260 boto.log.debug("Credentials need to be refreshed.")
261 return True
262 else:
263 return False
264
265 def get_credentials(self, access_key=None, secret_key=None,
266 security_token=None, profile_name=None):
267 access_key_name, secret_key_name, security_token_name, \
268 profile_name_name = self.CredentialMap[self.name]
269
270 # Load profile from shared environment variable if it was not
271 # already passed in and the environment variable exists
272 if profile_name is None and profile_name_name is not None and \
273 profile_name_name.upper() in os.environ:
274 profile_name = os.environ[profile_name_name.upper()]
275
276 shared = self.shared_credentials
277
278 if access_key is not None:
279 self.access_key = access_key
280 boto.log.debug("Using access key provided by client.")
281 elif access_key_name.upper() in os.environ:
282 self.access_key = os.environ[access_key_name.upper()]
283 boto.log.debug("Using access key found in environment variable.")
284 elif profile_name is not None:
285 if shared.has_option(profile_name, access_key_name):
286 self.access_key = shared.get(profile_name, access_key_name)
287 boto.log.debug("Using access key found in shared credential "
288 "file for profile %s." % profile_name)
289 elif config.has_option("profile %s" % profile_name,
290 access_key_name):
291 self.access_key = config.get("profile %s" % profile_name,
292 access_key_name)
293 boto.log.debug("Using access key found in config file: "
294 "profile %s." % profile_name)
295 else:
296 raise ProfileNotFoundError('Profile "%s" not found!' %
297 profile_name)
298 elif shared.has_option('default', access_key_name):
299 self.access_key = shared.get('default', access_key_name)
300 boto.log.debug("Using access key found in shared credential file.")
301 elif config.has_option('Credentials', access_key_name):
302 self.access_key = config.get('Credentials', access_key_name)
303 boto.log.debug("Using access key found in config file.")
304
305 if secret_key is not None:
306 self.secret_key = secret_key
307 boto.log.debug("Using secret key provided by client.")
308 elif secret_key_name.upper() in os.environ:
309 self.secret_key = os.environ[secret_key_name.upper()]
310 boto.log.debug("Using secret key found in environment variable.")
311 elif profile_name is not None:
312 if shared.has_option(profile_name, secret_key_name):
313 self.secret_key = shared.get(profile_name, secret_key_name)
314 boto.log.debug("Using secret key found in shared credential "
315 "file for profile %s." % profile_name)
316 elif config.has_option("profile %s" % profile_name, secret_key_name):
317 self.secret_key = config.get("profile %s" % profile_name,
318 secret_key_name)
319 boto.log.debug("Using secret key found in config file: "
320 "profile %s." % profile_name)
321 else:
322 raise ProfileNotFoundError('Profile "%s" not found!' %
323 profile_name)
324 elif shared.has_option('default', secret_key_name):
325 self.secret_key = shared.get('default', secret_key_name)
326 boto.log.debug("Using secret key found in shared credential file.")
327 elif config.has_option('Credentials', secret_key_name):
328 self.secret_key = config.get('Credentials', secret_key_name)
329 boto.log.debug("Using secret key found in config file.")
330 elif config.has_option('Credentials', 'keyring'):
331 keyring_name = config.get('Credentials', 'keyring')
332 try:
333 import keyring
334 except ImportError:
335 boto.log.error("The keyring module could not be imported. "
336 "For keyring support, install the keyring "
337 "module.")
338 raise
339 self.secret_key = keyring.get_password(
340 keyring_name, self.access_key)
341 boto.log.debug("Using secret key found in keyring.")
342
343 if security_token is not None:
344 self.security_token = security_token
345 boto.log.debug("Using security token provided by client.")
346 elif ((security_token_name is not None) and
347 (access_key is None) and (secret_key is None)):
348 # Only provide a token from the environment/config if the
349 # caller did not specify a key and secret. Otherwise an
350 # environment/config token could be paired with a
351 # different set of credentials provided by the caller
352 if security_token_name.upper() in os.environ:
353 self.security_token = os.environ[security_token_name.upper()]
354 boto.log.debug("Using security token found in environment"
355 " variable.")
356 elif shared.has_option(profile_name or 'default',
357 security_token_name):
358 self.security_token = shared.get(profile_name or 'default',
359 security_token_name)
360 boto.log.debug("Using security token found in shared "
361 "credential file.")
362 elif profile_name is not None:
363 if config.has_option("profile %s" % profile_name,
364 security_token_name):
365 boto.log.debug("config has option")
366 self.security_token = config.get("profile %s" % profile_name,
367 security_token_name)
368 boto.log.debug("Using security token found in config file: "
369 "profile %s." % profile_name)
370 elif config.has_option('Credentials', security_token_name):
371 self.security_token = config.get('Credentials',
372 security_token_name)
373 boto.log.debug("Using security token found in config file.")
374
375 if ((self._access_key is None or self._secret_key is None) and
376 self.MetadataServiceSupport[self.name]):
377 self._populate_keys_from_metadata_server()
378 self._secret_key = self._convert_key_to_str(self._secret_key)
379
380 def _populate_keys_from_metadata_server(self):
381 # get_instance_metadata is imported here because of a circular
382 # dependency.
383 boto.log.debug("Retrieving credentials from metadata server.")
384 from boto.utils import get_instance_metadata
385 timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
386 attempts = config.getint('Boto', 'metadata_service_num_attempts', 1)
387 # The num_retries arg is actually the total number of attempts made,
388 # so the config options is named *_num_attempts to make this more
389 # clear to users.
390 metadata = get_instance_metadata(
391 timeout=timeout, num_retries=attempts,
392 data='meta-data/iam/security-credentials/')
393 if metadata:
394 creds = self._get_credentials_from_metadata(metadata)
395 self._access_key = creds[0]
396 self._secret_key = creds[1]
397 self._security_token = creds[2]
398 expires_at = creds[3]
399 # I'm assuming there's only one role on the instance profile.
400 self._credential_expiry_time = datetime.strptime(
401 expires_at, "%Y-%m-%dT%H:%M:%SZ")
402 boto.log.debug("Retrieved credentials will expire in %s at: %s",
403 self._credential_expiry_time - datetime.now(),
404 expires_at)
405
406 def _get_credentials_from_metadata(self, metadata):
407 # Given metadata, return a tuple of (access, secret, token, expiration)
408 # On errors, an InvalidInstanceMetadataError will be raised.
409 # The "metadata" is a lazy loaded dictionary means that it's possible
410 # to still encounter errors as we traverse through the metadata dict.
411 # We try to be careful and raise helpful error messages when this
412 # happens.
413 creds = list(metadata.values())[0]
414 if not isinstance(creds, dict):
415 # We want to special case a specific error condition which is
416 # where get_instance_metadata() returns an empty string on
417 # error conditions.
418 if creds == '':
419 msg = 'an empty string'
420 else:
421 msg = 'type: %s' % creds
422 raise InvalidInstanceMetadataError("Expected a dict type of "
423 "credentials instead received "
424 "%s" % (msg))
425 try:
426 access_key = creds['AccessKeyId']
427 secret_key = self._convert_key_to_str(creds['SecretAccessKey'])
428 security_token = creds['Token']
429 expires_at = creds['Expiration']
430 except KeyError as e:
431 raise InvalidInstanceMetadataError(
432 "Credentials from instance metadata missing "
433 "required key: %s" % e)
434 return access_key, secret_key, security_token, expires_at
435
436 def _convert_key_to_str(self, key):
437 if isinstance(key, six.text_type):
438 # the secret key must be bytes and not unicode to work
439 # properly with hmac.new (see http://bugs.python.org/issue5285)
440 return str(key)
441 return key
442
443 def configure_headers(self):
444 header_info_map = self.HeaderInfoMap[self.name]
445 self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY]
446 self.header_prefix = header_info_map[HEADER_PREFIX_KEY]
447 self.acl_header = header_info_map[ACL_HEADER_KEY]
448 self.auth_header = header_info_map[AUTH_HEADER_KEY]
449 self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY]
450 self.copy_source_version_id = header_info_map[
451 COPY_SOURCE_VERSION_ID_HEADER_KEY]
452 self.copy_source_range_header = header_info_map[
453 COPY_SOURCE_RANGE_HEADER_KEY]
454 self.date_header = header_info_map[DATE_HEADER_KEY]
455 self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY]
456 self.metadata_directive_header = (
457 header_info_map[METADATA_DIRECTIVE_HEADER_KEY])
458 self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY]
459 self.resumable_upload_header = (
460 header_info_map[RESUMABLE_UPLOAD_HEADER_KEY])
461 self.server_side_encryption_header = header_info_map[SERVER_SIDE_ENCRYPTION_KEY]
462 self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY]
463 self.version_id = header_info_map[VERSION_ID_HEADER_KEY]
464 self.mfa_header = header_info_map[MFA_HEADER_KEY]
465 self.restore_header = header_info_map[RESTORE_HEADER_KEY]
466
467 def configure_errors(self):
468 error_map = self.ErrorMap[self.name]
469 self.storage_copy_error = error_map[STORAGE_COPY_ERROR]
470 self.storage_create_error = error_map[STORAGE_CREATE_ERROR]
471 self.storage_data_error = error_map[STORAGE_DATA_ERROR]
472 self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR]
473 self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR]
474
475 def get_provider_name(self):
476 return self.HostKeyMap[self.name]
477
478 def supports_chunked_transfer(self):
479 return self.ChunkedTransferSupport[self.name]
480
481
482 # Static utility method for getting default Provider.
483 def get_default():
484 return Provider('aws')