Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/boto/s3/connection.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 # Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
2 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates.
3 # Copyright (c) 2010, Eucalyptus Systems, Inc.
4 # All rights reserved.
5 #
6 # Permission is hereby granted, free of charge, to any person obtaining a
7 # copy of this software and associated documentation files (the
8 # "Software"), to deal in the Software without restriction, including
9 # without limitation the rights to use, copy, modify, merge, publish, dis-
10 # tribute, sublicense, and/or sell copies of the Software, and to permit
11 # persons to whom the Software is furnished to do so, subject to the fol-
12 # lowing conditions:
13 #
14 # The above copyright notice and this permission notice shall be included
15 # in all copies or substantial portions of the Software.
16 #
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
19 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
20 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
21 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 # IN THE SOFTWARE.
24
25 import xml.sax
26 import base64
27 from boto.compat import six, urllib
28 import time
29
30 from boto.auth import detect_potential_s3sigv4
31 import boto.utils
32 from boto.connection import AWSAuthConnection
33 from boto import handler
34 from boto.s3.bucket import Bucket
35 from boto.s3.key import Key
36 from boto.resultset import ResultSet
37 from boto.exception import BotoClientError, S3ResponseError
38
39
40 def check_lowercase_bucketname(n):
41 """
42 Bucket names must not contain uppercase characters. We check for
43 this by appending a lowercase character and testing with islower().
44 Note this also covers cases like numeric bucket names with dashes.
45
46 >>> check_lowercase_bucketname("Aaaa")
47 Traceback (most recent call last):
48 ...
49 BotoClientError: S3Error: Bucket names cannot contain upper-case
50 characters when using either the sub-domain or virtual hosting calling
51 format.
52
53 >>> check_lowercase_bucketname("1234-5678-9123")
54 True
55 >>> check_lowercase_bucketname("abcdefg1234")
56 True
57 """
58 if not (n + 'a').islower():
59 raise BotoClientError("Bucket names cannot contain upper-case " \
60 "characters when using either the sub-domain or virtual " \
61 "hosting calling format.")
62 return True
63
64
65 def assert_case_insensitive(f):
66 def wrapper(*args, **kwargs):
67 if len(args) == 3 and check_lowercase_bucketname(args[2]):
68 pass
69 return f(*args, **kwargs)
70 return wrapper
71
72
73 class _CallingFormat(object):
74
75 def get_bucket_server(self, server, bucket):
76 return ''
77
78 def build_url_base(self, connection, protocol, server, bucket, key=''):
79 url_base = '%s://' % protocol
80 url_base += self.build_host(server, bucket)
81 url_base += connection.get_path(self.build_path_base(bucket, key))
82 return url_base
83
84 def build_host(self, server, bucket):
85 if bucket == '':
86 return server
87 else:
88 return self.get_bucket_server(server, bucket)
89
90 def build_auth_path(self, bucket, key=''):
91 key = boto.utils.get_utf8_value(key)
92 path = ''
93 if bucket != '':
94 path = '/' + bucket
95 return path + '/%s' % urllib.parse.quote(key)
96
97 def build_path_base(self, bucket, key=''):
98 key = boto.utils.get_utf8_value(key)
99 return '/%s' % urllib.parse.quote(key)
100
101
102 class SubdomainCallingFormat(_CallingFormat):
103
104 @assert_case_insensitive
105 def get_bucket_server(self, server, bucket):
106 return '%s.%s' % (bucket, server)
107
108
109 class VHostCallingFormat(_CallingFormat):
110
111 @assert_case_insensitive
112 def get_bucket_server(self, server, bucket):
113 return bucket
114
115
116 class OrdinaryCallingFormat(_CallingFormat):
117
118 def get_bucket_server(self, server, bucket):
119 return server
120
121 def build_path_base(self, bucket, key=''):
122 key = boto.utils.get_utf8_value(key)
123 path_base = '/'
124 if bucket:
125 path_base += "%s/" % bucket
126 return path_base + urllib.parse.quote(key)
127
128
129 class ProtocolIndependentOrdinaryCallingFormat(OrdinaryCallingFormat):
130
131 def build_url_base(self, connection, protocol, server, bucket, key=''):
132 url_base = '//'
133 url_base += self.build_host(server, bucket)
134 url_base += connection.get_path(self.build_path_base(bucket, key))
135 return url_base
136
137
138 class Location(object):
139
140 DEFAULT = '' # US Classic Region
141 EU = 'EU' # Ireland
142 EUCentral1 = 'eu-central-1' # Frankfurt
143 USWest = 'us-west-1'
144 USWest2 = 'us-west-2'
145 SAEast = 'sa-east-1'
146 APNortheast = 'ap-northeast-1'
147 APSoutheast = 'ap-southeast-1'
148 APSoutheast2 = 'ap-southeast-2'
149 CNNorth1 = 'cn-north-1'
150
151
152 class NoHostProvided(object):
153 # An identifying object to help determine whether the user provided a
154 # ``host`` or not. Never instantiated.
155 pass
156
157
158 class HostRequiredError(BotoClientError):
159 pass
160
161
162 class S3Connection(AWSAuthConnection):
163
164 DefaultHost = 's3.amazonaws.com'
165 DefaultCallingFormat = boto.config.get('s3', 'calling_format', 'boto.s3.connection.SubdomainCallingFormat')
166 QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s'
167
168 def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
169 is_secure=True, port=None, proxy=None, proxy_port=None,
170 proxy_user=None, proxy_pass=None,
171 host=NoHostProvided, debug=0, https_connection_factory=None,
172 calling_format=DefaultCallingFormat, path='/',
173 provider='aws', bucket_class=Bucket, security_token=None,
174 suppress_consec_slashes=True, anon=False,
175 validate_certs=None, profile_name=None):
176 no_host_provided = False
177 # Try falling back to the boto config file's value, if present.
178 if host is NoHostProvided:
179 host = boto.config.get('s3', 'host')
180 if host is None:
181 host = self.DefaultHost
182 no_host_provided = True
183 if isinstance(calling_format, six.string_types):
184 calling_format=boto.utils.find_class(calling_format)()
185 self.calling_format = calling_format
186 self.bucket_class = bucket_class
187 self.anon = anon
188 super(S3Connection, self).__init__(host,
189 aws_access_key_id, aws_secret_access_key,
190 is_secure, port, proxy, proxy_port, proxy_user, proxy_pass,
191 debug=debug, https_connection_factory=https_connection_factory,
192 path=path, provider=provider, security_token=security_token,
193 suppress_consec_slashes=suppress_consec_slashes,
194 validate_certs=validate_certs, profile_name=profile_name)
195 # We need to delay until after the call to ``super`` before checking
196 # to see if SigV4 is in use.
197 if no_host_provided:
198 if 'hmac-v4-s3' in self._required_auth_capability():
199 raise HostRequiredError(
200 "When using SigV4, you must specify a 'host' parameter."
201 )
202
203 @detect_potential_s3sigv4
204 def _required_auth_capability(self):
205 if self.anon:
206 return ['anon']
207 else:
208 return ['s3']
209
210 def __iter__(self):
211 for bucket in self.get_all_buckets():
212 yield bucket
213
214 def __contains__(self, bucket_name):
215 return not (self.lookup(bucket_name) is None)
216
217 def set_bucket_class(self, bucket_class):
218 """
219 Set the Bucket class associated with this bucket. By default, this
220 would be the boto.s3.key.Bucket class but if you want to subclass that
221 for some reason this allows you to associate your new class.
222
223 :type bucket_class: class
224 :param bucket_class: A subclass of Bucket that can be more specific
225 """
226 self.bucket_class = bucket_class
227
228 def build_post_policy(self, expiration_time, conditions):
229 """
230 Taken from the AWS book Python examples and modified for use with boto
231 """
232 assert isinstance(expiration_time, time.struct_time), \
233 'Policy document must include a valid expiration Time object'
234
235 # Convert conditions object mappings to condition statements
236
237 return '{"expiration": "%s",\n"conditions": [%s]}' % \
238 (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions))
239
240 def build_post_form_args(self, bucket_name, key, expires_in=6000,
241 acl=None, success_action_redirect=None,
242 max_content_length=None,
243 http_method='http', fields=None,
244 conditions=None, storage_class='STANDARD',
245 server_side_encryption=None):
246 """
247 Taken from the AWS book Python examples and modified for use with boto
248 This only returns the arguments required for the post form, not the
249 actual form. This does not return the file input field which also
250 needs to be added
251
252 :type bucket_name: string
253 :param bucket_name: Bucket to submit to
254
255 :type key: string
256 :param key: Key name, optionally add ${filename} to the end to
257 attach the submitted filename
258
259 :type expires_in: integer
260 :param expires_in: Time (in seconds) before this expires, defaults
261 to 6000
262
263 :type acl: string
264 :param acl: A canned ACL. One of:
265 * private
266 * public-read
267 * public-read-write
268 * authenticated-read
269 * bucket-owner-read
270 * bucket-owner-full-control
271
272 :type success_action_redirect: string
273 :param success_action_redirect: URL to redirect to on success
274
275 :type max_content_length: integer
276 :param max_content_length: Maximum size for this file
277
278 :type http_method: string
279 :param http_method: HTTP Method to use, "http" or "https"
280
281 :type storage_class: string
282 :param storage_class: Storage class to use for storing the object.
283 Valid values: STANDARD | REDUCED_REDUNDANCY
284
285 :type server_side_encryption: string
286 :param server_side_encryption: Specifies server-side encryption
287 algorithm to use when Amazon S3 creates an object.
288 Valid values: None | AES256
289
290 :rtype: dict
291 :return: A dictionary containing field names/values as well as
292 a url to POST to
293
294 .. code-block:: python
295
296
297 """
298 if fields is None:
299 fields = []
300 if conditions is None:
301 conditions = []
302 expiration = time.gmtime(int(time.time() + expires_in))
303
304 # Generate policy document
305 conditions.append('{"bucket": "%s"}' % bucket_name)
306 if key.endswith("${filename}"):
307 conditions.append('["starts-with", "$key", "%s"]' % key[:-len("${filename}")])
308 else:
309 conditions.append('{"key": "%s"}' % key)
310 if acl:
311 conditions.append('{"acl": "%s"}' % acl)
312 fields.append({"name": "acl", "value": acl})
313 if success_action_redirect:
314 conditions.append('{"success_action_redirect": "%s"}' % success_action_redirect)
315 fields.append({"name": "success_action_redirect", "value": success_action_redirect})
316 if max_content_length:
317 conditions.append('["content-length-range", 0, %i]' % max_content_length)
318
319 if self.provider.security_token:
320 fields.append({'name': 'x-amz-security-token',
321 'value': self.provider.security_token})
322 conditions.append('{"x-amz-security-token": "%s"}' % self.provider.security_token)
323
324 if storage_class:
325 fields.append({'name': 'x-amz-storage-class',
326 'value': storage_class})
327 conditions.append('{"x-amz-storage-class": "%s"}' % storage_class)
328
329 if server_side_encryption:
330 fields.append({'name': 'x-amz-server-side-encryption',
331 'value': server_side_encryption})
332 conditions.append('{"x-amz-server-side-encryption": "%s"}' % server_side_encryption)
333
334 policy = self.build_post_policy(expiration, conditions)
335
336 # Add the base64-encoded policy document as the 'policy' field
337 policy_b64 = base64.b64encode(policy)
338 fields.append({"name": "policy", "value": policy_b64})
339
340 # Add the AWS access key as the 'AWSAccessKeyId' field
341 fields.append({"name": "AWSAccessKeyId",
342 "value": self.aws_access_key_id})
343
344 # Add signature for encoded policy document as the
345 # 'signature' field
346 signature = self._auth_handler.sign_string(policy_b64)
347 fields.append({"name": "signature", "value": signature})
348 fields.append({"name": "key", "value": key})
349
350 # HTTPS protocol will be used if the secure HTTP option is enabled.
351 url = '%s://%s/' % (http_method,
352 self.calling_format.build_host(self.server_name(),
353 bucket_name))
354
355 return {"action": url, "fields": fields}
356
357 def generate_url_sigv4(self, expires_in, method, bucket='', key='',
358 headers=None, force_http=False,
359 response_headers=None, version_id=None,
360 iso_date=None):
361 path = self.calling_format.build_path_base(bucket, key)
362 auth_path = self.calling_format.build_auth_path(bucket, key)
363 host = self.calling_format.build_host(self.server_name(), bucket)
364
365 # For presigned URLs we should ignore the port if it's HTTPS
366 if host.endswith(':443'):
367 host = host[:-4]
368
369 params = {}
370 if version_id is not None:
371 params['VersionId'] = version_id
372
373 if response_headers is not None:
374 params.update(response_headers)
375
376 http_request = self.build_base_http_request(method, path, auth_path,
377 headers=headers, host=host,
378 params=params)
379
380 return self._auth_handler.presign(http_request, expires_in,
381 iso_date=iso_date)
382
383 def generate_url(self, expires_in, method, bucket='', key='', headers=None,
384 query_auth=True, force_http=False, response_headers=None,
385 expires_in_absolute=False, version_id=None):
386 if self._auth_handler.capability[0] == 'hmac-v4-s3' and query_auth:
387 # Handle the special sigv4 case
388 return self.generate_url_sigv4(expires_in, method, bucket=bucket,
389 key=key, headers=headers, force_http=force_http,
390 response_headers=response_headers, version_id=version_id)
391
392 headers = headers or {}
393 if expires_in_absolute:
394 expires = int(expires_in)
395 else:
396 expires = int(time.time() + expires_in)
397 auth_path = self.calling_format.build_auth_path(bucket, key)
398 auth_path = self.get_path(auth_path)
399 # optional version_id and response_headers need to be added to
400 # the query param list.
401 extra_qp = []
402 if version_id is not None:
403 extra_qp.append("versionId=%s" % version_id)
404 if response_headers:
405 for k, v in response_headers.items():
406 extra_qp.append("%s=%s" % (k, urllib.parse.quote(v)))
407 if self.provider.security_token:
408 headers['x-amz-security-token'] = self.provider.security_token
409 if extra_qp:
410 delimiter = '?' if '?' not in auth_path else '&'
411 auth_path += delimiter + '&'.join(extra_qp)
412 self.calling_format.build_path_base(bucket, key)
413 if query_auth and not self.anon:
414 c_string = boto.utils.canonical_string(method, auth_path, headers,
415 expires, self.provider)
416 b64_hmac = self._auth_handler.sign_string(c_string)
417 encoded_canonical = urllib.parse.quote(b64_hmac, safe='')
418 query_part = '?' + self.QueryString % (encoded_canonical, expires,
419 self.aws_access_key_id)
420 else:
421 query_part = ''
422 if headers:
423 hdr_prefix = self.provider.header_prefix
424 for k, v in headers.items():
425 if k.startswith(hdr_prefix):
426 # headers used for sig generation must be
427 # included in the url also.
428 extra_qp.append("%s=%s" % (k, urllib.parse.quote(v)))
429 if extra_qp:
430 delimiter = '?' if not query_part else '&'
431 query_part += delimiter + '&'.join(extra_qp)
432 if force_http:
433 protocol = 'http'
434 port = 80
435 else:
436 protocol = self.protocol
437 port = self.port
438 return self.calling_format.build_url_base(self, protocol,
439 self.server_name(port),
440 bucket, key) + query_part
441
442 def get_all_buckets(self, headers=None):
443 response = self.make_request('GET', headers=headers)
444 body = response.read()
445 if response.status > 300:
446 raise self.provider.storage_response_error(
447 response.status, response.reason, body)
448 rs = ResultSet([('Bucket', self.bucket_class)])
449 h = handler.XmlHandler(rs, self)
450 if not isinstance(body, bytes):
451 body = body.encode('utf-8')
452 xml.sax.parseString(body, h)
453 return rs
454
455 def get_canonical_user_id(self, headers=None):
456 """
457 Convenience method that returns the "CanonicalUserID" of the
458 user who's credentials are associated with the connection.
459 The only way to get this value is to do a GET request on the
460 service which returns all buckets associated with the account.
461 As part of that response, the canonical userid is returned.
462 This method simply does all of that and then returns just the
463 user id.
464
465 :rtype: string
466 :return: A string containing the canonical user id.
467 """
468 rs = self.get_all_buckets(headers=headers)
469 return rs.owner.id
470
471 def get_bucket(self, bucket_name, validate=True, headers=None):
472 """
473 Retrieves a bucket by name.
474
475 If the bucket does not exist, an ``S3ResponseError`` will be raised. If
476 you are unsure if the bucket exists or not, you can use the
477 ``S3Connection.lookup`` method, which will either return a valid bucket
478 or ``None``.
479
480 If ``validate=False`` is passed, no request is made to the service (no
481 charge/communication delay). This is only safe to do if you are **sure**
482 the bucket exists.
483
484 If the default ``validate=True`` is passed, a request is made to the
485 service to ensure the bucket exists. Prior to Boto v2.25.0, this fetched
486 a list of keys (but with a max limit set to ``0``, always returning an empty
487 list) in the bucket (& included better error messages), at an
488 increased expense. As of Boto v2.25.0, this now performs a HEAD request
489 (less expensive but worse error messages).
490
491 If you were relying on parsing the error message before, you should call
492 something like::
493
494 bucket = conn.get_bucket('<bucket_name>', validate=False)
495 bucket.get_all_keys(maxkeys=0)
496
497 :type bucket_name: string
498 :param bucket_name: The name of the bucket
499
500 :type headers: dict
501 :param headers: Additional headers to pass along with the request to
502 AWS.
503
504 :type validate: boolean
505 :param validate: If ``True``, it will try to verify the bucket exists
506 on the service-side. (Default: ``True``)
507 """
508 if validate:
509 return self.head_bucket(bucket_name, headers=headers)
510 else:
511 return self.bucket_class(self, bucket_name)
512
513 def head_bucket(self, bucket_name, headers=None):
514 """
515 Determines if a bucket exists by name.
516
517 If the bucket does not exist, an ``S3ResponseError`` will be raised.
518
519 :type bucket_name: string
520 :param bucket_name: The name of the bucket
521
522 :type headers: dict
523 :param headers: Additional headers to pass along with the request to
524 AWS.
525
526 :returns: A <Bucket> object
527 """
528 response = self.make_request('HEAD', bucket_name, headers=headers)
529 body = response.read()
530 if response.status == 200:
531 return self.bucket_class(self, bucket_name)
532 elif response.status == 403:
533 # For backward-compatibility, we'll populate part of the exception
534 # with the most-common default.
535 err = self.provider.storage_response_error(
536 response.status,
537 response.reason,
538 body
539 )
540 err.error_code = 'AccessDenied'
541 err.error_message = 'Access Denied'
542 raise err
543 elif response.status == 404:
544 # For backward-compatibility, we'll populate part of the exception
545 # with the most-common default.
546 err = self.provider.storage_response_error(
547 response.status,
548 response.reason,
549 body
550 )
551 err.error_code = 'NoSuchBucket'
552 err.error_message = 'The specified bucket does not exist'
553 raise err
554 else:
555 raise self.provider.storage_response_error(
556 response.status, response.reason, body)
557
558 def lookup(self, bucket_name, validate=True, headers=None):
559 """
560 Attempts to get a bucket from S3.
561
562 Works identically to ``S3Connection.get_bucket``, save for that it
563 will return ``None`` if the bucket does not exist instead of throwing
564 an exception.
565
566 :type bucket_name: string
567 :param bucket_name: The name of the bucket
568
569 :type headers: dict
570 :param headers: Additional headers to pass along with the request to
571 AWS.
572
573 :type validate: boolean
574 :param validate: If ``True``, it will try to fetch all keys within the
575 given bucket. (Default: ``True``)
576 """
577 try:
578 bucket = self.get_bucket(bucket_name, validate, headers=headers)
579 except:
580 bucket = None
581 return bucket
582
583 def create_bucket(self, bucket_name, headers=None,
584 location=Location.DEFAULT, policy=None):
585 """
586 Creates a new located bucket. By default it's in the USA. You can pass
587 Location.EU to create a European bucket (S3) or European Union bucket
588 (GCS).
589
590 :type bucket_name: string
591 :param bucket_name: The name of the new bucket
592
593 :type headers: dict
594 :param headers: Additional headers to pass along with the request to AWS.
595
596 :type location: str
597 :param location: The location of the new bucket. You can use one of the
598 constants in :class:`boto.s3.connection.Location` (e.g. Location.EU,
599 Location.USWest, etc.).
600
601 :type policy: :class:`boto.s3.acl.CannedACLStrings`
602 :param policy: A canned ACL policy that will be applied to the
603 new key in S3.
604
605 """
606 check_lowercase_bucketname(bucket_name)
607
608 if policy:
609 if headers:
610 headers[self.provider.acl_header] = policy
611 else:
612 headers = {self.provider.acl_header: policy}
613 if location == Location.DEFAULT:
614 data = ''
615 else:
616 data = '<CreateBucketConfiguration><LocationConstraint>' + \
617 location + '</LocationConstraint></CreateBucketConfiguration>'
618 response = self.make_request('PUT', bucket_name, headers=headers,
619 data=data)
620 body = response.read()
621 if response.status == 409:
622 raise self.provider.storage_create_error(
623 response.status, response.reason, body)
624 if response.status == 200:
625 return self.bucket_class(self, bucket_name)
626 else:
627 raise self.provider.storage_response_error(
628 response.status, response.reason, body)
629
630 def delete_bucket(self, bucket, headers=None):
631 """
632 Removes an S3 bucket.
633
634 In order to remove the bucket, it must first be empty. If the bucket is
635 not empty, an ``S3ResponseError`` will be raised.
636
637 :type bucket_name: string
638 :param bucket_name: The name of the bucket
639
640 :type headers: dict
641 :param headers: Additional headers to pass along with the request to
642 AWS.
643 """
644 response = self.make_request('DELETE', bucket, headers=headers)
645 body = response.read()
646 if response.status != 204:
647 raise self.provider.storage_response_error(
648 response.status, response.reason, body)
649
650 def make_request(self, method, bucket='', key='', headers=None, data='',
651 query_args=None, sender=None, override_num_retries=None,
652 retry_handler=None):
653 if isinstance(bucket, self.bucket_class):
654 bucket = bucket.name
655 if isinstance(key, Key):
656 key = key.name
657 path = self.calling_format.build_path_base(bucket, key)
658 boto.log.debug('path=%s' % path)
659 auth_path = self.calling_format.build_auth_path(bucket, key)
660 boto.log.debug('auth_path=%s' % auth_path)
661 host = self.calling_format.build_host(self.server_name(), bucket)
662 if query_args:
663 path += '?' + query_args
664 boto.log.debug('path=%s' % path)
665 auth_path += '?' + query_args
666 boto.log.debug('auth_path=%s' % auth_path)
667 return super(S3Connection, self).make_request(
668 method, path, headers,
669 data, host, auth_path, sender,
670 override_num_retries=override_num_retries,
671 retry_handler=retry_handler
672 )