guppy_basecaller: env/lib/python3.7/site-packages/boto/s3/key.py comparison

comparison env/lib/python3.7/site-packages/boto/s3/key.py @ 0:26e78fe6e8c4 draft

"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"

author	shellac
date	Sat, 02 May 2020 07:14:21 -0400
parents
children

comparison

equal deleted inserted replaced

--1:000000000000
+:26e78fe6e8c4
+# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2011, Nexenta Systems Inc.
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+import email.utils
+import errno
+import hashlib
+import mimetypes
+import os
+import re
+import base64
+import binascii
+import math
+from hashlib import md5
+import boto.utils
+from boto.compat import BytesIO, six, urllib, encodebytes
+from boto.exception import BotoClientError
+from boto.exception import StorageDataError
+from boto.exception import PleaseRetryException
+from boto.provider import Provider
+from boto.s3.keyfile import KeyFile
+from boto.s3.user import User
+from boto import UserAgent
+from boto.utils import compute_md5, compute_hash
+from boto.utils import find_matching_headers
+from boto.utils import merge_headers_by_name
+class Key(object):
+"""
+Represents a key (object) in an S3 bucket.
+:ivar bucket: The parent :class:`boto.s3.bucket.Bucket`.
+:ivar name: The name of this Key object.
+:ivar metadata: A dictionary containing user metadata that you
+wish to store with the object or that has been retrieved from
+an existing object.
+:ivar cache_control: The value of the `Cache-Control` HTTP header.
+:ivar content_type: The value of the `Content-Type` HTTP header.
+:ivar content_encoding: The value of the `Content-Encoding` HTTP header.
+:ivar content_disposition: The value of the `Content-Disposition` HTTP
+header.
+:ivar content_language: The value of the `Content-Language` HTTP header.
+:ivar etag: The `etag` associated with this object.
+:ivar last_modified: The string timestamp representing the last
+time this object was modified in S3.
+:ivar owner: The ID of the owner of this object.
+:ivar storage_class: The storage class of the object.  Currently, one of:
+STANDARD | REDUCED_REDUNDANCY | GLACIER
+:ivar md5: The MD5 hash of the contents of the object.
+:ivar size: The size, in bytes, of the object.
+:ivar version_id: The version ID of this object, if it is a versioned
+object.
+:ivar encrypted: Whether the object is encrypted while at rest on
+the server.
+"""
+DefaultContentType = 'application/octet-stream'
+RestoreBody = """<?xml version="1.0" encoding="UTF-8"?>
+<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+<Days>%s</Days>
+</RestoreRequest>"""
+BufferSize = boto.config.getint('Boto', 'key_buffer_size', 8192)
+# The object metadata fields a user can set, other than custom metadata
+# fields (i.e., those beginning with a provider-specific prefix like
+# x-amz-meta).
+base_user_settable_fields = set(["cache-control", "content-disposition",
+"content-encoding", "content-language",
+"content-md5", "content-type",
+"x-robots-tag", "expires"])
+_underscore_base_user_settable_fields = set()
+for f in base_user_settable_fields:
+_underscore_base_user_settable_fields.add(f.replace('-', '_'))
+# Metadata fields, whether user-settable or not, other than custom
+# metadata fields (i.e., those beginning with a provider specific prefix
+# like x-amz-meta).
+base_fields = (base_user_settable_fields |
+set(["last-modified", "content-length", "date", "etag"]))
+def __init__(self, bucket=None, name=None):
+self.bucket = bucket
+self.name = name
+self.metadata = {}
+self.cache_control = None
+self.content_type = self.DefaultContentType
+self.content_encoding = None
+self.content_disposition = None
+self.content_language = None
+self.filename = None
+self.etag = None
+self.is_latest = False
+self.last_modified = None
+self.owner = None
+self._storage_class = None
+self.path = None
+self.resp = None
+self.mode = None
+self.size = None
+self.version_id = None
+self.source_version_id = None
+self.delete_marker = False
+self.encrypted = None
+# If the object is being restored, this attribute will be set to True.
+# If the object is restored, it will be set to False.  Otherwise this
+# value will be None. If the restore is completed (ongoing_restore =
+# False), the expiry_date will be populated with the expiry date of the
+# restored object.
+self.ongoing_restore = None
+self.expiry_date = None
+self.local_hashes = {}
+def __repr__(self):
+if self.bucket:
+name = u'<Key: %s,%s>' % (self.bucket.name, self.name)
+else:
+name = u'<Key: None,%s>' % self.name
+# Encode to bytes for Python 2 to prevent display decoding issues
+if not isinstance(name, str):
+name = name.encode('utf-8')
+return name
+def __iter__(self):
+return self
+@property
+def provider(self):
+provider = None
+if self.bucket and self.bucket.connection:
+provider = self.bucket.connection.provider
+return provider
+def _get_key(self):
+return self.name
+def _set_key(self, value):
+self.name = value
+key = property(_get_key, _set_key);
+def _get_md5(self):
+if 'md5' in self.local_hashes and self.local_hashes['md5']:
+return binascii.b2a_hex(self.local_hashes['md5'])
+def _set_md5(self, value):
+if value:
+self.local_hashes['md5'] = binascii.a2b_hex(value)
+elif 'md5' in self.local_hashes:
+self.local_hashes.pop('md5', None)
+md5 = property(_get_md5, _set_md5);
+def _get_base64md5(self):
+if 'md5' in self.local_hashes and self.local_hashes['md5']:
+md5 = self.local_hashes['md5']
+if not isinstance(md5, bytes):
+md5 = md5.encode('utf-8')
+return binascii.b2a_base64(md5).decode('utf-8').rstrip('\n')
+def _set_base64md5(self, value):
+if value:
+if not isinstance(value, six.string_types):
+value = value.decode('utf-8')
+self.local_hashes['md5'] = binascii.a2b_base64(value)
+elif 'md5' in self.local_hashes:
+del self.local_hashes['md5']
+base64md5 = property(_get_base64md5, _set_base64md5);
+def _get_storage_class(self):
+if self._storage_class is None and self.bucket:
+# Attempt to fetch storage class
+list_items = list(self.bucket.list(self.name.encode('utf-8')))
+if len(list_items) and getattr(list_items[0], '_storage_class',
+None):
+self._storage_class = list_items[0]._storage_class
+else:
+# Key is not yet saved? Just use default...
+self._storage_class = 'STANDARD'
+return self._storage_class
+def _set_storage_class(self, value):
+self._storage_class = value
+storage_class = property(_get_storage_class, _set_storage_class)
+def get_md5_from_hexdigest(self, md5_hexdigest):
+"""
+A utility function to create the 2-tuple (md5hexdigest, base64md5)
+from just having a precalculated md5_hexdigest.
+"""
+digest = binascii.unhexlify(md5_hexdigest)
+base64md5 = encodebytes(digest)
+if base64md5[-1] == '\n':
+base64md5 = base64md5[0:-1]
+return (md5_hexdigest, base64md5)
+def handle_encryption_headers(self, resp):
+provider = self.bucket.connection.provider
+if provider.server_side_encryption_header:
+self.encrypted = resp.getheader(
+provider.server_side_encryption_header, None)
+else:
+self.encrypted = None
+def handle_storage_class_header(self, resp):
+provider = self.bucket.connection.provider
+if provider.storage_class_header:
+self._storage_class = resp.getheader(
+provider.storage_class_header, None)
+if (self._storage_class is None and
+provider.get_provider_name() == 'aws'):
+# S3 docs for HEAD object requests say S3 will return this
+# header for all objects except Standard storage class objects.
+self._storage_class = 'STANDARD'
+def handle_version_headers(self, resp, force=False):
+provider = self.bucket.connection.provider
+# If the Key object already has a version_id attribute value, it
+# means that it represents an explicit version and the user is
+# doing a get_contents_*(version_id=<foo>) to retrieve another
+# version of the Key.  In that case, we don't really want to
+# overwrite the version_id in this Key object.  Comprende?
+if self.version_id is None or force:
+self.version_id = resp.getheader(provider.version_id, None)
+self.source_version_id = resp.getheader(provider.copy_source_version_id,
+None)
+if resp.getheader(provider.delete_marker, 'false') == 'true':
+self.delete_marker = True
+else:
+self.delete_marker = False
+def handle_restore_headers(self, response):
+provider = self.bucket.connection.provider
+header = response.getheader(provider.restore_header)
+if header is None:
+return
+parts = header.split(',', 1)
+for part in parts:
+key, val = [i.strip() for i in part.split('=')]
+val = val.replace('"', '')
+if key == 'ongoing-request':
+self.ongoing_restore = True if val.lower() == 'true' else False
+elif key == 'expiry-date':
+self.expiry_date = val
+def handle_addl_headers(self, headers):
+"""
+Used by Key subclasses to do additional, provider-specific
+processing of response headers. No-op for this base class.
+"""
+pass
+def open_read(self, headers=None, query_args='',
+override_num_retries=None, response_headers=None):
+"""
+Open this key for reading
+:type headers: dict
+:param headers: Headers to pass in the web request
+:type query_args: string
+:param query_args: Arguments to pass in the query string
+(ie, 'torrent')
+:type override_num_retries: int
+:param override_num_retries: If not None will override configured
+num_retries parameter for underlying GET.
+:type response_headers: dict
+:param response_headers: A dictionary containing HTTP
+headers/values that will override any headers associated
+with the stored object in the response.  See
+http://goo.gl/EWOPb for details.
+"""
+if self.resp is None:
+self.mode = 'r'
+provider = self.bucket.connection.provider
+self.resp = self.bucket.connection.make_request(
+'GET', self.bucket.name, self.name, headers,
+query_args=query_args,
+override_num_retries=override_num_retries)
+if self.resp.status < 199 or self.resp.status > 299:
+body = self.resp.read()
+raise provider.storage_response_error(self.resp.status,
+self.resp.reason, body)
+response_headers = self.resp.msg
+self.metadata = boto.utils.get_aws_metadata(response_headers,
+provider)
+for name, value in response_headers.items():
+# To get correct size for Range GETs, use Content-Range
+# header if one was returned. If not, use Content-Length
+# header.
+if (name.lower() == 'content-length' and
+'Content-Range' not in response_headers):
+self.size = int(value)
+elif name.lower() == 'content-range':
+end_range = re.sub('.*/(.*)', '\\1', value)
+self.size = int(end_range)
+elif name.lower() in Key.base_fields:
+self.__dict__[name.lower().replace('-', '_')] = value
+self.handle_version_headers(self.resp)
+self.handle_encryption_headers(self.resp)
+self.handle_restore_headers(self.resp)
+self.handle_addl_headers(self.resp.getheaders())
+def open_write(self, headers=None, override_num_retries=None):
+"""
+Open this key for writing.
+Not yet implemented
+:type headers: dict
+:param headers: Headers to pass in the write request
+:type override_num_retries: int
+:param override_num_retries: If not None will override configured
+num_retries parameter for underlying PUT.
+"""
+raise BotoClientError('Not Implemented')
+def open(self, mode='r', headers=None, query_args=None,
+override_num_retries=None):
+if mode == 'r':
+self.mode = 'r'
+self.open_read(headers=headers, query_args=query_args,
+override_num_retries=override_num_retries)
+elif mode == 'w':
+self.mode = 'w'
+self.open_write(headers=headers,
+override_num_retries=override_num_retries)
+else:
+raise BotoClientError('Invalid mode: %s' % mode)
+closed = False
+def close(self, fast=False):
+"""
+Close this key.
+:type fast: bool
+:param fast: True if you want the connection to be closed without first
+reading the content. This should only be used in cases where subsequent
+calls don't need to return the content from the open HTTP connection.
+Note: As explained at
+http://docs.python.org/2/library/httplib.html#httplib.HTTPConnection.getresponse,
+callers must read the whole response before sending a new request to the
+server. Calling Key.close(fast=True) and making a subsequent request to
+the server will work because boto will get an httplib exception and
+close/reopen the connection.
+"""
+if self.resp and not fast:
+self.resp.read()
+self.resp = None
+self.mode = None
+self.closed = True
+def next(self):
+"""
+By providing a next method, the key object supports use as an iterator.
+For example, you can now say:
+for bytes in key:
+write bytes to a file or whatever
+All of the HTTP connection stuff is handled for you.
+"""
+self.open_read()
+data = self.resp.read(self.BufferSize)
+if not data:
+self.close()
+raise StopIteration
+return data
+# Python 3 iterator support
+__next__ = next
+def read(self, size=0):
+self.open_read()
+if size == 0:
+data = self.resp.read()
+else:
+data = self.resp.read(size)
+if not data:
+self.close()
+return data
+def change_storage_class(self, new_storage_class, dst_bucket=None,
+validate_dst_bucket=True):
+"""
+Change the storage class of an existing key.
+Depending on whether a different destination bucket is supplied
+or not, this will either move the item within the bucket, preserving
+all metadata and ACL info bucket changing the storage class or it
+will copy the item to the provided destination bucket, also
+preserving metadata and ACL info.
+:type new_storage_class: string
+:param new_storage_class: The new storage class for the Key.
+Possible values are:
+* STANDARD
+* REDUCED_REDUNDANCY
+:type dst_bucket: string
+:param dst_bucket: The name of a destination bucket.  If not
+provided the current bucket of the key will be used.
+:type validate_dst_bucket: bool
+:param validate_dst_bucket: If True, will validate the dst_bucket
+by using an extra list request.
+"""
+bucket_name = dst_bucket or self.bucket.name
+if new_storage_class == 'STANDARD':
+return self.copy(bucket_name, self.name,
+reduced_redundancy=False, preserve_acl=True,
+validate_dst_bucket=validate_dst_bucket)
+elif new_storage_class == 'REDUCED_REDUNDANCY':
+return self.copy(bucket_name, self.name,
+reduced_redundancy=True, preserve_acl=True,
+validate_dst_bucket=validate_dst_bucket)
+else:
+raise BotoClientError('Invalid storage class: %s' %
+new_storage_class)
+def copy(self, dst_bucket, dst_key, metadata=None,
+reduced_redundancy=False, preserve_acl=False,
+encrypt_key=False, validate_dst_bucket=True):
+"""
+Copy this Key to another bucket.
+:type dst_bucket: string
+:param dst_bucket: The name of the destination bucket
+:type dst_key: string
+:param dst_key: The name of the destination key
+:type metadata: dict
+:param metadata: Metadata to be associated with new key.  If
+metadata is supplied, it will replace the metadata of the
+source key being copied.  If no metadata is supplied, the
+source key's metadata will be copied to the new key.
+:type reduced_redundancy: bool
+:param reduced_redundancy: If True, this will force the
+storage class of the new Key to be REDUCED_REDUNDANCY
+regardless of the storage class of the key being copied.
+The Reduced Redundancy Storage (RRS) feature of S3,
+provides lower redundancy at lower storage cost.
+:type preserve_acl: bool
+:param preserve_acl: If True, the ACL from the source key will
+be copied to the destination key.  If False, the
+destination key will have the default ACL.  Note that
+preserving the ACL in the new key object will require two
+additional API calls to S3, one to retrieve the current
+ACL and one to set that ACL on the new object.  If you
+don't care about the ACL, a value of False will be
+significantly more efficient.
+:type encrypt_key: bool
+:param encrypt_key: If True, the new copy of the object will
+be encrypted on the server-side by S3 and will be stored
+in an encrypted form while at rest in S3.
+:type validate_dst_bucket: bool
+:param validate_dst_bucket: If True, will validate the dst_bucket
+by using an extra list request.
+:rtype: :class:`boto.s3.key.Key` or subclass
+:returns: An instance of the newly created key object
+"""
+dst_bucket = self.bucket.connection.lookup(dst_bucket,
+validate_dst_bucket)
+if reduced_redundancy:
+storage_class = 'REDUCED_REDUNDANCY'
+else:
+storage_class = self.storage_class
+return dst_bucket.copy_key(dst_key, self.bucket.name,
+self.name, metadata,
+storage_class=storage_class,
+preserve_acl=preserve_acl,
+encrypt_key=encrypt_key,
+src_version_id=self.version_id)
+def startElement(self, name, attrs, connection):
+if name == 'Owner':
+self.owner = User(self)
+return self.owner
+else:
+return None
+def endElement(self, name, value, connection):
+if name == 'Key':
+self.name = value
+elif name == 'ETag':
+self.etag = value
+elif name == 'IsLatest':
+if value == 'true':
+self.is_latest = True
+else:
+self.is_latest = False
+elif name == 'LastModified':
+self.last_modified = value
+elif name == 'Size':
+self.size = int(value)
+elif name == 'StorageClass':
+self.storage_class = value
+elif name == 'Owner':
+pass
+elif name == 'VersionId':
+self.version_id = value
+else:
+setattr(self, name, value)
+def exists(self, headers=None):
+"""
+Returns True if the key exists
+:rtype: bool
+:return: Whether the key exists on S3
+"""
+return bool(self.bucket.lookup(self.name, headers=headers))
+def delete(self, headers=None):
+"""
+Delete this key from S3
+"""
+return self.bucket.delete_key(self.name, version_id=self.version_id,
+headers=headers)
+def get_metadata(self, name):
+return self.metadata.get(name)
+def set_metadata(self, name, value):
+# Ensure that metadata that is vital to signing is in the correct
+# case. Applies to ``Content-Type`` & ``Content-MD5``.
+if name.lower() == 'content-type':
+self.metadata['Content-Type'] = value
+elif name.lower() == 'content-md5':
+self.metadata['Content-MD5'] = value
+else:
+self.metadata[name] = value
+if name.lower() in Key.base_user_settable_fields:
+self.__dict__[name.lower().replace('-', '_')] = value
+def update_metadata(self, d):
+self.metadata.update(d)
+# convenience methods for setting/getting ACL
+def set_acl(self, acl_str, headers=None):
+if self.bucket is not None:
+self.bucket.set_acl(acl_str, self.name, headers=headers)
+def get_acl(self, headers=None):
+if self.bucket is not None:
+return self.bucket.get_acl(self.name, headers=headers)
+def get_xml_acl(self, headers=None):
+if self.bucket is not None:
+return self.bucket.get_xml_acl(self.name, headers=headers)
+def set_xml_acl(self, acl_str, headers=None):
+if self.bucket is not None:
+return self.bucket.set_xml_acl(acl_str, self.name, headers=headers)
+def set_canned_acl(self, acl_str, headers=None):
+return self.bucket.set_canned_acl(acl_str, self.name, headers)
+def get_redirect(self):
+"""Return the redirect location configured for this key.
+If no redirect is configured (via set_redirect), then None
+will be returned.
+"""
+response = self.bucket.connection.make_request(
+'HEAD', self.bucket.name, self.name)
+if response.status == 200:
+return response.getheader('x-amz-website-redirect-location')
+else:
+raise self.provider.storage_response_error(
+response.status, response.reason, response.read())
+def set_redirect(self, redirect_location, headers=None):
+"""Configure this key to redirect to another location.
+When the bucket associated with this key is accessed from the website
+endpoint, a 301 redirect will be issued to the specified
+`redirect_location`.
+:type redirect_location: string
+:param redirect_location: The location to redirect.
+"""
+if headers is None:
+headers = {}
+else:
+headers = headers.copy()
+headers['x-amz-website-redirect-location'] = redirect_location
+response = self.bucket.connection.make_request('PUT', self.bucket.name,
+self.name, headers)
+if response.status == 200:
+return True
+else:
+raise self.provider.storage_response_error(
+response.status, response.reason, response.read())
+def make_public(self, headers=None):
+return self.bucket.set_canned_acl('public-read', self.name, headers)
+def generate_url(self, expires_in, method='GET', headers=None,
+query_auth=True, force_http=False, response_headers=None,
+expires_in_absolute=False, version_id=None,
+policy=None, reduced_redundancy=False, encrypt_key=False):
+"""
+Generate a URL to access this key.
+:type expires_in: int
+:param expires_in: How long the url is valid for, in seconds.
+:type method: string
+:param method: The method to use for retrieving the file
+(default is GET).
+:type headers: dict
+:param headers: Any headers to pass along in the request.
+:type query_auth: bool
+:param query_auth: If True, signs the request in the URL.
+:type force_http: bool
+:param force_http: If True, http will be used instead of https.
+:type response_headers: dict
+:param response_headers: A dictionary containing HTTP
+headers/values that will override any headers associated
+with the stored object in the response.  See
+http://goo.gl/EWOPb for details.
+:type expires_in_absolute: bool
+:param expires_in_absolute:
+:type version_id: string
+:param version_id: The version_id of the object to GET. If specified
+this overrides any value in the key.
+:type policy: :class:`boto.s3.acl.CannedACLStrings`
+:param policy: A canned ACL policy that will be applied to the
+new key in S3.
+:type reduced_redundancy: bool
+:param reduced_redundancy: If True, this will set the storage
+class of the new Key to be REDUCED_REDUNDANCY. The Reduced
+Redundancy Storage (RRS) feature of S3, provides lower
+redundancy at lower storage cost.
+:type encrypt_key: bool
+:param encrypt_key: If True, the new copy of the object will
+be encrypted on the server-side by S3 and will be stored
+in an encrypted form while at rest in S3.
+:rtype: string
+:return: The URL to access the key
+"""
+provider = self.bucket.connection.provider
+version_id = version_id or self.version_id
+if headers is None:
+headers = {}
+else:
+headers = headers.copy()
+# add headers accordingly (usually PUT case)
+if policy:
+headers[provider.acl_header] = policy
+if reduced_redundancy:
+self.storage_class = 'REDUCED_REDUNDANCY'
+if provider.storage_class_header:
+headers[provider.storage_class_header] = self.storage_class
+if encrypt_key:
+headers[provider.server_side_encryption_header] = 'AES256'
+headers = boto.utils.merge_meta(headers, self.metadata, provider)
+return self.bucket.connection.generate_url(expires_in, method,
+self.bucket.name, self.name,
+headers, query_auth,
+force_http,
+response_headers,
+expires_in_absolute,
+version_id)
+def send_file(self, fp, headers=None, cb=None, num_cb=10,
+query_args=None, chunked_transfer=False, size=None):
+"""
+Upload a file to a key into a bucket on S3.
+:type fp: file
+:param fp: The file pointer to upload. The file pointer must
+point at the offset from which you wish to upload.
+ie. if uploading the full file, it should point at the
+start of the file. Normally when a file is opened for
+reading, the fp will point at the first byte.  See the
+bytes parameter below for more info.
+:type headers: dict
+:param headers: The headers to pass along with the PUT request
+:type num_cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file
+transfer. Providing a negative integer will cause your
+callback to be called with each buffer read.
+:type query_args: string
+:param query_args: (optional) Arguments to pass in the query string.
+:type chunked_transfer: boolean
+:param chunked_transfer: (optional) If true, we use chunked
+Transfer-Encoding.
+:type size: int
+:param size: (optional) The Maximum number of bytes to read
+from the file pointer (fp). This is useful when uploading
+a file in multiple parts where you are splitting the file
+up into different ranges to be uploaded. If not specified,
+the default behaviour is to read all bytes from the file
+pointer. Less bytes may be available.
+"""
+self._send_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb,
+query_args=query_args,
+chunked_transfer=chunked_transfer, size=size)
+def _send_file_internal(self, fp, headers=None, cb=None, num_cb=10,
+query_args=None, chunked_transfer=False, size=None,
+hash_algs=None):
+provider = self.bucket.connection.provider
+try:
+spos = fp.tell()
+except IOError:
+spos = None
+self.read_from_stream = False
+# If hash_algs is unset and the MD5 hasn't already been computed,
+# default to an MD5 hash_alg to hash the data on-the-fly.
+if hash_algs is None and not self.md5:
+hash_algs = {'md5': md5}
+digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {})
+def sender(http_conn, method, path, data, headers):
+# This function is called repeatedly for temporary retries
+# so we must be sure the file pointer is pointing at the
+# start of the data.
+if spos is not None and spos != fp.tell():
+fp.seek(spos)
+elif spos is None and self.read_from_stream:
+# if seek is not supported, and we've read from this
+# stream already, then we need to abort retries to
+# avoid setting bad data.
+raise provider.storage_data_error(
+'Cannot retry failed request. fp does not support seeking.')
+# If the caller explicitly specified host header, tell putrequest
+# not to add a second host header. Similarly for accept-encoding.
+skips = {}
+if boto.utils.find_matching_headers('host', headers):
+skips['skip_host'] = 1
+if boto.utils.find_matching_headers('accept-encoding', headers):
+skips['skip_accept_encoding'] = 1
+http_conn.putrequest(method, path, **skips)
+for key in headers:
+http_conn.putheader(key, headers[key])
+http_conn.endheaders()
+save_debug = self.bucket.connection.debug
+self.bucket.connection.debug = 0
+# If the debuglevel < 4 we don't want to show connection
+# payload, so turn off HTTP connection-level debug output (to
+# be restored below).
+# Use the getattr approach to allow this to work in AppEngine.
+if getattr(http_conn, 'debuglevel', 0) < 4:
+http_conn.set_debuglevel(0)
+data_len = 0
+if cb:
+if size:
+cb_size = size
+elif self.size:
+cb_size = self.size
+else:
+cb_size = 0
+if chunked_transfer and cb_size == 0:
+# For chunked Transfer, we call the cb for every 1MB
+# of data transferred, except when we know size.
+cb_count = (1024 * 1024) / self.BufferSize
+elif num_cb > 1:
+cb_count = int(
+math.ceil(cb_size / self.BufferSize / (num_cb - 1.0)))
+elif num_cb < 0:
+cb_count = -1
+else:
+cb_count = 0
+i = 0
+cb(data_len, cb_size)
+bytes_togo = size
+if bytes_togo and bytes_togo < self.BufferSize:
+chunk = fp.read(bytes_togo)
+else:
+chunk = fp.read(self.BufferSize)
+if not isinstance(chunk, bytes):
+chunk = chunk.encode('utf-8')
+if spos is None:
+# read at least something from a non-seekable fp.
+self.read_from_stream = True
+while chunk:
+chunk_len = len(chunk)
+data_len += chunk_len
+if chunked_transfer:
+http_conn.send('%x;\r\n' % chunk_len)
+http_conn.send(chunk)
+http_conn.send('\r\n')
+else:
+http_conn.send(chunk)
+for alg in digesters:
+digesters[alg].update(chunk)
+if bytes_togo:
+bytes_togo -= chunk_len
+if bytes_togo <= 0:
+break
+if cb:
+i += 1
+if i == cb_count or cb_count == -1:
+cb(data_len, cb_size)
+i = 0
+if bytes_togo and bytes_togo < self.BufferSize:
+chunk = fp.read(bytes_togo)
+else:
+chunk = fp.read(self.BufferSize)
+if not isinstance(chunk, bytes):
+chunk = chunk.encode('utf-8')
+self.size = data_len
+for alg in digesters:
+self.local_hashes[alg] = digesters[alg].digest()
+if chunked_transfer:
+http_conn.send('0\r\n')
+# http_conn.send("Content-MD5: %s\r\n" % self.base64md5)
+http_conn.send('\r\n')
+if cb and (cb_count <= 1 or i > 0) and data_len > 0:
+cb(data_len, cb_size)
+http_conn.set_debuglevel(save_debug)
+self.bucket.connection.debug = save_debug
+response = http_conn.getresponse()
+body = response.read()
+if not self.should_retry(response, chunked_transfer):
+raise provider.storage_response_error(
+response.status, response.reason, body)
+return response
+if not headers:
+headers = {}
+else:
+headers = headers.copy()
+# Overwrite user-supplied user-agent.
+for header in find_matching_headers('User-Agent', headers):
+del headers[header]
+headers['User-Agent'] = UserAgent
+# If storage_class is None, then a user has not explicitly requested
+# a storage class, so we can assume STANDARD here
+if self._storage_class not in [None, 'STANDARD']:
+headers[provider.storage_class_header] = self.storage_class
+if find_matching_headers('Content-Encoding', headers):
+self.content_encoding = merge_headers_by_name(
+'Content-Encoding', headers)
+if find_matching_headers('Content-Language', headers):
+self.content_language = merge_headers_by_name(
+'Content-Language', headers)
+content_type_headers = find_matching_headers('Content-Type', headers)
+if content_type_headers:
+# Some use cases need to suppress sending of the Content-Type
+# header and depend on the receiving server to set the content
+# type. This can be achieved by setting headers['Content-Type']
+# to None when calling this method.
+if (len(content_type_headers) == 1 and
+headers[content_type_headers[0]] is None):
+# Delete null Content-Type value to skip sending that header.
+del headers[content_type_headers[0]]
+else:
+self.content_type = merge_headers_by_name(
+'Content-Type', headers)
+elif self.path:
+self.content_type = mimetypes.guess_type(self.path)[0]
+if self.content_type is None:
+self.content_type = self.DefaultContentType
+headers['Content-Type'] = self.content_type
+else:
+headers['Content-Type'] = self.content_type
+if self.base64md5:
+headers['Content-MD5'] = self.base64md5
+if chunked_transfer:
+headers['Transfer-Encoding'] = 'chunked'
+#if not self.base64md5:
+#    headers['Trailer'] = "Content-MD5"
+else:
+headers['Content-Length'] = str(self.size)
+# This is terrible. We need a SHA256 of the body for SigV4, but to do
+# the chunked ``sender`` behavior above, the ``fp`` isn't available to
+# the auth mechanism (because closures). Detect if it's SigV4 & embelish
+# while we can before the auth calculations occur.
+if 'hmac-v4-s3' in self.bucket.connection._required_auth_capability():
+kwargs = {'fp': fp, 'hash_algorithm': hashlib.sha256}
+if size is not None:
+kwargs['size'] = size
+headers['_sha256'] = compute_hash(**kwargs)[0]
+headers['Expect'] = '100-Continue'
+headers = boto.utils.merge_meta(headers, self.metadata, provider)
+resp = self.bucket.connection.make_request(
+'PUT',
+self.bucket.name,
+self.name,
+headers,
+sender=sender,
+query_args=query_args
+)
+self.handle_version_headers(resp, force=True)
+self.handle_addl_headers(resp.getheaders())
+def should_retry(self, response, chunked_transfer=False):
+provider = self.bucket.connection.provider
+if not chunked_transfer:
+if response.status in [500, 503]:
+# 500 & 503 can be plain retries.
+return True
+if response.getheader('location'):
+# If there's a redirect, plain retry.
+return True
+if 200 <= response.status <= 299:
+self.etag = response.getheader('etag')
+md5 = self.md5
+if isinstance(md5, bytes):
+md5 = md5.decode('utf-8')
+# If you use customer-provided encryption keys, the ETag value that
+# Amazon S3 returns in the response will not be the MD5 of the
+# object.
+amz_server_side_encryption_customer_algorithm = response.getheader(
+'x-amz-server-side-encryption-customer-algorithm', None)
+# The same is applicable for KMS-encrypted objects in gs buckets.
+goog_customer_managed_encryption = response.getheader(
+'x-goog-encryption-kms-key-name', None)
+if (amz_server_side_encryption_customer_algorithm is None and
+goog_customer_managed_encryption is None):
+if self.etag != '"%s"' % md5:
+raise provider.storage_data_error(
+'ETag from S3 did not match computed MD5. '
+'%s vs. %s' % (self.etag, self.md5))
+return True
+if response.status == 400:
+# The 400 must be trapped so the retry handler can check to
+# see if it was a timeout.
+# If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb
+# out.
+body = response.read()
+err = provider.storage_response_error(
+response.status,
+response.reason,
+body
+)
+if err.error_code in ['RequestTimeout']:
+raise PleaseRetryException(
+"Saw %s, retrying" % err.error_code,
+response=response
+)
+return False
+def compute_md5(self, fp, size=None):
+"""
+:type fp: file
+:param fp: File pointer to the file to MD5 hash.  The file
+pointer will be reset to the same position before the
+method returns.
+:type size: int
+:param size: (optional) The Maximum number of bytes to read
+from the file pointer (fp). This is useful when uploading
+a file in multiple parts where the file is being split
+in place into different parts. Less bytes may be available.
+"""
+hex_digest, b64_digest, data_size = compute_md5(fp, size=size)
+# Returned values are MD5 hash, base64 encoded MD5 hash, and data size.
+# The internal implementation of compute_md5() needs to return the
+# data size but we don't want to return that value to the external
+# caller because it changes the class interface (i.e. it might
+# break some code) so we consume the third tuple value here and
+# return the remainder of the tuple to the caller, thereby preserving
+# the existing interface.
+self.size = data_size
+return (hex_digest, b64_digest)
+def set_contents_from_stream(self, fp, headers=None, replace=True,
+cb=None, num_cb=10, policy=None,
+reduced_redundancy=False, query_args=None,
+size=None):
+"""
+Store an object using the name of the Key object as the key in
+cloud and the contents of the data stream pointed to by 'fp' as
+the contents.
+The stream object is not seekable and total size is not known.
+This has the implication that we can't specify the
+Content-Size and Content-MD5 in the header. So for huge
+uploads, the delay in calculating MD5 is avoided but with a
+penalty of inability to verify the integrity of the uploaded
+data.
+:type fp: file
+:param fp: the file whose contents are to be uploaded
+:type headers: dict
+:param headers: additional HTTP headers to be sent with the
+PUT request.
+:type replace: bool
+:param replace: If this parameter is False, the method will first check
+to see if an object exists in the bucket with the same key. If it
+does, it won't overwrite it. The default value is True which will
+overwrite the object.
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload. The callback should accept two integer
+parameters, the first representing the number of bytes that have
+been successfully transmitted to GS and the second representing the
+total number of bytes that need to be transmitted.
+:type num_cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter, this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type policy: :class:`boto.gs.acl.CannedACLStrings`
+:param policy: A canned ACL policy that will be applied to the new key
+in GS.
+:type reduced_redundancy: bool
+:param reduced_redundancy: If True, this will set the storage
+class of the new Key to be REDUCED_REDUNDANCY. The Reduced
+Redundancy Storage (RRS) feature of S3, provides lower
+redundancy at lower storage cost.
+:type size: int
+:param size: (optional) The Maximum number of bytes to read from
+the file pointer (fp). This is useful when uploading a
+file in multiple parts where you are splitting the file up
+into different ranges to be uploaded. If not specified,
+the default behaviour is to read all bytes from the file
+pointer. Less bytes may be available.
+"""
+provider = self.bucket.connection.provider
+if not provider.supports_chunked_transfer():
+raise BotoClientError('%s does not support chunked transfer'
+% provider.get_provider_name())
+# Name of the Object should be specified explicitly for Streams.
+if not self.name or self.name == '':
+raise BotoClientError('Cannot determine the destination '
+'object name for the given stream')
+if headers is None:
+headers = {}
+if policy:
+headers[provider.acl_header] = policy
+if reduced_redundancy:
+self.storage_class = 'REDUCED_REDUNDANCY'
+if provider.storage_class_header:
+headers[provider.storage_class_header] = self.storage_class
+if self.bucket is not None:
+if not replace:
+if self.bucket.lookup(self.name):
+return
+self.send_file(fp, headers, cb, num_cb, query_args,
+chunked_transfer=True, size=size)
+def set_contents_from_file(self, fp, headers=None, replace=True,
+cb=None, num_cb=10, policy=None, md5=None,
+reduced_redundancy=False, query_args=None,
+encrypt_key=False, size=None, rewind=False):
+"""
+Store an object in S3 using the name of the Key object as the
+key in S3 and the contents of the file pointed to by 'fp' as the
+contents. The data is read from 'fp' from its current position until
+'size' bytes have been read or EOF.
+:type fp: file
+:param fp: the file whose contents to upload
+:type headers: dict
+:param headers: Additional HTTP headers that will be sent with
+the PUT request.
+:type replace: bool
+:param replace: If this parameter is False, the method will
+first check to see if an object exists in the bucket with
+the same key.  If it does, it won't overwrite it.  The
+default value is True which will overwrite the object.
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type num_cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type policy: :class:`boto.s3.acl.CannedACLStrings`
+:param policy: A canned ACL policy that will be applied to the
+new key in S3.
+:type md5: A tuple containing the hexdigest version of the MD5
+checksum of the file as the first element and the
+Base64-encoded version of the plain checksum as the second
+element.  This is the same format returned by the
+compute_md5 method.
+:param md5: If you need to compute the MD5 for any reason
+prior to upload, it's silly to have to do it twice so this
+param, if present, will be used as the MD5 values of the
+file.  Otherwise, the checksum will be computed.
+:type reduced_redundancy: bool
+:param reduced_redundancy: If True, this will set the storage
+class of the new Key to be REDUCED_REDUNDANCY. The Reduced
+Redundancy Storage (RRS) feature of S3, provides lower
+redundancy at lower storage cost.
+:type encrypt_key: bool
+:param encrypt_key: If True, the new copy of the object will
+be encrypted on the server-side by S3 and will be stored
+in an encrypted form while at rest in S3.
+:type size: int
+:param size: (optional) The Maximum number of bytes to read
+from the file pointer (fp). This is useful when uploading
+a file in multiple parts where you are splitting the file
+up into different ranges to be uploaded. If not specified,
+the default behaviour is to read all bytes from the file
+pointer. Less bytes may be available.
+:type rewind: bool
+:param rewind: (optional) If True, the file pointer (fp) will
+be rewound to the start before any bytes are read from
+it. The default behaviour is False which reads from the
+current position of the file pointer (fp).
+:rtype: int
+:return: The number of bytes written to the key.
+"""
+provider = self.bucket.connection.provider
+headers = headers or {}
+if policy:
+headers[provider.acl_header] = policy
+if encrypt_key:
+headers[provider.server_side_encryption_header] = 'AES256'
+if rewind:
+# caller requests reading from beginning of fp.
+fp.seek(0, os.SEEK_SET)
+else:
+# The following seek/tell/seek logic is intended
+# to detect applications using the older interface to
+# set_contents_from_file(), which automatically rewound the
+# file each time the Key was reused. This changed with commit
+# 14ee2d03f4665fe20d19a85286f78d39d924237e, to support uploads
+# split into multiple parts and uploaded in parallel, and at
+# the time of that commit this check was added because otherwise
+# older programs would get a success status and upload an empty
+# object. Unfortuantely, it's very inefficient for fp's implemented
+# by KeyFile (used, for example, by gsutil when copying between
+# providers). So, we skip the check for the KeyFile case.
+# TODO: At some point consider removing this seek/tell/seek
+# logic, after enough time has passed that it's unlikely any
+# programs remain that assume the older auto-rewind interface.
+if not isinstance(fp, KeyFile):
+spos = fp.tell()
+fp.seek(0, os.SEEK_END)
+if fp.tell() == spos:
+fp.seek(0, os.SEEK_SET)
+if fp.tell() != spos:
+# Raise an exception as this is likely a programming
+# error whereby there is data before the fp but nothing
+# after it.
+fp.seek(spos)
+raise AttributeError('fp is at EOF. Use rewind option '
+'or seek() to data start.')
+# seek back to the correct position.
+fp.seek(spos)
+if reduced_redundancy:
+self.storage_class = 'REDUCED_REDUNDANCY'
+if provider.storage_class_header:
+headers[provider.storage_class_header] = self.storage_class
+# TODO - What if provider doesn't support reduced reduncancy?
+# What if different providers provide different classes?
+if hasattr(fp, 'name'):
+self.path = fp.name
+if self.bucket is not None:
+if not md5 and provider.supports_chunked_transfer():
+# defer md5 calculation to on the fly and
+# we don't know anything about size yet.
+chunked_transfer = True
+self.size = None
+else:
+chunked_transfer = False
+if isinstance(fp, KeyFile):
+# Avoid EOF seek for KeyFile case as it's very inefficient.
+key = fp.getkey()
+size = key.size - fp.tell()
+self.size = size
+# At present both GCS and S3 use MD5 for the etag for
+# non-multipart-uploaded objects. If the etag is 32 hex
+# chars use it as an MD5, to avoid having to read the file
+# twice while transferring.
+if (re.match('^"[a-fA-F0-9]{32}"$', key.etag)):
+etag = key.etag.strip('"')
+md5 = (etag, base64.b64encode(binascii.unhexlify(etag)))
+if not md5:
+# compute_md5() and also set self.size to actual
+# size of the bytes read computing the md5.
+md5 = self.compute_md5(fp, size)
+# adjust size if required
+size = self.size
+elif size:
+self.size = size
+else:
+# If md5 is provided, still need to size so
+# calculate based on bytes to end of content
+spos = fp.tell()
+fp.seek(0, os.SEEK_END)
+self.size = fp.tell() - spos
+fp.seek(spos)
+size = self.size
+self.md5 = md5[0]
+self.base64md5 = md5[1]
+if self.name is None:
+self.name = self.md5
+if not replace:
+if self.bucket.lookup(self.name):
+return
+self.send_file(fp, headers=headers, cb=cb, num_cb=num_cb,
+query_args=query_args,
+chunked_transfer=chunked_transfer, size=size)
+# return number of bytes written.
+return self.size
+def set_contents_from_filename(self, filename, headers=None, replace=True,
+cb=None, num_cb=10, policy=None, md5=None,
+reduced_redundancy=False,
+encrypt_key=False):
+"""
+Store an object in S3 using the name of the Key object as the
+key in S3 and the contents of the file named by 'filename'.
+See set_contents_from_file method for details about the
+parameters.
+:type filename: string
+:param filename: The name of the file that you want to put onto S3
+:type headers: dict
+:param headers: Additional headers to pass along with the
+request to AWS.
+:type replace: bool
+:param replace: If True, replaces the contents of the file
+if it already exists.
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type policy: :class:`boto.s3.acl.CannedACLStrings`
+:param policy: A canned ACL policy that will be applied to the
+new key in S3.
+:type md5: A tuple containing the hexdigest version of the MD5
+checksum of the file as the first element and the
+Base64-encoded version of the plain checksum as the second
+element.  This is the same format returned by the
+compute_md5 method.
+:param md5: If you need to compute the MD5 for any reason
+prior to upload, it's silly to have to do it twice so this
+param, if present, will be used as the MD5 values of the
+file.  Otherwise, the checksum will be computed.
+:type reduced_redundancy: bool
+:param reduced_redundancy: If True, this will set the storage
+class of the new Key to be REDUCED_REDUNDANCY. The Reduced
+Redundancy Storage (RRS) feature of S3, provides lower
+redundancy at lower storage cost.  :type encrypt_key: bool
+:param encrypt_key: If True, the new copy of the object
+will be encrypted on the server-side by S3 and will be
+stored in an encrypted form while at rest in S3.
+:rtype: int
+:return: The number of bytes written to the key.
+"""
+with open(filename, 'rb') as fp:
+return self.set_contents_from_file(fp, headers, replace, cb,
+num_cb, policy, md5,
+reduced_redundancy,
+encrypt_key=encrypt_key)
+def set_contents_from_string(self, string_data, headers=None, replace=True,
+cb=None, num_cb=10, policy=None, md5=None,
+reduced_redundancy=False,
+encrypt_key=False):
+"""
+Store an object in S3 using the name of the Key object as the
+key in S3 and the string 's' as the contents.
+See set_contents_from_file method for details about the
+parameters.
+:type headers: dict
+:param headers: Additional headers to pass along with the
+request to AWS.
+:type replace: bool
+:param replace: If True, replaces the contents of the file if
+it already exists.
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type num_cb: int
+:param num_cb: (optional) If a callback is specified with the
+num_cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type policy: :class:`boto.s3.acl.CannedACLStrings`
+:param policy: A canned ACL policy that will be applied to the
+new key in S3.
+:type md5: A tuple containing the hexdigest version of the MD5
+checksum of the file as the first element and the
+Base64-encoded version of the plain checksum as the second
+element.  This is the same format returned by the
+compute_md5 method.
+:param md5: If you need to compute the MD5 for any reason
+prior to upload, it's silly to have to do it twice so this
+param, if present, will be used as the MD5 values of the
+file.  Otherwise, the checksum will be computed.
+:type reduced_redundancy: bool
+:param reduced_redundancy: If True, this will set the storage
+class of the new Key to be REDUCED_REDUNDANCY. The Reduced
+Redundancy Storage (RRS) feature of S3, provides lower
+redundancy at lower storage cost.
+:type encrypt_key: bool
+:param encrypt_key: If True, the new copy of the object will
+be encrypted on the server-side by S3 and will be stored
+in an encrypted form while at rest in S3.
+"""
+if not isinstance(string_data, bytes):
+string_data = string_data.encode("utf-8")
+fp = BytesIO(string_data)
+r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
+policy, md5, reduced_redundancy,
+encrypt_key=encrypt_key)
+fp.close()
+return r
+def get_file(self, fp, headers=None, cb=None, num_cb=10,
+torrent=False, version_id=None, override_num_retries=None,
+response_headers=None):
+"""
+Retrieves a file from an S3 Key
+:type fp: file
+:param fp: File pointer to put the data into
+:type headers: string
+:param: headers to send when retrieving the files
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type torrent: bool
+:param torrent: Flag for whether to get a torrent for the file
+:type override_num_retries: int
+:param override_num_retries: If not None will override configured
+num_retries parameter for underlying GET.
+:type response_headers: dict
+:param response_headers: A dictionary containing HTTP
+headers/values that will override any headers associated
+with the stored object in the response.  See
+http://goo.gl/EWOPb for details.
+:type version_id: str
+:param version_id: The ID of a particular version of the object.
+If this parameter is not supplied but the Key object has
+a ``version_id`` attribute, that value will be used when
+retrieving the object.  You can set the Key object's
+``version_id`` attribute to None to always grab the latest
+version from a version-enabled bucket.
+"""
+self._get_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb,
+torrent=torrent, version_id=version_id,
+override_num_retries=override_num_retries,
+response_headers=response_headers,
+hash_algs=None,
+query_args=None)
+def _get_file_internal(self, fp, headers=None, cb=None, num_cb=10,
+torrent=False, version_id=None, override_num_retries=None,
+response_headers=None, hash_algs=None, query_args=None):
+if headers is None:
+headers = {}
+save_debug = self.bucket.connection.debug
+if self.bucket.connection.debug == 1:
+self.bucket.connection.debug = 0
+query_args = query_args or []
+if torrent:
+query_args.append('torrent')
+if hash_algs is None and not torrent:
+hash_algs = {'md5': md5}
+digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {})
+# If a version_id is passed in, use that.  If not, check to see
+# if the Key object has an explicit version_id and, if so, use that.
+# Otherwise, don't pass a version_id query param.
+if version_id is None:
+version_id = self.version_id
+if version_id:
+query_args.append('versionId=%s' % version_id)
+if response_headers:
+for key in response_headers:
+query_args.append('%s=%s' % (
+key, urllib.parse.quote(response_headers[key])))
+query_args = '&'.join(query_args)
+self.open('r', headers, query_args=query_args,
+override_num_retries=override_num_retries)
+data_len = 0
+if cb:
+if self.size is None:
+cb_size = 0
+else:
+cb_size = self.size
+if self.size is None and num_cb != -1:
+# If size is not available due to chunked transfer for example,
+# we'll call the cb for every 1MB of data transferred.
+cb_count = (1024 * 1024) / self.BufferSize
+elif num_cb > 1:
+cb_count = int(math.ceil(cb_size/self.BufferSize/(num_cb-1.0)))
+elif num_cb < 0:
+cb_count = -1
+else:
+cb_count = 0
+i = 0
+cb(data_len, cb_size)
+try:
+for bytes in self:
+fp.write(bytes)
+data_len += len(bytes)
+for alg in digesters:
+digesters[alg].update(bytes)
+if cb:
+if cb_size > 0 and data_len >= cb_size:
+break
+i += 1
+if i == cb_count or cb_count == -1:
+cb(data_len, cb_size)
+i = 0
+except IOError as e:
+if e.errno == errno.ENOSPC:
+raise StorageDataError('Out of space for destination file '
+'%s' % fp.name)
+raise
+if cb and (cb_count <= 1 or i > 0) and data_len > 0:
+cb(data_len, cb_size)
+for alg in digesters:
+self.local_hashes[alg] = digesters[alg].digest()
+if self.size is None and not torrent and "Range" not in headers:
+self.size = data_len
+self.close()
+self.bucket.connection.debug = save_debug
+def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10):
+"""
+Get a torrent file (see to get_file)
+:type fp: file
+:param fp: The file pointer of where to put the torrent
+:type headers: dict
+:param headers: Headers to be passed
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+"""
+return self.get_file(fp, headers, cb, num_cb, torrent=True)
+def get_contents_to_file(self, fp, headers=None,
+cb=None, num_cb=10,
+torrent=False,
+version_id=None,
+res_download_handler=None,
+response_headers=None):
+"""
+Retrieve an object from S3 using the name of the Key object as the
+key in S3.  Write the contents of the object to the file pointed
+to by 'fp'.
+:type fp: File -like object
+:param fp:
+:type headers: dict
+:param headers: additional HTTP headers that will be sent with
+the GET request.
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type torrent: bool
+:param torrent: If True, returns the contents of a torrent
+file as a string.
+:type res_upload_handler: ResumableDownloadHandler
+:param res_download_handler: If provided, this handler will
+perform the download.
+:type response_headers: dict
+:param response_headers: A dictionary containing HTTP
+headers/values that will override any headers associated
+with the stored object in the response.  See
+http://goo.gl/EWOPb for details.
+:type version_id: str
+:param version_id: The ID of a particular version of the object.
+If this parameter is not supplied but the Key object has
+a ``version_id`` attribute, that value will be used when
+retrieving the object.  You can set the Key object's
+``version_id`` attribute to None to always grab the latest
+version from a version-enabled bucket.
+"""
+if self.bucket is not None:
+if res_download_handler:
+res_download_handler.get_file(self, fp, headers, cb, num_cb,
+torrent=torrent,
+version_id=version_id)
+else:
+self.get_file(fp, headers, cb, num_cb, torrent=torrent,
+version_id=version_id,
+response_headers=response_headers)
+def get_contents_to_filename(self, filename, headers=None,
+cb=None, num_cb=10,
+torrent=False,
+version_id=None,
+res_download_handler=None,
+response_headers=None):
+"""
+Retrieve an object from S3 using the name of the Key object as the
+key in S3.  Store contents of the object to a file named by 'filename'.
+See get_contents_to_file method for details about the
+parameters.
+:type filename: string
+:param filename: The filename of where to put the file contents
+:type headers: dict
+:param headers: Any additional headers to send in the request
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type num_cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type torrent: bool
+:param torrent: If True, returns the contents of a torrent file
+as a string.
+:type res_upload_handler: ResumableDownloadHandler
+:param res_download_handler: If provided, this handler will
+perform the download.
+:type response_headers: dict
+:param response_headers: A dictionary containing HTTP
+headers/values that will override any headers associated
+with the stored object in the response.  See
+http://goo.gl/EWOPb for details.
+:type version_id: str
+:param version_id: The ID of a particular version of the object.
+If this parameter is not supplied but the Key object has
+a ``version_id`` attribute, that value will be used when
+retrieving the object.  You can set the Key object's
+``version_id`` attribute to None to always grab the latest
+version from a version-enabled bucket.
+"""
+try:
+with open(filename, 'wb') as fp:
+self.get_contents_to_file(fp, headers, cb, num_cb,
+torrent=torrent,
+version_id=version_id,
+res_download_handler=res_download_handler,
+response_headers=response_headers)
+except Exception:
+os.remove(filename)
+raise
+# if last_modified date was sent from s3, try to set file's timestamp
+if self.last_modified is not None:
+try:
+modified_tuple = email.utils.parsedate_tz(self.last_modified)
+modified_stamp = int(email.utils.mktime_tz(modified_tuple))
+os.utime(fp.name, (modified_stamp, modified_stamp))
+except Exception:
+pass
+def get_contents_as_string(self, headers=None,
+cb=None, num_cb=10,
+torrent=False,
+version_id=None,
+response_headers=None, encoding=None):
+"""
+Retrieve an object from S3 using the name of the Key object as the
+key in S3.  Return the contents of the object as a string.
+See get_contents_to_file method for details about the
+parameters.
+:type headers: dict
+:param headers: Any additional headers to send in the request
+:type cb: function
+:param cb: a callback function that will be called to report
+progress on the upload.  The callback should accept two
+integer parameters, the first representing the number of
+bytes that have been successfully transmitted to S3 and
+the second representing the size of the to be transmitted
+object.
+:type cb: int
+:param num_cb: (optional) If a callback is specified with the
+cb parameter this parameter determines the granularity of
+the callback by defining the maximum number of times the
+callback will be called during the file transfer.
+:type torrent: bool
+:param torrent: If True, returns the contents of a torrent file
+as a string.
+:type response_headers: dict
+:param response_headers: A dictionary containing HTTP
+headers/values that will override any headers associated
+with the stored object in the response.  See
+http://goo.gl/EWOPb for details.
+:type version_id: str
+:param version_id: The ID of a particular version of the object.
+If this parameter is not supplied but the Key object has
+a ``version_id`` attribute, that value will be used when
+retrieving the object.  You can set the Key object's
+``version_id`` attribute to None to always grab the latest
+version from a version-enabled bucket.
+:type encoding: str
+:param encoding: The text encoding to use, such as ``utf-8``
+or ``iso-8859-1``. If set, then a string will be returned.
+Defaults to ``None`` and returns bytes.
+:rtype: bytes or str
+:returns: The contents of the file as bytes or a string
+"""
+fp = BytesIO()
+self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent,
+version_id=version_id,
+response_headers=response_headers)
+value = fp.getvalue()
+if encoding is not None:
+value = value.decode(encoding)
+return value
+def add_email_grant(self, permission, email_address, headers=None):
+"""
+Convenience method that provides a quick way to add an email grant
+to a key. This method retrieves the current ACL, creates a new
+grant based on the parameters passed in, adds that grant to the ACL
+and then PUT's the new ACL back to S3.
+:type permission: string
+:param permission: The permission being granted. Should be one of:
+(READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL).
+:type email_address: string
+:param email_address: The email address associated with the AWS
+account your are granting the permission to.
+:type recursive: boolean
+:param recursive: A boolean value to controls whether the
+command will apply the grant to all keys within the bucket
+or not.  The default value is False.  By passing a True
+value, the call will iterate through all keys in the
+bucket and apply the same grant to each key.  CAUTION: If
+you have a lot of keys, this could take a long time!
+"""
+policy = self.get_acl(headers=headers)
+policy.acl.add_email_grant(permission, email_address)
+self.set_acl(policy, headers=headers)
+def add_user_grant(self, permission, user_id, headers=None,
+display_name=None):
+"""
+Convenience method that provides a quick way to add a canonical
+user grant to a key.  This method retrieves the current ACL,
+creates a new grant based on the parameters passed in, adds that
+grant to the ACL and then PUT's the new ACL back to S3.
+:type permission: string
+:param permission: The permission being granted. Should be one of:
+(READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL).
+:type user_id: string
+:param user_id: The canonical user id associated with the AWS
+account your are granting the permission to.
+:type display_name: string
+:param display_name: An option string containing the user's
+Display Name.  Only required on Walrus.
+"""
+policy = self.get_acl(headers=headers)
+policy.acl.add_user_grant(permission, user_id,
+display_name=display_name)
+self.set_acl(policy, headers=headers)
+def _normalize_metadata(self, metadata):
+if type(metadata) == set:
+norm_metadata = set()
+for k in metadata:
+norm_metadata.add(k.lower())
+else:
+norm_metadata = {}
+for k in metadata:
+norm_metadata[k.lower()] = metadata[k]
+return norm_metadata
+def _get_remote_metadata(self, headers=None):
+"""
+Extracts metadata from existing URI into a dict, so we can
+overwrite/delete from it to form the new set of metadata to apply to a
+key.
+"""
+metadata = {}
+for underscore_name in self._underscore_base_user_settable_fields:
+if hasattr(self, underscore_name):
+value = getattr(self, underscore_name)
+if value:
+# Generate HTTP field name corresponding to "_" named field.
+field_name = underscore_name.replace('_', '-')
+metadata[field_name.lower()] = value
+# self.metadata contains custom metadata, which are all user-settable.
+prefix = self.provider.metadata_prefix
+for underscore_name in self.metadata:
+field_name = underscore_name.replace('_', '-')
+metadata['%s%s' % (prefix, field_name.lower())] = (
+self.metadata[underscore_name])
+return metadata
+def set_remote_metadata(self, metadata_plus, metadata_minus, preserve_acl,
+headers=None):
+metadata_plus = self._normalize_metadata(metadata_plus)
+metadata_minus = self._normalize_metadata(metadata_minus)
+metadata = self._get_remote_metadata()
+metadata.update(metadata_plus)
+for h in metadata_minus:
+if h in metadata:
+del metadata[h]
+src_bucket = self.bucket
+# Boto prepends the meta prefix when adding headers, so strip prefix in
+# metadata before sending back in to copy_key() call.
+rewritten_metadata = {}
+for h in metadata:
+if (h.startswith('x-goog-meta-') or h.startswith('x-amz-meta-')):
+rewritten_h = (h.replace('x-goog-meta-', '')
+.replace('x-amz-meta-', ''))
+else:
+rewritten_h = h
+rewritten_metadata[rewritten_h] = metadata[h]
+metadata = rewritten_metadata
+src_bucket.copy_key(self.name, self.bucket.name, self.name,
+metadata=metadata, preserve_acl=preserve_acl,
+headers=headers)
+def restore(self, days, headers=None):
+"""Restore an object from an archive.
+:type days: int
+:param days: The lifetime of the restored object (must
+be at least 1 day).  If the object is already restored
+then this parameter can be used to readjust the lifetime
+of the restored object.  In this case, the days
+param is with respect to the initial time of the request.
+If the object has not been restored, this param is with
+respect to the completion time of the request.
+"""
+response = self.bucket.connection.make_request(
+'POST', self.bucket.name, self.name,
+data=self.RestoreBody % days,
+headers=headers, query_args='restore')
+if response.status not in (200, 202):
+provider = self.bucket.connection.provider
+raise provider.storage_response_error(response.status,
+response.reason,
+response.read())

Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/boto/s3/key.py @ 0:26e78fe6e8c4 draft