Mercurial > repos > guerler > springsuite
diff planemo/lib/python3.7/site-packages/boto/glacier/job.py @ 0:d30785e31577 draft
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author | guerler |
---|---|
date | Fri, 31 Jul 2020 00:18:57 -0400 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/planemo/lib/python3.7/site-packages/boto/glacier/job.py Fri Jul 31 00:18:57 2020 -0400 @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import math +import socket + +from boto.glacier.exceptions import TreeHashDoesNotMatchError, \ + DownloadArchiveError +from boto.glacier.utils import tree_hash_from_str + + +class Job(object): + + DefaultPartSize = 4 * 1024 * 1024 + + ResponseDataElements = (('Action', 'action', None), + ('ArchiveId', 'archive_id', None), + ('ArchiveSizeInBytes', 'archive_size', 0), + ('Completed', 'completed', False), + ('CompletionDate', 'completion_date', None), + ('CreationDate', 'creation_date', None), + ('InventorySizeInBytes', 'inventory_size', 0), + ('JobDescription', 'description', None), + ('JobId', 'id', None), + ('SHA256TreeHash', 'sha256_treehash', None), + ('SNSTopic', 'sns_topic', None), + ('StatusCode', 'status_code', None), + ('StatusMessage', 'status_message', None), + ('VaultARN', 'arn', None)) + + def __init__(self, vault, response_data=None): + self.vault = vault + if response_data: + for response_name, attr_name, default in self.ResponseDataElements: + setattr(self, attr_name, response_data[response_name]) + else: + for response_name, attr_name, default in self.ResponseDataElements: + setattr(self, attr_name, default) + + def __repr__(self): + return 'Job(%s)' % self.arn + + def get_output(self, byte_range=None, validate_checksum=False): + """ + This operation downloads the output of the job. Depending on + the job type you specified when you initiated the job, the + output will be either the content of an archive or a vault + inventory. + + You can download all the job output or download a portion of + the output by specifying a byte range. In the case of an + archive retrieval job, depending on the byte range you + specify, Amazon Glacier returns the checksum for the portion + of the data. You can compute the checksum on the client and + verify that the values match to ensure the portion you + downloaded is the correct data. + + :type byte_range: tuple + :param range: A tuple of integer specifying the slice (in bytes) + of the archive you want to receive + + :type validate_checksum: bool + :param validate_checksum: Specify whether or not to validate + the associate tree hash. If the response does not contain + a TreeHash, then no checksum will be verified. + + """ + response = self.vault.layer1.get_job_output(self.vault.name, + self.id, + byte_range) + if validate_checksum and 'TreeHash' in response: + data = response.read() + actual_tree_hash = tree_hash_from_str(data) + if response['TreeHash'] != actual_tree_hash: + raise TreeHashDoesNotMatchError( + "The calculated tree hash %s does not match the " + "expected tree hash %s for the byte range %s" % ( + actual_tree_hash, response['TreeHash'], byte_range)) + return response + + def _calc_num_chunks(self, chunk_size): + return int(math.ceil(self.archive_size / float(chunk_size))) + + def download_to_file(self, filename, chunk_size=DefaultPartSize, + verify_hashes=True, retry_exceptions=(socket.error,)): + """Download an archive to a file by name. + + :type filename: str + :param filename: The name of the file where the archive + contents will be saved. + + :type chunk_size: int + :param chunk_size: The chunk size to use when downloading + the archive. + + :type verify_hashes: bool + :param verify_hashes: Indicates whether or not to verify + the tree hashes for each downloaded chunk. + + """ + num_chunks = self._calc_num_chunks(chunk_size) + with open(filename, 'wb') as output_file: + self._download_to_fileob(output_file, num_chunks, chunk_size, + verify_hashes, retry_exceptions) + + def download_to_fileobj(self, output_file, chunk_size=DefaultPartSize, + verify_hashes=True, + retry_exceptions=(socket.error,)): + """Download an archive to a file object. + + :type output_file: file + :param output_file: The file object where the archive + contents will be saved. + + :type chunk_size: int + :param chunk_size: The chunk size to use when downloading + the archive. + + :type verify_hashes: bool + :param verify_hashes: Indicates whether or not to verify + the tree hashes for each downloaded chunk. + + """ + num_chunks = self._calc_num_chunks(chunk_size) + self._download_to_fileob(output_file, num_chunks, chunk_size, + verify_hashes, retry_exceptions) + + def _download_to_fileob(self, fileobj, num_chunks, chunk_size, verify_hashes, + retry_exceptions): + for i in range(num_chunks): + byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1) + data, expected_tree_hash = self._download_byte_range( + byte_range, retry_exceptions) + if verify_hashes: + actual_tree_hash = tree_hash_from_str(data) + if expected_tree_hash != actual_tree_hash: + raise TreeHashDoesNotMatchError( + "The calculated tree hash %s does not match the " + "expected tree hash %s for the byte range %s" % ( + actual_tree_hash, expected_tree_hash, byte_range)) + fileobj.write(data) + + def _download_byte_range(self, byte_range, retry_exceptions): + # You can occasionally get socket.errors when downloading + # chunks from Glacier, so each chunk can be retried up + # to 5 times. + for _ in range(5): + try: + response = self.get_output(byte_range) + data = response.read() + expected_tree_hash = response['TreeHash'] + return data, expected_tree_hash + except retry_exceptions as e: + continue + else: + raise DownloadArchiveError("There was an error downloading" + "byte range %s: %s" % (byte_range, + e))