Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/boto/glacier/job.py @ 0:d30785e31577 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:18:57 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d30785e31577
1 # -*- coding: utf-8 -*-
2 # Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/
3 #
4 # Permission is hereby granted, free of charge, to any person obtaining a
5 # copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish, dis-
8 # tribute, sublicense, and/or sell copies of the Software, and to permit
9 # persons to whom the Software is furnished to do so, subject to the fol-
10 # lowing conditions:
11 #
12 # The above copyright notice and this permission notice shall be included
13 # in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 # IN THE SOFTWARE.
22 #
23 import math
24 import socket
25
26 from boto.glacier.exceptions import TreeHashDoesNotMatchError, \
27 DownloadArchiveError
28 from boto.glacier.utils import tree_hash_from_str
29
30
31 class Job(object):
32
33 DefaultPartSize = 4 * 1024 * 1024
34
35 ResponseDataElements = (('Action', 'action', None),
36 ('ArchiveId', 'archive_id', None),
37 ('ArchiveSizeInBytes', 'archive_size', 0),
38 ('Completed', 'completed', False),
39 ('CompletionDate', 'completion_date', None),
40 ('CreationDate', 'creation_date', None),
41 ('InventorySizeInBytes', 'inventory_size', 0),
42 ('JobDescription', 'description', None),
43 ('JobId', 'id', None),
44 ('SHA256TreeHash', 'sha256_treehash', None),
45 ('SNSTopic', 'sns_topic', None),
46 ('StatusCode', 'status_code', None),
47 ('StatusMessage', 'status_message', None),
48 ('VaultARN', 'arn', None))
49
50 def __init__(self, vault, response_data=None):
51 self.vault = vault
52 if response_data:
53 for response_name, attr_name, default in self.ResponseDataElements:
54 setattr(self, attr_name, response_data[response_name])
55 else:
56 for response_name, attr_name, default in self.ResponseDataElements:
57 setattr(self, attr_name, default)
58
59 def __repr__(self):
60 return 'Job(%s)' % self.arn
61
62 def get_output(self, byte_range=None, validate_checksum=False):
63 """
64 This operation downloads the output of the job. Depending on
65 the job type you specified when you initiated the job, the
66 output will be either the content of an archive or a vault
67 inventory.
68
69 You can download all the job output or download a portion of
70 the output by specifying a byte range. In the case of an
71 archive retrieval job, depending on the byte range you
72 specify, Amazon Glacier returns the checksum for the portion
73 of the data. You can compute the checksum on the client and
74 verify that the values match to ensure the portion you
75 downloaded is the correct data.
76
77 :type byte_range: tuple
78 :param range: A tuple of integer specifying the slice (in bytes)
79 of the archive you want to receive
80
81 :type validate_checksum: bool
82 :param validate_checksum: Specify whether or not to validate
83 the associate tree hash. If the response does not contain
84 a TreeHash, then no checksum will be verified.
85
86 """
87 response = self.vault.layer1.get_job_output(self.vault.name,
88 self.id,
89 byte_range)
90 if validate_checksum and 'TreeHash' in response:
91 data = response.read()
92 actual_tree_hash = tree_hash_from_str(data)
93 if response['TreeHash'] != actual_tree_hash:
94 raise TreeHashDoesNotMatchError(
95 "The calculated tree hash %s does not match the "
96 "expected tree hash %s for the byte range %s" % (
97 actual_tree_hash, response['TreeHash'], byte_range))
98 return response
99
100 def _calc_num_chunks(self, chunk_size):
101 return int(math.ceil(self.archive_size / float(chunk_size)))
102
103 def download_to_file(self, filename, chunk_size=DefaultPartSize,
104 verify_hashes=True, retry_exceptions=(socket.error,)):
105 """Download an archive to a file by name.
106
107 :type filename: str
108 :param filename: The name of the file where the archive
109 contents will be saved.
110
111 :type chunk_size: int
112 :param chunk_size: The chunk size to use when downloading
113 the archive.
114
115 :type verify_hashes: bool
116 :param verify_hashes: Indicates whether or not to verify
117 the tree hashes for each downloaded chunk.
118
119 """
120 num_chunks = self._calc_num_chunks(chunk_size)
121 with open(filename, 'wb') as output_file:
122 self._download_to_fileob(output_file, num_chunks, chunk_size,
123 verify_hashes, retry_exceptions)
124
125 def download_to_fileobj(self, output_file, chunk_size=DefaultPartSize,
126 verify_hashes=True,
127 retry_exceptions=(socket.error,)):
128 """Download an archive to a file object.
129
130 :type output_file: file
131 :param output_file: The file object where the archive
132 contents will be saved.
133
134 :type chunk_size: int
135 :param chunk_size: The chunk size to use when downloading
136 the archive.
137
138 :type verify_hashes: bool
139 :param verify_hashes: Indicates whether or not to verify
140 the tree hashes for each downloaded chunk.
141
142 """
143 num_chunks = self._calc_num_chunks(chunk_size)
144 self._download_to_fileob(output_file, num_chunks, chunk_size,
145 verify_hashes, retry_exceptions)
146
147 def _download_to_fileob(self, fileobj, num_chunks, chunk_size, verify_hashes,
148 retry_exceptions):
149 for i in range(num_chunks):
150 byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1)
151 data, expected_tree_hash = self._download_byte_range(
152 byte_range, retry_exceptions)
153 if verify_hashes:
154 actual_tree_hash = tree_hash_from_str(data)
155 if expected_tree_hash != actual_tree_hash:
156 raise TreeHashDoesNotMatchError(
157 "The calculated tree hash %s does not match the "
158 "expected tree hash %s for the byte range %s" % (
159 actual_tree_hash, expected_tree_hash, byte_range))
160 fileobj.write(data)
161
162 def _download_byte_range(self, byte_range, retry_exceptions):
163 # You can occasionally get socket.errors when downloading
164 # chunks from Glacier, so each chunk can be retried up
165 # to 5 times.
166 for _ in range(5):
167 try:
168 response = self.get_output(byte_range)
169 data = response.read()
170 expected_tree_hash = response['TreeHash']
171 return data, expected_tree_hash
172 except retry_exceptions as e:
173 continue
174 else:
175 raise DownloadArchiveError("There was an error downloading"
176 "byte range %s: %s" % (byte_range,
177 e))