Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/boto/sqs/bigmessage.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 # Copyright (c) 2013 Mitch Garnaat http://garnaat.org/
2 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates.
3 #
4 # Permission is hereby granted, free of charge, to any person obtaining a
5 # copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish, dis-
8 # tribute, sublicense, and/or sell copies of the Software, and to permit
9 # persons to whom the Software is furnished to do so, subject to the fol-
10 # lowing conditions:
11 #
12 # The above copyright notice and this permission notice shall be included
13 # in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 # IN THE SOFTWARE.
22
23 import uuid
24
25 import boto
26 from boto.sqs.message import RawMessage
27 from boto.exception import SQSDecodeError
28
29
30 class BigMessage(RawMessage):
31 """
32 The BigMessage class provides large payloads (up to 5GB)
33 by storing the payload itself in S3 and then placing a reference
34 to the S3 object in the actual SQS message payload.
35
36 To create a BigMessage, you should create a BigMessage object
37 and pass in a file-like object as the ``body`` param and also
38 pass in the an S3 URL specifying the bucket in which to store
39 the message body::
40
41 import boto.sqs
42 from boto.sqs.bigmessage import BigMessage
43
44 sqs = boto.sqs.connect_to_region('us-west-2')
45 queue = sqs.get_queue('myqueue')
46 fp = open('/path/to/bigmessage/data')
47 msg = BigMessage(queue, fp, 's3://mybucket')
48 queue.write(msg)
49
50 Passing in a fully-qualified S3 URL (e.g. s3://mybucket/foo)
51 is interpreted to mean that the body of the message is already
52 stored in S3 and the that S3 URL is then used directly with no
53 content uploaded by BigMessage.
54 """
55
56 def __init__(self, queue=None, body=None, s3_url=None):
57 self.s3_url = s3_url
58 super(BigMessage, self).__init__(queue, body)
59
60 def _get_bucket_key(self, s3_url):
61 bucket_name = key_name = None
62 if s3_url:
63 if s3_url.startswith('s3://'):
64 # We need to split out the bucket from the key (if
65 # supplied). We also have to be aware that someone
66 # may provide a trailing '/' character as in:
67 # s3://foo/ and we want to handle that.
68 s3_components = s3_url[5:].split('/', 1)
69 bucket_name = s3_components[0]
70 if len(s3_components) > 1:
71 if s3_components[1]:
72 key_name = s3_components[1]
73 else:
74 msg = 's3_url parameter should start with s3://'
75 raise SQSDecodeError(msg, self)
76 return bucket_name, key_name
77
78 def encode(self, value):
79 """
80 :type value: file-like object
81 :param value: A file-like object containing the content
82 of the message. The actual content will be stored
83 in S3 and a link to the S3 object will be stored in
84 the message body.
85 """
86 bucket_name, key_name = self._get_bucket_key(self.s3_url)
87 if bucket_name and key_name:
88 return self.s3_url
89 key_name = uuid.uuid4()
90 s3_conn = boto.connect_s3()
91 s3_bucket = s3_conn.get_bucket(bucket_name)
92 key = s3_bucket.new_key(key_name)
93 key.set_contents_from_file(value)
94 self.s3_url = 's3://%s/%s' % (bucket_name, key_name)
95 return self.s3_url
96
97 def _get_s3_object(self, s3_url):
98 bucket_name, key_name = self._get_bucket_key(s3_url)
99 if bucket_name and key_name:
100 s3_conn = boto.connect_s3()
101 s3_bucket = s3_conn.get_bucket(bucket_name)
102 key = s3_bucket.get_key(key_name)
103 return key
104 else:
105 msg = 'Unable to decode S3 URL: %s' % s3_url
106 raise SQSDecodeError(msg, self)
107
108 def decode(self, value):
109 self.s3_url = value
110 key = self._get_s3_object(value)
111 return key.get_contents_as_string()
112
113 def delete(self):
114 # Delete the object in S3 first, then delete the SQS message
115 if self.s3_url:
116 key = self._get_s3_object(self.s3_url)
117 key.delete()
118 super(BigMessage, self).delete()
119