Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/boto/dynamodb2/results.py @ 2:6af9afd405e9 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 0a63dd5f4d38a1f6944587f52a8cd79874177fc1"
author shellac
date Thu, 14 May 2020 14:56:58 -0400
parents 26e78fe6e8c4
children
comparison
equal deleted inserted replaced
1:75ca89e9b81c 2:6af9afd405e9
1 class ResultSet(object):
2 """
3 A class used to lazily handle page-to-page navigation through a set of
4 results.
5
6 It presents a transparent iterator interface, so that all the user has
7 to do is use it in a typical ``for`` loop (or list comprehension, etc.)
8 to fetch results, even if they weren't present in the current page of
9 results.
10
11 This is used by the ``Table.query`` & ``Table.scan`` methods.
12
13 Example::
14
15 >>> users = Table('users')
16 >>> results = ResultSet()
17 >>> results.to_call(users.query, username__gte='johndoe')
18 # Now iterate. When it runs out of results, it'll fetch the next page.
19 >>> for res in results:
20 ... print res['username']
21
22 """
23 def __init__(self, max_page_size=None):
24 super(ResultSet, self).__init__()
25 self.the_callable = None
26 self.call_args = []
27 self.call_kwargs = {}
28 self._results = []
29 self._offset = -1
30 self._results_left = True
31 self._last_key_seen = None
32 self._fetches = 0
33 self._max_page_size = max_page_size
34 self._limit = None
35
36 @property
37 def first_key(self):
38 return 'exclusive_start_key'
39
40 def _reset(self):
41 """
42 Resets the internal state of the ``ResultSet``.
43
44 This prevents results from being cached long-term & consuming
45 excess memory.
46
47 Largely internal.
48 """
49 self._results = []
50 self._offset = 0
51
52 def __iter__(self):
53 return self
54
55 def __next__(self):
56 self._offset += 1
57
58 if self._offset >= len(self._results):
59 if self._results_left is False:
60 raise StopIteration()
61
62 self.fetch_more()
63
64 # It's possible that previous call to ``fetch_more`` may not return
65 # anything useful but there may be more results. Loop until we get
66 # something back, making sure we guard for no results left.
67 while not len(self._results) and self._results_left:
68 self.fetch_more()
69
70 if self._offset < len(self._results):
71 if self._limit is not None:
72 self._limit -= 1
73
74 if self._limit < 0:
75 raise StopIteration()
76
77 return self._results[self._offset]
78 else:
79 raise StopIteration()
80
81 next = __next__
82
83 def to_call(self, the_callable, *args, **kwargs):
84 """
85 Sets up the callable & any arguments to run it with.
86
87 This is stored for subsequent calls so that those queries can be
88 run without requiring user intervention.
89
90 Example::
91
92 # Just an example callable.
93 >>> def squares_to(y):
94 ... for x in range(1, y):
95 ... yield x**2
96 >>> rs = ResultSet()
97 # Set up what to call & arguments.
98 >>> rs.to_call(squares_to, y=3)
99
100 """
101 if not callable(the_callable):
102 raise ValueError(
103 'You must supply an object or function to be called.'
104 )
105
106 # We pop the ``limit``, if present, to track how many we should return
107 # to the user. This isn't the same as the ``limit`` that the low-level
108 # DDB api calls use (which limit page size, not the overall result set).
109 self._limit = kwargs.pop('limit', None)
110
111 if self._limit is not None and self._limit < 0:
112 self._limit = None
113
114 self.the_callable = the_callable
115 self.call_args = args
116 self.call_kwargs = kwargs
117
118 def fetch_more(self):
119 """
120 When the iterator runs out of results, this method is run to re-execute
121 the callable (& arguments) to fetch the next page.
122
123 Largely internal.
124 """
125 self._reset()
126
127 args = self.call_args[:]
128 kwargs = self.call_kwargs.copy()
129
130 if self._last_key_seen is not None:
131 kwargs[self.first_key] = self._last_key_seen
132
133 # If the page size is greater than limit set them
134 # to the same value
135 if self._limit and self._max_page_size and self._max_page_size > self._limit:
136 self._max_page_size = self._limit
137
138 # Put in the max page size.
139 if self._max_page_size is not None:
140 kwargs['limit'] = self._max_page_size
141 elif self._limit is not None:
142 # If max_page_size is not set and limit is available
143 # use it as the page size
144 kwargs['limit'] = self._limit
145
146 results = self.the_callable(*args, **kwargs)
147 self._fetches += 1
148 new_results = results.get('results', [])
149 self._last_key_seen = results.get('last_key', None)
150
151 if len(new_results):
152 self._results.extend(results['results'])
153
154 # Check the limit, if it's present.
155 if self._limit is not None and self._limit >= 0:
156 limit = self._limit
157 limit -= len(results['results'])
158 # If we've exceeded the limit, we don't have any more
159 # results to look for.
160 if limit <= 0:
161 self._results_left = False
162
163 if self._last_key_seen is None:
164 self._results_left = False
165
166
167 class BatchGetResultSet(ResultSet):
168 def __init__(self, *args, **kwargs):
169 self._keys_left = kwargs.pop('keys', [])
170 self._max_batch_get = kwargs.pop('max_batch_get', 100)
171 super(BatchGetResultSet, self).__init__(*args, **kwargs)
172
173 def fetch_more(self):
174 self._reset()
175
176 args = self.call_args[:]
177 kwargs = self.call_kwargs.copy()
178
179 # Slice off the max we can fetch.
180 kwargs['keys'] = self._keys_left[:self._max_batch_get]
181 self._keys_left = self._keys_left[self._max_batch_get:]
182
183 if len(self._keys_left) <= 0:
184 self._results_left = False
185
186 results = self.the_callable(*args, **kwargs)
187
188 if not len(results.get('results', [])):
189 return
190
191 self._results.extend(results['results'])
192
193 for offset, key_data in enumerate(results.get('unprocessed_keys', [])):
194 # We've got an unprocessed key. Reinsert it into the list.
195 # DynamoDB only returns valid keys, so there should be no risk of
196 # missing keys ever making it here.
197 self._keys_left.insert(offset, key_data)
198
199 if len(self._keys_left) > 0:
200 self._results_left = True
201
202 # Decrease the limit, if it's present.
203 if self.call_kwargs.get('limit'):
204 self.call_kwargs['limit'] -= len(results['results'])