Mercurial > repos > guerler > hhblits

comparison lib/python3.8/site-packages/pip/_vendor/msgpack/fallback.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:9e54283cc701
1 """Fallback pure Python implementation of msgpack"""
2
3 import sys
4 import struct
5 import warnings
6
7
8 if sys.version_info[0] == 2:
9 PY2 = True
10 int_types = (int, long)
11 def dict_iteritems(d):
12 return d.iteritems()
13 else:
14 PY2 = False
15 int_types = int
16 unicode = str
17 xrange = range
18 def dict_iteritems(d):
19 return d.items()
20
21 if sys.version_info < (3, 5):
22 # Ugly hack...
23 RecursionError = RuntimeError
24
25 def _is_recursionerror(e):
26 return len(e.args) == 1 and isinstance(e.args[0], str) and \
27 e.args[0].startswith('maximum recursion depth exceeded')
28 else:
29 def _is_recursionerror(e):
30 return True
31
32 if hasattr(sys, 'pypy_version_info'):
33 # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own
34 # StringBuilder is fastest.
35 from __pypy__ import newlist_hint
36 try:
37 from __pypy__.builders import BytesBuilder as StringBuilder
38 except ImportError:
39 from __pypy__.builders import StringBuilder
40 USING_STRINGBUILDER = True
41 class StringIO(object):
42 def __init__(self, s=b''):
43 if s:
44 self.builder = StringBuilder(len(s))
45 self.builder.append(s)
46 else:
47 self.builder = StringBuilder()
48 def write(self, s):
49 if isinstance(s, memoryview):
50 s = s.tobytes()
51 elif isinstance(s, bytearray):
52 s = bytes(s)
53 self.builder.append(s)
54 def getvalue(self):
55 return self.builder.build()
56 else:
57 USING_STRINGBUILDER = False
58 from io import BytesIO as StringIO
59 newlist_hint = lambda size: []
60
61
62 from .exceptions import (
63 BufferFull,
64 OutOfData,
65 ExtraData,
66 FormatError,
67 StackError,
68 )
69
70 from . import ExtType
71
72
73 EX_SKIP = 0
74 EX_CONSTRUCT = 1
75 EX_READ_ARRAY_HEADER = 2
76 EX_READ_MAP_HEADER = 3
77
78 TYPE_IMMEDIATE = 0
79 TYPE_ARRAY = 1
80 TYPE_MAP = 2
81 TYPE_RAW = 3
82 TYPE_BIN = 4
83 TYPE_EXT = 5
84
85 DEFAULT_RECURSE_LIMIT = 511
86
87
88 def _check_type_strict(obj, t, type=type, tuple=tuple):
89 if type(t) is tuple:
90 return type(obj) in t
91 else:
92 return type(obj) is t
93
94
95 def _get_data_from_buffer(obj):
96 try:
97 view = memoryview(obj)
98 except TypeError:
99 # try to use legacy buffer protocol if 2.7, otherwise re-raise
100 if PY2:
101 view = memoryview(buffer(obj))
102 warnings.warn("using old buffer interface to unpack %s; "
103 "this leads to unpacking errors if slicing is used and "
104 "will be removed in a future version" % type(obj),
105 RuntimeWarning, stacklevel=3)
106 else:
107 raise
108 if view.itemsize != 1:
109 raise ValueError("cannot unpack from multi-byte object")
110 return view
111
112
113 def unpack(stream, **kwargs):
114 warnings.warn(
115 "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
116 DeprecationWarning, stacklevel=2)
117 data = stream.read()
118 return unpackb(data, **kwargs)
119
120
121 def unpackb(packed, **kwargs):
122 """
123 Unpack an object from `packed`.
124
125 Raises ``ExtraData`` when *packed* contains extra bytes.
126 Raises ``ValueError`` when *packed* is incomplete.
127 Raises ``FormatError`` when *packed* is not valid msgpack.
128 Raises ``StackError`` when *packed* contains too nested.
129 Other exceptions can be raised during unpacking.
130
131 See :class:`Unpacker` for options.
132 """
133 unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs)
134 unpacker.feed(packed)
135 try:
136 ret = unpacker._unpack()
137 except OutOfData:
138 raise ValueError("Unpack failed: incomplete input")
139 except RecursionError as e:
140 if _is_recursionerror(e):
141 raise StackError
142 raise
143 if unpacker._got_extradata():
144 raise ExtraData(ret, unpacker._get_extradata())
145 return ret
146
147
148 if sys.version_info < (2, 7, 6):
149 def _unpack_from(f, b, o=0):
150 """Explicit typcast for legacy struct.unpack_from"""
151 return struct.unpack_from(f, bytes(b), o)
152 else:
153 _unpack_from = struct.unpack_from
154
155
156 class Unpacker(object):
157 """Streaming unpacker.
158
159 arguments:
160
161 :param file_like:
162 File-like object having `.read(n)` method.
163 If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
164
165 :param int read_size:
166 Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
167
168 :param bool use_list:
169 If true, unpack msgpack array to Python list.
170 Otherwise, unpack to Python tuple. (default: True)
171
172 :param bool raw:
173 If true, unpack msgpack raw to Python bytes (default).
174 Otherwise, unpack to Python str (or unicode on Python 2) by decoding
175 with UTF-8 encoding (recommended).
176 Currently, the default is true, but it will be changed to false in
177 near future. So you must specify it explicitly for keeping backward
178 compatibility.
179
180 *encoding* option which is deprecated overrides this option.
181
182 :param bool strict_map_key:
183 If true, only str or bytes are accepted for map (dict) keys.
184 It's False by default for backward-compatibility.
185 But it will be True from msgpack 1.0.
186
187 :param callable object_hook:
188 When specified, it should be callable.
189 Unpacker calls it with a dict argument after unpacking msgpack map.
190 (See also simplejson)
191
192 :param callable object_pairs_hook:
193 When specified, it should be callable.
194 Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
195 (See also simplejson)
196
197 :param str encoding:
198 Encoding used for decoding msgpack raw.
199 If it is None (default), msgpack raw is deserialized to Python bytes.
200
201 :param str unicode_errors:
202 (deprecated) Used for decoding msgpack raw with *encoding*.
203 (default: `'strict'`)
204
205 :param int max_buffer_size:
206 Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
207 Raises `BufferFull` exception when it is insufficient.
208 You should set this parameter when unpacking data from untrusted source.
209
210 :param int max_str_len:
211 Deprecated, use *max_buffer_size* instead.
212 Limits max length of str. (default: max_buffer_size or 1024*1024)
213
214 :param int max_bin_len:
215 Deprecated, use *max_buffer_size* instead.
216 Limits max length of bin. (default: max_buffer_size or 1024*1024)
217
218 :param int max_array_len:
219 Limits max length of array.
220 (default: max_buffer_size or 128*1024)
221
222 :param int max_map_len:
223 Limits max length of map.
224 (default: max_buffer_size//2 or 32*1024)
225
226 :param int max_ext_len:
227 Deprecated, use *max_buffer_size* instead.
228 Limits max size of ext type. (default: max_buffer_size or 1024*1024)
229
230 Example of streaming deserialize from file-like object::
231
232 unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024)
233 for o in unpacker:
234 process(o)
235
236 Example of streaming deserialize from socket::
237
238 unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024)
239 while True:
240 buf = sock.recv(1024**2)
241 if not buf:
242 break
243 unpacker.feed(buf)
244 for o in unpacker:
245 process(o)
246
247 Raises ``ExtraData`` when *packed* contains extra bytes.
248 Raises ``OutOfData`` when *packed* is incomplete.
249 Raises ``FormatError`` when *packed* is not valid msgpack.
250 Raises ``StackError`` when *packed* contains too nested.
251 Other exceptions can be raised during unpacking.
252 """
253
254 def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False,
255 object_hook=None, object_pairs_hook=None, list_hook=None,
256 encoding=None, unicode_errors=None, max_buffer_size=0,
257 ext_hook=ExtType,
258 max_str_len=-1,
259 max_bin_len=-1,
260 max_array_len=-1,
261 max_map_len=-1,
262 max_ext_len=-1):
263 if encoding is not None:
264 warnings.warn(
265 "encoding is deprecated, Use raw=False instead.",
266 DeprecationWarning, stacklevel=2)
267
268 if unicode_errors is None:
269 unicode_errors = 'strict'
270
271 if file_like is None:
272 self._feeding = True
273 else:
274 if not callable(file_like.read):
275 raise TypeError("`file_like.read` must be callable")
276 self.file_like = file_like
277 self._feeding = False
278
279 #: array of bytes fed.
280 self._buffer = bytearray()
281 #: Which position we currently reads
282 self._buff_i = 0
283
284 # When Unpacker is used as an iterable, between the calls to next(),
285 # the buffer is not "consumed" completely, for efficiency sake.
286 # Instead, it is done sloppily. To make sure we raise BufferFull at
287 # the correct moments, we have to keep track of how sloppy we were.
288 # Furthermore, when the buffer is incomplete (that is: in the case
289 # we raise an OutOfData) we need to rollback the buffer to the correct
290 # state, which _buf_checkpoint records.
291 self._buf_checkpoint = 0
292
293 if max_str_len == -1:
294 max_str_len = max_buffer_size or 1024*1024
295 if max_bin_len == -1:
296 max_bin_len = max_buffer_size or 1024*1024
297 if max_array_len == -1:
298 max_array_len = max_buffer_size or 128*1024
299 if max_map_len == -1:
300 max_map_len = max_buffer_size//2 or 32*1024
301 if max_ext_len == -1:
302 max_ext_len = max_buffer_size or 1024*1024
303
304 self._max_buffer_size = max_buffer_size or 2**31-1
305 if read_size > self._max_buffer_size:
306 raise ValueError("read_size must be smaller than max_buffer_size")
307 self._read_size = read_size or min(self._max_buffer_size, 16*1024)
308 self._raw = bool(raw)
309 self._strict_map_key = bool(strict_map_key)
310 self._encoding = encoding
311 self._unicode_errors = unicode_errors
312 self._use_list = use_list
313 self._list_hook = list_hook
314 self._object_hook = object_hook
315 self._object_pairs_hook = object_pairs_hook
316 self._ext_hook = ext_hook
317 self._max_str_len = max_str_len
318 self._max_bin_len = max_bin_len
319 self._max_array_len = max_array_len
320 self._max_map_len = max_map_len
321 self._max_ext_len = max_ext_len
322 self._stream_offset = 0
323
324 if list_hook is not None and not callable(list_hook):
325 raise TypeError('`list_hook` is not callable')
326 if object_hook is not None and not callable(object_hook):
327 raise TypeError('`object_hook` is not callable')
328 if object_pairs_hook is not None and not callable(object_pairs_hook):
329 raise TypeError('`object_pairs_hook` is not callable')
330 if object_hook is not None and object_pairs_hook is not None:
331 raise TypeError("object_pairs_hook and object_hook are mutually "
332 "exclusive")
333 if not callable(ext_hook):
334 raise TypeError("`ext_hook` is not callable")
335
336 def feed(self, next_bytes):
337 assert self._feeding
338 view = _get_data_from_buffer(next_bytes)
339 if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
340 raise BufferFull
341
342 # Strip buffer before checkpoint before reading file.
343 if self._buf_checkpoint > 0:
344 del self._buffer[:self._buf_checkpoint]
345 self._buff_i -= self._buf_checkpoint
346 self._buf_checkpoint = 0
347
348 # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython
349 self._buffer.extend(view)
350
351 def _consume(self):
352 """ Gets rid of the used parts of the buffer. """
353 self._stream_offset += self._buff_i - self._buf_checkpoint
354 self._buf_checkpoint = self._buff_i
355
356 def _got_extradata(self):
357 return self._buff_i < len(self._buffer)
358
359 def _get_extradata(self):
360 return self._buffer[self._buff_i:]
361
362 def read_bytes(self, n):
363 return self._read(n)
364
365 def _read(self, n):
366 # (int) -> bytearray
367 self._reserve(n)
368 i = self._buff_i
369 self._buff_i = i+n
370 return self._buffer[i:i+n]
371
372 def _reserve(self, n):
373 remain_bytes = len(self._buffer) - self._buff_i - n
374
375 # Fast path: buffer has n bytes already
376 if remain_bytes >= 0:
377 return
378
379 if self._feeding:
380 self._buff_i = self._buf_checkpoint
381 raise OutOfData
382
383 # Strip buffer before checkpoint before reading file.
384 if self._buf_checkpoint > 0:
385 del self._buffer[:self._buf_checkpoint]
386 self._buff_i -= self._buf_checkpoint
387 self._buf_checkpoint = 0
388
389 # Read from file
390 remain_bytes = -remain_bytes
391 while remain_bytes > 0:
392 to_read_bytes = max(self._read_size, remain_bytes)
393 read_data = self.file_like.read(to_read_bytes)
394 if not read_data:
395 break
396 assert isinstance(read_data, bytes)
397 self._buffer += read_data
398 remain_bytes -= len(read_data)
399
400 if len(self._buffer) < n + self._buff_i:
401 self._buff_i = 0 # rollback
402 raise OutOfData
403
404 def _read_header(self, execute=EX_CONSTRUCT):
405 typ = TYPE_IMMEDIATE
406 n = 0
407 obj = None
408 self._reserve(1)
409 b = self._buffer[self._buff_i]
410 self._buff_i += 1
411 if b & 0b10000000 == 0:
412 obj = b
413 elif b & 0b11100000 == 0b11100000:
414 obj = -1 - (b ^ 0xff)
415 elif b & 0b11100000 == 0b10100000:
416 n = b & 0b00011111
417 typ = TYPE_RAW
418 if n > self._max_str_len:
419 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
420 obj = self._read(n)
421 elif b & 0b11110000 == 0b10010000:
422 n = b & 0b00001111
423 typ = TYPE_ARRAY
424 if n > self._max_array_len:
425 raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
426 elif b & 0b11110000 == 0b10000000:
427 n = b & 0b00001111
428 typ = TYPE_MAP
429 if n > self._max_map_len:
430 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
431 elif b == 0xc0:
432 obj = None
433 elif b == 0xc2:
434 obj = False
435 elif b == 0xc3:
436 obj = True
437 elif b == 0xc4:
438 typ = TYPE_BIN
439 self._reserve(1)
440 n = self._buffer[self._buff_i]
441 self._buff_i += 1
442 if n > self._max_bin_len:
443 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
444 obj = self._read(n)
445 elif b == 0xc5:
446 typ = TYPE_BIN
447 self._reserve(2)
448 n = _unpack_from(">H", self._buffer, self._buff_i)[0]
449 self._buff_i += 2
450 if n > self._max_bin_len:
451 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
452 obj = self._read(n)
453 elif b == 0xc6:
454 typ = TYPE_BIN
455 self._reserve(4)
456 n = _unpack_from(">I", self._buffer, self._buff_i)[0]
457 self._buff_i += 4
458 if n > self._max_bin_len:
459 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
460 obj = self._read(n)
461 elif b == 0xc7: # ext 8
462 typ = TYPE_EXT
463 self._reserve(2)
464 L, n = _unpack_from('Bb', self._buffer, self._buff_i)
465 self._buff_i += 2
466 if L > self._max_ext_len:
467 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
468 obj = self._read(L)
469 elif b == 0xc8: # ext 16
470 typ = TYPE_EXT
471 self._reserve(3)
472 L, n = _unpack_from('>Hb', self._buffer, self._buff_i)
473 self._buff_i += 3
474 if L > self._max_ext_len:
475 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
476 obj = self._read(L)
477 elif b == 0xc9: # ext 32
478 typ = TYPE_EXT
479 self._reserve(5)
480 L, n = _unpack_from('>Ib', self._buffer, self._buff_i)
481 self._buff_i += 5
482 if L > self._max_ext_len:
483 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
484 obj = self._read(L)
485 elif b == 0xca:
486 self._reserve(4)
487 obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
488 self._buff_i += 4
489 elif b == 0xcb:
490 self._reserve(8)
491 obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
492 self._buff_i += 8
493 elif b == 0xcc:
494 self._reserve(1)
495 obj = self._buffer[self._buff_i]
496 self._buff_i += 1
497 elif b == 0xcd:
498 self._reserve(2)
499 obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
500 self._buff_i += 2
501 elif b == 0xce:
502 self._reserve(4)
503 obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
504 self._buff_i += 4
505 elif b == 0xcf:
506 self._reserve(8)
507 obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
508 self._buff_i += 8
509 elif b == 0xd0:
510 self._reserve(1)
511 obj = _unpack_from("b", self._buffer, self._buff_i)[0]
512 self._buff_i += 1
513 elif b == 0xd1:
514 self._reserve(2)
515 obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
516 self._buff_i += 2
517 elif b == 0xd2:
518 self._reserve(4)
519 obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
520 self._buff_i += 4
521 elif b == 0xd3:
522 self._reserve(8)
523 obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
524 self._buff_i += 8
525 elif b == 0xd4: # fixext 1
526 typ = TYPE_EXT
527 if self._max_ext_len < 1:
528 raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
529 self._reserve(2)
530 n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
531 self._buff_i += 2
532 elif b == 0xd5: # fixext 2
533 typ = TYPE_EXT
534 if self._max_ext_len < 2:
535 raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
536 self._reserve(3)
537 n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
538 self._buff_i += 3
539 elif b == 0xd6: # fixext 4
540 typ = TYPE_EXT
541 if self._max_ext_len < 4:
542 raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
543 self._reserve(5)
544 n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
545 self._buff_i += 5
546 elif b == 0xd7: # fixext 8
547 typ = TYPE_EXT
548 if self._max_ext_len < 8:
549 raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
550 self._reserve(9)
551 n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
552 self._buff_i += 9
553 elif b == 0xd8: # fixext 16
554 typ = TYPE_EXT
555 if self._max_ext_len < 16:
556 raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
557 self._reserve(17)
558 n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
559 self._buff_i += 17
560 elif b == 0xd9:
561 typ = TYPE_RAW
562 self._reserve(1)
563 n = self._buffer[self._buff_i]
564 self._buff_i += 1
565 if n > self._max_str_len:
566 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
567 obj = self._read(n)
568 elif b == 0xda:
569 typ = TYPE_RAW
570 self._reserve(2)
571 n, = _unpack_from(">H", self._buffer, self._buff_i)
572 self._buff_i += 2
573 if n > self._max_str_len:
574 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
575 obj = self._read(n)
576 elif b == 0xdb:
577 typ = TYPE_RAW
578 self._reserve(4)
579 n, = _unpack_from(">I", self._buffer, self._buff_i)
580 self._buff_i += 4
581 if n > self._max_str_len:
582 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
583 obj = self._read(n)
584 elif b == 0xdc:
585 typ = TYPE_ARRAY
586 self._reserve(2)
587 n, = _unpack_from(">H", self._buffer, self._buff_i)
588 self._buff_i += 2
589 if n > self._max_array_len:
590 raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
591 elif b == 0xdd:
592 typ = TYPE_ARRAY
593 self._reserve(4)
594 n, = _unpack_from(">I", self._buffer, self._buff_i)
595 self._buff_i += 4
596 if n > self._max_array_len:
597 raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
598 elif b == 0xde:
599 self._reserve(2)
600 n, = _unpack_from(">H", self._buffer, self._buff_i)
601 self._buff_i += 2
602 if n > self._max_map_len:
603 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
604 typ = TYPE_MAP
605 elif b == 0xdf:
606 self._reserve(4)
607 n, = _unpack_from(">I", self._buffer, self._buff_i)
608 self._buff_i += 4
609 if n > self._max_map_len:
610 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
611 typ = TYPE_MAP
612 else:
613 raise FormatError("Unknown header: 0x%x" % b)
614 return typ, n, obj
615
616 def _unpack(self, execute=EX_CONSTRUCT):
617 typ, n, obj = self._read_header(execute)
618
619 if execute == EX_READ_ARRAY_HEADER:
620 if typ != TYPE_ARRAY:
621 raise ValueError("Expected array")
622 return n
623 if execute == EX_READ_MAP_HEADER:
624 if typ != TYPE_MAP:
625 raise ValueError("Expected map")
626 return n
627 # TODO should we eliminate the recursion?
628 if typ == TYPE_ARRAY:
629 if execute == EX_SKIP:
630 for i in xrange(n):
631 # TODO check whether we need to call `list_hook`
632 self._unpack(EX_SKIP)
633 return
634 ret = newlist_hint(n)
635 for i in xrange(n):
636 ret.append(self._unpack(EX_CONSTRUCT))
637 if self._list_hook is not None:
638 ret = self._list_hook(ret)
639 # TODO is the interaction between `list_hook` and `use_list` ok?
640 return ret if self._use_list else tuple(ret)
641 if typ == TYPE_MAP:
642 if execute == EX_SKIP:
643 for i in xrange(n):
644 # TODO check whether we need to call hooks
645 self._unpack(EX_SKIP)
646 self._unpack(EX_SKIP)
647 return
648 if self._object_pairs_hook is not None:
649 ret = self._object_pairs_hook(
650 (self._unpack(EX_CONSTRUCT),
651 self._unpack(EX_CONSTRUCT))
652 for _ in xrange(n))
653 else:
654 ret = {}
655 for _ in xrange(n):
656 key = self._unpack(EX_CONSTRUCT)
657 if self._strict_map_key and type(key) not in (unicode, bytes):
658 raise ValueError("%s is not allowed for map key" % str(type(key)))
659 ret[key] = self._unpack(EX_CONSTRUCT)
660 if self._object_hook is not None:
661 ret = self._object_hook(ret)
662 return ret
663 if execute == EX_SKIP:
664 return
665 if typ == TYPE_RAW:
666 if self._encoding is not None:
667 obj = obj.decode(self._encoding, self._unicode_errors)
668 elif self._raw:
669 obj = bytes(obj)
670 else:
671 obj = obj.decode('utf_8')
672 return obj
673 if typ == TYPE_EXT:
674 return self._ext_hook(n, bytes(obj))
675 if typ == TYPE_BIN:
676 return bytes(obj)
677 assert typ == TYPE_IMMEDIATE
678 return obj
679
680 def __iter__(self):
681 return self
682
683 def __next__(self):
684 try:
685 ret = self._unpack(EX_CONSTRUCT)
686 self._consume()
687 return ret
688 except OutOfData:
689 self._consume()
690 raise StopIteration
691 except RecursionError:
692 raise StackError
693
694 next = __next__
695
696 def skip(self):
697 self._unpack(EX_SKIP)
698 self._consume()
699
700 def unpack(self):
701 try:
702 ret = self._unpack(EX_CONSTRUCT)
703 except RecursionError:
704 raise StackError
705 self._consume()
706 return ret
707
708 def read_array_header(self):
709 ret = self._unpack(EX_READ_ARRAY_HEADER)
710 self._consume()
711 return ret
712
713 def read_map_header(self):
714 ret = self._unpack(EX_READ_MAP_HEADER)
715 self._consume()
716 return ret
717
718 def tell(self):
719 return self._stream_offset
720
721
722 class Packer(object):
723 """
724 MessagePack Packer
725
726 usage:
727
728 packer = Packer()
729 astream.write(packer.pack(a))
730 astream.write(packer.pack(b))
731
732 Packer's constructor has some keyword arguments:
733
734 :param callable default:
735 Convert user type to builtin type that Packer supports.
736 See also simplejson's document.
737
738 :param bool use_single_float:
739 Use single precision float type for float. (default: False)
740
741 :param bool autoreset:
742 Reset buffer after each pack and return its content as `bytes`. (default: True).
743 If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
744
745 :param bool use_bin_type:
746 Use bin type introduced in msgpack spec 2.0 for bytes.
747 It also enables str8 type for unicode.
748
749 :param bool strict_types:
750 If set to true, types will be checked to be exact. Derived classes
751 from serializeable types will not be serialized and will be
752 treated as unsupported type and forwarded to default.
753 Additionally tuples will not be serialized as lists.
754 This is useful when trying to implement accurate serialization
755 for python types.
756
757 :param str encoding:
758 (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
759
760 :param str unicode_errors:
761 Error handler for encoding unicode. (default: 'strict')
762 """
763 def __init__(self, default=None, encoding=None, unicode_errors=None,
764 use_single_float=False, autoreset=True, use_bin_type=False,
765 strict_types=False):
766 if encoding is None:
767 encoding = 'utf_8'
768 else:
769 warnings.warn(
770 "encoding is deprecated, Use raw=False instead.",
771 DeprecationWarning, stacklevel=2)
772
773 if unicode_errors is None:
774 unicode_errors = 'strict'
775
776 self._strict_types = strict_types
777 self._use_float = use_single_float
778 self._autoreset = autoreset
779 self._use_bin_type = use_bin_type
780 self._encoding = encoding
781 self._unicode_errors = unicode_errors
782 self._buffer = StringIO()
783 if default is not None:
784 if not callable(default):
785 raise TypeError("default must be callable")
786 self._default = default
787
788 def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
789 check=isinstance, check_type_strict=_check_type_strict):
790 default_used = False
791 if self._strict_types:
792 check = check_type_strict
793 list_types = list
794 else:
795 list_types = (list, tuple)
796 while True:
797 if nest_limit < 0:
798 raise ValueError("recursion limit exceeded")
799 if obj is None:
800 return self._buffer.write(b"\xc0")
801 if check(obj, bool):
802 if obj:
803 return self._buffer.write(b"\xc3")
804 return self._buffer.write(b"\xc2")
805 if check(obj, int_types):
806 if 0 <= obj < 0x80:
807 return self._buffer.write(struct.pack("B", obj))
808 if -0x20 <= obj < 0:
809 return self._buffer.write(struct.pack("b", obj))
810 if 0x80 <= obj <= 0xff:
811 return self._buffer.write(struct.pack("BB", 0xcc, obj))
812 if -0x80 <= obj < 0:
813 return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
814 if 0xff < obj <= 0xffff:
815 return self._buffer.write(struct.pack(">BH", 0xcd, obj))
816 if -0x8000 <= obj < -0x80:
817 return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
818 if 0xffff < obj <= 0xffffffff:
819 return self._buffer.write(struct.pack(">BI", 0xce, obj))
820 if -0x80000000 <= obj < -0x8000:
821 return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
822 if 0xffffffff < obj <= 0xffffffffffffffff:
823 return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
824 if -0x8000000000000000 <= obj < -0x80000000:
825 return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
826 if not default_used and self._default is not None:
827 obj = self._default(obj)
828 default_used = True
829 continue
830 raise OverflowError("Integer value out of range")
831 if check(obj, (bytes, bytearray)):
832 n = len(obj)
833 if n >= 2**32:
834 raise ValueError("%s is too large" % type(obj).__name__)
835 self._pack_bin_header(n)
836 return self._buffer.write(obj)
837 if check(obj, unicode):
838 if self._encoding is None:
839 raise TypeError(
840 "Can't encode unicode string: "
841 "no encoding is specified")
842 obj = obj.encode(self._encoding, self._unicode_errors)
843 n = len(obj)
844 if n >= 2**32:
845 raise ValueError("String is too large")
846 self._pack_raw_header(n)
847 return self._buffer.write(obj)
848 if check(obj, memoryview):
849 n = len(obj) * obj.itemsize
850 if n >= 2**32:
851 raise ValueError("Memoryview is too large")
852 self._pack_bin_header(n)
853 return self._buffer.write(obj)
854 if check(obj, float):
855 if self._use_float:
856 return self._buffer.write(struct.pack(">Bf", 0xca, obj))
857 return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
858 if check(obj, ExtType):
859 code = obj.code
860 data = obj.data
861 assert isinstance(code, int)
862 assert isinstance(data, bytes)
863 L = len(data)
864 if L == 1:
865 self._buffer.write(b'\xd4')
866 elif L == 2:
867 self._buffer.write(b'\xd5')
868 elif L == 4:
869 self._buffer.write(b'\xd6')
870 elif L == 8:
871 self._buffer.write(b'\xd7')
872 elif L == 16:
873 self._buffer.write(b'\xd8')
874 elif L <= 0xff:
875 self._buffer.write(struct.pack(">BB", 0xc7, L))
876 elif L <= 0xffff:
877 self._buffer.write(struct.pack(">BH", 0xc8, L))
878 else:
879 self._buffer.write(struct.pack(">BI", 0xc9, L))
880 self._buffer.write(struct.pack("b", code))
881 self._buffer.write(data)
882 return
883 if check(obj, list_types):
884 n = len(obj)
885 self._pack_array_header(n)
886 for i in xrange(n):
887 self._pack(obj[i], nest_limit - 1)
888 return
889 if check(obj, dict):
890 return self._pack_map_pairs(len(obj), dict_iteritems(obj),
891 nest_limit - 1)
892 if not default_used and self._default is not None:
893 obj = self._default(obj)
894 default_used = 1
895 continue
896 raise TypeError("Cannot serialize %r" % (obj, ))
897
898 def pack(self, obj):
899 try:
900 self._pack(obj)
901 except:
902 self._buffer = StringIO() # force reset
903 raise
904 if self._autoreset:
905 ret = self._buffer.getvalue()
906 self._buffer = StringIO()
907 return ret
908
909 def pack_map_pairs(self, pairs):
910 self._pack_map_pairs(len(pairs), pairs)
911 if self._autoreset:
912 ret = self._buffer.getvalue()
913 self._buffer = StringIO()
914 return ret
915
916 def pack_array_header(self, n):
917 if n >= 2**32:
918 raise ValueError
919 self._pack_array_header(n)
920 if self._autoreset:
921 ret = self._buffer.getvalue()
922 self._buffer = StringIO()
923 return ret
924
925 def pack_map_header(self, n):
926 if n >= 2**32:
927 raise ValueError
928 self._pack_map_header(n)
929 if self._autoreset:
930 ret = self._buffer.getvalue()
931 self._buffer = StringIO()
932 return ret
933
934 def pack_ext_type(self, typecode, data):
935 if not isinstance(typecode, int):
936 raise TypeError("typecode must have int type.")
937 if not 0 <= typecode <= 127:
938 raise ValueError("typecode should be 0-127")
939 if not isinstance(data, bytes):
940 raise TypeError("data must have bytes type")
941 L = len(data)
942 if L > 0xffffffff:
943 raise ValueError("Too large data")
944 if L == 1:
945 self._buffer.write(b'\xd4')
946 elif L == 2:
947 self._buffer.write(b'\xd5')
948 elif L == 4:
949 self._buffer.write(b'\xd6')
950 elif L == 8:
951 self._buffer.write(b'\xd7')
952 elif L == 16:
953 self._buffer.write(b'\xd8')
954 elif L <= 0xff:
955 self._buffer.write(b'\xc7' + struct.pack('B', L))
956 elif L <= 0xffff:
957 self._buffer.write(b'\xc8' + struct.pack('>H', L))
958 else:
959 self._buffer.write(b'\xc9' + struct.pack('>I', L))
960 self._buffer.write(struct.pack('B', typecode))
961 self._buffer.write(data)
962
963 def _pack_array_header(self, n):
964 if n <= 0x0f:
965 return self._buffer.write(struct.pack('B', 0x90 + n))
966 if n <= 0xffff:
967 return self._buffer.write(struct.pack(">BH", 0xdc, n))
968 if n <= 0xffffffff:
969 return self._buffer.write(struct.pack(">BI", 0xdd, n))
970 raise ValueError("Array is too large")
971
972 def _pack_map_header(self, n):
973 if n <= 0x0f:
974 return self._buffer.write(struct.pack('B', 0x80 + n))
975 if n <= 0xffff:
976 return self._buffer.write(struct.pack(">BH", 0xde, n))
977 if n <= 0xffffffff:
978 return self._buffer.write(struct.pack(">BI", 0xdf, n))
979 raise ValueError("Dict is too large")
980
981 def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
982 self._pack_map_header(n)
983 for (k, v) in pairs:
984 self._pack(k, nest_limit - 1)
985 self._pack(v, nest_limit - 1)
986
987 def _pack_raw_header(self, n):
988 if n <= 0x1f:
989 self._buffer.write(struct.pack('B', 0xa0 + n))
990 elif self._use_bin_type and n <= 0xff:
991 self._buffer.write(struct.pack('>BB', 0xd9, n))
992 elif n <= 0xffff:
993 self._buffer.write(struct.pack(">BH", 0xda, n))
994 elif n <= 0xffffffff:
995 self._buffer.write(struct.pack(">BI", 0xdb, n))
996 else:
997 raise ValueError('Raw is too large')
998
999 def _pack_bin_header(self, n):
1000 if not self._use_bin_type:
1001 return self._pack_raw_header(n)
1002 elif n <= 0xff:
1003 return self._buffer.write(struct.pack('>BB', 0xc4, n))
1004 elif n <= 0xffff:
1005 return self._buffer.write(struct.pack(">BH", 0xc5, n))
1006 elif n <= 0xffffffff:
1007 return self._buffer.write(struct.pack(">BI", 0xc6, n))
1008 else:
1009 raise ValueError('Bin is too large')
1010
1011 def bytes(self):
1012 """Return internal buffer contents as bytes object"""
1013 return self._buffer.getvalue()
1014
1015 def reset(self):
1016 """Reset internal buffer.
1017
1018 This method is usaful only when autoreset=False.
1019 """
1020 self._buffer = StringIO()
1021
1022 def getbuffer(self):
1023 """Return view of internal buffer."""
1024 if USING_STRINGBUILDER or PY2:
1025 return memoryview(self.bytes())
1026 else:
1027 return self._buffer.getbuffer()