Skip to content

Commit 6740b90

Browse files
methanemethane
committed
Merge branch 'purepython'
2 parents 266eaf8 + a865f8f commit 6740b90

File tree

4 files changed

+140
-94
lines changed

4 files changed

+140
-94
lines changed

‎benchmark/benchmark.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
from msgpack import fallback
2+
try:
3+
from msgpack import _unpacker, _packer
4+
has_ext = True
5+
except ImportError:
6+
has_ext = False
7+
import timeit
8+
9+
10+
def profile(name, func):
11+
times = timeit.repeat(func, number=1000, repeat=4)
12+
times = ', '.join(["%8f" % t for t in times])
13+
print("%-30s %40s" % (name, times))
14+
15+
16+
def simple(name, data):
17+
if has_ext:
18+
profile("packing %s (ext)" % name, lambda: _packer.packb(data))
19+
profile('packing %s (fallback)' % name, lambda: fallback.packb(data))
20+
21+
data = fallback.packb(data)
22+
if has_ext:
23+
profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
24+
profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))
25+
26+
def main():
27+
simple("integers", [7]*10000)
28+
simple("bytes", [b'x'*n for n in range(100)]*10)
29+
simple("lists", [[]]*10000)
30+
simple("dicts", [{}]*10000)
31+
32+
main()

‎msgpack/fallback.py‎

Lines changed: 88 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -168,18 +168,18 @@ def __init__(self, file_like=None, read_size=0, use_list=True,
168168
self._fb_buf_o = 0
169169
self._fb_buf_i = 0
170170
self._fb_buf_n = 0
171-
self.max_buffer_size = (2**31-1 if max_buffer_size == 0
172-
else max_buffer_size)
173-
self.read_size = (read_size if read_size != 0
174-
else min(self.max_buffer_size, 2048))
175-
if read_size > self.max_buffer_size:
171+
self._max_buffer_size = (2**31-1 if max_buffer_size == 0
172+
else max_buffer_size)
173+
self._read_size = (read_size if read_size != 0
174+
else min(self._max_buffer_size, 2048))
175+
if read_size > self._max_buffer_size:
176176
raise ValueError("read_size must be smaller than max_buffer_size")
177-
self.encoding = encoding
178-
self.unicode_errors = unicode_errors
179-
self.use_list = use_list
180-
self.list_hook = list_hook
181-
self.object_hook = object_hook
182-
self.object_pairs_hook = object_pairs_hook
177+
self._encoding = encoding
178+
self._unicode_errors = unicode_errors
179+
self._use_list = use_list
180+
self._list_hook = list_hook
181+
self._object_hook = object_hook
182+
self._object_pairs_hook = object_pairs_hook
183183

184184
if list_hook is not None and not callable(list_hook):
185185
raise ValueError('`list_hook` is not callable')
@@ -195,7 +195,7 @@ def feed(self, next_bytes):
195195
if isinstance(next_bytes, array.array):
196196
next_bytes = next_bytes.tostring()
197197
assert self._fb_feeding
198-
if self._fb_buf_n + len(next_bytes) > self.max_buffer_size:
198+
if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
199199
raise BufferFull
200200
self._fb_buf_n += len(next_bytes)
201201
self._fb_buffers.append(next_bytes)
@@ -246,7 +246,7 @@ def _fb_read(self, n, write_bytes=None):
246246
if self._fb_buf_i == len(self._fb_buffers):
247247
if self._fb_feeding:
248248
break
249-
tmp = self.file_like.read(self.read_size)
249+
tmp = self.file_like.read(self._read_size)
250250
if not tmp:
251251
break
252252
self._fb_buffers.append(tmp)
@@ -349,33 +349,36 @@ def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
349349
ret = []
350350
for i in xrange(n):
351351
ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
352-
if self.list_hook is not None:
353-
ret = self.list_hook(ret)
352+
if self._list_hook is not None:
353+
ret = self._list_hook(ret)
354354
# TODO is the interaction between `list_hook` and `use_list` ok?
355-
return ret if self.use_list else tuple(ret)
355+
return ret if self._use_list else tuple(ret)
356356
if typ == TYPE_MAP:
357357
if execute == EX_SKIP:
358358
for i in xrange(n):
359359
# TODO check whether we need to call hooks
360360
self._fb_unpack(EX_SKIP, write_bytes)
361361
self._fb_unpack(EX_SKIP, write_bytes)
362362
return
363-
ret = []
364-
for i in xrange(n):
365-
ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes),
366-
self._fb_unpack(EX_CONSTRUCT, write_bytes)))
367-
if self.object_pairs_hook is not None:
368-
ret = self.object_pairs_hook(ret)
363+
if self._object_pairs_hook is not None:
364+
ret = self._object_pairs_hook(
365+
(self._fb_unpack(EX_CONSTRUCT, write_bytes),
366+
self._fb_unpack(EX_CONSTRUCT, write_bytes))
367+
for _ in xrange(n)
368+
)
369369
else:
370-
ret = dict(ret)
371-
if self.object_hook is not None:
372-
ret = self.object_hook(ret)
370+
ret = {}
371+
for _ in xrange(n):
372+
key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
373+
ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
374+
if self._object_hook is not None:
375+
ret = self._object_hook(ret)
373376
return ret
374377
if execute == EX_SKIP:
375378
return
376379
if typ == TYPE_RAW:
377-
if self.encoding is not None:
378-
obj = obj.decode(self.encoding, self.unicode_errors)
380+
if self._encoding is not None:
381+
obj = obj.decode(self._encoding, self._unicode_errors)
379382
return obj
380383
assert typ == TYPE_IMMEDIATE
381384
return obj
@@ -408,68 +411,73 @@ def read_map_header(self, write_bytes=None):
408411
self._fb_consume()
409412
return ret
410413

414+
411415
class Packer(object):
412416
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
413417
use_single_float=False, autoreset=True):
414-
self.use_float = use_single_float
415-
self.autoreset = autoreset
416-
self.encoding = encoding
417-
self.unicode_errors = unicode_errors
418-
self.buffer = StringIO()
418+
self._use_float = use_single_float
419+
self._autoreset = autoreset
420+
self._encoding = encoding
421+
self._unicode_errors = unicode_errors
422+
self._buffer = StringIO()
419423
if default is not None:
420424
if not callable(default):
421425
raise TypeError("default must be callable")
422426
self._default = default
423427

424-
def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
428+
def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
425429
if nest_limit < 0:
426430
raise PackValueError("recursion limit exceeded")
427431
if obj is None:
428-
return self.buffer.write(b"\xc0")
432+
return self._buffer.write(b"\xc0")
429433
if isinstance(obj, bool):
430434
if obj:
431-
return self.buffer.write(b"\xc3")
432-
return self.buffer.write(b"\xc2")
435+
return self._buffer.write(b"\xc3")
436+
return self._buffer.write(b"\xc2")
433437
if isinstance(obj, int_types):
434438
if 0 <= obj < 0x80:
435-
return self.buffer.write(struct.pack("B", obj))
439+
return self._buffer.write(struct.pack("B", obj))
436440
if -0x20 <= obj < 0:
437-
return self.buffer.write(struct.pack("b", obj))
441+
return self._buffer.write(struct.pack("b", obj))
438442
if 0x80 <= obj <= 0xff:
439-
return self.buffer.write(struct.pack("BB", 0xcc, obj))
443+
return self._buffer.write(struct.pack("BB", 0xcc, obj))
440444
if -0x80 <= obj < 0:
441-
return self.buffer.write(struct.pack(">Bb", 0xd0, obj))
445+
return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
442446
if 0xff < obj <= 0xffff:
443-
return self.buffer.write(struct.pack(">BH", 0xcd, obj))
447+
return self._buffer.write(struct.pack(">BH", 0xcd, obj))
444448
if -0x8000 <= obj < -0x80:
445-
return self.buffer.write(struct.pack(">Bh", 0xd1, obj))
449+
return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
446450
if 0xffff < obj <= 0xffffffff:
447-
return self.buffer.write(struct.pack(">BI", 0xce, obj))
451+
return self._buffer.write(struct.pack(">BI", 0xce, obj))
448452
if -0x80000000 <= obj < -0x8000:
449-
return self.buffer.write(struct.pack(">Bi", 0xd2, obj))
453+
return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
450454
if 0xffffffff < obj <= 0xffffffffffffffff:
451-
return self.buffer.write(struct.pack(">BQ", 0xcf, obj))
455+
return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
452456
if -0x8000000000000000 <= obj < -0x80000000:
453-
return self.buffer.write(struct.pack(">Bq", 0xd3, obj))
457+
return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
454458
raise PackValueError("Integer value out of range")
455459
if isinstance(obj, (Unicode, bytes)):
456460
if isinstance(obj, Unicode):
457-
obj = obj.encode(self.encoding, self.unicode_errors)
461+
if self._encoding is None:
462+
raise TypeError(
463+
"Can't encode unicode string: "
464+
"no encoding is specified")
465+
obj = obj.encode(self._encoding, self._unicode_errors)
458466
n = len(obj)
459467
if n <= 0x1f:
460-
self.buffer.write(struct.pack('B', 0xa0 + n))
461-
return self.buffer.write(obj)
468+
self._buffer.write(struct.pack('B', 0xa0 + n))
469+
return self._buffer.write(obj)
462470
if n <= 0xffff:
463-
self.buffer.write(struct.pack(">BH", 0xda, n))
464-
return self.buffer.write(obj)
471+
self._buffer.write(struct.pack(">BH", 0xda, n))
472+
return self._buffer.write(obj)
465473
if n <= 0xffffffff:
466-
self.buffer.write(struct.pack(">BI", 0xdb, n))
467-
return self.buffer.write(obj)
474+
self._buffer.write(struct.pack(">BI", 0xdb, n))
475+
return self._buffer.write(obj)
468476
raise PackValueError("String is too large")
469477
if isinstance(obj, float):
470-
if self.use_float:
471-
return self.buffer.write(struct.pack(">Bf", 0xca, obj))
472-
return self.buffer.write(struct.pack(">Bd", 0xcb, obj))
478+
if self._use_float:
479+
return self._buffer.write(struct.pack(">Bf", 0xca, obj))
480+
return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
473481
if isinstance(obj, list) or isinstance(obj, tuple):
474482
n = len(obj)
475483
self._fb_pack_array_header(n)
@@ -485,56 +493,56 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
485493

486494
def pack(self, obj):
487495
self._pack(obj)
488-
ret = self.buffer.getvalue()
489-
if self.autoreset:
490-
self.buffer = StringIO()
496+
ret = self._buffer.getvalue()
497+
if self._autoreset:
498+
self._buffer = StringIO()
491499
elif USING_STRINGBUILDER:
492-
self.buffer = StringIO(ret)
500+
self._buffer = StringIO(ret)
493501
return ret
494502

495503
def pack_map_pairs(self, pairs):
496504
self._fb_pack_map_pairs(len(pairs), pairs)
497-
ret = self.buffer.getvalue()
498-
if self.autoreset:
499-
self.buffer = StringIO()
505+
ret = self._buffer.getvalue()
506+
if self._autoreset:
507+
self._buffer = StringIO()
500508
elif USING_STRINGBUILDER:
501-
self.buffer = StringIO(ret)
509+
self._buffer = StringIO(ret)
502510
return ret
503511

504512
def pack_array_header(self, n):
505513
self._fb_pack_array_header(n)
506-
ret = self.buffer.getvalue()
507-
if self.autoreset:
508-
self.buffer = StringIO()
514+
ret = self._buffer.getvalue()
515+
if self._autoreset:
516+
self._buffer = StringIO()
509517
elif USING_STRINGBUILDER:
510-
self.buffer = StringIO(ret)
518+
self._buffer = StringIO(ret)
511519
return ret
512520

513521
def pack_map_header(self, n):
514522
self._fb_pack_map_header(n)
515-
ret = self.buffer.getvalue()
516-
if self.autoreset:
517-
self.buffer = StringIO()
523+
ret = self._buffer.getvalue()
524+
if self._autoreset:
525+
self._buffer = StringIO()
518526
elif USING_STRINGBUILDER:
519-
self.buffer = StringIO(ret)
527+
self._buffer = StringIO(ret)
520528
return ret
521529

522530
def _fb_pack_array_header(self, n):
523531
if n <= 0x0f:
524-
return self.buffer.write(struct.pack('B', 0x90 + n))
532+
return self._buffer.write(struct.pack('B', 0x90 + n))
525533
if n <= 0xffff:
526-
return self.buffer.write(struct.pack(">BH", 0xdc, n))
534+
return self._buffer.write(struct.pack(">BH", 0xdc, n))
527535
if n <= 0xffffffff:
528-
return self.buffer.write(struct.pack(">BI", 0xdd, n))
536+
return self._buffer.write(struct.pack(">BI", 0xdd, n))
529537
raise PackValueError("Array is too large")
530538

531539
def _fb_pack_map_header(self, n):
532540
if n <= 0x0f:
533-
return self.buffer.write(struct.pack('B', 0x80 + n))
541+
return self._buffer.write(struct.pack('B', 0x80 + n))
534542
if n <= 0xffff:
535-
return self.buffer.write(struct.pack(">BH", 0xde, n))
543+
return self._buffer.write(struct.pack(">BH", 0xde, n))
536544
if n <= 0xffffffff:
537-
return self.buffer.write(struct.pack(">BI", 0xdf, n))
545+
return self._buffer.write(struct.pack(">BI", 0xdf, n))
538546
raise PackValueError("Dict is too large")
539547

540548
def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
@@ -544,7 +552,7 @@ def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
544552
self._pack(v, nest_limit - 1)
545553

546554
def bytes(self):
547-
return self.buffer.getvalue()
555+
return self._buffer.getvalue()
548556

549557
def reset(self):
550-
self.buffer = StringIO()
558+
self._buffer = StringIO()

‎setup.py‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,12 @@ def build_extension(self, ext):
4646
print("Install Cython >= 0.16 or install msgpack from PyPI.")
4747
print("Falling back to pure Python implementation.")
4848
return
49-
return build_ext.build_extension(self, ext)
49+
try:
50+
return build_ext.build_extension(self, ext)
51+
except Exception as e:
52+
print("WARNING: Failed to compile extensiom modules.")
53+
print("msgpack uses fallback pure python implementation.")
54+
print(e)
5055

5156

5257
exec(open('msgpack/_version.py').read())
@@ -75,18 +80,19 @@ def __init__(self, *args, **kwargs):
7580
macros = [('__LITTLE_ENDIAN__', '1')]
7681

7782
ext_modules = []
78-
ext_modules.append(Extension('msgpack._packer',
79-
sources=['msgpack/_packer.cpp'],
80-
libraries=libraries,
81-
include_dirs=['.'],
82-
define_macros=macros,
83-
))
84-
ext_modules.append(Extension('msgpack._unpacker',
85-
sources=['msgpack/_unpacker.cpp'],
86-
libraries=libraries,
87-
include_dirs=['.'],
88-
define_macros=macros,
89-
))
83+
if not hasattr(sys, 'pypy_version_info'):
84+
ext_modules.append(Extension('msgpack._packer',
85+
sources=['msgpack/_packer.cpp'],
86+
libraries=libraries,
87+
include_dirs=['.'],
88+
define_macros=macros,
89+
))
90+
ext_modules.append(Extension('msgpack._unpacker',
91+
sources=['msgpack/_unpacker.cpp'],
92+
libraries=libraries,
93+
include_dirs=['.'],
94+
define_macros=macros,
95+
))
9096
del libraries, macros
9197

9298

‎test/test_pack.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ def test_odict():
151151
od = odict(seq)
152152
assert unpackb(packb(od), use_list=1) == dict(seq)
153153
def pair_hook(seq):
154-
return seq
154+
return list(seq)
155155
assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
156156

157157

0 commit comments

Comments
 (0)