Merge pull request #45 from msgpack/purepython

fallback enhancements.

Author: methane

benchmark/benchmark.py

@@ -0,0 +1,32 @@
+from msgpack import fallback
+try:
+    from msgpack import _unpacker, _packer
+    has_ext = True
+except ImportError:
+    has_ext = False
+import timeit
+
+
+def profile(name, func):
+    times = timeit.repeat(func, number=1000, repeat=4)
+    times = ', '.join(["%8f" % t for t in times])
+    print("%-30s %40s" % (name, times))
+
+
+def simple(name, data):
+    if has_ext:
+        profile("packing %s (ext)" % name, lambda: _packer.packb(data))
+    profile('packing %s (fallback)' % name, lambda: fallback.packb(data))
+
+    data = fallback.packb(data)
+    if has_ext:
+        profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
+    profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))
+
+def main():
+    simple("integers", [7]*10000)
+    simple("bytes", [b'x'*n for n in range(100)]*10)
+    simple("lists", [[]]*10000)
+    simple("dicts", [{}]*10000)
+
+main()

msgpack/fallback.py

@@ -360,16 +360,19 @@ def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
                     self._fb_unpack(EX_SKIP, write_bytes)
                     self._fb_unpack(EX_SKIP, write_bytes)
                 return
-            ret = []
-            for i in xrange(n):
-                ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes),
-                            self._fb_unpack(EX_CONSTRUCT, write_bytes)))
             if self.object_pairs_hook is not None:
-                ret = self.object_pairs_hook(ret)
+                ret = self.object_pairs_hook(
+                        (self._fb_unpack(EX_CONSTRUCT, write_bytes),
+                         self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                            for _ in xrange(n)
+                        )
             else:
-                ret = dict(ret)
-            if self.object_hook is not None:
-                ret = self.object_hook(ret)
+                ret = {}
+                for _ in xrange(n):
+                    key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                    ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                if self.object_hook is not None:
+                    ret = self.object_hook(ret)
             return ret
         if execute == EX_SKIP:
             return
@@ -421,7 +424,7 @@ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                 raise TypeError("default must be callable")
         self._default = default
 
-    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
+    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded")
         if obj is None:
@@ -454,6 +457,10 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
             raise PackValueError("Integer value out of range")
         if isinstance(obj, (Unicode, bytes)):
             if isinstance(obj, Unicode):
+                if self.encoding is None:
+                    raise TypeError(
+                            "Can't encode unicode string: "
+                            "no encoding is specified")
                 obj = obj.encode(self.encoding, self.unicode_errors)
             n = len(obj)
             if n <= 0x1f:

setup.py

@@ -46,7 +46,12 @@ def build_extension(self, ext):
             print("Install Cython >= 0.16 or install msgpack from PyPI.")
             print("Falling back to pure Python implementation.")
             return
-        return build_ext.build_extension(self, ext)
+        try:
+            return build_ext.build_extension(self, ext)
+        except Exception as e:
+            print("WARNING: Failed to compile extensiom modules.")
+            print("msgpack uses fallback pure python implementation.")
+            print(e)
 
 
 exec(open('msgpack/_version.py').read())
@@ -75,18 +80,19 @@ def __init__(self, *args, **kwargs):
     macros = [('__LITTLE_ENDIAN__', '1')]
 
 ext_modules = []
-ext_modules.append(Extension('msgpack._packer',
-                             sources=['msgpack/_packer.cpp'],
-                             libraries=libraries,
-                             include_dirs=['.'],
-                             define_macros=macros,
-                             ))
-ext_modules.append(Extension('msgpack._unpacker',
-                             sources=['msgpack/_unpacker.cpp'],
-                             libraries=libraries,
-                             include_dirs=['.'],
-                             define_macros=macros,
-                             ))
+if not hasattr(sys, 'pypy_version_info'):
+    ext_modules.append(Extension('msgpack._packer',
+                                 sources=['msgpack/_packer.cpp'],
+                                 libraries=libraries,
+                                 include_dirs=['.'],
+                                 define_macros=macros,
+                                 ))
+    ext_modules.append(Extension('msgpack._unpacker',
+                                 sources=['msgpack/_unpacker.cpp'],
+                                 libraries=libraries,
+                                 include_dirs=['.'],
+                                 define_macros=macros,
+                                 ))
 del libraries, macros
 
 

test/test_pack.py

@@ -151,7 +151,7 @@ def test_odict():
     od = odict(seq)
     assert unpackb(packb(od), use_list=1) == dict(seq)
     def pair_hook(seq):
-        return seq
+        return list(seq)
     assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq