Alternate fixes for jython and legacy CPython (#310)

Python 3.4 is not supported officially.
But keep running test for a while, to know when msgpack-python
stop working on Python 3.4 actually.

The current patches did not work under jython-2.7.1 where implicit
casting of buffer or memoryview doesn't work. It may also be the
jython is a little pickier about string casting non string bytes
due to the underlying strong typing of java.

See issues #303 & #304.

Author: devendor

msgpack/fallback.py

@@ -100,13 +100,6 @@ def _get_data_from_buffer(obj):
     return view
 
 
-# Jython's memoryview support is incomplete
-# See https://github.com/msgpack/msgpack-python/issues/303
-_is_jython = sys.platform.startswith('java')
-if _is_jython:
-    _get_data_from_buffer = bytes
-
-
 def unpack(stream, **kwargs):
     warnings.warn(
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
@@ -133,6 +126,14 @@ def unpackb(packed, **kwargs):
     return ret
 
 
+if sys.version_info < (2, 7, 6):
+    def _unpack_from(f, b, o=0):
+        """Explicit typcast for legacy struct.unpack_from"""
+        return struct.unpack_from(f, bytes(b), o)
+else:
+    _unpack_from = struct.unpack_from
+
+
 class Unpacker(object):
     """Streaming unpacker.
 
@@ -241,12 +242,6 @@ def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
 
         #: array of bytes fed.
         self._buffer = bytearray()
-        # Some very old pythons don't support `struct.unpack_from()` with a
-        # `bytearray`. So we wrap it in a `buffer()` there.
-        if sys.version_info < (2, 7, 6):
-            self._buffer_view = buffer(self._buffer)
-        else:
-            self._buffer_view = self._buffer
         #: Which position we currently reads
         self._buff_i = 0
 
@@ -302,7 +297,8 @@ def feed(self, next_bytes):
             self._buff_i -= self._buf_checkpoint
             self._buf_checkpoint = 0
 
-        self._buffer += view
+        # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython
+        self._buffer.extend(view)
 
     def _consume(self):
         """ Gets rid of the used parts of the buffer. """
@@ -401,117 +397,117 @@ def _read_header(self, execute=EX_CONSTRUCT):
         elif b == 0xc5:
             typ = TYPE_BIN
             self._reserve(2)
-            n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
+            n = _unpack_from(">H", self._buffer, self._buff_i)[0]
             self._buff_i += 2
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc6:
             typ = TYPE_BIN
             self._reserve(4)
-            n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
+            n = _unpack_from(">I", self._buffer, self._buff_i)[0]
             self._buff_i += 4
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
             self._reserve(2)
-            L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i)
+            L, n = _unpack_from('Bb', self._buffer, self._buff_i)
             self._buff_i += 2
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
             self._reserve(3)
-            L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i)
+            L, n = _unpack_from('>Hb', self._buffer, self._buff_i)
             self._buff_i += 3
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
             self._reserve(5)
-            L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i)
+            L, n = _unpack_from('>Ib', self._buffer, self._buff_i)
             self._buff_i += 5
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xca:
             self._reserve(4)
-            obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xcb:
             self._reserve(8)
-            obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xcc:
             self._reserve(1)
             obj = self._buffer[self._buff_i]
             self._buff_i += 1
         elif b == 0xcd:
             self._reserve(2)
-            obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
             self._buff_i += 2
         elif b == 0xce:
             self._reserve(4)
-            obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xcf:
             self._reserve(8)
-            obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xd0:
             self._reserve(1)
-            obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from("b", self._buffer, self._buff_i)[0]
             self._buff_i += 1
         elif b == 0xd1:
             self._reserve(2)
-            obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
             self._buff_i += 2
         elif b == 0xd2:
             self._reserve(4)
-            obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xd3:
             self._reserve(8)
-            obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             self._reserve(2)
-            n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
             self._buff_i += 2
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             self._reserve(3)
-            n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
             self._buff_i += 3
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             self._reserve(5)
-            n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
             self._buff_i += 5
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             self._reserve(9)
-            n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
             self._buff_i += 9
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             self._reserve(17)
-            n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
             self._buff_i += 17
         elif b == 0xd9:
             typ = TYPE_RAW
@@ -524,43 +520,43 @@ def _read_header(self, execute=EX_CONSTRUCT):
         elif b == 0xda:
             typ = TYPE_RAW
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xdb:
             typ = TYPE_RAW
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xdc:
             typ = TYPE_ARRAY
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xdd:
             typ = TYPE_ARRAY
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xde:
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)