implement unpacking for all the fixtext formats

Author: antocuni

msgpack/_unpacker.pyx

@@ -25,6 +25,7 @@ cdef extern from "unpack.h":
         PyObject* object_hook
         bint has_pairs_hook # call object_hook with k-v pairs
         PyObject* list_hook
+        PyObject* ext_type_hook
         char *encoding
         char *unicode_errors
 
@@ -46,6 +47,7 @@ cdef extern from "unpack.h":
 
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook, object list_hook,
+                     object ext_type_hook,
                      bint use_list, char* encoding, char* unicode_errors):
     unpack_init(ctx)
     ctx.user.use_list = use_list
@@ -72,9 +74,17 @@ cdef inline init_ctx(unpack_context *ctx,
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = <PyObject*>list_hook
 
+    if ext_type_hook is not None:
+        if not PyCallable_Check(ext_type_hook):
+            raise TypeError("ext_type_hook must be a callable.")
+        ctx.user.ext_type_hook = <PyObject*>ext_type_hook
+
     ctx.user.encoding = encoding
     ctx.user.unicode_errors = unicode_errors
 
+def default_read_extended_type(typecode, data):
+    raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
+
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
             object_pairs_hook=None,
@@ -107,7 +117,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             unicode_errors = unicode_errors.encode('ascii')
         cerr = PyBytes_AsString(unicode_errors)
 
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type,
+              use_list, cenc, cerr)
     ret = unpack_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = unpack_data(&ctx)
@@ -249,7 +260,10 @@ cdef class Unpacker(object):
                 self.unicode_errors = unicode_errors
             cerr = PyBytes_AsString(self.unicode_errors)
 
-        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+        ext_type_hook = self.read_extended_type
+        Py_INCREF(ext_type_hook)
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
+                  ext_type_hook, use_list, cenc, cerr)
 
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
@@ -404,6 +418,9 @@ cdef class Unpacker(object):
         """
         return self._unpack(read_map_header, write_bytes)
 
+    def read_extended_type(self, typecode, data):
+        return default_read_extended_type(typecode, data)
+
     def __iter__(self):
         return self
 

msgpack/unpack.h

@@ -24,6 +24,7 @@ typedef struct unpack_user {
     PyObject *object_hook;
     bool has_pairs_hook;
     PyObject *list_hook;
+    PyObject *ext_type_hook;
     const char *encoding;
     const char *unicode_errors;
 } unpack_user;
@@ -226,4 +227,21 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     return 0;
 }
 
+static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
+                                      unsigned int lenght, msgpack_unpack_object* o)
+{
+    PyObject *py;
+    int8_t typecode = (int8_t)*pos++;
+    if (!u->ext_type_hook) {
+        PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL");
+        return -1;
+    }
+    // lenght also includes the typecode, so the actual data is lenght-1
+    py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
 #include "unpack_template.h"

msgpack/unpack_define.h

@@ -59,12 +59,12 @@ typedef enum {
     CS_INT_32            = 0x12,
     CS_INT_64            = 0x13,
 
-    //CS_                = 0x14,
-    //CS_                = 0x15,
-    //CS_BIG_INT_16        = 0x16,
-    //CS_BIG_INT_32        = 0x17,
-    //CS_BIG_FLOAT_16      = 0x18,
-    //CS_BIG_FLOAT_32      = 0x19,
+    CS_FIXEXT1           = 0x14,
+    CS_FIXEXT2           = 0x15,
+    CS_FIXEXT4           = 0x16,
+    CS_FIXEXT8           = 0x17,
+    CS_FIXEXT16          = 0x18,
+
     CS_RAW_16            = 0x1a,
     CS_RAW_32            = 0x1b,
     CS_ARRAY_16          = 0x1c,
@@ -75,6 +75,8 @@ typedef enum {
     //ACS_BIG_INT_VALUE,
     //ACS_BIG_FLOAT_VALUE,
     ACS_RAW_VALUE,
+    ACS_EXT_VALUE,
+
 } msgpack_unpack_state;
 
 

msgpack/unpack_template.h

@@ -202,12 +202,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 case 0xd2:  // signed int 32
                 case 0xd3:  // signed int 64
                     again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-                //case 0xd4:
-                //case 0xd5:
-                //case 0xd6:  // big integer 16
-                //case 0xd7:  // big integer 32
-                //case 0xd8:  // big float 16
-                //case 0xd9:  // big float 32
+                case 0xd4:  // fixext 1
+                case 0xd5:  // fixext 2
+                case 0xd6:  // fixext 4
+                case 0xd7:  // fixext 8
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 
+                                              (1 << (((unsigned int)*p) & 0x03))+1,
+                                              _ext_zero);
+                case 0xd8:  // fixext 16
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
+                //case 0xd9:
                 case 0xda:  // raw 16
                 case 0xdb:  // raw 32
                 case 0xdc:  // array 16
@@ -298,6 +302,10 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             _raw_zero:
                 push_variable_value(_raw, data, n, trail);
 
+            case ACS_EXT_VALUE:
+            _ext_zero:
+                push_variable_value(_ext, data, n, trail);
+
             case CS_ARRAY_16:
                 start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
             case CS_ARRAY_32:
@@ -309,7 +317,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             case CS_MAP_32:
                 /* FIXME security guard */
                 start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
-
+            
             default:
                 goto _failed;
             }

setup.py

@@ -92,6 +92,7 @@ def __init__(self, *args, **kwargs):
                                  libraries=libraries,
                                  include_dirs=['.'],
                                  define_macros=macros,
+                                 extra_compile_args=['-O0'],
                                  ))
 del libraries, macros
 

test/test_extension.py

@@ -1,3 +1,4 @@
+import py
 import array
 import struct
 import msgpack