Skip to content

Commit 27f0cba

Browse files
methanemethane
committed
Merge branch 'master' of https://github.com/antocuni/msgpack-python into newspec
Conflicts: msgpack/fallback.py msgpack/unpack.h msgpack/unpack_define.h msgpack/unpack_template.h
2 parents 7123341 + 6386481 commit 27f0cba

File tree

11 files changed

+261
-41
lines changed

11 files changed

+261
-41
lines changed

‎README.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,14 @@ It is also possible to pack/unpack custom data types. Here is an example for
140140
``object_pairs_hook`` callback may instead be used to receive a list of
141141
key-value pairs.
142142

143+
Extended types
144+
^^^^^^^^^^^^^^^
145+
146+
It is also possible to pack/unpack custom data types using the msgpack feature
147+
of "extended types". For example, msgpack-pypy uses it to provide very fast serialization of int/float lists on top of PyPy (experimental for now):
148+
149+
https://bitbucket.org/antocuni/msgpack-pypy/src/default/msgpack_pypy.py
150+
143151

144152
Advanced unpacking control
145153
^^^^^^^^^^^^^^^^^^^^^^^^^^

‎msgpack/_packer.pyx‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ from cpython cimport *
55
from libc.stdlib cimport *
66
from libc.string cimport *
77
from libc.limits cimport *
8+
from libc.stdint cimport int8_t
89

910
from msgpack.exceptions import PackValueError
1011

@@ -29,6 +30,7 @@ cdef extern from "pack.h":
2930
int msgpack_pack_raw(msgpack_packer* pk, size_t l)
3031
int msgpack_pack_bin(msgpack_packer* pk, size_t l)
3132
int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
33+
int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l)
3234

3335
cdef int DEFAULT_RECURSE_LIMIT=511
3436

@@ -183,6 +185,9 @@ cdef class Packer(object):
183185
for v in o:
184186
ret = self._pack(v, nest_limit-1)
185187
if ret != 0: break
188+
elif self.handle_unknown_type(o):
189+
# it means that obj was succesfully packed, so we are done
190+
return 0
186191
elif self._default:
187192
o = self._default(o)
188193
ret = self._pack(o, nest_limit-1)
@@ -202,6 +207,13 @@ cdef class Packer(object):
202207
self.pk.length = 0
203208
return buf
204209

210+
def handle_unknown_type(self, obj):
211+
return None
212+
213+
def pack_extended_type(self, typecode, data):
214+
msgpack_pack_ext(&self.pk, typecode, len(data))
215+
msgpack_pack_raw_body(&self.pk, data, len(data))
216+
205217
def pack_array_header(self, size_t size):
206218
cdef int ret = msgpack_pack_array(&self.pk, size)
207219
if ret == -1:

‎msgpack/_unpacker.pyx‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ cdef extern from "unpack.h":
2424
PyObject* object_hook
2525
bint has_pairs_hook # call object_hook with k-v pairs
2626
PyObject* list_hook
27+
PyObject* ext_type_hook
2728
char *encoding
2829
char *unicode_errors
2930

@@ -45,6 +46,7 @@ cdef extern from "unpack.h":
4546

4647
cdef inline init_ctx(unpack_context *ctx,
4748
object object_hook, object object_pairs_hook, object list_hook,
49+
object ext_type_hook,
4850
bint use_list, char* encoding, char* unicode_errors):
4951
unpack_init(ctx)
5052
ctx.user.use_list = use_list
@@ -71,9 +73,17 @@ cdef inline init_ctx(unpack_context *ctx,
7173
raise TypeError("list_hook must be a callable.")
7274
ctx.user.list_hook = <PyObject*>list_hook
7375

76+
if ext_type_hook is not None:
77+
if not PyCallable_Check(ext_type_hook):
78+
raise TypeError("ext_type_hook must be a callable.")
79+
ctx.user.ext_type_hook = <PyObject*>ext_type_hook
80+
7481
ctx.user.encoding = encoding
7582
ctx.user.unicode_errors = unicode_errors
7683

84+
def default_read_extended_type(typecode, data):
85+
raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
86+
7787
def unpackb(object packed, object object_hook=None, object list_hook=None,
7888
bint use_list=1, encoding=None, unicode_errors="strict",
7989
object_pairs_hook=None,
@@ -106,7 +116,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
106116
unicode_errors = unicode_errors.encode('ascii')
107117
cerr = PyBytes_AsString(unicode_errors)
108118

109-
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
119+
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type,
120+
use_list, cenc, cerr)
110121
ret = unpack_construct(&ctx, buf, buf_len, &off)
111122
if ret == 1:
112123
obj = unpack_data(&ctx)
@@ -248,7 +259,10 @@ cdef class Unpacker(object):
248259
self.unicode_errors = unicode_errors
249260
cerr = PyBytes_AsString(self.unicode_errors)
250261

251-
init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
262+
ext_type_hook = self.read_extended_type
263+
Py_INCREF(ext_type_hook)
264+
init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
265+
ext_type_hook, use_list, cenc, cerr)
252266

253267
def feed(self, object next_bytes):
254268
"""Append `next_bytes` to internal buffer."""
@@ -358,6 +372,24 @@ cdef class Unpacker(object):
358372
"""
359373
return self._unpack(unpack_construct, write_bytes)
360374

375+
def unpack_one(self, object write_bytes=None):
376+
"""
377+
unpack one object
378+
379+
If write_bytes is not None, it will be called with parts of the raw
380+
message as it is unpacked.
381+
382+
Raises `UnpackValueError` if there are no more bytes to unpack.
383+
Raises ``ExtraData`` if there are still bytes left after the unpacking.
384+
"""
385+
try:
386+
result = self.unpack()
387+
except OutOfData:
388+
raise UnpackValueError("Data is not enough")
389+
if self.buf_head < self.buf_tail:
390+
raise ExtraData(result, self.buf[self.buf_head:])
391+
return result
392+
361393
def skip(self, object write_bytes=None):
362394
"""
363395
read and ignore one object, returning None
@@ -385,6 +417,9 @@ cdef class Unpacker(object):
385417
"""
386418
return self._unpack(read_map_header, write_bytes)
387419

420+
def read_extended_type(self, typecode, data):
421+
return default_read_extended_type(typecode, data)
422+
388423
def __iter__(self):
389424
return self
390425

‎msgpack/pack.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
7171
static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
7272
static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
7373

74+
static inline int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l);
75+
7476
static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
7577
{
7678
char* buf = pk->buf;

‎msgpack/pack_template.h‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -708,6 +708,66 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t
708708
msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
709709
}
710710

711+
/*
712+
* Ext
713+
*/
714+
715+
static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l)
716+
{
717+
if (l == 1) {
718+
unsigned char buf[2];
719+
buf[0] = 0xd4;
720+
buf[1] = (unsigned char)typecode;
721+
msgpack_pack_append_buffer(x, buf, 2);
722+
}
723+
else if(l == 2) {
724+
unsigned char buf[2];
725+
buf[0] = 0xd5;
726+
buf[1] = (unsigned char)typecode;
727+
msgpack_pack_append_buffer(x, buf, 2);
728+
}
729+
else if(l == 4) {
730+
unsigned char buf[2];
731+
buf[0] = 0xd6;
732+
buf[1] = (unsigned char)typecode;
733+
msgpack_pack_append_buffer(x, buf, 2);
734+
}
735+
else if(l == 8) {
736+
unsigned char buf[2];
737+
buf[0] = 0xd7;
738+
buf[1] = (unsigned char)typecode;
739+
msgpack_pack_append_buffer(x, buf, 2);
740+
}
741+
else if(l == 16) {
742+
unsigned char buf[2];
743+
buf[0] = 0xd8;
744+
buf[1] = (unsigned char)typecode;
745+
msgpack_pack_append_buffer(x, buf, 2);
746+
}
747+
else if(l < 256) {
748+
unsigned char buf[3];
749+
buf[0] = 0xc7;
750+
buf[1] = l;
751+
buf[2] = (unsigned char)typecode;
752+
msgpack_pack_append_buffer(x, buf, 3);
753+
} else if(l < 65536) {
754+
unsigned char buf[4];
755+
buf[0] = 0xc8;
756+
_msgpack_store16(&buf[1], (uint16_t)l);
757+
buf[3] = (unsigned char)typecode;
758+
msgpack_pack_append_buffer(x, buf, 4);
759+
} else {
760+
unsigned char buf[6];
761+
buf[0] = 0xc9;
762+
_msgpack_store32(&buf[1], (uint32_t)l);
763+
buf[5] = (unsigned char)typecode;
764+
msgpack_pack_append_buffer(x, buf, 6);
765+
}
766+
767+
}
768+
769+
770+
711771
#undef msgpack_pack_append_buffer
712772

713773
#undef TAKE8_8

‎msgpack/unpack.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ typedef struct unpack_user {
2424
PyObject *object_hook;
2525
bool has_pairs_hook;
2626
PyObject *list_hook;
27+
PyObject *ext_type_hook;
2728
const char *encoding;
2829
const char *unicode_errors;
2930
} unpack_user;
@@ -235,4 +236,21 @@ static inline int unpack_callback_bin(unpack_user* u, const char* b, const char*
235236
return 0;
236237
}
237238

239+
static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
240+
unsigned int lenght, msgpack_unpack_object* o)
241+
{
242+
PyObject *py;
243+
int8_t typecode = (int8_t)*pos++;
244+
if (!u->ext_type_hook) {
245+
PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL");
246+
return -1;
247+
}
248+
// lenght also includes the typecode, so the actual data is lenght-1
249+
py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1);
250+
if (!py)
251+
return -1;
252+
*o = py;
253+
return 0;
254+
}
255+
238256
#include "unpack_template.h"

‎msgpack/unpack_define.h‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,11 @@ typedef enum {
4545
CS_BIN_8 = 0x04,
4646
CS_BIN_16 = 0x05,
4747
CS_BIN_32 = 0x06,
48-
//CS_ = 0x07,
4948

50-
//CS_ = 0x08,
51-
//CS_ = 0x09,
49+
CS_EXT_8 = 0x07,
50+
CS_EXT_16 = 0x08,
51+
CS_EXT_32 = 0x09,
52+
5253
CS_FLOAT = 0x0a,
5354
CS_DOUBLE = 0x0b,
5455
CS_UINT_8 = 0x0c,
@@ -60,6 +61,12 @@ typedef enum {
6061
CS_INT_32 = 0x12,
6162
CS_INT_64 = 0x13,
6263

64+
//CS_FIXEXT1 = 0x14,
65+
//CS_FIXEXT2 = 0x15,
66+
//CS_FIXEXT4 = 0x16,
67+
//CS_FIXEXT8 = 0x17,
68+
//CS_FIXEXT16 = 0x18,
69+
6370
CS_RAW_8 = 0x19,
6471
CS_RAW_16 = 0x1a,
6572
CS_RAW_32 = 0x1b,
@@ -70,6 +77,7 @@ typedef enum {
7077

7178
ACS_RAW_VALUE,
7279
ACS_BIN_VALUE,
80+
ACS_EXT_VALUE,
7381
} msgpack_unpack_state;
7482

7583

‎msgpack/unpack_template.h‎

Lines changed: 34 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
184184
push_simple_value(_false);
185185
case 0xc3: // true
186186
push_simple_value(_true);
187-
//case 0xc7:
188-
//case 0xc8:
189-
//case 0xc9:
187+
//case 0xc4:
188+
//case 0xc5:
189+
//case 0xc6:
190+
case 0xc7: // ext 8
191+
again_fixed_trail(NEXT_CS(p), 1);
192+
case 0xc8: // ext 16
193+
again_fixed_trail(NEXT_CS(p), 2);
194+
case 0xc9: // ext 32
195+
again_fixed_trail(NEXT_CS(p), 4);
190196
case 0xca: // float
191197
case 0xcb: // double
192198
case 0xcc: // unsigned int 8
@@ -198,15 +204,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
198204
case 0xd2: // signed int 32
199205
case 0xd3: // signed int 64
200206
again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
201-
case 0xc4: // bin 8
202-
case 0xc5: // bin 16
203-
case 0xc6: // bin 32
204-
//case 0xd4:
205-
//case 0xd5:
206-
//case 0xd6: // big integer 16
207-
//case 0xd7: // big integer 32
208-
//case 0xd8: // big float 16
209-
case 0xd9: // raw 8
207+
case 0xd4: // fixext 1
208+
case 0xd5: // fixext 2
209+
case 0xd6: // fixext 4
210+
case 0xd7: // fixext 8
211+
again_fixed_trail_if_zero(ACS_EXT_VALUE,
212+
(1 << (((unsigned int)*p) & 0x03))+1,
213+
_ext_zero);
214+
case 0xd8: // fixext 16
215+
again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
216+
//case 0xd9:
210217
case 0xda: // raw 16
211218
case 0xdb: // raw 32
212219
case 0xdc: // array 16
@@ -237,8 +244,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
237244
if((size_t)(pe - p) < trail) { goto _out; }
238245
n = p; p += trail - 1;
239246
switch(cs) {
240-
//case CS_
241-
//case CS_
247+
case CS_EXT_8:
248+
again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero);
249+
case CS_EXT_16:
250+
again_fixed_trail_if_zero(ACS_EXT_VALUE,
251+
_msgpack_load16(uint16_t,n)+1,
252+
_ext_zero);
253+
case CS_EXT_32:
254+
again_fixed_trail_if_zero(ACS_EXT_VALUE,
255+
_msgpack_load32(uint32_t,n)+1,
256+
_ext_zero);
242257
case CS_FLOAT: {
243258
union { uint32_t i; float f; } mem;
244259
mem.i = _msgpack_load32(uint32_t,n);
@@ -269,26 +284,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
269284
case CS_INT_64:
270285
push_fixed_value(_int64, _msgpack_load64(int64_t,n));
271286

272-
//case CS_
273-
//case CS_
274-
//case CS_BIG_INT_16:
275-
// again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
276-
//case CS_BIG_INT_32:
277-
// again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
278-
//case ACS_BIG_INT_VALUE:
279-
//_big_int_zero:
280-
// // FIXME
281-
// push_variable_value(_big_int, data, n, trail);
282-
283-
//case CS_BIG_FLOAT_16:
284-
// again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
285-
//case CS_BIG_FLOAT_32:
286-
// again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
287-
//case ACS_BIG_FLOAT_VALUE:
288-
//_big_float_zero:
289-
// // FIXME
290-
// push_variable_value(_big_float, data, n, trail);
291-
292287
case CS_BIN_8:
293288
again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
294289
case CS_BIN_16:
@@ -309,6 +304,10 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
309304
_raw_zero:
310305
push_variable_value(_raw, data, n, trail);
311306

307+
case ACS_EXT_VALUE:
308+
_ext_zero:
309+
push_variable_value(_ext, data, n, trail);
310+
312311
case CS_ARRAY_16:
313312
start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
314313
case CS_ARRAY_32:
@@ -320,7 +319,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
320319
case CS_MAP_32:
321320
/* FIXME security guard */
322321
start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
323-
322+
324323
default:
325324
goto _failed;
326325
}

0 commit comments

Comments
 (0)