From 167c0b9bdabfafd32cfe64ddb61fbf6a1f664516 Mon Sep 17 00:00:00 2001 From: lipan02 Date: Mon, 26 Jan 2026 20:14:48 +0800 Subject: [PATCH 1/6] perf(python): optimize int64 list/tuple serialization with C++ batch processing Signed-off-by: lipan02 --- cpp/fory/python/pyfory.cc | 124 ++++++++++++++++++++ cpp/fory/python/pyfory.h | 13 ++ python/pyfory/collection.pxi | 21 +++- python/pyfory/includes/libserialization.pxd | 2 + python/pyfory/serialization.pyx | 2 +- 5 files changed, 159 insertions(+), 3 deletions(-) diff --git a/cpp/fory/python/pyfory.cc b/cpp/fory/python/pyfory.cc index 561cda8a8d..ced5ed1b7b 100644 --- a/cpp/fory/python/pyfory.cc +++ b/cpp/fory/python/pyfory.cc @@ -58,4 +58,128 @@ int Fory_PyFloatSequenceWriteToBuffer(PyObject *collection, Buffer *buffer, } return 0; } + +// Write varint64 with ZigZag encoding inline +// Returns number of bytes written +static inline uint32_t WriteVarint64ZigZag(uint8_t *arr, int64_t value) { + // ZigZag encoding: (value << 1) ^ (value >> 63) + uint64_t v = (static_cast(value) << 1) ^ + (static_cast(value >> 63)); + + if (v < 0x80) { + arr[0] = static_cast(v); + return 1; + } + arr[0] = static_cast((v & 0x7F) | 0x80); + if (v < 0x4000) { + arr[1] = static_cast(v >> 7); + return 2; + } + arr[1] = static_cast((v >> 7) | 0x80); + if (v < 0x200000) { + arr[2] = static_cast(v >> 14); + return 3; + } + arr[2] = static_cast((v >> 14) | 0x80); + if (v < 0x10000000) { + arr[3] = static_cast(v >> 21); + return 4; + } + arr[3] = static_cast((v >> 21) | 0x80); + if (v < 0x800000000ULL) { + arr[4] = static_cast(v >> 28); + return 5; + } + arr[4] = static_cast((v >> 28) | 0x80); + if (v < 0x40000000000ULL) { + arr[5] = static_cast(v >> 35); + return 6; + } + arr[5] = static_cast((v >> 35) | 0x80); + if (v < 0x2000000000000ULL) { + arr[6] = static_cast(v >> 42); + return 7; + } + arr[6] = static_cast((v >> 42) | 0x80); + if (v < 0x100000000000000ULL) { + arr[7] = static_cast(v >> 49); + return 8; + } + arr[7] = static_cast((v >> 49) | 0x80); + arr[8] = static_cast(v >> 56); + return 9; +} + +Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, + Buffer *buffer, + Py_ssize_t start_index) { + PyObject **items = PySequenceGetItems(collection); + if (items == nullptr) { + return -1; + } + Py_ssize_t size = Py_SIZE(collection); + + uint8_t *data = buffer->data() + start_index; + Py_ssize_t total_bytes = 0; + + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *item = items[i]; + int64_t value = PyLong_AsLongLong(item); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + uint32_t bytes_written = WriteVarint64ZigZag(data + total_bytes, value); + total_bytes += bytes_written; + } + + return total_bytes; +} + +// Read varint64 with ZigZag decoding inline +// Returns the number of bytes read +static inline uint32_t ReadVarint64ZigZag(const uint8_t *arr, int64_t *result) { + uint64_t v = 0; + uint32_t shift = 0; + uint32_t bytes_read = 0; + + // Read up to 9 bytes for varint64 + for (int i = 0; i < 9; i++) { + uint8_t b = arr[bytes_read++]; + v |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + + // ZigZag decoding: (v >> 1) ^ -(v & 1) + *result = static_cast((v >> 1) ^ (~(v & 1) + 1)); + return bytes_read; +} + +Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, + Py_ssize_t start_index, + Py_ssize_t count) { + if (!PyList_CheckExact(list)) { + return -1; + } + + const uint8_t *data = buffer->data() + start_index; + Py_ssize_t total_bytes = 0; + + for (Py_ssize_t i = 0; i < count; i++) { + int64_t value; + uint32_t bytes_read = ReadVarint64ZigZag(data + total_bytes, &value); + total_bytes += bytes_read; + + PyObject *py_int = PyLong_FromLongLong(value); + if (py_int == nullptr) { + return -1; + } + // Use PyList_SET_ITEM which steals the reference + PyList_SET_ITEM(list, i, py_int); + } + + return total_bytes; +} } // namespace fory diff --git a/cpp/fory/python/pyfory.h b/cpp/fory/python/pyfory.h index 467a86970f..cb2aa8736d 100644 --- a/cpp/fory/python/pyfory.h +++ b/cpp/fory/python/pyfory.h @@ -26,4 +26,17 @@ int Fory_PyBooleanSequenceWriteToBuffer(PyObject *collection, Buffer *buffer, Py_ssize_t start_index); int Fory_PyFloatSequenceWriteToBuffer(PyObject *collection, Buffer *buffer, Py_ssize_t start_index); +// Write a sequence of Python integers to buffer using varint64 encoding with +// ZigZag. Returns number of bytes written on success, -1 on error. +// start_index is the offset in the buffer where writing begins. +Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, + Buffer *buffer, + Py_ssize_t start_index); + +// Read a sequence of integers from buffer into a pre-allocated Python list. +// Uses varint64 encoding with ZigZag decoding. +// Returns number of bytes read on success, -1 on error. +Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, + Py_ssize_t start_index, + Py_ssize_t count); } // namespace fory \ No newline at end of file diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index 45b95654d2..09352ea204 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -191,10 +191,27 @@ cdef class CollectionSerializer(Serializer): self._add_element(collection_, i, buffer.read_string()) cdef inline _write_int(self, Buffer buffer, value): - for s in value: - buffer.write_varint64(s) + cdef Py_ssize_t bytes_written + cdef Py_ssize_t max_size + value_type = type(value) + if value_type is list or value_type is tuple: + # Reserve maximum possible size (9 bytes per varint64) + max_size = Py_SIZE(value) * 9 + buffer.grow(max_size) + bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, buffer.c_buffer.get(), buffer.writer_index) + if bytes_written >= 0: + buffer.writer_index += bytes_written + else: + for s in value: + buffer.write_varint64(s) cdef inline _read_int(self, Buffer buffer, int64_t len_, object collection_): + cdef Py_ssize_t bytes_read + if type(collection_) is list: + bytes_read = Fory_PyInt64SequenceReadFromBuffer(collection_, buffer.c_buffer.get(), buffer.reader_index, len_) + if bytes_read >= 0: + buffer.reader_index += bytes_read + return for i in range(len_): self._add_element(collection_, i, buffer.read_varint64()) diff --git a/python/pyfory/includes/libserialization.pxd b/python/pyfory/includes/libserialization.pxd index d1926911c5..1fc816bd5d 100644 --- a/python/pyfory/includes/libserialization.pxd +++ b/python/pyfory/includes/libserialization.pxd @@ -82,3 +82,5 @@ cdef extern from "fory/type/type.h" namespace "fory" nogil: cdef extern from "fory/python/pyfory.h" namespace "fory": int Fory_PyBooleanSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) int Fory_PyFloatSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) + Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) + Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(object list, CBuffer *buffer, Py_ssize_t start_index, Py_ssize_t count) diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 65e36fcbba..87a6bc41cb 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -36,7 +36,7 @@ from pyfory.meta.metastring import Encoding from pyfory.types import is_primitive_type from pyfory.policy import DeserializationPolicy, DEFAULT_POLICY from pyfory.includes.libserialization cimport \ - (TypeId, IsNamespacedType, IsTypeShareMeta, Fory_PyBooleanSequenceWriteToBuffer, Fory_PyFloatSequenceWriteToBuffer) + (TypeId, IsNamespacedType, IsTypeShareMeta, Fory_PyBooleanSequenceWriteToBuffer, Fory_PyFloatSequenceWriteToBuffer, Fory_PyInt64SequenceWriteToBuffer, Fory_PyInt64SequenceReadFromBuffer) from libc.stdint cimport int8_t, int16_t, int32_t, int64_t, uint64_t from libc.stdint cimport * From c1526661a0f9a55afdb8a5592a4a0d0232c2e8c5 Mon Sep 17 00:00:00 2001 From: lipan02 Date: Tue, 27 Jan 2026 11:20:02 +0800 Subject: [PATCH 2/6] perf(python): optimize int64 list/tuple serialization with C++ batch processing Signed-off-by: lipan02 --- cpp/fory/python/pyfory.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/fory/python/pyfory.cc b/cpp/fory/python/pyfory.cc index ced5ed1b7b..0fe7a370dc 100644 --- a/cpp/fory/python/pyfory.cc +++ b/cpp/fory/python/pyfory.cc @@ -143,14 +143,19 @@ static inline uint32_t ReadVarint64ZigZag(const uint8_t *arr, int64_t *result) { uint32_t bytes_read = 0; // Read up to 9 bytes for varint64 - for (int i = 0; i < 9; i++) { + for (int i = 0; i < 8; i++) { uint8_t b = arr[bytes_read++]; v |= static_cast(b & 0x7F) << shift; if ((b & 0x80) == 0) { - break; + // ZigZag decoding: (v >> 1) ^ -(v & 1) + *result = static_cast((v >> 1) ^ (~(v & 1) + 1)); + return bytes_read; } shift += 7; } + // 9th byte: use all 8 bits (no continuation bit masking) + uint8_t b = arr[bytes_read++]; + v |= static_cast(b) << 56; // ZigZag decoding: (v >> 1) ^ -(v & 1) *result = static_cast((v >> 1) ^ (~(v & 1) + 1)); From 55c00620d8abc47b07bd6f8b008c5669a6575cbb Mon Sep 17 00:00:00 2001 From: lipan02 Date: Tue, 27 Jan 2026 11:32:12 +0800 Subject: [PATCH 3/6] clean codestyle Signed-off-by: lipan02 --- cpp/fory/python/pyfory.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cpp/fory/python/pyfory.h b/cpp/fory/python/pyfory.h index cb2aa8736d..ba79fe64cd 100644 --- a/cpp/fory/python/pyfory.h +++ b/cpp/fory/python/pyfory.h @@ -26,16 +26,11 @@ int Fory_PyBooleanSequenceWriteToBuffer(PyObject *collection, Buffer *buffer, Py_ssize_t start_index); int Fory_PyFloatSequenceWriteToBuffer(PyObject *collection, Buffer *buffer, Py_ssize_t start_index); -// Write a sequence of Python integers to buffer using varint64 encoding with -// ZigZag. Returns number of bytes written on success, -1 on error. -// start_index is the offset in the buffer where writing begins. + Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, Buffer *buffer, Py_ssize_t start_index); -// Read a sequence of integers from buffer into a pre-allocated Python list. -// Uses varint64 encoding with ZigZag decoding. -// Returns number of bytes read on success, -1 on error. Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, Py_ssize_t start_index, Py_ssize_t count); From ca6159458c5627cbf71a991a3ee67df78ad0dbc2 Mon Sep 17 00:00:00 2001 From: lipan02 Date: Tue, 27 Jan 2026 15:25:31 +0800 Subject: [PATCH 4/6] fix: issue1 and issue2 Signed-off-by: lipan02 --- cpp/fory/python/pyfory.cc | 31 +++++++++++++++------ cpp/fory/python/pyfory.h | 3 +- python/pyfory/collection.pxi | 21 +++++++++----- python/pyfory/includes/libserialization.pxd | 4 +-- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/cpp/fory/python/pyfory.cc b/cpp/fory/python/pyfory.cc index 0fe7a370dc..d2f1e39d17 100644 --- a/cpp/fory/python/pyfory.cc +++ b/cpp/fory/python/pyfory.cc @@ -136,52 +136,67 @@ Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, } // Read varint64 with ZigZag decoding inline -// Returns the number of bytes read -static inline uint32_t ReadVarint64ZigZag(const uint8_t *arr, int64_t *result) { +// Returns the number of bytes read, or 0 on buffer overflow +static inline uint32_t +ReadVarint64ZigZag(const uint8_t *arr, Py_ssize_t remaining, int64_t *result) { + if (remaining <= 0) { + return 0; + } + uint64_t v = 0; uint32_t shift = 0; uint32_t bytes_read = 0; - // Read up to 9 bytes for varint64 + // Read up to 8 bytes with continuation bit for (int i = 0; i < 8; i++) { + if (bytes_read >= static_cast(remaining)) { + return 0; // Buffer overflow + } uint8_t b = arr[bytes_read++]; v |= static_cast(b & 0x7F) << shift; if ((b & 0x80) == 0) { - // ZigZag decoding: (v >> 1) ^ -(v & 1) *result = static_cast((v >> 1) ^ (~(v & 1) + 1)); return bytes_read; } shift += 7; } // 9th byte: use all 8 bits (no continuation bit masking) + if (bytes_read >= static_cast(remaining)) { + return 0; // Buffer overflow + } uint8_t b = arr[bytes_read++]; v |= static_cast(b) << 56; - // ZigZag decoding: (v >> 1) ^ -(v & 1) *result = static_cast((v >> 1) ^ (~(v & 1) + 1)); return bytes_read; } Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, Py_ssize_t start_index, - Py_ssize_t count) { + Py_ssize_t count, + Py_ssize_t buffer_len) { if (!PyList_CheckExact(list)) { return -1; } const uint8_t *data = buffer->data() + start_index; + Py_ssize_t remaining = buffer_len - start_index; Py_ssize_t total_bytes = 0; for (Py_ssize_t i = 0; i < count; i++) { int64_t value; - uint32_t bytes_read = ReadVarint64ZigZag(data + total_bytes, &value); + uint32_t bytes_read = + ReadVarint64ZigZag(data + total_bytes, remaining - total_bytes, &value); + if (bytes_read == 0) { + PyErr_SetString(PyExc_ValueError, "Buffer overflow reading varint64"); + return -1; + } total_bytes += bytes_read; PyObject *py_int = PyLong_FromLongLong(value); if (py_int == nullptr) { return -1; } - // Use PyList_SET_ITEM which steals the reference PyList_SET_ITEM(list, i, py_int); } diff --git a/cpp/fory/python/pyfory.h b/cpp/fory/python/pyfory.h index ba79fe64cd..a7801ae377 100644 --- a/cpp/fory/python/pyfory.h +++ b/cpp/fory/python/pyfory.h @@ -33,5 +33,6 @@ Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, Py_ssize_t start_index, - Py_ssize_t count); + Py_ssize_t count, + Py_ssize_t buffer_len); } // namespace fory \ No newline at end of file diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index 09352ea204..c479cc8fdc 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -198,17 +198,24 @@ cdef class CollectionSerializer(Serializer): # Reserve maximum possible size (9 bytes per varint64) max_size = Py_SIZE(value) * 9 buffer.grow(max_size) - bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, buffer.c_buffer.get(), buffer.writer_index) - if bytes_written >= 0: - buffer.writer_index += bytes_written - else: - for s in value: - buffer.write_varint64(s) + # Try batch write, fall back to slow path on error (e.g., int overflow) + try: + bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, buffer.c_buffer.get(), buffer.writer_index) + if bytes_written >= 0: + buffer.writer_index += bytes_written + return + except OverflowError: + pass # Fall through to slow path + # Slow path: write elements one by one + for s in value: + buffer.write_varint64(s) cdef inline _read_int(self, Buffer buffer, int64_t len_, object collection_): cdef Py_ssize_t bytes_read if type(collection_) is list: - bytes_read = Fory_PyInt64SequenceReadFromBuffer(collection_, buffer.c_buffer.get(), buffer.reader_index, len_) + # Pass buffer size for bounds checking + bytes_read = Fory_PyInt64SequenceReadFromBuffer( + collection_, buffer.c_buffer.get(), buffer.reader_index, len_, buffer.size()) if bytes_read >= 0: buffer.reader_index += bytes_read return diff --git a/python/pyfory/includes/libserialization.pxd b/python/pyfory/includes/libserialization.pxd index e70fe01bbe..08504f0385 100644 --- a/python/pyfory/includes/libserialization.pxd +++ b/python/pyfory/includes/libserialization.pxd @@ -84,5 +84,5 @@ cdef extern from "fory/type/type.h" namespace "fory" nogil: cdef extern from "fory/python/pyfory.h" namespace "fory": int Fory_PyBooleanSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) int Fory_PyFloatSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) - Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) - Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(object list, CBuffer *buffer, Py_ssize_t start_index, Py_ssize_t count) + Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) except? -1 + Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(object list, CBuffer *buffer, Py_ssize_t start_index, Py_ssize_t count, Py_ssize_t buffer_len) except? -1 From 0b233096092e41adbe0f4d0c751772a0b16e90f9 Mon Sep 17 00:00:00 2001 From: lipan02 Date: Tue, 27 Jan 2026 15:57:42 +0800 Subject: [PATCH 5/6] fix api error Signed-off-by: lipan02 --- python/pyfory/collection.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index a631b24ad3..e8a76fffdf 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -200,7 +200,7 @@ cdef class CollectionSerializer(Serializer): buffer.grow(max_size) # Try batch write, fall back to slow path on error (e.g., int overflow) try: - bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, buffer.c_buffer.get(), buffer.writer_index) + bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, &buffer.c_buffer, buffer.writer_index) if bytes_written >= 0: buffer.writer_index += bytes_written return @@ -215,7 +215,7 @@ cdef class CollectionSerializer(Serializer): if type(collection_) is list: # Pass buffer size for bounds checking bytes_read = Fory_PyInt64SequenceReadFromBuffer( - collection_, buffer.c_buffer.get(), buffer.reader_index, len_, buffer.size()) + collection_, &buffer.c_buffer, buffer.reader_index, len_, buffer.size()) if bytes_read >= 0: buffer.reader_index += bytes_read return From 57f48c415b9a73fcd3665e70832346530d79fd98 Mon Sep 17 00:00:00 2001 From: lipan02 Date: Tue, 27 Jan 2026 18:19:32 +0800 Subject: [PATCH 6/6] fix api error Signed-off-by: lipan02 --- cpp/fory/python/pyfory.cc | 14 +++++++------- cpp/fory/python/pyfory.h | 7 ++----- python/pyfory/collection.pxi | 8 ++------ python/pyfory/includes/libserialization.pxd | 4 ++-- 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/cpp/fory/python/pyfory.cc b/cpp/fory/python/pyfory.cc index d2f1e39d17..2367f4c7c7 100644 --- a/cpp/fory/python/pyfory.cc +++ b/cpp/fory/python/pyfory.cc @@ -111,14 +111,13 @@ static inline uint32_t WriteVarint64ZigZag(uint8_t *arr, int64_t value) { } Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, - Buffer *buffer, - Py_ssize_t start_index) { + Buffer *buffer) { PyObject **items = PySequenceGetItems(collection); if (items == nullptr) { return -1; } Py_ssize_t size = Py_SIZE(collection); - + uint32_t start_index = buffer->writer_index(); uint8_t *data = buffer->data() + start_index; Py_ssize_t total_bytes = 0; @@ -132,6 +131,7 @@ Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, total_bytes += bytes_written; } + buffer->IncreaseWriterIndex(total_bytes); return total_bytes; } @@ -172,15 +172,14 @@ ReadVarint64ZigZag(const uint8_t *arr, Py_ssize_t remaining, int64_t *result) { } Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, - Py_ssize_t start_index, - Py_ssize_t count, - Py_ssize_t buffer_len) { + Py_ssize_t count) { if (!PyList_CheckExact(list)) { return -1; } + uint32_t start_index = buffer->reader_index(); const uint8_t *data = buffer->data() + start_index; - Py_ssize_t remaining = buffer_len - start_index; + Py_ssize_t remaining = buffer->size() - start_index; Py_ssize_t total_bytes = 0; for (Py_ssize_t i = 0; i < count; i++) { @@ -200,6 +199,7 @@ Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, PyList_SET_ITEM(list, i, py_int); } + buffer->IncreaseReaderIndex(total_bytes); return total_bytes; } } // namespace fory diff --git a/cpp/fory/python/pyfory.h b/cpp/fory/python/pyfory.h index a7801ae377..502dca2530 100644 --- a/cpp/fory/python/pyfory.h +++ b/cpp/fory/python/pyfory.h @@ -28,11 +28,8 @@ int Fory_PyFloatSequenceWriteToBuffer(PyObject *collection, Buffer *buffer, Py_ssize_t start_index); Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(PyObject *collection, - Buffer *buffer, - Py_ssize_t start_index); + Buffer *buffer); Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(PyObject *list, Buffer *buffer, - Py_ssize_t start_index, - Py_ssize_t count, - Py_ssize_t buffer_len); + Py_ssize_t count); } // namespace fory \ No newline at end of file diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index e8a76fffdf..ea0e1df644 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -200,9 +200,8 @@ cdef class CollectionSerializer(Serializer): buffer.grow(max_size) # Try batch write, fall back to slow path on error (e.g., int overflow) try: - bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, &buffer.c_buffer, buffer.writer_index) + bytes_written = Fory_PyInt64SequenceWriteToBuffer(value, &buffer.c_buffer) if bytes_written >= 0: - buffer.writer_index += bytes_written return except OverflowError: pass # Fall through to slow path @@ -213,11 +212,8 @@ cdef class CollectionSerializer(Serializer): cdef inline _read_int(self, Buffer buffer, int64_t len_, object collection_): cdef Py_ssize_t bytes_read if type(collection_) is list: - # Pass buffer size for bounds checking - bytes_read = Fory_PyInt64SequenceReadFromBuffer( - collection_, &buffer.c_buffer, buffer.reader_index, len_, buffer.size()) + bytes_read = Fory_PyInt64SequenceReadFromBuffer(collection_, &buffer.c_buffer, len_) if bytes_read >= 0: - buffer.reader_index += bytes_read return for i in range(len_): self._add_element(collection_, i, buffer.read_varint64()) diff --git a/python/pyfory/includes/libserialization.pxd b/python/pyfory/includes/libserialization.pxd index 08504f0385..601d4af6c9 100644 --- a/python/pyfory/includes/libserialization.pxd +++ b/python/pyfory/includes/libserialization.pxd @@ -84,5 +84,5 @@ cdef extern from "fory/type/type.h" namespace "fory" nogil: cdef extern from "fory/python/pyfory.h" namespace "fory": int Fory_PyBooleanSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) int Fory_PyFloatSequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) - Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(object collection, CBuffer *buffer, Py_ssize_t start_index) except? -1 - Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(object list, CBuffer *buffer, Py_ssize_t start_index, Py_ssize_t count, Py_ssize_t buffer_len) except? -1 + Py_ssize_t Fory_PyInt64SequenceWriteToBuffer(object collection, CBuffer *buffer) except? -1 + Py_ssize_t Fory_PyInt64SequenceReadFromBuffer(object list, CBuffer *buffer, Py_ssize_t count) except? -1