Skip to content

Commit a434045

Browse files
committed
Remove compress/decompress and mark module as not reliant on the GIL
The `compress`/`decompress` functions will be moved to Python code for simplicity. C implementations can always be re-added in the future. Also, mark _zstd as not requiring the GIL.
1 parent eb7892a commit a434045

File tree

2 files changed

+3
-379
lines changed

2 files changed

+3
-379
lines changed

Modules/_zstd/_zstdmodule.c

Lines changed: 2 additions & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -588,198 +588,13 @@ _zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
588588
Py_RETURN_NONE;
589589
}
590590

591-
/*[clinic input]
592-
_zstd.compress
593-
594-
data: Py_buffer
595-
A bytes-like object, data to be compressed.
596-
level: object = None
597-
The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.
598-
options: object = None
599-
A dict object that contains advanced compression parameters.
600-
zstd_dict: object = None
601-
A ZstdDict object, a pre-trained zstd dictionary.
602-
603-
Compress a block of data, return a bytes object of zstd compressed data.
604-
605-
Refer to ZstdCompressor's docstring for a description of the
606-
optional arguments *level*, *options*, and *zstd_dict*.
607-
608-
For incremental compression, use an ZstdCompressor instead.
609-
[clinic start generated code]*/
610-
611-
static PyObject *
612-
_zstd_compress_impl(PyObject *module, Py_buffer *data, PyObject *level,
613-
PyObject *options, PyObject *zstd_dict)
614-
/*[clinic end generated code: output=0cca9399ca5c95cc input=e8a7c59073af923c]*/
615-
{
616-
_zstd_state* const _module_state = PyModule_GetState(module);
617-
if (_module_state == NULL) {
618-
return NULL;
619-
}
620-
PyObject *ret = NULL;
621-
ZstdCompressor self = {0};
622-
623-
/* Initialize & set ZstdCompressor */
624-
self.cctx = ZSTD_createCCtx();
625-
if (self.cctx == NULL) {
626-
PyErr_SetString(_module_state->ZstdError,
627-
"Unable to create ZSTD_CCtx instance.");
628-
goto error;
629-
}
630-
631-
if (level != Py_None && options != Py_None) {
632-
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
633-
return NULL;
634-
}
635-
636-
/* Set compressLevel/options to compression context */
637-
if (level != Py_None) {
638-
if (_PyZstd_set_c_parameters(&self, level, "level", "int") < 0) {
639-
goto error;
640-
}
641-
}
642-
643-
if (options != Py_None) {
644-
if (_PyZstd_set_c_parameters(&self, options, "options", "dict") < 0) {
645-
goto error;
646-
}
647-
}
648-
649-
/* Load dictionary to compression context */
650-
if (zstd_dict != Py_None) {
651-
if (_PyZstd_load_c_dict(&self, zstd_dict) < 0) {
652-
goto error;
653-
}
654-
self.dict = zstd_dict;
655-
}
656-
657-
ret = compress_impl(&self, data, ZSTD_e_end);
658-
if (ret == NULL) {
659-
goto error;
660-
} else {
661-
goto success;
662-
}
663-
error:
664-
Py_CLEAR(ret);
665-
success:
666-
/* Free decompression context */
667-
ZSTD_freeCCtx(self.cctx);
668-
return ret;
669-
}
670-
671-
/*[clinic input]
672-
_zstd.decompress
673-
674-
data: Py_buffer
675-
A bytes-like object, zstd data to be decompressed.
676-
zstd_dict: object = None
677-
A ZstdDict object, a pre-trained zstd dictionary.
678-
options: object = None
679-
A dict object that contains advanced decompression parameters.
680-
681-
Decompress one or more frames of data.
682-
683-
Refer to ZstdDecompressor's docstring for a description of the
684-
optional arguments *zstd_dict*, *options*.
685-
686-
For incremental decompression, use an ZstdDecompressor instead.
687-
[clinic start generated code]*/
688-
689-
static PyObject *
690-
_zstd_decompress_impl(PyObject *module, Py_buffer *data, PyObject *zstd_dict,
691-
PyObject *options)
692-
/*[clinic end generated code: output=2e8423588fb3b178 input=c18346e620e59039]*/
693-
{
694-
uint64_t decompressed_size;
695-
Py_ssize_t initial_size;
696-
ZstdDecompressor self = {0};
697-
ZSTD_inBuffer in;
698-
_zstd_state* const _module_state = PyModule_GetState(module);
699-
if (_module_state == NULL) {
700-
return NULL;
701-
}
702-
PyObject *ret = NULL;
703-
704-
/* Initialize & set ZstdDecompressor */
705-
self.dctx = ZSTD_createDCtx();
706-
if (self.dctx == NULL) {
707-
PyErr_SetString(_module_state->ZstdError,
708-
"Unable to create ZSTD_DCtx instance.");
709-
goto error;
710-
}
711-
self.at_frame_edge = 1;
712-
713-
/* Load dictionary to decompression context */
714-
if (zstd_dict != Py_None) {
715-
if (_PyZstd_load_d_dict(&self, zstd_dict) < 0) {
716-
goto error;
717-
}
718-
}
719-
720-
/* Set option to decompression context */
721-
if (options != Py_None) {
722-
if (_PyZstd_set_d_parameters(&self, options) < 0) {
723-
goto error;
724-
}
725-
}
726-
727-
/* Prepare input data */
728-
in.src = data->buf;
729-
in.size = data->len;
730-
in.pos = 0;
731-
732-
/* Get decompressed size */
733-
decompressed_size = ZSTD_getFrameContentSize(data->buf, data->len);
734-
/* These two zstd constants always > PY_SSIZE_T_MAX:
735-
ZSTD_CONTENTSIZE_UNKNOWN is (0ULL - 1)
736-
ZSTD_CONTENTSIZE_ERROR is (0ULL - 2) */
737-
if (decompressed_size <= (uint64_t) PY_SSIZE_T_MAX) {
738-
initial_size = (Py_ssize_t) decompressed_size;
739-
} else {
740-
initial_size = -1;
741-
}
742-
743-
/* Decompress */
744-
ret = decompress_impl(&self, &in, -1, initial_size,
745-
TYPE_ENDLESS_DECOMPRESSOR);
746-
if (ret == NULL) {
747-
goto error;
748-
}
749-
750-
/* Check data integrity. at_frame_edge flag is 1 when both the input and
751-
output streams are at a frame edge. */
752-
if (self.at_frame_edge == 0) {
753-
char *extra_msg = (Py_SIZE(ret) == 0) ? "." :
754-
", if want to output these decompressed data, use "
755-
"the ZstdDecompressor class to decompress.";
756-
PyErr_Format(_module_state->ZstdError,
757-
"Decompression failed: zstd data ends in an incomplete "
758-
"frame, maybe the input data was truncated. Decompressed "
759-
"data is %zd bytes%s",
760-
Py_SIZE(ret), extra_msg);
761-
goto error;
762-
}
763-
764-
goto success;
765-
766-
error:
767-
Py_CLEAR(ret);
768-
success:
769-
/* Free decompression context */
770-
ZSTD_freeDCtx(self.dctx);
771-
return ret;
772-
}
773-
774591
static PyMethodDef _zstd_methods[] = {
775592
_ZSTD__TRAIN_DICT_METHODDEF
776593
_ZSTD__FINALIZE_DICT_METHODDEF
777594
_ZSTD__GET_PARAM_BOUNDS_METHODDEF
778595
_ZSTD_GET_FRAME_SIZE_METHODDEF
779596
_ZSTD__GET_FRAME_INFO_METHODDEF
780597
_ZSTD__SET_PARAMETER_TYPES_METHODDEF
781-
_ZSTD_COMPRESS_METHODDEF
782-
_ZSTD_DECOMPRESS_METHODDEF
783598

784599
{0}
785600
};
@@ -1130,6 +945,8 @@ _zstd_free(void *module)
1130945

1131946
static struct PyModuleDef_Slot _zstd_slots[] = {
1132947
{Py_mod_exec, _zstd_exec},
948+
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
949+
1133950
{0}
1134951
};
1135952

0 commit comments

Comments
 (0)