From 251272a77d37f771dd7d306596df4824a351698f Mon Sep 17 00:00:00 2001 From: Koen Vossen Date: Wed, 9 Apr 2025 16:50:23 +0200 Subject: [PATCH] Temporary fix for Arrow issue 46057 --- pyiceberg/io/pyarrow.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a1fa696f38..99223f1253 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1441,11 +1441,15 @@ def _task_to_record_batches( # Apply the user filter if pyarrow_filter is not None: - current_batch = current_batch.filter(pyarrow_filter) + # Temporary fix until PyArrow 21 is released ( https://github.com/apache/arrow/pull/46057 ) + table = pa.Table.from_batches([current_batch]) + table = table.filter(pyarrow_filter) # skip empty batches - if current_batch.num_rows == 0: + if table.num_rows == 0: continue + current_batch = table.combine_chunks().to_batches()[0] + result_batch = _to_requested_schema( projected_schema, file_project_schema,