Skip to content

Commit 2cd2137

Browse files
committed
Make sure the 'infer the types when reading (#1669)' works again
1 parent 6b8dace commit 2cd2137

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

pyiceberg/io/pyarrow.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1576,7 +1576,11 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
15761576
# Empty
15771577
return pa.Table.from_batches([], schema=arrow_schema)
15781578

1579-
result = pa.Table.from_batches(itertools.chain([first_batch], batches))
1579+
# Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
1580+
# as different batches can use different schema's (due to large_ types)
1581+
result = pa.concat_tables(
1582+
(pa.Table.from_batches([batch]) for batch in itertools.chain([first_batch], batches)), promote_options="permissive"
1583+
)
15801584

15811585
if property_as_bool(self._io.properties, PYARROW_USE_LARGE_TYPES_ON_READ, False):
15821586
deprecation_message(

0 commit comments

Comments
 (0)