We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 6b8dace commit 2cd2137Copy full SHA for 2cd2137
pyiceberg/io/pyarrow.py
@@ -1576,7 +1576,11 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
1576
# Empty
1577
return pa.Table.from_batches([], schema=arrow_schema)
1578
1579
- result = pa.Table.from_batches(itertools.chain([first_batch], batches))
+ # Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
1580
+ # as different batches can use different schema's (due to large_ types)
1581
+ result = pa.concat_tables(
1582
+ (pa.Table.from_batches([batch]) for batch in itertools.chain([first_batch], batches)), promote_options="permissive"
1583
+ )
1584
1585
if property_as_bool(self._io.properties, PYARROW_USE_LARGE_TYPES_ON_READ, False):
1586
deprecation_message(
0 commit comments