@@ -52,8 +52,6 @@ def arrow_batches(self) -> Iterator[pyarrow.RecordBatch]:
5252 result_rows = 0
5353
5454 for batch in self ._arrow_batches :
55- # Convert JSON columns to strings before casting
56- batch = self ._convert_json_to_string (batch )
5755 batch = pyarrow_utils .cast_batch (batch , self .schema .to_pyarrow ())
5856 result_rows += batch .num_rows
5957
@@ -69,38 +67,6 @@ def arrow_batches(self) -> Iterator[pyarrow.RecordBatch]:
6967
7068 yield batch
7169
72- def _convert_json_to_string (
73- self , batch : pyarrow .RecordBatch
74- ) -> pyarrow .RecordBatch :
75- """Convert JSON arrow extension types to string to avoid PyArrow compatibility issues."""
76- import logging
77-
78- new_arrays = []
79- new_fields = []
80-
81- for i , field in enumerate (batch .schema ):
82- array = batch .column (i )
83-
84- # Check if this column should be JSON based on our schema
85- schema_item = next (
86- (item for item in self .schema .items if item .column == field .name ), None
87- )
88-
89- if schema_item and schema_item .dtype == bigframes .dtypes .JSON_DTYPE :
90- logging .info (f"Converting JSON column: { field .name } " )
91- # Convert JSONArrowType to string
92- if array .type == bigframes .dtypes .JSON_ARROW_TYPE :
93- array = array .cast (pyarrow .string ())
94- new_fields .append (pyarrow .field (field .name , pyarrow .string ()))
95- else :
96- new_fields .append (field )
97-
98- new_arrays .append (array )
99-
100- return pyarrow .RecordBatch .from_arrays (
101- new_arrays , schema = pyarrow .schema (new_fields )
102- )
103-
10470 def to_arrow_table (self ) -> pyarrow .Table :
10571 # Need to provide schema if no result rows, as arrow can't infer
10672 # If ther are rows, it is safest to infer schema from batches.
0 commit comments