convert json parse to classes

Author: tswast

google/cloud/bigquery/_helpers.py

@@ -759,9 +759,6 @@ def _row_iterator_page_to_arrow(page, column_names, arrow_types):
 
     arrays = []
     for column_index, arrow_type in enumerate(arrow_types):
-        # RowIterator parses JSON, but for arrow, we actually want to keep them
-        # as strings.
-        # TODO: Support STRUCT<JSON> and ARRAY<JSON>.
         arrays.append(pyarrow.array(page._columns[column_index], type=arrow_type))
 
     if isinstance(column_names, pyarrow.Schema):

google/cloud/bigquery/_pandas_helpers.py

@@ -21,8 +21,8 @@
 from typing import Any, Optional, Dict, Union
 
 from google.cloud.bigquery.table import _parse_schema_resource
+from google.cloud.bigquery import _helpers
 from google.cloud.bigquery._helpers import _rows_from_json
-from google.cloud.bigquery._helpers import _QUERY_PARAMS_FROM_JSON
 from google.cloud.bigquery._helpers import _SCALAR_VALUE_TO_JSON_PARAM
 from google.cloud.bigquery._helpers import _SUPPORTED_RANGE_ELEMENTS
 
@@ -571,14 +571,19 @@ def from_api_repr(cls, resource: dict) -> "ScalarQueryParameter":
         Returns:
             google.cloud.bigquery.query.ScalarQueryParameter: Instance
         """
+        # Import here to avoid circular imports.
+        from google.cloud.bigquery import schema
+
         name = resource.get("name")
         type_ = resource["parameterType"]["type"]
 
         # parameterValue might not be present if JSON resource originates
         # from the back-end - the latter omits it for None values.
         value = resource.get("parameterValue", {}).get("value")
         if value is not None:
-            converted = _QUERY_PARAMS_FROM_JSON[type_](value, None)
+            converted = _helpers.SCALAR_QUERY_PARAM_PARSER.to_py(
+                value, schema.SchemaField(name, type_)
+            )
         else:
             converted = None
 
@@ -693,13 +698,20 @@ def _from_api_repr_struct(cls, resource):
 
     @classmethod
     def _from_api_repr_scalar(cls, resource):
+        """Converts REST resource into a list of scalar values."""
+        # Import here to avoid circular imports.
+        from google.cloud.bigquery import schema
+
         name = resource.get("name")
         array_type = resource["parameterType"]["arrayType"]["type"]
         parameter_value = resource.get("parameterValue", {})
         array_values = parameter_value.get("arrayValues", ())
         values = [value["value"] for value in array_values]
         converted = [
-            _QUERY_PARAMS_FROM_JSON[array_type](value, None) for value in values
+            _helpers.SCALAR_QUERY_PARAM_PARSER.to_py(
+                value, schema.SchemaField(name, array_type)
+            )
+            for value in values
         ]
         return cls(name, array_type, converted)
 
@@ -850,6 +862,9 @@ def from_api_repr(cls, resource: dict) -> "StructQueryParameter":
         Returns:
             google.cloud.bigquery.query.StructQueryParameter: Instance
         """
+        # Import here to avoid circular imports.
+        from google.cloud.bigquery import schema
+
         name = resource.get("name")
         instance = cls(name)
         type_resources = {}
@@ -877,7 +892,9 @@ def from_api_repr(cls, resource: dict) -> "StructQueryParameter":
                 converted = ArrayQueryParameter.from_api_repr(struct_resource)
             else:
                 value = value["value"]
-                converted = _QUERY_PARAMS_FROM_JSON[type_](value, None)
+                converted = _helpers.SCALAR_QUERY_PARAM_PARSER.to_py(
+                    value, schema.SchemaField(name, type_)
+                )
             instance.struct_values[key] = converted
         return instance
 

google/cloud/bigquery/query.py

@@ -3531,22 +3531,14 @@ def _row_iterator_page_columns(schema, response):
     columns = []
     rows = response.get("rows", [])
 
-    def get_column_data(field_index):
+    def get_column_data(field_index, field):
         for row in rows:
-            yield row["f"][field_index]["v"]
-
-    def parse_column_data(column, field):
-        # pyarrow.json_() type needs to keep the data as a string, not parsed.
-        # TODO: support STRUCT<JSON> and ARRAY<JSON>
-        if field.field_type.casefold() == "json":
-            for value in column:
-                yield value
-
-        for value in column:
-            yield _helpers._field_from_json(value, field)
+            yield _helpers.DATA_FRAME_CELL_DATA_PARSER.to_py(
+                row["f"][field_index]["v"], field
+            )
 
     for field_index, field in enumerate(schema):
-        columns.append(parse_column_data(get_column_data(field_index), field))
+        columns.append(get_column_data(field_index, field))
 
     return columns