From 4d369a2fb5dc594b71e8c3c970562fdcb0eeb1ce Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 7 Aug 2025 16:13:33 +0000 Subject: [PATCH 01/21] draft proof of concept microgenerator 2 --- .../microgenerator/bigqueryclient_config.yaml | 21 +++ .../bigqueryclient_generator.py | 169 ++++++++++++++++++ scripts/microgenerator/config_helper.py | 39 ++++ 3 files changed, 229 insertions(+) create mode 100644 scripts/microgenerator/bigqueryclient_config.yaml create mode 100644 scripts/microgenerator/bigqueryclient_generator.py create mode 100644 scripts/microgenerator/config_helper.py diff --git a/scripts/microgenerator/bigqueryclient_config.yaml b/scripts/microgenerator/bigqueryclient_config.yaml new file mode 100644 index 000000000..5a0e08f0b --- /dev/null +++ b/scripts/microgenerator/bigqueryclient_config.yaml @@ -0,0 +1,21 @@ +# TODO: add header if needed. + +include_class_name_patterns: + - Client + +exclude_class_name_patterns: [] + +include_method_name_patterns: + - batch_delete_ + - cancel_ + - create_ + - delete_ + - get_ + - insert_ + - list_ + - patch_ + - undelete_ + - update_ + +exclude_method_name_patterns: + - get_mtls_endpoint_and_cert_source diff --git a/scripts/microgenerator/bigqueryclient_generator.py b/scripts/microgenerator/bigqueryclient_generator.py new file mode 100644 index 000000000..c5ab7b04c --- /dev/null +++ b/scripts/microgenerator/bigqueryclient_generator.py @@ -0,0 +1,169 @@ +import ast +import os +from collections import defaultdict + +import jinja2 + +from config_helper import ( + CLASSES_TO_INCLUDE, + # CLASSES_TO_EXCLUDE, # Not currently being used. + METHODS_TO_INCLUDE, + METHODS_TO_EXCLUDE, +) + +# Constants +BASE_DIR = "google/cloud/bigquery_v2/services" +FILES_TO_PARSE = [ + os.path.join(root, file) + for root, _, files in os.walk(BASE_DIR) + for file in files + if file.endswith(".py") +] + + +def create_tree(file_path): + with open(file_path, "r") as source: + tree = ast.parse(source.read()) + return tree + + +def _extract_classes(tree): + """Extracts class nodes from an AST.""" + classes = [] + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name.endswith( + *CLASSES_TO_INCLUDE + ): # TODO: currently this is one class. Refactor if necessary + classes.append(node) + return classes + + +def _extract_methods(class_node): + """Extracts method nodes from a class node.""" + return (m for m in class_node.body if isinstance(m, ast.FunctionDef)) + + +def _process_method(method, class_name, parsed_data): + """Processes a single method and updates parsed_data.""" + method_name = method.name + if any(method_name.startswith(prefix) for prefix in METHODS_TO_INCLUDE) and not any( + method_name.startswith(prefix) for prefix in METHODS_TO_EXCLUDE + ): + parameters = [arg.arg for arg in method.args.args + method.args.kwonlyargs] + parsed_data[class_name][method_name] = parameters + + +def parse_files(file_paths): + """ + Parse a list of Python files and extract information about classes, + methods, and parameters. + + Args: + file_paths (list): List of file paths to parse. + + Returns: + Defaultdict with zero or more entries. + """ + + parsed_data = defaultdict(dict) + + for file_path in file_paths: + tree = create_tree(file_path) + + for class_ in _extract_classes(tree): + class_name = class_.name + parsed_data[class_name] + + for method in _extract_methods(class_): + _process_method(method, class_name, parsed_data) + + return parsed_data + + +def _format_args(method_args): + """Formats method arguments for use in creating a method definition + and a method call. + """ + args_for_def = ", ".join(method_args) + args_for_call = ", ".join([f"{arg}={arg}" for arg in method_args if arg != "self"]) + return args_for_def, args_for_call + + +def _format_class_name(method_name, suffix="Request"): + """Formats a class name from a method name. + + Example: + list_datasets -> ListDatasetsRequest + """ + return "".join(word.capitalize() for word in method_name.split("_")) + suffix + + +def generate_client_class_source(data): + """ + Generates the BigQueryClient source code using a Jinja2 template. + + Args: + data: A dictionary where keys are *ServiceClient class names and + values are dictionaries of methods for that client. + + Returns: + A string containing the complete, formatted Python source code + for the BigQueryClient class. + """ + + # TODO: move template strings to a separate file. + class_template_string = """\ +class BigQueryClient: + def __init__(self): + self._clients = {} + +{% for method in methods %} + def {{ method.name }}({{ method.args_for_def }}): + \"\"\"A generated method to call the BigQuery API.\"\"\" + + if "{{ method.class_name }}" not in self._clients: + from google.cloud.bigquery_v2 import {{ method.class_name }} + self._clients["{{ method.class_name }}"] = {{ method.class_name }}() + + client = self._clients["{{ method.class_name }}"] + from google.cloud.bigquery_v2 import types + request = types.{{ method.request_class_name }}({{ method.args_for_call }}) + return client.{{ method.name }}(request=request) + +{% endfor %} +""" + + # Prepare the context for the template. + # We transform the input data into a flat list of methods + methods_context = [] + for class_name, methods in data.items(): + for method_name, method_args in methods.items(): + args_for_def, args_for_call = _format_args(method_args) + request_class_name = _format_class_name(method_name) + methods_context.append( + { + "name": method_name, + "class_name": class_name, + "args_for_def": args_for_def, + "args_for_call": args_for_call, + "request_class_name": request_class_name, + } + ) + + # Create a Jinja2 Template object and render it with the context. + template = jinja2.Template(class_template_string, trim_blocks=True) + generated_code = template.render(methods=methods_context) + + return generated_code + + +if __name__ == "__main__": + data = parse_files(FILES_TO_PARSE) + + final_code = generate_client_class_source(data) + # TODO: write final code to file. + + print(final_code) + + # Ensure black gets called on the generated source files as a final step. diff --git a/scripts/microgenerator/config_helper.py b/scripts/microgenerator/config_helper.py new file mode 100644 index 000000000..fcad4673c --- /dev/null +++ b/scripts/microgenerator/config_helper.py @@ -0,0 +1,39 @@ +# config_helper.py + +import yaml +import os + + +def load_config_yaml(filepath): + """Loads configuration from a YAML file.""" + try: + with open(filepath, "r") as f: + config = yaml.safe_load(f) + return config + except FileNotFoundError: + print(f"Error: Configuration file '{filepath}' not found.") + return None + except yaml.YAMLError as e: + print(f"Error: Could not load YAML from '{filepath}': {e}") + return None + + +# Determine the absolute path to the config file relative to this file. +# This makes the path robust to where the script is run from. +_CONFIG_FILE_PATH = os.path.join( + os.path.dirname(__file__), "bigqueryclient_config.yaml" +) + +config_data = load_config_yaml(_CONFIG_FILE_PATH) + +if config_data: + CLASSES_TO_INCLUDE = config_data.get("include_class_name_patterns", []) + CLASSES_TO_EXCLUDE = config_data.get("exclude_class_name_patterns", []) + METHODS_TO_INCLUDE = config_data.get("include_method_name_patterns", []) + METHODS_TO_EXCLUDE = config_data.get("exclude_method_name_patterns", []) +else: + # Define default empty values if the config fails to load + CLASSES_TO_INCLUDE = [] + CLASSES_TO_EXCLUDE = [] + METHODS_TO_INCLUDE = [] + METHODS_TO_EXCLUDE = [] From be1f56848cd5489f64607d42a0bfc8f4ceef6e7a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 7 Aug 2025 16:20:53 +0000 Subject: [PATCH 02/21] minor update to docstring --- scripts/microgenerator/bigqueryclient_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/microgenerator/bigqueryclient_generator.py b/scripts/microgenerator/bigqueryclient_generator.py index c5ab7b04c..2011aa72f 100644 --- a/scripts/microgenerator/bigqueryclient_generator.py +++ b/scripts/microgenerator/bigqueryclient_generator.py @@ -34,7 +34,7 @@ def _extract_classes(tree): for node in ast.walk(tree): if isinstance(node, ast.ClassDef) and node.name.endswith( *CLASSES_TO_INCLUDE - ): # TODO: currently this is one class. Refactor if necessary + ): # TODO: currently this is variable includes only one class. Refactor if necessary classes.append(node) return classes From bef908153a3925d1e8ef619b66caa6c5c05a2ac8 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 7 Aug 2025 17:09:59 +0000 Subject: [PATCH 03/21] adds templating utility and template file --- scripts/microgenerator/bigqueryclient.py.j2 | 18 ++++++++++++ .../bigqueryclient_generator.py | 28 ++----------------- scripts/microgenerator/template_utils.py | 15 ++++++++++ 3 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 scripts/microgenerator/bigqueryclient.py.j2 create mode 100644 scripts/microgenerator/template_utils.py diff --git a/scripts/microgenerator/bigqueryclient.py.j2 b/scripts/microgenerator/bigqueryclient.py.j2 new file mode 100644 index 000000000..0166e0842 --- /dev/null +++ b/scripts/microgenerator/bigqueryclient.py.j2 @@ -0,0 +1,18 @@ +class BigQueryClient: + def __init__(self): + self._clients = {} + +{% for method in methods %} + def {{ method.name }}({{ method.args_for_def }}): + """A generated method to call the BigQuery API.""" + + if "{{ method.class_name }}" not in self._clients: + from google.cloud.bigquery_v2 import {{ method.class_name }} + self._clients["{{ method.class_name }}"] = {{ method.class_name }}() + + client = self._clients["{{ method.class_name }}"] + from google.cloud.bigquery_v2 import types + request = types.{{ method.request_class_name }}({{ method.args_for_call }}) + return client.{{ method.name }}(request=request) + +{% endfor %} diff --git a/scripts/microgenerator/bigqueryclient_generator.py b/scripts/microgenerator/bigqueryclient_generator.py index 2011aa72f..d3eed4ecf 100644 --- a/scripts/microgenerator/bigqueryclient_generator.py +++ b/scripts/microgenerator/bigqueryclient_generator.py @@ -2,14 +2,13 @@ import os from collections import defaultdict -import jinja2 - from config_helper import ( CLASSES_TO_INCLUDE, # CLASSES_TO_EXCLUDE, # Not currently being used. METHODS_TO_INCLUDE, METHODS_TO_EXCLUDE, ) +from template_utils import load_template # Constants BASE_DIR = "google/cloud/bigquery_v2/services" @@ -112,27 +111,7 @@ def generate_client_class_source(data): for the BigQueryClient class. """ - # TODO: move template strings to a separate file. - class_template_string = """\ -class BigQueryClient: - def __init__(self): - self._clients = {} - -{% for method in methods %} - def {{ method.name }}({{ method.args_for_def }}): - \"\"\"A generated method to call the BigQuery API.\"\"\" - - if "{{ method.class_name }}" not in self._clients: - from google.cloud.bigquery_v2 import {{ method.class_name }} - self._clients["{{ method.class_name }}"] = {{ method.class_name }}() - - client = self._clients["{{ method.class_name }}"] - from google.cloud.bigquery_v2 import types - request = types.{{ method.request_class_name }}({{ method.args_for_call }}) - return client.{{ method.name }}(request=request) - -{% endfor %} -""" + template = load_template("bigqueryclient.py.j2") # Prepare the context for the template. # We transform the input data into a flat list of methods @@ -151,8 +130,7 @@ def {{ method.name }}({{ method.args_for_def }}): } ) - # Create a Jinja2 Template object and render it with the context. - template = jinja2.Template(class_template_string, trim_blocks=True) + # Render the template with the context. generated_code = template.render(methods=methods_context) return generated_code diff --git a/scripts/microgenerator/template_utils.py b/scripts/microgenerator/template_utils.py new file mode 100644 index 000000000..cd095624b --- /dev/null +++ b/scripts/microgenerator/template_utils.py @@ -0,0 +1,15 @@ +import os +import jinja2 + + +def load_template(template_name): + """ + Loads a Jinja2 template from the same directory as the script. + """ + template_dir = os.path.dirname(os.path.abspath(__file__)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir), + trim_blocks=True, + lstrip_blocks=True, + ) + return env.get_template(template_name) From c421c23615f83b30266379de5b52b6727c1a0724 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 8 Aug 2025 10:35:03 +0000 Subject: [PATCH 04/21] Minor revision to TODO comment --- scripts/microgenerator/bigqueryclient_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/microgenerator/bigqueryclient_config.yaml b/scripts/microgenerator/bigqueryclient_config.yaml index 5a0e08f0b..4df44e88c 100644 --- a/scripts/microgenerator/bigqueryclient_config.yaml +++ b/scripts/microgenerator/bigqueryclient_config.yaml @@ -1,4 +1,4 @@ -# TODO: add header if needed. +# TODO: Add a header if needed. include_class_name_patterns: - Client From 3b8718980c0b2953f2b3fd286300e547f1adb1cb Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 14 Aug 2025 11:51:41 +0000 Subject: [PATCH 05/21] Updates several comments --- scripts/microgenerator/bigqueryclient.py.j2 | 7 +++++++ scripts/microgenerator/bigqueryclient_generator.py | 6 ++++-- scripts/microgenerator/config_helper.py | 2 +- scripts/microgenerator/template_utils.py | 6 ++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/microgenerator/bigqueryclient.py.j2 b/scripts/microgenerator/bigqueryclient.py.j2 index 0166e0842..728708ca8 100644 --- a/scripts/microgenerator/bigqueryclient.py.j2 +++ b/scripts/microgenerator/bigqueryclient.py.j2 @@ -1,3 +1,10 @@ +# TODO: Add a header if needed. + +# ======== 🦕 HERE THERE BE DINOSAURS 🦖 ========= +# This content is subject to significant change. Not for review yet. +# Included as a proof of concept for context or testing ONLY. +# ================================================ + class BigQueryClient: def __init__(self): self._clients = {} diff --git a/scripts/microgenerator/bigqueryclient_generator.py b/scripts/microgenerator/bigqueryclient_generator.py index d3eed4ecf..0138948a7 100644 --- a/scripts/microgenerator/bigqueryclient_generator.py +++ b/scripts/microgenerator/bigqueryclient_generator.py @@ -1,3 +1,5 @@ +# TODO: Add a header if needed. + import ast import os from collections import defaultdict @@ -140,8 +142,8 @@ def generate_client_class_source(data): data = parse_files(FILES_TO_PARSE) final_code = generate_client_class_source(data) - # TODO: write final code to file. + # TODO: write final code to file instead of print to screen. print(final_code) - # Ensure black gets called on the generated source files as a final step. + # TODO: Ensure blacken gets called on the generated source files as a final step. diff --git a/scripts/microgenerator/config_helper.py b/scripts/microgenerator/config_helper.py index fcad4673c..ce5ab0806 100644 --- a/scripts/microgenerator/config_helper.py +++ b/scripts/microgenerator/config_helper.py @@ -1,4 +1,4 @@ -# config_helper.py +# TODO: Add a header if needed. import yaml import os diff --git a/scripts/microgenerator/template_utils.py b/scripts/microgenerator/template_utils.py index cd095624b..2cfd9cd87 100644 --- a/scripts/microgenerator/template_utils.py +++ b/scripts/microgenerator/template_utils.py @@ -1,3 +1,5 @@ +# TODO: Add a header if needed. + import os import jinja2 @@ -9,7 +11,7 @@ def load_template(template_name): template_dir = os.path.dirname(os.path.abspath(__file__)) env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_dir), - trim_blocks=True, - lstrip_blocks=True, + trim_blocks=True, # prevents blank lines by removing '\n' after block tags (e.g. {% if condition %}\n) + lstrip_blocks=True, # prevents unwanted empty spaces before lines of text by removing non-explicit spaces, tabs, etc ) return env.get_template(template_name) From 698f4013ad5be81b1b3c914688d8e899136db384 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 14 Aug 2025 13:07:03 +0000 Subject: [PATCH 06/21] Adds example of generated method --- scripts/microgenerator/bigqueryclient.py.j2 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/microgenerator/bigqueryclient.py.j2 b/scripts/microgenerator/bigqueryclient.py.j2 index 728708ca8..55379062d 100644 --- a/scripts/microgenerator/bigqueryclient.py.j2 +++ b/scripts/microgenerator/bigqueryclient.py.j2 @@ -11,7 +11,8 @@ class BigQueryClient: {% for method in methods %} def {{ method.name }}({{ method.args_for_def }}): - """A generated method to call the BigQuery API.""" + """TODO: extract docstring for use here. + A generated method to call the BigQuery API.""" if "{{ method.class_name }}" not in self._clients: from google.cloud.bigquery_v2 import {{ method.class_name }} @@ -23,3 +24,18 @@ class BigQueryClient: return client.{{ method.name }}(request=request) {% endfor %} + +# ======== An example of the code generated ====== + + def get_dataset(self, request, retry, timeout, metadata): + """TODO: extract docstring for use here. + A generated method to call the BigQuery API.""" + + if "DatasetServiceClient" not in self._clients: + from google.cloud.bigquery_v2 import DatasetServiceClient + self._clients["DatasetServiceClient"] = DatasetServiceClient() + + client = self._clients["DatasetServiceClient"] + from google.cloud.bigquery_v2 import types + request = types.GetDatasetRequest(request=request, retry=retry, timeout=timeout, metadata=metadata) + return client.get_dataset(request=request) From b736b235e71d0ce5f807a04d40ed3f9d804e75ba Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 7 Aug 2025 13:56:43 -0400 Subject: [PATCH 07/21] feat: adds centralized client and tests (#2249) --- google/cloud/bigquery_v2/__init__.py | 2 + .../services/centralized_services/__init__.py | 18 ++ .../services/centralized_services/client.py | 180 ++++++++++++ .../bigquery_v2/test_centralized_services.py | 259 ++++++++++++++++++ 4 files changed, 459 insertions(+) create mode 100644 google/cloud/bigquery_v2/services/centralized_services/__init__.py create mode 100644 google/cloud/bigquery_v2/services/centralized_services/client.py create mode 100644 tests/unit/gapic/bigquery_v2/test_centralized_services.py diff --git a/google/cloud/bigquery_v2/__init__.py b/google/cloud/bigquery_v2/__init__.py index 83c82e729..442609adc 100644 --- a/google/cloud/bigquery_v2/__init__.py +++ b/google/cloud/bigquery_v2/__init__.py @@ -24,6 +24,7 @@ from .services.routine_service import RoutineServiceClient from .services.row_access_policy_service import RowAccessPolicyServiceClient from .services.table_service import TableServiceClient +from .services.centralized_services import BigQueryClient from .types.biglake_config import BigLakeConfiguration from .types.clustering import Clustering @@ -214,6 +215,7 @@ "BiEngineReason", "BiEngineStatistics", "BigLakeConfiguration", + "BigQueryClient", "BigtableColumn", "BigtableColumnFamily", "BigtableOptions", diff --git a/google/cloud/bigquery_v2/services/centralized_services/__init__.py b/google/cloud/bigquery_v2/services/centralized_services/__init__.py new file mode 100644 index 000000000..e82098176 --- /dev/null +++ b/google/cloud/bigquery_v2/services/centralized_services/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryClient + +__all__ = ("BigQueryClient",) diff --git a/google/cloud/bigquery_v2/services/centralized_services/client.py b/google/cloud/bigquery_v2/services/centralized_services/client.py new file mode 100644 index 000000000..aa581d2ab --- /dev/null +++ b/google/cloud/bigquery_v2/services/centralized_services/client.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from typing import ( + Optional, + Sequence, + Tuple, + Union, +) + +# Import service clients +from google.cloud.bigquery_v2.services.dataset_service import DatasetServiceClient +from google.cloud.bigquery_v2.services.job_service import JobServiceClient +from google.cloud.bigquery_v2.services.model_service import ModelServiceClient + +# Import Request classes +from google.cloud.bigquery_v2.types import ( + # DatasetService Request classes + GetDatasetRequest, + # JobService Request classes + ListJobsRequest, + # ModelService Request classes + DeleteModelRequest, + GetModelRequest, + PatchModelRequest, + ListModelsRequest, +) + +from google.api_core import gapic_v1 +from google.api_core import retry as retries + +# Create a type alias +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +# TODO: revise this universally. +PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT") + +DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT +DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT +DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () + + +def _drop_self_key(kwargs): + "Drops 'self' key from a given kwargs dict." + + if not isinstance(kwargs, dict): + raise TypeError("kwargs must be a dict.") + kwargs.pop("self", None) # Essentially a no-op if 'self' key does not exist + return kwargs + + +# Create Centralized Client +class BigQueryClient: + def __init__(self): + # Dataset service related init attributes + self.dataset_service_client = DatasetServiceClient() + self.job_service_client = JobServiceClient() + self.model_service_client = ModelServiceClient() + + def get_dataset( + self, + request: Optional[Union[GetDatasetRequest, dict]] = None, + *, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ): + """ + TODO: Add docstring. + """ + kwargs = _drop_self_key(locals()) + return self.dataset_service_client.get_dataset(**kwargs) + + def list_jobs( + self, + request: Optional[Union[ListJobsRequest, dict]] = None, + *, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ): + """ + TODO: Add docstring. + """ + kwargs = _drop_self_key(locals()) + return self.job_service_client.list_jobs(**kwargs) + + def get_model( + self, + request: Optional[Union[GetModelRequest, dict]] = None, + *, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ): + """ + TODO: Add docstring. + """ + kwargs = _drop_self_key(locals()) + return self.model_service_client.get_model(**kwargs) + + def delete_model( + self, + request: Optional[Union[DeleteModelRequest, dict]] = None, + *, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ): + """ + TODO: Add docstring. + """ + kwargs = _drop_self_key(locals()) + # The underlying GAPIC client returns None on success. + return self.model_service_client.delete_model(**kwargs) + + def patch_model( + self, + request: Optional[Union[PatchModelRequest, dict]] = None, + *, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ): + """ + TODO: Add docstring. + """ + kwargs = _drop_self_key(locals()) + return self.model_service_client.patch_model(**kwargs) + + def list_models( + self, + request: Optional[Union[ListModelsRequest, dict]] = None, + *, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ): + """ + TODO: Add docstring. + """ + kwargs = _drop_self_key(locals()) + return self.model_service_client.list_models(**kwargs) + + +# =============================================== +# Sample TODO: Relocate this to a samples file +# =============================================== + +# Instantiate BQClient class +bqclient = BigQueryClient() + +# Instantiate Request class +get_dataset_request = GetDatasetRequest( + project_id=PROJECT_ID, + dataset_id="experimental", +) + +# Generate response +dataset = bqclient.get_dataset(request=get_dataset_request) + +# Display response +print(f"GET DATASET:\n\t{dataset.id=}\n") diff --git a/tests/unit/gapic/bigquery_v2/test_centralized_services.py b/tests/unit/gapic/bigquery_v2/test_centralized_services.py new file mode 100644 index 000000000..e3883da5f --- /dev/null +++ b/tests/unit/gapic/bigquery_v2/test_centralized_services.py @@ -0,0 +1,259 @@ +import pytest +from typing import ( + Optional, + Sequence, + Tuple, + Union, +) +from unittest import mock + +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.cloud.bigquery_v2 import BigQueryClient +from google.cloud.bigquery_v2.types import Dataset, Job, Model +from google.cloud.bigquery_v2 import DatasetServiceClient +from google.cloud.bigquery_v2 import JobServiceClient +from google.cloud.bigquery_v2 import ModelServiceClient + +from google.cloud.bigquery_v2 import GetDatasetRequest + +from google.cloud.bigquery_v2 import ListJobsRequest + +from google.cloud.bigquery_v2 import DeleteModelRequest +from google.cloud.bigquery_v2 import GetModelRequest +from google.cloud.bigquery_v2 import PatchModelRequest +from google.cloud.bigquery_v2 import ListModelsRequest + + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + + +# --- CONSTANTS --- +PROJECT_ID = "test-project" +DATASET_ID = "test_dataset" +JOB_ID = "test_job" +MODEL_ID = "test_model" +DEFAULT_ETAG = "test_etag" + +DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT +DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT +DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () + +# --- HELPERS --- + + +def assert_client_called_once_with( + mock_method: mock.Mock, + request: Union[ + GetDatasetRequest, + GetModelRequest, + DeleteModelRequest, + PatchModelRequest, + ListJobsRequest, + ListModelsRequest, + ], # TODO this needs to be simplified. + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, +): + """Helper to assert a client method was called with default args.""" + mock_method.assert_called_once_with( + request=request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + +# --- FIXTURES --- + + +@pytest.fixture +def mock_dataset_service_client(): + """Mocks the DatasetServiceClient.""" + with mock.patch( + "google.cloud.bigquery_v2.services.centralized_services.client.DatasetServiceClient", + autospec=True, + ) as mock_client: + yield mock_client + + +@pytest.fixture +def mock_job_service_client(): + """Mocks the JobServiceClient.""" + with mock.patch( + "google.cloud.bigquery_v2.services.centralized_services.client.JobServiceClient", + autospec=True, + ) as mock_client: + yield mock_client + + +@pytest.fixture +def mock_model_service_client(): + """Mocks the ModelServiceClient.""" + with mock.patch( + "google.cloud.bigquery_v2.services.centralized_services.client.ModelServiceClient", + autospec=True, + ) as mock_client: + yield mock_client + + +# TODO: figure out a solution for this... is there an easier way to feed in clients? +# TODO: is there an easier way to make mock_x_service_clients? +@pytest.fixture +def bq_client( + mock_dataset_service_client, mock_job_service_client, mock_model_service_client +): + """Provides a BigQueryClient with mocked underlying services.""" + client = BigQueryClient() + client.dataset_service_client = mock_dataset_service_client + client.job_service_client = mock_job_service_client + client.model_service_client = mock_model_service_client + ... + return client + + +# --- TEST CLASSES --- + + +class TestCentralizedClientDatasetService: + def test_get_dataset(self, bq_client, mock_dataset_service_client): + # Arrange + expected_dataset = Dataset( + kind="bigquery#dataset", id=f"{PROJECT_ID}:{DATASET_ID}" + ) + mock_dataset_service_client.get_dataset.return_value = expected_dataset + get_dataset_request = GetDatasetRequest( + project_id=PROJECT_ID, dataset_id=DATASET_ID + ) + + # Act + dataset = bq_client.get_dataset(request=get_dataset_request) + + # Assert + assert dataset == expected_dataset + assert_client_called_once_with( + mock_dataset_service_client.get_dataset, get_dataset_request + ) + + +class TestCentralizedClientJobService: + def test_list_jobs(self, bq_client, mock_job_service_client): + # Arrange + expected_jobs = [Job(kind="bigquery#job", id=f"{PROJECT_ID}:{JOB_ID}")] + mock_job_service_client.list_jobs.return_value = expected_jobs + list_jobs_request = ListJobsRequest(project_id=PROJECT_ID) + + # Act + jobs = bq_client.list_jobs(request=list_jobs_request) + + # Assert + assert jobs == expected_jobs + assert_client_called_once_with( + mock_job_service_client.list_jobs, list_jobs_request + ) + + +class TestCentralizedClientModelService: + def test_get_model(self, bq_client, mock_model_service_client): + # Arrange + expected_model = Model( + etag=DEFAULT_ETAG, + model_reference={ + "project_id": PROJECT_ID, + "dataset_id": DATASET_ID, + "model_id": MODEL_ID, + }, + ) + mock_model_service_client.get_model.return_value = expected_model + get_model_request = GetModelRequest( + project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID + ) + + # Act + model = bq_client.get_model(request=get_model_request) + + # Assert + assert model == expected_model + assert_client_called_once_with( + mock_model_service_client.get_model, get_model_request + ) + + def test_delete_model(self, bq_client, mock_model_service_client): + # Arrange + # The underlying service call returns nothing on success. + mock_model_service_client.delete_model.return_value = None + delete_model_request = DeleteModelRequest( + project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID + ) + + # Act + # The wrapper method should also return nothing. + result = bq_client.delete_model(request=delete_model_request) + + # Assert + # 1. Assert the return value is None. This fails if the method doesn't exist. + assert result is None + # 2. Assert the underlying service was called correctly. + assert_client_called_once_with( + mock_model_service_client.delete_model, + delete_model_request, + ) + + def test_patch_model(self, bq_client, mock_model_service_client): + # Arrange + expected_model = Model( + etag="new_etag", + model_reference={ + "project_id": PROJECT_ID, + "dataset_id": DATASET_ID, + "model_id": MODEL_ID, + }, + description="A newly patched description.", + ) + mock_model_service_client.patch_model.return_value = expected_model + + model_patch = Model(description="A newly patched description.") + patch_model_request = PatchModelRequest( + project_id=PROJECT_ID, + dataset_id=DATASET_ID, + model_id=MODEL_ID, + model=model_patch, + ) + + # Act + patched_model = bq_client.patch_model(request=patch_model_request) + + # Assert + assert patched_model == expected_model + assert_client_called_once_with( + mock_model_service_client.patch_model, patch_model_request + ) + + def test_list_models(self, bq_client, mock_model_service_client): + # Arrange + expected_models = [ + Model( + etag=DEFAULT_ETAG, + model_reference={ + "project_id": PROJECT_ID, + "dataset_id": DATASET_ID, + "model_id": MODEL_ID, + }, + ) + ] + mock_model_service_client.list_models.return_value = expected_models + list_models_request = ListModelsRequest( + project_id=PROJECT_ID, dataset_id=DATASET_ID + ) + # Act + models = bq_client.list_models(request=list_models_request) + + # Assert + assert models == expected_models + assert_client_called_once_with( + mock_model_service_client.list_models, list_models_request + ) From 0e9abc0629cfb3c72f61038ea0568b84aac086c9 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 13 Aug 2025 09:45:36 -0400 Subject: [PATCH 08/21] chore: adds ci/cd related infra content (#2258) * chore: adds ci/cd related infra content * updates branch for github CI/CD check to autogen * update run commands * updates formatting in pagers.py files to pass blacken * removes unused check * adds unittest.yml * adds removes matrix.options * adds kokoro linting-typing.cfg * mods kokoro linting-typing.cfg * mods kokoro linting-typing.cfg again * adds kokoro .sh files * comments out a small snippet for debugging * Update google/cloud/bigquery_v2/services/job_service/pagers.py * Update .github/workflows/lint.yml * removes sample code * removes two currently unused ci shell scripts * removes .kokoro files for now --- .github/.OwlBot.lock.yaml | 17 ++++ .github/.OwlBot.yaml | 22 +++++ .github/CODEOWNERS | 11 +++ .github/CONTRIBUTING.md | 28 ++++++ .github/PULL_REQUEST_TEMPLATE.md | 7 ++ .github/release-please.yml | 14 +++ .github/release-trigger.yml | 2 + .github/workflows/lint.yml | 40 +++++++++ .github/workflows/unittest.yml | 87 +++++++++++++++++++ .../services/centralized_services/client.py | 20 ----- .../services/dataset_service/pagers.py | 1 - .../services/model_service/pagers.py | 1 - .../services/routine_service/pagers.py | 1 - .../row_access_policy_service/pagers.py | 1 - .../services/table_service/pagers.py | 1 - 15 files changed, 228 insertions(+), 25 deletions(-) create mode 100644 .github/.OwlBot.lock.yaml create mode 100644 .github/.OwlBot.yaml create mode 100644 .github/CODEOWNERS create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/release-please.yml create mode 100644 .github/release-trigger.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/unittest.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml new file mode 100644 index 000000000..cea9eb68f --- /dev/null +++ b/.github/.OwlBot.lock.yaml @@ -0,0 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:3b3a31be60853477bc39ed8d9bac162cac3ba083724cecaad54eb81d4e4dae9c +# created: 2025-04-16T22:40:03.123475241Z diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 000000000..8b142686c --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,22 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + +deep-remove-regex: + - /owl-bot-staging + +begin-after-commit-hash: f2de93abafa306b2ebadf1d10d947db8bcf2bf15 + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..6763f258c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,11 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + +# The @googleapis/api-bigquery is the default owner for changes in this repo +* @googleapis/api-bigquery @googleapis/yoshi-python + +# The python-samples-reviewers team is the default owner for samples changes +/samples/ @googleapis/api-bigquery @googleapis/python-samples-owners @googleapis/yoshi-python diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000..939e5341e --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..65ceeeb5e --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,7 @@ +Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: +- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea +- [ ] Ensure the tests and linter pass +- [ ] Code coverage does not decrease (if any source code was changed) +- [ ] Appropriate docs were updated (if necessary) + +Fixes # 🦕 diff --git a/.github/release-please.yml b/.github/release-please.yml new file mode 100644 index 000000000..5161ab347 --- /dev/null +++ b/.github/release-please.yml @@ -0,0 +1,14 @@ +releaseType: python +handleGHRelease: true +# NOTE: this section is generated by synthtool.languages.python +# See https://github.com/googleapis/synthtool/blob/master/synthtool/languages/python.py +branches: +- branch: v2 + handleGHRelease: true + releaseType: python +- branch: v1 + handleGHRelease: true + releaseType: python +- branch: v0 + handleGHRelease: true + releaseType: python diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 000000000..b975c190d --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1,2 @@ +enabled: true +multiScmName: python-bigquery diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..d06fd2a7c --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,40 @@ +on: + pull_request: + branches: + - autogen + # Trigger workflow on GitHub merge queue events + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group + merge_group: + types: [checks_requested] +name: lint + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run lint + env: + BUILD_TYPE: presubmit + TEST_TYPE: lint + # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session. + PY_VERSION: "unused" + run: | + nox -s blacken lint diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 000000000..4f118eca0 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,87 @@ +on: + pull_request: + branches: + - autogen + # Trigger workflow on GitHub merge queue events + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group + merge_group: + types: [checks_requested] +name: unittest + +permissions: + contents: read + +jobs: + unit: + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2303): use `ubuntu-latest` once this bug is fixed. + # Use ubuntu-22.04 until Python 3.7 is removed from the test matrix + # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories + runs-on: ubuntu-22.04 + strategy: + matrix: + python: ['3.9', '3.10', "3.11", "3.12", "3.13"] + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run unit tests + env: + COVERAGE_FILE: .coverage-${{ matrix.python }} + BUILD_TYPE: presubmit + TEST_TYPE: unit + PY_VERSION: ${{ matrix.python }} + run: | + nox -s unit-${{ matrix.python }} + - name: Upload coverage results + uses: actions/upload-artifact@v4 + with: + name: coverage-artifact-${{ '{{' }} matrix.python {{ '}}' }} + path: .coverage-${{ matrix.python }} + + cover: + runs-on: ubuntu-latest + needs: + - unit + steps: + - name: Checkout + uses: actions/checkout@v4 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Set number of files changes in packages directory + id: packages + run: echo "::set-output name=num_files_changed::$(git diff HEAD~1 -- packages | wc -l)" + - name: Install coverage + if: steps.packages.num_files_changed > 0 + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + - name: Download coverage results + if: ${{ steps.date.packages.num_files_changed > 0 }} + uses: actions/download-artifact@v4 + with: + path: .coverage-results/ + - name: Report coverage results + if: ${{ steps.date.packages.num_files_changed > 0 }} + run: | + find .coverage-results -type f -name '*.zip' -exec unzip {} \; + coverage combine .coverage-results/**/.coverage* + coverage report --show-missing --fail-under=100 \ No newline at end of file diff --git a/google/cloud/bigquery_v2/services/centralized_services/client.py b/google/cloud/bigquery_v2/services/centralized_services/client.py index aa581d2ab..63c403dcb 100644 --- a/google/cloud/bigquery_v2/services/centralized_services/client.py +++ b/google/cloud/bigquery_v2/services/centralized_services/client.py @@ -158,23 +158,3 @@ def list_models( """ kwargs = _drop_self_key(locals()) return self.model_service_client.list_models(**kwargs) - - -# =============================================== -# Sample TODO: Relocate this to a samples file -# =============================================== - -# Instantiate BQClient class -bqclient = BigQueryClient() - -# Instantiate Request class -get_dataset_request = GetDatasetRequest( - project_id=PROJECT_ID, - dataset_id="experimental", -) - -# Generate response -dataset = bqclient.get_dataset(request=get_dataset_request) - -# Display response -print(f"GET DATASET:\n\t{dataset.id=}\n") diff --git a/google/cloud/bigquery_v2/services/dataset_service/pagers.py b/google/cloud/bigquery_v2/services/dataset_service/pagers.py index c94c0b76a..de0afd87f 100644 --- a/google/cloud/bigquery_v2/services/dataset_service/pagers.py +++ b/google/cloud/bigquery_v2/services/dataset_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/model_service/pagers.py b/google/cloud/bigquery_v2/services/model_service/pagers.py index 567324a11..2a3fc6233 100644 --- a/google/cloud/bigquery_v2/services/model_service/pagers.py +++ b/google/cloud/bigquery_v2/services/model_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/routine_service/pagers.py b/google/cloud/bigquery_v2/services/routine_service/pagers.py index 7a67a9cd5..807f01503 100644 --- a/google/cloud/bigquery_v2/services/routine_service/pagers.py +++ b/google/cloud/bigquery_v2/services/routine_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py b/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py index 13a621b23..0b7d99f81 100644 --- a/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py +++ b/google/cloud/bigquery_v2/services/row_access_policy_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/google/cloud/bigquery_v2/services/table_service/pagers.py b/google/cloud/bigquery_v2/services/table_service/pagers.py index 1555d74c0..108e79eff 100644 --- a/google/cloud/bigquery_v2/services/table_service/pagers.py +++ b/google/cloud/bigquery_v2/services/table_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. From c688cff5eeaf3c4968ba9b8b4099794406611678 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 18 Aug 2025 14:30:58 +0000 Subject: [PATCH 09/21] removes example code from jinja template --- scripts/microgenerator/bigqueryclient.py.j2 | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/scripts/microgenerator/bigqueryclient.py.j2 b/scripts/microgenerator/bigqueryclient.py.j2 index 55379062d..ca1be2ca9 100644 --- a/scripts/microgenerator/bigqueryclient.py.j2 +++ b/scripts/microgenerator/bigqueryclient.py.j2 @@ -24,18 +24,3 @@ class BigQueryClient: return client.{{ method.name }}(request=request) {% endfor %} - -# ======== An example of the code generated ====== - - def get_dataset(self, request, retry, timeout, metadata): - """TODO: extract docstring for use here. - A generated method to call the BigQuery API.""" - - if "DatasetServiceClient" not in self._clients: - from google.cloud.bigquery_v2 import DatasetServiceClient - self._clients["DatasetServiceClient"] = DatasetServiceClient() - - client = self._clients["DatasetServiceClient"] - from google.cloud.bigquery_v2 import types - request = types.GetDatasetRequest(request=request, retry=retry, timeout=timeout, metadata=metadata) - return client.get_dataset(request=request) From 05aae1bd6563ffcecb9d5cf346657d2bd21e4b37 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:19:45 +0000 Subject: [PATCH 10/21] adds endmatter to template for simplicity in assessing generated code --- google/cloud/bigquery_v2/services/job_service/pagers.py | 1 - scripts/microgenerator/bigqueryclient.py.j2 | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigquery_v2/services/job_service/pagers.py b/google/cloud/bigquery_v2/services/job_service/pagers.py index 83f723917..b976c1d75 100644 --- a/google/cloud/bigquery_v2/services/job_service/pagers.py +++ b/google/cloud/bigquery_v2/services/job_service/pagers.py @@ -67,7 +67,6 @@ def __init__( retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), - ): """Instantiate the pager. diff --git a/scripts/microgenerator/bigqueryclient.py.j2 b/scripts/microgenerator/bigqueryclient.py.j2 index ca1be2ca9..840977a9a 100644 --- a/scripts/microgenerator/bigqueryclient.py.j2 +++ b/scripts/microgenerator/bigqueryclient.py.j2 @@ -24,3 +24,8 @@ class BigQueryClient: return client.{{ method.name }}(request=request) {% endfor %} + +# ======== 🦕 HERE THERE WERE DINOSAURS 🦖 ========= +# The above content is subject to significant change. Not for review yet. +# Included as a proof of concept for context or testing ONLY. +# ================================================ \ No newline at end of file From 49e398cf708d2f3a68b1256caaa9a1256a6b90e4 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:22:02 +0000 Subject: [PATCH 11/21] removes generated c_services content --- .../services/centralized_services/__init__.py | 18 -- .../services/centralized_services/client.py | 160 ------------------ 2 files changed, 178 deletions(-) delete mode 100644 google/cloud/bigquery_v2/services/centralized_services/__init__.py delete mode 100644 google/cloud/bigquery_v2/services/centralized_services/client.py diff --git a/google/cloud/bigquery_v2/services/centralized_services/__init__.py b/google/cloud/bigquery_v2/services/centralized_services/__init__.py deleted file mode 100644 index e82098176..000000000 --- a/google/cloud/bigquery_v2/services/centralized_services/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .client import BigQueryClient - -__all__ = ("BigQueryClient",) diff --git a/google/cloud/bigquery_v2/services/centralized_services/client.py b/google/cloud/bigquery_v2/services/centralized_services/client.py deleted file mode 100644 index 63c403dcb..000000000 --- a/google/cloud/bigquery_v2/services/centralized_services/client.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -from typing import ( - Optional, - Sequence, - Tuple, - Union, -) - -# Import service clients -from google.cloud.bigquery_v2.services.dataset_service import DatasetServiceClient -from google.cloud.bigquery_v2.services.job_service import JobServiceClient -from google.cloud.bigquery_v2.services.model_service import ModelServiceClient - -# Import Request classes -from google.cloud.bigquery_v2.types import ( - # DatasetService Request classes - GetDatasetRequest, - # JobService Request classes - ListJobsRequest, - # ModelService Request classes - DeleteModelRequest, - GetModelRequest, - PatchModelRequest, - ListModelsRequest, -) - -from google.api_core import gapic_v1 -from google.api_core import retry as retries - -# Create a type alias -try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] -except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object, None] # type: ignore - -# TODO: revise this universally. -PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT") - -DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT -DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT -DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () - - -def _drop_self_key(kwargs): - "Drops 'self' key from a given kwargs dict." - - if not isinstance(kwargs, dict): - raise TypeError("kwargs must be a dict.") - kwargs.pop("self", None) # Essentially a no-op if 'self' key does not exist - return kwargs - - -# Create Centralized Client -class BigQueryClient: - def __init__(self): - # Dataset service related init attributes - self.dataset_service_client = DatasetServiceClient() - self.job_service_client = JobServiceClient() - self.model_service_client = ModelServiceClient() - - def get_dataset( - self, - request: Optional[Union[GetDatasetRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Add docstring. - """ - kwargs = _drop_self_key(locals()) - return self.dataset_service_client.get_dataset(**kwargs) - - def list_jobs( - self, - request: Optional[Union[ListJobsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Add docstring. - """ - kwargs = _drop_self_key(locals()) - return self.job_service_client.list_jobs(**kwargs) - - def get_model( - self, - request: Optional[Union[GetModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Add docstring. - """ - kwargs = _drop_self_key(locals()) - return self.model_service_client.get_model(**kwargs) - - def delete_model( - self, - request: Optional[Union[DeleteModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Add docstring. - """ - kwargs = _drop_self_key(locals()) - # The underlying GAPIC client returns None on success. - return self.model_service_client.delete_model(**kwargs) - - def patch_model( - self, - request: Optional[Union[PatchModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Add docstring. - """ - kwargs = _drop_self_key(locals()) - return self.model_service_client.patch_model(**kwargs) - - def list_models( - self, - request: Optional[Union[ListModelsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Add docstring. - """ - kwargs = _drop_self_key(locals()) - return self.model_service_client.list_models(**kwargs) From d91e6f7f1abc98ad33598e48bd3a4206438325f9 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:25:05 +0000 Subject: [PATCH 12/21] removes unused c_services tests --- .../bigquery_v2/test_centralized_services.py | 259 ------------------ 1 file changed, 259 deletions(-) delete mode 100644 tests/unit/gapic/bigquery_v2/test_centralized_services.py diff --git a/tests/unit/gapic/bigquery_v2/test_centralized_services.py b/tests/unit/gapic/bigquery_v2/test_centralized_services.py deleted file mode 100644 index e3883da5f..000000000 --- a/tests/unit/gapic/bigquery_v2/test_centralized_services.py +++ /dev/null @@ -1,259 +0,0 @@ -import pytest -from typing import ( - Optional, - Sequence, - Tuple, - Union, -) -from unittest import mock - -from google.api_core import gapic_v1 -from google.api_core import retry as retries -from google.cloud.bigquery_v2 import BigQueryClient -from google.cloud.bigquery_v2.types import Dataset, Job, Model -from google.cloud.bigquery_v2 import DatasetServiceClient -from google.cloud.bigquery_v2 import JobServiceClient -from google.cloud.bigquery_v2 import ModelServiceClient - -from google.cloud.bigquery_v2 import GetDatasetRequest - -from google.cloud.bigquery_v2 import ListJobsRequest - -from google.cloud.bigquery_v2 import DeleteModelRequest -from google.cloud.bigquery_v2 import GetModelRequest -from google.cloud.bigquery_v2 import PatchModelRequest -from google.cloud.bigquery_v2 import ListModelsRequest - - -try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] -except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object, None] # type: ignore - - -# --- CONSTANTS --- -PROJECT_ID = "test-project" -DATASET_ID = "test_dataset" -JOB_ID = "test_job" -MODEL_ID = "test_model" -DEFAULT_ETAG = "test_etag" - -DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT -DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT -DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () - -# --- HELPERS --- - - -def assert_client_called_once_with( - mock_method: mock.Mock, - request: Union[ - GetDatasetRequest, - GetModelRequest, - DeleteModelRequest, - PatchModelRequest, - ListJobsRequest, - ListModelsRequest, - ], # TODO this needs to be simplified. - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, -): - """Helper to assert a client method was called with default args.""" - mock_method.assert_called_once_with( - request=request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - -# --- FIXTURES --- - - -@pytest.fixture -def mock_dataset_service_client(): - """Mocks the DatasetServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.centralized_services.client.DatasetServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -@pytest.fixture -def mock_job_service_client(): - """Mocks the JobServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.centralized_services.client.JobServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -@pytest.fixture -def mock_model_service_client(): - """Mocks the ModelServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.centralized_services.client.ModelServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -# TODO: figure out a solution for this... is there an easier way to feed in clients? -# TODO: is there an easier way to make mock_x_service_clients? -@pytest.fixture -def bq_client( - mock_dataset_service_client, mock_job_service_client, mock_model_service_client -): - """Provides a BigQueryClient with mocked underlying services.""" - client = BigQueryClient() - client.dataset_service_client = mock_dataset_service_client - client.job_service_client = mock_job_service_client - client.model_service_client = mock_model_service_client - ... - return client - - -# --- TEST CLASSES --- - - -class TestCentralizedClientDatasetService: - def test_get_dataset(self, bq_client, mock_dataset_service_client): - # Arrange - expected_dataset = Dataset( - kind="bigquery#dataset", id=f"{PROJECT_ID}:{DATASET_ID}" - ) - mock_dataset_service_client.get_dataset.return_value = expected_dataset - get_dataset_request = GetDatasetRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID - ) - - # Act - dataset = bq_client.get_dataset(request=get_dataset_request) - - # Assert - assert dataset == expected_dataset - assert_client_called_once_with( - mock_dataset_service_client.get_dataset, get_dataset_request - ) - - -class TestCentralizedClientJobService: - def test_list_jobs(self, bq_client, mock_job_service_client): - # Arrange - expected_jobs = [Job(kind="bigquery#job", id=f"{PROJECT_ID}:{JOB_ID}")] - mock_job_service_client.list_jobs.return_value = expected_jobs - list_jobs_request = ListJobsRequest(project_id=PROJECT_ID) - - # Act - jobs = bq_client.list_jobs(request=list_jobs_request) - - # Assert - assert jobs == expected_jobs - assert_client_called_once_with( - mock_job_service_client.list_jobs, list_jobs_request - ) - - -class TestCentralizedClientModelService: - def test_get_model(self, bq_client, mock_model_service_client): - # Arrange - expected_model = Model( - etag=DEFAULT_ETAG, - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - ) - mock_model_service_client.get_model.return_value = expected_model - get_model_request = GetModelRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID - ) - - # Act - model = bq_client.get_model(request=get_model_request) - - # Assert - assert model == expected_model - assert_client_called_once_with( - mock_model_service_client.get_model, get_model_request - ) - - def test_delete_model(self, bq_client, mock_model_service_client): - # Arrange - # The underlying service call returns nothing on success. - mock_model_service_client.delete_model.return_value = None - delete_model_request = DeleteModelRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID - ) - - # Act - # The wrapper method should also return nothing. - result = bq_client.delete_model(request=delete_model_request) - - # Assert - # 1. Assert the return value is None. This fails if the method doesn't exist. - assert result is None - # 2. Assert the underlying service was called correctly. - assert_client_called_once_with( - mock_model_service_client.delete_model, - delete_model_request, - ) - - def test_patch_model(self, bq_client, mock_model_service_client): - # Arrange - expected_model = Model( - etag="new_etag", - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - description="A newly patched description.", - ) - mock_model_service_client.patch_model.return_value = expected_model - - model_patch = Model(description="A newly patched description.") - patch_model_request = PatchModelRequest( - project_id=PROJECT_ID, - dataset_id=DATASET_ID, - model_id=MODEL_ID, - model=model_patch, - ) - - # Act - patched_model = bq_client.patch_model(request=patch_model_request) - - # Assert - assert patched_model == expected_model - assert_client_called_once_with( - mock_model_service_client.patch_model, patch_model_request - ) - - def test_list_models(self, bq_client, mock_model_service_client): - # Arrange - expected_models = [ - Model( - etag=DEFAULT_ETAG, - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - ) - ] - mock_model_service_client.list_models.return_value = expected_models - list_models_request = ListModelsRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID - ) - # Act - models = bq_client.list_models(request=list_models_request) - - # Assert - assert models == expected_models - assert_client_called_once_with( - mock_model_service_client.list_models, list_models_request - ) From 332a7335ab8acf34ec0b75870f8304159e07c0cc Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:29:06 +0000 Subject: [PATCH 13/21] removes unneeded init lines --- google/cloud/bigquery_v2/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/google/cloud/bigquery_v2/__init__.py b/google/cloud/bigquery_v2/__init__.py index 442609adc..83c82e729 100644 --- a/google/cloud/bigquery_v2/__init__.py +++ b/google/cloud/bigquery_v2/__init__.py @@ -24,7 +24,6 @@ from .services.routine_service import RoutineServiceClient from .services.row_access_policy_service import RowAccessPolicyServiceClient from .services.table_service import TableServiceClient -from .services.centralized_services import BigQueryClient from .types.biglake_config import BigLakeConfiguration from .types.clustering import Clustering @@ -215,7 +214,6 @@ "BiEngineReason", "BiEngineStatistics", "BigLakeConfiguration", - "BigQueryClient", "BigtableColumn", "BigtableColumnFamily", "BigtableOptions", From 4045bdeae7ff36df272fa9de712e350b7f667422 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:53:48 +0000 Subject: [PATCH 14/21] adds utils.py file --- scripts/microgenerator/utils.py | 120 ++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 scripts/microgenerator/utils.py diff --git a/scripts/microgenerator/utils.py b/scripts/microgenerator/utils.py new file mode 100644 index 000000000..324c20662 --- /dev/null +++ b/scripts/microgenerator/utils.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utility functions for the microgenerator.""" + +import os +import sys +import yaml +import jinja2 +from typing import Dict, Any, Iterator, Callable + + +def _load_resource( + loader_func: Callable, + path: str, + not_found_exc: type, + parse_exc: type, + resource_type_name: str, +) -> Any: + """ + Generic resource loader with common error handling. + + Args: + loader_func: A callable that performs the loading and returns the resource. + It should raise appropriate exceptions on failure. + path: The path/name of the resource for use in error messages. + not_found_exc: The exception type to catch for a missing resource. + parse_exc: The exception type to catch for a malformed resource. + resource_type_name: A human-readable name for the resource type. + """ + try: + return loader_func() + except not_found_exc: + print(f"Error: {resource_type_name} '{path}' not found.", file=sys.stderr) + sys.exit(1) + except parse_exc as e: + print( + f"Error: Could not load {resource_type_name.lower()} from '{path}': {e}", + file=sys.stderr, + ) + sys.exit(1) + + +def load_template(template_path: str) -> jinja2.Template: + """ + Loads a Jinja2 template from a given file path. + """ + template_dir = os.path.dirname(template_path) + template_name = os.path.basename(template_path) + + def _loader() -> jinja2.Template: + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir or "."), + trim_blocks=True, + lstrip_blocks=True, + ) + return env.get_template(template_name) + + return _load_resource( + loader_func=_loader, + path=template_path, + not_found_exc=jinja2.exceptions.TemplateNotFound, + parse_exc=jinja2.exceptions.TemplateError, + resource_type_name="Template file", + ) + + +def load_config(config_path: str) -> Dict[str, Any]: + """Loads the generator's configuration from a YAML file.""" + + def _loader() -> Dict[str, Any]: + with open(config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + return _load_resource( + loader_func=_loader, + path=config_path, + not_found_exc=FileNotFoundError, + parse_exc=yaml.YAMLError, + resource_type_name="Configuration file", + ) + + +def walk_codebase(path: str) -> Iterator[str]: + """Yields all .py file paths in a directory.""" + for root, _, files in os.walk(path): + for file in files: + if file.endswith(".py"): + yield os.path.join(root, file) + + +def write_code_to_file(output_path: str, content: str): + """Ensures the output directory exists and writes content to the file.""" + output_dir = os.path.dirname(output_path) + + # An empty output_dir means the file is in the current directory. + if output_dir: + print(f" Ensuring output directory exists: {os.path.abspath(output_dir)}") + os.makedirs(output_dir, exist_ok=True) + if not os.path.isdir(output_dir): + print(f" Error: Output directory was not created.", file=sys.stderr) + sys.exit(1) + + print(f" Writing generated code to: {os.path.abspath(output_path)}") + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) + print(f"Successfully generated {output_path}") \ No newline at end of file From eed1713b9a48fca340e9d360e453949029cec591 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:55:06 +0000 Subject: [PATCH 15/21] adds generate.py file --- scripts/microgenerator/generate.py | 362 +++++++++++++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 scripts/microgenerator/generate.py diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py new file mode 100644 index 000000000..6273f71a1 --- /dev/null +++ b/scripts/microgenerator/generate.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +A dual-purpose module for Python code analysis and BigQuery client generation. + +When run as a script, it generates the BigQueryClient source code. +When imported, it provides utility functions for parsing and exploring +any Python codebase using the `ast` module. +""" + +import ast +import os +import argparse +import glob +from collections import defaultdict +from typing import List, Dict, Any, Iterator + +import utils + +# ============================================================================= +# Section 1: Generic AST Analysis Utilities +# ============================================================================= + +class CodeAnalyzer(ast.NodeVisitor): + """ + A node visitor to traverse an AST and extract structured information + about classes, methods, and their arguments. + """ + + def __init__(self): + self.structure: List[Dict[str, Any]] = [] + self._current_class_info: Dict[str, Any] | None = None + self._is_in_method: bool = False + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + """Visits a class definition node.""" + class_info = { + "class_name": node.name, + "methods": [], + "attributes": [], + } + self.structure.append(class_info) + self._current_class_info = class_info + self.generic_visit(node) + self._current_class_info = None + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + """Visits a function/method definition node.""" + if self._current_class_info: # This is a method + method_info = { + "method_name": node.name, + "args": [arg.arg for arg in node.args.args], + } + self._current_class_info["methods"].append(method_info) + + # Visit nodes inside the method to find instance attributes + self._is_in_method = True + self.generic_visit(node) + self._is_in_method = False + + def _add_attribute(self, attr_name: str): + """Adds a unique attribute to the current class context.""" + if self._current_class_info: + if attr_name not in self._current_class_info["attributes"]: + self._current_class_info["attributes"].append(attr_name) + + def visit_Assign(self, node: ast.Assign) -> None: + """Handles attribute assignments: `x = ...` and `self.x = ...`.""" + if self._current_class_info: + for target in node.targets: + # Instance attribute: self.x = ... + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + ): + self._add_attribute(target.attr) + # Class attribute: x = ... (only if not inside a method) + elif isinstance(target, ast.Name) and not self._is_in_method: + self._add_attribute(target.id) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + """Handles annotated assignments: `x: int = ...` and `self.x: int = ...`.""" + if self._current_class_info: + target = node.target + # Instance attribute: self.x: int = ... + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + ): + self._add_attribute(target.attr) + # Class attribute: x: int = ... (only if not inside a method) + elif isinstance(target, ast.Name) and not self._is_in_method: + self._add_attribute(target.id) + self.generic_visit(node) + + +def parse_code(code: str) -> List[Dict[str, Any]]: + """ + Parses a string of Python code into a structured list of classes. + + Args: + code: A string containing Python code. + + Returns: + A list of dictionaries, where each dictionary represents a class. + """ + tree = ast.parse(code) + analyzer = CodeAnalyzer() + analyzer.visit(tree) + return analyzer.structure + + +def parse_file(file_path: str) -> List[Dict[str, Any]]: + """ + Parses a Python file into a structured list of classes. + + Args: + file_path: The absolute path to the Python file. + + Returns: + A list of dictionaries representing the classes in the file. + """ + with open(file_path, "r", encoding="utf-8") as source: + code = source.read() + return parse_code(code) + + +def list_classes(path: str) -> List[str]: + """Lists all classes in a given Python file or directory.""" + class_names = [] + if os.path.isfile(path) and path.endswith(".py"): + structure = parse_file(path) + for class_info in structure: + class_names.append(class_info["class_name"]) + elif os.path.isdir(path): + for file_path in utils.walk_codebase(path): + structure = parse_file(file_path) + for class_info in structure: + class_names.append( + f"{class_info['class_name']} (in {os.path.basename(file_path)})" + ) + return sorted(class_names) + + +def list_classes_and_methods(path: str) -> Dict[str, List[str]]: + """Lists all classes and their methods in a given Python file or directory.""" + results = defaultdict(list) + + def process_structure(structure, file_name=None): + for class_info in structure: + key = class_info["class_name"] + if file_name: + key = f"{key} (in {file_name})" + + results[key] = sorted([m["method_name"] for m in class_info["methods"]]) + + if os.path.isfile(path) and path.endswith(".py"): + process_structure(parse_file(path)) + elif os.path.isdir(path): + for file_path in utils.walk_codebase(path): + process_structure( + parse_file(file_path), file_name=os.path.basename(file_path) + ) + + return results + + +def list_classes_methods_and_attributes(path: str) -> Dict[str, Dict[str, List[str]]]: + """Lists classes, methods, and attributes in a file or directory.""" + results = defaultdict(lambda: defaultdict(list)) + + def process_structure(structure, file_name=None): + for class_info in structure: + key = class_info["class_name"] + if file_name: + key = f"{key} (in {file_name})" + + results[key]["attributes"] = sorted(class_info["attributes"]) + results[key]["methods"] = sorted( + [m["method_name"] for m in class_info["methods"]] + ) + + if os.path.isfile(path) and path.endswith(".py"): + process_structure(parse_file(path)) + elif os.path.isdir(path): + for file_path in utils.walk_codebase(path): + process_structure( + parse_file(file_path), file_name=os.path.basename(file_path) + ) + + return results + + +def list_classes_methods_attributes_and_arguments( + path: str, +) -> Dict[str, Dict[str, Any]]: + """Lists classes, methods, attributes, and arguments in a file or directory.""" + results = defaultdict(lambda: defaultdict(list)) + + def process_structure(structure, file_name=None): + for class_info in structure: + key = class_info["class_name"] + if file_name: + key = f"{key} (in {file_name})" + + results[key]["attributes"] = sorted(class_info["attributes"]) + method_details = {} + # Sort methods by name for consistent output + for method in sorted(class_info["methods"], key=lambda m: m["method_name"]): + method_details[method["method_name"]] = method["args"] + results[key]["methods"] = method_details + + if os.path.isfile(path) and path.endswith(".py"): + process_structure(parse_file(path)) + elif os.path.isdir(path): + for file_path in utils.walk_codebase(path): + process_structure( + parse_file(file_path), file_name=os.path.basename(file_path) + ) + + return results + + +# ============================================================================= +# Section 2: Generic Code Generation Logic +# ============================================================================= + +def analyze_source_files(config: Dict[str, Any]) -> Dict[str, Any]: + """ + Analyzes source files as per the configuration to extract class and method info. + + Args: + config: The generator's configuration dictionary. + + Returns: + A dictionary containing the data needed for template rendering. + """ + parsed_data = defaultdict(dict) + source_patterns = config.get("source_files", []) + filter_rules = config.get("filter", {}) + class_filters = filter_rules.get("classes", {}) + method_filters = filter_rules.get("methods", {}) + + source_files = [] + for pattern in source_patterns: + source_files.extend(glob.glob(pattern, recursive=True)) + + for file_path in source_files: + structure = parse_file(file_path) + + for class_info in structure: + class_name = class_info["class_name"] + # Apply class filters + if class_filters.get("include_suffixes"): + if not class_name.endswith(tuple(class_filters["include_suffixes"])): + continue + + parsed_data[class_name] # Ensure class is in dict + + for method in class_info["methods"]: + method_name = method["method_name"] + # Apply method filters + if method_filters.get("include_prefixes"): + if not any( + method_name.startswith(p) + for p in method_filters["include_prefixes"] + ): + continue + if method_filters.get("exclude_prefixes"): + if any( + method_name.startswith(p) + for p in method_filters["exclude_prefixes"] + ): + continue + parsed_data[class_name][method_name] = method["args"] + return parsed_data + + +def _format_args(method_args: List[str]) -> tuple[str, str]: + """Formats method arguments for use in creating a method definition and a method call.""" + args_for_def = ", ".join(method_args) + args_for_call = ", ".join([f"{arg}={arg}" for arg in method_args if arg != "self"]) + return args_for_def, args_for_call + + +def _format_class_name(method_name: str, suffix: str = "Request") -> str: + """Formats a class name from a method name.""" + return "".join(word.capitalize() for word in method_name.split("_")) + suffix + + +def generate_code(config: Dict[str, Any], data: Dict[str, Any]) -> None: + """ + Generates source code files using Jinja2 templates. + """ + templates_config = config.get("templates", []) + for item in templates_config: + template_path = item["template"] + output_path = item["output"] + + print(f"Processing template: {template_path}.") + + template = utils.load_template(template_path) + methods_context = [] + for class_name, methods in data.items(): + for method_name, method_args in methods.items(): + args_for_def, args_for_call = _format_args(method_args) + request_class_name = _format_class_name(method_name) + methods_context.append( + { + "name": method_name, + "class_name": class_name, + "args_for_def": args_for_def, + "args_for_call": args_for_call, + "request_class_name": request_class_name, + } + ) + + print(f"Found {len(methods_context)} methods to generate.") + + final_code = template.render( + service_name=config.get("service_name"), + methods=methods_context + ) + + utils.write_code_to_file(output_path, final_code) + + +# ============================================================================= +# Section 3: Main Execution +# ============================================================================= + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="A generic Python code generator for clients." + ) + parser.add_argument( + "config", help="Path to the YAML configuration file." + ) + args = parser.parse_args() + + config = utils.load_config(args.config) + data = analyze_source_files(config) + generate_code(config, data) + + # TODO: Ensure blacken gets called on the generated source files as a final step. From a8a76d18e42ff3033d3bf3573793ee8c447dbb83 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:56:09 +0000 Subject: [PATCH 16/21] removes bigqueryclient_generator.py file in deference to generate.py --- .../bigqueryclient_generator.py | 149 ------------------ 1 file changed, 149 deletions(-) delete mode 100644 scripts/microgenerator/bigqueryclient_generator.py diff --git a/scripts/microgenerator/bigqueryclient_generator.py b/scripts/microgenerator/bigqueryclient_generator.py deleted file mode 100644 index 0138948a7..000000000 --- a/scripts/microgenerator/bigqueryclient_generator.py +++ /dev/null @@ -1,149 +0,0 @@ -# TODO: Add a header if needed. - -import ast -import os -from collections import defaultdict - -from config_helper import ( - CLASSES_TO_INCLUDE, - # CLASSES_TO_EXCLUDE, # Not currently being used. - METHODS_TO_INCLUDE, - METHODS_TO_EXCLUDE, -) -from template_utils import load_template - -# Constants -BASE_DIR = "google/cloud/bigquery_v2/services" -FILES_TO_PARSE = [ - os.path.join(root, file) - for root, _, files in os.walk(BASE_DIR) - for file in files - if file.endswith(".py") -] - - -def create_tree(file_path): - with open(file_path, "r") as source: - tree = ast.parse(source.read()) - return tree - - -def _extract_classes(tree): - """Extracts class nodes from an AST.""" - classes = [] - - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef) and node.name.endswith( - *CLASSES_TO_INCLUDE - ): # TODO: currently this is variable includes only one class. Refactor if necessary - classes.append(node) - return classes - - -def _extract_methods(class_node): - """Extracts method nodes from a class node.""" - return (m for m in class_node.body if isinstance(m, ast.FunctionDef)) - - -def _process_method(method, class_name, parsed_data): - """Processes a single method and updates parsed_data.""" - method_name = method.name - if any(method_name.startswith(prefix) for prefix in METHODS_TO_INCLUDE) and not any( - method_name.startswith(prefix) for prefix in METHODS_TO_EXCLUDE - ): - parameters = [arg.arg for arg in method.args.args + method.args.kwonlyargs] - parsed_data[class_name][method_name] = parameters - - -def parse_files(file_paths): - """ - Parse a list of Python files and extract information about classes, - methods, and parameters. - - Args: - file_paths (list): List of file paths to parse. - - Returns: - Defaultdict with zero or more entries. - """ - - parsed_data = defaultdict(dict) - - for file_path in file_paths: - tree = create_tree(file_path) - - for class_ in _extract_classes(tree): - class_name = class_.name - parsed_data[class_name] - - for method in _extract_methods(class_): - _process_method(method, class_name, parsed_data) - - return parsed_data - - -def _format_args(method_args): - """Formats method arguments for use in creating a method definition - and a method call. - """ - args_for_def = ", ".join(method_args) - args_for_call = ", ".join([f"{arg}={arg}" for arg in method_args if arg != "self"]) - return args_for_def, args_for_call - - -def _format_class_name(method_name, suffix="Request"): - """Formats a class name from a method name. - - Example: - list_datasets -> ListDatasetsRequest - """ - return "".join(word.capitalize() for word in method_name.split("_")) + suffix - - -def generate_client_class_source(data): - """ - Generates the BigQueryClient source code using a Jinja2 template. - - Args: - data: A dictionary where keys are *ServiceClient class names and - values are dictionaries of methods for that client. - - Returns: - A string containing the complete, formatted Python source code - for the BigQueryClient class. - """ - - template = load_template("bigqueryclient.py.j2") - - # Prepare the context for the template. - # We transform the input data into a flat list of methods - methods_context = [] - for class_name, methods in data.items(): - for method_name, method_args in methods.items(): - args_for_def, args_for_call = _format_args(method_args) - request_class_name = _format_class_name(method_name) - methods_context.append( - { - "name": method_name, - "class_name": class_name, - "args_for_def": args_for_def, - "args_for_call": args_for_call, - "request_class_name": request_class_name, - } - ) - - # Render the template with the context. - generated_code = template.render(methods=methods_context) - - return generated_code - - -if __name__ == "__main__": - data = parse_files(FILES_TO_PARSE) - - final_code = generate_client_class_source(data) - - # TODO: write final code to file instead of print to screen. - print(final_code) - - # TODO: Ensure blacken gets called on the generated source files as a final step. From 0bfe9892506af76dc1bb4c7eda924e2efb030042 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:57:15 +0000 Subject: [PATCH 17/21] adds config removes bqclient_config --- .../microgenerator/bigqueryclient_config.yaml | 21 --------- scripts/microgenerator/config.yaml | 45 +++++++++++++++++++ 2 files changed, 45 insertions(+), 21 deletions(-) delete mode 100644 scripts/microgenerator/bigqueryclient_config.yaml create mode 100644 scripts/microgenerator/config.yaml diff --git a/scripts/microgenerator/bigqueryclient_config.yaml b/scripts/microgenerator/bigqueryclient_config.yaml deleted file mode 100644 index 4df44e88c..000000000 --- a/scripts/microgenerator/bigqueryclient_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# TODO: Add a header if needed. - -include_class_name_patterns: - - Client - -exclude_class_name_patterns: [] - -include_method_name_patterns: - - batch_delete_ - - cancel_ - - create_ - - delete_ - - get_ - - insert_ - - list_ - - patch_ - - undelete_ - - update_ - -exclude_method_name_patterns: - - get_mtls_endpoint_and_cert_source diff --git a/scripts/microgenerator/config.yaml b/scripts/microgenerator/config.yaml new file mode 100644 index 000000000..f93e39735 --- /dev/null +++ b/scripts/microgenerator/config.yaml @@ -0,0 +1,45 @@ +# config.yaml + +# The name of the service, used for variable names and comments. +service_name: "bigquery" + +# A list of paths to the source code files to be parsed. +# Globs are supported. +source_files: + - "autogen/google/cloud/bigquery_v2/services/dataset_service/client.py" + - "autogen/google/cloud/bigquery_v2/services/job_service/client.py" + - "autogen/google/cloud/bigquery_v2/services/model_service/client.py" + - "autogen/google/cloud/bigquery_v2/services/project_service/client.py" + - "autogen/google/cloud/bigquery_v2/services/routine_service/client.py" + - "autogen/google/cloud/bigquery_v2/services/row_access_policy_service/client.py" + - "autogen/google/cloud/bigquery_v2/services/table_service/client.py" + +# Filtering rules for classes and methods. +filter: + classes: + # Only include classes with these suffixes. + include_suffixes: + - "Client" + methods: + # Include methods with these prefixes. + include_prefixes: + - "batch_delete_" + - "cancel_" + - "create_" + - "delete_" + - "get_" + - "insert_" + - "list_" + - "patch_" + - "undelete_" + - "update_" + # Exclude methods with these prefixes. + exclude_prefixes: + - "get_mtls_endpoint_and_cert_source" + +# A list of templates to render and their corresponding output files. +templates: + - template: "autogen/scripts/microgenerator/bigqueryclient.py.j2" + output: "autogen/google/cloud/bigquery_v2/services/centralized_service/client.py" + # - template: "test_bigqueryclient.py.j2" + # output: "tests/unit/test_bigqueryclient.py" From fc10cd422d657c715844aa565dc27ab80d5b978f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 10:58:19 +0000 Subject: [PATCH 18/21] removes unused helper and template_utils files --- scripts/microgenerator/config_helper.py | 39 ------------------------ scripts/microgenerator/template_utils.py | 17 ----------- 2 files changed, 56 deletions(-) delete mode 100644 scripts/microgenerator/config_helper.py delete mode 100644 scripts/microgenerator/template_utils.py diff --git a/scripts/microgenerator/config_helper.py b/scripts/microgenerator/config_helper.py deleted file mode 100644 index ce5ab0806..000000000 --- a/scripts/microgenerator/config_helper.py +++ /dev/null @@ -1,39 +0,0 @@ -# TODO: Add a header if needed. - -import yaml -import os - - -def load_config_yaml(filepath): - """Loads configuration from a YAML file.""" - try: - with open(filepath, "r") as f: - config = yaml.safe_load(f) - return config - except FileNotFoundError: - print(f"Error: Configuration file '{filepath}' not found.") - return None - except yaml.YAMLError as e: - print(f"Error: Could not load YAML from '{filepath}': {e}") - return None - - -# Determine the absolute path to the config file relative to this file. -# This makes the path robust to where the script is run from. -_CONFIG_FILE_PATH = os.path.join( - os.path.dirname(__file__), "bigqueryclient_config.yaml" -) - -config_data = load_config_yaml(_CONFIG_FILE_PATH) - -if config_data: - CLASSES_TO_INCLUDE = config_data.get("include_class_name_patterns", []) - CLASSES_TO_EXCLUDE = config_data.get("exclude_class_name_patterns", []) - METHODS_TO_INCLUDE = config_data.get("include_method_name_patterns", []) - METHODS_TO_EXCLUDE = config_data.get("exclude_method_name_patterns", []) -else: - # Define default empty values if the config fails to load - CLASSES_TO_INCLUDE = [] - CLASSES_TO_EXCLUDE = [] - METHODS_TO_INCLUDE = [] - METHODS_TO_EXCLUDE = [] diff --git a/scripts/microgenerator/template_utils.py b/scripts/microgenerator/template_utils.py deleted file mode 100644 index 2cfd9cd87..000000000 --- a/scripts/microgenerator/template_utils.py +++ /dev/null @@ -1,17 +0,0 @@ -# TODO: Add a header if needed. - -import os -import jinja2 - - -def load_template(template_name): - """ - Loads a Jinja2 template from the same directory as the script. - """ - template_dir = os.path.dirname(os.path.abspath(__file__)) - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(template_dir), - trim_blocks=True, # prevents blank lines by removing '\n' after block tags (e.g. {% if condition %}\n) - lstrip_blocks=True, # prevents unwanted empty spaces before lines of text by removing non-explicit spaces, tabs, etc - ) - return env.get_template(template_name) From 240c496ad977908aa50419cf66a36684e73072ba Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 22 Aug 2025 11:09:46 +0000 Subject: [PATCH 19/21] adds minor mod (.) for testing purposes --- scripts/microgenerator/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 6273f71a1..4e641f37d 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -67,7 +67,7 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> None: } self._current_class_info["methods"].append(method_info) - # Visit nodes inside the method to find instance attributes + # Visit nodes inside the method to find instance attributes. self._is_in_method = True self.generic_visit(node) self._is_in_method = False From e33a574f86850ad76d7e64fb7b542c4c96d95cbb Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 5 Sep 2025 15:18:52 -0400 Subject: [PATCH 20/21] refactors the generator for clarity and to gen code per design --- scripts/microgenerator/name_utils.py | 45 ++++++++ scripts/microgenerator/templates/client.py.j2 | 103 ++++++++++++++++++ .../templates/partials/_client_helpers.j2 | 49 +++++++++ 3 files changed, 197 insertions(+) create mode 100644 scripts/microgenerator/name_utils.py create mode 100644 scripts/microgenerator/templates/client.py.j2 create mode 100644 scripts/microgenerator/templates/partials/_client_helpers.j2 diff --git a/scripts/microgenerator/name_utils.py b/scripts/microgenerator/name_utils.py new file mode 100644 index 000000000..560a18751 --- /dev/null +++ b/scripts/microgenerator/name_utils.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility module for handling name transformations.""" + +import re +from typing import Dict + +def to_snake_case(name: str) -> str: + """Converts a PascalCase name to snake_case.""" + return re.sub(r"(? Dict[str, str]: + """ + Generates various name formats for a service based on its client class name. + + Args: + class_name: The PascalCase name of the service client class + (e.g., 'DatasetServiceClient'). + + Returns: + A dictionary containing different name variations. + """ + snake_case_name = to_snake_case(class_name) + module_name = snake_case_name.replace("_client", "") + service_name = module_name.replace("_service", "") + + return { + "service_name": service_name, + "service_module_name": module_name, + "service_client_class": class_name, + "property_name": snake_case_name, # Direct use of snake_case_name + } diff --git a/scripts/microgenerator/templates/client.py.j2 b/scripts/microgenerator/templates/client.py.j2 new file mode 100644 index 000000000..98c2b4e99 --- /dev/null +++ b/scripts/microgenerator/templates/client.py.j2 @@ -0,0 +1,103 @@ +# TODO: Add a header if needed. + +# ======== 🦕 HERE THERE BE DINOSAURS 🦖 ========= +# This content is subject to significant change. Not for review yet. +# Included as a proof of concept for context or testing ONLY. +# ================================================ + +# Imports +import os + +from typing import ( + Dict, + Optional, + Sequence, + Tuple, + Union, +) + + +{% for imp in service_imports %} +{{ imp }} +{% endfor %} +from google.cloud.bigquery_v2.services.centralized_service import _helpers + +{% for imp in type_imports %} +{{ imp }} +{% endfor %} +from google.cloud.bigquery_v2.types import dataset_reference + +from google.api_core import client_options as client_options_lib +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as auth_credentials + +# Create type aliases +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +DatasetIdentifier = Union[str, dataset_reference.DatasetReference] + +DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT +DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT +DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () + + +class BigQueryClient: + def __init__(self, credentials=None, client_options=None): + self._clients = {} + self._credentials = credentials + self._client_options = client_options + + # --- *METHOD SECTION --- +{% for method in methods %} + def {{ method.name }}( + self, + *, + request: Optional["{{ method.service_module_name.replace('_service', '') }}.{{ method.name.replace('_', ' ').title().replace(' ', '') }}Request"] = None, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, + ) -> "{{ method.return_type }}": + """ + TODO: Docstring is purposefully blank. microgenerator will add automatically. + """ + + return self.{{ method.service_module_name }}_client.{{ method.name }}( + request=request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) +{% endfor %} + +{#- *ServiceClient Properties Section: methods to get/set service clients -#} + # --- *SERVICECLIENT PROPERTIES --- +{% for service in services %} + @property + def {{ service.property_name }}(self): + if "{{ service.service_name }}" not in self._clients: + self._clients["{{ service.service_name }}"] = {{ service.service_module_name }}.{{ service.service_client_class }}( + credentials=self._credentials, client_options=self._client_options + ) + return self._clients["{{ service.service_name }}"] + + @{{ service.property_name }}.setter + def {{ service.property_name }}(self, value): + if not isinstance(value, {{ service.service_module_name }}.{{ service.service_client_class }}): + raise TypeError( + "Expected an instance of {{ service.service_module_name }}.{{ service.service_client_class }}." + ) + self._clients["{{ service.service_name }}"] = value +{% endfor %} + +{#- Helper Section: methods included from partial template -#} + {%- include "partials/_client_helpers.j2" %} + + +# ======== 🦕 HERE THERE WERE DINOSAURS 🦖 ========= +# The above content is subject to significant change. Not for review yet. +# Included as a proof of concept for context or testing ONLY. +# ================================================ \ No newline at end of file diff --git a/scripts/microgenerator/templates/partials/_client_helpers.j2 b/scripts/microgenerator/templates/partials/_client_helpers.j2 new file mode 100644 index 000000000..a6e6343b5 --- /dev/null +++ b/scripts/microgenerator/templates/partials/_client_helpers.j2 @@ -0,0 +1,49 @@ + {# + This is a partial template file intended to be included in other templates. + It contains helper methods for the BigQueryClient class. + #} + + # --- HELPER METHODS --- + def _parse_dataset_path(self, dataset_path: str) -> Tuple[Optional[str], str]: + """ + Helper to parse project_id and/or dataset_id from a string identifier. + + Args: + dataset_path: A string in the format 'project_id.dataset_id' or + 'dataset_id'. + + Returns: + A tuple of (project_id, dataset_id). + """ + if "." in dataset_path: + # Use rsplit to handle legacy paths like `google.com:my-project.my_dataset`. + project_id, dataset_id = dataset_path.rsplit(".", 1) + return project_id, dataset_id + return self.project, dataset_path + + def _parse_dataset_id_to_dict(self, dataset_id: "DatasetIdentifier") -> dict: + """ + Helper to create a dictionary from a project_id and dataset_id to pass + internally between helper functions. + + Args: + dataset_id: A string or DatasetReference. + + Returns: + A dict of {"project_id": project_id, "dataset_id": dataset_id_str }. + """ + if isinstance(dataset_id, str): + project_id, dataset_id_str = self._parse_dataset_path(dataset_id) + return {"project_id": project_id, "dataset_id": dataset_id_str} + elif isinstance(dataset_id, dataset_reference.DatasetReference): + return { + "project_id": dataset_id.project_id, + "dataset_id": dataset_id.dataset_id, + } + else: + raise TypeError(f"Invalid type for dataset_id: {type(dataset_id)}") + + def _parse_project_id_to_dict(self, project_id: Optional[str] = None) -> dict: + """Helper to create a request dictionary from a project_id.""" + final_project_id = project_id or self.project + return {"project_id": final_project_id} \ No newline at end of file From 79fe5dda9bf8ea56fb31caefbdf08233c2842e5e Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 5 Sep 2025 15:22:45 -0400 Subject: [PATCH 21/21] cleans up more refactoring --- scripts/microgenerator/bigqueryclient.py.j2 | 31 -- scripts/microgenerator/config.yaml | 18 +- scripts/microgenerator/generate.py | 412 +++++++++++++------- 3 files changed, 290 insertions(+), 171 deletions(-) delete mode 100644 scripts/microgenerator/bigqueryclient.py.j2 diff --git a/scripts/microgenerator/bigqueryclient.py.j2 b/scripts/microgenerator/bigqueryclient.py.j2 deleted file mode 100644 index 840977a9a..000000000 --- a/scripts/microgenerator/bigqueryclient.py.j2 +++ /dev/null @@ -1,31 +0,0 @@ -# TODO: Add a header if needed. - -# ======== 🦕 HERE THERE BE DINOSAURS 🦖 ========= -# This content is subject to significant change. Not for review yet. -# Included as a proof of concept for context or testing ONLY. -# ================================================ - -class BigQueryClient: - def __init__(self): - self._clients = {} - -{% for method in methods %} - def {{ method.name }}({{ method.args_for_def }}): - """TODO: extract docstring for use here. - A generated method to call the BigQuery API.""" - - if "{{ method.class_name }}" not in self._clients: - from google.cloud.bigquery_v2 import {{ method.class_name }} - self._clients["{{ method.class_name }}"] = {{ method.class_name }}() - - client = self._clients["{{ method.class_name }}"] - from google.cloud.bigquery_v2 import types - request = types.{{ method.request_class_name }}({{ method.args_for_call }}) - return client.{{ method.name }}(request=request) - -{% endfor %} - -# ======== 🦕 HERE THERE WERE DINOSAURS 🦖 ========= -# The above content is subject to significant change. Not for review yet. -# Included as a proof of concept for context or testing ONLY. -# ================================================ \ No newline at end of file diff --git a/scripts/microgenerator/config.yaml b/scripts/microgenerator/config.yaml index f93e39735..a36e7c728 100644 --- a/scripts/microgenerator/config.yaml +++ b/scripts/microgenerator/config.yaml @@ -19,27 +19,29 @@ filter: classes: # Only include classes with these suffixes. include_suffixes: - - "Client" + - "DatasetServiceClient" + - "JobServiceClient" + - "ModelServiceClient" methods: # Include methods with these prefixes. include_prefixes: - - "batch_delete_" - - "cancel_" - - "create_" - - "delete_" + # - "batch_delete_" + # - "cancel_" + # - "create_" + # - "delete_" - "get_" - "insert_" - "list_" - "patch_" - - "undelete_" - - "update_" + # - "undelete_" + # - "update_" # Exclude methods with these prefixes. exclude_prefixes: - "get_mtls_endpoint_and_cert_source" # A list of templates to render and their corresponding output files. templates: - - template: "autogen/scripts/microgenerator/bigqueryclient.py.j2" + - template: "autogen/scripts/microgenerator/templates/client.py.j2" output: "autogen/google/cloud/bigquery_v2/services/centralized_service/client.py" # - template: "test_bigqueryclient.py.j2" # output: "tests/unit/test_bigqueryclient.py" diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 4e641f37d..6c394f3f4 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -26,9 +26,11 @@ import os import argparse import glob +import re from collections import defaultdict from typing import List, Dict, Any, Iterator +import name_utils import utils # ============================================================================= @@ -43,9 +45,95 @@ class CodeAnalyzer(ast.NodeVisitor): def __init__(self): self.structure: List[Dict[str, Any]] = [] + self.imports: set[str] = set() + self.types: set[str] = set() self._current_class_info: Dict[str, Any] | None = None self._is_in_method: bool = False + def _get_type_str(self, node: ast.AST | None) -> str | None: + """Recursively reconstructs a type annotation string from an AST node.""" + if node is None: + return None + # Handles simple names like 'str', 'int', 'HttpRequest' + if isinstance(node, ast.Name): + return node.id + # Handles dotted names like 'service.GetDatasetRequest' + if isinstance(node, ast.Attribute): + # Attempt to reconstruct the full dotted path + parts = [] + curr = node + while isinstance(curr, ast.Attribute): + parts.append(curr.attr) + curr = curr.value + if isinstance(curr, ast.Name): + parts.append(curr.id) + return ".".join(reversed(parts)) + # Handles subscripted types like 'list[str]', 'Optional[...]' + if isinstance(node, ast.Subscript): + value_str = self._get_type_str(node.value) + slice_str = self._get_type_str(node.slice) + return f"{value_str}[{slice_str}]" + # Handles tuples inside subscripts, e.g., 'dict[str, int]' + if isinstance(node, ast.Tuple): + return ", ".join( + [s for s in (self._get_type_str(e) for e in node.elts) if s] + ) + # Handles forward references as strings, e.g., '"Dataset"' + if isinstance(node, ast.Constant): + return str(node.value) + return None # Fallback for unhandled types + + def _collect_types_from_node(self, node: ast.AST | None) -> None: + """Recursively traverses an annotation node to find and collect all type names.""" + if node is None: + return + + if isinstance(node, ast.Name): + self.types.add(node.id) + elif isinstance(node, ast.Attribute): + type_str = self._get_type_str(node) + if type_str: + self.types.add(type_str) + elif isinstance(node, ast.Subscript): + self._collect_types_from_node(node.value) + self._collect_types_from_node(node.slice) + elif isinstance(node, (ast.Tuple, ast.List)): + for elt in node.elts: + self._collect_types_from_node(elt) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + self.types.add(node.value) + elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): # For | union type + self._collect_types_from_node(node.left) + self._collect_types_from_node(node.right) + + def visit_Import(self, node: ast.Import) -> None: + """Catches 'import X' and 'import X as Y' statements.""" + for alias in node.names: + if alias.asname: + self.imports.add(f"import {alias.name} as {alias.asname}") + else: + self.imports.add(f"import {alias.name}") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + """Catches 'from X import Y' statements.""" + module = node.module or "" + if not module: + module = "." * node.level + else: + module = "." * node.level + module + + names = [] + for alias in node.names: + if alias.asname: + names.append(f"{alias.name} as {alias.asname}") + else: + names.append(alias.name) + + if names: + self.imports.add(f"from {module} import {', '.join(names)}") + self.generic_visit(node) + def visit_ClassDef(self, node: ast.ClassDef) -> None: """Visits a class definition node.""" class_info = { @@ -61,9 +149,21 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: def visit_FunctionDef(self, node: ast.FunctionDef) -> None: """Visits a function/method definition node.""" if self._current_class_info: # This is a method + + args_info = [] + for arg in node.args.args: + type_str = self._get_type_str(arg.annotation) + args_info.append({"name": arg.arg, "type": type_str}) + self._collect_types_from_node(arg.annotation) + + # Collect return type + return_type = self._get_type_str(node.returns) + self._collect_types_from_node(node.returns) + method_info = { "method_name": node.name, - "args": [arg.arg for arg in node.args.args], + "args": args_info, + "return_type": return_type, } self._current_class_info["methods"].append(method_info) @@ -111,148 +211,164 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> None: self.generic_visit(node) -def parse_code(code: str) -> List[Dict[str, Any]]: +def parse_code(code: str) -> tuple[List[Dict[str, Any]], set[str], set[str]]: """ - Parses a string of Python code into a structured list of classes. + Parses a string of Python code into a structured list of classes, a set of imports, + and a set of all type annotations found. Args: code: A string containing Python code. Returns: - A list of dictionaries, where each dictionary represents a class. + A tuple containing: + - A list of dictionaries, where each dictionary represents a class. + - A set of strings, where each string is an import statement. + - A set of strings, where each string is a type annotation. """ tree = ast.parse(code) analyzer = CodeAnalyzer() analyzer.visit(tree) - return analyzer.structure + return analyzer.structure, analyzer.imports, analyzer.types -def parse_file(file_path: str) -> List[Dict[str, Any]]: +def parse_file(file_path: str) -> tuple[List[Dict[str, Any]], set[str], set[str]]: """ - Parses a Python file into a structured list of classes. + Parses a Python file into a structured list of classes, a set of imports, + and a set of all type annotations found. Args: file_path: The absolute path to the Python file. Returns: - A list of dictionaries representing the classes in the file. + A tuple containing the class structure, a set of import statements, + and a set of type annotations. """ with open(file_path, "r", encoding="utf-8") as source: code = source.read() return parse_code(code) -def list_classes(path: str) -> List[str]: - """Lists all classes in a given Python file or directory.""" - class_names = [] - if os.path.isfile(path) and path.endswith(".py"): - structure = parse_file(path) - for class_info in structure: - class_names.append(class_info["class_name"]) - elif os.path.isdir(path): - for file_path in utils.walk_codebase(path): - structure = parse_file(file_path) - for class_info in structure: - class_names.append( - f"{class_info['class_name']} (in {os.path.basename(file_path)})" - ) - return sorted(class_names) - - -def list_classes_and_methods(path: str) -> Dict[str, List[str]]: - """Lists all classes and their methods in a given Python file or directory.""" - results = defaultdict(list) - - def process_structure(structure, file_name=None): - for class_info in structure: - key = class_info["class_name"] - if file_name: - key = f"{key} (in {file_name})" - - results[key] = sorted([m["method_name"] for m in class_info["methods"]]) +def list_code_objects( + path: str, + show_methods: bool = False, + show_attributes: bool = False, + show_arguments: bool = False, +) -> Any: + """ + Lists classes and optionally their methods, attributes, and arguments + from a given Python file or directory. - if os.path.isfile(path) and path.endswith(".py"): - process_structure(parse_file(path)) - elif os.path.isdir(path): - for file_path in utils.walk_codebase(path): - process_structure( - parse_file(file_path), file_name=os.path.basename(file_path) - ) + This function consolidates the functionality of the various `list_*` functions. - return results + Args: + path (str): The absolute path to a Python file or directory. + show_methods (bool): Whether to include methods in the output. + show_attributes (bool): Whether to include attributes in the output. + show_arguments (bool): If True, includes method arguments. Implies show_methods. + Returns: + - If `show_methods` and `show_attributes` are both False, returns a + sorted `List[str]` of class names (mimicking `list_classes`). + - Otherwise, returns a `Dict[str, Dict[str, Any]]` containing the + requested details about each class. + """ + # If show_arguments is True, we must show methods. + if show_arguments: + show_methods = True -def list_classes_methods_and_attributes(path: str) -> Dict[str, Dict[str, List[str]]]: - """Lists classes, methods, and attributes in a file or directory.""" - results = defaultdict(lambda: defaultdict(list)) + results = defaultdict(dict) + all_class_keys = [] - def process_structure(structure, file_name=None): + def process_structure(structure: List[Dict[str, Any]], file_name: str | None = None): + """Populates the results dictionary from the parsed AST structure.""" for class_info in structure: key = class_info["class_name"] if file_name: key = f"{key} (in {file_name})" - - results[key]["attributes"] = sorted(class_info["attributes"]) - results[key]["methods"] = sorted( - [m["method_name"] for m in class_info["methods"]] - ) - + + all_class_keys.append(key) + + # Skip filling details if not needed for the dictionary. + if not show_methods and not show_attributes: + continue + + if show_attributes: + results[key]["attributes"] = sorted(class_info["attributes"]) + + if show_methods: + if show_arguments: + method_details = {} + # Sort methods by name for consistent output + for method in sorted(class_info["methods"], key=lambda m: m["method_name"]): + method_details[method["method_name"]] = method["args"] + results[key]["methods"] = method_details + else: + results[key]["methods"] = sorted( + [m["method_name"] for m in class_info["methods"]] + ) + + # Determine if the path is a file or directory and process accordingly if os.path.isfile(path) and path.endswith(".py"): - process_structure(parse_file(path)) + structure, _, _ = parse_file(path) + process_structure(structure) elif os.path.isdir(path): + # This assumes `utils.walk_codebase` is defined elsewhere. for file_path in utils.walk_codebase(path): - process_structure( - parse_file(file_path), file_name=os.path.basename(file_path) - ) + structure, _, _ = parse_file(file_path) + process_structure(structure, file_name=os.path.basename(file_path)) - return results + # Return the data in the desired format based on the flags + if not show_methods and not show_attributes: + return sorted(all_class_keys) + else: + return dict(results) -def list_classes_methods_attributes_and_arguments( - path: str, -) -> Dict[str, Dict[str, Any]]: - """Lists classes, methods, attributes, and arguments in a file or directory.""" - results = defaultdict(lambda: defaultdict(list)) +# ============================================================================= +# Section 2: Source file data gathering +# ============================================================================= - def process_structure(structure, file_name=None): - for class_info in structure: - key = class_info["class_name"] - if file_name: - key = f"{key} (in {file_name})" - results[key]["attributes"] = sorted(class_info["attributes"]) - method_details = {} - # Sort methods by name for consistent output - for method in sorted(class_info["methods"], key=lambda m: m["method_name"]): - method_details[method["method_name"]] = method["args"] - results[key]["methods"] = method_details +def _should_include_class(class_name: str, class_filters: Dict[str, Any]) -> bool: + """Checks if a class should be included based on filter criteria.""" + if class_filters.get("include_suffixes"): + if not class_name.endswith(tuple(class_filters["include_suffixes"])): + return False + if class_filters.get("exclude_suffixes"): + if class_name.endswith(tuple(class_filters["exclude_suffixes"])): + return False + return True - if os.path.isfile(path) and path.endswith(".py"): - process_structure(parse_file(path)) - elif os.path.isdir(path): - for file_path in utils.walk_codebase(path): - process_structure( - parse_file(file_path), file_name=os.path.basename(file_path) - ) - - return results +def _should_include_method(method_name: str, method_filters: Dict[str, Any]) -> bool: + """Checks if a method should be included based on filter criteria.""" + if method_filters.get("include_prefixes"): + if not any(method_name.startswith(p) for p in method_filters["include_prefixes"]): + return False + if method_filters.get("exclude_prefixes"): + if any(method_name.startswith(p) for p in method_filters["exclude_prefixes"]): + return False + return True -# ============================================================================= -# Section 2: Generic Code Generation Logic -# ============================================================================= -def analyze_source_files(config: Dict[str, Any]) -> Dict[str, Any]: +def analyze_source_files(config: Dict[str, Any]) -> tuple[Dict[str, Any], set[str], set[str]]: """ - Analyzes source files as per the configuration to extract class and method info. + Analyzes source files per the configuration to extract class and method info, + as well as information on imports and typehints. Args: config: The generator's configuration dictionary. Returns: - A dictionary containing the data needed for template rendering. + A tuple containing: + - A dictionary containing the data needed for template rendering. + - A set of all import statements required by the parsed methods. + - A set of all type annotations found in the parsed methods. """ parsed_data = defaultdict(dict) + all_imports: set[str] = set() + all_types: set[str] = set() + source_patterns = config.get("source_files", []) filter_rules = config.get("filter", {}) class_filters = filter_rules.get("classes", {}) @@ -261,89 +377,121 @@ def analyze_source_files(config: Dict[str, Any]) -> Dict[str, Any]: source_files = [] for pattern in source_patterns: source_files.extend(glob.glob(pattern, recursive=True)) - + for file_path in source_files: - structure = parse_file(file_path) + structure, imports, types = parse_file(file_path) + all_imports.update(imports) + all_types.update(types) for class_info in structure: class_name = class_info["class_name"] - # Apply class filters - if class_filters.get("include_suffixes"): - if not class_name.endswith(tuple(class_filters["include_suffixes"])): - continue + if not _should_include_class(class_name, class_filters): + continue parsed_data[class_name] # Ensure class is in dict for method in class_info["methods"]: method_name = method["method_name"] - # Apply method filters - if method_filters.get("include_prefixes"): - if not any( - method_name.startswith(p) - for p in method_filters["include_prefixes"] - ): - continue - if method_filters.get("exclude_prefixes"): - if any( - method_name.startswith(p) - for p in method_filters["exclude_prefixes"] - ): - continue - parsed_data[class_name][method_name] = method["args"] - return parsed_data - - -def _format_args(method_args: List[str]) -> tuple[str, str]: - """Formats method arguments for use in creating a method definition and a method call.""" - args_for_def = ", ".join(method_args) - args_for_call = ", ".join([f"{arg}={arg}" for arg in method_args if arg != "self"]) - return args_for_def, args_for_call + if not _should_include_method(method_name, method_filters): + continue + parsed_data[class_name][method_name] = method + + return parsed_data, all_imports, all_types def _format_class_name(method_name: str, suffix: str = "Request") -> str: """Formats a class name from a method name.""" return "".join(word.capitalize() for word in method_name.split("_")) + suffix +# ============================================================================= +# Section 3: Code Generation +# ============================================================================= + +def _generate_import_statement(context: List[Dict[str, Any]], key: str, path: str) -> str: + """Generates a formatted import statement from a list of context dictionaries. + + Args: + context: A list of dictionaries containing the data. + key: The key to extract from each dictionary in the context. + path: The base import path (e.g., "google.cloud.bigquery_v2.services"). + + Returns: + A formatted, multi-line import statement string. + """ + names = sorted(list(set([item[key] for item in context]))) + names_str = ",\n ".join(names) + return f"from {path} import (\n {names_str}\n)" + -def generate_code(config: Dict[str, Any], data: Dict[str, Any]) -> None: +def generate_code(config: Dict[str, Any], analysis_results: tuple) -> None: """ Generates source code files using Jinja2 templates. """ + data, all_imports, all_types = analysis_results + templates_config = config.get("templates", []) for item in templates_config: template_path = item["template"] output_path = item["output"] - print(f"Processing template: {template_path}.") - template = utils.load_template(template_path) methods_context = [] for class_name, methods in data.items(): - for method_name, method_args in methods.items(): - args_for_def, args_for_call = _format_args(method_args) - request_class_name = _format_class_name(method_name) + for method_name, method_info in methods.items(): + methods_context.append( { "name": method_name, "class_name": class_name, - "args_for_def": args_for_def, - "args_for_call": args_for_call, - "request_class_name": request_class_name, + "return_type": method_info["return_type"], } ) + + # Prepare imports for the template + services_context = [] + client_class_names = sorted(list(set([m['class_name'] for m in methods_context]))) + + for class_name in client_class_names: + service_name_cluster = name_utils.generate_service_names(class_name) + services_context.append(service_name_cluster) + + # Also need to update methods_context to include the service_name and module_name + # so the template knows which client to use for each method. + class_to_service_map = {s['service_client_class']: s for s in services_context} + for method in methods_context: + service_info = class_to_service_map.get(method['class_name']) + if service_info: + method['service_name'] = service_info['service_name'] + method['service_module_name'] = service_info['service_module_name'] + + # Prepare new imports + service_imports = [ + _generate_import_statement( + services_context, "service_module_name", "google.cloud.bigquery_v2.services" + ) + ] - print(f"Found {len(methods_context)} methods to generate.") + # Prepare type imports + type_imports = [ + _generate_import_statement( + services_context, "service_name", "google.cloud.bigquery_v2.types" + ) + ] final_code = template.render( service_name=config.get("service_name"), - methods=methods_context + methods=methods_context, + services=services_context, + service_imports=service_imports, + type_imports=type_imports, ) utils.write_code_to_file(output_path, final_code) + # ============================================================================= -# Section 3: Main Execution +# Section 4: Main Execution # ============================================================================= if __name__ == "__main__": @@ -356,7 +504,7 @@ def generate_code(config: Dict[str, Any], data: Dict[str, Any]) -> None: args = parser.parse_args() config = utils.load_config(args.config) - data = analyze_source_files(config) - generate_code(config, data) + analysis_results = analyze_source_files(config) + generate_code(config, analysis_results) # TODO: Ensure blacken gets called on the generated source files as a final step.