From b6fcf5747d7c6b205ffe5c7a1fdb7da5de5396c2 Mon Sep 17 00:00:00 2001 From: Tim Swena Date: Tue, 5 Aug 2025 20:06:18 +0000 Subject: [PATCH 1/7] feat: retry AI/ML jobs that fail more often --- .pre-commit-config.yaml | 2 +- bigframes/session/__init__.py | 2 + bigframes/session/_io/bigquery/__init__.py | 15 +- pyproject.toml | 8 +- .../google_cloud_bigquery/retry.py | 222 ++++++++++++++++++ 5 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 third_party/bigframes_vendored/google_cloud_bigquery/retry.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f839c3c0a4..d4f37464da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index d27cd48cdd..10a112c779 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -40,6 +40,7 @@ import weakref import bigframes_vendored.constants as constants +import bigframes_vendored.google_cloud_bigquery.retry as third_party_gcb_retry import bigframes_vendored.ibis.backends.bigquery as ibis_bigquery # noqa import bigframes_vendored.pandas.io.gbq as third_party_pandas_gbq import bigframes_vendored.pandas.io.parquet as third_party_pandas_parquet @@ -2051,6 +2052,7 @@ def _start_query_ml_ddl( project=None, timeout=None, query_with_job=True, + job_retry=third_party_gcb_retry.DEFAULT_ML_JOB_RETRY, ) return iterator, query_job diff --git a/bigframes/session/_io/bigquery/__init__.py b/bigframes/session/_io/bigquery/__init__.py index fdc240fa69..099f58d207 100644 --- a/bigframes/session/_io/bigquery/__init__.py +++ b/bigframes/session/_io/bigquery/__init__.py @@ -24,8 +24,10 @@ import typing from typing import Dict, Iterable, Literal, Mapping, Optional, overload, Tuple, Union +import bigframes_vendored.google_cloud_bigquery.retry as third_party_gcb_retry import bigframes_vendored.pandas.io.gbq as third_party_pandas_gbq import google.api_core.exceptions +import google.api_core.retry import google.cloud.bigquery as bigquery from bigframes.core import log_adapter @@ -245,8 +247,9 @@ def start_query_with_client( location: Optional[str], project: Optional[str], timeout: Optional[float], - metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], query_with_job: Literal[True], + job_retry: Optional[google.api_core.retry.Retry], ) -> Tuple[bigquery.table.RowIterator, bigquery.QueryJob]: ... @@ -260,8 +263,9 @@ def start_query_with_client( location: Optional[str], project: Optional[str], timeout: Optional[float], - metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], query_with_job: Literal[False], + job_retry: Optional[google.api_core.retry.Retry], ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: ... @@ -276,6 +280,11 @@ def start_query_with_client( timeout: Optional[float] = None, metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, query_with_job: bool = True, + # TODO(tswast): We can stop providing our own default once we use a + # google-cloud-bigquery version with + # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely + # version 3.36.0 or later. + job_retry: Optional[google.api_core.retry.Retry] = third_party_gcb_retry.DEFAULT_JOB_RETRY, ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: """ Starts query job and waits for results. @@ -292,6 +301,7 @@ def start_query_with_client( location=location, project=project, api_timeout=timeout, + job_retry=job_retry, ) if metrics is not None: metrics.count_job_stats(row_iterator=results_iterator) @@ -303,6 +313,7 @@ def start_query_with_client( location=location, project=project, timeout=timeout, + job_retry=job_retry, ) except google.api_core.exceptions.Forbidden as ex: if "Drive credentials" in ex.message: diff --git a/pyproject.toml b/pyproject.toml index fed528d4a7..c43b50459b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools", "setuptools_scm"] build-backend = "setuptools.build_meta" + +[project] +dynamic=['version'] + +[tool.setuptools_scm] +version_file = "bigframes/version.py" diff --git a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py new file mode 100644 index 0000000000..278b2a9be2 --- /dev/null +++ b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py @@ -0,0 +1,222 @@ +# Original: https://github.com/googleapis/python-bigquery/blob/main/google/cloud/bigquery/retry.py +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.api_core import exceptions +from google.api_core import retry +import google.api_core.future.polling +from google.auth import exceptions as auth_exceptions # type: ignore +import requests.exceptions + + +_RETRYABLE_REASONS = frozenset( + ["rateLimitExceeded", "backendError", "internalError", "badGateway"] +) + +_UNSTRUCTURED_RETRYABLE_TYPES = ( + ConnectionError, + exceptions.TooManyRequests, + exceptions.InternalServerError, + exceptions.BadGateway, + exceptions.ServiceUnavailable, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + auth_exceptions.TransportError, +) + +_MINUTE_IN_SECONDS = 60.0 +_HOUR_IN_SECONDS = 60.0 * _MINUTE_IN_SECONDS +_DEFAULT_RETRY_DEADLINE = 10.0 * _MINUTE_IN_SECONDS + +# Ambiguous errors (e.g. internalError, backendError, rateLimitExceeded) retry +# until the full `_DEFAULT_RETRY_DEADLINE`. This is because the +# `jobs.getQueryResults` REST API translates a job failure into an HTTP error. +# +# TODO(https://github.com/googleapis/python-bigquery/issues/1903): Investigate +# if we can fail early for ambiguous errors in `QueryJob.result()`'s call to +# the `jobs.getQueryResult` API. +# +# We need `_DEFAULT_JOB_DEADLINE` to be some multiple of +# `_DEFAULT_RETRY_DEADLINE` to allow for a few retries after the retry +# timeout is reached. +# +# Note: This multiple should actually be a multiple of +# (2 * _DEFAULT_RETRY_DEADLINE). After an ambiguous exception, the first +# call from `job_retry()` refreshes the job state without actually restarting +# the query. The second `job_retry()` actually restarts the query. For a more +# detailed explanation, see the comments where we set `restart_query_job = True` +# in `QueryJob.result()`'s inner `is_job_done()` function. +_DEFAULT_JOB_DEADLINE = 2.0 * (2.0 * _DEFAULT_RETRY_DEADLINE) + + +def _should_retry(exc): + """Predicate for determining when to retry. + + We retry if and only if the 'reason' is 'backendError' + or 'rateLimitExceeded'. + """ + if not hasattr(exc, "errors") or len(exc.errors) == 0: + # Check for unstructured error returns, e.g. from GFE + return isinstance(exc, _UNSTRUCTURED_RETRYABLE_TYPES) + + reason = exc.errors[0]["reason"] + return reason in _RETRYABLE_REASONS + + +DEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=_DEFAULT_RETRY_DEADLINE) +"""The default retry object. + +Any method with a ``retry`` parameter will be retried automatically, +with reasonable defaults. To disable retry, pass ``retry=None``. +To modify the default retry behavior, call a ``with_XXX`` method +on ``DEFAULT_RETRY``. For example, to change the deadline to 30 seconds, +pass ``retry=bigquery.DEFAULT_RETRY.with_deadline(30)``. +""" + + +def _should_retry_get_job_conflict(exc): + """Predicate for determining when to retry a jobs.get call after a conflict error. + + Sometimes we get a 404 after a Conflict. In this case, we + have pretty high confidence that by retrying the 404, we'll + (hopefully) eventually recover the job. + https://github.com/googleapis/python-bigquery/issues/2134 + + Note: we may be able to extend this to user-specified predicates + after https://github.com/googleapis/python-api-core/issues/796 + to tweak existing Retry object predicates. + """ + return isinstance(exc, exceptions.NotFound) or _should_retry(exc) + + +# Pick a deadline smaller than our other deadlines since we want to timeout +# before those expire. +_DEFAULT_GET_JOB_CONFLICT_DEADLINE = _DEFAULT_RETRY_DEADLINE / 3.0 +_DEFAULT_GET_JOB_CONFLICT_RETRY = retry.Retry( + predicate=_should_retry_get_job_conflict, + deadline=_DEFAULT_GET_JOB_CONFLICT_DEADLINE, +) +"""Private, may be removed in future.""" + + +# Note: Take care when updating DEFAULT_TIMEOUT to anything but None. We +# briefly had a default timeout, but even setting it at more than twice the +# theoretical server-side default timeout of 2 minutes was not enough for +# complex queries. See: +# https://github.com/googleapis/python-bigquery/issues/970#issuecomment-921934647 +DEFAULT_TIMEOUT = None +"""The default API timeout. + +This is the time to wait per request. To adjust the total wait time, set a +deadline on the retry object. +""" + +job_retry_reasons = ( + "rateLimitExceeded", + "backendError", + "internalError", + "jobBackendError", + "jobInternalError", + "jobRateLimitExceeded", +) + + +def _job_should_retry(exc): + # Sometimes we have ambiguous errors, such as 'backendError' which could + # be due to an API problem or a job problem. For these, make sure we retry + # our is_job_done() function. + # + # Note: This won't restart the job unless we know for sure it's because of + # the job status and set restart_query_job = True in that loop. This means + # that we might end up calling this predicate twice for the same job + # but from different paths: (1) from jobs.getQueryResults RetryError and + # (2) from translating the job error from the body of a jobs.get response. + # + # Note: If we start retrying job types other than queries where we don't + # call the problematic getQueryResults API to check the status, we need + # to provide a different predicate, as there shouldn't be ambiguous + # errors in those cases. + if isinstance(exc, exceptions.RetryError): + exc = exc.cause + + # Per https://github.com/googleapis/python-bigquery/issues/1929, sometimes + # retriable errors make their way here. Because of the separate + # `restart_query_job` logic to make sure we aren't restarting non-failed + # jobs, it should be safe to continue and not totally fail our attempt at + # waiting for the query to complete. + if _should_retry(exc): + return True + + if not hasattr(exc, "errors") or len(exc.errors) == 0: + return False + + reason = exc.errors[0]["reason"] + return reason in job_retry_reasons + + +DEFAULT_JOB_RETRY = retry.Retry( + predicate=_job_should_retry, deadline=_DEFAULT_JOB_DEADLINE +) +""" +The default job retry object. +""" + + +DEFAULT_ML_JOB_RETRY = retry.Retry( + predicate=_job_should_retry, deadline=_HOUR_IN_SECONDS +) +""" +The default job retry object for AI/ML jobs. + +Such jobs can take a long time to fail. See: b/436586523. +""" + + +def _query_job_insert_should_retry(exc): + # Per https://github.com/googleapis/python-bigquery/issues/2134, sometimes + # we get a 404 error. In this case, if we get this far, assume that the job + # doesn't actually exist and try again. We can't add 404 to the default + # job_retry because that happens for errors like "this table does not + # exist", which probably won't resolve with a retry. + if isinstance(exc, exceptions.RetryError): + exc = exc.cause + + if isinstance(exc, exceptions.NotFound): + message = exc.message + # Don't try to retry table/dataset not found, just job not found. + # The URL contains jobs, so use whitespace to disambiguate. + return message is not None and " job" in message.lower() + + return _job_should_retry(exc) + + +_DEFAULT_QUERY_JOB_INSERT_RETRY = retry.Retry( + predicate=_query_job_insert_should_retry, + # jobs.insert doesn't wait for the job to complete, so we don't need the + # long _DEFAULT_JOB_DEADLINE for this part. + deadline=_DEFAULT_RETRY_DEADLINE, +) +"""Private, may be removed in future.""" + + +DEFAULT_GET_JOB_TIMEOUT = 128 +""" +Default timeout for Client.get_job(). +""" + +POLLING_DEFAULT_VALUE = google.api_core.future.polling.PollingFuture._DEFAULT_VALUE +""" +Default value defined in google.api_core.future.polling.PollingFuture. +""" From d0528c682e36c9621b15a5d19ddcb2c152c07447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Tue, 5 Aug 2025 15:22:27 -0500 Subject: [PATCH 2/7] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d4f37464da..f839c3c0a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v4.0.1 hooks: - id: trailing-whitespace - id: end-of-file-fixer From f7e3e36033e22d8d7e11939f2cc8ced331926354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Tue, 5 Aug 2025 15:23:16 -0500 Subject: [PATCH 3/7] Update pyproject.toml --- pyproject.toml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c43b50459b..fed528d4a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,3 @@ [build-system] -requires = ["setuptools", "setuptools_scm"] +requires = ["setuptools"] build-backend = "setuptools.build_meta" - -[project] -dynamic=['version'] - -[tool.setuptools_scm] -version_file = "bigframes/version.py" From b8c5da81a9ce8d57664fdbce9fedce8db3a62cc8 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 5 Aug 2025 20:23:50 +0000 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- bigframes/session/_io/bigquery/__init__.py | 4 +++- third_party/bigframes_vendored/google_cloud_bigquery/retry.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigframes/session/_io/bigquery/__init__.py b/bigframes/session/_io/bigquery/__init__.py index 099f58d207..3783740422 100644 --- a/bigframes/session/_io/bigquery/__init__.py +++ b/bigframes/session/_io/bigquery/__init__.py @@ -284,7 +284,9 @@ def start_query_with_client( # google-cloud-bigquery version with # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely # version 3.36.0 or later. - job_retry: Optional[google.api_core.retry.Retry] = third_party_gcb_retry.DEFAULT_JOB_RETRY, + job_retry: Optional[ + google.api_core.retry.Retry + ] = third_party_gcb_retry.DEFAULT_JOB_RETRY, ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: """ Starts query job and waits for results. diff --git a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py index 278b2a9be2..15ecda4fbc 100644 --- a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py +++ b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py @@ -13,13 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.api_core import exceptions -from google.api_core import retry +from google.api_core import exceptions, retry import google.api_core.future.polling from google.auth import exceptions as auth_exceptions # type: ignore import requests.exceptions - _RETRYABLE_REASONS = frozenset( ["rateLimitExceeded", "backendError", "internalError", "badGateway"] ) From 4d223cdf523e4fff3f8399d29f78fa684d6860ff Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 5 Aug 2025 20:25:00 +0000 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- bigframes/session/_io/bigquery/__init__.py | 4 +++- third_party/bigframes_vendored/google_cloud_bigquery/retry.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigframes/session/_io/bigquery/__init__.py b/bigframes/session/_io/bigquery/__init__.py index 099f58d207..3783740422 100644 --- a/bigframes/session/_io/bigquery/__init__.py +++ b/bigframes/session/_io/bigquery/__init__.py @@ -284,7 +284,9 @@ def start_query_with_client( # google-cloud-bigquery version with # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely # version 3.36.0 or later. - job_retry: Optional[google.api_core.retry.Retry] = third_party_gcb_retry.DEFAULT_JOB_RETRY, + job_retry: Optional[ + google.api_core.retry.Retry + ] = third_party_gcb_retry.DEFAULT_JOB_RETRY, ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: """ Starts query job and waits for results. diff --git a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py index 278b2a9be2..15ecda4fbc 100644 --- a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py +++ b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py @@ -13,13 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.api_core import exceptions -from google.api_core import retry +from google.api_core import exceptions, retry import google.api_core.future.polling from google.auth import exceptions as auth_exceptions # type: ignore import requests.exceptions - _RETRYABLE_REASONS = frozenset( ["rateLimitExceeded", "backendError", "internalError", "badGateway"] ) From bd5f73788b9ed84bbdf1daa863e74e60e5c0b97c Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 5 Aug 2025 20:25:40 +0000 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- bigframes/session/_io/bigquery/__init__.py | 4 +++- third_party/bigframes_vendored/google_cloud_bigquery/retry.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigframes/session/_io/bigquery/__init__.py b/bigframes/session/_io/bigquery/__init__.py index 099f58d207..3783740422 100644 --- a/bigframes/session/_io/bigquery/__init__.py +++ b/bigframes/session/_io/bigquery/__init__.py @@ -284,7 +284,9 @@ def start_query_with_client( # google-cloud-bigquery version with # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely # version 3.36.0 or later. - job_retry: Optional[google.api_core.retry.Retry] = third_party_gcb_retry.DEFAULT_JOB_RETRY, + job_retry: Optional[ + google.api_core.retry.Retry + ] = third_party_gcb_retry.DEFAULT_JOB_RETRY, ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: """ Starts query job and waits for results. diff --git a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py index 278b2a9be2..15ecda4fbc 100644 --- a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py +++ b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py @@ -13,13 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.api_core import exceptions -from google.api_core import retry +from google.api_core import exceptions, retry import google.api_core.future.polling from google.auth import exceptions as auth_exceptions # type: ignore import requests.exceptions - _RETRYABLE_REASONS = frozenset( ["rateLimitExceeded", "backendError", "internalError", "badGateway"] ) From 800aff9dbe1c7e6b99532d999a0f59010044e70a Mon Sep 17 00:00:00 2001 From: Tim Swena Date: Tue, 5 Aug 2025 20:47:20 +0000 Subject: [PATCH 7/7] fix mypy --- bigframes/session/_io/bigquery/__init__.py | 38 +++++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/bigframes/session/_io/bigquery/__init__.py b/bigframes/session/_io/bigquery/__init__.py index 3783740422..83f63e8b9a 100644 --- a/bigframes/session/_io/bigquery/__init__.py +++ b/bigframes/session/_io/bigquery/__init__.py @@ -249,7 +249,6 @@ def start_query_with_client( timeout: Optional[float], metrics: Optional[bigframes.session.metrics.ExecutionMetrics], query_with_job: Literal[True], - job_retry: Optional[google.api_core.retry.Retry], ) -> Tuple[bigquery.table.RowIterator, bigquery.QueryJob]: ... @@ -265,7 +264,38 @@ def start_query_with_client( timeout: Optional[float], metrics: Optional[bigframes.session.metrics.ExecutionMetrics], query_with_job: Literal[False], - job_retry: Optional[google.api_core.retry.Retry], +) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: + ... + + +@overload +def start_query_with_client( + bq_client: bigquery.Client, + sql: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project: Optional[str], + timeout: Optional[float], + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], + query_with_job: Literal[True], + job_retry: google.api_core.retry.Retry, +) -> Tuple[bigquery.table.RowIterator, bigquery.QueryJob]: + ... + + +@overload +def start_query_with_client( + bq_client: bigquery.Client, + sql: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project: Optional[str], + timeout: Optional[float], + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], + query_with_job: Literal[False], + job_retry: google.api_core.retry.Retry, ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: ... @@ -284,9 +314,7 @@ def start_query_with_client( # google-cloud-bigquery version with # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely # version 3.36.0 or later. - job_retry: Optional[ - google.api_core.retry.Retry - ] = third_party_gcb_retry.DEFAULT_JOB_RETRY, + job_retry: google.api_core.retry.Retry = third_party_gcb_retry.DEFAULT_JOB_RETRY, ) -> Tuple[bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: """ Starts query job and waits for results.