From 9007ab4c5ec2900e4a70787c47aa3b1ed2612b07 Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Mon, 15 Dec 2025 16:49:14 -0800 Subject: [PATCH 1/8] Improve the error message when User Default Credentials are not provided --- google/cloud/dataproc_spark_connect/session.py | 10 ++++++++++ tests/unit/test_session.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/google/cloud/dataproc_spark_connect/session.py b/google/cloud/dataproc_spark_connect/session.py index 2fd2211..9ea051e 100644 --- a/google/cloud/dataproc_spark_connect/session.py +++ b/google/cloud/dataproc_spark_connect/session.py @@ -39,6 +39,7 @@ PermissionDenied, ) from google.api_core.future.polling import POLLING_PREDICATE +from google.auth.exceptions import DefaultCredentialsError from google.cloud.dataproc_spark_connect.client import DataprocChannelBuilder from google.cloud.dataproc_spark_connect.exceptions import DataprocSparkConnectException from google.cloud.dataproc_spark_connect.pypi_artifacts import PyPiArtifacts @@ -456,6 +457,15 @@ def create_session_pbar(): raise DataprocSparkConnectException( f"Error while creating Dataproc Session: {e.message}" ) + except DefaultCredentialsError as e: + stop_create_session_pbar_event.set() + if create_session_pbar_thread.is_alive(): + create_session_pbar_thread.join() + DataprocSparkSession._active_s8s_session_id = None + DataprocSparkSession._active_session_uses_custom_id = False + raise DataprocSparkConnectException( + f"Error while creating Dataproc Session: {e}" + ) except Exception as e: stop_create_session_pbar_event.set() if create_session_pbar_thread.is_alive(): diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 703276f..6222f47 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1487,6 +1487,15 @@ def test_create_session_without_location(self): except DataprocSparkConnectException as e: self.assertIn("location is not set", str(e)) + def test_create_session_without_application_default_credentials(self): + """Tests that an exception is raised when application default credentials is not provided.""" + os.environ.clear() + try: + DataprocSparkSession.builder.location("test-region").projectId( + "test-project" + ).getOrCreate() + except DataprocSparkConnectException as e: + self.assertIn("Your default credentials were not found", str(e)) class DataprocSparkConnectClientTest(unittest.TestCase): From 9348464a8bc9e067b2b8b59d6e4b0ecdc2890ea0 Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Mon, 15 Dec 2025 16:49:14 -0800 Subject: [PATCH 2/8] Improve the error message when User Default Credentials are not provided --- google/cloud/dataproc_spark_connect/session.py | 10 ++++++++++ tests/unit/test_session.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/google/cloud/dataproc_spark_connect/session.py b/google/cloud/dataproc_spark_connect/session.py index 2fd2211..9ea051e 100644 --- a/google/cloud/dataproc_spark_connect/session.py +++ b/google/cloud/dataproc_spark_connect/session.py @@ -39,6 +39,7 @@ PermissionDenied, ) from google.api_core.future.polling import POLLING_PREDICATE +from google.auth.exceptions import DefaultCredentialsError from google.cloud.dataproc_spark_connect.client import DataprocChannelBuilder from google.cloud.dataproc_spark_connect.exceptions import DataprocSparkConnectException from google.cloud.dataproc_spark_connect.pypi_artifacts import PyPiArtifacts @@ -456,6 +457,15 @@ def create_session_pbar(): raise DataprocSparkConnectException( f"Error while creating Dataproc Session: {e.message}" ) + except DefaultCredentialsError as e: + stop_create_session_pbar_event.set() + if create_session_pbar_thread.is_alive(): + create_session_pbar_thread.join() + DataprocSparkSession._active_s8s_session_id = None + DataprocSparkSession._active_session_uses_custom_id = False + raise DataprocSparkConnectException( + f"Error while creating Dataproc Session: {e}" + ) except Exception as e: stop_create_session_pbar_event.set() if create_session_pbar_thread.is_alive(): diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 703276f..623253b 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1487,6 +1487,16 @@ def test_create_session_without_location(self): except DataprocSparkConnectException as e: self.assertIn("location is not set", str(e)) + def test_create_session_without_application_default_credentials(self): + """Tests that an exception is raised when application default credentials is not provided.""" + os.environ.clear() + try: + DataprocSparkSession.builder.location("test-region").projectId( + "test-project" + ).getOrCreate() + except DataprocSparkConnectException as e: + self.assertIn("Your default credentials were not found", str(e)) + class DataprocSparkConnectClientTest(unittest.TestCase): From 627df5c7eadf9266238c6b0afca3dac1d0a7b898 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 27 Jan 2026 12:54:57 -0800 Subject: [PATCH 3/8] Update google/cloud/dataproc_spark_connect/session.py Co-authored-by: Igor Dvorzhak --- google/cloud/dataproc_spark_connect/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/dataproc_spark_connect/session.py b/google/cloud/dataproc_spark_connect/session.py index b68599b..22c42c3 100644 --- a/google/cloud/dataproc_spark_connect/session.py +++ b/google/cloud/dataproc_spark_connect/session.py @@ -464,8 +464,8 @@ def create_session_pbar(): DataprocSparkSession._active_s8s_session_id = None DataprocSparkSession._active_session_uses_custom_id = False raise DataprocSparkConnectException( - f"Error while creating Dataproc Session (see https://docs.cloud.google.com/docs/authentication/provide-credentials-adc for more info): {e}" - ) + "Credentials error while creating Dataproc Session (see https://docs.cloud.google.com/docs/authentication/provide-credentials-adc for more info)" + ) from e except Exception as e: stop_create_session_pbar_event.set() if create_session_pbar_thread.is_alive(): From a8643905eca2963a837bb54d9fa1663b8edd1bd8 Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Tue, 27 Jan 2026 12:58:36 -0800 Subject: [PATCH 4/8] Formatting --- google/cloud/dataproc_spark_connect/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/dataproc_spark_connect/session.py b/google/cloud/dataproc_spark_connect/session.py index 22c42c3..7ad24fe 100644 --- a/google/cloud/dataproc_spark_connect/session.py +++ b/google/cloud/dataproc_spark_connect/session.py @@ -465,7 +465,7 @@ def create_session_pbar(): DataprocSparkSession._active_session_uses_custom_id = False raise DataprocSparkConnectException( "Credentials error while creating Dataproc Session (see https://docs.cloud.google.com/docs/authentication/provide-credentials-adc for more info)" - ) from e + ) from e except Exception as e: stop_create_session_pbar_event.set() if create_session_pbar_thread.is_alive(): From 4a3cd2986a2dfc4e0f0e374f1939f815ad1eaf6d Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Thu, 29 Jan 2026 18:14:14 -0800 Subject: [PATCH 5/8] Fix broken tests --- tests/unit/test_session.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 623253b..9bd9282 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1212,7 +1212,7 @@ def test_display_button_with_aiplatform_installed_ipython_non_interactive( @mock.patch( "IPython.core.interactiveshell.InteractiveShell.initialized", - return_value=False, + return_value=True, ) @mock.patch("IPython.display.display") def test_display_session_link_on_creation_colab_enterprise( @@ -1238,7 +1238,7 @@ def test_display_session_link_on_creation_colab_enterprise( @mock.patch( "IPython.core.interactiveshell.InteractiveShell.initialized", - return_value=False, + return_value=True, ) @mock.patch("IPython.display.display") def test_display_session_link_on_creation_not_colab_enterprise( @@ -1495,7 +1495,7 @@ def test_create_session_without_application_default_credentials(self): "test-project" ).getOrCreate() except DataprocSparkConnectException as e: - self.assertIn("Your default credentials were not found", str(e)) + self.assertIn("Credentials error while creating Dataproc Session", str(e)) class DataprocSparkConnectClientTest(unittest.TestCase): From 1854520f9e4a0af9c5df85ec21b2e9ef984bae2c Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Thu, 29 Jan 2026 19:14:52 -0800 Subject: [PATCH 6/8] Formatting --- tests/unit/test_session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 9bd9282..2b1a624 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1495,7 +1495,9 @@ def test_create_session_without_application_default_credentials(self): "test-project" ).getOrCreate() except DataprocSparkConnectException as e: - self.assertIn("Credentials error while creating Dataproc Session", str(e)) + self.assertIn( + "Credentials error while creating Dataproc Session", str(e) + ) class DataprocSparkConnectClientTest(unittest.TestCase): From 975066df981d0d9db36384b48bd6bfee26f55ab5 Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Thu, 29 Jan 2026 19:23:07 -0800 Subject: [PATCH 7/8] Fix broken test --- tests/unit/test_pypi_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_pypi_artifacts.py b/tests/unit/test_pypi_artifacts.py index 0d578e9..7156742 100644 --- a/tests/unit/test_pypi_artifacts.py +++ b/tests/unit/test_pypi_artifacts.py @@ -26,7 +26,7 @@ def test_valid_inputs(): def test_bad_format(self): with self.assertRaisesRegex( InvalidRequirement, - "Expected end or semicolon \(after name and no valid version specifier\).*", + "Expected semicolon \(after name and no valid version specifier\).*", ): PyPiArtifacts({"pypi://spacy:23"}) From e6471d98f1df48da8d8657783f33f42971824899 Mon Sep 17 00:00:00 2001 From: Tim Utegenov Date: Thu, 29 Jan 2026 19:28:45 -0800 Subject: [PATCH 8/8] Fix broken test --- tests/unit/test_pypi_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_pypi_artifacts.py b/tests/unit/test_pypi_artifacts.py index 7156742..22ef360 100644 --- a/tests/unit/test_pypi_artifacts.py +++ b/tests/unit/test_pypi_artifacts.py @@ -26,7 +26,7 @@ def test_valid_inputs(): def test_bad_format(self): with self.assertRaisesRegex( InvalidRequirement, - "Expected semicolon \(after name and no valid version specifier\).*", + r"Expected semicolon \(after name with no version specifier\) or end", ): PyPiArtifacts({"pypi://spacy:23"})