Skip to content

Commit 0a1b1fa

Browse files
yenkins-adminyenkins-admin
authored
Merge pull request #938 from gooddata/snapshot-master-f97c924b-to-rel/dev
[bot] Merge master/f97c924b into rel/dev
2 parents b53a3e1 + f97c924 commit 0a1b1fa

19 files changed

+886
-60
lines changed

‎gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,13 @@
1212
from gooddata_api_client.model.test_definition_request import TestDefinitionRequest
1313

1414
from gooddata_sdk.catalog.base import Base, value_in_allowed
15-
from gooddata_sdk.catalog.entity import TokenCredentialsFromFile
15+
from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile, TokenCredentialsFromFile
1616
from gooddata_sdk.catalog.parameter import CatalogParameter
1717
from gooddata_sdk.catalog.permission.declarative_model.permission import CatalogDeclarativeDataSourcePermission
1818
from gooddata_sdk.utils import create_directory, get_ds_credentials, read_layout_from_file, write_layout_to_file
1919

2020
BIGQUERY_TYPE = "BIGQUERY"
21+
DATABRICKS_TYPE = "DATABRICKS"
2122
LAYOUT_DATA_SOURCES_DIR = "data_sources"
2223

2324

@@ -33,6 +34,15 @@ def _inject_base(self, credentials: dict[str, Any]) -> DeclarativeDataSources:
3334
if data_source.type == BIGQUERY_TYPE:
3435
token = TokenCredentialsFromFile.token_from_file(credentials[data_source.id])
3536
data_sources.append(data_source.to_api(token=token))
37+
elif data_source.type == DATABRICKS_TYPE:
38+
if data_source.client_id is not None:
39+
client_secret = ClientSecretCredentialsFromFile.client_secret_from_file(
40+
credentials[data_source.id]
41+
)
42+
data_sources.append(data_source.to_api(client_secret=client_secret))
43+
else:
44+
token = TokenCredentialsFromFile.token_from_file(credentials[data_source.id])
45+
data_sources.append(data_source.to_api(token=token))
3646
else:
3747
data_sources.append(data_source.to_api(password=credentials[data_source.id]))
3848
else:
@@ -125,8 +135,13 @@ def to_test_request(
125135
kwargs["private_key"] = private_key
126136
if private_key_passphrase is not None:
127137
kwargs["private_key_passphrase"] = private_key
138+
if self.client_id is not None:
139+
kwargs["client_id"] = self.client_id
128140
if client_secret is not None:
129141
kwargs["client_secret"] = client_secret
142+
if self.parameters is not None:
143+
kwargs["parameters"] = [param.to_data_source_parameter() for param in self.parameters]
144+
130145
return TestDefinitionRequest(type=self.type, url=self.url, **kwargs)
131146

132147
@staticmethod

‎gooddata-sdk/gooddata_sdk/catalog/data_source/service.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from gooddata_sdk.catalog.data_source.action_model.responses.scan_sql_response import ScanSqlResponse
1818
from gooddata_sdk.catalog.data_source.declarative_model.data_source import (
1919
BIGQUERY_TYPE,
20+
DATABRICKS_TYPE,
2021
CatalogDeclarativeDataSource,
2122
CatalogDeclarativeDataSources,
2223
)
@@ -25,7 +26,7 @@
2526
CatalogScanResultPdm,
2627
)
2728
from gooddata_sdk.catalog.data_source.entity_model.data_source import CatalogDataSource
28-
from gooddata_sdk.catalog.entity import TokenCredentialsFromFile
29+
from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile, TokenCredentialsFromFile
2930
from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm import CatalogDeclarativeModel
3031
from gooddata_sdk.client import GoodDataApiClient
3132
from gooddata_sdk.utils import get_ds_credentials, load_all_entities_dict, read_layout_from_file
@@ -474,6 +475,19 @@ def test_data_sources_connection(
474475
response = self._actions_api.test_data_source_definition(
475476
declarative_data_source.to_test_request(token=token)
476477
)
478+
elif declarative_data_source.type == DATABRICKS_TYPE:
479+
if declarative_data_source.client_id is not None:
480+
client_secret = ClientSecretCredentialsFromFile.client_secret_from_file(
481+
credentials[declarative_data_source.id]
482+
)
483+
response = self._actions_api.test_data_source_definition(
484+
declarative_data_source.to_test_request(client_secret=client_secret)
485+
)
486+
else:
487+
token = TokenCredentialsFromFile.token_from_file(credentials[declarative_data_source.id])
488+
response = self._actions_api.test_data_source_definition(
489+
declarative_data_source.to_test_request(token=token)
490+
)
477491
else:
478492
response = self._actions_api.test_data_source_definition(
479493
declarative_data_source.to_test_request(password=credentials[declarative_data_source.id])
@@ -528,6 +542,6 @@ def _credentials_from_file(credentials_path: Path) -> dict[str, Any]:
528542
if data.get("data_sources") is None:
529543
raise ValueError("The file has a wrong structure. There should be a root key 'data_sources'.")
530544
if len(data["data_sources"]) == 0:
531-
raise ValueError("There are no pairs of data source id and token.")
545+
raise ValueError("There are no pairs of data source id and credentials.")
532546
credentials = data["data_sources"]
533547
return credentials

‎gooddata-sdk/gooddata_sdk/catalog/entity.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,3 +274,31 @@ def from_api(cls, attributes: dict[str, Any]) -> ClientSecretCredentials:
274274
# You have to fill it to keep it or update it
275275
client_secret="",
276276
)
277+
278+
279+
@attr.s(auto_attribs=True, kw_only=True)
280+
class ClientSecretCredentialsFromFile(Credentials):
281+
file_path: Path
282+
client_secret: str = attr.field(init=False, repr=lambda value: "***")
283+
284+
def __attrs_post_init__(self) -> None:
285+
self.client_secret = self.client_secret_from_file(self.file_path)
286+
287+
def to_api_args(self) -> dict[str, Any]:
288+
return {
289+
self.CLIENT_SECRET: self.client_secret,
290+
}
291+
292+
@classmethod
293+
def is_part_of_api(cls, entity: dict[str, Any]) -> bool:
294+
return cls.CLIENT_SECRET in entity
295+
296+
@classmethod
297+
def from_api(cls, entity: dict[str, Any]) -> ClientSecretCredentialsFromFile:
298+
# Credentials are not returned for security reasons
299+
raise NotImplementedError
300+
301+
@staticmethod
302+
def client_secret_from_file(file_path: Union[str, Path]) -> str:
303+
with open(file_path, "rb") as fp:
304+
return base64.b64encode(fp.read()).decode("utf-8")

‎gooddata-sdk/gooddata_sdk/catalog/parameter.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# (C) 2022 GoodData Corporation
22
import attr
3+
from gooddata_api_client.model.data_source_parameter import DataSourceParameter
34
from gooddata_api_client.model.parameter import Parameter
45

56
from gooddata_sdk.catalog.base import Base
@@ -13,3 +14,6 @@ class CatalogParameter(Base):
1314
@staticmethod
1415
def client_class() -> type[Parameter]:
1516
return Parameter
17+
18+
def to_data_source_parameter(self) -> DataSourceParameter:
19+
return DataSourceParameter(name=self.name, value=self.value)

‎gooddata-sdk/tests/catalog/expected/declarative_data_sources.json‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,65 @@
2323
"type": "POSTGRESQL",
2424
"url": "jdbc:postgresql://localhost:5432/demo",
2525
"username": "demouser"
26+
},
27+
{
28+
"id": "demo-test-ds-databricks-client-secret",
29+
"name": "demo-test-ds-databricks-client-secret",
30+
"permissions": [
31+
{
32+
"assignee": {
33+
"id": "demo2",
34+
"type": "user"
35+
},
36+
"name": "MANAGE"
37+
},
38+
{
39+
"assignee": {
40+
"id": "demoGroup",
41+
"type": "userGroup"
42+
},
43+
"name": "USE"
44+
}
45+
],
46+
"schema": "demo",
47+
"type": "DATABRICKS",
48+
"url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
49+
"clientId": "client-id",
50+
"parameters": [
51+
{
52+
"name": "catalog",
53+
"value": "demo"
54+
}
55+
]
56+
},
57+
{
58+
"id": "demo-test-ds-databricks-token",
59+
"name": "demo-test-ds-databricks-token",
60+
"permissions": [
61+
{
62+
"assignee": {
63+
"id": "demo2",
64+
"type": "user"
65+
},
66+
"name": "MANAGE"
67+
},
68+
{
69+
"assignee": {
70+
"id": "demoGroup",
71+
"type": "userGroup"
72+
},
73+
"name": "USE"
74+
}
75+
],
76+
"schema": "demo",
77+
"type": "DATABRICKS",
78+
"url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
79+
"parameters": [
80+
{
81+
"name": "catalog",
82+
"value": "demo"
83+
}
84+
]
2685
}
2786
]
2887
}

‎gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_client_secret.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"dataSources": [
3+
{
4+
"id": "demo-test-ds-databricks-client-secret",
5+
"name": "demo-test-ds-databricks-client-secret",
6+
"permissions": [
7+
{
8+
"assignee": {
9+
"id": "demo2",
10+
"type": "user"
11+
},
12+
"name": "MANAGE"
13+
},
14+
{
15+
"assignee": {
16+
"id": "demoGroup",
17+
"type": "userGroup"
18+
},
19+
"name": "USE"
20+
}
21+
],
22+
"schema": "demo",
23+
"type": "DATABRICKS",
24+
"url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
25+
"clientId": "client-id",
26+
"parameters": [
27+
{
28+
"name": "catalog",
29+
"value": "demo"
30+
}
31+
]
32+
}
33+
]
34+
}

‎gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_token.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"dataSources": [
3+
{
4+
"id": "demo-test-ds-databricks-token",
5+
"name": "demo-test-ds-databricks-token",
6+
"permissions": [
7+
{
8+
"assignee": {
9+
"id": "demo2",
10+
"type": "user"
11+
},
12+
"name": "MANAGE"
13+
},
14+
{
15+
"assignee": {
16+
"id": "demoGroup",
17+
"type": "userGroup"
18+
},
19+
"name": "USE"
20+
}
21+
],
22+
"schema": "demo",
23+
"type": "DATABRICKS",
24+
"url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
25+
"parameters": [
26+
{
27+
"name": "catalog",
28+
"value": "demo"
29+
}
30+
]
31+
}
32+
]
33+
}

‎gooddata-sdk/tests/catalog/fixtures/data_sources/demo_cache_strategy.yaml‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# (C) 2024 GoodData Corporation
1+
# (C) 2025 GoodData Corporation
22
version: 1
33
interactions:
44
- request:
@@ -69,7 +69,7 @@ interactions:
6969
X-XSS-Protection:
7070
- '0'
7171
set-cookie:
72-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
72+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT;
7373
Path=/; HTTPOnly; SameSite=Lax
7474
body:
7575
string:
@@ -161,7 +161,7 @@ interactions:
161161
X-XSS-Protection:
162162
- '0'
163163
set-cookie:
164-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
164+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT;
165165
Path=/; HTTPOnly; SameSite=Lax
166166
body:
167167
string:
@@ -245,7 +245,7 @@ interactions:
245245
X-XSS-Protection:
246246
- '0'
247247
set-cookie:
248-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
248+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT;
249249
Path=/; HTTPOnly; SameSite=Lax
250250
body:
251251
string:
@@ -283,6 +283,43 @@ interactions:
283283
type: userGroup
284284
name: USE
285285
password: demopass
286+
- id: demo-test-ds-databricks-client-secret
287+
name: demo-test-ds-databricks-client-secret
288+
schema: demo
289+
type: DATABRICKS
290+
url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;
291+
parameters:
292+
- name: catalog
293+
value: demo
294+
permissions:
295+
- assignee:
296+
id: demo2
297+
type: user
298+
name: MANAGE
299+
- assignee:
300+
id: demoGroup
301+
type: userGroup
302+
name: USE
303+
clientId: client-id
304+
clientSecret: databricks-client-secret
305+
- id: demo-test-ds-databricks-token
306+
name: demo-test-ds-databricks-token
307+
schema: demo
308+
type: DATABRICKS
309+
url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;
310+
parameters:
311+
- name: catalog
312+
value: demo
313+
permissions:
314+
- assignee:
315+
id: demo2
316+
type: user
317+
name: MANAGE
318+
- assignee:
319+
id: demoGroup
320+
type: userGroup
321+
name: USE
322+
token: databricks-token
286323
headers:
287324
Accept-Encoding:
288325
- br, gzip, deflate
@@ -342,7 +379,7 @@ interactions:
342379
X-XSS-Protection:
343380
- '0'
344381
set-cookie:
345-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
382+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:51 GMT;
346383
Path=/; HTTPOnly; SameSite=Lax
347384
body:
348385
string: ''

0 commit comments

Comments
 (0)