Skip to content

Commit d8ee104

Browse files
jacspa96Jacek Spalinski
jacspa96
and
Jacek Spalinski
authored
feat(dataplex): add code samples for Entries (#12766)
* feat(dataplex): add sample for list Entries * feat(dataplex): add sample for get Entry * feat(dataplex): add sample for delete Entry * feat(dataplex): add sample for create Entry * feat(dataplex): add sample for update Entry * feat(dataplex): add sample for lookup Entry * feat(dataplex): add integration test for Entries * feat(dataplex): change method comments into docstrings --------- Co-authored-by: Jacek Spalinski <jspa@google.com>
1 parent 894b911 commit d8ee104

File tree

7 files changed

+484
-0
lines changed

7 files changed

+484
-0
lines changed

‎dataplex/snippets/create_entry.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START dataplex_create_entry]
16+
from google.cloud import dataplex_v1
17+
from google.protobuf import struct_pb2
18+
19+
20+
def create_entry(
21+
project_id: str, location: str, entry_group_id: str, entry_id: str
22+
) -> dataplex_v1.Entry:
23+
"""Method to create Entry located in project_id, location, entry_group_id and with entry_id"""
24+
25+
# Initialize client that will be used to send requests across threads. This
26+
# client only needs to be created once, and can be reused for multiple requests.
27+
# After completing all of your requests, call the "__exit__()" method to safely
28+
# clean up any remaining background resources. Alternatively, use the client as
29+
# a context manager.
30+
with dataplex_v1.CatalogServiceClient() as client:
31+
# The resource name of the Entry Group
32+
parent = (
33+
f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}"
34+
)
35+
entry = dataplex_v1.Entry(
36+
# Example of system Entry Type.
37+
# It is also possible to specify custom Entry Type.
38+
entry_type="projects/dataplex-types/locations/global/entryTypes/generic",
39+
entry_source=dataplex_v1.EntrySource(
40+
description="description of the entry"
41+
),
42+
aspects={
43+
"dataplex-types.global.generic": dataplex_v1.Aspect(
44+
# This is required Aspect Type for "generic" Entry Type.
45+
# For custom Aspect Type required Entry Type would be different.
46+
aspect_type="projects/dataplex-types/locations/global/aspectTypes/generic",
47+
data=struct_pb2.Struct(
48+
fields={
49+
# "Generic" Aspect Type have fields called "type" and "system.
50+
# The values below are a sample of possible options.
51+
"type": struct_pb2.Value(string_value="example value"),
52+
"system": struct_pb2.Value(string_value="example system"),
53+
}
54+
),
55+
)
56+
},
57+
)
58+
return client.create_entry(parent=parent, entry=entry, entry_id=entry_id)
59+
60+
61+
if __name__ == "__main__":
62+
# TODO(developer): Replace these variables before running the sample.
63+
project_id = "MY_PROJECT_ID"
64+
# Available locations: https://cloud.google.com/dataplex/docs/locations
65+
location = "MY_LOCATION"
66+
entry_group_id = "MY_ENTRY_GROUP_ID"
67+
entry_id = "MY_ENTRY_ID"
68+
69+
created_entry = create_entry(project_id, location, entry_group_id, entry_id)
70+
print(f"Successfully created entry: {created_entry.name}")
71+
# [END dataplex_create_entry]

‎dataplex/snippets/delete_entry.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START dataplex_delete_entry]
16+
from google.cloud import dataplex_v1
17+
18+
19+
def delete_entry(
20+
project_id: str, location: str, entry_group_id: str, entry_id: str
21+
) -> None:
22+
"""Method to delete Entry located in project_id, location, entry_group_id and with entry_id"""
23+
24+
# Initialize client that will be used to send requests across threads. This
25+
# client only needs to be created once, and can be reused for multiple requests.
26+
# After completing all of your requests, call the "__exit__()" method to safely
27+
# clean up any remaining background resources. Alternatively, use the client as
28+
# a context manager.
29+
with dataplex_v1.CatalogServiceClient() as client:
30+
# The resource name of the Entry
31+
name = f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}/entries/{entry_id}"
32+
client.delete_entry(name=name)
33+
34+
35+
if __name__ == "__main__":
36+
# TODO(developer): Replace these variables before running the sample.
37+
project_id = "MY_PROJECT_ID"
38+
# Available locations: https://cloud.google.com/dataplex/docs/locations
39+
location = "MY_LOCATION"
40+
entry_group_id = "MY_ENTRY_TYPE_ID"
41+
entry_id = "MY_ENTRY_ID"
42+
43+
delete_entry(project_id, location, entry_group_id, entry_id)
44+
print("Successfully deleted entry")
45+
# [END dataplex_delete_entry]

‎dataplex/snippets/entry_test.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
import os
15+
16+
import uuid
17+
18+
from google.api_core.retry import Retry
19+
20+
import pytest
21+
22+
import create_entry
23+
import create_entry_group
24+
import delete_entry
25+
import delete_entry_group
26+
import get_entry
27+
import list_entries
28+
import lookup_entry
29+
import update_entry
30+
31+
PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
32+
LOCATION = "us-central1"
33+
ID = str(uuid.uuid4()).split("-")[0]
34+
ENTRY_GROUP_ID = f"test-entry-group-{ID}"
35+
ENTRY_ID = f"test-entry-{ID}"
36+
EXPECTED_ENTRY = f"projects/{PROJECT_ID}/locations/{LOCATION}/entryGroups/{ENTRY_GROUP_ID}/entries/{ENTRY_ID}"
37+
38+
39+
@pytest.fixture(autouse=True, scope="session")
40+
def setup_and_teardown_entry_group() -> None:
41+
try:
42+
# Create Entry Group resource that will be used for creating Entry
43+
create_entry_group.create_entry_group(PROJECT_ID, LOCATION, ENTRY_GROUP_ID)
44+
# Create Entry that will be used in tests for "get", "lookup", "list" and "update" methods
45+
create_entry.create_entry(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID)
46+
yield
47+
finally:
48+
# Clean-up Entry Group resource created above
49+
# Entry inside this Entry Group will be deleted automatically
50+
delete_entry_group.delete_entry_group(PROJECT_ID, LOCATION, ENTRY_GROUP_ID)
51+
52+
53+
@Retry()
54+
def test_list_entries() -> None:
55+
entries = list_entries.list_entries(PROJECT_ID, LOCATION, ENTRY_GROUP_ID)
56+
assert EXPECTED_ENTRY in [entry.name for entry in entries]
57+
58+
59+
@Retry()
60+
def test_get_entry() -> None:
61+
entry = get_entry.get_entry(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID)
62+
assert EXPECTED_ENTRY == entry.name
63+
64+
65+
@Retry()
66+
def test_lookup_entry() -> None:
67+
entry = lookup_entry.lookup_entry(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID)
68+
assert EXPECTED_ENTRY == entry.name
69+
70+
71+
@Retry()
72+
def test_update_entry() -> None:
73+
entry = update_entry.update_entry(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID)
74+
assert EXPECTED_ENTRY in entry.name
75+
76+
77+
@Retry()
78+
def test_create_entry() -> None:
79+
entry_id_to_create = f"test-entry-{str(uuid.uuid4()).split('-')[0]}"
80+
expected_entry_to_create = f"projects/{PROJECT_ID}/locations/{LOCATION}/entryGroups/{ENTRY_GROUP_ID}/entries/{entry_id_to_create}"
81+
try:
82+
entry = create_entry.create_entry(
83+
PROJECT_ID, LOCATION, ENTRY_GROUP_ID, entry_id_to_create
84+
)
85+
assert expected_entry_to_create == entry.name
86+
finally:
87+
# Clean-up created Entry
88+
delete_entry.delete_entry(
89+
PROJECT_ID, LOCATION, ENTRY_GROUP_ID, entry_id_to_create
90+
)
91+
92+
93+
@Retry()
94+
def test_delete_entry() -> None:
95+
entry_id_to_delete = f"test-entry-{str(uuid.uuid4()).split('-')[0]}"
96+
# Create Entry to be deleted
97+
create_entry.create_entry(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, entry_id_to_delete)
98+
# No exception means successful call
99+
delete_entry.delete_entry(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, entry_id_to_delete)

‎dataplex/snippets/get_entry.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START dataplex_get_entry]
16+
from google.cloud import dataplex_v1
17+
18+
19+
def get_entry(
20+
project_id: str, location: str, entry_group_id: str, entry_id: str
21+
) -> dataplex_v1.Entry:
22+
"""Method to retrieve Entry located in project_id, location, entry_group_id and with entry_id
23+
24+
When Entry is created in Dataplex for example for BigQuery table,
25+
access permissions might differ between Dataplex and source system.
26+
"Get" method checks permissions in Dataplex.
27+
Please also refer how to lookup an Entry, which checks permissions in source system.
28+
"""
29+
30+
# Initialize client that will be used to send requests across threads. This
31+
# client only needs to be created once, and can be reused for multiple requests.
32+
# After completing all of your requests, call the "__exit__()" method to safely
33+
# clean up any remaining background resources. Alternatively, use the client as
34+
# a context manager.
35+
with dataplex_v1.CatalogServiceClient() as client:
36+
# The resource name of the Entry
37+
name = f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}/entries/{entry_id}"
38+
get_entry_request = dataplex_v1.GetEntryRequest(
39+
name=name,
40+
# View determines which Aspects are returned with the Entry.
41+
# For all available options, see:
42+
# https://cloud.google.com/sdk/gcloud/reference/dataplex/entries/lookup#--view
43+
view=dataplex_v1.EntryView.FULL,
44+
# Following 2 lines will be ignored, because "View" is set to FULL.
45+
# Their purpose is to demonstrate how to filter the Aspects returned for Entry
46+
# when "View" is set to CUSTOM.
47+
aspect_types=[
48+
"projects/dataplex-types/locations/global/aspectTypes/generic"
49+
],
50+
paths=["my_path"],
51+
)
52+
return client.get_entry(request=get_entry_request)
53+
54+
55+
if __name__ == "__main__":
56+
# TODO(developer): Replace these variables before running the sample.
57+
project_id = "MY_PROJECT_ID"
58+
# Available locations: https://cloud.google.com/dataplex/docs/locations
59+
location = "MY_LOCATION"
60+
entry_group_id = "MY_ENTRY_GROUP_ID"
61+
entry_id = "MY_ENTRY_ID"
62+
63+
entry = get_entry(project_id, location, entry_group_id, entry_id)
64+
print(f"Entry retrieved successfully: {entry.name}")
65+
for aspect_key in entry.aspects.keys():
66+
print(f"Retrieved aspect for entry: {aspect_key}")
67+
# [END dataplex_get_entry]

‎dataplex/snippets/list_entries.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START dataplex_list_entries]
16+
from typing import List
17+
18+
from google.cloud import dataplex_v1
19+
20+
21+
def list_entries(
22+
project_id: str, location: str, entry_group_id: str
23+
) -> List[dataplex_v1.Entry]:
24+
"""Method to list Entries located in project_id, location and entry_group_id"""
25+
26+
# Initialize client that will be used to send requests across threads. This
27+
# client only needs to be created once, and can be reused for multiple requests.
28+
# After completing all of your requests, call the "__exit__()" method to safely
29+
# clean up any remaining background resources. Alternatively, use the client as
30+
# a context manager.
31+
with dataplex_v1.CatalogServiceClient() as client:
32+
# The resource name of the Entries location
33+
parent = (
34+
f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}"
35+
)
36+
list_entries_request = dataplex_v1.ListEntriesRequest(
37+
parent=parent,
38+
# A filter on the entries to return. Filters are case-sensitive.
39+
# You can filter the request by the following fields:
40+
# * entry_type
41+
# * entry_source.display_name
42+
# To learn more about filters in general, see:
43+
# https://cloud.google.com/sdk/gcloud/reference/topic/filters
44+
filter="entry_type=projects/dataplex-types/locations/global/entryTypes/generic",
45+
)
46+
47+
results = client.list_entries(request=list_entries_request)
48+
return list(results)
49+
50+
51+
if __name__ == "__main__":
52+
# TODO(developer): Replace these variables before running the sample.
53+
project_id = "MY_PROJECT_ID"
54+
# Available locations: https://cloud.google.com/dataplex/docs/locations
55+
location = "MY_LOCATION"
56+
entry_group_id = "MY_ENTRY_GROUP_ID"
57+
58+
entries = list_entries(project_id, location, entry_group_id)
59+
for entry in entries:
60+
print(f"Entry name: {entry.name}")
61+
# [END dataplex_list_entries]

0 commit comments

Comments
 (0)