From 6c42c811e9155258d485682b71adf9f0e8b9bc0e Mon Sep 17 00:00:00 2001 From: janmatzek Date: Fri, 29 Aug 2025 16:22:17 +0200 Subject: [PATCH 1/3] feat(gooddata-pipelines): standardize provisioning input models --- gooddata-pipelines/TODO.md | 2 +- .../gooddata_pipelines/__init__.py | 2 + .../gooddata_pipelines/api/gooddata_api.py | 51 ----- .../entities/users/models/permissions.py | 102 +++------- .../entities/users/models/user_groups.py | 73 +++---- .../entities/users/models/users.py | 58 +----- .../entities/users/permissions.py | 15 +- .../entities/users/user_groups.py | 3 +- .../provisioning/entities/users/users.py | 3 +- .../entities/workspaces/models.py | 31 +-- .../entities/workspaces/workspace.py | 3 +- .../provisioning/provisioning.py | 6 +- .../provisioning/utils/context_objects.py | 12 +- .../provisioning/utils/utils.py | 18 +- .../permissions_input_full_load.json | 28 +-- .../permissions_input_incremental_load.json | 35 ++-- .../entities/users/users_input_full_load.json | 6 +- .../users/users_input_incremental_load.json | 8 +- .../entities/users/test_permissions.py | 182 ++++++++---------- .../entities/users/test_user_groups.py | 146 +++++--------- .../provisioning/entities/users/test_users.py | 8 +- 21 files changed, 284 insertions(+), 508 deletions(-) diff --git a/gooddata-pipelines/TODO.md b/gooddata-pipelines/TODO.md index ef51a2c4a..e6c5e9a64 100644 --- a/gooddata-pipelines/TODO.md +++ b/gooddata-pipelines/TODO.md @@ -10,7 +10,7 @@ A list of outstanding tasks, features, or technical debt to be addressed in this - [ ] Integrate with GoodDataApiClient - [ ] Consider replacing the SdkMethods wrapper with direct calls to the SDK methods -- [ ] Consider using orjson library instead of json +- [ ] Consider using orjson library instead of json to load test data - [ ] Cleanup custom exceptions - [ ] Improve test coverage. Write missing unit tests for legacy code (e.g., user data filters) diff --git a/gooddata-pipelines/gooddata_pipelines/__init__.py b/gooddata-pipelines/gooddata_pipelines/__init__.py index bbaa5e495..5aa82737d 100644 --- a/gooddata-pipelines/gooddata_pipelines/__init__.py +++ b/gooddata-pipelines/gooddata_pipelines/__init__.py @@ -19,6 +19,7 @@ UserDataFilterProvisioner, ) from .provisioning.entities.users.models.permissions import ( + EntityType, PermissionFullLoad, PermissionIncrementalLoad, ) @@ -55,5 +56,6 @@ "PermissionProvisioner", "UserDataFilterProvisioner", "UserDataFilterFullLoad", + "EntityType", "__version__", ] diff --git a/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py index 89197750f..a2563422c 100644 --- a/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py +++ b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py @@ -55,42 +55,6 @@ def _get_url(self, endpoint: str) -> str: """ return f"{self.base_url}{endpoint}" - def get_custom_application_setting( - self, workspace_id: str, setting_id: str - ) -> requests.Response: - """Gets a custom application setting. - - Args: - workspace_id (str): The ID of the workspace. - setting_id (str): The ID of the custom application setting. - Returns: - requests.Response: The response from the server containing the - custom application setting. - """ - url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}" - return self._get(url) - - def put_custom_application_setting( - self, workspace_id: str, setting_id: str, data: dict[str, Any] - ) -> requests.Response: - url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}" - return self._put(url, data, self.headers) - - def post_custom_application_setting( - self, workspace_id: str, data: dict[str, Any] - ) -> requests.Response: - """Creates a custom application setting for a given workspace. - - Args: - workspace_id (str): The ID of the workspace. - data (dict[str, Any]): The data for the custom application setting. - Returns: - requests.Response: The response from the server containing the - created custom application setting. - """ - url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/" - return self._post(url, data, self.headers) - def get_all_workspace_data_filters( self, workspace_id: str ) -> requests.Response: @@ -201,21 +165,6 @@ def delete_workspace_data_filter_setting( endpoint, ) - def post_workspace_data_filter( - self, workspace_id: str, data: dict[str, Any] - ) -> requests.Response: - """Creates a workspace data filter for a given workspace. - - Args: - workspace_id (str): The ID of the workspace. - data (dict[str, Any]): The data for the workspace data filter. - Returns: - requests.Response: The response from the server containing the - created workspace data filter. - """ - endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilters" - return self._post(endpoint, data, self.headers) - def get_user_data_filters(self, workspace_id: str) -> requests.Response: """Gets the user data filters for a given workspace.""" endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters" diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py index 7eccc2331..dcd42bd2d 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py @@ -1,7 +1,7 @@ # (C) 2025 GoodData Corporation -from abc import abstractmethod + from enum import Enum -from typing import Any, Iterator, TypeAlias, TypeVar +from typing import Iterator, TypeAlias import attrs from gooddata_sdk.catalog.identifier import CatalogAssigneeIdentifier @@ -14,85 +14,29 @@ from gooddata_pipelines.provisioning.utils.exceptions import BaseUserException TargetsPermissionDict: TypeAlias = dict[str, dict[str, bool]] -ConstructorType = TypeVar("ConstructorType", bound="ConstructorMixin") -class PermissionType(str, Enum): +class EntityType(str, Enum): # NOTE: Start using StrEnum with Python 3.11 user = "user" user_group = "userGroup" -class ConstructorMixin: - @staticmethod - def _get_id_and_type( - permission: dict[str, Any], - ) -> tuple[str, PermissionType]: - user_id: str | None = permission.get("user_id") - user_group_id: str | None = permission.get("ug_id") - if user_id and user_group_id: - raise ValueError("Only one of user_id or ug_id must be present") - elif user_id: - return user_id, PermissionType.user - elif user_group_id: - return user_group_id, PermissionType.user_group - else: - raise ValueError("Either user_id or ug_id must be present") - - @classmethod - def from_list_of_dicts( - cls: type[ConstructorType], data: list[dict[str, Any]] - ) -> list[ConstructorType]: - """Creates a list of instances from list of dicts.""" - # NOTE: We can use typing.Self for the return type in Python 3.11 - permissions = [] - for permission in data: - permissions.append(cls.from_dict(permission)) - return permissions - - @classmethod - @abstractmethod - def from_dict(cls, data: dict[str, Any]) -> Any: - """Construction form a dictionary to be implemented by subclasses.""" - pass - - -class PermissionIncrementalLoad(BaseModel, ConstructorMixin): +class BasePermission(BaseModel): permission: str workspace_id: str - id_: str - type_: PermissionType - is_active: bool + entity_id: str + entity_type: EntityType - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "PermissionIncrementalLoad": - """Returns an instance of PermissionIncrementalLoad from a dictionary.""" - id_, target_type = cls._get_id_and_type(data) - return cls( - permission=data["ws_permissions"], - workspace_id=data["ws_id"], - id_=id_, - type_=target_type, - is_active=data["is_active"], - ) +class PermissionFullLoad(BasePermission): + """Input validator for full load of workspace permissions provisioning.""" -class PermissionFullLoad(BaseModel, ConstructorMixin): - permission: str - workspace_id: str - id_: str - type_: PermissionType - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "PermissionFullLoad": - """Returns an instance of PermissionFullLoad from a dictionary.""" - id_, target_type = cls._get_id_and_type(data) - return cls( - permission=data["ws_permissions"], - workspace_id=data["ws_id"], - id_=id_, - type_=target_type, - ) +class PermissionIncrementalLoad(BasePermission): + """Input validator for incremental load of workspace permissions provisioning.""" + + is_active: bool @attrs.define @@ -117,7 +61,7 @@ def from_sdk_api( permission.assignee.id, ) - if permission_type == PermissionType.user.value: + if permission_type == EntityType.user.value: target_dict = users else: target_dict = user_groups @@ -170,7 +114,7 @@ def to_sdk_api(self) -> CatalogDeclarativeWorkspacePermissions: for user_id, permissions in self.users.items(): assignee = CatalogAssigneeIdentifier( - id=user_id, type=PermissionType.user.value + id=user_id, type=EntityType.user.value ) for declaration in self._permissions_for_target( permissions, assignee @@ -179,7 +123,7 @@ def to_sdk_api(self) -> CatalogDeclarativeWorkspacePermissions: for ug_id, permissions in self.user_groups.items(): assignee = CatalogAssigneeIdentifier( - id=ug_id, type=PermissionType.user_group.value + id=ug_id, type=EntityType.user_group.value ) for declaration in self._permissions_for_target( permissions, assignee @@ -200,15 +144,15 @@ def add_incremental_permission( """ target_dict = ( self.users - if permission.type_ == PermissionType.user + if permission.entity_type == EntityType.user else self.user_groups ) - if permission.id_ not in target_dict: - target_dict[permission.id_] = {} + if permission.entity_id not in target_dict: + target_dict[permission.entity_id] = {} is_active = permission.is_active - target_permissions = target_dict[permission.id_] + target_permissions = target_dict[permission.entity_id] permission_value = permission.permission if permission_value not in target_permissions: @@ -233,14 +177,14 @@ def add_full_load_permission(self, permission: PermissionFullLoad) -> None: """ target_dict = ( self.users - if permission.type_ == PermissionType.user + if permission.entity_type == EntityType.user else self.user_groups ) - if permission.id_ not in target_dict: - target_dict[permission.id_] = {} + if permission.entity_id not in target_dict: + target_dict[permission.entity_id] = {} - target_permissions = target_dict[permission.id_] + target_permissions = target_dict[permission.entity_id] permission_value = permission.permission if permission_value not in target_permissions: diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py index 015b7aff9..792431673 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py @@ -1,64 +1,37 @@ # (C) 2025 GoodData Corporation -from typing import Any +from pydantic import BaseModel, Field, ValidationInfo, field_validator -from pydantic import BaseModel -from gooddata_pipelines.provisioning.utils.utils import SplitMixin - - -class BaseUserGroup(BaseModel, SplitMixin): +class UserGroupBase(BaseModel): user_group_id: str user_group_name: str - parent_user_groups: list[str] + parent_user_groups: list[str] = Field(default_factory=list) + @field_validator("user_group_name", mode="before") @classmethod - def _create_from_dict_data( - cls, user_group_data: dict[str, Any], delimiter: str = "," - ) -> dict[str, Any]: - """Helper method to extract common data from dict.""" - parent_user_groups = cls.split( - user_group_data["parent_user_groups"], delimiter=delimiter - ) - user_group_name = user_group_data["user_group_name"] - if not user_group_name: - user_group_name = user_group_data["user_group_id"] - - return { - "user_group_id": user_group_data["user_group_id"], - "user_group_name": user_group_name, - "parent_user_groups": parent_user_groups, - } - - -class UserGroupIncrementalLoad(BaseUserGroup): - is_active: bool - + def validate_user_group_name( + cls, v: str | None, info: ValidationInfo + ) -> str: + """If user_group_name is None or empty, default to user_group_id.""" + if not v: # handles None and empty string + return info.data.get("user_group_id", "") + return v + + @field_validator("parent_user_groups", mode="before") @classmethod - def from_list_of_dicts( - cls, data: list[dict[str, Any]], delimiter: str = "," - ) -> list["UserGroupIncrementalLoad"]: - """Creates a list of User objects from list of dicts.""" - user_groups = [] - for user_group in data: - base_data = cls._create_from_dict_data(user_group, delimiter) - base_data["is_active"] = user_group["is_active"] + def validate_parent_user_groups(cls, v: list[str] | None) -> list[str]: + """If parent_user_groups is None or empty, default to empty list.""" + if not v: + return [] + return v - user_groups.append(UserGroupIncrementalLoad(**base_data)) - return user_groups +class UserGroupFullLoad(UserGroupBase): + """Input validator for full load of user group provisioning.""" -class UserGroupFullLoad(BaseUserGroup): - @classmethod - def from_list_of_dicts( - cls, data: list[dict[str, Any]], delimiter: str = "," - ) -> list["UserGroupFullLoad"]: - """Creates a list of User objects from list of dicts.""" - user_groups = [] - for user_group in data: - base_data = cls._create_from_dict_data(user_group, delimiter) +class UserGroupIncrementalLoad(UserGroupBase): + """Input validator for incremental load of user group provisioning.""" - user_groups.append(UserGroupFullLoad(**base_data)) - - return user_groups + is_active: bool diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py index b0278341f..a29943f77 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py @@ -5,10 +5,8 @@ from gooddata_sdk.catalog.user.entity_model.user import CatalogUser from pydantic import BaseModel -from gooddata_pipelines.provisioning.utils.utils import SplitMixin - -class BaseUser(BaseModel, SplitMixin): +class BaseUser(BaseModel): """Base class containing shared user fields and functionality.""" user_id: str @@ -18,21 +16,6 @@ class BaseUser(BaseModel, SplitMixin): auth_id: str | None user_groups: list[str] - @classmethod - def _create_from_dict_data( - cls, user_data: dict[str, Any], delimiter: str = "," - ) -> dict[str, Any]: - """Helper method to extract common data from dict.""" - user_groups = cls.split(user_data["user_groups"], delimiter=delimiter) - return { - "user_id": user_data["user_id"], - "firstname": user_data["firstname"], - "lastname": user_data["lastname"], - "email": user_data["email"], - "auth_id": user_data["auth_id"], - "user_groups": user_groups, - } - @classmethod def _create_from_sdk_data(cls, obj: CatalogUser) -> dict[str, Any]: """Helper method to extract common data from SDK object.""" @@ -68,47 +51,24 @@ def to_sdk_obj(self) -> CatalogUser: ) -class UserIncrementalLoad(BaseUser): - """User model for incremental load operations with active status tracking.""" - - is_active: bool - - @classmethod - def from_list_of_dicts( - cls, data: list[dict[str, Any]], delimiter: str = "," - ) -> list["UserIncrementalLoad"]: - """Creates a list of User objects from list of dicts.""" - converted_users = [] - for user in data: - base_data = cls._create_from_dict_data(user, delimiter) - base_data["is_active"] = user["is_active"] - converted_users.append(cls(**base_data)) - return converted_users +class UserFullLoad(BaseUser): + """Input validator for full load of user provisioning.""" @classmethod - def from_sdk_obj(cls, obj: CatalogUser) -> "UserIncrementalLoad": + def from_sdk_obj(cls, obj: CatalogUser) -> "UserFullLoad": """Creates GDUserTarget from CatalogUser SDK object.""" base_data = cls._create_from_sdk_data(obj) - base_data["is_active"] = True return cls(**base_data) -class UserFullLoad(BaseUser): - """User model for full load operations.""" +class UserIncrementalLoad(BaseUser): + """Input validator for incremental load of user provisioning.""" - @classmethod - def from_list_of_dicts( - cls, data: list[dict[str, Any]], delimiter: str = "," - ) -> list["UserFullLoad"]: - """Creates a list of User objects from list of dicts.""" - converted_users = [] - for user in data: - base_data = cls._create_from_dict_data(user, delimiter) - converted_users.append(cls(**base_data)) - return converted_users + is_active: bool @classmethod - def from_sdk_obj(cls, obj: CatalogUser) -> "UserFullLoad": + def from_sdk_obj(cls, obj: CatalogUser) -> "UserIncrementalLoad": """Creates GDUserTarget from CatalogUser SDK object.""" base_data = cls._create_from_sdk_data(obj) + base_data["is_active"] = True return cls(**base_data) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py index b4fad7db3..5246ad177 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py @@ -6,10 +6,10 @@ from gooddata_pipelines.api.exceptions import GoodDataApiException from gooddata_pipelines.provisioning.entities.users.models.permissions import ( + EntityType, PermissionDeclaration, PermissionFullLoad, PermissionIncrementalLoad, - PermissionType, TargetsPermissionDict, WSPermissionsDeclarations, ) @@ -28,7 +28,8 @@ class PermissionProvisioner( """Provisioning class for user permissions in GoodData workspaces. This class handles the provisioning of user permissions based on the provided - source data. + source data. Use the `full_load` or `incremental_load` + methods to run the provisioning. """ source_group_incremental: list[PermissionIncrementalLoad] @@ -47,7 +48,7 @@ def _get_ws_declaration(self, ws_id: str) -> PermissionDeclaration: ) target_dict = ( users - if permission_type == PermissionType.user.value + if permission_type == EntityType.user.value else user_groups ) @@ -105,11 +106,13 @@ def _validate_permission( self, permission: PermissionFullLoad | PermissionIncrementalLoad ) -> None: """Validates if the permission is correctly defined.""" - if permission.type_ == PermissionType.user: - self._api.get_user(permission.id_, error_message="User not found") + if permission.entity_type == EntityType.user: + self._api.get_user( + permission.entity_id, error_message="User not found" + ) else: self._api.get_user_group( - permission.id_, error_message="User group not found" + permission.entity_id, error_message="User group not found" ) self._api.get_workspace( diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py index 44463f69b..db1d7d337 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py @@ -21,7 +21,8 @@ class UserGroupProvisioner( """Provisioning class for user groups in GoodData workspaces. This class handles the creation, update, and deletion of user groups - based on the provided source data. + based on the provided source data. Use the `full_load` or `incremental_load` + methods to run the provisioning. """ source_group_incremental: list[UserGroupIncrementalLoad] diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py index e443bcfcc..30fad3964 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py @@ -24,7 +24,8 @@ class UserProvisioner(Provisioning[UserFullLoad, UserIncrementalLoad]): """Provisioning class for users in GoodData workspaces. This class handles the creation, update, and deletion of users - based on the provided source data. + based on the provided source data. Use the `full_load` or `incremental_load` + methods to run the provisioning. """ source_group_incremental: list[UserIncrementalLoad] diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py index 788c7b44b..6c7deb52c 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py @@ -1,29 +1,27 @@ # (C) 2025 GoodData Corporation """Module containing models related to workspace provisioning in GoodData Cloud.""" -from dataclasses import dataclass, field from typing import Literal +import attrs from pydantic import BaseModel, ConfigDict -@dataclass +@attrs.define class WorkspaceDataMaps: """Dataclass to hold various mappings related to workspace data.""" - child_to_parent_id_map: dict[str, str] = field(default_factory=dict) - workspace_id_to_wdf_map: dict[str, dict[str, list[str]]] = field( - default_factory=dict + child_to_parent_id_map: dict[str, str] = attrs.field(factory=dict) + workspace_id_to_wdf_map: dict[str, dict[str, list[str]]] = attrs.field( + factory=dict ) - parent_ids: set[str] = field(default_factory=set) - source_ids: set[str] = field(default_factory=set) - workspace_id_to_name_map: dict[str, str] = field(default_factory=dict) - upstream_ids: set[str] = field(default_factory=set) + parent_ids: set[str] = attrs.field(factory=set) + source_ids: set[str] = attrs.field(factory=set) + workspace_id_to_name_map: dict[str, str] = attrs.field(factory=dict) + upstream_ids: set[str] = attrs.field(factory=set) -class WorkspaceFullLoad(BaseModel): - """Model representing input for provisioning of workspaces in GoodData Cloud.""" - +class WorkspaceBase(BaseModel): model_config = ConfigDict(coerce_numbers_to_str=True) parent_id: str @@ -33,10 +31,13 @@ class WorkspaceFullLoad(BaseModel): workspace_data_filter_values: list[str] | None = None -class WorkspaceIncrementalLoad(WorkspaceFullLoad): - """Model representing input for incremental provisioning of workspaces in GoodData Cloud.""" +class WorkspaceFullLoad(WorkspaceBase): + """Input validator for full load of workspace provisioning.""" + + +class WorkspaceIncrementalLoad(WorkspaceBase): + """Input validator for incremental load of workspace provisioning.""" - # TODO: double check that the model loads the data correctly, write a test is_active: bool diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py index 7324d41b0..f359059b9 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py @@ -39,7 +39,8 @@ def __init__(self, *args: str, **kwargs: str) -> None: """Creates an instance of the WorkspaceProvisioner. Calls the superclass constructor and initializes the validator, parser, - and maps for workspace data. + and maps for workspace data. Use the `full_load` or `incremental_load` + methods to run the provisioning. """ super().__init__(*args, **kwargs) self.validator: WorkspaceDataValidator = WorkspaceDataValidator( diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py index 3c24991e7..cc3c8fe9e 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py @@ -100,7 +100,8 @@ def full_load(self, source_data: list[TFullLoadSourceData]) -> None: That means: - All workspaces declared in the source data are created if missing, or updated to match the source data - - All workspaces not declared in the source data are deleted + - All child workspaces not declared under the parent workspace in the + source data are deleted """ self.source_group_full = source_data @@ -116,7 +117,8 @@ def incremental_load( """Runs incremental provisioning workflow with the provided source data. Incremental provisioning is used to modify a subset of the upstream workspaces - based on the source data provided. + based on the source data provided. Only changes requested in the source + data will be applied. """ # TODO: validate the data type of source group at runtime self.source_group_incremental = source_data diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py index b54894ac4..ebc37ac5c 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py @@ -16,10 +16,10 @@ def __init__( wdf_id: str | None = None, wdf_values: list[str] | None = None, ): - self.workspace_id: str = workspace_id if workspace_id else "NA" - self.workspace_name: str | None = workspace_name - self.wdf_id: str | None = wdf_id - self.wdf_values: list[str] | None = wdf_values + self.workspace_id = workspace_id if workspace_id else "NA" + self.workspace_name = workspace_name + self.wdf_id = wdf_id + self.wdf_values = wdf_values class UserContext: @@ -28,5 +28,5 @@ class UserContext: def __init__(self, user_id: str, user_groups: list[str]): """User context object, stringifies list of user groups""" - self.user_id: str = user_id - self.user_groups: str = ",".join(user_groups) + self.user_id = user_id + self.user_groups = ",".join(user_groups) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py index 2050fb6aa..4cc44be58 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py @@ -2,7 +2,7 @@ """Module for utilities used in GoodData Pipelines provisioning.""" -from pydantic import BaseModel +import attrs from requests import Response @@ -61,20 +61,8 @@ def get_attrs( return attrs -class SplitMixin: - @staticmethod - def split(string_value: str, delimiter: str = ",") -> list[str]: - """ - Splits a string by the given delimiter and returns a list of stripped values. - If the input is empty, returns an empty list. - """ - if not string_value: - return [] - - return [value.strip() for value in string_value.split(delimiter)] - - -class EntityGroupIds(BaseModel): +@attrs.define +class EntityGroupIds: ids_in_both_systems: set[str] ids_to_delete: set[str] ids_to_create: set[str] diff --git a/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_full_load.json b/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_full_load.json index c899e3213..812e6f92b 100644 --- a/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_full_load.json +++ b/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_full_load.json @@ -1,22 +1,26 @@ [ { - "user_id": "user_1", - "ws_id": "child_workspace_id_1", - "ws_permissions": "ANALYZE" + "entity_id": "user_1", + "entity_type": "user", + "workspace_id": "child_workspace_id_1", + "permission": "ANALYZE" }, { - "user_id": "user_2", - "ws_id": "child_workspace_id_1", - "ws_permissions": "VIEW" + "entity_id": "user_2", + "entity_type": "user", + "workspace_id": "child_workspace_id_1", + "permission": "VIEW" }, { - "user_id": "user_3", - "ws_id": "child_workspace_id_2", - "ws_permissions": "MANAGE" + "entity_id": "user_3", + "entity_type": "user", + "workspace_id": "child_workspace_id_2", + "permission": "MANAGE" }, { - "user_id": "user_4", - "ws_id": "child_workspace_id_2", - "ws_permissions": "ANALYZE" + "entity_id": "user_4", + "entity_type": "user", + "workspace_id": "child_workspace_id_2", + "permission": "ANALYZE" } ] diff --git a/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_incremental_load.json b/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_incremental_load.json index 3f1d765de..3c5eab124 100644 --- a/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_incremental_load.json +++ b/gooddata-pipelines/tests/data/provisioning/entities/permissions/permissions_input_incremental_load.json @@ -1,32 +1,37 @@ [ { - "user_id": "user_1", - "ws_id": "child_workspace_id_1", - "ws_permissions": "ANALYZE", + "entity_id": "user_1", + "entity_type": "user", + "workspace_id": "child_workspace_id_1", + "permission": "ANALYZE", "is_active": true }, { - "user_id": "user_2", - "ws_id": "child_workspace_id_1", - "ws_permissions": "VIEW", + "entity_id": "user_2", + "entity_type": "user", + "workspace_id": "child_workspace_id_1", + "permission": "VIEW", "is_active": true }, { - "user_id": "user_2", - "ws_id": "child_workspace_id_1", - "ws_permissions": "VIEW", + "entity_id": "user_2", + "entity_type": "user", + "workspace_id": "child_workspace_id_1", + "permission": "VIEW", "is_active": false }, { - "user_id": "user_3", - "ws_id": "child_workspace_id_2", - "ws_permissions": "MANAGE", + "entity_id": "user_3", + "entity_type": "user", + "workspace_id": "child_workspace_id_2", + "permission": "MANAGE", "is_active": true }, { - "user_id": "user_4", - "ws_id": "child_workspace_id_2", - "ws_permissions": "ANALYZE", + "entity_id": "user_4", + "entity_type": "user", + "workspace_id": "child_workspace_id_2", + "permission": "ANALYZE", "is_active": true } ] diff --git a/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_full_load.json b/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_full_load.json index e92483a72..c92afb293 100644 --- a/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_full_load.json +++ b/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_full_load.json @@ -5,7 +5,7 @@ "lastname": "Doe", "email": "john.doe@example.com", "auth_id": "auth_1", - "user_groups": "group_1,group_2" + "user_groups": ["group_1", "group_2"] }, { "user_id": "user_2", @@ -13,7 +13,7 @@ "lastname": "Doe", "email": "jane.doe@example.com", "auth_id": "auth_2", - "user_groups": "group_2,group_3" + "user_groups": ["group_2", "group_3"] }, { "user_id": "user_3", @@ -21,6 +21,6 @@ "lastname": "Rock", "email": "jim.rock@example.com", "auth_id": "auth_3", - "user_groups": "group_3,group_4" + "user_groups": ["group_3", "group_4"] } ] diff --git a/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_incremental_load.json b/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_incremental_load.json index 670867780..1b2f3f7b5 100644 --- a/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_incremental_load.json +++ b/gooddata-pipelines/tests/data/provisioning/entities/users/users_input_incremental_load.json @@ -5,7 +5,7 @@ "lastname": "Doe", "email": "john.doe@example.com", "auth_id": "auth_1", - "user_groups": "group_1,group_2", + "user_groups": ["group_1", "group_2"], "is_active": true }, { @@ -14,7 +14,7 @@ "lastname": "Doe", "email": "jane.doe@example.com", "auth_id": "auth_2", - "user_groups": "group_2,group_3", + "user_groups": ["group_2", "group_3"], "is_active": true }, { @@ -23,7 +23,7 @@ "lastname": "Rock", "email": "jim.rock@example.com", "auth_id": "auth_3", - "user_groups": "group_3,group_4", + "user_groups": ["group_3", "group_4"], "is_active": true }, { @@ -32,7 +32,7 @@ "lastname": "Cliff", "email": "jack.cliff@example.com", "auth_id": "auth_4", - "user_groups": "group_4,group_5", + "user_groups": ["group_4", "group_5"], "is_active": false } ] diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py b/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py index d24d466ba..b30b8bb73 100644 --- a/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py @@ -14,6 +14,7 @@ from pytest_mock import MockerFixture from gooddata_pipelines.provisioning.entities.users.models.permissions import ( + EntityType, PermissionDeclaration, PermissionFullLoad, PermissionIncrementalLoad, @@ -147,15 +148,14 @@ def test_add_new_active_user_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "MANAGE", - "ug_id": "", - "user_id": "user_1", - "is_active": True, - } + permission = PermissionIncrementalLoad( + permission="MANAGE", + workspace_id="", + entity_id="user_1", + is_active=True, + entity_type=EntityType.user, ) + declaration.add_incremental_permission(permission) assert declaration.users == { "user_1": {"ANALYZE": True, "VIEW": False, "MANAGE": True} @@ -168,14 +168,12 @@ def test_add_new_inactive_user_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "MANAGE", - "ug_id": "", - "user_id": "user_1", - "is_active": False, - } + permission = PermissionIncrementalLoad( + permission="MANAGE", + workspace_id="", + entity_id="user_1", + entity_type=EntityType.user, + is_active=False, ) declaration.add_incremental_permission(permission) @@ -190,15 +188,14 @@ def test_overwrite_inactive_user_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "VIEW", - "ug_id": "", - "user_id": "user_1", - "is_active": True, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="VIEW", + entity_id="user_1", + entity_type=EntityType.user, + is_active=True, ) + declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": True}} assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} @@ -209,15 +206,14 @@ def test_overwrite_active_user_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "ANALYZE", - "ug_id": "", - "user_id": "user_1", - "is_active": False, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="ANALYZE", + entity_id="user_1", + entity_type=EntityType.user, + is_active=False, ) + declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} @@ -228,15 +224,15 @@ def test_add_new_user_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "VIEW", - "ug_id": "", - "user_id": "user_2", - "is_active": True, - } + + permission = PermissionIncrementalLoad( + workspace_id="", + permission="VIEW", + entity_id="user_2", + entity_type=EntityType.user, + is_active=True, ) + declaration.add_incremental_permission(permission) assert declaration.users == { "user_1": {"ANALYZE": True, "VIEW": False}, @@ -250,14 +246,12 @@ def test_modify_one_of_user_perms() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}, "user_2": {"VIEW": True}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "MANAGE", - "ug_id": "", - "user_id": "user_1", - "is_active": True, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="MANAGE", + entity_id="user_1", + entity_type=EntityType.user, + is_active=True, ) declaration.add_incremental_permission(permission) assert declaration.users == { @@ -275,14 +269,12 @@ def test_add_new_active_ug_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "MANAGE", - "ug_id": "ug_1", - "user_id": "", - "is_active": True, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="MANAGE", + entity_id="ug_1", + entity_type=EntityType.user_group, + is_active=True, ) declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} @@ -296,14 +288,12 @@ def test_add_new_inactive_ug_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "MANAGE", - "ug_id": "ug_1", - "user_id": "", - "is_active": False, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="MANAGE", + entity_id="ug_1", + entity_type=EntityType.user_group, + is_active=False, ) declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} @@ -317,14 +307,12 @@ def test_overwrite_inactive_ug_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "ANALYZE", - "ug_id": "ug_1", - "user_id": "", - "is_active": True, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="ANALYZE", + entity_id="ug_1", + entity_type=EntityType.user_group, + is_active=True, ) declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} @@ -336,14 +324,12 @@ def test_overwrite_active_ug_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "VIEW", - "ug_id": "ug_1", - "user_id": "", - "is_active": False, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="VIEW", + entity_id="ug_1", + entity_type=EntityType.user_group, + is_active=True, ) declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} @@ -355,14 +341,12 @@ def test_add_new_ug_perm() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "VIEW", - "ug_id": "ug_2", - "user_id": "", - "is_active": True, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="VIEW", + entity_id="ug_2", + entity_type=EntityType.user_group, + is_active=True, ) declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} @@ -377,14 +361,12 @@ def test_modify_one_of_ug_perms() -> None: {"user_1": {"ANALYZE": True, "VIEW": False}}, {"ug_1": {"VIEW": True, "ANALYZE": False}, "ug_2": {"VIEW": True}}, ) - permission = PermissionIncrementalLoad.from_dict( - { - "ws_id": "", - "ws_permissions": "MANAGE", - "ug_id": "ug_1", - "user_id": "", - "is_active": True, - } + permission = PermissionIncrementalLoad( + workspace_id="", + permission="MANAGE", + entity_id="ug_1", + entity_type=EntityType.user_group, + is_active=True, ) declaration.add_incremental_permission(permission) assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} @@ -540,10 +522,10 @@ def compare_permissions( ) if load_method == "incremental_load": - incremental_load_data = PermissionIncrementalLoad.from_list_of_dicts( - source_data - ) + incremental_load_data = [ + PermissionIncrementalLoad(**row) for row in source_data + ] permission_provisioner.incremental_load(incremental_load_data) else: - full_load_data = PermissionFullLoad.from_list_of_dicts(source_data) + full_load_data = [PermissionFullLoad(**row) for row in source_data] permission_provisioner.full_load(full_load_data) diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py b/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py index d23d8f1ce..f02a8b934 100644 --- a/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py @@ -1,119 +1,79 @@ # (C) 2025 GoodData Corporation -from dataclasses import dataclass -from unittest import mock import pytest -from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup from gooddata_pipelines.provisioning.entities.users.models.user_groups import ( UserGroupIncrementalLoad, ) -TEST_CSV_PATH = "tests/data/user_group_mgmt/input.csv" +# TODO: Test business logic -@dataclass -class MockUserGroup: - id: str - name: str - parent_ids: list[str] - - def to_sdk(self): - return CatalogUserGroup.init( - user_group_id=self.id, - user_group_name=self.name, - user_group_parent_ids=self.parent_ids, - ) +def test_missing_key_no_parent_groups() -> None: + # NOTE: Type ignore because of the missing parent_user_groups key. In Python + # 3.11, we can use NotRequired to make the key optional. + result = UserGroupIncrementalLoad( + user_group_id="ug_2", user_group_name="Developers", is_active=True + ) + expected = UserGroupIncrementalLoad( + user_group_id="ug_2", + user_group_name="Developers", + parent_user_groups=[], + is_active=True, + ) -def test_from_csv_row_standard(): - input = [ - { - "user_group_id": "ug_1", - "user_group_name": "Admins", - "parent_user_groups": "ug_2|ug_3", - "is_active": "True", - } - ] - result = UserGroupIncrementalLoad.from_list_of_dicts(input, "|") - expected = [ - UserGroupIncrementalLoad( - user_group_id="ug_1", - user_group_name="Admins", - parent_user_groups=["ug_2", "ug_3"], - is_active=True, - ) - ] - assert result == expected, "Standard row should be parsed correctly" - - -def test_from_csv_row_no_parent_groups(): - input = [ - { - "user_group_id": "ug_2", - "user_group_name": "Developers", - "parent_user_groups": "", - "is_active": "True", - } - ] - result = UserGroupIncrementalLoad.from_list_of_dicts(input, "|") - expected = [ - UserGroupIncrementalLoad( - user_group_id="ug_2", - user_group_name="Developers", - parent_user_groups=[], - is_active=True, - ) - ] assert result == expected, ( "Row without parent user groups should be parsed correctly" ) -def test_from_csv_row_fallback_name(): - input = [ - { - "user_group_id": "ug_3", - "user_group_name": "", - "parent_user_groups": "", - "is_active": "False", - } - ] - result = UserGroupIncrementalLoad.from_list_of_dicts(input, "|") - expected = [ - UserGroupIncrementalLoad( - user_group_id="ug_3", - user_group_name="ug_3", - parent_user_groups=[], - is_active=False, - ) - ] +@pytest.mark.parametrize("invalid_name", [None, ""]) +def test_fallback_name(invalid_name) -> None: + result = UserGroupIncrementalLoad( + user_group_id="ug_3", + user_group_name=invalid_name, + parent_user_groups=[], + is_active=False, + ) + expected = UserGroupIncrementalLoad( + user_group_id="ug_3", + user_group_name="ug_3", + parent_user_groups=[], + is_active=False, + ) + assert result == expected, ( "Row with empty name should fallback to user group ID" ) -def test_from_csv_row_invalid_is_active(): - input = [ - { - "user_group_id": "ug_4", - "user_group_name": "Testers", - "parent_user_groups": "ug_1", - "is_active": "not_a_boolean", - } - ] - with pytest.raises(ValueError): - UserGroupIncrementalLoad.from_list_of_dicts(input, "|") +@pytest.mark.parametrize("empty_parent_groups", [None, "", []]) +def test_no_parent_user_groups(empty_parent_groups) -> None: + result = UserGroupIncrementalLoad( + user_group_id="ug_3", + user_group_name="ug_3", + parent_user_groups=empty_parent_groups, + is_active=False, + ) + expected = UserGroupIncrementalLoad( + user_group_id="ug_3", + user_group_name="ug_3", + parent_user_groups=[], + is_active=False, + ) + assert result == expected, ( + "Row with empty parent user groups should be parsed correctly" + ) -def prepare_sdk(): - def mock_list_user_groups(): - return [ - MockUserGroup("ug_1", "Admins", []).to_sdk(), - MockUserGroup("ug_4", "TemporaryAccess", ["ug_2"]).to_sdk(), - ] - sdk = mock.Mock() - sdk.catalog_user.list_user_groups = mock_list_user_groups - return sdk +def test_row_invalid_is_active() -> None: + with pytest.raises(ValueError): + UserGroupIncrementalLoad( + user_group_id="ug_4", + user_group_name="Testers", + parent_user_groups=[], + is_active="not_a_boolean", # type: ignore + ) diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_users.py b/gooddata-pipelines/tests/provisioning/entities/users/test_users.py index 864a511e8..26cd18a19 100644 --- a/gooddata-pipelines/tests/provisioning/entities/users/test_users.py +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_users.py @@ -269,12 +269,12 @@ def patch_delete_user(user_id: str, *args, **kwargs): # Run the provisioning if load_method == "incremental_load": - incremental_load_data = UserIncrementalLoad.from_list_of_dicts( - input_data - ) + incremental_load_data = [ + UserIncrementalLoad(**row) for row in input_data + ] user_provisioner.incremental_load(incremental_load_data) else: - full_load_data = UserFullLoad.from_list_of_dicts(input_data) + full_load_data = [UserFullLoad(**row) for row in input_data] user_provisioner.full_load(full_load_data) # Compare list lengths From 7ad6bac23738a4845230ee4daa1d8fdc24c513b3 Mon Sep 17 00:00:00 2001 From: janmatzek Date: Wed, 3 Sep 2025 08:37:43 +0200 Subject: [PATCH 2/3] feat(gooddata-pipelines): runtime validation of provisioning inputs --- .../entities/users/permissions.py | 5 + .../entities/users/user_groups.py | 5 + .../provisioning/entities/users/users.py | 6 +- .../entities/workspaces/workspace.py | 5 + .../provisioning/provisioning.py | 23 +++- gooddata-pipelines/tests/conftest.py | 60 +++++++++- .../entities/users/test_permissions.py | 12 -- .../entities/workspaces/test_provisioning.py | 27 +++-- .../entities/workspaces/test_workspace.py | 34 +++--- .../tests/provisioning/test_provisioning.py | 104 ++++++++++++++++++ 10 files changed, 236 insertions(+), 45 deletions(-) create mode 100644 gooddata-pipelines/tests/provisioning/test_provisioning.py diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py index 5246ad177..034c22a48 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py @@ -35,6 +35,11 @@ class PermissionProvisioner( source_group_incremental: list[PermissionIncrementalLoad] source_group_full: list[PermissionFullLoad] + FULL_LOAD_TYPE: type[PermissionFullLoad] = PermissionFullLoad + INCREMENTAL_LOAD_TYPE: type[PermissionIncrementalLoad] = ( + PermissionIncrementalLoad + ) + def _get_ws_declaration(self, ws_id: str) -> PermissionDeclaration: users: TargetsPermissionDict = {} user_groups: TargetsPermissionDict = {} diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py index db1d7d337..b739e3e38 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py @@ -29,6 +29,11 @@ class UserGroupProvisioner( source_group_full: list[UserGroupFullLoad] upstream_user_groups: list[CatalogUserGroup] + FULL_LOAD_TYPE: type[UserGroupFullLoad] = UserGroupFullLoad + INCREMENTAL_LOAD_TYPE: type[UserGroupIncrementalLoad] = ( + UserGroupIncrementalLoad + ) + @staticmethod def _is_changed( group: UserGroupModel, existing_group: CatalogUserGroup diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py index 30fad3964..e63afd5d7 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py @@ -24,13 +24,15 @@ class UserProvisioner(Provisioning[UserFullLoad, UserIncrementalLoad]): """Provisioning class for users in GoodData workspaces. This class handles the creation, update, and deletion of users - based on the provided source data. Use the `full_load` or `incremental_load` - methods to run the provisioning. + based on the provided source data. """ source_group_incremental: list[UserIncrementalLoad] source_group_full: list[UserFullLoad] + FULL_LOAD_TYPE: type[UserFullLoad] = UserFullLoad + INCREMENTAL_LOAD_TYPE: type[UserIncrementalLoad] = UserIncrementalLoad + def __init__(self, host: str, token: str) -> None: super().__init__(host, token) self.upstream_user_cache: dict[UserId, UserModel] = {} diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py index f359059b9..6c173908c 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py @@ -35,6 +35,11 @@ class WorkspaceProvisioner( source_group_full: list[WorkspaceFullLoad] source_group_incremental: list[WorkspaceIncrementalLoad] + FULL_LOAD_TYPE: type[WorkspaceFullLoad] = WorkspaceFullLoad + INCREMENTAL_LOAD_TYPE: type[WorkspaceIncrementalLoad] = ( + WorkspaceIncrementalLoad + ) + def __init__(self, *args: str, **kwargs: str) -> None: """Creates an instance of the WorkspaceProvisioner. diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py index cc3c8fe9e..81b509da7 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py @@ -23,6 +23,8 @@ class Provisioning(Generic[TFullLoadSourceData, TIncrementalSourceData]): TProvisioning = TypeVar("TProvisioning", bound="Provisioning") source_group_full: list[TFullLoadSourceData] source_group_incremental: list[TIncrementalSourceData] + FULL_LOAD_TYPE: type[TFullLoadSourceData] + INCREMENTAL_LOAD_TYPE: type[TIncrementalSourceData] def __init__(self, host: str, token: str) -> None: self.source_id: set[str] = set() @@ -80,6 +82,17 @@ def _create_groups( ids_to_create=ids_to_create, ) + def _validate_source_data_type( + self, + source_data: list[TFullLoadSourceData] | list[TIncrementalSourceData], + model: type[TFullLoadSourceData] | type[TIncrementalSourceData], + ) -> None: + """Validates data type of the source data.""" + if not all(isinstance(record, model) for record in source_data): + raise TypeError( + f"Not all elements in source data are instances of {model.__name__}" + ) + def _provision_incremental_load(self) -> None: raise NotImplementedError( "Provisioning method to be implemented in the subclass." @@ -103,9 +116,10 @@ def full_load(self, source_data: list[TFullLoadSourceData]) -> None: - All child workspaces not declared under the parent workspace in the source data are deleted """ - self.source_group_full = source_data try: + self._validate_source_data_type(source_data, self.FULL_LOAD_TYPE) + self.source_group_full = source_data self._provision_full_load() self.logger.info("Provisioning completed.") except Exception as e: @@ -120,10 +134,11 @@ def incremental_load( based on the source data provided. Only changes requested in the source data will be applied. """ - # TODO: validate the data type of source group at runtime - self.source_group_incremental = source_data - try: + self._validate_source_data_type( + source_data, self.INCREMENTAL_LOAD_TYPE + ) + self.source_group_incremental = source_data self._provision_incremental_load() self.logger.info("Provisioning completed.") except Exception as e: diff --git a/gooddata-pipelines/tests/conftest.py b/gooddata-pipelines/tests/conftest.py index 261146b4d..b3ce833b8 100644 --- a/gooddata-pipelines/tests/conftest.py +++ b/gooddata-pipelines/tests/conftest.py @@ -6,7 +6,15 @@ import boto3 import pytest from moto import mock_aws - +from pytest_mock import MockerFixture + +from gooddata_pipelines import ( + PermissionProvisioner, + UserDataFilterProvisioner, + UserGroupProvisioner, + UserProvisioner, + WorkspaceProvisioner, +) from gooddata_pipelines.api import GoodDataApi TEST_DATA_DIR = str((Path(__file__).parent / "data").absolute()) @@ -69,3 +77,53 @@ def error(self, msg: str) -> None: print(msg) return MockLogger() + + +@pytest.fixture +def workspace_provisioner(mocker: MockerFixture): + provisioner_instance = WorkspaceProvisioner.create("host", "token") + + # Patch the API + mocker.patch.object(provisioner_instance, "_api", return_value=None) + + return provisioner_instance + + +@pytest.fixture +def user_provisioner(mocker: MockerFixture): + provisioner_instance = UserProvisioner.create("host", "token") + + # Patch the API + mocker.patch.object(provisioner_instance, "_api", return_value=None) + + return UserProvisioner.create("host", "token") + + +@pytest.fixture +def user_group_provisioner(mocker: MockerFixture): + provisioner_instance = UserGroupProvisioner.create("host", "token") + + # Patch the API + mocker.patch.object(provisioner_instance, "_api", return_value=None) + + return provisioner_instance + + +@pytest.fixture +def permission_provisioner(mocker: MockerFixture) -> PermissionProvisioner: + provisioner_instance = PermissionProvisioner.create("host", "token") + + # Patch the API + mocker.patch.object(provisioner_instance, "_api", return_value=None) + + return provisioner_instance + + +@pytest.fixture +def user_data_filter_provisioner(mocker: MockerFixture): + provisioner_instance = UserDataFilterProvisioner.create("host", "token") + + # Patch the API + mocker.patch.object(provisioner_instance, "_api", return_value=None) + + return provisioner_instance diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py b/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py index b30b8bb73..2e042ac92 100644 --- a/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py @@ -402,18 +402,6 @@ def mock_upstream_perms(ws_id: str) -> CatalogDeclarativeWorkspacePermissions: return UPSTREAM_WS_PERMISSIONS[ws_id] -@pytest.fixture -def permission_provisioner(mocker: MockerFixture) -> PermissionProvisioner: - provisioner_instance = PermissionProvisioner.create( - host="https://localhost:3000", token="token" - ) - - # Patch the API - mocker.patch.object(provisioner_instance, "_api", return_value=None) - - return provisioner_instance - - def parse_expected_permissions( raw_data: dict, ) -> dict[str, list[CatalogDeclarativeSingleWorkspacePermission]]: diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py index 45183f810..f94064bfd 100644 --- a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py @@ -1,12 +1,13 @@ # (C) 2025 GoodData Corporation +from gooddata_pipelines import WorkspaceProvisioner from gooddata_pipelines.provisioning.provisioning import Provisioning -MOCK_PROVISIONER: Provisioning = Provisioning.create("host", "token") - -def test_create_groups_base_case() -> None: - provisioner: Provisioning = MOCK_PROVISIONER +def test_create_groups_base_case( + workspace_provisioner: WorkspaceProvisioner, +) -> None: + provisioner: Provisioning = workspace_provisioner test_source_ids = {"source_id_1", "source_id_2", "source_id_3"} test_panther_ids = {"source_id_2", "source_id_3", "source_id_4"} @@ -17,8 +18,10 @@ def test_create_groups_base_case() -> None: assert id_groups.ids_to_create == {"source_id_1"} -def test_create_groups_empty_sets() -> None: - provisioner: Provisioning = MOCK_PROVISIONER +def test_create_groups_empty_sets( + workspace_provisioner: WorkspaceProvisioner, +) -> None: + provisioner: Provisioning = workspace_provisioner test_source_ids: set[str] = set() test_panther_ids: set[str] = set() @@ -29,8 +32,10 @@ def test_create_groups_empty_sets() -> None: assert id_groups.ids_to_create == set() -def test_create_groups_no_overlap() -> None: - provisioner: Provisioning = MOCK_PROVISIONER +def test_create_groups_no_overlap( + workspace_provisioner: WorkspaceProvisioner, +) -> None: + provisioner: Provisioning = workspace_provisioner test_source_ids = {"source_id_1", "source_id_2"} test_panther_ids = {"source_id_3", "source_id_4"} @@ -41,8 +46,10 @@ def test_create_groups_no_overlap() -> None: assert id_groups.ids_to_create == {"source_id_1", "source_id_2"} -def test_create_groups_full_overlap() -> None: - provisioner: Provisioning = MOCK_PROVISIONER +def test_create_groups_full_overlap( + workspace_provisioner: WorkspaceProvisioner, +) -> None: + provisioner: Provisioning = workspace_provisioner test_source_ids = {"source_id_1", "source_id_2"} test_panther_ids = {"source_id_1", "source_id_2"} diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py index 81f73b67f..5e79dc366 100644 --- a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py @@ -1,5 +1,6 @@ # (C) 2025 GoodData Corporation +import pytest from gooddata_sdk.catalog.workspace.entity_model.workspace import ( CatalogWorkspace, ) @@ -9,13 +10,11 @@ WorkspaceFullLoad, ) -MOCK_WORKSPACE_PROVISIONER: WorkspaceProvisioner = WorkspaceProvisioner.create( - "host", "token" -) - -def test_find_workspaces_to_update_same_ids_and_names() -> None: - provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER +@pytest.fixture +def test_find_workspaces_to_update_same_ids_and_names( + workspace_provisioner: WorkspaceProvisioner, +) -> None: ids_in_both_systems = {"workspace_id1", "workspace_id2"} source_group: list[WorkspaceFullLoad] = [ WorkspaceFullLoad( @@ -42,15 +41,16 @@ def test_find_workspaces_to_update_same_ids_and_names() -> None: ), ] - workspaces_to_update = provisioner._find_workspaces_to_update( + workspaces_to_update = workspace_provisioner._find_workspaces_to_update( source_group, panther_group, ids_in_both_systems ) assert workspaces_to_update == set() -def test_find_workspaces_to_update_different_ids() -> None: - provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER +def test_find_workspaces_to_update_different_ids( + workspace_provisioner: WorkspaceProvisioner, +) -> None: ids_in_both_systems = {"workspace_id1", "workspace_id2"} source_group: list[WorkspaceFullLoad] = [ WorkspaceFullLoad( @@ -77,15 +77,16 @@ def test_find_workspaces_to_update_different_ids() -> None: ), ] - workspaces_to_update = provisioner._find_workspaces_to_update( + workspaces_to_update = workspace_provisioner._find_workspaces_to_update( source_group, panther_group, ids_in_both_systems ) assert workspaces_to_update == set() -def test_find_workspaces_to_update_same_ids_different_names() -> None: - provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER +def test_find_workspaces_to_update_same_ids_different_names( + workspace_provisioner: WorkspaceProvisioner, +) -> None: ids_in_both_systems: set[str] = {"workspace_id1", "workspace_id2"} source_group: list[WorkspaceFullLoad] = [ WorkspaceFullLoad( @@ -112,15 +113,16 @@ def test_find_workspaces_to_update_same_ids_different_names() -> None: ), ] - workspaces_to_update = provisioner._find_workspaces_to_update( + workspaces_to_update = workspace_provisioner._find_workspaces_to_update( source_group, panther_group, ids_in_both_systems ) assert workspaces_to_update == {"workspace_id1", "workspace_id2"} -def test_find_workspaces_to_update_no_panther() -> None: - provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER +def test_find_workspaces_to_update_no_panther( + workspace_provisioner: WorkspaceProvisioner, +) -> None: ids_in_both_systems: set[str] = set() source_group: list[WorkspaceFullLoad] = [ WorkspaceFullLoad( @@ -136,7 +138,7 @@ def test_find_workspaces_to_update_no_panther() -> None: ] panther_group: list[CatalogWorkspace] = [] - workspaces_to_update = provisioner._find_workspaces_to_update( + workspaces_to_update = workspace_provisioner._find_workspaces_to_update( source_group, panther_group, ids_in_both_systems ) diff --git a/gooddata-pipelines/tests/provisioning/test_provisioning.py b/gooddata-pipelines/tests/provisioning/test_provisioning.py new file mode 100644 index 000000000..6ae79a657 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/test_provisioning.py @@ -0,0 +1,104 @@ +# (C) 2025 GoodData Corporation +import pytest + +from gooddata_pipelines import ( + EntityType, + PermissionFullLoad, + PermissionIncrementalLoad, + UserFullLoad, + UserGroupFullLoad, + UserGroupIncrementalLoad, + UserIncrementalLoad, + WorkspaceFullLoad, +) +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceIncrementalLoad, +) + +WORKSPACE_DATA_TO_FAIL = [ + WorkspaceFullLoad( + parent_id="client_id1", + workspace_id="workspace_id1", + workspace_name="workspace_title1", + ), + WorkspaceIncrementalLoad( # type: ignore + parent_id="client_id1", + workspace_id="workspace_id1", + workspace_name="workspace_title1", + is_active=True, + ), +] +USER_DATA_TO_FAIL = [ + UserFullLoad( + user_id="user_id1", + firstname="user_name1", + lastname="user_name1", + email="user_email1", + auth_id="auth_id1", + user_groups=["user_group_id1"], + ), + UserIncrementalLoad( + user_id="user_id1", + firstname="user_name1", + lastname="user_name1", + email="user_email1", + auth_id="auth_id1", + user_groups=["user_group_id1"], + is_active=True, + ), +] + +USER_GROUP_DATA_TO_FAIL = [ + UserGroupFullLoad( + user_group_id="user_group_id1", + user_group_name="user_group_name1", + ), + UserGroupIncrementalLoad( + user_group_id="user_group_id1", + user_group_name="user_group_name1", + is_active=True, + ), +] + +PERMISSION_DATA_TO_FAIL = [ + PermissionFullLoad( + permission="permission_id1", + workspace_id="workspace_id1", + entity_id="entity_id1", + entity_type=EntityType.user, + ), + PermissionIncrementalLoad( + permission="permission_id1", + workspace_id="workspace_id1", + entity_id="entity_id1", + entity_type=EntityType.user, + is_active=True, + ), +] + + +TEST_CASES = [ + ("workspace_provisioner", WORKSPACE_DATA_TO_FAIL), + ("user_provisioner", USER_DATA_TO_FAIL), + ("user_group_provisioner", USER_GROUP_DATA_TO_FAIL), + ("permission_provisioner", PERMISSION_DATA_TO_FAIL), +] + + +@pytest.mark.parametrize( + "provisioner_name, data_to_fail", + TEST_CASES, +) +def test_fail_type_validation( + request: pytest.FixtureRequest, provisioner_name: str, data_to_fail: list +) -> None: + """Data type validation of source data should fail when input data is not + all of the same type.""" + provisioner = request.getfixturevalue(provisioner_name) + with pytest.raises(TypeError) as e: + provisioner._validate_source_data_type( + data_to_fail, + WorkspaceFullLoad, + ) + + assert "Not all elements in source data are instances of" in str(e) From 148c853bf36af51d6e879f823db1e350f37d9705 Mon Sep 17 00:00:00 2001 From: janmatzek Date: Wed, 3 Sep 2025 12:30:42 +0200 Subject: [PATCH 3/3] feat(gooddata-pipelines): incremental workspace provisioning --- .../gooddata_pipelines/__init__.py | 6 ++- .../gooddata_pipelines/api/gooddata_api.py | 3 -- .../entities/workspaces/workspace.py | 49 +++++++++++++++++-- .../workspaces/workspace_data_parser.py | 15 +++--- .../provisioning/provisioning.py | 1 + .../entities/users/test_user_groups.py | 5 -- 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/gooddata-pipelines/gooddata_pipelines/__init__.py b/gooddata-pipelines/gooddata_pipelines/__init__.py index 5aa82737d..26c8a5eff 100644 --- a/gooddata-pipelines/gooddata_pipelines/__init__.py +++ b/gooddata-pipelines/gooddata_pipelines/__init__.py @@ -34,7 +34,10 @@ from .provisioning.entities.users.permissions import PermissionProvisioner from .provisioning.entities.users.user_groups import UserGroupProvisioner from .provisioning.entities.users.users import UserProvisioner -from .provisioning.entities.workspaces.models import WorkspaceFullLoad +from .provisioning.entities.workspaces.models import ( + WorkspaceFullLoad, + WorkspaceIncrementalLoad, +) from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner __all__ = [ @@ -53,6 +56,7 @@ "UserGroupFullLoad", "UserProvisioner", "UserGroupProvisioner", + "WorkspaceIncrementalLoad", "PermissionProvisioner", "UserDataFilterProvisioner", "UserDataFilterFullLoad", diff --git a/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py index a2563422c..93ffb5487 100644 --- a/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py +++ b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py @@ -7,9 +7,6 @@ import requests -# TODO: Limit the use of "typing.Any". Improve readability by using either models -# or typed dicts. - TIMEOUT = 60 REQUEST_PAGE_SIZE = 250 API_VERSION = "v1" diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py index 6c173908c..b1f52d7c5 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py @@ -40,6 +40,8 @@ class WorkspaceProvisioner( WorkspaceIncrementalLoad ) + upstream_group: list[CatalogWorkspace] + def __init__(self, *args: str, **kwargs: str) -> None: """Creates an instance of the WorkspaceProvisioner. @@ -97,10 +99,11 @@ def _create_or_update_panther_workspaces( workspace_ids_to_update: set[str], child_to_parent_map: dict[str, str], workspace_id_to_wdf_map: dict[str, dict[str, list[str]]], + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], ) -> None: action: Literal["CREATE", "UPDATE"] - for source_workspace in self.source_group_full: + for source_workspace in source_group: if source_workspace.workspace_id in workspace_ids_to_update: action = "UPDATE" elif source_workspace.workspace_id in workspace_ids_to_create: @@ -205,8 +208,8 @@ def _provision_full_load(self) -> None: ) # Get upstream children of all parent workspaces. - self.upstream_group: list[CatalogWorkspace] = ( - self._api.get_panther_children_workspaces(self.maps.parent_ids) + self.upstream_group = self._api.get_panther_children_workspaces( + self.maps.parent_ids ) # Set maps that require upstream data. @@ -240,6 +243,7 @@ def _provision_full_load(self) -> None: self.ids_to_update, self.maps.child_to_parent_id_map, self.maps.workspace_id_to_wdf_map, + self.source_group_full, ) # Check WDF settings of ignored workspaces. @@ -265,5 +269,42 @@ def _provision_full_load(self) -> None: def _provision_incremental_load(self) -> None: """Incremental workspace provisioning.""" + # Set the maps based on the source data. + self.maps = self.parser.set_maps_based_on_source( + self.maps, self.source_group_incremental + ) + + # Get upstream children of all parent workspaces. + self.upstream_group = self._api.get_panther_children_workspaces( + self.maps.parent_ids + ) - raise NotImplementedError("Not implemented yet.") + # Set maps that require upstream data. + self.maps = self.parser.set_maps_with_upstream_data( + self.maps, self.source_group_incremental, self.upstream_group + ) + + # Create an instance of WDF manager with the created maps. + self.wdf_manager = WorkspaceDataFilterManager(self._api, self.maps) + + # Iterate through the source data and sort workspace ID to groups + ids_to_update: set[str] = set() + ids_to_delete: set[str] = set() + + for workspace in self.source_group_incremental: + if workspace.is_active: + ids_to_update.add(workspace.workspace_id) + else: + ids_to_delete.add(workspace.workspace_id) + + self._create_or_update_panther_workspaces( + set(), + ids_to_update, + self.maps.child_to_parent_id_map, + self.maps.workspace_id_to_wdf_map, + self.source_group_incremental, + ) + + self.delete_panther_workspaces( + ids_to_delete, self.maps.workspace_id_to_name_map + ) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py index 32bea2230..c2a1adee7 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py @@ -9,6 +9,7 @@ from gooddata_pipelines.provisioning.entities.workspaces.models import ( WorkspaceDataMaps, WorkspaceFullLoad, + WorkspaceIncrementalLoad, ) @@ -17,7 +18,7 @@ class WorkspaceDataParser: @staticmethod def _get_id_to_name_map( - source_group: list[WorkspaceFullLoad], + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], upstream_group: list[CatalogWorkspace], ) -> dict[str, str]: """Creates a map of workspace IDs to their names for all known workspaces.""" @@ -33,7 +34,7 @@ def _get_id_to_name_map( @staticmethod def _get_child_to_parent_map( - source_group: list[WorkspaceFullLoad], + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], ) -> dict[str, str]: """Creates a map of child workspace IDs to their parent workspace IDs.""" child_to_parent_map: dict[str, str] = { @@ -45,7 +46,8 @@ def _get_child_to_parent_map( @staticmethod def _get_set_of_ids_from_source( - source_group: list[WorkspaceFullLoad], column_name: str + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], + column_name: str, ) -> set[str]: """Creates a set of unique parent workspace IDs.""" set_of_ids: set[str] = { @@ -64,7 +66,8 @@ def get_set_of_upstream_workspace_ids( return set_of_ids def _get_child_to_wdfs_map( - self, source_group: list[WorkspaceFullLoad] + self, + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], ) -> dict[str, dict[str, list[str]]]: """Creates a map of child workspace IDs to their WDF IDs.""" # TODO: Use objects or a more transparent data structure instead of this. @@ -88,7 +91,7 @@ def _get_child_to_wdfs_map( def set_maps_based_on_source( self, map_object: WorkspaceDataMaps, - source_group: list[WorkspaceFullLoad], + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], ) -> WorkspaceDataMaps: """Creates maps which are dependent on the source group only.""" map_object.child_to_parent_id_map = self._get_child_to_parent_map( @@ -109,7 +112,7 @@ def set_maps_based_on_source( def set_maps_with_upstream_data( self, map_object: WorkspaceDataMaps, - source_group: list[WorkspaceFullLoad], + source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad], upstream_group: list[CatalogWorkspace], ) -> WorkspaceDataMaps: """Creates maps which are dependent on both the source group and upstream group.""" diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py index 81b509da7..75b34229d 100644 --- a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py @@ -23,6 +23,7 @@ class Provisioning(Generic[TFullLoadSourceData, TIncrementalSourceData]): TProvisioning = TypeVar("TProvisioning", bound="Provisioning") source_group_full: list[TFullLoadSourceData] source_group_incremental: list[TIncrementalSourceData] + FULL_LOAD_TYPE: type[TFullLoadSourceData] INCREMENTAL_LOAD_TYPE: type[TIncrementalSourceData] diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py b/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py index f02a8b934..bd02e87e8 100644 --- a/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py @@ -7,13 +7,8 @@ UserGroupIncrementalLoad, ) -# TODO: Test business logic - def test_missing_key_no_parent_groups() -> None: - # NOTE: Type ignore because of the missing parent_user_groups key. In Python - # 3.11, we can use NotRequired to make the key optional. - result = UserGroupIncrementalLoad( user_group_id="ug_2", user_group_name="Developers", is_active=True )