update: full load provisioning logic of users and groups

Author: janmatzek

gooddata-pipelines/gooddata_pipelines/__init__.py

@@ -2,28 +2,42 @@
 
 from ._version import __version__
 from .provisioning.entities.user_data_filters.models.udf_models import (
-    UserDataFilter,
+    UserDataFilterFullLoad,
 )
 from .provisioning.entities.user_data_filters.user_data_filters import (
     UserDataFilterProvisioner,
 )
-from .provisioning.entities.users.models import Permission, User, UserGroup
+from .provisioning.entities.users.models.permissions import (
+    PermissionFullLoad,
+    PermissionIncrementalLoad,
+)
+from .provisioning.entities.users.models.user_groups import (
+    UserGroupFullLoad,
+    UserGroupIncrementalLoad,
+)
+from .provisioning.entities.users.models.users import (
+    UserFullLoad,
+    UserIncrementalLoad,
+)
 from .provisioning.entities.users.permissions import PermissionProvisioner
 from .provisioning.entities.users.user_groups import UserGroupProvisioner
 from .provisioning.entities.users.users import UserProvisioner
-from .provisioning.entities.workspaces.models import Workspace
+from .provisioning.entities.workspaces.models import WorkspaceFullLoad
 from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner
 
 __all__ = [
-    "Workspace",
+    "WorkspaceFullLoad",
     "WorkspaceProvisioner",
-    "User",
-    "UserGroup",
-    "Permission",
+    "UserIncrementalLoad",
+    "UserGroupIncrementalLoad",
+    "PermissionFullLoad",
+    "PermissionIncrementalLoad",
+    "UserFullLoad",
+    "UserGroupFullLoad",
     "UserProvisioner",
     "UserGroupProvisioner",
     "PermissionProvisioner",
     "UserDataFilterProvisioner",
-    "UserDataFilter",
+    "UserDataFilterFullLoad",
     "__version__",
 ]

gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py

@@ -216,17 +216,22 @@ def post_workspace_data_filter(
         endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilters"
         return self._post(endpoint, data, self.headers)
 
-    def _get(self, endpoint: str) -> requests.Response:
+    def _get(
+        self, endpoint: str, headers: dict[str, str] | None = None
+    ) -> requests.Response:
         """Sends a GET request to the server.
 
         Args:
             endpoint (str): The API endpoint to send the GET request to.
+            headers (dict[str, str] | None): Headers to include in the request.
+                If no headers are provided, the default headers will be used.
         Returns:
             requests.Response: The response from the server.
         """
         url = self._get_url(endpoint)
+        request_headers = headers if headers else self.headers
 
-        return requests.get(url, headers=self.headers, timeout=TIMEOUT)
+        return requests.get(url, headers=request_headers, timeout=TIMEOUT)
 
     def _post(
         self,
@@ -239,7 +244,8 @@ def _post(
         Args:
             endpoint (str): The API endpoint to send the POST request to.
             data (Any): The JSON data to send in the request body.
-            headers (dict | None): Optional headers to include in the request.
+            headers (dict | None): Headers to include in the request.
+                If no headers are provided, the default headers will be used.
         Returns:
             requests.Response: The response from the server.
         """
@@ -262,7 +268,8 @@ def _put(
         Args:
             endpoint (str): The API endpoint to send the PUT request to.
             data (Any): The JSON data to send in the request body.
-            headers (dict | None): Optional headers to include in the request.
+            headers (dict | None): Headers to include in the request.
+                If no headers are provided, the default headers will be used.
         Returns:
             requests.Response: The response from the server.
         """

gooddata-pipelines/gooddata_pipelines/api/gooddata_sdk.py

@@ -246,6 +246,15 @@ def list_user_groups(self) -> list[CatalogUserGroup]:
         """
         return self._sdk.catalog_user.list_user_groups()
 
+    @raise_with_context()
+    def list_users(self) -> list[CatalogUser]:
+        """Calls GoodData Python SDK to retrieve all users.
+
+        Returns:
+            list[CatalogUser]: A list of all users in the domain.
+        """
+        return self._sdk.catalog_user.list_users()
+
     @raise_with_context()
     def create_or_update_user_group(
         self, catalog_user_group: CatalogUserGroup, **_: str

gooddata-pipelines/gooddata_pipelines/provisioning/__init__.py

@@ -1,16 +1,30 @@
 # (C) 2025 GoodData Corporation
 
-from .entities.users.models import Permission, User, UserGroup
+from .entities.users.models.permissions import (
+    PermissionFullLoad,
+    PermissionIncrementalLoad,
+)
+from .entities.users.models.user_groups import (
+    UserGroupFullLoad,
+    UserGroupIncrementalLoad,
+)
+from .entities.users.models.users import (
+    UserFullLoad,
+    UserIncrementalLoad,
+)
 from .entities.users.permissions import PermissionProvisioner
 from .entities.users.user_groups import UserGroupProvisioner
 from .entities.users.users import UserProvisioner
 from .entities.workspaces.workspace import WorkspaceProvisioner
 
 __all__ = [
-    "Permission",
+    "PermissionFullLoad",
+    "PermissionIncrementalLoad",
     "PermissionProvisioner",
-    "User",
-    "UserGroup",
+    "UserFullLoad",
+    "UserGroupFullLoad",
+    "UserIncrementalLoad",
+    "UserGroupIncrementalLoad",
     "UserGroupProvisioner",
     "UserProvisioner",
     "WorkspaceProvisioner",

gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/udf_models.py

@@ -18,7 +18,12 @@ class WorkspaceUserDataFilters:
 
 
 @dataclass
-class UserDataFilter:
+class UserDataFilterFullLoad:
     workspace_id: str
     udf_id: str
     udf_value: str
+
+
+@dataclass
+class UserDataFilterIncrementalLoad(UserDataFilterFullLoad):
+    is_active: bool

gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/user_data_filters.py

@@ -16,15 +16,18 @@
 )
 
 from gooddata_pipelines.provisioning.entities.user_data_filters.models.udf_models import (
-    UserDataFilter,
+    UserDataFilterFullLoad,
     UserDataFilterGroup,
+    UserDataFilterIncrementalLoad,
     WorkspaceUserDataFilters,
 )
 from gooddata_pipelines.provisioning.provisioning import Provisioning
 from gooddata_pipelines.provisioning.utils.exceptions import ContextException
 
 
-class UserDataFilterProvisioner(Provisioning[UserDataFilter]):
+class UserDataFilterProvisioner(
+    Provisioning[UserDataFilterFullLoad, UserDataFilterIncrementalLoad]
+):
     """Provisioning class for user data filters in GoodData workspaces.
 
     This class handles the creation, update, and deletion of user data filters
@@ -42,7 +45,8 @@ class UserDataFilterProvisioner(Provisioning[UserDataFilter]):
     ```
     """
 
-    source_group: list[UserDataFilter]
+    source_group_full: list[UserDataFilterFullLoad]
+    source_group_incremental: list[UserDataFilterIncrementalLoad]
     ldm_column_name: str = ""
     maql_column_name: str = ""
 
@@ -64,7 +68,7 @@ def set_maql_column_name(self, maql_column_name: str) -> None:
 
     @staticmethod
     def _group_db_user_data_filters_by_ws_id(
-        user_data_filters: list[UserDataFilter],
+        user_data_filters: list[UserDataFilterFullLoad],
     ) -> list[WorkspaceUserDataFilters]:
         """Group user data filters by workspace ID and user ID."""
         ws_map: dict[str, dict[str, set[str]]] = {}
@@ -193,11 +197,8 @@ def _delete_user_data_filters(
                     f"Failed to delete user data filters: {e}"
                 ) from e
 
-    def _provision(self) -> None:
+    def _provision_full_load(self) -> None:
         """Provision user data filters in GoodData workspaces."""
-        grouped_db_user_data_filters = (
-            self._group_db_user_data_filters_by_ws_id(self.source_group)
-        )
 
         if not self.maql_column_name:
             raise ContextException(
@@ -208,6 +209,13 @@ def _provision(self) -> None:
                 "LDM column name is not set. Please set it before provisioning."
             )
 
+        grouped_db_user_data_filters = (
+            self._group_db_user_data_filters_by_ws_id(self.source_group_full)
+        )
         self._create_user_data_filters(grouped_db_user_data_filters)
 
         self.logger.info("User data filters provisioning completed")
+
+    def _provision_incremental_load(self) -> None:
+        """Provision user data filters in GoodData workspaces."""
+        raise NotImplementedError("Not implemented yet.")

gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation