feat(gooddata-pipelines): incremental workspace provisioning

Author: janmatzek

gooddata-pipelines/gooddata_pipelines/__init__.py

@@ -34,7 +34,10 @@
 from .provisioning.entities.users.permissions import PermissionProvisioner
 from .provisioning.entities.users.user_groups import UserGroupProvisioner
 from .provisioning.entities.users.users import UserProvisioner
-from .provisioning.entities.workspaces.models import WorkspaceFullLoad
+from .provisioning.entities.workspaces.models import (
+    WorkspaceFullLoad,
+    WorkspaceIncrementalLoad,
+)
 from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner
 
 __all__ = [
@@ -53,6 +56,7 @@
     "UserGroupFullLoad",
     "UserProvisioner",
     "UserGroupProvisioner",
+    "WorkspaceIncrementalLoad",
     "PermissionProvisioner",
     "UserDataFilterProvisioner",
     "UserDataFilterFullLoad",

gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py

@@ -7,9 +7,6 @@
 
 import requests
 
-# TODO: Limit the use of "typing.Any". Improve readability by using either models
-#  or typed dicts.
-
 TIMEOUT = 60
 REQUEST_PAGE_SIZE = 250
 API_VERSION = "v1"

gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py

@@ -40,6 +40,8 @@ class WorkspaceProvisioner(
         WorkspaceIncrementalLoad
     )
 
+    upstream_group: list[CatalogWorkspace]
+
     def __init__(self, *args: str, **kwargs: str) -> None:
         """Creates an instance of the WorkspaceProvisioner.
 
@@ -97,10 +99,11 @@ def _create_or_update_panther_workspaces(
         workspace_ids_to_update: set[str],
         child_to_parent_map: dict[str, str],
         workspace_id_to_wdf_map: dict[str, dict[str, list[str]]],
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
     ) -> None:
         action: Literal["CREATE", "UPDATE"]
 
-        for source_workspace in self.source_group_full:
+        for source_workspace in source_group:
             if source_workspace.workspace_id in workspace_ids_to_update:
                 action = "UPDATE"
             elif source_workspace.workspace_id in workspace_ids_to_create:
@@ -205,8 +208,8 @@ def _provision_full_load(self) -> None:
         )
 
         # Get upstream children of all parent workspaces.
-        self.upstream_group: list[CatalogWorkspace] = (
-            self._api.get_panther_children_workspaces(self.maps.parent_ids)
+        self.upstream_group = self._api.get_panther_children_workspaces(
+            self.maps.parent_ids
         )
 
         # Set maps that require upstream data.
@@ -240,6 +243,7 @@ def _provision_full_load(self) -> None:
             self.ids_to_update,
             self.maps.child_to_parent_id_map,
             self.maps.workspace_id_to_wdf_map,
+            self.source_group_full,
         )
 
         # Check WDF settings of ignored workspaces.
@@ -265,5 +269,42 @@ def _provision_full_load(self) -> None:
 
     def _provision_incremental_load(self) -> None:
         """Incremental workspace provisioning."""
+        # Set the maps based on the source data.
+        self.maps = self.parser.set_maps_based_on_source(
+            self.maps, self.source_group_incremental
+        )
+
+        # Get upstream children of all parent workspaces.
+        self.upstream_group = self._api.get_panther_children_workspaces(
+            self.maps.parent_ids
+        )
 
-        raise NotImplementedError("Not implemented yet.")
+        # Set maps that require upstream data.
+        self.maps = self.parser.set_maps_with_upstream_data(
+            self.maps, self.source_group_incremental, self.upstream_group
+        )
+
+        # Create an instance of WDF manager with the created maps.
+        self.wdf_manager = WorkspaceDataFilterManager(self._api, self.maps)
+
+        # Iterate through the source data and sort workspace ID to groups
+        ids_to_update: set[str] = set()
+        ids_to_delete: set[str] = set()
+
+        for workspace in self.source_group_incremental:
+            if workspace.is_active:
+                ids_to_update.add(workspace.workspace_id)
+            else:
+                ids_to_delete.add(workspace.workspace_id)
+
+        self._create_or_update_panther_workspaces(
+            set(),
+            ids_to_update,
+            self.maps.child_to_parent_id_map,
+            self.maps.workspace_id_to_wdf_map,
+            self.source_group_incremental,
+        )
+
+        self.delete_panther_workspaces(
+            ids_to_delete, self.maps.workspace_id_to_name_map
+        )

gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py

@@ -9,6 +9,7 @@
 from gooddata_pipelines.provisioning.entities.workspaces.models import (
     WorkspaceDataMaps,
     WorkspaceFullLoad,
+    WorkspaceIncrementalLoad,
 )
 
 
@@ -17,7 +18,7 @@ class WorkspaceDataParser:
 
     @staticmethod
     def _get_id_to_name_map(
-        source_group: list[WorkspaceFullLoad],
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
         upstream_group: list[CatalogWorkspace],
     ) -> dict[str, str]:
         """Creates a map of workspace IDs to their names for all known workspaces."""
@@ -33,7 +34,7 @@ def _get_id_to_name_map(
 
     @staticmethod
     def _get_child_to_parent_map(
-        source_group: list[WorkspaceFullLoad],
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
     ) -> dict[str, str]:
         """Creates a map of child workspace IDs to their parent workspace IDs."""
         child_to_parent_map: dict[str, str] = {
@@ -45,7 +46,8 @@ def _get_child_to_parent_map(
 
     @staticmethod
     def _get_set_of_ids_from_source(
-        source_group: list[WorkspaceFullLoad], column_name: str
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
+        column_name: str,
     ) -> set[str]:
         """Creates a set of unique parent workspace IDs."""
         set_of_ids: set[str] = {
@@ -64,7 +66,8 @@ def get_set_of_upstream_workspace_ids(
         return set_of_ids
 
     def _get_child_to_wdfs_map(
-        self, source_group: list[WorkspaceFullLoad]
+        self,
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
     ) -> dict[str, dict[str, list[str]]]:
         """Creates a map of child workspace IDs to their WDF IDs."""
         # TODO: Use objects or a more transparent data structure instead of this.
@@ -88,7 +91,7 @@ def _get_child_to_wdfs_map(
     def set_maps_based_on_source(
         self,
         map_object: WorkspaceDataMaps,
-        source_group: list[WorkspaceFullLoad],
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
     ) -> WorkspaceDataMaps:
         """Creates maps which are dependent on the source group only."""
         map_object.child_to_parent_id_map = self._get_child_to_parent_map(
@@ -109,7 +112,7 @@ def set_maps_based_on_source(
     def set_maps_with_upstream_data(
         self,
         map_object: WorkspaceDataMaps,
-        source_group: list[WorkspaceFullLoad],
+        source_group: list[WorkspaceFullLoad] | list[WorkspaceIncrementalLoad],
         upstream_group: list[CatalogWorkspace],
     ) -> WorkspaceDataMaps:
         """Creates maps which are dependent on both the source group and upstream group."""

gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py

@@ -23,6 +23,7 @@ class Provisioning(Generic[TFullLoadSourceData, TIncrementalSourceData]):
     TProvisioning = TypeVar("TProvisioning", bound="Provisioning")
     source_group_full: list[TFullLoadSourceData]
     source_group_incremental: list[TIncrementalSourceData]
+
     FULL_LOAD_TYPE: type[TFullLoadSourceData]
     INCREMENTAL_LOAD_TYPE: type[TIncrementalSourceData]
 

gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py

@@ -7,13 +7,8 @@
     UserGroupIncrementalLoad,
 )
 
-# TODO: Test business logic
-
 
 def test_missing_key_no_parent_groups() -> None:
-    # NOTE: Type ignore because of the missing parent_user_groups key. In Python
-    # 3.11, we can use NotRequired to make the key optional.
-
     result = UserGroupIncrementalLoad(
         user_group_id="ug_2", user_group_name="Developers", is_active=True
     )