add workspace backup functionality

Author: janmatzek

gooddata-pipelines/TODO.md

@@ -4,19 +4,15 @@ A list of outstanding tasks, features, or technical debt to be addressed in this
 
 ## Pre-release
 
-- [ ] License file
+- [ ] License file (FOSSA)
 
 ## Features
 
-- [ ] Workspace backup
 - [ ] Workspace restore
-- [ ] Handle input from csv files [?]
 
 ## Refactoring / Debt
 
 - [ ] Cleanup custom exception
-- [ ] Use objects to make `workspace_data_parses.py` more transparent
-- [ ] Use objects in API integration. Consider using existing Python SDK objects where possible, otherwise create pydantic models to use instead of `dict[str, Any]` stand-ins for json values
 - [ ] Improve test coverage. Write missing unit tests for legacy code (e.g., user data filters)
 
 ## Documentation

gooddata-pipelines/gooddata_pipelines/__init__.py

@@ -1,6 +1,17 @@
 # (C) 2025 GoodData Corporation
 
 from ._version import __version__
+
+# -------- Backup and Restore --------
+from .backup_and_restore.backup_manager import BackupManager
+from .backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    StorageType,
+)
+from .backup_and_restore.storage.local_storage import LocalStorage
+from .backup_and_restore.storage.s3_storage import S3Storage
+
+# -------- Provisioning --------
 from .provisioning.entities.user_data_filters.models.udf_models import (
     UserDataFilterFullLoad,
 )
@@ -26,6 +37,11 @@
 from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner
 
 __all__ = [
+    "BackupManager",
+    "BackupRestoreConfig",
+    "StorageType",
+    "LocalStorage",
+    "S3Storage",
     "WorkspaceFullLoad",
     "WorkspaceProvisioner",
     "UserIncrementalLoad",

gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py

@@ -11,8 +11,8 @@
 #  or typed dicts.
 
 TIMEOUT = 60
-PANTHER_REQUEST_PAGE_SIZE = 250
-PANTHER_API_VERSION = "v1"
+REQUEST_PAGE_SIZE = 250
+API_VERSION = "v1"
 
 
 class APIMethods:
@@ -43,7 +43,7 @@ def _get_base_url(domain: str) -> str:
         if domain.startswith("http://") and not domain.startswith("https://"):
             domain = domain.replace("http://", "https://")
 
-        return f"{domain}/api/{PANTHER_API_VERSION}"
+        return f"{domain}/api/{API_VERSION}"
 
     def _get_url(self, endpoint: str) -> str:
         """Returns the full URL for a given API endpoint.
@@ -216,6 +216,18 @@ def post_workspace_data_filter(
         endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilters"
         return self._post(endpoint, data, self.headers)
 
+    def get_user_data_filters(self, workspace_id: str) -> requests.Response:
+        """Gets the user data filters for a given workspace."""
+        endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters"
+        return self._get(endpoint)
+
+    def get_automations(self, workspace_id: str) -> requests.Response:
+        """Gets the automations for a given workspace."""
+        endpoint = (
+            f"/entities/workspaces/{workspace_id}/automations?include=ALL"
+        )
+        return self._get(endpoint)
+
     def _get(
         self, endpoint: str, headers: dict[str, str] | None = None
     ) -> requests.Response:

gooddata-pipelines/gooddata_pipelines/api/gooddata_sdk.py

@@ -2,6 +2,8 @@
 
 """Interaction with GoodData Cloud via the Gooddata Python SDK."""
 
+from pathlib import Path
+
 from gooddata_sdk.catalog.permission.declarative_model.permission import (
     CatalogDeclarativeWorkspacePermissions,
 )
@@ -21,21 +23,34 @@
 from gooddata_pipelines.api.utils import raise_with_context
 
 
+def apply_to_all_methods(decorator):
+    def decorate(cls):
+        for attr in cls.__dict__:
+            if callable(getattr(cls, attr)) and not attr.startswith("__"):
+                setattr(cls, attr, decorator(getattr(cls, attr)))
+        return cls
+
+    return decorate
+
+
+@apply_to_all_methods(raise_with_context())
 class SDKMethods:
     """
     Class to intaract with GoodData Cloud via the Gooddata Python SDK.
     """
 
     _sdk: GoodDataSdk
 
+    def get_organization_id(self) -> str:
+        return self._sdk.catalog_organization.organization_id
+
     def check_workspace_exists(self, workspace_id: str) -> bool:
         try:
             self._sdk.catalog_workspace.get_workspace(workspace_id)
             return True
         except Exception:
             return False
 
-    @raise_with_context()
     def get_workspace(self, workspace_id: str, **_: str) -> CatalogWorkspace:
         """
         Calls GoodData Python SDK to retrieve a workspace by its ID.
@@ -50,7 +65,6 @@ def get_workspace(self, workspace_id: str, **_: str) -> CatalogWorkspace:
         """
         return self._sdk.catalog_workspace.get_workspace(workspace_id)
 
-    @raise_with_context()
     def delete_panther_workspace(self, workspace_id: str) -> None:
         """
         Calls GoodData Python SDK to delete a workspace by its ID.
@@ -63,7 +77,6 @@ def delete_panther_workspace(self, workspace_id: str) -> None:
         """
         self._sdk.catalog_workspace.delete_workspace(workspace_id)
 
-    @raise_with_context()
     def create_or_update_panther_workspace(
         self,
         workspace_id: str,
@@ -116,7 +129,6 @@ def get_panther_children_workspaces(
 
         return children
 
-    @raise_with_context()
     def list_workspaces(self) -> list[CatalogWorkspace]:
         """Retrieves all workspaces in the GoodData Cloud domain.
 
@@ -128,7 +140,6 @@ def list_workspaces(self) -> list[CatalogWorkspace]:
         """
         return self._sdk.catalog_workspace.list_workspaces()
 
-    @raise_with_context()
     def get_declarative_permissions(
         self, workspace_id: str
     ) -> CatalogDeclarativeWorkspacePermissions:
@@ -149,7 +160,6 @@ def get_declarative_permissions(
             workspace_id
         )
 
-    @raise_with_context()
     def put_declarative_permissions(
         self,
         workspace_id: str,
@@ -173,7 +183,6 @@ def put_declarative_permissions(
             workspace_id, ws_permissions
         )
 
-    @raise_with_context()
     def get_user(self, user_id: str, **_: str) -> CatalogUser:
         """
         Calls GoodData Python SDK to retrieve a user by its ID.
@@ -188,7 +197,6 @@ def get_user(self, user_id: str, **_: str) -> CatalogUser:
         """
         return self._sdk.catalog_user.get_user(user_id)
 
-    @raise_with_context()
     def create_or_update_user(self, user: CatalogUser, **_: str) -> None:
         """
         Calls GoodData Python SDK to create or update a user.
@@ -203,7 +211,6 @@ def create_or_update_user(self, user: CatalogUser, **_: str) -> None:
         """
         return self._sdk.catalog_user.create_or_update_user(user)
 
-    @raise_with_context()
     def delete_user(self, user_id: str, **_: str) -> None:
         """
         Calls GoodData Python SDK to delete a user by its ID.
@@ -218,7 +225,6 @@ def delete_user(self, user_id: str, **_: str) -> None:
         """
         return self._sdk.catalog_user.delete_user(user_id)
 
-    @raise_with_context()
     def get_user_group(self, user_group_id: str, **_: str) -> CatalogUserGroup:
         """
         Calls GoodData Python SDK to retrieve a user group by its ID.
@@ -233,7 +239,6 @@ def get_user_group(self, user_group_id: str, **_: str) -> CatalogUserGroup:
         """
         return self._sdk.catalog_user.get_user_group(user_group_id)
 
-    @raise_with_context()
     def list_user_groups(self) -> list[CatalogUserGroup]:
         """
         Calls GoodData Python SDK to retrieve all user groups.
@@ -246,7 +251,6 @@ def list_user_groups(self) -> list[CatalogUserGroup]:
         """
         return self._sdk.catalog_user.list_user_groups()
 
-    @raise_with_context()
     def list_users(self) -> list[CatalogUser]:
         """Calls GoodData Python SDK to retrieve all users.
 
@@ -255,7 +259,6 @@ def list_users(self) -> list[CatalogUser]:
         """
         return self._sdk.catalog_user.list_users()
 
-    @raise_with_context()
     def create_or_update_user_group(
         self, catalog_user_group: CatalogUserGroup, **_: str
     ) -> None:
@@ -273,7 +276,6 @@ def create_or_update_user_group(
             catalog_user_group
         )
 
-    @raise_with_context()
     def delete_user_group(self, user_group_id: str) -> None:
         """Calls GoodData Python SDK to delete a user group by its ID.
 
@@ -287,7 +289,6 @@ def delete_user_group(self, user_group_id: str) -> None:
         """
         return self._sdk.catalog_user.delete_user_group(user_group_id)
 
-    @raise_with_context()
     def get_declarative_workspace_data_filters(
         self,
     ) -> CatalogDeclarativeWorkspaceDataFilters:
@@ -303,7 +304,6 @@ def get_declarative_workspace_data_filters(
             self._sdk.catalog_workspace.get_declarative_workspace_data_filters()
         )
 
-    @raise_with_context()
     def list_user_data_filters(
         self, workspace_id: str
     ) -> list[CatalogUserDataFilter]:
@@ -319,7 +319,6 @@ def list_user_data_filters(
         """
         return self._sdk.catalog_workspace.list_user_data_filters(workspace_id)
 
-    @raise_with_context()
     def delete_user_data_filter(
         self, workspace_id: str, user_data_filter_id: str
     ) -> None:
@@ -338,7 +337,6 @@ def delete_user_data_filter(
             workspace_id, user_data_filter_id
         )
 
-    @raise_with_context()
     def create_or_update_user_data_filter(
         self, workspace_id: str, user_data_filter: CatalogUserDataFilter
     ) -> None:
@@ -357,3 +355,19 @@ def create_or_update_user_data_filter(
         self._sdk.catalog_workspace.create_or_update_user_data_filter(
             workspace_id, user_data_filter
         )
+
+    def store_declarative_workspace(
+        self, workspace_id: str, export_path: Path
+    ) -> None:
+        """Stores the declarative workspace in the specified export path."""
+        self._sdk.catalog_workspace.store_declarative_workspace(
+            workspace_id, export_path
+        )
+
+    def store_declarative_filter_views(
+        self, workspace_id: str, export_path: Path
+    ) -> None:
+        """Stores the declarative filter views in the specified export path."""
+        self._sdk.catalog_workspace.store_declarative_filter_views(
+            workspace_id, export_path
+        )

gooddata-pipelines/gooddata_pipelines/backup_and_restore/__init__.py

@@ -0,0 +1 @@
+# (C) 2025 GoodData Corporation