feat(workspace-backup): Backup to Azure blob storage

Author: benkeanna

.gitignore

@@ -12,6 +12,16 @@ packages/gooddata-sdk/tests/catalog/translate
 .vscode
 .ruff_cache
 
+# Python build artifacts
+.tox
+*.egg-info
+dist/
+build/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
 docs/node_modules
 docs/public
 docs/resources/_gen

packages/gooddata-pipelines/README.md

@@ -11,7 +11,7 @@ You can use the package to manage following resources in GDC:
    - User Data Filters
    - Child workspaces (incl. Workspace Data Filter settings)
 1. Backup and restore of workspaces
-   - Create and backup snapshots of workspace metadata.
+   - Create and backup snapshots of workspace metadata to local storage, AWS S3, or Azure Blob Storage
 1. LDM Extension
    - extend the Logical Data Model of a child workspace with custom datasets and fields
 
@@ -34,7 +34,7 @@ import logging
 from csv import DictReader
 from pathlib import Path
 
-# Import the Entity Provisioner class and corresponding model from gooddata_pipelines library
+# Import the Entity Provisioner class and corresponding model from the gooddata_pipelines library
 from gooddata_pipelines import UserFullLoad, UserProvisioner
 
 # Create the Provisioner instance - you can also create the instance from a GDC yaml profile
@@ -62,12 +62,51 @@ provisioner.full_load(full_load_data)
 
 ```
 
+Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository.
+
+## Backup and Restore of Workspaces
+
+The backup and restore module allows you to create snapshots of GoodData Cloud workspaces and restore them later. Backups can be stored locally, in AWS S3, or Azure Blob Storage.
+
+```python
+import os
+
+from gooddata_pipelines import BackupManager
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    LocalStorageConfig,
+    StorageType,
+)
+
+# Configure backup storage
+config = BackupRestoreConfig(
+    storage_type=StorageType.LOCAL,
+    storage=LocalStorageConfig(),
+)
+
+# Create the BackupManager instance
+backup_manager = BackupManager.create(
+    config=config,
+    host=os.environ["GDC_HOSTNAME"],
+    token=os.environ["GDC_AUTH_TOKEN"]
+)
+
+# Backup specific workspaces
+backup_manager.backup_workspaces(workspace_ids=["workspace1", "workspace2"])
+
+# Backup workspace hierarchies (workspace + all children)
+backup_manager.backup_hierarchies(workspace_ids=["parent_workspace"])
+
+# Backup entire organization
+backup_manager.backup_entire_organization()
+```
+
+For S3 or Azure Blob Storage, configure the appropriate storage type and credentials in `BackupRestoreConfig`.
+
 ## Bugs & Requests
 
-Please use the [GitHub issue tracker](https://github.com/gooddata/gooddata-python-sdk/issues) to submit bugs
-or request features.
+Please use the [GitHub issue tracker](https://github.com/gooddata/gooddata-python-sdk/issues) to submit bugs or request features.
 
 ## Changelog
 
-See [Github releases](https://github.com/gooddata/gooddata-python-sdk/releases) for released versions
-and a list of changes.
+See [GitHub releases](https://github.com/gooddata/gooddata-python-sdk/releases) for released versions and a list of changes.

packages/gooddata-pipelines/pyproject.toml

@@ -14,6 +14,8 @@ dependencies = [
     "gooddata-sdk~=1.54.0",
     "boto3 (>=1.39.3,<2.0.0)",
     "boto3-stubs (>=1.39.3,<2.0.0)",
+    "azure-storage-blob (>=12.19.0,<13.0.0)",
+    "azure-identity (>=1.15.0,<2.0.0)",
     "types-pyyaml (>=6.0.12.20250326,<7.0.0)",
 ]
 

packages/gooddata-pipelines/src/gooddata_pipelines/__init__.py

@@ -5,6 +5,7 @@
 # -------- Backup and Restore --------
 from .backup_and_restore.backup_manager import BackupManager
 from .backup_and_restore.models.storage import (
+    AzureStorageConfig,
     BackupRestoreConfig,
     LocalStorageConfig,
     S3StorageConfig,
@@ -14,6 +15,7 @@
     RestoreManager,
     WorkspaceToRestore,
 )
+from .backup_and_restore.storage.azure_storage import AzureStorage
 from .backup_and_restore.storage.local_storage import LocalStorage
 from .backup_and_restore.storage.s3_storage import S3Storage
 
@@ -67,13 +69,15 @@
     "StorageType",
     "LocalStorage",
     "S3Storage",
+    "AzureStorage",
     "WorkspaceFullLoad",
     "WorkspaceProvisioner",
     "UserIncrementalLoad",
     "UserGroupIncrementalLoad",
     "PermissionFullLoad",
     "LocalStorageConfig",
     "S3StorageConfig",
+    "AzureStorageConfig",
     "PermissionIncrementalLoad",
     "UserFullLoad",
     "UserGroupFullLoad",

packages/gooddata-pipelines/src/gooddata_pipelines/backup_and_restore/base_manager.py

@@ -6,7 +6,7 @@
 
 from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
 
-from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataApi
+from gooddata_pipelines.api import GoodDataApi
 from gooddata_pipelines.backup_and_restore.models.storage import (
     BackupRestoreConfig,
     StorageType,
@@ -18,6 +18,9 @@
     LocalStorage,
 )
 from gooddata_pipelines.backup_and_restore.storage.s3_storage import S3Storage
+from gooddata_pipelines.backup_and_restore.storage.azure_storage import (
+    AzureStorage,
+)
 from gooddata_pipelines.logger import LogObserver
 from gooddata_pipelines.utils.file_utils import JsonUtils, YamlUtils
 
@@ -44,6 +47,8 @@ def _get_storage(self, conf: BackupRestoreConfig) -> BackupStorage:
         """Returns the storage class based on the storage type."""
         if conf.storage_type == StorageType.S3:
             return S3Storage(conf)
+        elif conf.storage_type == StorageType.AZURE:
+            return AzureStorage(conf)
         elif conf.storage_type == StorageType.LOCAL:
             return LocalStorage(conf)
         else:

packages/gooddata-pipelines/src/gooddata_pipelines/backup_and_restore/models/storage.py

@@ -14,6 +14,7 @@ class StorageType(Enum):
 
     S3 = "s3"
     LOCAL = "local"
+    AZURE = "azure"
 
 
 class S3StorageConfig(BaseModel):
@@ -91,6 +92,102 @@ def from_aws_profile(
         return cls(backup_path=backup_path, bucket=bucket, profile=profile)
 
 
+class AzureStorageConfig(BaseModel):
+    """Configuration for Azure Blob Storage.
+
+    Can be created using the following constructor methods:
+    - `from_workload_identity`
+    - `from_connection_string`
+    - `from_service_principal`
+    """
+
+    backup_path: str
+    account_name: str
+    container: str
+    connection_string: str | None = None
+    client_id: str | None = None
+    client_secret: str | None = None
+    tenant_id: str | None = None
+
+    @classmethod
+    def from_workload_identity(
+        cls, backup_path: str, account_name: str, container: str
+    ) -> "AzureStorageConfig":
+        """Use Azure Workload Identity (for Kubernetes).
+
+        Args:
+            backup_path: The path to the backup directory.
+            account_name: The Azure storage account name.
+            container: The name of the blob container.
+
+        Returns:
+            AzureStorageConfig: The Azure storage configuration.
+        """
+        return cls(
+            backup_path=backup_path,
+            account_name=account_name,
+            container=container,
+        )
+
+    @classmethod
+    def from_connection_string(
+        cls,
+        backup_path: str,
+        account_name: str,
+        container: str,
+        connection_string: str,
+    ) -> "AzureStorageConfig":
+        """Use Azure Storage connection string.
+
+        Args:
+            backup_path: The path to the backup directory.
+            account_name: The Azure storage account name.
+            container: The name of the blob container.
+            connection_string: The Azure Storage connection string.
+
+        Returns:
+            AzureStorageConfig: The Azure storage configuration.
+        """
+        return cls(
+            backup_path=backup_path,
+            account_name=account_name,
+            container=container,
+            connection_string=connection_string,
+        )
+
+    @classmethod
+    def from_service_principal(
+        cls,
+        backup_path: str,
+        account_name: str,
+        container: str,
+        client_id: str,
+        client_secret: str,
+        tenant_id: str,
+    ) -> "AzureStorageConfig":
+        """Use Azure Service Principal credentials.
+
+        Args:
+            backup_path: The path to the backup directory.
+            account_name: The Azure storage account name.
+            container: The name of the blob container.
+            client_id: The Azure client ID.
+            client_secret: The Azure client secret.
+            tenant_id: The Azure tenant ID.
+
+        Returns:
+            AzureStorageConfig: The Azure storage configuration.
+        """
+        return cls(
+            backup_path=backup_path,
+            account_name=account_name,
+            container=container,
+            client_id=client_id,
+            client_secret=client_secret,
+            tenant_id=tenant_id,
+        )
+
+
 class LocalStorageConfig(BaseModel):
     """Placeholder for local storage config."""
 
@@ -102,14 +199,14 @@ class BackupRestoreConfig(BaseModel):
 
     Args:
         storage_type: The type of storage to use. Defaults to `StorageType.LOCAL`.
-        storage: Storage configuration. Either `S3StorageConfig` or `LocalStorageConfig`. Defaults to `LocalStorageConfig()`.
+        storage: Storage configuration. Either `S3StorageConfig`, `AzureStorageConfig`, or `LocalStorageConfig`. Defaults to `LocalStorageConfig()`.
         api_page_size: The page size for fetching workspace relationships. Defaults to `BackupSettings.API.PAGE_SIZE`.
         batch_size: The batch size for fetching workspace relationships. Defaults to `BackupSettings.API.BATCH_SIZE`.
         api_calls_per_second: The maximum API calls per second (rate limiting). Defaults to `BackupSettings.API.CALLS_PER_SECOND`.
     """
 
     storage_type: StorageType = Field(default=StorageType.LOCAL)
-    storage: S3StorageConfig | LocalStorageConfig = Field(
+    storage: S3StorageConfig | AzureStorageConfig | LocalStorageConfig = Field(
         default_factory=LocalStorageConfig
     )
     api_page_size: Annotated[
@@ -142,12 +239,17 @@ def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig":
 
     @model_validator(mode="after")
     def validate_storage(self) -> "BackupRestoreConfig":
-        """Check that the storage gets correct configuration when using S3 storage"""
+        """Check that the storage gets correct configuration"""
         if self.storage_type == StorageType.S3:
             if not isinstance(self.storage, S3StorageConfig):
                 raise ValueError(
                     "S3 storage must be configured with S3StorageConfig object"
                 )
+        elif self.storage_type == StorageType.AZURE:
+            if not isinstance(self.storage, AzureStorageConfig):
+                raise ValueError(
+                    "Azure storage must be configured with AzureStorageConfig object"
+                )
         elif self.storage_type == StorageType.LOCAL:
             if not isinstance(self.storage, LocalStorageConfig):
                 raise ValueError(