feat(workspace-backup): Backup to Azure blob storage

Author: benkeanna

.gitignore

@@ -12,6 +12,16 @@ packages/gooddata-sdk/tests/catalog/translate
 .vscode
 .ruff_cache
 
+# Python build artifacts
+.tox
+*.egg-info
+dist/
+build/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
 docs/node_modules
 docs/public
 docs/resources/_gen

packages/gooddata-pipelines/README.md

@@ -11,7 +11,7 @@ You can use the package to manage following resources in GDC:
    - User Data Filters
    - Child workspaces (incl. Workspace Data Filter settings)
 1. Backup and restore of workspaces
-   - Create and backup snapshots of workspace metadata.
+   - Create and backup snapshots of workspace metadata to local storage, AWS S3, or Azure Blob Storage
 1. LDM Extension
    - extend the Logical Data Model of a child workspace with custom datasets and fields
 
@@ -34,7 +34,7 @@ import logging
 from csv import DictReader
 from pathlib import Path
 
-# Import the Entity Provisioner class and corresponding model from gooddata_pipelines library
+# Import the Entity Provisioner class and corresponding model from the gooddata_pipelines library
 from gooddata_pipelines import UserFullLoad, UserProvisioner
 
 # Create the Provisioner instance - you can also create the instance from a GDC yaml profile
@@ -62,12 +62,200 @@ provisioner.full_load(full_load_data)
 
 ```
 
+Ready-made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository.
+
+## Backup and Restore of Workspaces
+
+The backup and restore module allows you to create snapshots of GoodData Cloud workspaces and restore them later. This is useful for:
+
+- Creating backups before major changes
+- Migrating workspaces between environments
+- Disaster recovery scenarios
+- Copying workspace configurations
+
+### Backup
+
+The module supports three backup modes:
+
+1. **List of workspaces** - Backup specific workspaces by providing a list of workspace IDs
+2. **Workspace hierarchies** - Backup a workspace and all its direct and indirect children
+3. **Entire organization** - Backup all workspaces in the organization
+
+Each backup includes:
+- Workspace declarative model (logical data model, analytics model, permissions)
+- User data filters
+- Filter views
+- Automations
+
+#### Storage Options
+
+Backups can be stored in:
+- **Local storage** - Save backups to a local directory
+- **S3 storage** - Upload backups to an AWS S3 bucket
+- **Azure Blob Storage** - Upload backups to Azure Blob Storage
+
+#### Basic Usage
+
+```python
+import os
+from pathlib import Path
+
+from gooddata_pipelines import BackupManager
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    LocalStorageConfig,
+    StorageType,
+)
+from gooddata_pipelines.logger.logger import LogObserver
+
+# Optionally, subscribe a standard Python logger to the LogObserver
+import logging
+logger = logging.getLogger(__name__)
+LogObserver().subscribe(logger)
+
+# Configure backup storage
+config = BackupRestoreConfig(
+    storage_type=StorageType.LOCAL,
+    storage=LocalStorageConfig(),
+    batch_size=10,  # Number of workspaces to process in one batch
+    api_calls_per_second=10,  # Rate limit for API calls
+)
+
+# Create the BackupManager instance
+backup_manager = BackupManager.create(
+    config=config,
+    host=os.environ["GDC_HOSTNAME"],
+    token=os.environ["GDC_AUTH_TOKEN"]
+)
+
+# Backup specific workspaces
+workspace_ids = ["workspace1", "workspace2", "workspace3"]
+backup_manager.backup_workspaces(workspace_ids=workspace_ids)
+
+# Or read workspace IDs from a CSV file
+backup_manager.backup_workspaces(path_to_csv="workspaces.csv")
+
+# Backup workspace hierarchies (workspace + all children)
+backup_manager.backup_hierarchies(workspace_ids=["parent_workspace"])
+
+# Backup entire organization
+backup_manager.backup_entire_organization()
+```
+
+#### Using S3 Storage
+
+```python
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    S3StorageConfig,
+    StorageType,
+)
+
+# Configure S3 storage with explicit credentials
+config = BackupRestoreConfig(
+    storage_type=StorageType.S3,
+    storage=S3StorageConfig(
+        bucket="my-backup-bucket",
+        backup_path="gooddata-backups/",
+        aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+        aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+        aws_default_region="us-east-1"
+    ),
+)
+
+# Or use AWS profile
+config = BackupRestoreConfig(
+    storage_type=StorageType.S3,
+    storage=S3StorageConfig(
+        bucket="my-backup-bucket",
+        backup_path="gooddata-backups/",
+        profile="my-aws-profile"
+    ),
+)
+
+backup_manager = BackupManager.create(
+    config=config,
+    host=os.environ["GDC_HOSTNAME"],
+    token=os.environ["GDC_AUTH_TOKEN"]
+)
+
+backup_manager.backup_workspaces(workspace_ids=["workspace1"])
+```
+
+#### Using Azure Blob Storage
+
+```python
+from gooddata_pipelines.backup_and_restore.models.storage import (
+    BackupRestoreConfig,
+    AzureStorageConfig,
+    StorageType,
+)
+
+# Configure Azure storage
+config = BackupRestoreConfig(
+    storage_type=StorageType.AZURE,
+    storage=AzureStorageConfig(
+        account_name="mystorageaccount",
+        container="my-backup-container",
+        backup_path="gooddata-backups/"
+    ),
+)
+
+backup_manager = BackupManager.create(
+    config=config,
+    host=os.environ["GDC_HOSTNAME"],
+    token=os.environ["GDC_AUTH_TOKEN"]
+)
+
+backup_manager.backup_workspaces(workspace_ids=["workspace1"])
+```
+
+#### Using GoodData Profile
+
+You can also create the BackupManager from a GoodData profile file:
+
+```python
+from pathlib import Path
+
+backup_manager = BackupManager.create_from_profile(
+    config=config,
+    profile="production",
+    profiles_path=Path.home() / ".gooddata" / "profiles.yaml"
+)
+```
+
+#### CSV File Format
+
+When providing workspace IDs via a CSV file, the file should have a `workspace_id` column:
+
+```csv
+workspace_id
+workspace1
+workspace2
+workspace3
+```
+
+#### Configuration Options
+
+The `BackupRestoreConfig` class accepts the following parameters:
+- `storage_type` - Type of storage (`StorageType.LOCAL`, `StorageType.S3`, or `StorageType.AZURE`)
+- `storage` - Storage-specific configuration (`LocalStorageConfig`, `S3StorageConfig`, or `AzureStorageConfig`)
+- `batch_size` (optional, default: 10) - Number of workspaces to process in one batch
+- `api_calls_per_second` (optional, default: 10) - Rate limit for API calls to avoid throttling
+- `api_page_size` (optional, default: 500) - Page size for paginated API calls
+
+#### Error Handling and Retries
+
+The backup process includes automatic retry logic with exponential backoff. If a batch fails, it will retry up to 3 times before failing completely. Individual workspace errors are logged but don't stop the entire backup process.
+
+### Restore
+
+Note: Restore functionality is currently in development.
+
 ## Bugs & Requests
 
-Please use the [GitHub issue tracker](https://github.com/gooddata/gooddata-python-sdk/issues) to submit bugs
-or request features.
+Please use the [GitHub issue tracker](https://github.com/gooddata/gooddata-python-sdk/issues) to submit bugs or request features.
 
 ## Changelog
 
-See [Github releases](https://github.com/gooddata/gooddata-python-sdk/releases) for released versions
-and a list of changes.
+See [GitHub releases](https://github.com/gooddata/gooddata-python-sdk/releases) for released versions and a list of changes.

packages/gooddata-pipelines/pyproject.toml

@@ -14,6 +14,8 @@ dependencies = [
     "gooddata-sdk~=1.54.0",
     "boto3 (>=1.39.3,<2.0.0)",
     "boto3-stubs (>=1.39.3,<2.0.0)",
+    "azure-storage-blob (>=12.19.0,<13.0.0)",
+    "azure-identity (>=1.15.0,<2.0.0)",
     "types-pyyaml (>=6.0.12.20250326,<7.0.0)",
 ]
 

packages/gooddata-pipelines/src/gooddata_pipelines/__init__.py

@@ -5,6 +5,7 @@
 # -------- Backup and Restore --------
 from .backup_and_restore.backup_manager import BackupManager
 from .backup_and_restore.models.storage import (
+    AzureStorageConfig,
     BackupRestoreConfig,
     LocalStorageConfig,
     S3StorageConfig,
@@ -14,6 +15,7 @@
     RestoreManager,
     WorkspaceToRestore,
 )
+from .backup_and_restore.storage.azure_storage import AzureStorage
 from .backup_and_restore.storage.local_storage import LocalStorage
 from .backup_and_restore.storage.s3_storage import S3Storage
 
@@ -67,13 +69,15 @@
     "StorageType",
     "LocalStorage",
     "S3Storage",
+    "AzureStorage",
     "WorkspaceFullLoad",
     "WorkspaceProvisioner",
     "UserIncrementalLoad",
     "UserGroupIncrementalLoad",
     "PermissionFullLoad",
     "LocalStorageConfig",
     "S3StorageConfig",
+    "AzureStorageConfig",
     "PermissionIncrementalLoad",
     "UserFullLoad",
     "UserGroupFullLoad",

packages/gooddata-pipelines/src/gooddata_pipelines/backup_and_restore/base_manager.py

@@ -6,7 +6,7 @@
 
 from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content
 
-from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataApi
+from gooddata_pipelines.api import GoodDataApi
 from gooddata_pipelines.backup_and_restore.models.storage import (
     BackupRestoreConfig,
     StorageType,
@@ -18,6 +18,9 @@
     LocalStorage,
 )
 from gooddata_pipelines.backup_and_restore.storage.s3_storage import S3Storage
+from gooddata_pipelines.backup_and_restore.storage.azure_storage import (
+    AzureStorage,
+)
 from gooddata_pipelines.logger import LogObserver
 from gooddata_pipelines.utils.file_utils import JsonUtils, YamlUtils
 
@@ -44,6 +47,8 @@ def _get_storage(self, conf: BackupRestoreConfig) -> BackupStorage:
         """Returns the storage class based on the storage type."""
         if conf.storage_type == StorageType.S3:
             return S3Storage(conf)
+        elif conf.storage_type == StorageType.AZURE:
+            return AzureStorage(conf)
         elif conf.storage_type == StorageType.LOCAL:
             return LocalStorage(conf)
         else: