feat(gooddata-pipelines): Configurable parallelism of backup

Author: benkeanna

gooddata-pipelines/gooddata_pipelines/backup_and_restore/constants.py

@@ -1,3 +1,4 @@
+# (C) 2025 GoodData Corporation
 import datetime
 from dataclasses import dataclass
 
@@ -22,7 +23,7 @@ class DirNames:
 
 @dataclass(frozen=True)
 class ConcurrencyDefaults:
-    MAX_WORKERS = 2
+    MAX_WORKERS = 1
     DEFAULT_BATCH_SIZE = 100
 
 

gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/storage.py

@@ -6,7 +6,10 @@
 import yaml
 from pydantic import BaseModel, Field
 
-from gooddata_pipelines.backup_and_restore.constants import BackupSettings
+from gooddata_pipelines.backup_and_restore.constants import (
+    BackupSettings,
+    ConcurrencyDefaults,
+)
 
 
 class StorageType(Enum):
@@ -83,6 +86,14 @@ class BackupRestoreConfig(BaseModel):
             description="Batch size must be greater than 0",
         ),
     ] = Field(default=BackupSettings.DEFAULT_BATCH_SIZE)
+    max_workers: Annotated[
+        int,
+        Field(
+            gt=0,
+            lt=3,
+            description="Max workers must be greater than 0 and less than 3",
+        ),
+    ] = Field(default=ConcurrencyDefaults.MAX_WORKERS)
 
     @classmethod
     def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig":