feat(gooddata-pipelines): Configurable parallelism of backup

Author: benkeanna

gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_manager.py

@@ -347,7 +347,7 @@ def _process_batches_in_parallel(
         stop_event = threading.Event()
 
         with ThreadPoolExecutor(
-            max_workers=BackupSettings.MAX_WORKERS
+            max_workers=self.config.max_workers
         ) as executor:
             # Set the futures tasks.
             futures = []

gooddata-pipelines/gooddata_pipelines/backup_and_restore/constants.py

@@ -1,3 +1,4 @@
+# (C) 2025 GoodData Corporation
 import datetime
 from dataclasses import dataclass
 
@@ -22,7 +23,7 @@ class DirNames:
 
 @dataclass(frozen=True)
 class ConcurrencyDefaults:
-    MAX_WORKERS = 2
+    MAX_WORKERS = 1
     DEFAULT_BATCH_SIZE = 100
 
 

gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/storage.py

@@ -83,6 +83,14 @@ class BackupRestoreConfig(BaseModel):
             description="Batch size must be greater than 0",
         ),
     ] = Field(default=BackupSettings.DEFAULT_BATCH_SIZE)
+    max_workers: Annotated[
+        int,
+        Field(
+            gt=0,
+            lt=3,
+            description="Max workers must be greater than 0 and less than 3",
+        ),
+    ] = Field(default=BackupSettings.MAX_WORKERS)
 
     @classmethod
     def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig":