Skip to content

Commit 40427d4

Browse files
kanterovkanterov
committed
Add pipelines
1 parent 38cac79 commit 40427d4

26 files changed

+2254
-0
lines changed

‎experimental/python/databricks/bundles/pipelines/__init__.py‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
__all__ = [
2+
"Adlsgen2Info",
3+
"Adlsgen2InfoDict",
4+
"Adlsgen2InfoParam",
5+
"AwsAttributes",
6+
"AwsAttributesDict",
7+
"AwsAttributesParam",
8+
"AwsAvailability",
9+
"AwsAvailabilityParam",
10+
"AzureAttributes",
11+
"AzureAttributesDict",
12+
"AzureAttributesParam",
13+
"AzureAvailability",
14+
"AzureAvailabilityParam",
15+
"ClusterLogConf",
16+
"ClusterLogConfDict",
17+
"ClusterLogConfParam",
18+
"CronTrigger",
19+
"CronTriggerDict",
20+
"CronTriggerParam",
21+
"DayOfWeek",
22+
"DayOfWeekParam",
23+
"DbfsStorageInfo",
24+
"DbfsStorageInfoDict",
25+
"DbfsStorageInfoParam",
26+
"DeploymentKind",
27+
"DeploymentKindParam",
28+
"EbsVolumeType",
29+
"EbsVolumeTypeParam",
30+
"FileLibrary",
31+
"FileLibraryDict",
32+
"FileLibraryParam",
33+
"Filters",
34+
"FiltersDict",
35+
"FiltersParam",
36+
"GcpAttributes",
37+
"GcpAttributesDict",
38+
"GcpAttributesParam",
39+
"GcpAvailability",
40+
"GcpAvailabilityParam",
41+
"GcsStorageInfo",
42+
"GcsStorageInfoDict",
43+
"GcsStorageInfoParam",
44+
"IngestionConfig",
45+
"IngestionConfigDict",
46+
"IngestionConfigParam",
47+
"IngestionGatewayPipelineDefinition",
48+
"IngestionGatewayPipelineDefinitionDict",
49+
"IngestionGatewayPipelineDefinitionParam",
50+
"IngestionPipelineDefinition",
51+
"IngestionPipelineDefinitionDict",
52+
"IngestionPipelineDefinitionParam",
53+
"InitScriptInfo",
54+
"InitScriptInfoDict",
55+
"InitScriptInfoParam",
56+
"LocalFileInfo",
57+
"LocalFileInfoDict",
58+
"LocalFileInfoParam",
59+
"LogAnalyticsInfo",
60+
"LogAnalyticsInfoDict",
61+
"LogAnalyticsInfoParam",
62+
"ManualTrigger",
63+
"ManualTriggerDict",
64+
"ManualTriggerParam",
65+
"MavenLibrary",
66+
"MavenLibraryDict",
67+
"MavenLibraryParam",
68+
"NotebookLibrary",
69+
"NotebookLibraryDict",
70+
"NotebookLibraryParam",
71+
"Notifications",
72+
"NotificationsDict",
73+
"NotificationsParam",
74+
"Permission",
75+
"PermissionDict",
76+
"PermissionParam",
77+
"Pipeline",
78+
"PipelineCluster",
79+
"PipelineClusterAutoscale",
80+
"PipelineClusterAutoscaleDict",
81+
"PipelineClusterAutoscaleMode",
82+
"PipelineClusterAutoscaleModeParam",
83+
"PipelineClusterAutoscaleParam",
84+
"PipelineClusterDict",
85+
"PipelineClusterParam",
86+
"PipelineDeployment",
87+
"PipelineDeploymentDict",
88+
"PipelineDeploymentParam",
89+
"PipelineDict",
90+
"PipelineLibrary",
91+
"PipelineLibraryDict",
92+
"PipelineLibraryParam",
93+
"PipelineParam",
94+
"PipelineTrigger",
95+
"PipelineTriggerDict",
96+
"PipelineTriggerParam",
97+
"ReportSpec",
98+
"ReportSpecDict",
99+
"ReportSpecParam",
100+
"RestartWindow",
101+
"RestartWindowDict",
102+
"RestartWindowParam",
103+
"RunAs",
104+
"RunAsDict",
105+
"RunAsParam",
106+
"S3StorageInfo",
107+
"S3StorageInfoDict",
108+
"S3StorageInfoParam",
109+
"SchemaSpec",
110+
"SchemaSpecDict",
111+
"SchemaSpecParam",
112+
"TableSpec",
113+
"TableSpecDict",
114+
"TableSpecParam",
115+
"TableSpecificConfig",
116+
"TableSpecificConfigDict",
117+
"TableSpecificConfigParam",
118+
"TableSpecificConfigScdType",
119+
"TableSpecificConfigScdTypeParam",
120+
"VolumesStorageInfo",
121+
"VolumesStorageInfoDict",
122+
"VolumesStorageInfoParam",
123+
"WorkspaceStorageInfo",
124+
"WorkspaceStorageInfoDict",
125+
"WorkspaceStorageInfoParam",
126+
]
127+
128+
129+
from databricks.bundles.compute._models.adlsgen2_info import (
130+
Adlsgen2Info,
131+
Adlsgen2InfoDict,
132+
Adlsgen2InfoParam,
133+
)
134+
from databricks.bundles.compute._models.aws_attributes import (
135+
AwsAttributes,
136+
AwsAttributesDict,
137+
AwsAttributesParam,
138+
)
139+
from databricks.bundles.compute._models.aws_availability import (
140+
AwsAvailability,
141+
AwsAvailabilityParam,
142+
)
143+
from databricks.bundles.compute._models.azure_attributes import (
144+
AzureAttributes,
145+
AzureAttributesDict,
146+
AzureAttributesParam,
147+
)
148+
from databricks.bundles.compute._models.azure_availability import (
149+
AzureAvailability,
150+
AzureAvailabilityParam,
151+
)
152+
from databricks.bundles.compute._models.cluster_log_conf import (
153+
ClusterLogConf,
154+
ClusterLogConfDict,
155+
ClusterLogConfParam,
156+
)
157+
from databricks.bundles.compute._models.dbfs_storage_info import (
158+
DbfsStorageInfo,
159+
DbfsStorageInfoDict,
160+
DbfsStorageInfoParam,
161+
)
162+
from databricks.bundles.compute._models.ebs_volume_type import (
163+
EbsVolumeType,
164+
EbsVolumeTypeParam,
165+
)
166+
from databricks.bundles.compute._models.gcp_attributes import (
167+
GcpAttributes,
168+
GcpAttributesDict,
169+
GcpAttributesParam,
170+
)
171+
from databricks.bundles.compute._models.gcp_availability import (
172+
GcpAvailability,
173+
GcpAvailabilityParam,
174+
)
175+
from databricks.bundles.compute._models.gcs_storage_info import (
176+
GcsStorageInfo,
177+
GcsStorageInfoDict,
178+
GcsStorageInfoParam,
179+
)
180+
from databricks.bundles.compute._models.init_script_info import (
181+
InitScriptInfo,
182+
InitScriptInfoDict,
183+
InitScriptInfoParam,
184+
)
185+
from databricks.bundles.compute._models.local_file_info import (
186+
LocalFileInfo,
187+
LocalFileInfoDict,
188+
LocalFileInfoParam,
189+
)
190+
from databricks.bundles.compute._models.log_analytics_info import (
191+
LogAnalyticsInfo,
192+
LogAnalyticsInfoDict,
193+
LogAnalyticsInfoParam,
194+
)
195+
from databricks.bundles.compute._models.maven_library import (
196+
MavenLibrary,
197+
MavenLibraryDict,
198+
MavenLibraryParam,
199+
)
200+
from databricks.bundles.compute._models.s3_storage_info import (
201+
S3StorageInfo,
202+
S3StorageInfoDict,
203+
S3StorageInfoParam,
204+
)
205+
from databricks.bundles.compute._models.volumes_storage_info import (
206+
VolumesStorageInfo,
207+
VolumesStorageInfoDict,
208+
VolumesStorageInfoParam,
209+
)
210+
from databricks.bundles.compute._models.workspace_storage_info import (
211+
WorkspaceStorageInfo,
212+
WorkspaceStorageInfoDict,
213+
WorkspaceStorageInfoParam,
214+
)
215+
from databricks.bundles.jobs._models.permission import (
216+
Permission,
217+
PermissionDict,
218+
PermissionParam,
219+
)
220+
from databricks.bundles.pipelines._models.cron_trigger import (
221+
CronTrigger,
222+
CronTriggerDict,
223+
CronTriggerParam,
224+
)
225+
from databricks.bundles.pipelines._models.day_of_week import DayOfWeek, DayOfWeekParam
226+
from databricks.bundles.pipelines._models.deployment_kind import (
227+
DeploymentKind,
228+
DeploymentKindParam,
229+
)
230+
from databricks.bundles.pipelines._models.file_library import (
231+
FileLibrary,
232+
FileLibraryDict,
233+
FileLibraryParam,
234+
)
235+
from databricks.bundles.pipelines._models.filters import (
236+
Filters,
237+
FiltersDict,
238+
FiltersParam,
239+
)
240+
from databricks.bundles.pipelines._models.ingestion_config import (
241+
IngestionConfig,
242+
IngestionConfigDict,
243+
IngestionConfigParam,
244+
)
245+
from databricks.bundles.pipelines._models.ingestion_gateway_pipeline_definition import (
246+
IngestionGatewayPipelineDefinition,
247+
IngestionGatewayPipelineDefinitionDict,
248+
IngestionGatewayPipelineDefinitionParam,
249+
)
250+
from databricks.bundles.pipelines._models.ingestion_pipeline_definition import (
251+
IngestionPipelineDefinition,
252+
IngestionPipelineDefinitionDict,
253+
IngestionPipelineDefinitionParam,
254+
)
255+
from databricks.bundles.pipelines._models.manual_trigger import (
256+
ManualTrigger,
257+
ManualTriggerDict,
258+
ManualTriggerParam,
259+
)
260+
from databricks.bundles.pipelines._models.notebook_library import (
261+
NotebookLibrary,
262+
NotebookLibraryDict,
263+
NotebookLibraryParam,
264+
)
265+
from databricks.bundles.pipelines._models.notifications import (
266+
Notifications,
267+
NotificationsDict,
268+
NotificationsParam,
269+
)
270+
from databricks.bundles.pipelines._models.pipeline import (
271+
Pipeline,
272+
PipelineDict,
273+
PipelineParam,
274+
)
275+
from databricks.bundles.pipelines._models.pipeline_cluster import (
276+
PipelineCluster,
277+
PipelineClusterDict,
278+
PipelineClusterParam,
279+
)
280+
from databricks.bundles.pipelines._models.pipeline_cluster_autoscale import (
281+
PipelineClusterAutoscale,
282+
PipelineClusterAutoscaleDict,
283+
PipelineClusterAutoscaleParam,
284+
)
285+
from databricks.bundles.pipelines._models.pipeline_cluster_autoscale_mode import (
286+
PipelineClusterAutoscaleMode,
287+
PipelineClusterAutoscaleModeParam,
288+
)
289+
from databricks.bundles.pipelines._models.pipeline_deployment import (
290+
PipelineDeployment,
291+
PipelineDeploymentDict,
292+
PipelineDeploymentParam,
293+
)
294+
from databricks.bundles.pipelines._models.pipeline_library import (
295+
PipelineLibrary,
296+
PipelineLibraryDict,
297+
PipelineLibraryParam,
298+
)
299+
from databricks.bundles.pipelines._models.pipeline_trigger import (
300+
PipelineTrigger,
301+
PipelineTriggerDict,
302+
PipelineTriggerParam,
303+
)
304+
from databricks.bundles.pipelines._models.report_spec import (
305+
ReportSpec,
306+
ReportSpecDict,
307+
ReportSpecParam,
308+
)
309+
from databricks.bundles.pipelines._models.restart_window import (
310+
RestartWindow,
311+
RestartWindowDict,
312+
RestartWindowParam,
313+
)
314+
from databricks.bundles.pipelines._models.run_as import RunAs, RunAsDict, RunAsParam
315+
from databricks.bundles.pipelines._models.schema_spec import (
316+
SchemaSpec,
317+
SchemaSpecDict,
318+
SchemaSpecParam,
319+
)
320+
from databricks.bundles.pipelines._models.table_spec import (
321+
TableSpec,
322+
TableSpecDict,
323+
TableSpecParam,
324+
)
325+
from databricks.bundles.pipelines._models.table_specific_config import (
326+
TableSpecificConfig,
327+
TableSpecificConfigDict,
328+
TableSpecificConfigParam,
329+
)
330+
from databricks.bundles.pipelines._models.table_specific_config_scd_type import (
331+
TableSpecificConfigScdType,
332+
TableSpecificConfigScdTypeParam,
333+
)

‎experimental/python/databricks/bundles/pipelines/_models/cron_trigger.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
from dataclasses import dataclass
2+
from typing import TYPE_CHECKING, TypedDict
3+
4+
from databricks.bundles.core._transform import _transform
5+
from databricks.bundles.core._transform_to_json import _transform_to_json_value
6+
from databricks.bundles.core._variable import VariableOrOptional
7+
8+
if TYPE_CHECKING:
9+
from typing_extensions import Self
10+
11+
12+
@dataclass(kw_only=True)
13+
class CronTrigger:
14+
""""""
15+
16+
quartz_cron_schedule: VariableOrOptional[str] = None
17+
18+
timezone_id: VariableOrOptional[str] = None
19+
20+
@classmethod
21+
def from_dict(cls, value: "CronTriggerDict") -> "Self":
22+
return _transform(cls, value)
23+
24+
def as_dict(self) -> "CronTriggerDict":
25+
return _transform_to_json_value(self) # type:ignore
26+
27+
28+
class CronTriggerDict(TypedDict, total=False):
29+
""""""
30+
31+
quartz_cron_schedule: VariableOrOptional[str]
32+
33+
timezone_id: VariableOrOptional[str]
34+
35+
36+
CronTriggerParam = CronTriggerDict | CronTrigger

‎experimental/python/databricks/bundles/pipelines/_models/day_of_week.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
from enum import Enum
2+
from typing import Literal
3+
4+
5+
class DayOfWeek(Enum):
6+
"""
7+
Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).
8+
If not specified all days of the week will be used.
9+
"""
10+
11+
MONDAY = "MONDAY"
12+
TUESDAY = "TUESDAY"
13+
WEDNESDAY = "WEDNESDAY"
14+
THURSDAY = "THURSDAY"
15+
FRIDAY = "FRIDAY"
16+
SATURDAY = "SATURDAY"
17+
SUNDAY = "SUNDAY"
18+
19+
20+
DayOfWeekParam = (
21+
Literal[
22+
"MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", "SUNDAY"
23+
]
24+
| DayOfWeek
25+
)

0 commit comments

Comments
 (0)