From a21573d0197e3111fcd2dff4739e6779e7138e68 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 8 Aug 2025 13:15:47 +0200 Subject: [PATCH 1/6] [Python] Keep newly deprecated fields in Python code --- bundle/docsgen/main.go | 2 +- bundle/internal/annotation/descriptor.go | 11 ++++- bundle/internal/schema/annotations.go | 5 +-- .../schema/annotations_openapi_overrides.yml | 37 ++++++++++++++- bundle/internal/schema/parser.go | 35 +++++++++++++-- bundle/schema/jsonschema.json | 6 ++- .../codegen/codegen/generated_dataclass.py | 45 ++++++++++++++++--- .../python/codegen/codegen/generated_enum.py | 9 +++- .../python/codegen/codegen/jsonschema.py | 2 + experimental/python/codegen/codegen/main.py | 10 ++++- .../codegen_tests/test_generated_dataclass.py | 2 + .../codegen_tests/test_generated_enum.py | 1 + .../bundles/jobs/_models/environment.py | 10 +++++ .../bundles/pipelines/_models/pipeline.py | 4 +- libs/jsonschema/extension.go | 4 ++ 15 files changed, 161 insertions(+), 22 deletions(-) diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index 593f3e1dd1..697f716819 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -137,7 +137,7 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Deprecated = true s.DeprecationMessage = a.DeprecationMessage } - if a.ForceNotDeprecated { + if a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedTrue { s.Deprecated = false s.DeprecationMessage = "" } diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go index 562d992c4c..0dfd0f92e6 100644 --- a/bundle/internal/annotation/descriptor.go +++ b/bundle/internal/annotation/descriptor.go @@ -1,5 +1,12 @@ package annotation +type PythonKeepDeprecated string + +const ( + PythonKeepDeprecatedTrue PythonKeepDeprecated = "true" + PythonKeepDeprecatedUnset PythonKeepDeprecated = "" +) + type Descriptor struct { Description string `json:"description,omitempty"` MarkdownDescription string `json:"markdown_description,omitempty"` @@ -10,8 +17,8 @@ type Descriptor struct { DeprecationMessage string `json:"deprecation_message,omitempty"` Preview string `json:"x-databricks-preview,omitempty"` - // If true, takes priority over 'DeprecationMessage' - ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"` + // If true, keep the field in Python code even if it is deprecated. + PythonKeepDeprecated PythonKeepDeprecated `json:"x-databricks-python-keep-deprecated,omitempty"` } const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 8f65368a74..ae57a54598 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -138,9 +138,8 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Preview = a.Preview } - if a.ForceNotDeprecated { - s.Deprecated = false - s.DeprecationMessage = "" + if a.PythonKeepDeprecated != annotation.PythonKeepDeprecatedUnset { + s.PythonKeepDeprecated = a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedTrue } s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription) diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index bddcec5784..ad02c8e60e 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -210,6 +210,8 @@ github.com/databricks/cli/bundle/config/resources.Job: ``` For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md). + "format": + "x-databricks-python-keep-deprecated": "false" "health": "description": |- PLACEHOLDER @@ -311,6 +313,8 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: "permissions": "description": |- PLACEHOLDER + "rate_limits": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/cli/bundle/config/resources.ModelServingEndpointPermissionLevel: "_": "enum": @@ -354,11 +358,11 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "description": |- PLACEHOLDER "target": - "force_not_deprecated": |- - true + "x-databricks-python-keep-deprecated": "true" "trigger": "deprecation_message": |- Use continuous instead + "x-databricks-python-keep-deprecated": "false" github.com/databricks/cli/bundle/config/resources.PipelinePermissionLevel: "_": "enum": @@ -710,6 +714,8 @@ github.com/databricks/databricks-sdk-go/service/compute.DockerImage: "description": |- PLACEHOLDER github.com/databricks/databricks-sdk-go/service/compute.Environment: + "client": + "x-databricks-python-keep-deprecated": "true" "dependencies": "description": |- List of pip dependencies, as supported by the version of pip in this environment. @@ -717,10 +723,17 @@ github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes: "availability": "description": |- PLACEHOLDER + "use_preemptible_executors": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo: "abfss": "description": |- Contains the Azure Data Lake Storage destination path + "dbfs": + "x-databricks-python-keep-deprecated": "false" +github.com/databricks/databricks-sdk-go/service/compute.Library: + "egg": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo: "log_analytics_primary_key": "description": |- @@ -743,6 +756,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.GitSource: "git_snapshot": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications: + "no_alert_for_skipped_runs": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment: "spec": "description": |- @@ -762,6 +778,11 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: "python_named_params": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask: + "jar_uri": + "x-databricks-python-keep-deprecated": "false" + "run_as_repl": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/jobs.Subscription: "subscribers": "description": |- @@ -783,6 +804,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "health": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications: + "no_alert_for_skipped_runs": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings: "table_update": "description": |- @@ -798,10 +822,14 @@ github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger: "timezone_id": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition: + "connection_id": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary: "whl": "deprecation_message": |- This field is deprecated + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "cron": "description": |- @@ -809,6 +837,11 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters: + "invalid_keywords": + "x-databricks-python-keep-deprecated": "false" + "valid_topics": + "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/serving.Route: "served_entity_name": "description": |- diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 8db276f061..27ac5ce8c6 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -196,6 +196,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid if description == "" { addEmptyOverride(k, basePath, overrides) } + + // if field got deprecated outside of preview it can't be just removed and we keep it + if refProp.Deprecated && refProp.Preview != "PRIVATE" { + addPythonKeepDeprecatedOverride(k, basePath, overrides) + } } else { addEmptyOverride(k, basePath, overrides) } @@ -241,22 +246,46 @@ func prependCommentToFile(outputPath, comment string) error { return err } -func addEmptyOverride(key, pkg string, overridesFile annotation.File) { +// addPythonKeepDeprecatedOverride adds an override with "x-databricks-python-keep-deprecated: true" to +// annotations_openapi_overrides.yaml unless it already specifies a value for "x-databricks-python-keep-deprecated". +// +// This way if field got deprecated, we add an override to keep it in Python code, until we revisit deprecation. +func addPythonKeepDeprecatedOverride(key, pkg string, overridesFile annotation.File) { if overridesFile[pkg] == nil { overridesFile[pkg] = map[string]annotation.Descriptor{} } overrides := overridesFile[pkg] - if overrides[key].Description == "" { - overrides[key] = annotation.Descriptor{Description: annotation.Placeholder} + + a, ok := overrides[key] + if !ok { + a = annotation.Descriptor{} + } + + // if we don't have explicit value, it means that field just got deprecated, + // and we should keep it in Python code + if a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedUnset { + a.PythonKeepDeprecated = annotation.PythonKeepDeprecatedTrue + } + + overrides[key] = a +} + +func addEmptyOverride(key, pkg string, overridesFile annotation.File) { + if overridesFile[pkg] == nil { + overridesFile[pkg] = map[string]annotation.Descriptor{} } + overrides := overridesFile[pkg] + a, ok := overrides[key] if !ok { a = annotation.Descriptor{} } + if a.Description == "" { a.Description = annotation.Placeholder } + overrides[key] = a } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c0aec94eb8..aeb0d5eff9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1273,7 +1273,10 @@ }, "target": { "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "x-databricks-python-keep-deprecated": true, + "deprecated": true }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", @@ -3760,6 +3763,7 @@ "description": "Use `environment_version` instead.", "$ref": "#/$defs/string", "deprecationMessage": "This field is deprecated", + "x-databricks-python-keep-deprecated": true, "deprecated": true }, "dependencies": { diff --git a/experimental/python/codegen/codegen/generated_dataclass.py b/experimental/python/codegen/codegen/generated_dataclass.py index e1888aadab..9070a1e38c 100644 --- a/experimental/python/codegen/codegen/generated_dataclass.py +++ b/experimental/python/codegen/codegen/generated_dataclass.py @@ -102,6 +102,11 @@ class GeneratedField: be marked as experimental in docstring. """ + deprecated: bool + """ + If true, the field is deprecated and should be marked as deprecated in docstring. + """ + def __post_init__(self): if self.default_factory is not None and self.default is not None: raise ValueError("Can't have both default and default_factory", self) @@ -131,6 +136,7 @@ class GeneratedDataclass: fields: list[GeneratedField] extends: list[GeneratedType] experimental: bool + deprecated: bool def generate_field( @@ -156,6 +162,7 @@ def generate_field( default_factory="dict", create_func_default="None", experimental=prop.stage == Stage.PRIVATE, + deprecated=prop.deprecated or False, ) elif field_type.name == "VariableOrList": return GeneratedField( @@ -168,6 +175,7 @@ def generate_field( default_factory="list", create_func_default="None", experimental=prop.stage == Stage.PRIVATE, + deprecated=prop.deprecated or False, ) elif is_required: return GeneratedField( @@ -180,6 +188,7 @@ def generate_field( default_factory=None, create_func_default=None, experimental=prop.stage == Stage.PRIVATE, + deprecated=prop.deprecated or False, ) else: return GeneratedField( @@ -192,6 +201,7 @@ def generate_field( default_factory=None, create_func_default="None", experimental=prop.stage == Stage.PRIVATE, + deprecated=prop.deprecated or False, ) @@ -326,6 +336,7 @@ def generate_dataclass( fields=fields, extends=extends, experimental=schema.stage == Stage.PRIVATE, + deprecated=schema.deprecated or False, ) @@ -365,10 +376,19 @@ def _append_dataclass(b: CodeBuilder, generated: GeneratedDataclass): b.append(":").newline() # FIXME should contain class docstring - if not generated.description and not generated.experimental: + if ( + not generated.description + and not generated.experimental + and not generated.deprecated + ): b.indent().append_triple_quote().append_triple_quote().newline().newline() else: - _append_description(b, generated.description, generated.experimental) + _append_description( + b, + generated.description, + experimental=generated.experimental, + deprecated=generated.deprecated, + ) def _append_field(b: CodeBuilder, field: GeneratedField): @@ -446,7 +466,12 @@ def _append_typed_dict(b: CodeBuilder, generated: GeneratedDataclass): b.indent().append_triple_quote().append_triple_quote().newline().newline() -def _append_description(b: CodeBuilder, description: Optional[str], experimental: bool): +def _append_description( + b: CodeBuilder, description: Optional[str], *, experimental: bool, deprecated: bool +): + if deprecated: + description = "[DEPRECATED] " + (description or "") + if description or experimental: b.indent().append_triple_quote().newline() if experimental: @@ -472,7 +497,12 @@ def get_code(generated: GeneratedDataclass) -> str: for field in generated.fields: _append_field(b, field) - _append_description(b, field.description, field.experimental) + _append_description( + b, + field.description, + experimental=field.experimental, + deprecated=field.deprecated, + ) b.newline() @@ -485,7 +515,12 @@ def get_code(generated: GeneratedDataclass) -> str: for field in generated.fields: _append_typed_dict_field(b, field) - _append_description(b, field.description, field.experimental) + _append_description( + b, + field.description, + experimental=field.experimental, + deprecated=field.deprecated, + ) b.newline() diff --git a/experimental/python/codegen/codegen/generated_enum.py b/experimental/python/codegen/codegen/generated_enum.py index 7f413d5b38..f2eecf82e4 100644 --- a/experimental/python/codegen/codegen/generated_enum.py +++ b/experimental/python/codegen/codegen/generated_enum.py @@ -15,6 +15,7 @@ class GeneratedEnum: values: dict[str, str] description: Optional[str] experimental: bool + deprecated: bool def generate_enum(namespace: str, schema_name: str, schema: Schema) -> GeneratedEnum: @@ -35,6 +36,7 @@ def generate_enum(namespace: str, schema_name: str, schema: Schema) -> Generated values=values, description=schema.description, experimental=schema.stage == Stage.PRIVATE, + deprecated=schema.deprecated or False, ) @@ -48,7 +50,12 @@ def get_code(generated: GeneratedEnum) -> str: b.append(f"class {generated.class_name}(Enum):") b.newline() - _append_description(b, generated.description, generated.experimental) + _append_description( + b, + generated.description, + experimental=generated.experimental, + deprecated=generated.deprecated, + ) # Example: # diff --git a/experimental/python/codegen/codegen/jsonschema.py b/experimental/python/codegen/codegen/jsonschema.py index f07f8d2d1d..5a252a2b58 100644 --- a/experimental/python/codegen/codegen/jsonschema.py +++ b/experimental/python/codegen/codegen/jsonschema.py @@ -16,6 +16,7 @@ class Property: ref: str description: Optional[str] = None deprecated: Optional[bool] = None + keep_deprecated: Optional[bool] = None stage: Optional[str] = None @@ -101,6 +102,7 @@ def _parse_bool(value) -> Optional[bool]: ref=v["$ref"], description=v.get("description"), deprecated=_parse_bool(v.get("deprecated")), + keep_deprecated=_parse_bool(v.get("x-databricks-python-keep-deprecated")), stage=v.get("x-databricks-preview"), ) diff --git a/experimental/python/codegen/codegen/main.py b/experimental/python/codegen/codegen/main.py index 18a11cc627..5380b0b968 100644 --- a/experimental/python/codegen/codegen/main.py +++ b/experimental/python/codegen/codegen/main.py @@ -83,7 +83,7 @@ def _remove_deprecated_fields( if schema.type == openapi.SchemaType.OBJECT: new_properties = {} for field_name, field in schema.properties.items(): - if field.deprecated: + if field.deprecated and not field.keep_deprecated: continue new_properties[field_name] = field @@ -243,7 +243,13 @@ def _collect_reachable_schemas( if not include_private and field.stage == openapi.Stage.PRIVATE: continue - if not include_deprecated and field.deprecated: + if ( + not include_deprecated + and field.deprecated + # we don't remove keep_deprecated fields so they should be considered + # reachable + and not field.keep_deprecated + ): continue if name not in reachable: diff --git a/experimental/python/codegen/codegen_tests/test_generated_dataclass.py b/experimental/python/codegen/codegen_tests/test_generated_dataclass.py index c41fdfd6d6..22de11b074 100644 --- a/experimental/python/codegen/codegen_tests/test_generated_dataclass.py +++ b/experimental/python/codegen/codegen_tests/test_generated_dataclass.py @@ -69,9 +69,11 @@ def test_generate_dataclass(): param_type_name=variable_or_type(str_type(), is_required=True), type_name=variable_or_type(str_type(), is_required=True), experimental=False, + deprecated=False, ), ], experimental=False, + deprecated=False, ) diff --git a/experimental/python/codegen/codegen_tests/test_generated_enum.py b/experimental/python/codegen/codegen_tests/test_generated_enum.py index f3f122b1a1..300b021e6a 100644 --- a/experimental/python/codegen/codegen_tests/test_generated_enum.py +++ b/experimental/python/codegen/codegen_tests/test_generated_enum.py @@ -19,4 +19,5 @@ def test_generate_enum(): values={"MY_ENUM_VALUE": "myEnumValue"}, description="enum description", experimental=False, + deprecated=False, ) diff --git a/experimental/python/databricks/bundles/jobs/_models/environment.py b/experimental/python/databricks/bundles/jobs/_models/environment.py index c8bdee0917..f7606430ec 100644 --- a/experimental/python/databricks/bundles/jobs/_models/environment.py +++ b/experimental/python/databricks/bundles/jobs/_models/environment.py @@ -16,6 +16,11 @@ class Environment: In this minimal environment spec, only pip dependencies are supported. """ + client: VariableOrOptional[str] = None + """ + [DEPRECATED] Use `environment_version` instead. + """ + dependencies: VariableOrList[str] = field(default_factory=list) """ List of pip dependencies, as supported by the version of pip in this environment. @@ -46,6 +51,11 @@ def as_dict(self) -> "EnvironmentDict": class EnvironmentDict(TypedDict, total=False): """""" + client: VariableOrOptional[str] + """ + [DEPRECATED] Use `environment_version` instead. + """ + dependencies: VariableOrList[str] """ List of pip dependencies, as supported by the version of pip in this environment. diff --git a/experimental/python/databricks/bundles/pipelines/_models/pipeline.py b/experimental/python/databricks/bundles/pipelines/_models/pipeline.py index 6ca71f4dae..817689f338 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/experimental/python/databricks/bundles/pipelines/_models/pipeline.py @@ -203,7 +203,7 @@ class Pipeline(Resource): target: VariableOrOptional[str] = None """ - Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. + [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ @classmethod @@ -361,7 +361,7 @@ class PipelineDict(TypedDict, total=False): target: VariableOrOptional[str] """ - Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. + [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ diff --git a/libs/jsonschema/extension.go b/libs/jsonschema/extension.go index d400e31e56..ddd55fd13e 100644 --- a/libs/jsonschema/extension.go +++ b/libs/jsonschema/extension.go @@ -55,4 +55,8 @@ type Extension struct { // This field is not in the JSON schema spec, but it is supported in VSCode // It hides a property from IntelliSense (autocomplete suggestions). DoNotSuggest bool `json:"doNotSuggest,omitempty"` + + // PythonKeepDeprecated is a custom extension that forces the field to be + // present in the generated Python code, even if it is deprecated. + PythonKeepDeprecated bool `json:"x-databricks-python-keep-deprecated,omitempty"` } From afedb6aa72da8f4760fa0c4ae357738bd689f5da Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 22 Aug 2025 12:49:20 +0200 Subject: [PATCH 2/6] Add x-databricks-python-keep-deprecated to keywords --- .github/workflows/push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index dea881e1a4..5c62d25f14 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -180,6 +180,7 @@ jobs: a.addKeyword('doNotSuggest'); a.addKeyword('markdownDescription'); a.addKeyword('x-databricks-preview'); + a.addKeyword('x-databricks-python-keep-deprecated'); }" >> keywords.js for file in ./bundle/internal/schema/testdata/pass/*.yml; do From 3948db9198d7d81720fc03604b57b8db1ea806f1 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 29 Aug 2025 10:44:42 +0200 Subject: [PATCH 3/6] Revert --- bundle/docsgen/main.go | 2 +- bundle/internal/annotation/descriptor.go | 11 +----- bundle/internal/schema/annotations.go | 5 ++- .../schema/annotations_openapi_overrides.yml | 37 +------------------ bundle/internal/schema/parser.go | 35 ++---------------- bundle/schema/jsonschema.json | 6 +-- 6 files changed, 12 insertions(+), 84 deletions(-) diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index 697f716819..593f3e1dd1 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -137,7 +137,7 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Deprecated = true s.DeprecationMessage = a.DeprecationMessage } - if a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedTrue { + if a.ForceNotDeprecated { s.Deprecated = false s.DeprecationMessage = "" } diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go index 0dfd0f92e6..562d992c4c 100644 --- a/bundle/internal/annotation/descriptor.go +++ b/bundle/internal/annotation/descriptor.go @@ -1,12 +1,5 @@ package annotation -type PythonKeepDeprecated string - -const ( - PythonKeepDeprecatedTrue PythonKeepDeprecated = "true" - PythonKeepDeprecatedUnset PythonKeepDeprecated = "" -) - type Descriptor struct { Description string `json:"description,omitempty"` MarkdownDescription string `json:"markdown_description,omitempty"` @@ -17,8 +10,8 @@ type Descriptor struct { DeprecationMessage string `json:"deprecation_message,omitempty"` Preview string `json:"x-databricks-preview,omitempty"` - // If true, keep the field in Python code even if it is deprecated. - PythonKeepDeprecated PythonKeepDeprecated `json:"x-databricks-python-keep-deprecated,omitempty"` + // If true, takes priority over 'DeprecationMessage' + ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"` } const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index ae57a54598..8f65368a74 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -138,8 +138,9 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Preview = a.Preview } - if a.PythonKeepDeprecated != annotation.PythonKeepDeprecatedUnset { - s.PythonKeepDeprecated = a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedTrue + if a.ForceNotDeprecated { + s.Deprecated = false + s.DeprecationMessage = "" } s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription) diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index ad02c8e60e..bddcec5784 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -210,8 +210,6 @@ github.com/databricks/cli/bundle/config/resources.Job: ``` For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md). - "format": - "x-databricks-python-keep-deprecated": "false" "health": "description": |- PLACEHOLDER @@ -313,8 +311,6 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: "permissions": "description": |- PLACEHOLDER - "rate_limits": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/cli/bundle/config/resources.ModelServingEndpointPermissionLevel: "_": "enum": @@ -358,11 +354,11 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "description": |- PLACEHOLDER "target": - "x-databricks-python-keep-deprecated": "true" + "force_not_deprecated": |- + true "trigger": "deprecation_message": |- Use continuous instead - "x-databricks-python-keep-deprecated": "false" github.com/databricks/cli/bundle/config/resources.PipelinePermissionLevel: "_": "enum": @@ -714,8 +710,6 @@ github.com/databricks/databricks-sdk-go/service/compute.DockerImage: "description": |- PLACEHOLDER github.com/databricks/databricks-sdk-go/service/compute.Environment: - "client": - "x-databricks-python-keep-deprecated": "true" "dependencies": "description": |- List of pip dependencies, as supported by the version of pip in this environment. @@ -723,17 +717,10 @@ github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes: "availability": "description": |- PLACEHOLDER - "use_preemptible_executors": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo: "abfss": "description": |- Contains the Azure Data Lake Storage destination path - "dbfs": - "x-databricks-python-keep-deprecated": "false" -github.com/databricks/databricks-sdk-go/service/compute.Library: - "egg": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo: "log_analytics_primary_key": "description": |- @@ -756,9 +743,6 @@ github.com/databricks/databricks-sdk-go/service/jobs.GitSource: "git_snapshot": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications: - "no_alert_for_skipped_runs": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment: "spec": "description": |- @@ -778,11 +762,6 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: "python_named_params": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask: - "jar_uri": - "x-databricks-python-keep-deprecated": "false" - "run_as_repl": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/jobs.Subscription: "subscribers": "description": |- @@ -804,9 +783,6 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "health": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications: - "no_alert_for_skipped_runs": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings: "table_update": "description": |- @@ -822,14 +798,10 @@ github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger: "timezone_id": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition: - "connection_id": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary: "whl": "deprecation_message": |- This field is deprecated - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "cron": "description": |- @@ -837,11 +809,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters: - "invalid_keywords": - "x-databricks-python-keep-deprecated": "false" - "valid_topics": - "x-databricks-python-keep-deprecated": "false" github.com/databricks/databricks-sdk-go/service/serving.Route: "served_entity_name": "description": |- diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 27ac5ce8c6..8db276f061 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -196,11 +196,6 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid if description == "" { addEmptyOverride(k, basePath, overrides) } - - // if field got deprecated outside of preview it can't be just removed and we keep it - if refProp.Deprecated && refProp.Preview != "PRIVATE" { - addPythonKeepDeprecatedOverride(k, basePath, overrides) - } } else { addEmptyOverride(k, basePath, overrides) } @@ -246,46 +241,22 @@ func prependCommentToFile(outputPath, comment string) error { return err } -// addPythonKeepDeprecatedOverride adds an override with "x-databricks-python-keep-deprecated: true" to -// annotations_openapi_overrides.yaml unless it already specifies a value for "x-databricks-python-keep-deprecated". -// -// This way if field got deprecated, we add an override to keep it in Python code, until we revisit deprecation. -func addPythonKeepDeprecatedOverride(key, pkg string, overridesFile annotation.File) { - if overridesFile[pkg] == nil { - overridesFile[pkg] = map[string]annotation.Descriptor{} - } - - overrides := overridesFile[pkg] - - a, ok := overrides[key] - if !ok { - a = annotation.Descriptor{} - } - - // if we don't have explicit value, it means that field just got deprecated, - // and we should keep it in Python code - if a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedUnset { - a.PythonKeepDeprecated = annotation.PythonKeepDeprecatedTrue - } - - overrides[key] = a -} - func addEmptyOverride(key, pkg string, overridesFile annotation.File) { if overridesFile[pkg] == nil { overridesFile[pkg] = map[string]annotation.Descriptor{} } overrides := overridesFile[pkg] + if overrides[key].Description == "" { + overrides[key] = annotation.Descriptor{Description: annotation.Placeholder} + } a, ok := overrides[key] if !ok { a = annotation.Descriptor{} } - if a.Description == "" { a.Description = annotation.Placeholder } - overrides[key] = a } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index aeb0d5eff9..c0aec94eb8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1273,10 +1273,7 @@ }, "target": { "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", - "$ref": "#/$defs/string", - "deprecationMessage": "This field is deprecated", - "x-databricks-python-keep-deprecated": true, - "deprecated": true + "$ref": "#/$defs/string" }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", @@ -3763,7 +3760,6 @@ "description": "Use `environment_version` instead.", "$ref": "#/$defs/string", "deprecationMessage": "This field is deprecated", - "x-databricks-python-keep-deprecated": true, "deprecated": true }, "dependencies": { From 20e1856e182c707cc882452037bfe629202d36a6 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 29 Aug 2025 10:50:23 +0200 Subject: [PATCH 4/6] Add all deprecated fields except private ones --- .github/workflows/push.yml | 1 - bundle/docsgen/main.go | 4 ---- bundle/internal/annotation/descriptor.go | 3 --- bundle/internal/schema/annotations.go | 5 ----- .../schema/annotations_openapi_overrides.yml | 3 --- bundle/schema/jsonschema.json | 4 +++- .../python/codegen/codegen/jsonschema.py | 2 -- .../python/codegen/codegen/jsonschema_patch.py | 9 +++++++++ experimental/python/codegen/codegen/main.py | 9 +++++---- .../bundles/jobs/_models/gcp_attributes.py | 14 ++++++++++++++ .../bundles/jobs/_models/init_script_info.py | 16 ++++++++++++++++ .../jobs/_models/job_email_notifications.py | 14 +++++++++++++- .../databricks/bundles/jobs/_models/library.py | 10 ++++++++++ .../jobs/_models/task_email_notifications.py | 14 +++++++++++++- .../bundles/pipelines/_models/gcp_attributes.py | 14 ++++++++++++++ .../ingestion_gateway_pipeline_definition.py | 10 ++++++++++ .../pipelines/_models/init_script_info.py | 16 ++++++++++++++++ libs/jsonschema/extension.go | 4 ---- 18 files changed, 123 insertions(+), 29 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5c62d25f14..dea881e1a4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -180,7 +180,6 @@ jobs: a.addKeyword('doNotSuggest'); a.addKeyword('markdownDescription'); a.addKeyword('x-databricks-preview'); - a.addKeyword('x-databricks-python-keep-deprecated'); }" >> keywords.js for file in ./bundle/internal/schema/testdata/pass/*.yml; do diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index 593f3e1dd1..9f585830d7 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -137,10 +137,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Deprecated = true s.DeprecationMessage = a.DeprecationMessage } - if a.ForceNotDeprecated { - s.Deprecated = false - s.DeprecationMessage = "" - } if a.Preview == "PRIVATE" { s.DoNotSuggest = true s.Preview = a.Preview diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go index 562d992c4c..7b5dd607bb 100644 --- a/bundle/internal/annotation/descriptor.go +++ b/bundle/internal/annotation/descriptor.go @@ -9,9 +9,6 @@ type Descriptor struct { MarkdownExamples string `json:"markdown_examples,omitempty"` DeprecationMessage string `json:"deprecation_message,omitempty"` Preview string `json:"x-databricks-preview,omitempty"` - - // If true, takes priority over 'DeprecationMessage' - ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"` } const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 8f65368a74..a40c5c7c8e 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -138,11 +138,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Preview = a.Preview } - if a.ForceNotDeprecated { - s.Deprecated = false - s.DeprecationMessage = "" - } - s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription) s.Title = a.Title s.Enum = a.Enum diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index bddcec5784..a3b69140f6 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -353,9 +353,6 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "run_as": "description": |- PLACEHOLDER - "target": - "force_not_deprecated": |- - true "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c0aec94eb8..f3072ef23d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1273,7 +1273,9 @@ }, "target": { "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", diff --git a/experimental/python/codegen/codegen/jsonschema.py b/experimental/python/codegen/codegen/jsonschema.py index 5a252a2b58..f07f8d2d1d 100644 --- a/experimental/python/codegen/codegen/jsonschema.py +++ b/experimental/python/codegen/codegen/jsonschema.py @@ -16,7 +16,6 @@ class Property: ref: str description: Optional[str] = None deprecated: Optional[bool] = None - keep_deprecated: Optional[bool] = None stage: Optional[str] = None @@ -102,7 +101,6 @@ def _parse_bool(value) -> Optional[bool]: ref=v["$ref"], description=v.get("description"), deprecated=_parse_bool(v.get("deprecated")), - keep_deprecated=_parse_bool(v.get("x-databricks-python-keep-deprecated")), stage=v.get("x-databricks-preview"), ) diff --git a/experimental/python/codegen/codegen/jsonschema_patch.py b/experimental/python/codegen/codegen/jsonschema_patch.py index 848df05069..8cfe2981c3 100644 --- a/experimental/python/codegen/codegen/jsonschema_patch.py +++ b/experimental/python/codegen/codegen/jsonschema_patch.py @@ -7,6 +7,15 @@ # doesn't work, openapi schema needs to be updated to be enum "kind", }, + # fields that were deprecated a long time ago + "resources.Pipeline": { + # 'trigger' is deprecated, use 'continuous' or schedule pipeline refresh using job instead + "trigger", + }, + "pipelines.PipelineLibrary": [ + # 'whl' is deprecated, install libraries through notebooks and %pip command + "whl", + ], } EXTRA_REQUIRED_FIELDS: dict[str, list[str]] = { diff --git a/experimental/python/codegen/codegen/main.py b/experimental/python/codegen/codegen/main.py index 5380b0b968..eafcf69026 100644 --- a/experimental/python/codegen/codegen/main.py +++ b/experimental/python/codegen/codegen/main.py @@ -77,13 +77,17 @@ def _transitively_mark_deprecated_and_private( def _remove_deprecated_fields( schemas: dict[str, openapi.Schema], ) -> dict[str, openapi.Schema]: + """ + Remove fields that were deprecated during Private Preview. + """ + new_schemas = {} for name, schema in schemas.items(): if schema.type == openapi.SchemaType.OBJECT: new_properties = {} for field_name, field in schema.properties.items(): - if field.deprecated and not field.keep_deprecated: + if field.deprecated and field.stage == openapi.Stage.PRIVATE: continue new_properties[field_name] = field @@ -246,9 +250,6 @@ def _collect_reachable_schemas( if ( not include_deprecated and field.deprecated - # we don't remove keep_deprecated fields so they should be considered - # reachable - and not field.keep_deprecated ): continue diff --git a/experimental/python/databricks/bundles/jobs/_models/gcp_attributes.py b/experimental/python/databricks/bundles/jobs/_models/gcp_attributes.py index cbc2a29630..6f390c1cf8 100644 --- a/experimental/python/databricks/bundles/jobs/_models/gcp_attributes.py +++ b/experimental/python/databricks/bundles/jobs/_models/gcp_attributes.py @@ -53,6 +53,13 @@ class GcpAttributes: for the supported number of local SSDs for each instance type. """ + use_preemptible_executors: VariableOrOptional[bool] = None + """ + [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible + VMs (when set to true) versus standard compute engine VMs (when set to false; default). + Note: Soon to be deprecated, use the 'availability' field instead. + """ + zone_id: VariableOrOptional[str] = None """ Identifier for the availability zone in which the cluster resides. @@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False): for the supported number of local SSDs for each instance type. """ + use_preemptible_executors: VariableOrOptional[bool] + """ + [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible + VMs (when set to true) versus standard compute engine VMs (when set to false; default). + Note: Soon to be deprecated, use the 'availability' field instead. + """ + zone_id: VariableOrOptional[str] """ Identifier for the availability zone in which the cluster resides. diff --git a/experimental/python/databricks/bundles/jobs/_models/init_script_info.py b/experimental/python/databricks/bundles/jobs/_models/init_script_info.py index 124741eaf7..f14f185487 100644 --- a/experimental/python/databricks/bundles/jobs/_models/init_script_info.py +++ b/experimental/python/databricks/bundles/jobs/_models/init_script_info.py @@ -8,6 +8,10 @@ Adlsgen2Info, Adlsgen2InfoParam, ) +from databricks.bundles.jobs._models.dbfs_storage_info import ( + DbfsStorageInfo, + DbfsStorageInfoParam, +) from databricks.bundles.jobs._models.gcs_storage_info import ( GcsStorageInfo, GcsStorageInfoParam, @@ -45,6 +49,12 @@ class InitScriptInfo: Contains the Azure Data Lake Storage destination path """ + dbfs: VariableOrOptional[DbfsStorageInfo] = None + """ + [DEPRECATED] destination needs to be provided. e.g. + `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` + """ + file: VariableOrOptional[LocalFileInfo] = None """ destination needs to be provided, e.g. @@ -93,6 +103,12 @@ class InitScriptInfoDict(TypedDict, total=False): Contains the Azure Data Lake Storage destination path """ + dbfs: VariableOrOptional[DbfsStorageInfoParam] + """ + [DEPRECATED] destination needs to be provided. e.g. + `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` + """ + file: VariableOrOptional[LocalFileInfoParam] """ destination needs to be provided, e.g. diff --git a/experimental/python/databricks/bundles/jobs/_models/job_email_notifications.py b/experimental/python/databricks/bundles/jobs/_models/job_email_notifications.py index 4cca930023..b97648f4dc 100644 --- a/experimental/python/databricks/bundles/jobs/_models/job_email_notifications.py +++ b/experimental/python/databricks/bundles/jobs/_models/job_email_notifications.py @@ -3,7 +3,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import VariableOrList +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional if TYPE_CHECKING: from typing_extensions import Self @@ -13,6 +13,12 @@ class JobEmailNotifications: """""" + no_alert_for_skipped_runs: VariableOrOptional[bool] = None + """ + [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped. + This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + """ + on_duration_warning_threshold_exceeded: VariableOrList[str] = field( default_factory=list ) @@ -53,6 +59,12 @@ def as_dict(self) -> "JobEmailNotificationsDict": class JobEmailNotificationsDict(TypedDict, total=False): """""" + no_alert_for_skipped_runs: VariableOrOptional[bool] + """ + [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped. + This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + """ + on_duration_warning_threshold_exceeded: VariableOrList[str] """ A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent. diff --git a/experimental/python/databricks/bundles/jobs/_models/library.py b/experimental/python/databricks/bundles/jobs/_models/library.py index bde8cbaad9..c8d4b5f5b5 100644 --- a/experimental/python/databricks/bundles/jobs/_models/library.py +++ b/experimental/python/databricks/bundles/jobs/_models/library.py @@ -30,6 +30,11 @@ class Library: Specification of a CRAN library to be installed as part of the library """ + egg: VariableOrOptional[str] = None + """ + [DEPRECATED] Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above. + """ + jar: VariableOrOptional[str] = None """ URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. @@ -82,6 +87,11 @@ class LibraryDict(TypedDict, total=False): Specification of a CRAN library to be installed as part of the library """ + egg: VariableOrOptional[str] + """ + [DEPRECATED] Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above. + """ + jar: VariableOrOptional[str] """ URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. diff --git a/experimental/python/databricks/bundles/jobs/_models/task_email_notifications.py b/experimental/python/databricks/bundles/jobs/_models/task_email_notifications.py index a0ead50c37..3583798515 100644 --- a/experimental/python/databricks/bundles/jobs/_models/task_email_notifications.py +++ b/experimental/python/databricks/bundles/jobs/_models/task_email_notifications.py @@ -3,7 +3,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import VariableOrList +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional if TYPE_CHECKING: from typing_extensions import Self @@ -13,6 +13,12 @@ class TaskEmailNotifications: """""" + no_alert_for_skipped_runs: VariableOrOptional[bool] = None + """ + [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped. + This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + """ + on_duration_warning_threshold_exceeded: VariableOrList[str] = field( default_factory=list ) @@ -53,6 +59,12 @@ def as_dict(self) -> "TaskEmailNotificationsDict": class TaskEmailNotificationsDict(TypedDict, total=False): """""" + no_alert_for_skipped_runs: VariableOrOptional[bool] + """ + [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped. + This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + """ + on_duration_warning_threshold_exceeded: VariableOrList[str] """ A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent. diff --git a/experimental/python/databricks/bundles/pipelines/_models/gcp_attributes.py b/experimental/python/databricks/bundles/pipelines/_models/gcp_attributes.py index 53b2e9c76d..1deae0c124 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/gcp_attributes.py +++ b/experimental/python/databricks/bundles/pipelines/_models/gcp_attributes.py @@ -53,6 +53,13 @@ class GcpAttributes: for the supported number of local SSDs for each instance type. """ + use_preemptible_executors: VariableOrOptional[bool] = None + """ + [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible + VMs (when set to true) versus standard compute engine VMs (when set to false; default). + Note: Soon to be deprecated, use the 'availability' field instead. + """ + zone_id: VariableOrOptional[str] = None """ Identifier for the availability zone in which the cluster resides. @@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False): for the supported number of local SSDs for each instance type. """ + use_preemptible_executors: VariableOrOptional[bool] + """ + [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible + VMs (when set to true) versus standard compute engine VMs (when set to false; default). + Note: Soon to be deprecated, use the 'availability' field instead. + """ + zone_id: VariableOrOptional[str] """ Identifier for the availability zone in which the cluster resides. diff --git a/experimental/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py b/experimental/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py index 9793c02fe7..fd278e5e6d 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py +++ b/experimental/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py @@ -30,6 +30,11 @@ class IngestionGatewayPipelineDefinition: Required, Immutable. The name of the schema for the gateway pipelines's storage location. """ + connection_id: VariableOrOptional[str] = None + """ + [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + """ + gateway_storage_name: VariableOrOptional[str] = None """ Optional. The Unity Catalog-compatible name for the gateway storage location. @@ -63,6 +68,11 @@ class IngestionGatewayPipelineDefinitionDict(TypedDict, total=False): Required, Immutable. The name of the schema for the gateway pipelines's storage location. """ + connection_id: VariableOrOptional[str] + """ + [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + """ + gateway_storage_name: VariableOrOptional[str] """ Optional. The Unity Catalog-compatible name for the gateway storage location. diff --git a/experimental/python/databricks/bundles/pipelines/_models/init_script_info.py b/experimental/python/databricks/bundles/pipelines/_models/init_script_info.py index 91bc383e42..6a6297a30b 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/init_script_info.py +++ b/experimental/python/databricks/bundles/pipelines/_models/init_script_info.py @@ -8,6 +8,10 @@ Adlsgen2Info, Adlsgen2InfoParam, ) +from databricks.bundles.pipelines._models.dbfs_storage_info import ( + DbfsStorageInfo, + DbfsStorageInfoParam, +) from databricks.bundles.pipelines._models.gcs_storage_info import ( GcsStorageInfo, GcsStorageInfoParam, @@ -45,6 +49,12 @@ class InitScriptInfo: Contains the Azure Data Lake Storage destination path """ + dbfs: VariableOrOptional[DbfsStorageInfo] = None + """ + [DEPRECATED] destination needs to be provided. e.g. + `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` + """ + file: VariableOrOptional[LocalFileInfo] = None """ destination needs to be provided, e.g. @@ -93,6 +103,12 @@ class InitScriptInfoDict(TypedDict, total=False): Contains the Azure Data Lake Storage destination path """ + dbfs: VariableOrOptional[DbfsStorageInfoParam] + """ + [DEPRECATED] destination needs to be provided. e.g. + `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` + """ + file: VariableOrOptional[LocalFileInfoParam] """ destination needs to be provided, e.g. diff --git a/libs/jsonschema/extension.go b/libs/jsonschema/extension.go index ddd55fd13e..d400e31e56 100644 --- a/libs/jsonschema/extension.go +++ b/libs/jsonschema/extension.go @@ -55,8 +55,4 @@ type Extension struct { // This field is not in the JSON schema spec, but it is supported in VSCode // It hides a property from IntelliSense (autocomplete suggestions). DoNotSuggest bool `json:"doNotSuggest,omitempty"` - - // PythonKeepDeprecated is a custom extension that forces the field to be - // present in the generated Python code, even if it is deprecated. - PythonKeepDeprecated bool `json:"x-databricks-python-keep-deprecated,omitempty"` } From 30c090d21ae9fca694231155a3dac8c33724fc3b Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 29 Aug 2025 14:01:39 +0200 Subject: [PATCH 5/6] Fix lint/fmt --- experimental/python/codegen/codegen/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/experimental/python/codegen/codegen/main.py b/experimental/python/codegen/codegen/main.py index eafcf69026..df26810338 100644 --- a/experimental/python/codegen/codegen/main.py +++ b/experimental/python/codegen/codegen/main.py @@ -247,10 +247,7 @@ def _collect_reachable_schemas( if not include_private and field.stage == openapi.Stage.PRIVATE: continue - if ( - not include_deprecated - and field.deprecated - ): + if not include_deprecated and field.deprecated: continue if name not in reachable: From 8b75eccdfe056339f21a02c8f2f599e6728fff6f Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 29 Aug 2025 14:02:13 +0200 Subject: [PATCH 6/6] Update codegen --- .../bundles/jobs/_models/spark_jar_task.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/experimental/python/databricks/bundles/jobs/_models/spark_jar_task.py b/experimental/python/databricks/bundles/jobs/_models/spark_jar_task.py index c4ebcaaf75..40bbe92ba0 100644 --- a/experimental/python/databricks/bundles/jobs/_models/spark_jar_task.py +++ b/experimental/python/databricks/bundles/jobs/_models/spark_jar_task.py @@ -3,7 +3,11 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import VariableOr, VariableOrList +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) if TYPE_CHECKING: from typing_extensions import Self @@ -20,6 +24,11 @@ class SparkJarTask: The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail. """ + jar_uri: VariableOrOptional[str] = None + """ + [DEPRECATED] Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create. + """ + parameters: VariableOrList[str] = field(default_factory=list) """ Parameters passed to the main method. @@ -27,6 +36,11 @@ class SparkJarTask: Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. """ + run_as_repl: VariableOrOptional[bool] = None + """ + [DEPRECATED] Deprecated. A value of `false` is no longer supported. + """ + @classmethod def from_dict(cls, value: "SparkJarTaskDict") -> "Self": return _transform(cls, value) @@ -45,6 +59,11 @@ class SparkJarTaskDict(TypedDict, total=False): The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail. """ + jar_uri: VariableOrOptional[str] + """ + [DEPRECATED] Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create. + """ + parameters: VariableOrList[str] """ Parameters passed to the main method. @@ -52,5 +71,10 @@ class SparkJarTaskDict(TypedDict, total=False): Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. """ + run_as_repl: VariableOrOptional[bool] + """ + [DEPRECATED] Deprecated. A value of `false` is no longer supported. + """ + SparkJarTaskParam = SparkJarTaskDict | SparkJarTask