Skip to content

[Python] Keep deprecated fields in Python code #3384

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kanterov merged 6 commits into from
Sep 2, 2025
Merged

[Python] Keep deprecated fields in Python code #3384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a21573d
[Python] Keep newly deprecated fields in Python code
kanterov Aug 8, 2025
afedb6a
Add x-databricks-python-keep-deprecated to keywords
kanterov Aug 22, 2025
3948db9
Revert
kanterov Aug 29, 2025
20e1856
Add all deprecated fields except private ones
kanterov Aug 29, 2025
30c090d
Fix lint/fmt
kanterov Aug 29, 2025
8b75ecc
Update codegen
kanterov Aug 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions bundle/docsgen/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
s.Deprecated = true
s.DeprecationMessage = a.DeprecationMessage
}
if a.ForceNotDeprecated {
s.Deprecated = false
s.DeprecationMessage = ""
}
if a.Preview == "PRIVATE" {
s.DoNotSuggest = true
s.Preview = a.Preview
Expand Down
3 changes: 0 additions & 3 deletions bundle/internal/annotation/descriptor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ type Descriptor struct {
MarkdownExamples string `json:"markdown_examples,omitempty"`
DeprecationMessage string `json:"deprecation_message,omitempty"`
Preview string `json:"x-databricks-preview,omitempty"`

// If true, takes priority over 'DeprecationMessage'
ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"`
}

const Placeholder = "PLACEHOLDER"
5 changes: 0 additions & 5 deletions bundle/internal/schema/annotations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
s.Preview = a.Preview
}

if a.ForceNotDeprecated {
s.Deprecated = false
s.DeprecationMessage = ""
}

s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
s.Title = a.Title
s.Enum = a.Enum
Expand Down
3 changes: 0 additions & 3 deletions bundle/internal/schema/annotations_openapi_overrides.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,9 +353,6 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
"run_as":
"description": |-
PLACEHOLDER
"target":
"force_not_deprecated": |-
true
"trigger":
"deprecation_message": |-
Use continuous instead
Expand Down
4 changes: 3 additions & 1 deletion bundle/schema/jsonschema.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 40 additions & 5 deletions experimental/python/codegen/codegen/generated_dataclass.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,11 @@ class GeneratedField:
be marked as experimental in docstring.
"""

deprecated: bool
"""
If true, the field is deprecated and should be marked as deprecated in docstring.
"""

def __post_init__(self):
if self.default_factory is not None and self.default is not None:
raise ValueError("Can't have both default and default_factory", self)
Expand Down Expand Up @@ -131,6 +136,7 @@ class GeneratedDataclass:
fields: list[GeneratedField]
extends: list[GeneratedType]
experimental: bool
deprecated: bool


def generate_field(
Expand All @@ -156,6 +162,7 @@ def generate_field(
default_factory="dict",
create_func_default="None",
experimental=prop.stage == Stage.PRIVATE,
deprecated=prop.deprecated or False,
)
elif field_type.name == "VariableOrList":
return GeneratedField(
Expand All @@ -168,6 +175,7 @@ def generate_field(
default_factory="list",
create_func_default="None",
experimental=prop.stage == Stage.PRIVATE,
deprecated=prop.deprecated or False,
)
elif is_required:
return GeneratedField(
Expand All @@ -180,6 +188,7 @@ def generate_field(
default_factory=None,
create_func_default=None,
experimental=prop.stage == Stage.PRIVATE,
deprecated=prop.deprecated or False,
)
else:
return GeneratedField(
Expand All @@ -192,6 +201,7 @@ def generate_field(
default_factory=None,
create_func_default="None",
experimental=prop.stage == Stage.PRIVATE,
deprecated=prop.deprecated or False,
)


Expand Down Expand Up @@ -326,6 +336,7 @@ def generate_dataclass(
fields=fields,
extends=extends,
experimental=schema.stage == Stage.PRIVATE,
deprecated=schema.deprecated or False,
)


Expand Down Expand Up @@ -365,10 +376,19 @@ def _append_dataclass(b: CodeBuilder, generated: GeneratedDataclass):
b.append(":").newline()

# FIXME should contain class docstring
if not generated.description and not generated.experimental:
if (
not generated.description
and not generated.experimental
and not generated.deprecated
):
b.indent().append_triple_quote().append_triple_quote().newline().newline()
else:
_append_description(b, generated.description, generated.experimental)
_append_description(
b,
generated.description,
experimental=generated.experimental,
deprecated=generated.deprecated,
)


def _append_field(b: CodeBuilder, field: GeneratedField):
Expand Down Expand Up @@ -446,7 +466,12 @@ def _append_typed_dict(b: CodeBuilder, generated: GeneratedDataclass):
b.indent().append_triple_quote().append_triple_quote().newline().newline()


def _append_description(b: CodeBuilder, description: Optional[str], experimental: bool):
def _append_description(
b: CodeBuilder, description: Optional[str], *, experimental: bool, deprecated: bool
):
if deprecated:
description = "[DEPRECATED] " + (description or "")

if description or experimental:
b.indent().append_triple_quote().newline()
if experimental:
Expand All @@ -472,7 +497,12 @@ def get_code(generated: GeneratedDataclass) -> str:

for field in generated.fields:
_append_field(b, field)
_append_description(b, field.description, field.experimental)
_append_description(
b,
field.description,
experimental=field.experimental,
deprecated=field.deprecated,
)

b.newline()

Expand All @@ -485,7 +515,12 @@ def get_code(generated: GeneratedDataclass) -> str:

for field in generated.fields:
_append_typed_dict_field(b, field)
_append_description(b, field.description, field.experimental)
_append_description(
b,
field.description,
experimental=field.experimental,
deprecated=field.deprecated,
)

b.newline()

Expand Down
9 changes: 8 additions & 1 deletion experimental/python/codegen/codegen/generated_enum.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class GeneratedEnum:
values: dict[str, str]
description: Optional[str]
experimental: bool
deprecated: bool


def generate_enum(namespace: str, schema_name: str, schema: Schema) -> GeneratedEnum:
Expand All @@ -35,6 +36,7 @@ def generate_enum(namespace: str, schema_name: str, schema: Schema) -> Generated
values=values,
description=schema.description,
experimental=schema.stage == Stage.PRIVATE,
deprecated=schema.deprecated or False,
)


Expand All @@ -48,7 +50,12 @@ def get_code(generated: GeneratedEnum) -> str:
b.append(f"class {generated.class_name}(Enum):")
b.newline()

_append_description(b, generated.description, generated.experimental)
_append_description(
b,
generated.description,
experimental=generated.experimental,
deprecated=generated.deprecated,
)

# Example:
#
Expand Down
9 changes: 9 additions & 0 deletions experimental/python/codegen/codegen/jsonschema_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@
# doesn't work, openapi schema needs to be updated to be enum
"kind",
},
# fields that were deprecated a long time ago
"resources.Pipeline": {
# 'trigger' is deprecated, use 'continuous' or schedule pipeline refresh using job instead
"trigger",
},
"pipelines.PipelineLibrary": [
# 'whl' is deprecated, install libraries through notebooks and %pip command
"whl",
],
}

EXTRA_REQUIRED_FIELDS: dict[str, list[str]] = {
Expand Down
6 changes: 5 additions & 1 deletion experimental/python/codegen/codegen/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,17 @@ def _transitively_mark_deprecated_and_private(
def _remove_deprecated_fields(
schemas: dict[str, openapi.Schema],
) -> dict[str, openapi.Schema]:
"""
Remove fields that were deprecated during Private Preview.
"""

Copy link
Contributor

@pietern pietern Sep 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have examples of these?

They should rather be removed from the spec entirely.

Copy link
Collaborator Author

@kanterov kanterov Sep 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, there are many of them. Should we remove them from the JSON schema altogether? For example, in jobs.TriggerSettings:

                    "table": {
                      "description": "Old table trigger settings name. Deprecated in favor of `table_update`.",
                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration",
                      "x-databricks-preview": "PRIVATE",
                      "deprecationMessage": "This field is deprecated",
                      "doNotSuggest": true,
                      "deprecated": true
                    },
                    "table_update": {
                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration",
                      "x-databricks-preview": "PRIVATE",
                      "doNotSuggest": true
                    }

I think following the principle, we should remove them from SDK v1

new_schemas = {}

for name, schema in schemas.items():
if schema.type == openapi.SchemaType.OBJECT:
new_properties = {}
for field_name, field in schema.properties.items():
if field.deprecated:
if field.deprecated and field.stage == openapi.Stage.PRIVATE:
continue

new_properties[field_name] = field
Expand Down
2 changes: 2 additions & 0 deletions experimental/python/codegen/codegen_tests/test_generated_dataclass.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,11 @@ def test_generate_dataclass():
param_type_name=variable_or_type(str_type(), is_required=True),
type_name=variable_or_type(str_type(), is_required=True),
experimental=False,
deprecated=False,
),
],
experimental=False,
deprecated=False,
)


Expand Down
1 change: 1 addition & 0 deletions experimental/python/codegen/codegen_tests/test_generated_enum.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ def test_generate_enum():
values={"MY_ENUM_VALUE": "myEnumValue"},
description="enum description",
experimental=False,
deprecated=False,
)
10 changes: 10 additions & 0 deletions experimental/python/databricks/bundles/jobs/_models/environment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ class Environment:
In this minimal environment spec, only pip dependencies are supported.
"""

client: VariableOrOptional[str] = None
"""
[DEPRECATED] Use `environment_version` instead.
"""

dependencies: VariableOrList[str] = field(default_factory=list)
"""
List of pip dependencies, as supported by the version of pip in this environment.
Expand Down Expand Up @@ -46,6 +51,11 @@ def as_dict(self) -> "EnvironmentDict":
class EnvironmentDict(TypedDict, total=False):
""""""

client: VariableOrOptional[str]
"""
[DEPRECATED] Use `environment_version` instead.
"""

dependencies: VariableOrList[str]
"""
List of pip dependencies, as supported by the version of pip in this environment.
Expand Down
14 changes: 14 additions & 0 deletions experimental/python/databricks/bundles/jobs/_models/gcp_attributes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@ class GcpAttributes:
for the supported number of local SSDs for each instance type.
"""

use_preemptible_executors: VariableOrOptional[bool] = None
"""
[DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
VMs (when set to true) versus standard compute engine VMs (when set to false; default).
Note: Soon to be deprecated, use the 'availability' field instead.
"""

zone_id: VariableOrOptional[str] = None
"""
Identifier for the availability zone in which the cluster resides.
Expand Down Expand Up @@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False):
for the supported number of local SSDs for each instance type.
"""

use_preemptible_executors: VariableOrOptional[bool]
"""
[DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
VMs (when set to true) versus standard compute engine VMs (when set to false; default).
Note: Soon to be deprecated, use the 'availability' field instead.
"""

zone_id: VariableOrOptional[str]
"""
Identifier for the availability zone in which the cluster resides.
Expand Down
16 changes: 16 additions & 0 deletions experimental/python/databricks/bundles/jobs/_models/init_script_info.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
Adlsgen2Info,
Adlsgen2InfoParam,
)
from databricks.bundles.jobs._models.dbfs_storage_info import (
DbfsStorageInfo,
DbfsStorageInfoParam,
)
from databricks.bundles.jobs._models.gcs_storage_info import (
GcsStorageInfo,
GcsStorageInfoParam,
Expand Down Expand Up @@ -45,6 +49,12 @@ class InitScriptInfo:
Contains the Azure Data Lake Storage destination path
"""

dbfs: VariableOrOptional[DbfsStorageInfo] = None
"""
[DEPRECATED] destination needs to be provided. e.g.
`{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`
"""

file: VariableOrOptional[LocalFileInfo] = None
"""
destination needs to be provided, e.g.
Expand Down Expand Up @@ -93,6 +103,12 @@ class InitScriptInfoDict(TypedDict, total=False):
Contains the Azure Data Lake Storage destination path
"""

dbfs: VariableOrOptional[DbfsStorageInfoParam]
"""
[DEPRECATED] destination needs to be provided. e.g.
`{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`
"""

file: VariableOrOptional[LocalFileInfoParam]
"""
destination needs to be provided, e.g.
Expand Down
14 changes: 13 additions & 1 deletion experimental/python/databricks/bundles/jobs/_models/job_email_notifications.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from databricks.bundles.core._transform import _transform
from databricks.bundles.core._transform_to_json import _transform_to_json_value
from databricks.bundles.core._variable import VariableOrList
from databricks.bundles.core._variable import VariableOrList, VariableOrOptional

if TYPE_CHECKING:
from typing_extensions import Self
Expand All @@ -13,6 +13,12 @@
class JobEmailNotifications:
""""""

no_alert_for_skipped_runs: VariableOrOptional[bool] = None
"""
[DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
"""

on_duration_warning_threshold_exceeded: VariableOrList[str] = field(
default_factory=list
)
Expand Down Expand Up @@ -53,6 +59,12 @@ def as_dict(self) -> "JobEmailNotificationsDict":
class JobEmailNotificationsDict(TypedDict, total=False):
""""""

no_alert_for_skipped_runs: VariableOrOptional[bool]
"""
[DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
"""

on_duration_warning_threshold_exceeded: VariableOrList[str]
"""
A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.
Expand Down
Loading
Loading