diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 19009050ac..6d4ada17b6 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -59c4c0f3d5f0ef00cd5350b5674e941a7606d91a \ No newline at end of file +8f5eedbc991c4f04ce1284406577b0c92d59a224 \ No newline at end of file diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 8271361883..f1a7936ac7 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -1550,6 +1550,13 @@ resources.jobs.*.settings.trigger.file_arrival *jobs.FileArrivalTriggerConfigura resources.jobs.*.settings.trigger.file_arrival.min_time_between_triggers_seconds int REMOTE resources.jobs.*.settings.trigger.file_arrival.url string REMOTE resources.jobs.*.settings.trigger.file_arrival.wait_after_last_change_seconds int REMOTE +resources.jobs.*.settings.trigger.model *jobs.ModelTriggerConfiguration REMOTE +resources.jobs.*.settings.trigger.model.aliases []string REMOTE +resources.jobs.*.settings.trigger.model.aliases[*] string REMOTE +resources.jobs.*.settings.trigger.model.condition jobs.ModelTriggerConfigurationCondition REMOTE +resources.jobs.*.settings.trigger.model.min_time_between_triggers_seconds int REMOTE +resources.jobs.*.settings.trigger.model.securable_name string REMOTE +resources.jobs.*.settings.trigger.model.wait_after_last_change_seconds int REMOTE resources.jobs.*.settings.trigger.pause_status jobs.PauseStatus REMOTE resources.jobs.*.settings.trigger.periodic *jobs.PeriodicTriggerConfiguration REMOTE resources.jobs.*.settings.trigger.periodic.interval int REMOTE @@ -2215,6 +2222,13 @@ resources.jobs.*.trigger.file_arrival *jobs.FileArrivalTriggerConfiguration INPU resources.jobs.*.trigger.file_arrival.min_time_between_triggers_seconds int INPUT STATE resources.jobs.*.trigger.file_arrival.url string INPUT STATE resources.jobs.*.trigger.file_arrival.wait_after_last_change_seconds int INPUT STATE +resources.jobs.*.trigger.model *jobs.ModelTriggerConfiguration INPUT STATE +resources.jobs.*.trigger.model.aliases []string INPUT STATE +resources.jobs.*.trigger.model.aliases[*] string INPUT STATE +resources.jobs.*.trigger.model.condition jobs.ModelTriggerConfigurationCondition INPUT STATE +resources.jobs.*.trigger.model.min_time_between_triggers_seconds int INPUT STATE +resources.jobs.*.trigger.model.securable_name string INPUT STATE +resources.jobs.*.trigger.model.wait_after_last_change_seconds int INPUT STATE resources.jobs.*.trigger.pause_status jobs.PauseStatus INPUT STATE resources.jobs.*.trigger.periodic *jobs.PeriodicTriggerConfiguration INPUT STATE resources.jobs.*.trigger.periodic.interval int INPUT STATE @@ -2890,6 +2904,8 @@ resources.pipelines.*.filters.include[*] string INPUT STATE resources.pipelines.*.gateway_definition *pipelines.IngestionGatewayPipelineDefinition INPUT STATE resources.pipelines.*.gateway_definition.connection_id string INPUT STATE resources.pipelines.*.gateway_definition.connection_name string INPUT STATE +resources.pipelines.*.gateway_definition.connection_parameters *pipelines.ConnectionParameters INPUT STATE +resources.pipelines.*.gateway_definition.connection_parameters.source_catalog string INPUT STATE resources.pipelines.*.gateway_definition.gateway_storage_catalog string INPUT STATE resources.pipelines.*.gateway_definition.gateway_storage_name string INPUT STATE resources.pipelines.*.gateway_definition.gateway_storage_schema string INPUT STATE @@ -2897,6 +2913,7 @@ resources.pipelines.*.health pipelines.GetPipelineResponseHealth REMOTE resources.pipelines.*.id string INPUT STATE resources.pipelines.*.ingestion_definition *pipelines.IngestionPipelineDefinition INPUT STATE resources.pipelines.*.ingestion_definition.connection_name string INPUT STATE +resources.pipelines.*.ingestion_definition.ingest_from_uc_foreign_catalog bool INPUT STATE resources.pipelines.*.ingestion_definition.ingestion_gateway_id string INPUT STATE resources.pipelines.*.ingestion_definition.netsuite_jar_path string INPUT STATE resources.pipelines.*.ingestion_definition.objects []pipelines.IngestionConfig INPUT STATE @@ -3185,12 +3202,15 @@ resources.pipelines.*.spec.filters.include[*] string REMOTE resources.pipelines.*.spec.gateway_definition *pipelines.IngestionGatewayPipelineDefinition REMOTE resources.pipelines.*.spec.gateway_definition.connection_id string REMOTE resources.pipelines.*.spec.gateway_definition.connection_name string REMOTE +resources.pipelines.*.spec.gateway_definition.connection_parameters *pipelines.ConnectionParameters REMOTE +resources.pipelines.*.spec.gateway_definition.connection_parameters.source_catalog string REMOTE resources.pipelines.*.spec.gateway_definition.gateway_storage_catalog string REMOTE resources.pipelines.*.spec.gateway_definition.gateway_storage_name string REMOTE resources.pipelines.*.spec.gateway_definition.gateway_storage_schema string REMOTE resources.pipelines.*.spec.id string REMOTE resources.pipelines.*.spec.ingestion_definition *pipelines.IngestionPipelineDefinition REMOTE resources.pipelines.*.spec.ingestion_definition.connection_name string REMOTE +resources.pipelines.*.spec.ingestion_definition.ingest_from_uc_foreign_catalog bool REMOTE resources.pipelines.*.spec.ingestion_definition.ingestion_gateway_id string REMOTE resources.pipelines.*.spec.ingestion_definition.netsuite_jar_path string REMOTE resources.pipelines.*.spec.ingestion_definition.objects []pipelines.IngestionConfig REMOTE @@ -3565,6 +3585,7 @@ resources.synced_database_tables.*.spec *database.SyncedTableSpec ALL resources.synced_database_tables.*.spec.create_database_objects_if_missing bool ALL resources.synced_database_tables.*.spec.existing_pipeline_id string ALL resources.synced_database_tables.*.spec.new_pipeline_spec *database.NewPipelineSpec ALL +resources.synced_database_tables.*.spec.new_pipeline_spec.budget_policy_id string ALL resources.synced_database_tables.*.spec.new_pipeline_spec.storage_catalog string ALL resources.synced_database_tables.*.spec.new_pipeline_spec.storage_schema string ALL resources.synced_database_tables.*.spec.primary_key_columns []string ALL diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index 7f2f7426a9..ceb50dec6c 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -20,13 +20,11 @@ Compute policy-compliance-for-clusters The policy compliance APIs allow you to view and manage the policy compliance status of clusters in your workspace. policy-families View available policy families. -Workflows +Lakeflow jobs The Jobs API allows you to create, edit, and delete jobs. + pipelines The Lakeflow Spark Declarative Pipelines API allows you to create, edit, delete, start, and view details about pipelines. policy-compliance-for-jobs The compliance APIs allow you to view and manage the policy compliance status of jobs in your workspace. -Delta Live Tables - pipelines The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines. - Machine Learning experiments Experiments are the primary unit of organization in MLflow; all MLflow runs belong to an experiment. model-registry Note: This API reference documents APIs for the Workspace Model Registry. @@ -86,7 +84,7 @@ Unity Catalog quality-monitors A monitor computes and monitors data or model quality metrics for a table over time. registered-models Databricks provides a hosted version of MLflow Model Registry in Unity Catalog. resource-quotas Unity Catalog enforces resource quotas on all securable objects, which limits the number of resources that can be created. - rfa Request for Access enables customers to request access to and manage access request destinations for Unity Catalog securables. + rfa Request for Access enables users to request access for Unity Catalog securables. schemas A schema (also called a database) is the second layer of Unity Catalog’s three-level namespace. storage-credentials A storage credential represents an authentication and authorization mechanism for accessing data stored on your cloud tenant. system-schemas A system schema is a schema that lives within the system catalog. diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index a2704cf7f5..f13c29091c 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -450,8 +450,7 @@ github.com/databricks/cli/bundle/config/resources.Job: "environments": "description": |- A list of task execution environment specifications that can be referenced by serverless tasks of this job. - An environment is required to be present for serverless tasks. - For serverless notebook tasks, the environment is accessible in the notebook environment panel. + For serverless notebook tasks, if the environment_key is not specified, the notebook environment will be used if present. If a jobs environment is specified, it will override the notebook environment. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. "format": "description": |- @@ -1467,7 +1466,7 @@ github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. - This is an optional field at cluster creation, and if not specified, a default zone will be used. + This is an optional field at cluster creation, and if not specified, the zone "auto" will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. @@ -1841,8 +1840,6 @@ github.com/databricks/databricks-sdk-go/service/compute.Environment: "java_dependencies": "description": |- List of java dependencies. Each dependency is a string representing a java library path. For example: `/Volumes/path/to/test.jar`. - "x-databricks-preview": |- - PRIVATE github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes: "_": "description": |- @@ -2173,6 +2170,9 @@ github.com/databricks/databricks-sdk-go/service/database.NewPipelineSpec: "description": |- Custom fields that user can set for pipeline while creating SyncedDatabaseTable. Note that other fields of pipeline are still inferred by table def internally + "budget_policy_id": + "description": |- + Budget policy to set on the newly created pipeline. "storage_catalog": "description": |- This field needs to be specified if the destination catalog is a managed postgres catalog. @@ -2899,6 +2899,35 @@ github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules: "description": |- An optional set of health rules that can be defined for this job. "rules": {} +github.com/databricks/databricks-sdk-go/service/jobs.ModelTriggerConfiguration: + "aliases": + "description": |- + Aliases of the model versions to monitor. Can only be used in conjunction with condition MODEL_ALIAS_SET. + "condition": + "description": |- + The condition based on which to trigger a job run. + "min_time_between_triggers_seconds": + "description": |- + If set, the trigger starts a run only after the specified amount of time has passed since + the last time the trigger fired. The minimum allowed value is 60 seconds. + "securable_name": + "description": |- + Name of the securable to monitor ("mycatalog.myschema.mymodel" in the case of model-level triggers, + "mycatalog.myschema" in the case of schema-level triggers) or empty in the case of metastore-level triggers. + "wait_after_last_change_seconds": + "description": |- + If set, the trigger starts a run only after no model updates have occurred for the specified time + and can be used to wait for a series of model updates before triggering a run. The + minimum allowed value is 60 seconds. +github.com/databricks/databricks-sdk-go/service/jobs.ModelTriggerConfigurationCondition: + "_": + "enum": + - |- + MODEL_CREATED + - |- + MODEL_VERSION_READY + - |- + MODEL_ALIAS_SET github.com/databricks/databricks-sdk-go/service/jobs.NotebookTask: "base_parameters": "description": |- @@ -3516,6 +3545,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings: "file_arrival": "description": |- File arrival trigger settings. + "model": + "x-databricks-preview": |- + PRIVATE "pause_status": "description": |- Whether this trigger is paused or not. @@ -3564,6 +3596,15 @@ github.com/databricks/databricks-sdk-go/service/ml.ModelTag: "value": "description": |- The tag value. +github.com/databricks/databricks-sdk-go/service/pipelines.ConnectionParameters: + "source_catalog": + "description": |- + Source catalog for initial connection. + This is necessary for schema exploration in some database systems like Oracle, and optional but nice-to-have + in some other database systems like Postgres. + For Oracle databases, this maps to a service name. + "x-databricks-preview": |- + PRIVATE github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger: "quartz_cron_schedule": {} "timezone_id": {} @@ -3638,6 +3679,11 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipeli "connection_name": "description": |- Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + "connection_parameters": + "description": |- + Optional, Internal. Parameters required to establish an initial connection with the source. + "x-databricks-preview": |- + PRIVATE "gateway_storage_catalog": "description": |- Required, Immutable. The name of the catalog for the gateway pipeline's storage location. @@ -3645,7 +3691,7 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipeli "description": |- Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. - Delta Live Tables system will automatically create the storage location under the catalog and schema. + Spark Declarative Pipelines system will automatically create the storage location under the catalog and schema. "gateway_storage_schema": "description": |- Required, Immutable. The name of the schema for the gateway pipelines's storage location. @@ -3653,6 +3699,14 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefin "connection_name": "description": |- Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on. + "ingest_from_uc_foreign_catalog": + "description": |- + Immutable. If set to true, the pipeline will ingest tables from the + UC foreign catalogs directly without the need to specify a UC connection or ingestion gateway. + The `source_catalog` fields in objects of IngestionConfig are interpreted as + the UC foreign catalogs to ingest from. + "x-databricks-preview": |- + PRIVATE "ingestion_gateway_id": "description": |- Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server. @@ -3669,8 +3723,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefin "source_configurations": "description": |- Top-level source configurations - "x-databricks-preview": |- - PRIVATE "source_type": "description": |- The type of the foreign source. @@ -3783,6 +3835,8 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionSourceType: SHAREPOINT - |- DYNAMICS365 + - |- + FOREIGN_CATALOG github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger: {} github.com/databricks/databricks-sdk-go/service/pipelines.NotebookLibrary: "path": @@ -3985,8 +4039,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PostgresCatalogConfig: "slot_config": "description": |- Optional. The Postgres slot configuration to use for logical replication - "x-databricks-preview": |- - PRIVATE github.com/databricks/databricks-sdk-go/service/pipelines.PostgresSlotConfig: "_": "description": |- @@ -3994,13 +4046,9 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PostgresSlotConfig: "publication_name": "description": |- The name of the publication to use for the Postgres source - "x-databricks-preview": |- - PRIVATE "slot_name": "description": |- The name of the logical replication slot to use for the Postgres source - "x-databricks-preview": |- - PRIVATE github.com/databricks/databricks-sdk-go/service/pipelines.ReportSpec: "destination_catalog": "description": |- @@ -4065,19 +4113,13 @@ github.com/databricks/databricks-sdk-go/service/pipelines.SourceCatalogConfig: "postgres": "description": |- Postgres-specific catalog-level configuration parameters - "x-databricks-preview": |- - PRIVATE "source_catalog": "description": |- Source catalog name - "x-databricks-preview": |- - PRIVATE github.com/databricks/databricks-sdk-go/service/pipelines.SourceConfig: "catalog": "description": |- Catalog-level source configuration parameters - "x-databricks-preview": |- - PRIVATE github.com/databricks/databricks-sdk-go/service/pipelines.TableSpec: "destination_catalog": "description": |- @@ -4134,7 +4176,7 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig: PRIVATE "sequence_by": "description": |- - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. "workday_report_parameters": "description": |- (Optional) Additional custom parameters for Workday Report diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 72d50bf2ab..9778fa922f 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -891,6 +891,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "description": |- PLACEHOLDER github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings: + "model": + "description": |- + PLACEHOLDER "table_update": "description": |- PLACEHOLDER diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index c8b13327fc..f10be54bbd 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -11,7 +11,7 @@ var EnumFields = map[string][]string{ "resources.apps.*.active_deployment.mode": {"AUTO_SYNC", "SNAPSHOT"}, "resources.apps.*.active_deployment.status.state": {"CANCELLED", "FAILED", "IN_PROGRESS", "SUCCEEDED"}, "resources.apps.*.app_status.state": {"CRASHED", "DEPLOYING", "RUNNING", "UNAVAILABLE"}, - "resources.apps.*.compute_size": {"LARGE", "LIQUID", "MEDIUM"}, + "resources.apps.*.compute_size": {"LARGE", "MEDIUM"}, "resources.apps.*.compute_status.state": {"ACTIVE", "DELETING", "ERROR", "STARTING", "STOPPED", "STOPPING", "UPDATING"}, "resources.apps.*.pending_deployment.mode": {"AUTO_SYNC", "SNAPSHOT"}, "resources.apps.*.pending_deployment.status.state": {"CANCELLED", "FAILED", "IN_PROGRESS", "SUCCEEDED"}, @@ -92,6 +92,7 @@ var EnumFields = map[string][]string{ "resources.jobs.*.tasks[*].run_if": {"ALL_DONE", "ALL_FAILED", "ALL_SUCCESS", "AT_LEAST_ONE_FAILED", "AT_LEAST_ONE_SUCCESS", "NONE_FAILED"}, "resources.jobs.*.tasks[*].spark_python_task.source": {"GIT", "WORKSPACE"}, "resources.jobs.*.tasks[*].sql_task.file.source": {"GIT", "WORKSPACE"}, + "resources.jobs.*.trigger.model.condition": {"MODEL_ALIAS_SET", "MODEL_CREATED", "MODEL_VERSION_READY"}, "resources.jobs.*.trigger.pause_status": {"PAUSED", "UNPAUSED"}, "resources.jobs.*.trigger.periodic.unit": {"DAYS", "HOURS", "WEEKS"}, "resources.jobs.*.trigger.table_update.condition": {"ALL_UPDATED", "ANY_UPDATED"}, @@ -116,7 +117,7 @@ var EnumFields = map[string][]string{ "resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, "resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, "resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, - "resources.pipelines.*.ingestion_definition.source_type": {"BIGQUERY", "CONFLUENCE", "DYNAMICS365", "FOREIGN_CATALOG", "GA4_RAW_DATA", "GOOGLE_ADS", "GUIDEWIRE", "HUBSPOT", "MANAGED_POSTGRESQL", "META_MARKETING", "MYSQL", "NETSUITE", "ORACLE", "POSTGRESQL", "REDSHIFT", "SALESFORCE", "SALESFORCE_MARKETING_CLOUD", "SERVICENOW", "SHAREPOINT", "SQLDW", "SQLSERVER", "TERADATA", "TIKTOK_ADS", "WORKDAY_HCM", "WORKDAY_RAAS", "ZENDESK"}, + "resources.pipelines.*.ingestion_definition.source_type": {"BIGQUERY", "DYNAMICS365", "FOREIGN_CATALOG", "GA4_RAW_DATA", "MANAGED_POSTGRESQL", "MYSQL", "NETSUITE", "ORACLE", "POSTGRESQL", "SALESFORCE", "SERVICENOW", "SHAREPOINT", "SQLSERVER", "TERADATA", "WORKDAY_RAAS"}, "resources.pipelines.*.ingestion_definition.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, "resources.pipelines.*.restart_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 432dba0a7d..dcd81254be 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -143,7 +143,9 @@ var RequiredFields = map[string][]string{ "resources.jobs.*.tasks[*].webhook_notifications.on_streaming_backlog_exceeded[*]": {"id"}, "resources.jobs.*.tasks[*].webhook_notifications.on_success[*]": {"id"}, "resources.jobs.*.trigger.file_arrival": {"url"}, + "resources.jobs.*.trigger.model": {"condition"}, "resources.jobs.*.trigger.periodic": {"interval", "unit"}, + "resources.jobs.*.trigger.table_update": {"table_names"}, "resources.jobs.*.webhook_notifications.on_duration_warning_threshold_exceeded[*]": {"id"}, "resources.jobs.*.webhook_notifications.on_failure[*]": {"id"}, "resources.jobs.*.webhook_notifications.on_start[*]": {"id"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9c892a15ba..433d436dd1 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -653,7 +653,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" }, "environments": { - "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", + "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nFor serverless notebook tasks, if the environment_key is not specified, the notebook environment will be used if present. If a jobs environment is specified, it will override the notebook environment.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" }, "git_source": { @@ -3552,7 +3552,7 @@ "$ref": "#/$defs/int" }, "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\n\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, the zone \"auto\" will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\n\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", "$ref": "#/$defs/string" } }, @@ -3942,9 +3942,7 @@ }, "java_dependencies": { "description": "List of java dependencies. Each dependency is a string representing a java library path. For example: `/Volumes/path/to/test.jar`.", - "$ref": "#/$defs/slice/string", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false @@ -4492,6 +4490,10 @@ "type": "object", "description": "Custom fields that user can set for pipeline while creating SyncedDatabaseTable.\nNote that other fields of pipeline are still inferred by table def internally", "properties": { + "budget_policy_id": { + "description": "Budget policy to set on the newly created pipeline.", + "$ref": "#/$defs/string" + }, "storage_catalog": { "description": "This field needs to be specified if the destination catalog is a managed postgres catalog.\n\nUC catalog for the pipeline to store intermediate files (checkpoints, event logs etc).\nThis needs to be a standard catalog where the user has permissions to create Delta tables.", "$ref": "#/$defs/string" @@ -5594,6 +5596,59 @@ } ] }, + "jobs.ModelTriggerConfiguration": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": { + "description": "Aliases of the model versions to monitor. Can only be used in conjunction with condition MODEL_ALIAS_SET.", + "$ref": "#/$defs/slice/string" + }, + "condition": { + "description": "The condition based on which to trigger a job run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ModelTriggerConfigurationCondition" + }, + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + }, + "securable_name": { + "description": "Name of the securable to monitor (\"mycatalog.myschema.mymodel\" in the case of model-level triggers,\n\"mycatalog.myschema\" in the case of schema-level triggers) or empty in the case of metastore-level triggers.", + "$ref": "#/$defs/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no model updates have occurred for the specified time\nand can be used to wait for a series of model updates before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "condition" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ModelTriggerConfigurationCondition": { + "oneOf": [ + { + "type": "string", + "enum": [ + "MODEL_CREATED", + "MODEL_VERSION_READY", + "MODEL_ALIAS_SET" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.NotebookTask": { "oneOf": [ { @@ -6351,7 +6406,10 @@ "$ref": "#/$defs/int" } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "table_names" + ] }, { "type": "string", @@ -6645,6 +6703,11 @@ "description": "File arrival trigger settings.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.FileArrivalTriggerConfiguration" }, + "model": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ModelTriggerConfiguration", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "pause_status": { "description": "Whether this trigger is paused or not.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" @@ -6765,6 +6828,26 @@ } ] }, + "pipelines.ConnectionParameters": { + "oneOf": [ + { + "type": "object", + "properties": { + "source_catalog": { + "description": "Source catalog for initial connection.\nThis is necessary for schema exploration in some database systems like Oracle, and optional but nice-to-have\nin some other database systems like Postgres.\nFor Oracle databases, this maps to a service name.", + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.CronTrigger": { "oneOf": [ { @@ -6929,12 +7012,18 @@ "description": "Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.", "$ref": "#/$defs/string" }, + "connection_parameters": { + "description": "Optional, Internal. Parameters required to establish an initial connection with the source.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConnectionParameters", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "gateway_storage_catalog": { "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location.", "$ref": "#/$defs/string" }, "gateway_storage_name": { - "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.", + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nSpark Declarative Pipelines system will automatically create the storage location under the catalog and schema.", "$ref": "#/$defs/string" }, "gateway_storage_schema": { @@ -6964,6 +7053,12 @@ "description": "Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on.", "$ref": "#/$defs/string" }, + "ingest_from_uc_foreign_catalog": { + "description": "Immutable. If set to true, the pipeline will ingest tables from the\nUC foreign catalogs directly without the need to specify a UC connection or ingestion gateway.\nThe `source_catalog` fields in objects of IngestionConfig are interpreted as\nthe UC foreign catalogs to ingest from.", + "$ref": "#/$defs/bool", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "ingestion_gateway_id": { "description": "Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server.", "$ref": "#/$defs/string" @@ -6980,9 +7075,7 @@ }, "source_configurations": { "description": "Top-level source configurations", - "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.SourceConfig", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.SourceConfig" }, "table_configuration": { "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", @@ -7100,7 +7193,8 @@ "ORACLE", "TERADATA", "SHAREPOINT", - "DYNAMICS365" + "DYNAMICS365", + "FOREIGN_CATALOG" ] }, { @@ -7431,9 +7525,7 @@ "properties": { "slot_config": { "description": "Optional. The Postgres slot configuration to use for logical replication", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PostgresSlotConfig", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PostgresSlotConfig" } }, "additionalProperties": false @@ -7452,15 +7544,11 @@ "properties": { "publication_name": { "description": "The name of the publication to use for the Postgres source", - "$ref": "#/$defs/string", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/string" }, "slot_name": { "description": "The name of the logical replication slot to use for the Postgres source", - "$ref": "#/$defs/string", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/string" } }, "additionalProperties": false @@ -7609,15 +7697,11 @@ "properties": { "postgres": { "description": "Postgres-specific catalog-level configuration parameters", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PostgresCatalogConfig", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PostgresCatalogConfig" }, "source_catalog": { "description": "Source catalog name", - "$ref": "#/$defs/string", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/string" } }, "additionalProperties": false @@ -7635,9 +7719,7 @@ "properties": { "catalog": { "description": "Catalog-level source configuration parameters", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SourceCatalogConfig", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SourceCatalogConfig" } }, "additionalProperties": false @@ -7731,7 +7813,7 @@ "doNotSuggest": true }, "sequence_by": { - "description": "The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.", + "description": "The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order.", "$ref": "#/$defs/slice/string" }, "workday_report_parameters": { diff --git a/cmd/account/groups-v2/groups-v2.go b/cmd/account/groups-v2/groups-v2.go index 419f9ba242..c0c68e0994 100755 --- a/cmd/account/groups-v2/groups-v2.go +++ b/cmd/account/groups-v2/groups-v2.go @@ -260,8 +260,10 @@ func newList() *cobra.Command { cmd.Long = `List group details. Gets all details of the groups associated with the Databricks account. As of - 08/22/2025, this endpoint will not return members. Instead, members should be - retrieved by iterating through Get group details.` + 08/22/2025, this endpoint will no longer return members. Instead, members + should be retrieved by iterating through Get group details. Existing + accounts that rely on this attribute will not be impacted and will continue + receiving member data as before.` cmd.Annotations = make(map[string]string) diff --git a/cmd/account/settings-v2/settings-v2.go b/cmd/account/settings-v2/settings-v2.go index 2fe9dd1460..01dc85c5d7 100755 --- a/cmd/account/settings-v2/settings-v2.go +++ b/cmd/account/settings-v2/settings-v2.go @@ -187,7 +187,9 @@ func newPatchPublicAccountSetting() *cobra.Command { Patch a setting value at account level. See :method:settingsv2/listaccountsettingsmetadata for list of setting available - via public APIs at account level.` + via public APIs at account level. To determine the correct field to include in + a patch request, refer to the type field of the setting returned in the + :method:settingsv2/listaccountsettingsmetadata response.` cmd.Annotations = make(map[string]string) diff --git a/cmd/account/usage-dashboards/usage-dashboards.go b/cmd/account/usage-dashboards/usage-dashboards.go index 949f142c09..9abfba0bf9 100755 --- a/cmd/account/usage-dashboards/usage-dashboards.go +++ b/cmd/account/usage-dashboards/usage-dashboards.go @@ -56,6 +56,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().Var(&createReq.DashboardType, "dashboard-type", `Workspace level usage dashboard shows usage data for the specified workspace ID. Supported values: [USAGE_DASHBOARD_TYPE_GLOBAL, USAGE_DASHBOARD_TYPE_WORKSPACE]`) + cmd.Flags().Var(&createReq.MajorVersion, "major-version", `The major version of the usage dashboard template to use. Supported values: [USAGE_DASHBOARD_MAJOR_VERSION_1, USAGE_DASHBOARD_MAJOR_VERSION_2]`) cmd.Flags().Int64Var(&createReq.WorkspaceId, "workspace-id", createReq.WorkspaceId, `The workspace ID of the workspace in which the usage dashboard is created.`) cmd.Use = "create" diff --git a/cmd/workspace/feature-engineering/feature-engineering.go b/cmd/workspace/feature-engineering/feature-engineering.go index 893fdae760..46b2797c1e 100755 --- a/cmd/workspace/feature-engineering/feature-engineering.go +++ b/cmd/workspace/feature-engineering/feature-engineering.go @@ -4,11 +4,13 @@ package feature_engineering import ( "fmt" + "strings" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/spf13/cobra" ) @@ -31,14 +33,19 @@ func New() *cobra.Command { // Add methods cmd.AddCommand(newCreateFeature()) + cmd.AddCommand(newCreateKafkaConfig()) cmd.AddCommand(newCreateMaterializedFeature()) cmd.AddCommand(newDeleteFeature()) + cmd.AddCommand(newDeleteKafkaConfig()) cmd.AddCommand(newDeleteMaterializedFeature()) cmd.AddCommand(newGetFeature()) + cmd.AddCommand(newGetKafkaConfig()) cmd.AddCommand(newGetMaterializedFeature()) cmd.AddCommand(newListFeatures()) + cmd.AddCommand(newListKafkaConfigs()) cmd.AddCommand(newListMaterializedFeatures()) cmd.AddCommand(newUpdateFeature()) + cmd.AddCommand(newUpdateKafkaConfig()) cmd.AddCommand(newUpdateMaterializedFeature()) // Apply optional overrides to this command. @@ -69,6 +76,7 @@ func newCreateFeature() *cobra.Command { cmd.Flags().StringVar(&createFeatureReq.Feature.Description, "description", createFeatureReq.Feature.Description, `The description of the feature.`) cmd.Flags().StringVar(&createFeatureReq.Feature.FilterCondition, "filter-condition", createFeatureReq.Feature.FilterCondition, `The filter condition applied to the source data before aggregation.`) + // TODO: complex arg: lineage_context cmd.Use = "create-feature FULL_NAME SOURCE INPUTS FUNCTION TIME_WINDOW" cmd.Short = `Create a feature.` @@ -165,6 +173,111 @@ func newCreateFeature() *cobra.Command { return cmd } +// start create-kafka-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createKafkaConfigOverrides []func( + *cobra.Command, + *ml.CreateKafkaConfigRequest, +) + +func newCreateKafkaConfig() *cobra.Command { + cmd := &cobra.Command{} + + var createKafkaConfigReq ml.CreateKafkaConfigRequest + createKafkaConfigReq.KafkaConfig = ml.KafkaConfig{} + var createKafkaConfigJson flags.JsonFlag + + cmd.Flags().Var(&createKafkaConfigJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: map via StringToStringVar: extra_options + // TODO: complex arg: key_schema + // TODO: complex arg: value_schema + + cmd.Use = "create-kafka-config NAME BOOTSTRAP_SERVERS SUBSCRIPTION_MODE AUTH_CONFIG" + cmd.Short = `Create a Kafka config.` + cmd.Long = `Create a Kafka config. + + Arguments: + NAME: Name that uniquely identifies this Kafka config within the metastore. This + will be the identifier used from the Feature object to reference these + configs for a feature. Can be distinct from topic name. + BOOTSTRAP_SERVERS: A comma-separated list of host/port pairs pointing to Kafka cluster. + SUBSCRIPTION_MODE: Options to configure which Kafka topics to pull data from. + AUTH_CONFIG: Authentication configuration for connection to topics.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name', 'bootstrap_servers', 'subscription_mode', 'auth_config' in your JSON input") + } + return nil + } + check := root.ExactArgs(4) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createKafkaConfigJson.Unmarshal(&createKafkaConfigReq.KafkaConfig) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + if !cmd.Flags().Changed("json") { + createKafkaConfigReq.KafkaConfig.Name = args[0] + } + if !cmd.Flags().Changed("json") { + createKafkaConfigReq.KafkaConfig.BootstrapServers = args[1] + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[2], &createKafkaConfigReq.KafkaConfig.SubscriptionMode) + if err != nil { + return fmt.Errorf("invalid SUBSCRIPTION_MODE: %s", args[2]) + } + + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[3], &createKafkaConfigReq.KafkaConfig.AuthConfig) + if err != nil { + return fmt.Errorf("invalid AUTH_CONFIG: %s", args[3]) + } + + } + + response, err := w.FeatureEngineering.CreateKafkaConfig(ctx, createKafkaConfigReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createKafkaConfigOverrides { + fn(cmd, &createKafkaConfigReq) + } + + return cmd +} + // start create-materialized-feature command // Slice with functions to override default command behavior. @@ -304,6 +417,60 @@ func newDeleteFeature() *cobra.Command { return cmd } +// start delete-kafka-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteKafkaConfigOverrides []func( + *cobra.Command, + *ml.DeleteKafkaConfigRequest, +) + +func newDeleteKafkaConfig() *cobra.Command { + cmd := &cobra.Command{} + + var deleteKafkaConfigReq ml.DeleteKafkaConfigRequest + + cmd.Use = "delete-kafka-config NAME" + cmd.Short = `Delete a Kafka config.` + cmd.Long = `Delete a Kafka config. + + Arguments: + NAME: Name of the Kafka config to delete.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + deleteKafkaConfigReq.Name = args[0] + + err = w.FeatureEngineering.DeleteKafkaConfig(ctx, deleteKafkaConfigReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteKafkaConfigOverrides { + fn(cmd, &deleteKafkaConfigReq) + } + + return cmd +} + // start delete-materialized-feature command // Slice with functions to override default command behavior. @@ -414,6 +581,60 @@ func newGetFeature() *cobra.Command { return cmd } +// start get-kafka-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getKafkaConfigOverrides []func( + *cobra.Command, + *ml.GetKafkaConfigRequest, +) + +func newGetKafkaConfig() *cobra.Command { + cmd := &cobra.Command{} + + var getKafkaConfigReq ml.GetKafkaConfigRequest + + cmd.Use = "get-kafka-config NAME" + cmd.Short = `Get a Kafka config.` + cmd.Long = `Get a Kafka config. + + Arguments: + NAME: Name of the Kafka config to get.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getKafkaConfigReq.Name = args[0] + + response, err := w.FeatureEngineering.GetKafkaConfig(ctx, getKafkaConfigReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getKafkaConfigOverrides { + fn(cmd, &getKafkaConfigReq) + } + + return cmd +} + // start get-materialized-feature command // Slice with functions to override default command behavior. @@ -519,6 +740,55 @@ func newListFeatures() *cobra.Command { return cmd } +// start list-kafka-configs command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listKafkaConfigsOverrides []func( + *cobra.Command, + *ml.ListKafkaConfigsRequest, +) + +func newListKafkaConfigs() *cobra.Command { + cmd := &cobra.Command{} + + var listKafkaConfigsReq ml.ListKafkaConfigsRequest + + cmd.Flags().IntVar(&listKafkaConfigsReq.PageSize, "page-size", listKafkaConfigsReq.PageSize, `The maximum number of results to return.`) + cmd.Flags().StringVar(&listKafkaConfigsReq.PageToken, "page-token", listKafkaConfigsReq.PageToken, `Pagination token to go to the next page based on a previous query.`) + + cmd.Use = "list-kafka-configs" + cmd.Short = `List Kafka configs.` + cmd.Long = `List Kafka configs.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response := w.FeatureEngineering.ListKafkaConfigs(ctx, listKafkaConfigsReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listKafkaConfigsOverrides { + fn(cmd, &listKafkaConfigsReq) + } + + return cmd +} + // start list-materialized-features command // Slice with functions to override default command behavior. @@ -589,6 +859,7 @@ func newUpdateFeature() *cobra.Command { cmd.Flags().StringVar(&updateFeatureReq.Feature.Description, "description", updateFeatureReq.Feature.Description, `The description of the feature.`) cmd.Flags().StringVar(&updateFeatureReq.Feature.FilterCondition, "filter-condition", updateFeatureReq.Feature.FilterCondition, `The filter condition applied to the source data before aggregation.`) + // TODO: complex arg: lineage_context cmd.Use = "update-feature FULL_NAME UPDATE_MASK SOURCE INPUTS FUNCTION TIME_WINDOW" cmd.Short = `Update a feature's description (all other fields are immutable).` @@ -685,6 +956,114 @@ func newUpdateFeature() *cobra.Command { return cmd } +// start update-kafka-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateKafkaConfigOverrides []func( + *cobra.Command, + *ml.UpdateKafkaConfigRequest, +) + +func newUpdateKafkaConfig() *cobra.Command { + cmd := &cobra.Command{} + + var updateKafkaConfigReq ml.UpdateKafkaConfigRequest + updateKafkaConfigReq.KafkaConfig = ml.KafkaConfig{} + var updateKafkaConfigJson flags.JsonFlag + + cmd.Flags().Var(&updateKafkaConfigJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: map via StringToStringVar: extra_options + // TODO: complex arg: key_schema + // TODO: complex arg: value_schema + + cmd.Use = "update-kafka-config NAME UPDATE_MASK BOOTSTRAP_SERVERS SUBSCRIPTION_MODE AUTH_CONFIG" + cmd.Short = `Update a Kafka config.` + cmd.Long = `Update a Kafka config. + + Arguments: + NAME: Name that uniquely identifies this Kafka config within the metastore. This + will be the identifier used from the Feature object to reference these + configs for a feature. Can be distinct from topic name. + UPDATE_MASK: The list of fields to update. + BOOTSTRAP_SERVERS: A comma-separated list of host/port pairs pointing to Kafka cluster. + SUBSCRIPTION_MODE: Options to configure which Kafka topics to pull data from. + AUTH_CONFIG: Authentication configuration for connection to topics.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only NAME, UPDATE_MASK as positional arguments. Provide 'name', 'bootstrap_servers', 'subscription_mode', 'auth_config' in your JSON input") + } + return nil + } + check := root.ExactArgs(5) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateKafkaConfigJson.Unmarshal(&updateKafkaConfigReq.KafkaConfig) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + updateKafkaConfigReq.Name = args[0] + if args[1] != "" { + updateMaskArray := strings.Split(args[1], ",") + updateKafkaConfigReq.UpdateMask = *fieldmask.New(updateMaskArray) + } + if !cmd.Flags().Changed("json") { + updateKafkaConfigReq.KafkaConfig.BootstrapServers = args[2] + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[3], &updateKafkaConfigReq.KafkaConfig.SubscriptionMode) + if err != nil { + return fmt.Errorf("invalid SUBSCRIPTION_MODE: %s", args[3]) + } + + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[4], &updateKafkaConfigReq.KafkaConfig.AuthConfig) + if err != nil { + return fmt.Errorf("invalid AUTH_CONFIG: %s", args[4]) + } + + } + + response, err := w.FeatureEngineering.UpdateKafkaConfig(ctx, updateKafkaConfigReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateKafkaConfigOverrides { + fn(cmd, &updateKafkaConfigReq) + } + + return cmd +} + // start update-materialized-feature command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/feature-store/feature-store.go b/cmd/workspace/feature-store/feature-store.go index 96371c164c..b0033dc491 100755 --- a/cmd/workspace/feature-store/feature-store.go +++ b/cmd/workspace/feature-store/feature-store.go @@ -38,6 +38,7 @@ func New() *cobra.Command { // Add methods cmd.AddCommand(newCreateOnlineStore()) cmd.AddCommand(newDeleteOnlineStore()) + cmd.AddCommand(newDeleteOnlineTable()) cmd.AddCommand(newGetOnlineStore()) cmd.AddCommand(newListOnlineStores()) cmd.AddCommand(newPublishTable()) @@ -192,6 +193,62 @@ func newDeleteOnlineStore() *cobra.Command { return cmd } +// start delete-online-table command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOnlineTableOverrides []func( + *cobra.Command, + *ml.DeleteOnlineTableRequest, +) + +func newDeleteOnlineTable() *cobra.Command { + cmd := &cobra.Command{} + + var deleteOnlineTableReq ml.DeleteOnlineTableRequest + + cmd.Use = "delete-online-table ONLINE_TABLE_NAME" + cmd.Short = `Delete an online table.` + cmd.Long = `Delete an online table. + + Delete online table. + + Arguments: + ONLINE_TABLE_NAME: The full three-part (catalog, schema, table) name of the online table.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + deleteOnlineTableReq.OnlineTableName = args[0] + + err = w.FeatureStore.DeleteOnlineTable(ctx, deleteOnlineTableReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOnlineTableOverrides { + fn(cmd, &deleteOnlineTableReq) + } + + return cmd +} + // start get-online-store command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go index 7b7e84e4aa..04d5b684de 100755 --- a/cmd/workspace/genie/genie.go +++ b/cmd/workspace/genie/genie.go @@ -33,6 +33,7 @@ func New() *cobra.Command { // Add methods cmd.AddCommand(newCreateMessage()) + cmd.AddCommand(newCreateSpace()) cmd.AddCommand(newDeleteConversation()) cmd.AddCommand(newDeleteConversationMessage()) cmd.AddCommand(newExecuteMessageAttachmentQuery()) @@ -48,6 +49,7 @@ func New() *cobra.Command { cmd.AddCommand(newSendMessageFeedback()) cmd.AddCommand(newStartConversation()) cmd.AddCommand(newTrashSpace()) + cmd.AddCommand(newUpdateSpace()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -162,6 +164,98 @@ func newCreateMessage() *cobra.Command { return cmd } +// start create-space command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createSpaceOverrides []func( + *cobra.Command, + *dashboards.GenieCreateSpaceRequest, +) + +func newCreateSpace() *cobra.Command { + cmd := &cobra.Command{} + + var createSpaceReq dashboards.GenieCreateSpaceRequest + var createSpaceJson flags.JsonFlag + + cmd.Flags().Var(&createSpaceJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createSpaceReq.Description, "description", createSpaceReq.Description, `Optional description.`) + cmd.Flags().StringVar(&createSpaceReq.ParentPath, "parent-path", createSpaceReq.ParentPath, `Parent folder path where the space will be registered.`) + cmd.Flags().StringVar(&createSpaceReq.Title, "title", createSpaceReq.Title, `Optional title override.`) + + cmd.Use = "create-space WAREHOUSE_ID SERIALIZED_SPACE" + cmd.Short = `Create Genie Space.` + cmd.Long = `Create Genie Space. + + Creates a Genie space from a serialized payload. + + Arguments: + WAREHOUSE_ID: Warehouse to associate with the new space + SERIALIZED_SPACE: The contents of the Genie Space in serialized string form. Use the [Get + Genie Space](:method:genie/getspace) API to retrieve an example response, + which includes the serialized_space field. This field provides the + structure of the JSON string that represents the space's layout and + components.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'warehouse_id', 'serialized_space' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createSpaceJson.Unmarshal(&createSpaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + if !cmd.Flags().Changed("json") { + createSpaceReq.WarehouseId = args[0] + } + if !cmd.Flags().Changed("json") { + createSpaceReq.SerializedSpace = args[1] + } + + response, err := w.Genie.CreateSpace(ctx, createSpaceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createSpaceOverrides { + fn(cmd, &createSpaceReq) + } + + return cmd +} + // start delete-conversation command // Slice with functions to override default command behavior. @@ -677,6 +771,8 @@ func newGetSpace() *cobra.Command { var getSpaceReq dashboards.GenieGetSpaceRequest + cmd.Flags().BoolVar(&getSpaceReq.IncludeSerializedSpace, "include-serialized-space", getSpaceReq.IncludeSerializedSpace, `Whether to include the serialized space export in the response.`) + cmd.Use = "get-space SPACE_ID" cmd.Short = `Get Genie Space.` cmd.Long = `Get Genie Space. @@ -1143,4 +1239,80 @@ func newTrashSpace() *cobra.Command { return cmd } +// start update-space command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateSpaceOverrides []func( + *cobra.Command, + *dashboards.GenieUpdateSpaceRequest, +) + +func newUpdateSpace() *cobra.Command { + cmd := &cobra.Command{} + + var updateSpaceReq dashboards.GenieUpdateSpaceRequest + var updateSpaceJson flags.JsonFlag + + cmd.Flags().Var(&updateSpaceJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&updateSpaceReq.Description, "description", updateSpaceReq.Description, `Optional description.`) + cmd.Flags().StringVar(&updateSpaceReq.SerializedSpace, "serialized-space", updateSpaceReq.SerializedSpace, `The contents of the Genie Space in serialized string form (full replacement).`) + cmd.Flags().StringVar(&updateSpaceReq.Title, "title", updateSpaceReq.Title, `Optional title override.`) + cmd.Flags().StringVar(&updateSpaceReq.WarehouseId, "warehouse-id", updateSpaceReq.WarehouseId, `Optional warehouse override.`) + + cmd.Use = "update-space SPACE_ID" + cmd.Short = `Update Genie Space.` + cmd.Long = `Update Genie Space. + + Updates a Genie space with a serialized payload. + + Arguments: + SPACE_ID: Genie space ID` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateSpaceJson.Unmarshal(&updateSpaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + updateSpaceReq.SpaceId = args[0] + + response, err := w.Genie.UpdateSpace(ctx, updateSpaceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateSpaceOverrides { + fn(cmd, &updateSpaceReq) + } + + return cmd +} + // end service Genie diff --git a/cmd/workspace/groups.go b/cmd/workspace/groups.go index 01c1c6ad20..4b99544cd6 100644 --- a/cmd/workspace/groups.go +++ b/cmd/workspace/groups.go @@ -17,12 +17,8 @@ func Groups() []cobra.Group { Title: "Compute", }, { - ID: "jobs", - Title: "Workflows", - }, - { - ID: "pipelines", - Title: "Delta Live Tables", + ID: "lakeflow", + Title: "Lakeflow", }, { ID: "files", diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 029df5de87..803af77df4 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -30,7 +30,7 @@ func New() *cobra.Command { Databricks manages the task orchestration, cluster management, monitoring, and error reporting for all of your jobs. You can run your jobs immediately or periodically through an easy-to-use scheduling system. You can implement job - tasks using notebooks, JARS, Delta Live Tables pipelines, or Python, Scala, + tasks using notebooks, JARS, Spark Declarative Pipelines, or Python, Scala, Spark submit, and Java applications. You should never hard code secrets or store them in plain text. Use the @@ -40,7 +40,7 @@ func New() *cobra.Command { [Databricks CLI]: https://docs.databricks.com/dev-tools/cli/index.html [Secrets CLI]: https://docs.databricks.com/dev-tools/cli/secrets-cli.html [Secrets utility]: https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-secrets`, - GroupID: "jobs", + GroupID: "lakeflow", RunE: root.ReportUnknownSubcommand, } diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index 266a9064bb..b45364b348 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -21,22 +21,23 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "pipelines", - Short: `The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines.`, - Long: `The Delta Live Tables API allows you to create, edit, delete, start, and view - details about pipelines. + Short: `The Lakeflow Spark Declarative Pipelines API allows you to create, edit, delete, start, and view details about pipelines.`, + Long: `The Lakeflow Spark Declarative Pipelines API allows you to create, edit, + delete, start, and view details about pipelines. - Delta Live Tables is a framework for building reliable, maintainable, and - testable data processing pipelines. You define the transformations to perform - on your data, and Delta Live Tables manages task orchestration, cluster - management, monitoring, data quality, and error handling. + Spark Declarative Pipelines is a framework for building reliable, + maintainable, and testable data processing pipelines. You define the + transformations to perform on your data, and Spark Declarative Pipelines + manages task orchestration, cluster management, monitoring, data quality, and + error handling. Instead of defining your data pipelines using a series of separate Apache - Spark tasks, Delta Live Tables manages how your data is transformed based on a - target schema you define for each processing step. You can also enforce data - quality with Delta Live Tables expectations. Expectations allow you to define - expected data quality and specify how to handle records that fail those - expectations.`, - GroupID: "pipelines", + Spark tasks, Spark Declarative Pipelines manages how your data is transformed + based on a target schema you define for each processing step. You can also + enforce data quality with Spark Declarative Pipelines expectations. + Expectations allow you to define expected data quality and specify how to + handle records that fail those expectations.`, + GroupID: "lakeflow", RunE: root.ReportUnknownSubcommand, } @@ -147,8 +148,9 @@ func newDelete() *cobra.Command { cmd.Short = `Delete a pipeline.` cmd.Long = `Delete a pipeline. - Deletes a pipeline. Deleting a pipeline is a permanent action that stops and - removes the pipeline and its tables. You cannot undo this action.` + Deletes a pipeline. If the pipeline publishes to Unity Catalog, pipeline + deletion will cascade to all pipeline tables. Please reach out to Databricks + support for assistance to undo this action.` cmd.Annotations = make(map[string]string) @@ -546,7 +548,7 @@ func newListPipelines() *cobra.Command { cmd.Short = `List pipelines.` cmd.Long = `List pipelines. - Lists pipelines defined in the Delta Live Tables system.` + Lists pipelines defined in the Spark Declarative Pipelines system.` cmd.Annotations = make(map[string]string) @@ -764,6 +766,7 @@ func newStartUpdate() *cobra.Command { cmd.Flags().BoolVar(&startUpdateReq.FullRefresh, "full-refresh", startUpdateReq.FullRefresh, `If true, this update will reset all tables before running.`) // TODO: array: full_refresh_selection // TODO: array: refresh_selection + // TODO: complex arg: rewind_spec cmd.Flags().BoolVar(&startUpdateReq.ValidateOnly, "validate-only", startUpdateReq.ValidateOnly, `If true, this update only validates the correctness of pipeline source code but does not materialize or publish any datasets.`) cmd.Use = "start-update PIPELINE_ID" diff --git a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go index c57089a2e8..4861d1a96d 100755 --- a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go +++ b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go @@ -34,7 +34,7 @@ func New() *cobra.Command { The get and list compliance APIs allow you to view the policy compliance status of a job. The enforce compliance API allows you to update a job so that it becomes compliant with all of its policies.`, - GroupID: "jobs", + GroupID: "lakeflow", RunE: root.ReportUnknownSubcommand, } diff --git a/cmd/workspace/rfa/rfa.go b/cmd/workspace/rfa/rfa.go index c95a629e0a..d16e3c4604 100755 --- a/cmd/workspace/rfa/rfa.go +++ b/cmd/workspace/rfa/rfa.go @@ -20,13 +20,12 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "rfa", - Short: `Request for Access enables customers to request access to and manage access request destinations for Unity Catalog securables.`, - Long: `Request for Access enables customers to request access to and manage access - request destinations for Unity Catalog securables. + Short: `Request for Access enables users to request access for Unity Catalog securables.`, + Long: `Request for Access enables users to request access for Unity Catalog + securables. - These APIs provide a standardized way to update, get, and request to access - request destinations. Fine-grained authorization ensures that only users with - appropriate permissions can manage access request destinations.`, + These APIs provide a standardized way for securable owners (or users with + MANAGE privileges) to manage access request destinations.`, GroupID: "catalog", RunE: root.ReportUnknownSubcommand, } @@ -204,7 +203,9 @@ func newUpdateAccessRequestDestinations() *cobra.Command { cmd.Flags().Var(&updateAccessRequestDestinationsJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Use = "update-access-request-destinations UPDATE_MASK DESTINATIONS SECURABLE" + // TODO: array: destinations + + cmd.Use = "update-access-request-destinations UPDATE_MASK SECURABLE" cmd.Short = `Update Access Request Destinations.` cmd.Long = `Update Access Request Destinations. @@ -234,7 +235,6 @@ func newUpdateAccessRequestDestinations() *cobra.Command { always explicitly list the fields being updated and avoid using * wildcards, as it can lead to unintended results if the API changes in the future. - DESTINATIONS: The access request destinations for the securable. SECURABLE: The securable for which the access request destinations are being retrieved.` @@ -244,11 +244,11 @@ func newUpdateAccessRequestDestinations() *cobra.Command { if cmd.Flags().Changed("json") { err := root.ExactArgs(1)(cmd, args) if err != nil { - return fmt.Errorf("when --json flag is specified, provide only UPDATE_MASK as positional arguments. Provide 'destinations', 'securable' in your JSON input") + return fmt.Errorf("when --json flag is specified, provide only UPDATE_MASK as positional arguments. Provide 'securable' in your JSON input") } return nil } - check := root.ExactArgs(3) + check := root.ExactArgs(2) return check(cmd, args) } @@ -271,16 +271,9 @@ func newUpdateAccessRequestDestinations() *cobra.Command { } updateAccessRequestDestinationsReq.UpdateMask = args[0] if !cmd.Flags().Changed("json") { - _, err = fmt.Sscan(args[1], &updateAccessRequestDestinationsReq.AccessRequestDestinations.Destinations) - if err != nil { - return fmt.Errorf("invalid DESTINATIONS: %s", args[1]) - } - - } - if !cmd.Flags().Changed("json") { - _, err = fmt.Sscan(args[2], &updateAccessRequestDestinationsReq.AccessRequestDestinations.Securable) + _, err = fmt.Sscan(args[1], &updateAccessRequestDestinationsReq.AccessRequestDestinations.Securable) if err != nil { - return fmt.Errorf("invalid SECURABLE: %s", args[2]) + return fmt.Errorf("invalid SECURABLE: %s", args[1]) } } diff --git a/cmd/workspace/tag-policies/tag-policies.go b/cmd/workspace/tag-policies/tag-policies.go index 424f2a3ab7..049261bc97 100755 --- a/cmd/workspace/tag-policies/tag-policies.go +++ b/cmd/workspace/tag-policies/tag-policies.go @@ -22,10 +22,12 @@ func New() *cobra.Command { Use: "tag-policies", Short: `The Tag Policy API allows you to manage policies for governed tags in Databricks.`, Long: `The Tag Policy API allows you to manage policies for governed tags in - Databricks. Permissions for tag policies can be managed using the [Account - Access Control Proxy API]. + Databricks. For Terraform usage, see the [Tag Policy Terraform documentation]. + Permissions for tag policies can be managed using the [Account Access Control + Proxy API]. - [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy`, + [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy + [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/tag_policy`, GroupID: "tags", RunE: root.ReportUnknownSubcommand, } @@ -70,7 +72,12 @@ func newCreateTagPolicy() *cobra.Command { cmd.Short = `Create a new tag policy.` cmd.Long = `Create a new tag policy. - Creates a new tag policy, making the associated tag key governed.` + Creates a new tag policy, making the associated tag key governed. For + Terraform usage, see the [Tag Policy Terraform documentation]. To manage + permissions for tag policies, use the [Account Access Control Proxy API]. + + [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy + [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/tag_policy` cmd.Annotations = make(map[string]string) @@ -145,7 +152,10 @@ func newDeleteTagPolicy() *cobra.Command { cmd.Long = `Delete a tag policy. Deletes a tag policy by its associated governed tag's key, leaving that tag - key ungoverned.` + key ungoverned. For Terraform usage, see the [Tag Policy Terraform + documentation]. + + [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/tag_policy` cmd.Annotations = make(map[string]string) @@ -198,7 +208,12 @@ func newGetTagPolicy() *cobra.Command { cmd.Short = `Get a tag policy.` cmd.Long = `Get a tag policy. - Gets a single tag policy by its associated governed tag's key.` + Gets a single tag policy by its associated governed tag's key. For Terraform + usage, see the [Tag Policy Terraform documentation]. To list granted + permissions for tag policies, use the [Account Access Control Proxy API]. + + [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy + [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/tag_policy` cmd.Annotations = make(map[string]string) @@ -254,7 +269,12 @@ func newListTagPolicies() *cobra.Command { cmd.Short = `List tag policies.` cmd.Long = `List tag policies. - Lists the tag policies for all governed tags in the account.` + Lists the tag policies for all governed tags in the account. For Terraform + usage, see the [Tag Policy Terraform documentation]. To list granted + permissions for tag policies, use the [Account Access Control Proxy API]. + + [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy + [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/tag_policies` cmd.Annotations = make(map[string]string) @@ -309,7 +329,12 @@ func newUpdateTagPolicy() *cobra.Command { cmd.Short = `Update an existing tag policy.` cmd.Long = `Update an existing tag policy. - Updates an existing tag policy for a single governed tag. + Updates an existing tag policy for a single governed tag. For Terraform usage, + see the [Tag Policy Terraform documentation]. To manage permissions for tag + policies, use the [Account Access Control Proxy API]. + + [Account Access Control Proxy API]: https://docs.databricks.com/api/workspace/accountaccesscontrolproxy + [Tag Policy Terraform documentation]: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/tag_policy Arguments: TAG_KEY: diff --git a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go index 0e4696d154..dcdcf47498 100755 --- a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go +++ b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go @@ -32,6 +32,7 @@ func New() *cobra.Command { cmd.AddCommand(newDeleteEndpoint()) cmd.AddCommand(newGetEndpoint()) cmd.AddCommand(newListEndpoints()) + cmd.AddCommand(newRetrieveUserVisibleMetrics()) cmd.AddCommand(newUpdateEndpointBudgetPolicy()) cmd.AddCommand(newUpdateEndpointCustomTags()) @@ -321,6 +322,83 @@ func newListEndpoints() *cobra.Command { return cmd } +// start retrieve-user-visible-metrics command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var retrieveUserVisibleMetricsOverrides []func( + *cobra.Command, + *vectorsearch.RetrieveUserVisibleMetricsRequest, +) + +func newRetrieveUserVisibleMetrics() *cobra.Command { + cmd := &cobra.Command{} + + var retrieveUserVisibleMetricsReq vectorsearch.RetrieveUserVisibleMetricsRequest + var retrieveUserVisibleMetricsJson flags.JsonFlag + + cmd.Flags().Var(&retrieveUserVisibleMetricsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&retrieveUserVisibleMetricsReq.EndTime, "end-time", retrieveUserVisibleMetricsReq.EndTime, `End time for metrics query.`) + cmd.Flags().IntVar(&retrieveUserVisibleMetricsReq.GranularityInSeconds, "granularity-in-seconds", retrieveUserVisibleMetricsReq.GranularityInSeconds, `Granularity in seconds.`) + // TODO: array: metrics + cmd.Flags().StringVar(&retrieveUserVisibleMetricsReq.PageToken, "page-token", retrieveUserVisibleMetricsReq.PageToken, `Token for pagination.`) + cmd.Flags().StringVar(&retrieveUserVisibleMetricsReq.StartTime, "start-time", retrieveUserVisibleMetricsReq.StartTime, `Start time for metrics query.`) + + cmd.Use = "retrieve-user-visible-metrics NAME" + cmd.Short = `Retrieve user-visible metrics for an endpoint.` + cmd.Long = `Retrieve user-visible metrics for an endpoint. + + Retrieve user-visible metrics for an endpoint + + Arguments: + NAME: Vector search endpoint name` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := retrieveUserVisibleMetricsJson.Unmarshal(&retrieveUserVisibleMetricsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + retrieveUserVisibleMetricsReq.Name = args[0] + + response, err := w.VectorSearchEndpoints.RetrieveUserVisibleMetrics(ctx, retrieveUserVisibleMetricsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range retrieveUserVisibleMetricsOverrides { + fn(cmd, &retrieveUserVisibleMetricsReq) + } + + return cmd +} + // start update-endpoint-budget-policy command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go b/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go index 361a009c69..4c0c81c02b 100755 --- a/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go +++ b/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go @@ -57,7 +57,10 @@ func newGetPublicWorkspaceSetting() *cobra.Command { Get a setting value at workspace level. See :method:settingsv2/listworkspacesettingsmetadata for list of setting available - via public APIs.` + via public APIs. + + Arguments: + NAME: Name of the setting` cmd.Annotations = make(map[string]string) @@ -187,7 +190,12 @@ func newPatchPublicWorkspaceSetting() *cobra.Command { Patch a setting value at workspace level. See :method:settingsv2/listworkspacesettingsmetadata for list of setting available - via public APIs at workspace level.` + via public APIs at workspace level. To determine the correct field to include + in a patch request, refer to the type field of the setting returned in the + :method:settingsv2/listworkspacesettingsmetadata response. + + Arguments: + NAME: Name of the setting` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/workspace/workspace.go b/cmd/workspace/workspace/workspace.go index 192ba0525e..c22382a123 100755 --- a/cmd/workspace/workspace/workspace.go +++ b/cmd/workspace/workspace/workspace.go @@ -262,7 +262,9 @@ func newGetPermissionLevels() *cobra.Command { Gets the permission levels that a user can have on an object. Arguments: - WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. + WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. Could be + one of the following: alerts, alertsv2, dashboards, dbsql-dashboards, + directories, experiments, files, genie, notebooks, queries WORKSPACE_OBJECT_ID: The workspace object for which to get or manage permissions.` cmd.Annotations = make(map[string]string) @@ -321,7 +323,9 @@ func newGetPermissions() *cobra.Command { permissions from their parent objects or root object. Arguments: - WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. + WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. Could be + one of the following: alerts, alertsv2, dashboards, dbsql-dashboards, + directories, experiments, files, genie, notebooks, queries WORKSPACE_OBJECT_ID: The workspace object for which to get or manage permissions.` cmd.Annotations = make(map[string]string) @@ -701,7 +705,9 @@ func newSetPermissions() *cobra.Command { permissions from their parent objects or root object. Arguments: - WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. + WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. Could be + one of the following: alerts, alertsv2, dashboards, dbsql-dashboards, + directories, experiments, files, genie, notebooks, queries WORKSPACE_OBJECT_ID: The workspace object for which to get or manage permissions.` cmd.Annotations = make(map[string]string) @@ -777,7 +783,9 @@ func newUpdatePermissions() *cobra.Command { permissions from their parent objects or root object. Arguments: - WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. + WORKSPACE_OBJECT_TYPE: The workspace object type for which to get or manage permissions. Could be + one of the following: alerts, alertsv2, dashboards, dbsql-dashboards, + directories, experiments, files, genie, notebooks, queries WORKSPACE_OBJECT_ID: The workspace object for which to get or manage permissions.` cmd.Annotations = make(map[string]string) diff --git a/go.mod b/go.mod index b990b2c032..18cbf64c6f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/BurntSushi/toml v1.5.0 // MIT github.com/Masterminds/semver/v3 v3.4.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.91.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.92.0 // Apache 2.0 github.com/fatih/color v1.18.0 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/gorilla/mux v1.8.1 // BSD 3-Clause diff --git a/go.sum b/go.sum index 36e4789147..4727b8947e 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/databricks/databricks-sdk-go v0.91.0 h1:ygoPB5j1SlLFnfO1k9CkIoCekGJ4yi2N0Fj2mpL3bE4= -github.com/databricks/databricks-sdk-go v0.91.0/go.mod h1:hWoHnHbNLjPKiTm5K/7bcIv3J3Pkgo5x9pPzh8K3RVE= +github.com/databricks/databricks-sdk-go v0.92.0 h1:Pvm/LvEXetOYJPgQFeewhmtkKC4z79AK3PHC5tlC+VQ= +github.com/databricks/databricks-sdk-go v0.92.0/go.mod h1:hWoHnHbNLjPKiTm5K/7bcIv3J3Pkgo5x9pPzh8K3RVE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/python/databricks/bundles/jobs/__init__.py b/python/databricks/bundles/jobs/__init__.py index 860940222a..a1f36e2399 100644 --- a/python/databricks/bundles/jobs/__init__.py +++ b/python/databricks/bundles/jobs/__init__.py @@ -148,6 +148,11 @@ "MavenLibrary", "MavenLibraryDict", "MavenLibraryParam", + "ModelTriggerConfiguration", + "ModelTriggerConfigurationCondition", + "ModelTriggerConfigurationConditionParam", + "ModelTriggerConfigurationDict", + "ModelTriggerConfigurationParam", "NotebookTask", "NotebookTaskDict", "NotebookTaskParam", @@ -508,6 +513,15 @@ MavenLibraryDict, MavenLibraryParam, ) +from databricks.bundles.jobs._models.model_trigger_configuration import ( + ModelTriggerConfiguration, + ModelTriggerConfigurationDict, + ModelTriggerConfigurationParam, +) +from databricks.bundles.jobs._models.model_trigger_configuration_condition import ( + ModelTriggerConfigurationCondition, + ModelTriggerConfigurationConditionParam, +) from databricks.bundles.jobs._models.notebook_task import ( NotebookTask, NotebookTaskDict, diff --git a/python/databricks/bundles/jobs/_models/aws_attributes.py b/python/databricks/bundles/jobs/_models/aws_attributes.py index d38c3187e6..62ddf7a765 100644 --- a/python/databricks/bundles/jobs/_models/aws_attributes.py +++ b/python/databricks/bundles/jobs/_models/aws_attributes.py @@ -104,7 +104,7 @@ class AwsAttributes: This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. - This is an optional field at cluster creation, and if not specified, a default zone will be used. + This is an optional field at cluster creation, and if not specified, the zone "auto" will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. @@ -204,7 +204,7 @@ class AwsAttributesDict(TypedDict, total=False): This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. - This is an optional field at cluster creation, and if not specified, a default zone will be used. + This is an optional field at cluster creation, and if not specified, the zone "auto" will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. diff --git a/python/databricks/bundles/jobs/_models/dashboard_task.py b/python/databricks/bundles/jobs/_models/dashboard_task.py index b42ef0bdd5..6284ca36d3 100644 --- a/python/databricks/bundles/jobs/_models/dashboard_task.py +++ b/python/databricks/bundles/jobs/_models/dashboard_task.py @@ -4,7 +4,10 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrOptional -from databricks.bundles.jobs._models.subscription import Subscription, SubscriptionParam +from databricks.bundles.jobs._models.subscription import ( + Subscription, + SubscriptionParam, +) if TYPE_CHECKING: from typing_extensions import Self diff --git a/python/databricks/bundles/jobs/_models/environment.py b/python/databricks/bundles/jobs/_models/environment.py index 1521fffeda..8beefedfb6 100644 --- a/python/databricks/bundles/jobs/_models/environment.py +++ b/python/databricks/bundles/jobs/_models/environment.py @@ -35,8 +35,6 @@ class Environment: java_dependencies: VariableOrList[str] = field(default_factory=list) """ - :meta private: [EXPERIMENTAL] - List of java dependencies. Each dependency is a string representing a java library path. For example: `/Volumes/path/to/test.jar`. """ @@ -70,8 +68,6 @@ class EnvironmentDict(TypedDict, total=False): java_dependencies: VariableOrList[str] """ - :meta private: [EXPERIMENTAL] - List of java dependencies. Each dependency is a string representing a java library path. For example: `/Volumes/path/to/test.jar`. """ diff --git a/python/databricks/bundles/jobs/_models/job.py b/python/databricks/bundles/jobs/_models/job.py index a751c6c43e..88b6ee17a7 100644 --- a/python/databricks/bundles/jobs/_models/job.py +++ b/python/databricks/bundles/jobs/_models/job.py @@ -96,8 +96,7 @@ class Job(Resource): environments: VariableOrList[JobEnvironment] = field(default_factory=list) """ A list of task execution environment specifications that can be referenced by serverless tasks of this job. - An environment is required to be present for serverless tasks. - For serverless notebook tasks, the environment is accessible in the notebook environment panel. + For serverless notebook tasks, if the environment_key is not specified, the notebook environment will be used if present. If a jobs environment is specified, it will override the notebook environment. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. """ @@ -241,8 +240,7 @@ class JobDict(TypedDict, total=False): environments: VariableOrList[JobEnvironmentParam] """ A list of task execution environment specifications that can be referenced by serverless tasks of this job. - An environment is required to be present for serverless tasks. - For serverless notebook tasks, the environment is accessible in the notebook environment panel. + For serverless notebook tasks, if the environment_key is not specified, the notebook environment will be used if present. If a jobs environment is specified, it will override the notebook environment. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. """ diff --git a/python/databricks/bundles/jobs/_models/model_trigger_configuration.py b/python/databricks/bundles/jobs/_models/model_trigger_configuration.py new file mode 100644 index 0000000000..140e7e09ec --- /dev/null +++ b/python/databricks/bundles/jobs/_models/model_trigger_configuration.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.jobs._models.model_trigger_configuration_condition import ( + ModelTriggerConfigurationCondition, + ModelTriggerConfigurationConditionParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class ModelTriggerConfiguration: + """ + :meta private: [EXPERIMENTAL] + """ + + condition: VariableOr[ModelTriggerConfigurationCondition] + """ + The condition based on which to trigger a job run. + """ + + aliases: VariableOrList[str] = field(default_factory=list) + """ + Aliases of the model versions to monitor. Can only be used in conjunction with condition MODEL_ALIAS_SET. + """ + + min_time_between_triggers_seconds: VariableOrOptional[int] = None + """ + If set, the trigger starts a run only after the specified amount of time has passed since + the last time the trigger fired. The minimum allowed value is 60 seconds. + """ + + securable_name: VariableOrOptional[str] = None + """ + Name of the securable to monitor ("mycatalog.myschema.mymodel" in the case of model-level triggers, + "mycatalog.myschema" in the case of schema-level triggers) or empty in the case of metastore-level triggers. + """ + + wait_after_last_change_seconds: VariableOrOptional[int] = None + """ + If set, the trigger starts a run only after no model updates have occurred for the specified time + and can be used to wait for a series of model updates before triggering a run. The + minimum allowed value is 60 seconds. + """ + + @classmethod + def from_dict(cls, value: "ModelTriggerConfigurationDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "ModelTriggerConfigurationDict": + return _transform_to_json_value(self) # type:ignore + + +class ModelTriggerConfigurationDict(TypedDict, total=False): + """""" + + condition: VariableOr[ModelTriggerConfigurationConditionParam] + """ + The condition based on which to trigger a job run. + """ + + aliases: VariableOrList[str] + """ + Aliases of the model versions to monitor. Can only be used in conjunction with condition MODEL_ALIAS_SET. + """ + + min_time_between_triggers_seconds: VariableOrOptional[int] + """ + If set, the trigger starts a run only after the specified amount of time has passed since + the last time the trigger fired. The minimum allowed value is 60 seconds. + """ + + securable_name: VariableOrOptional[str] + """ + Name of the securable to monitor ("mycatalog.myschema.mymodel" in the case of model-level triggers, + "mycatalog.myschema" in the case of schema-level triggers) or empty in the case of metastore-level triggers. + """ + + wait_after_last_change_seconds: VariableOrOptional[int] + """ + If set, the trigger starts a run only after no model updates have occurred for the specified time + and can be used to wait for a series of model updates before triggering a run. The + minimum allowed value is 60 seconds. + """ + + +ModelTriggerConfigurationParam = ( + ModelTriggerConfigurationDict | ModelTriggerConfiguration +) diff --git a/python/databricks/bundles/jobs/_models/model_trigger_configuration_condition.py b/python/databricks/bundles/jobs/_models/model_trigger_configuration_condition.py new file mode 100644 index 0000000000..60d942a2c5 --- /dev/null +++ b/python/databricks/bundles/jobs/_models/model_trigger_configuration_condition.py @@ -0,0 +1,18 @@ +from enum import Enum +from typing import Literal + + +class ModelTriggerConfigurationCondition(Enum): + """ + :meta private: [EXPERIMENTAL] + """ + + MODEL_CREATED = "MODEL_CREATED" + MODEL_VERSION_READY = "MODEL_VERSION_READY" + MODEL_ALIAS_SET = "MODEL_ALIAS_SET" + + +ModelTriggerConfigurationConditionParam = ( + Literal["MODEL_CREATED", "MODEL_VERSION_READY", "MODEL_ALIAS_SET"] + | ModelTriggerConfigurationCondition +) diff --git a/python/databricks/bundles/jobs/_models/trigger_settings.py b/python/databricks/bundles/jobs/_models/trigger_settings.py index 18cccd14dd..67b3a4def7 100644 --- a/python/databricks/bundles/jobs/_models/trigger_settings.py +++ b/python/databricks/bundles/jobs/_models/trigger_settings.py @@ -8,6 +8,10 @@ FileArrivalTriggerConfiguration, FileArrivalTriggerConfigurationParam, ) +from databricks.bundles.jobs._models.model_trigger_configuration import ( + ModelTriggerConfiguration, + ModelTriggerConfigurationParam, +) from databricks.bundles.jobs._models.pause_status import PauseStatus, PauseStatusParam from databricks.bundles.jobs._models.periodic_trigger_configuration import ( PeriodicTriggerConfiguration, @@ -31,6 +35,11 @@ class TriggerSettings: File arrival trigger settings. """ + model: VariableOrOptional[ModelTriggerConfiguration] = None + """ + :meta private: [EXPERIMENTAL] + """ + pause_status: VariableOrOptional[PauseStatus] = None """ Whether this trigger is paused or not. @@ -59,6 +68,11 @@ class TriggerSettingsDict(TypedDict, total=False): File arrival trigger settings. """ + model: VariableOrOptional[ModelTriggerConfigurationParam] + """ + :meta private: [EXPERIMENTAL] + """ + pause_status: VariableOrOptional[PauseStatusParam] """ Whether this trigger is paused or not. diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 4fae28e74c..cadfc3e87b 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -15,6 +15,9 @@ "ClusterLogConf", "ClusterLogConfDict", "ClusterLogConfParam", + "ConnectionParameters", + "ConnectionParametersDict", + "ConnectionParametersParam", "DayOfWeek", "DayOfWeekParam", "DbfsStorageInfo", @@ -175,6 +178,11 @@ ClusterLogConfDict, ClusterLogConfParam, ) +from databricks.bundles.pipelines._models.connection_parameters import ( + ConnectionParameters, + ConnectionParametersDict, + ConnectionParametersParam, +) from databricks.bundles.pipelines._models.day_of_week import DayOfWeek, DayOfWeekParam from databricks.bundles.pipelines._models.dbfs_storage_info import ( DbfsStorageInfo, diff --git a/python/databricks/bundles/pipelines/_models/aws_attributes.py b/python/databricks/bundles/pipelines/_models/aws_attributes.py index 542f55aa2f..47bc0b97ce 100644 --- a/python/databricks/bundles/pipelines/_models/aws_attributes.py +++ b/python/databricks/bundles/pipelines/_models/aws_attributes.py @@ -104,7 +104,7 @@ class AwsAttributes: This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. - This is an optional field at cluster creation, and if not specified, a default zone will be used. + This is an optional field at cluster creation, and if not specified, the zone "auto" will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. @@ -204,7 +204,7 @@ class AwsAttributesDict(TypedDict, total=False): This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. - This is an optional field at cluster creation, and if not specified, a default zone will be used. + This is an optional field at cluster creation, and if not specified, the zone "auto" will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. diff --git a/python/databricks/bundles/pipelines/_models/connection_parameters.py b/python/databricks/bundles/pipelines/_models/connection_parameters.py new file mode 100644 index 0000000000..98ed0dbcda --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/connection_parameters.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class ConnectionParameters: + """ + :meta private: [EXPERIMENTAL] + """ + + source_catalog: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + Source catalog for initial connection. + This is necessary for schema exploration in some database systems like Oracle, and optional but nice-to-have + in some other database systems like Postgres. + For Oracle databases, this maps to a service name. + """ + + @classmethod + def from_dict(cls, value: "ConnectionParametersDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "ConnectionParametersDict": + return _transform_to_json_value(self) # type:ignore + + +class ConnectionParametersDict(TypedDict, total=False): + """""" + + source_catalog: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + Source catalog for initial connection. + This is necessary for schema exploration in some database systems like Oracle, and optional but nice-to-have + in some other database systems like Postgres. + For Oracle databases, this maps to a service name. + """ + + +ConnectionParametersParam = ConnectionParametersDict | ConnectionParameters diff --git a/python/databricks/bundles/pipelines/_models/ingestion_config.py b/python/databricks/bundles/pipelines/_models/ingestion_config.py index 988227c43e..c452222df9 100644 --- a/python/databricks/bundles/pipelines/_models/ingestion_config.py +++ b/python/databricks/bundles/pipelines/_models/ingestion_config.py @@ -4,10 +4,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrOptional -from databricks.bundles.pipelines._models.report_spec import ( - ReportSpec, - ReportSpecParam, -) +from databricks.bundles.pipelines._models.report_spec import ReportSpec, ReportSpecParam from databricks.bundles.pipelines._models.schema_spec import SchemaSpec, SchemaSpecParam from databricks.bundles.pipelines._models.table_spec import TableSpec, TableSpecParam diff --git a/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py b/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py index fd278e5e6d..abe37602e5 100644 --- a/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py +++ b/python/databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py @@ -4,6 +4,10 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOr, VariableOrOptional +from databricks.bundles.pipelines._models.connection_parameters import ( + ConnectionParameters, + ConnectionParametersParam, +) if TYPE_CHECKING: from typing_extensions import Self @@ -35,11 +39,18 @@ class IngestionGatewayPipelineDefinition: [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. """ + connection_parameters: VariableOrOptional[ConnectionParameters] = None + """ + :meta private: [EXPERIMENTAL] + + Optional, Internal. Parameters required to establish an initial connection with the source. + """ + gateway_storage_name: VariableOrOptional[str] = None """ Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. - Delta Live Tables system will automatically create the storage location under the catalog and schema. + Spark Declarative Pipelines system will automatically create the storage location under the catalog and schema. """ @classmethod @@ -73,11 +84,18 @@ class IngestionGatewayPipelineDefinitionDict(TypedDict, total=False): [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. """ + connection_parameters: VariableOrOptional[ConnectionParametersParam] + """ + :meta private: [EXPERIMENTAL] + + Optional, Internal. Parameters required to establish an initial connection with the source. + """ + gateway_storage_name: VariableOrOptional[str] """ Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. - Delta Live Tables system will automatically create the storage location under the catalog and schema. + Spark Declarative Pipelines system will automatically create the storage location under the catalog and schema. """ diff --git a/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py b/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py index 6a5e058890..9276bc0419 100644 --- a/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py +++ b/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py @@ -30,6 +30,16 @@ class IngestionPipelineDefinition: Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on. """ + ingest_from_uc_foreign_catalog: VariableOrOptional[bool] = None + """ + :meta private: [EXPERIMENTAL] + + Immutable. If set to true, the pipeline will ingest tables from the + UC foreign catalogs directly without the need to specify a UC connection or ingestion gateway. + The `source_catalog` fields in objects of IngestionConfig are interpreted as + the UC foreign catalogs to ingest from. + """ + ingestion_gateway_id: VariableOrOptional[str] = None """ Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server. @@ -51,8 +61,6 @@ class IngestionPipelineDefinition: source_configurations: VariableOrList[SourceConfig] = field(default_factory=list) """ - :meta private: [EXPERIMENTAL] - Top-level source configurations """ @@ -77,6 +85,16 @@ class IngestionPipelineDefinitionDict(TypedDict, total=False): Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on. """ + ingest_from_uc_foreign_catalog: VariableOrOptional[bool] + """ + :meta private: [EXPERIMENTAL] + + Immutable. If set to true, the pipeline will ingest tables from the + UC foreign catalogs directly without the need to specify a UC connection or ingestion gateway. + The `source_catalog` fields in objects of IngestionConfig are interpreted as + the UC foreign catalogs to ingest from. + """ + ingestion_gateway_id: VariableOrOptional[str] """ Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server. @@ -98,8 +116,6 @@ class IngestionPipelineDefinitionDict(TypedDict, total=False): source_configurations: VariableOrList[SourceConfigParam] """ - :meta private: [EXPERIMENTAL] - Top-level source configurations """ diff --git a/python/databricks/bundles/pipelines/_models/postgres_catalog_config.py b/python/databricks/bundles/pipelines/_models/postgres_catalog_config.py index e7216f14a8..87092e7d7a 100644 --- a/python/databricks/bundles/pipelines/_models/postgres_catalog_config.py +++ b/python/databricks/bundles/pipelines/_models/postgres_catalog_config.py @@ -16,15 +16,11 @@ @dataclass(kw_only=True) class PostgresCatalogConfig: """ - :meta private: [EXPERIMENTAL] - PG-specific catalog-level configuration parameters """ slot_config: VariableOrOptional[PostgresSlotConfig] = None """ - :meta private: [EXPERIMENTAL] - Optional. The Postgres slot configuration to use for logical replication """ @@ -41,8 +37,6 @@ class PostgresCatalogConfigDict(TypedDict, total=False): slot_config: VariableOrOptional[PostgresSlotConfigParam] """ - :meta private: [EXPERIMENTAL] - Optional. The Postgres slot configuration to use for logical replication """ diff --git a/python/databricks/bundles/pipelines/_models/postgres_slot_config.py b/python/databricks/bundles/pipelines/_models/postgres_slot_config.py index 329e3a5dbc..3c392c2d33 100644 --- a/python/databricks/bundles/pipelines/_models/postgres_slot_config.py +++ b/python/databricks/bundles/pipelines/_models/postgres_slot_config.py @@ -12,22 +12,16 @@ @dataclass(kw_only=True) class PostgresSlotConfig: """ - :meta private: [EXPERIMENTAL] - PostgresSlotConfig contains the configuration for a Postgres logical replication slot """ publication_name: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - The name of the publication to use for the Postgres source """ slot_name: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - The name of the logical replication slot to use for the Postgres source """ @@ -44,15 +38,11 @@ class PostgresSlotConfigDict(TypedDict, total=False): publication_name: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - The name of the publication to use for the Postgres source """ slot_name: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - The name of the logical replication slot to use for the Postgres source """ diff --git a/python/databricks/bundles/pipelines/_models/source_catalog_config.py b/python/databricks/bundles/pipelines/_models/source_catalog_config.py index 008893d461..246079cee0 100644 --- a/python/databricks/bundles/pipelines/_models/source_catalog_config.py +++ b/python/databricks/bundles/pipelines/_models/source_catalog_config.py @@ -16,22 +16,16 @@ @dataclass(kw_only=True) class SourceCatalogConfig: """ - :meta private: [EXPERIMENTAL] - SourceCatalogConfig contains catalog-level custom configuration parameters for each source """ postgres: VariableOrOptional[PostgresCatalogConfig] = None """ - :meta private: [EXPERIMENTAL] - Postgres-specific catalog-level configuration parameters """ source_catalog: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - Source catalog name """ @@ -48,15 +42,11 @@ class SourceCatalogConfigDict(TypedDict, total=False): postgres: VariableOrOptional[PostgresCatalogConfigParam] """ - :meta private: [EXPERIMENTAL] - Postgres-specific catalog-level configuration parameters """ source_catalog: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - Source catalog name """ diff --git a/python/databricks/bundles/pipelines/_models/source_config.py b/python/databricks/bundles/pipelines/_models/source_config.py index b404c90f0b..876fb460a7 100644 --- a/python/databricks/bundles/pipelines/_models/source_config.py +++ b/python/databricks/bundles/pipelines/_models/source_config.py @@ -15,14 +15,10 @@ @dataclass(kw_only=True) class SourceConfig: - """ - :meta private: [EXPERIMENTAL] - """ + """""" catalog: VariableOrOptional[SourceCatalogConfig] = None """ - :meta private: [EXPERIMENTAL] - Catalog-level source configuration parameters """ @@ -39,8 +35,6 @@ class SourceConfigDict(TypedDict, total=False): catalog: VariableOrOptional[SourceCatalogConfigParam] """ - :meta private: [EXPERIMENTAL] - Catalog-level source configuration parameters """ diff --git a/python/databricks/bundles/pipelines/_models/table_specific_config.py b/python/databricks/bundles/pipelines/_models/table_specific_config.py index 7211a9a951..0b7c6b1a69 100644 --- a/python/databricks/bundles/pipelines/_models/table_specific_config.py +++ b/python/databricks/bundles/pipelines/_models/table_specific_config.py @@ -72,7 +72,7 @@ class TableSpecificConfig: sequence_by: VariableOrList[str] = field(default_factory=list) """ - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. """ workday_report_parameters: VariableOrOptional[ @@ -142,7 +142,7 @@ class TableSpecificConfigDict(TypedDict, total=False): sequence_by: VariableOrList[str] """ - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. """ workday_report_parameters: VariableOrOptional[