[Python] Add pipelines to template (#2706)

## Changes
Add pipelines to `experimental-jobs-as-code` template

## Why
It makes template consistent with capabilities in `default-python`
template

## Tests
Acceptance tests

Author: kanterov

acceptance/bundle/templates/experimental-jobs-as-code/input.json

@@ -1,5 +1,6 @@
 {
   "project_name": "my_jobs_as_code",
   "include_notebook": "yes",
-  "include_python": "yes"
+  "include_python": "yes",
+  "include_dlt": "yes"
 }

acceptance/bundle/templates/experimental-jobs-as-code/output.txt

@@ -79,6 +79,29 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
         }
       }
     }
+  },
+  "pipelines": {
+    "my_jobs_as_code_pipeline": {
+      "catalog": "catalog_name",
+      "configuration": {
+        "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src"
+      },
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/state/metadata.json"
+      },
+      "development": true,
+      "libraries": [
+        {
+          "notebook": {
+            "path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/dlt_pipeline"
+          }
+        }
+      ],
+      "name": "[dev [USERNAME]] my_jobs_as_code_pipeline",
+      "permissions": [],
+      "target": "my_jobs_as_code_dev"
+    }
   }
 }
 

acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_pipeline.py

@@ -0,0 +1,20 @@
+from databricks.bundles.pipelines import Pipeline
+
+my_jobs_as_code_pipeline = Pipeline.from_dict(
+    {
+        "name": "my_jobs_as_code_pipeline",
+        "target": "my_jobs_as_code_${bundle.target}",
+        ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
+        "catalog": "catalog_name",
+        "libraries": [
+            {
+                "notebook": {
+                    "path": "src/dlt_pipeline.ipynb",
+                },
+            },
+        ],
+        "configuration": {
+            "bundle.sourcePath": "${workspace.file_path}/src",
+        },
+    }
+)

acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/dlt_pipeline.ipynb

@@ -0,0 +1,90 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# DLT pipeline\n",
+    "\n",
+    "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/my_jobs_as_code.pipeline.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Import DLT and src/my_jobs_as_code\n",
+    "import dlt\n",
+    "import sys\n",
+    "\n",
+    "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n",
+    "from pyspark.sql.functions import expr\n",
+    "from my_jobs_as_code import main"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "@dlt.view\n",
+    "def taxi_raw():\n",
+    "    return main.get_taxis(spark)\n",
+    "\n",
+    "\n",
+    "@dlt.table\n",
+    "def filtered_taxis():\n",
+    "    return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "dlt_pipeline",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

libs/template/templates/experimental-jobs-as-code/databricks_template_schema.json

@@ -16,12 +16,19 @@
             "description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'",
             "order": 2
         },
+        "include_dlt": {
+            "type": "string",
+            "default": "yes",
+            "enum": ["yes", "no"],
+            "description": "Include a stub (sample) Delta Live Tables pipeline in '{{.project_name}}{{path_separator}}src'",
+            "order": 3
+        },
         "include_python": {
             "type": "string",
             "default": "yes",
             "enum": ["yes", "no"],
             "description": "Include a stub (sample) Python package in '{{.project_name}}/src'",
-            "order": 3
+            "order": 4
         }
     },
     "success_message": "Workspace to use (auto-detected, edit in '{{.project_name}}/databricks.yml'): {{workspace_host}}\n\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html."

libs/template/templates/experimental-jobs-as-code/template/__preamble.tmpl

@@ -4,8 +4,7 @@ This file only template directives; it is skipped for the actual output.
 
 {{skip "__preamble"}}
 
-# TODO add DLT support, placeholder for now
-{{$notDLT := true }}
+{{$notDLT := not (eq .include_dlt "yes")}}
 {{$notNotebook := not (eq .include_notebook "yes")}}
 {{$notPython := not (eq .include_python "yes")}}