[Python] Run acceptance tests against the latest snapshot

acceptance/acceptance_test.go

@@ -101,6 +101,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	// Download terraform and provider and create config; this also creates build directory.
 	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".")
 
+	wheelPath, err := buildDatabricksBundlesWheel(t, buildDir)
+	require.NoError(t, err)
+	t.Setenv("DATABRICKS_BUNDLES_WHEEL", wheelPath)
+	repls.SetPath(wheelPath, "[DATABRICKS_BUNDLES_WHEEL]")
+
 	coverDir := os.Getenv("CLI_GOCOVERDIR")
 
 	if coverDir != "" {
@@ -453,7 +458,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	}
 
 	for _, keyvalue := range customEnv {
-		items := strings.Split(keyvalue, "=")
+		items := strings.SplitN(keyvalue, "=", 2)
 		require.Len(t, items, 2)
 		cmd.Env = append(cmd.Env, keyvalue)
 		repls.Set(items[1], "["+items[0]+"]")
@@ -878,3 +883,33 @@ func getNodeTypeID(cloudEnv string) string {
 		return "unknown-cloudEnv-" + cloudEnv
 	}
 }
+
+// buildDatabricksBundlesWheel builds the databricks-bundles wheel and returns the path to the wheel.
+// It's used to cache the wheel build between acceptance tests, because one build takes ~10 seconds.
+func buildDatabricksBundlesWheel(t *testing.T, buildDir string) (string, error) {
+	RunCommand(t, []string{"uv", "build", "-q", "--wheel", "--out-dir", buildDir}, "../experimental/python")
+
+	files, err := os.ReadDir(buildDir)
+	if err != nil {
+		return "", fmt.Errorf("failed to read directory %s: %s", buildDir, err)
+	}
+
+	// we can't control output file name, so we have to search for it
+
+	var wheelName string
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "databricks_bundles-") && strings.HasSuffix(file.Name(), ".whl") {
+			if wheelName != "" {
+				return "", fmt.Errorf("multiple wheels found: %s and %s", wheelName, file.Name())
+			} else {
+				wheelName = file.Name()
+			}
+		}
+	}
+
+	if wheelName != "" {
+		return filepath.Join(buildDir, wheelName), nil
+	} else {
+		return "", fmt.Errorf("databricks-bundles wheel not found in %s", buildDir)
+	}
+}

acceptance/bundle/python/mutator-ordering/output.txt

@@ -1,5 +1,5 @@
 
->>> uv run --python 3.10 --with databricks-bundles==0.7.2 -q [CLI] bundle validate --output json
+>>> uv run --python 3.10 [UV_ARGS] -q [CLI] bundle validate --output json
 {
   "experimental": {
     "python": {

acceptance/bundle/python/mutator-ordering/script

@@ -1,4 +1,6 @@
-trace uv run --python 3.10 --with databricks-bundles==0.7.2 -q $CLI bundle validate --output json | \
+UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+
+trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \
   jq "pick(.experimental.python, .resources)"
 
 rm -fr .databricks __pycache__

acceptance/bundle/python/resolve-variable/output.txt

@@ -1,5 +1,5 @@
 
->>> uv run --python 3.10 --with databricks-bundles==0.7.2 -q [CLI] bundle validate --output json
+>>> uv run --python 3.10 [UV_ARGS] -q [CLI] bundle validate --output json
 {
   "experimental": {
     "python": {

acceptance/bundle/python/resolve-variable/script

@@ -1,4 +1,6 @@
-trace uv run --python 3.10 --with databricks-bundles==0.7.2 -q $CLI bundle validate --output json | \
+UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+
+trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \
   jq "pick(.experimental.python, .resources)"
 
 rm -fr .databricks __pycache__

acceptance/bundle/python/resource-loading/output.txt

@@ -1,5 +1,5 @@
 
->>> uv run --python 3.10 --with databricks-bundles==0.7.2 -q [CLI] bundle validate --output json
+>>> uv run --python 3.10 [UV_ARGS] -q [CLI] bundle validate --output json
 {
   "experimental": {
     "python": {

acceptance/bundle/python/resource-loading/script

@@ -1,4 +1,6 @@
-trace uv run --python 3.10 --with databricks-bundles==0.7.2 -q $CLI bundle validate --output json | \
+UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+
+trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \
   jq "pick(.experimental.python, .resources)"
 
 rm -fr .databricks __pycache__

acceptance/bundle/python/test.toml

@@ -1,2 +1,9 @@
 Local = true
 Cloud = false # tests don't interact with APIs
+
+[EnvMatrix]
+UV_ARGS = [
+  "--with databricks-bundles==0.7.3",
+  # NB: test runner doesn't support substitutions, they are expanded in the script
+  "--with [DATABRICKS_BUNDLES_WHEEL]",
+]

experimental/python/LICENSE

@@ -0,0 +1,69 @@
+                               Databricks License
+                        Copyright (2022) Databricks, Inc.
+
+    Definitions.
+
+    Agreement: The agreement between Databricks, Inc., and you governing
+    the use of the Databricks Services, as that term is defined in
+    the Master Cloud Services Agreement (MCSA) located at
+    www.databricks.com/legal/mcsa.
+
+    Licensed Materials: The source code, object code, data, and/or other
+    works to which this license applies.
+
+    Scope of Use. You may not use the Licensed Materials except in
+    connection with your use of the Databricks Services pursuant to
+    the Agreement. Your use of the Licensed Materials must comply at all
+    times with any restrictions applicable to the Databricks Services,
+    generally, and must be used in accordance with any applicable
+    documentation. You may view, use, copy, modify, publish, and/or
+    distribute the Licensed Materials solely for the purposes of using
+    the Licensed Materials within or connecting to the Databricks Services.
+    If you do not agree to these terms, you may not view, use, copy,
+    modify, publish, and/or distribute the Licensed Materials.
+
+    Redistribution. You may redistribute and sublicense the Licensed
+    Materials so long as all use is in compliance with these terms.
+    In addition:
+
+        -   You must give any other recipients a copy of this License;
+        -   You must cause any modified files to carry prominent notices
+            stating that you changed the files;
+        -   You must retain, in any derivative works that you distribute,
+            all copyright, patent, trademark, and attribution notices,
+            excluding those notices that do not pertain to any part of
+            the derivative works; and
+        -   If a "NOTICE" text file is provided as part of its
+            distribution, then any derivative works that you distribute
+            must include a readable copy of the attribution notices
+            contained within such NOTICE file, excluding those notices
+            that do not pertain to any part of the derivative works.
+
+    You may add your own copyright statement to your modifications and may
+    provide additional license terms and conditions for use, reproduction,
+    or distribution of your modifications, or for any such derivative works
+    as a whole, provided your use, reproduction, and distribution of
+    the Licensed Materials otherwise complies with the conditions stated
+    in this License.
+
+    Termination. This license terminates automatically upon your breach of
+    these terms or upon the termination of your Agreement. Additionally,
+    Databricks may terminate this license at any time on notice. Upon
+    termination, you must permanently delete the Licensed Materials and
+    all copies thereof.
+
+    DISCLAIMER; LIMITATION OF LIABILITY.
+
+    THE LICENSED MATERIALS ARE PROVIDED “AS-IS” AND WITH ALL FAULTS.
+    DATABRICKS, ON BEHALF OF ITSELF AND ITS LICENSORS, SPECIFICALLY
+    DISCLAIMS ALL WARRANTIES RELATING TO THE LICENSED MATERIALS, EXPRESS
+    AND IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES,
+    CONDITIONS AND OTHER TERMS OF MERCHANTABILITY, SATISFACTORY QUALITY OR
+    FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. DATABRICKS AND
+    ITS LICENSORS TOTAL AGGREGATE LIABILITY RELATING TO OR ARISING OUT OF
+    YOUR USE OF OR DATABRICKS’ PROVISIONING OF THE LICENSED MATERIALS SHALL
+    BE LIMITED TO ONE THOUSAND ($1,000) DOLLARS.  IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+    OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+    ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS OR
+    THE USE OR OTHER DEALINGS IN THE LICENSED MATERIALS.

experimental/python/pyproject.toml

@@ -9,11 +9,15 @@ authors = [
 
 readme = "README.md"
 requires-python = ">=3.10"
-license = { name = "Databricks License", file = "LICENSE" }
+license = { file = "LICENSE" }
 
 dependencies = [
 ]
 
+[build-system]
+requires = ["flit_core >=3.11,<4"]
+build-backend = "flit_core.buildapi"
+
 [tool.uv]
 dev-dependencies = [
   "pyright==1.1.380",
@@ -24,11 +28,9 @@ dev-dependencies = [
   "typing_extensions==4.12.2",
 ]
 
-[tool.setuptools]
-py-modules = ["databricks"]
-
-[tool.setuptools.package-data]
-"databricks.bundles" = ["py.typed"]
+[tool.flit.module]
+name = "databricks"
+packages = ["databricks"]
 
 [tool.pytest.ini_options]
 addopts = "-ra -q -vv"