From 6d06c098ec20c4de2cd01f68f99023ca4b677d71 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 19 Mar 2025 14:20:02 +0100 Subject: [PATCH 1/8] [Python] Run acceptance tests against the latest snapshot --- acceptance/acceptance_test.go | 37 ++++++++++++++++++- .../bundle/python/mutator-ordering/output.txt | 2 +- .../bundle/python/mutator-ordering/script | 4 +- .../bundle/python/resolve-variable/output.txt | 2 +- .../bundle/python/resolve-variable/script | 4 +- .../bundle/python/resource-loading/output.txt | 2 +- .../bundle/python/resource-loading/script | 4 +- acceptance/bundle/python/test.toml | 7 ++++ experimental/python/README.md | 0 experimental/python/pyproject.toml | 5 ++- 10 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 experimental/python/README.md diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index c16e597ffa..66340eab0d 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -101,6 +101,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // Download terraform and provider and create config; this also creates build directory. RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".") + wheelPath, err := buildDatabricksBundlesWheel(t, buildDir) + require.NoError(t, err) + t.Setenv("DATABRICKS_BUNDLES_WHEEL", wheelPath) + repls.SetPath(wheelPath, "[DATABRICKS_BUNDLES_WHEEL]") + coverDir := os.Getenv("CLI_GOCOVERDIR") if coverDir != "" { @@ -453,7 +458,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont } for _, keyvalue := range customEnv { - items := strings.Split(keyvalue, "=") + items := strings.SplitN(keyvalue, "=", 2) require.Len(t, items, 2) cmd.Env = append(cmd.Env, keyvalue) repls.Set(items[1], "["+items[0]+"]") @@ -878,3 +883,33 @@ func getNodeTypeID(cloudEnv string) string { return "unknown-cloudEnv-" + cloudEnv } } + +// buildDatabricksBundlesWheel builds the databricks-bundles wheel and returns the path to the wheel. +// It's used to cache the wheel build between acceptance tests, because one build takes ~10 seconds. +func buildDatabricksBundlesWheel(t *testing.T, buildDir string) (string, error) { + RunCommand(t, []string{"uv", "build", "--wheel", "--out-dir", buildDir}, "../experimental/python") + + files, err := os.ReadDir(buildDir) + if err != nil { + return "", fmt.Errorf("failed to read directory %s: %s", buildDir, err) + } + + // we can't control output file name, so we have to search for it + + var wheelName string + for _, file := range files { + if strings.HasPrefix(file.Name(), "databricks_bundles-") && strings.HasSuffix(file.Name(), ".whl") { + if wheelName != "" { + return "", fmt.Errorf("multiple wheels found: %s and %s", wheelName, file.Name()) + } else { + wheelName = file.Name() + } + } + } + + if wheelName != "" { + return filepath.Join(buildDir, wheelName), nil + } else { + return "", fmt.Errorf("databricks-bundles wheel not found in %s", buildDir) + } +} diff --git a/acceptance/bundle/python/mutator-ordering/output.txt b/acceptance/bundle/python/mutator-ordering/output.txt index d7a11f30f1..9a4c8c5793 100644 --- a/acceptance/bundle/python/mutator-ordering/output.txt +++ b/acceptance/bundle/python/mutator-ordering/output.txt @@ -1,5 +1,5 @@ ->>> uv run --python 3.10 --with databricks-bundles==0.7.2 -q [CLI] bundle validate --output json +>>> uv run --python 3.10 [UV_ARGS] -q [CLI] bundle validate --output json { "experimental": { "python": { diff --git a/acceptance/bundle/python/mutator-ordering/script b/acceptance/bundle/python/mutator-ordering/script index c17738eac0..e6a186328e 100644 --- a/acceptance/bundle/python/mutator-ordering/script +++ b/acceptance/bundle/python/mutator-ordering/script @@ -1,4 +1,6 @@ -trace uv run --python 3.10 --with databricks-bundles==0.7.2 -q $CLI bundle validate --output json | \ +UV_ARGS=$(sed "s|\[DATABRICKS_BUNDLES_WHEEL\]|$DATABRICKS_BUNDLES_WHEEL|" <<< "$UV_ARGS") + +trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \ jq "pick(.experimental.python, .resources)" rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/resolve-variable/output.txt b/acceptance/bundle/python/resolve-variable/output.txt index 5ebf4fc113..79cdc2371c 100644 --- a/acceptance/bundle/python/resolve-variable/output.txt +++ b/acceptance/bundle/python/resolve-variable/output.txt @@ -1,5 +1,5 @@ ->>> uv run --python 3.10 --with databricks-bundles==0.7.2 -q [CLI] bundle validate --output json +>>> uv run --python 3.10 [UV_ARGS] -q [CLI] bundle validate --output json { "experimental": { "python": { diff --git a/acceptance/bundle/python/resolve-variable/script b/acceptance/bundle/python/resolve-variable/script index c17738eac0..e6a186328e 100644 --- a/acceptance/bundle/python/resolve-variable/script +++ b/acceptance/bundle/python/resolve-variable/script @@ -1,4 +1,6 @@ -trace uv run --python 3.10 --with databricks-bundles==0.7.2 -q $CLI bundle validate --output json | \ +UV_ARGS=$(sed "s|\[DATABRICKS_BUNDLES_WHEEL\]|$DATABRICKS_BUNDLES_WHEEL|" <<< "$UV_ARGS") + +trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \ jq "pick(.experimental.python, .resources)" rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/resource-loading/output.txt b/acceptance/bundle/python/resource-loading/output.txt index 88b99e4d30..16acb421f9 100644 --- a/acceptance/bundle/python/resource-loading/output.txt +++ b/acceptance/bundle/python/resource-loading/output.txt @@ -1,5 +1,5 @@ ->>> uv run --python 3.10 --with databricks-bundles==0.7.2 -q [CLI] bundle validate --output json +>>> uv run --python 3.10 [UV_ARGS] -q [CLI] bundle validate --output json { "experimental": { "python": { diff --git a/acceptance/bundle/python/resource-loading/script b/acceptance/bundle/python/resource-loading/script index c17738eac0..e6a186328e 100644 --- a/acceptance/bundle/python/resource-loading/script +++ b/acceptance/bundle/python/resource-loading/script @@ -1,4 +1,6 @@ -trace uv run --python 3.10 --with databricks-bundles==0.7.2 -q $CLI bundle validate --output json | \ +UV_ARGS=$(sed "s|\[DATABRICKS_BUNDLES_WHEEL\]|$DATABRICKS_BUNDLES_WHEEL|" <<< "$UV_ARGS") + +trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \ jq "pick(.experimental.python, .resources)" rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/test.toml b/acceptance/bundle/python/test.toml index 95d0b9d9f9..5d95df0fe4 100644 --- a/acceptance/bundle/python/test.toml +++ b/acceptance/bundle/python/test.toml @@ -1,2 +1,9 @@ Local = true Cloud = false # tests don't interact with APIs + +[EnvMatrix] +UV_ARGS = [ + "--with databricks-bundles==0.7.3", + # NB: test runner doesn't support substitutions, they are expanded in the script + "--with [DATABRICKS_BUNDLES_WHEEL]", +] diff --git a/experimental/python/README.md b/experimental/python/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/python/pyproject.toml b/experimental/python/pyproject.toml index e6f60ffab5..aab0472276 100644 --- a/experimental/python/pyproject.toml +++ b/experimental/python/pyproject.toml @@ -24,8 +24,9 @@ dev-dependencies = [ "typing_extensions==4.12.2", ] -[tool.setuptools] -py-modules = ["databricks"] +[tool.setuptools.packages.find] +where = ["."] +include = ["databricks.bundles*"] [tool.setuptools.package-data] "databricks.bundles" = ["py.typed"] From 13446e93dde0550c323fd70d478a23caa174e841 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 19 Mar 2025 15:18:27 +0100 Subject: [PATCH 2/8] Fix windows --- acceptance/bundle/python/mutator-ordering/script | 2 +- acceptance/bundle/python/resolve-variable/script | 2 +- acceptance/bundle/python/resource-loading/script | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/python/mutator-ordering/script b/acceptance/bundle/python/mutator-ordering/script index e6a186328e..795d18a56e 100644 --- a/acceptance/bundle/python/mutator-ordering/script +++ b/acceptance/bundle/python/mutator-ordering/script @@ -1,4 +1,4 @@ -UV_ARGS=$(sed "s|\[DATABRICKS_BUNDLES_WHEEL\]|$DATABRICKS_BUNDLES_WHEEL|" <<< "$UV_ARGS") +UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}" trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \ jq "pick(.experimental.python, .resources)" diff --git a/acceptance/bundle/python/resolve-variable/script b/acceptance/bundle/python/resolve-variable/script index e6a186328e..795d18a56e 100644 --- a/acceptance/bundle/python/resolve-variable/script +++ b/acceptance/bundle/python/resolve-variable/script @@ -1,4 +1,4 @@ -UV_ARGS=$(sed "s|\[DATABRICKS_BUNDLES_WHEEL\]|$DATABRICKS_BUNDLES_WHEEL|" <<< "$UV_ARGS") +UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}" trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \ jq "pick(.experimental.python, .resources)" diff --git a/acceptance/bundle/python/resource-loading/script b/acceptance/bundle/python/resource-loading/script index e6a186328e..795d18a56e 100644 --- a/acceptance/bundle/python/resource-loading/script +++ b/acceptance/bundle/python/resource-loading/script @@ -1,4 +1,4 @@ -UV_ARGS=$(sed "s|\[DATABRICKS_BUNDLES_WHEEL\]|$DATABRICKS_BUNDLES_WHEEL|" <<< "$UV_ARGS") +UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}" trace uv run --python 3.10 $UV_ARGS -q $CLI bundle validate --output json | \ jq "pick(.experimental.python, .resources)" From eab80f513f9aa78ffb7aea0601ae1abac326d85c Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 21 Mar 2025 10:11:35 +0100 Subject: [PATCH 3/8] Switch to flit --- experimental/python/pyproject.toml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/experimental/python/pyproject.toml b/experimental/python/pyproject.toml index aab0472276..4b3b07e8b4 100644 --- a/experimental/python/pyproject.toml +++ b/experimental/python/pyproject.toml @@ -14,6 +14,10 @@ license = { name = "Databricks License", file = "LICENSE" } dependencies = [ ] +[build-system] +requires = ["flit_core >=3.11,<4"] +build-backend = "flit_core.buildapi" + [tool.uv] dev-dependencies = [ "pyright==1.1.380", @@ -24,12 +28,9 @@ dev-dependencies = [ "typing_extensions==4.12.2", ] -[tool.setuptools.packages.find] -where = ["."] -include = ["databricks.bundles*"] - -[tool.setuptools.package-data] -"databricks.bundles" = ["py.typed"] +[tool.flit.module] +name = "databricks" +packages = ["databricks"] [tool.pytest.ini_options] addopts = "-ra -q -vv" From 7073409fec45960fcbce1009d93112aa0fcd1bf0 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 21 Mar 2025 11:19:21 +0100 Subject: [PATCH 4/8] Update uv.lock --- experimental/python/uv.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/experimental/python/uv.lock b/experimental/python/uv.lock index 99d168534c..bf595fe78a 100644 --- a/experimental/python/uv.lock +++ b/experimental/python/uv.lock @@ -1,4 +1,5 @@ version = 1 +revision = 1 requires-python = ">=3.10" [[package]] @@ -166,7 +167,7 @@ toml = [ [[package]] name = "databricks-bundles" version = "0.243.0.dev0" -source = { virtual = "." } +source = { editable = "." } [package.dev-dependencies] dev = [ From 0787bf42e3284f532b54e7cbdba0e237a7cb9d54 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 25 Mar 2025 09:56:14 +0100 Subject: [PATCH 5/8] Add -q to "uv build" --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 66340eab0d..29c2cce1c0 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -887,7 +887,7 @@ func getNodeTypeID(cloudEnv string) string { // buildDatabricksBundlesWheel builds the databricks-bundles wheel and returns the path to the wheel. // It's used to cache the wheel build between acceptance tests, because one build takes ~10 seconds. func buildDatabricksBundlesWheel(t *testing.T, buildDir string) (string, error) { - RunCommand(t, []string{"uv", "build", "--wheel", "--out-dir", buildDir}, "../experimental/python") + RunCommand(t, []string{"uv", "build", "-q", "--wheel", "--out-dir", buildDir}, "../experimental/python") files, err := os.ReadDir(buildDir) if err != nil { From 63caf9039eee996e0aad72719a05e962fb86ee47 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 25 Mar 2025 10:07:12 +0100 Subject: [PATCH 6/8] Revert "Add -q to "uv build"" This reverts commit f68a8ab3a8b84c6ed1b4ad2a31986002dd26571a. --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 29c2cce1c0..66340eab0d 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -887,7 +887,7 @@ func getNodeTypeID(cloudEnv string) string { // buildDatabricksBundlesWheel builds the databricks-bundles wheel and returns the path to the wheel. // It's used to cache the wheel build between acceptance tests, because one build takes ~10 seconds. func buildDatabricksBundlesWheel(t *testing.T, buildDir string) (string, error) { - RunCommand(t, []string{"uv", "build", "-q", "--wheel", "--out-dir", buildDir}, "../experimental/python") + RunCommand(t, []string{"uv", "build", "--wheel", "--out-dir", buildDir}, "../experimental/python") files, err := os.ReadDir(buildDir) if err != nil { From bde68e7fda61029e9621deef0752fee69966f99c Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 25 Mar 2025 10:08:14 +0100 Subject: [PATCH 7/8] Reapply "Add -q to "uv build"" This reverts commit da7b80f8affc5f8505bc4d10816f9aa27e107bba. --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 66340eab0d..29c2cce1c0 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -887,7 +887,7 @@ func getNodeTypeID(cloudEnv string) string { // buildDatabricksBundlesWheel builds the databricks-bundles wheel and returns the path to the wheel. // It's used to cache the wheel build between acceptance tests, because one build takes ~10 seconds. func buildDatabricksBundlesWheel(t *testing.T, buildDir string) (string, error) { - RunCommand(t, []string{"uv", "build", "--wheel", "--out-dir", buildDir}, "../experimental/python") + RunCommand(t, []string{"uv", "build", "-q", "--wheel", "--out-dir", buildDir}, "../experimental/python") files, err := os.ReadDir(buildDir) if err != nil { From fade5f451b589d9a115cef9dffb5daf03bfae7c3 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 25 Mar 2025 10:14:59 +0100 Subject: [PATCH 8/8] Remove symlink for LICENSE --- experimental/python/LICENSE | 70 +++++++++++++++++++++++++++++- experimental/python/pyproject.toml | 2 +- 2 files changed, 70 insertions(+), 2 deletions(-) mode change 120000 => 100644 experimental/python/LICENSE diff --git a/experimental/python/LICENSE b/experimental/python/LICENSE deleted file mode 120000 index 30cff7403d..0000000000 --- a/experimental/python/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE \ No newline at end of file diff --git a/experimental/python/LICENSE b/experimental/python/LICENSE new file mode 100644 index 0000000000..9878fb4747 --- /dev/null +++ b/experimental/python/LICENSE @@ -0,0 +1,69 @@ + Databricks License + Copyright (2022) Databricks, Inc. + + Definitions. + + Agreement: The agreement between Databricks, Inc., and you governing + the use of the Databricks Services, as that term is defined in + the Master Cloud Services Agreement (MCSA) located at + www.databricks.com/legal/mcsa. + + Licensed Materials: The source code, object code, data, and/or other + works to which this license applies. + + Scope of Use. You may not use the Licensed Materials except in + connection with your use of the Databricks Services pursuant to + the Agreement. Your use of the Licensed Materials must comply at all + times with any restrictions applicable to the Databricks Services, + generally, and must be used in accordance with any applicable + documentation. You may view, use, copy, modify, publish, and/or + distribute the Licensed Materials solely for the purposes of using + the Licensed Materials within or connecting to the Databricks Services. + If you do not agree to these terms, you may not view, use, copy, + modify, publish, and/or distribute the Licensed Materials. + + Redistribution. You may redistribute and sublicense the Licensed + Materials so long as all use is in compliance with these terms. + In addition: + + - You must give any other recipients a copy of this License; + - You must cause any modified files to carry prominent notices + stating that you changed the files; + - You must retain, in any derivative works that you distribute, + all copyright, patent, trademark, and attribution notices, + excluding those notices that do not pertain to any part of + the derivative works; and + - If a "NOTICE" text file is provided as part of its + distribution, then any derivative works that you distribute + must include a readable copy of the attribution notices + contained within such NOTICE file, excluding those notices + that do not pertain to any part of the derivative works. + + You may add your own copyright statement to your modifications and may + provide additional license terms and conditions for use, reproduction, + or distribution of your modifications, or for any such derivative works + as a whole, provided your use, reproduction, and distribution of + the Licensed Materials otherwise complies with the conditions stated + in this License. + + Termination. This license terminates automatically upon your breach of + these terms or upon the termination of your Agreement. Additionally, + Databricks may terminate this license at any time on notice. Upon + termination, you must permanently delete the Licensed Materials and + all copies thereof. + + DISCLAIMER; LIMITATION OF LIABILITY. + + THE LICENSED MATERIALS ARE PROVIDED “AS-IS” AND WITH ALL FAULTS. + DATABRICKS, ON BEHALF OF ITSELF AND ITS LICENSORS, SPECIFICALLY + DISCLAIMS ALL WARRANTIES RELATING TO THE LICENSED MATERIALS, EXPRESS + AND IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES, + CONDITIONS AND OTHER TERMS OF MERCHANTABILITY, SATISFACTORY QUALITY OR + FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. DATABRICKS AND + ITS LICENSORS TOTAL AGGREGATE LIABILITY RELATING TO OR ARISING OUT OF + YOUR USE OF OR DATABRICKS’ PROVISIONING OF THE LICENSED MATERIALS SHALL + BE LIMITED TO ONE THOUSAND ($1,000) DOLLARS. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS OR + THE USE OR OTHER DEALINGS IN THE LICENSED MATERIALS. diff --git a/experimental/python/pyproject.toml b/experimental/python/pyproject.toml index 4b3b07e8b4..16b6d4c2cb 100644 --- a/experimental/python/pyproject.toml +++ b/experimental/python/pyproject.toml @@ -9,7 +9,7 @@ authors = [ readme = "README.md" requires-python = ">=3.10" -license = { name = "Databricks License", file = "LICENSE" } +license = { file = "LICENSE" } dependencies = [ ]