diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/databricks.yml b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/databricks.yml new file mode 100644 index 0000000000..66622ab0bc --- /dev/null +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/databricks.yml @@ -0,0 +1,4 @@ +resources: + pipelines: + abc: + root_path: "foo/bar/doesnotexist" diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/output.txt b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/output.txt new file mode 100644 index 0000000000..043526e669 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle validate +Error: stat foo/bar/doesnotexist: no such file or directory + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/script b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/script new file mode 100644 index 0000000000..f52b452ee6 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/script @@ -0,0 +1 @@ +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/test.toml b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/test.toml new file mode 100644 index 0000000000..dfdc3476a1 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/test.toml @@ -0,0 +1,4 @@ +# Replace error message from windows +[[Repls]] +Old = "CreateFile foo/bar/doesnotexist: The system cannot find the path specified." +New = "stat foo/bar/doesnotexist: no such file or directory" diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/databricks.yml b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/databricks.yml new file mode 100644 index 0000000000..1f0a6bdc5e --- /dev/null +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: pipelines_relative_path_translation + +include: + - resources/*.yml diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/output.txt b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/output.txt new file mode 100644 index 0000000000..55a7d4cfa7 --- /dev/null +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle validate -o json +{ + "one_libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipelines_relative_path_translation/default/files/src/pipeline_one/a/b/c/**" + } + } + ], + "two_libraries": [ + { + "glob": { + "include": "/Workspace/Users/me@company.com/a/b/c/**" + } + } + ], + "one_root_path": "/Workspace/Users/[USERNAME]/.bundle/pipelines_relative_path_translation/default/files/src/pipeline_one", + "two_root_path": "/Workspace/Users/me@company.com/src" +} diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/resources/pipeline.yml b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/resources/pipeline.yml new file mode 100644 index 0000000000..13b47fd6bc --- /dev/null +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/resources/pipeline.yml @@ -0,0 +1,15 @@ +resources: + pipelines: + # relative paths should get translated to remote paths + one: + root_path: "../src/pipeline_one" + libraries: + - glob: + include: "../src/pipeline_one/a/b/c/**" + + # absolute paths should remain as-is + two: + root_path: "/Workspace/Users/me@company.com/src" + libraries: + - glob: + include: "/Workspace/Users/me@company.com/a/b/c/**" diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/script b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/script new file mode 100644 index 0000000000..92685bf297 --- /dev/null +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '{one_libraries: .resources.pipelines.one.libraries, two_libraries: .resources.pipelines.two.libraries, one_root_path: .resources.pipelines.one.root_path, two_root_path: .resources.pipelines.two.root_path}' diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/src/pipeline_one/.gitkeep b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/src/pipeline_one/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/a/b/c/databricks.yml b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/a/b/c/databricks.yml new file mode 100644 index 0000000000..c716045065 --- /dev/null +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/a/b/c/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: pipelines_root_path_outside_sync_root + +resources: + pipelines: + abc: + root_path: ../../../src diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/output.txt b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/output.txt new file mode 100644 index 0000000000..86caa8f7f9 --- /dev/null +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle validate +Error: path [TEST_TMP_DIR]/src is not contained in sync root path + +Name: pipelines_root_path_outside_sync_root +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipelines_root_path_outside_sync_root/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/script b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/script new file mode 100644 index 0000000000..5f9c79b27b --- /dev/null +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/script @@ -0,0 +1,3 @@ +cd a/b/c + +trace $CLI bundle validate diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/src/.gitkeep b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/src/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/test.toml b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/test.toml new file mode 100644 index 0000000000..a298217f21 --- /dev/null +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/selftest/basic/output.txt b/acceptance/selftest/basic/output.txt index 9ca32232b9..9c8ec242ac 100644 --- a/acceptance/selftest/basic/output.txt +++ b/acceptance/selftest/basic/output.txt @@ -29,7 +29,7 @@ Exit code: 7 === Custom output files - everything starting with out is captured and compared >>> echo HELLO -=== Custom regex can be specified in [[Repl]] section +=== Custom regex can be specified in [[Repls]] section 1234 CUSTOM_NUMBER_REGEX 123456 diff --git a/acceptance/selftest/basic/script b/acceptance/selftest/basic/script index a3ec984026..1ccb63815a 100644 --- a/acceptance/selftest/basic/script +++ b/acceptance/selftest/basic/script @@ -20,7 +20,7 @@ trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())' printf "\n=== Custom output files - everything starting with out is captured and compared" trace echo HELLO > out.hello.txt -printf "\n=== Custom regex can be specified in [[Repl]] section\n" +printf "\n=== Custom regex can be specified in [[Repls]] section\n" echo 1234 echo 12345 echo 123456 diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index fa1ded827a..b40f7027ef 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -15,20 +15,26 @@ func pipelineRewritePatterns() []pipelineRewritePattern { dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), - dyn.Key("libraries"), - dyn.AnyIndex(), ) // Compile list of configuration paths to rewrite. return []pipelineRewritePattern{ { - pattern: base.Append(dyn.Key("notebook"), dyn.Key("path")), + pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("notebook"), dyn.Key("path")), mode: TranslateModeNotebook, }, { - pattern: base.Append(dyn.Key("file"), dyn.Key("path")), + pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("file"), dyn.Key("path")), mode: TranslateModeFile, }, + { + pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + }, + { + pattern: base.Append(dyn.Key("root_path")), + mode: TranslateModeDirectory, + }, } } diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index 2a848a9f8c..a73ee1767a 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -16,6 +16,7 @@ func TestVisitPipelinePaths(t *testing.T) { Pipelines: map[string]*resources.Pipeline{ "pipeline0": { CreatePipeline: pipelines.CreatePipeline{ + RootPath: "src", Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -27,6 +28,11 @@ func TestVisitPipelinePaths(t *testing.T) { Path: "src/foo.py", }, }, + { + Glob: &pipelines.PathPattern{ + Include: "a/b/c/**", + }, + }, }, }, }, @@ -38,6 +44,8 @@ func TestVisitPipelinePaths(t *testing.T) { expected := []dyn.Path{ dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[0].file.path"), dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[1].notebook.path"), + dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[2].glob.include"), + dyn.MustPathFromString("resources.pipelines.pipeline0.root_path"), } assert.ElementsMatch(t, expected, actual) diff --git a/bundle/config/mutator/paths/translation_mode.go b/bundle/config/mutator/paths/translation_mode.go index 8f3f76dbcd..7382f5a80c 100644 --- a/bundle/config/mutator/paths/translation_mode.go +++ b/bundle/config/mutator/paths/translation_mode.go @@ -13,6 +13,10 @@ const ( // TranslateModeDirectory translates a path to a remote directory. TranslateModeDirectory + // TranslateModeGlob translates a relative glob pattern to a remote glob pattern. + // It does not perform any checks on the glob pattern itself. + TranslateModeGlob + // TranslateModeLocalAbsoluteDirectory translates a path to the local absolute directory path. // It returns an error if the path does not exist or is not a directory. TranslateModeLocalAbsoluteDirectory diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 1f772910c2..ad707636df 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -141,6 +141,8 @@ func (t *translateContext) rewritePath( interp, err = t.translateFilePath(ctx, input, localPath, localRelPath) case paths.TranslateModeDirectory: interp, err = t.translateDirectoryPath(ctx, input, localPath, localRelPath) + case paths.TranslateModeGlob: + interp, err = t.translateGlobPath(ctx, input, localPath, localRelPath) case paths.TranslateModeLocalAbsoluteDirectory: interp, err = t.translateLocalAbsoluteDirectoryPath(ctx, input, localPath, localRelPath) case paths.TranslateModeLocalRelative: @@ -230,6 +232,10 @@ func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, return path.Join(t.remoteRoot, localRelPath), nil } +func (t *translateContext) translateGlobPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { + return path.Join(t.remoteRoot, localRelPath), nil +} + func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) { info, err := os.Stat(filepath.FromSlash(localFullPath)) if errors.Is(err, fs.ErrNotExist) {