Translate relative paths root_path and glob.include for pipelines

acceptance/bundle/paths/pipeline_root_path_doesnotexist/databricks.yml

@@ -0,0 +1,4 @@
+resources:
+  pipelines:
+    abc:
+      root_path: "foo/bar/doesnotexist"

acceptance/bundle/paths/pipeline_root_path_doesnotexist/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Error: stat foo/bar/doesnotexist: no such file or directory
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 1 error
+
+Exit code: 1

acceptance/bundle/paths/pipeline_root_path_doesnotexist/script

@@ -0,0 +1 @@
+errcode trace $CLI bundle validate

acceptance/bundle/paths/pipeline_root_path_doesnotexist/test.toml

@@ -0,0 +1,4 @@
+# Replace error message from windows
+[[Repls]]
+Old = "CreateFile foo/bar/doesnotexist: The system cannot find the path specified."
+New = "stat foo/bar/doesnotexist: no such file or directory"

acceptance/bundle/paths/pipelines_glob_include_and_root_path/databricks.yml

@@ -0,0 +1,5 @@
+bundle:
+  name: pipelines_relative_path_translation
+
+include:
+  - resources/*.yml

acceptance/bundle/paths/pipelines_glob_include_and_root_path/output.txt

@@ -0,0 +1,20 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "one_libraries": [
+    {
+      "glob": {
+        "include": "/Workspace/Users/[USERNAME]/.bundle/pipelines_relative_path_translation/default/files/src/pipeline_one/a/b/c/**"
+      }
+    }
+  ],
+  "two_libraries": [
+    {
+      "glob": {
+        "include": "/Workspace/Users/me@company.com/a/b/c/**"
+      }
+    }
+  ],
+  "one_root_path": "/Workspace/Users/[USERNAME]/.bundle/pipelines_relative_path_translation/default/files/src/pipeline_one",
+  "two_root_path": "/Workspace/Users/me@company.com/src"
+}

acceptance/bundle/paths/pipelines_glob_include_and_root_path/resources/pipeline.yml

@@ -0,0 +1,15 @@
+resources:
+  pipelines:
+    # relative paths should get translated to remote paths
+    one:
+      root_path: "../src/pipeline_one"
+      libraries:
+        - glob:
+            include: "../src/pipeline_one/a/b/c/**"
+
+    # absolute paths should remain as-is
+    two:
+      root_path: "/Workspace/Users/me@company.com/src"
+      libraries:
+        - glob:
+            include: "/Workspace/Users/me@company.com/a/b/c/**"

acceptance/bundle/paths/pipelines_glob_include_and_root_path/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq '{one_libraries: .resources.pipelines.one.libraries, two_libraries: .resources.pipelines.two.libraries, one_root_path: .resources.pipelines.one.root_path, two_root_path: .resources.pipelines.two.root_path}'

acceptance/bundle/paths/pipelines_root_path_outside_sync_root/a/b/c/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: pipelines_root_path_outside_sync_root
+
+resources:
+  pipelines:
+    abc:
+      root_path: ../../../src

acceptance/bundle/paths/pipelines_root_path_outside_sync_root/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Error: path [TEST_TMP_DIR]/src is not contained in sync root path
+
+Name: pipelines_root_path_outside_sync_root
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/pipelines_root_path_outside_sync_root/default
+
+Found 1 error
+
+Exit code: 1

acceptance/bundle/paths/pipelines_root_path_outside_sync_root/script

@@ -0,0 +1,3 @@
+cd a/b/c
+
+trace $CLI bundle validate

acceptance/bundle/paths/pipelines_root_path_outside_sync_root/test.toml

@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '\\'
+New = '/'

acceptance/selftest/basic/output.txt

@@ -29,7 +29,7 @@ Exit code: 7
 === Custom output files - everything starting with out is captured and compared
 >>> echo HELLO
 
-=== Custom regex can be specified in [[Repl]] section
+=== Custom regex can be specified in [[Repls]] section
 1234
 CUSTOM_NUMBER_REGEX
 123456

acceptance/selftest/basic/script

@@ -20,7 +20,7 @@ trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())'
 printf "\n=== Custom output files - everything starting with out is captured and compared"
 trace echo HELLO > out.hello.txt
 
-printf "\n=== Custom regex can be specified in [[Repl]] section\n"
+printf "\n=== Custom regex can be specified in [[Repls]] section\n"
 echo 1234
 echo 12345
 echo 123456

bundle/config/mutator/paths/pipeline_paths_visitor.go

@@ -15,20 +15,26 @@ func pipelineRewritePatterns() []pipelineRewritePattern {
 		dyn.Key("resources"),
 		dyn.Key("pipelines"),
 		dyn.AnyKey(),
-		dyn.Key("libraries"),
-		dyn.AnyIndex(),
 	)
 
 	// Compile list of configuration paths to rewrite.
 	return []pipelineRewritePattern{
 		{
-			pattern: base.Append(dyn.Key("notebook"), dyn.Key("path")),
+			pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("notebook"), dyn.Key("path")),
 			mode:    TranslateModeNotebook,
 		},
 		{
-			pattern: base.Append(dyn.Key("file"), dyn.Key("path")),
+			pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("file"), dyn.Key("path")),
 			mode:    TranslateModeFile,
 		},
+		{
+			pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")),
+			mode:    TranslateModeGlob,
+		},
+		{
+			pattern: base.Append(dyn.Key("root_path")),
+			mode:    TranslateModeDirectory,
+		},
 	}
 }
 

bundle/config/mutator/paths/pipeline_paths_visitor_test.go

@@ -16,6 +16,7 @@ func TestVisitPipelinePaths(t *testing.T) {
 			Pipelines: map[string]*resources.Pipeline{
 				"pipeline0": {
 					CreatePipeline: pipelines.CreatePipeline{
+						RootPath: "src",
 						Libraries: []pipelines.PipelineLibrary{
 							{
 								File: &pipelines.FileLibrary{
@@ -27,6 +28,11 @@ func TestVisitPipelinePaths(t *testing.T) {
 									Path: "src/foo.py",
 								},
 							},
+							{
+								Glob: &pipelines.PathPattern{
+									Include: "a/b/c/**",
+								},
+							},
 						},
 					},
 				},
@@ -38,6 +44,8 @@ func TestVisitPipelinePaths(t *testing.T) {
 	expected := []dyn.Path{
 		dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[0].file.path"),
 		dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[1].notebook.path"),
+		dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[2].glob.include"),
+		dyn.MustPathFromString("resources.pipelines.pipeline0.root_path"),
 	}
 
 	assert.ElementsMatch(t, expected, actual)

bundle/config/mutator/paths/translation_mode.go

@@ -13,6 +13,10 @@ const (
 	// TranslateModeDirectory translates a path to a remote directory.
 	TranslateModeDirectory
 
+	// TranslateModeGlob translates a relative glob pattern to a remote glob pattern.
+	// It does not perform any checks on the glob pattern itself.
+	TranslateModeGlob
+
 	// TranslateModeLocalAbsoluteDirectory translates a path to the local absolute directory path.
 	// It returns an error if the path does not exist or is not a directory.
 	TranslateModeLocalAbsoluteDirectory

bundle/config/mutator/translate_paths.go

@@ -141,6 +141,8 @@ func (t *translateContext) rewritePath(
 		interp, err = t.translateFilePath(ctx, input, localPath, localRelPath)
 	case paths.TranslateModeDirectory:
 		interp, err = t.translateDirectoryPath(ctx, input, localPath, localRelPath)
+	case paths.TranslateModeGlob:
+		interp, err = t.translateGlobPath(ctx, input, localPath, localRelPath)
 	case paths.TranslateModeLocalAbsoluteDirectory:
 		interp, err = t.translateLocalAbsoluteDirectoryPath(ctx, input, localPath, localRelPath)
 	case paths.TranslateModeLocalRelative:
@@ -230,6 +232,10 @@ func (t *translateContext) translateDirectoryPath(ctx context.Context, literal,
 	return path.Join(t.remoteRoot, localRelPath), nil
 }
 
+func (t *translateContext) translateGlobPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
+	return path.Join(t.remoteRoot, localRelPath), nil
+}
+
 func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) {
 	info, err := os.Stat(filepath.FromSlash(localFullPath))
 	if errors.Is(err, fs.ErrNotExist) {