Skip to content

Commit f86cb03

Browse files
kanterovdenik
andauthored
Fix git source regression (#2699)
## Changes Fix git source regression. Jobs using the `git_source` feature shouldn't have their paths normalized because they are referring to files in git repositories, not local paths. There is still a problem with how libraries and other paths are translated, but this has been a problem for a long time and hasn't regressed recently. Fixes #2697 ## Tests Acceptance tests --------- Co-authored-by: Denis Bilenko <denis.bilenko@databricks.com>
1 parent 6e87c60 commit f86cb03

File tree

5 files changed

+84
-0
lines changed

5 files changed

+84
-0
lines changed
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
bundle:
2+
name: hello
3+
4+
include:
5+
- resources/included.yml
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"resources": {
3+
"jobs": {
4+
"my_job": {
5+
"deployment": {
6+
"kind": "BUNDLE",
7+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/hello/default/state/metadata.json"
8+
},
9+
"edit_mode": "UI_LOCKED",
10+
"format": "MULTI_TASK",
11+
"git_source": {
12+
"git_branch": "main",
13+
"git_provider": "gitHub",
14+
"git_url": "https://github.com/foo"
15+
},
16+
"name": "my_job",
17+
"permissions": [],
18+
"queue": {
19+
"enabled": true
20+
},
21+
"tags": {},
22+
"tasks": [
23+
{
24+
"libraries": [
25+
{
26+
"whl": "../mywheel.whl"
27+
}
28+
],
29+
"notebook_task": {
30+
"notebook_path": "path/in/repo/should/not/change",
31+
"source": "GIT"
32+
},
33+
"task_key": "my_task"
34+
}
35+
]
36+
}
37+
}
38+
}
39+
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
resources:
2+
jobs:
3+
my_job:
4+
name: my_job
5+
tasks:
6+
- task_key: my_task
7+
notebook_task:
8+
notebook_path: path/in/repo/should/not/change
9+
source: GIT
10+
libraries:
11+
# this path should become remote path, but it doesn't work
12+
- whl: ../mywheel.whl
13+
git_source:
14+
git_url: https://github.com/foo
15+
git_provider: gitHub
16+
git_branch: main
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
touch mywheel.whl
2+
$CLI bundle validate -o json | jq "pick(.resources)"
3+
rm mywheel.whl

bundle/config/mutator/normalize_paths.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,17 @@ func NormalizePaths() bundle.Mutator {
3737
}
3838

3939
func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
40+
// Do not normalize job task paths if using git source
41+
gitSourcePaths := collectGitSourcePaths(b)
42+
4043
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
4144
return paths.VisitPaths(v, func(path dyn.Path, kind paths.TranslateMode, v dyn.Value) (dyn.Value, error) {
45+
for _, gitSourcePrefix := range gitSourcePaths {
46+
if path.HasPrefix(gitSourcePrefix) {
47+
return v, nil
48+
}
49+
}
50+
4251
value, ok := v.AsString()
4352
if !ok {
4453
return dyn.InvalidValue, fmt.Errorf("value at %s is not a string", path.String())
@@ -59,6 +68,18 @@ func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnost
5968
return diag.FromErr(err)
6069
}
6170

71+
func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path {
72+
var jobs []dyn.Path
73+
74+
for name, job := range b.Config.Resources.Jobs {
75+
if job.GitSource != nil {
76+
jobs = append(jobs, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key(name)))
77+
}
78+
}
79+
80+
return jobs
81+
}
82+
6283
func normalizePath(path string, location dyn.Location, bundleRootPath string) (string, error) {
6384
pathAsUrl, err := url.Parse(path)
6485
if err != nil {

0 commit comments

Comments
 (0)