Skip to content

Commit f7de41f

Browse files
anton-107anton-107
authored
Add include/exclude flags to sync and bundle-sync commands (#2650)
## Why <!-- Why are these changes needed? Provide the context that the reviewer might be missing. For example, were there any decisions behind the change that are not reflected in the code itself? --> This allows users to specify file patterns to include / exclude from syncs using respective flags in sync and bundle sync commands: * `$ databricks sync <SRC> <DEST> --exclude "*.bak" --include "node_modules/mylib"` * `$ databricks bundle sync --exclude "*.bak" --include "node_modules/mylib"` [gitignore syntax](https://git-scm.com/docs/gitignore) for advanced pattern matching is supported. Note that in the current implementation excludes always takes higher priority over includes. The order in which the flags are provided in the prompt does not change this behavior. To overcome this behavior users can provide an `--include` flag with a negated pattern (prefixed by a `!`) instead of using an `--exclude` flag in the combinations of patterns. ## Tests <!-- How have you tested the changes? --> Manual testing + introduced new acceptance tests <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent f141221 commit f7de41f

File tree

9 files changed

+166
-2
lines changed

9 files changed

+166
-2
lines changed

‎NEXT_CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77
### Dependency updates
88

99
### CLI
10+
* Added include/exclude flags support to sync command ([#2650](https://github.com/databricks/cli/pull/2650))
1011

1112
### Bundles
1213
* Added support for model serving endpoints in deployment bind/unbind commands ([#2634](https://github.com/databricks/cli/pull/2634))
14+
* Added include/exclude flags support to bundle sync command ([#2650](https://github.com/databricks/cli/pull/2650))
1315

1416
### API Changes

‎acceptance/bundle/help/bundle-sync/output.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ Usage:
66
databricks bundle sync [flags]
77

88
Flags:
9+
--exclude strings patterns to exclude from sync (can be specified multiple times)
910
--full perform full synchronization (default is incremental)
1011
-h, --help help for sync
12+
--include strings patterns to include in sync (can be specified multiple times)
1113
--interval duration file system polling interval (for --watch) (default 1s)
1214
--output type type of the output format
1315
--watch watch local file system for changes

‎acceptance/bundle/sync/databricks.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
bundle:
2+
name: bundle-sync-test
3+
4+
resources:
5+
dashboards:
6+
dashboard1:
7+
display_name: My dashboard

‎acceptance/bundle/sync/output.txt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
2+
>>> [CLI] bundle sync --output text
3+
Initial Sync Complete
4+
Uploaded .gitignore
5+
Uploaded databricks.yml
6+
Uploaded project-folder
7+
Uploaded project-folder/app.py
8+
Uploaded project-folder/app.yaml
9+
Uploaded project-folder/query.sql
10+
11+
>>> [CLI] bundle sync --exclude project-folder/app.* --output text
12+
Deleted project-folder/app.py
13+
Deleted project-folder/app.yaml
14+
Initial Sync Complete
15+
16+
>>> [CLI] bundle sync --exclude project-folder/app.* --exclude project-folder/query.sql --output text
17+
Deleted project-folder
18+
Deleted project-folder/query.sql
19+
Initial Sync Complete
20+
21+
>>> [CLI] bundle sync --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/*.py --output text
22+
Initial Sync Complete
23+
Uploaded ignored-folder
24+
Uploaded ignored-folder/script.py
25+
26+
>>> [CLI] bundle sync --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/**/*.py --output text
27+
Initial Sync Complete
28+
Uploaded ignored-folder/folder1
29+
Uploaded ignored-folder/folder1/script.py
30+
31+
>>> [CLI] bundle sync --output text
32+
Deleted ignored-folder
33+
Deleted ignored-folder/folder1
34+
Deleted ignored-folder/folder1/script.py
35+
Deleted ignored-folder/script.py
36+
Initial Sync Complete
37+
Uploaded project-folder
38+
Uploaded project-folder/app.py
39+
Uploaded project-folder/app.yaml
40+
Uploaded project-folder/query.sql

‎acceptance/bundle/sync/script‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
mkdir "project-folder" "ignored-folder" "ignored-folder/folder1"
2+
touch "project-folder/app.yaml" "project-folder/app.py" "project-folder/query.sql"
3+
touch "ignored-folder/script.py" "ignored-folder/folder1/script.py"
4+
cat > .gitignore << EOF
5+
ignored-folder/
6+
script
7+
output.txt
8+
repls.json
9+
EOF
10+
11+
cleanup() {
12+
rm -rf project-folder ignored-folder .git
13+
}
14+
trap cleanup EXIT
15+
16+
# Note: output line starting with "Action: " lists files in non-deterministic order so we filter it out
17+
trace $CLI bundle sync --output text | grep -v "^Action: " | sort
18+
trace $CLI bundle sync --exclude 'project-folder/app.*' --output text | grep -v "^Action: " | sort
19+
trace $CLI bundle sync --exclude 'project-folder/app.*' --exclude 'project-folder/query.sql' --output text | grep -v "^Action: " | sort
20+
trace $CLI bundle sync --exclude 'project-folder/app.*' --exclude 'project-folder/query.sql' --include 'ignored-folder/*.py' --output text | grep -v "^Action: " | sort
21+
trace $CLI bundle sync --exclude 'project-folder/app.*' --exclude 'project-folder/query.sql' --include 'ignored-folder/**/*.py' --output text | grep -v "^Action: " | sort
22+
trace $CLI bundle sync --output text | grep -v "^Action: " | sort

‎acceptance/cmd/sync/output.txt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
2+
>>> [CLI] sync . /Users/[USERNAME]
3+
Initial Sync Complete
4+
Uploaded .gitignore
5+
Uploaded project-folder
6+
Uploaded project-folder/app.py
7+
Uploaded project-folder/app.yaml
8+
Uploaded project-folder/query.sql
9+
10+
>>> [CLI] sync . /Users/[USERNAME] --exclude project-folder/app.*
11+
Deleted project-folder/app.py
12+
Deleted project-folder/app.yaml
13+
Initial Sync Complete
14+
15+
>>> [CLI] sync . /Users/[USERNAME] --exclude project-folder/app.* --exclude project-folder/query.sql
16+
Deleted project-folder
17+
Deleted project-folder/query.sql
18+
Initial Sync Complete
19+
20+
>>> [CLI] sync . /Users/[USERNAME] --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/*.py
21+
Initial Sync Complete
22+
Uploaded ignored-folder
23+
Uploaded ignored-folder/script.py
24+
25+
>>> [CLI] sync . /Users/[USERNAME] --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/**/*.py
26+
Initial Sync Complete
27+
Uploaded ignored-folder/folder1
28+
Uploaded ignored-folder/folder1/script.py
29+
30+
>>> [CLI] sync . /Users/[USERNAME] --include ignored-folder/** --include !ignored-folder/folder1/big-blob
31+
Initial Sync Complete
32+
Uploaded ignored-folder/folder1/script.yaml
33+
Uploaded project-folder
34+
Uploaded project-folder/app.py
35+
Uploaded project-folder/app.yaml
36+
Uploaded project-folder/query.sql
37+
38+
>>> [CLI] sync . /Users/[USERNAME]
39+
Deleted ignored-folder
40+
Deleted ignored-folder/folder1
41+
Deleted ignored-folder/folder1/script.py
42+
Deleted ignored-folder/folder1/script.yaml
43+
Deleted ignored-folder/script.py
44+
Initial Sync Complete

‎acceptance/cmd/sync/script‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
mkdir "project-folder" "ignored-folder" "ignored-folder/folder1" ".git"
2+
touch "project-folder/app.yaml" "project-folder/app.py" "project-folder/query.sql"
3+
touch "ignored-folder/script.py" "ignored-folder/folder1/script.py" "ignored-folder/folder1/script.yaml" "ignored-folder/folder1/big-blob"
4+
cat > .gitignore << EOF
5+
ignored-folder/
6+
script
7+
output.txt
8+
repls.json
9+
EOF
10+
11+
cleanup() {
12+
rm -rf project-folder ignored-folder .git
13+
}
14+
trap cleanup EXIT
15+
16+
# Note: output line starting with "Action: " lists files in non-deterministic order so we filter it out
17+
trace $CLI sync . /Users/$CURRENT_USER_NAME | grep -v "^Action: " | sort
18+
19+
# excluding by mask:
20+
trace $CLI sync . /Users/$CURRENT_USER_NAME --exclude 'project-folder/app.*' | grep -v "^Action: " | sort
21+
22+
# combining excludes:
23+
trace $CLI sync . /Users/$CURRENT_USER_NAME --exclude 'project-folder/app.*' --exclude 'project-folder/query.sql' | grep -v "^Action: " | sort
24+
25+
# combining excludes and includes:
26+
trace $CLI sync . /Users/$CURRENT_USER_NAME --exclude 'project-folder/app.*' --exclude 'project-folder/query.sql' --include 'ignored-folder/*.py' | grep -v "^Action: " | sort
27+
28+
# include sub-folders:
29+
trace $CLI sync . /Users/$CURRENT_USER_NAME --exclude 'project-folder/app.*' --exclude 'project-folder/query.sql' --include 'ignored-folder/**/*.py' | grep -v "^Action: " | sort
30+
31+
# use negated include to exclude files from syncing:
32+
trace $CLI sync . /Users/$CURRENT_USER_NAME --include 'ignored-folder/**' --include '!ignored-folder/folder1/big-blob' | grep -v "^Action: " | sort
33+
34+
# subsequent call without include/exclude flag syncs files based on .gitignore:
35+
trace $CLI sync . /Users/$CURRENT_USER_NAME | grep -v "^Action: " | sort

‎cmd/bundle/sync.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ type syncFlags struct {
2222
full bool
2323
watch bool
2424
output flags.Output
25+
exclude []string
26+
include []string
2527
}
2628

2729
func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) {
@@ -47,6 +49,8 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle)
4749

4850
opts.Full = f.full
4951
opts.PollInterval = f.interval
52+
opts.Exclude = append(opts.Exclude, f.exclude...)
53+
opts.Include = append(opts.Include, f.include...)
5054
return opts, nil
5155
}
5256

@@ -62,6 +66,8 @@ func newSyncCommand() *cobra.Command {
6266
cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)")
6367
cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes")
6468
cmd.Flags().Var(&f.output, "output", "type of the output format")
69+
cmd.Flags().StringSliceVar(&f.exclude, "exclude", nil, "patterns to exclude from sync (can be specified multiple times)")
70+
cmd.Flags().StringSliceVar(&f.include, "include", nil, "patterns to include in sync (can be specified multiple times)")
6571

6672
cmd.RunE = func(cmd *cobra.Command, args []string) error {
6773
ctx := cmd.Context()

‎cmd/sync/sync.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ type syncFlags struct {
2727
full bool
2828
watch bool
2929
output flags.Output
30+
exclude []string
31+
include []string
3032
}
3133

3234
func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
@@ -42,6 +44,8 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *
4244
opts.Full = f.full
4345
opts.PollInterval = f.interval
4446
opts.WorktreeRoot = b.WorktreeRoot
47+
opts.Exclude = append(opts.Exclude, f.exclude...)
48+
opts.Include = append(opts.Include, f.include...)
4549
return opts, nil
4650
}
4751

@@ -86,8 +90,8 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn
8690
WorktreeRoot: worktreeRoot,
8791
LocalRoot: localRoot,
8892
Paths: []string{"."},
89-
Include: nil,
90-
Exclude: nil,
93+
Include: f.include,
94+
Exclude: f.exclude,
9195

9296
RemotePath: args[1],
9397
Full: f.full,
@@ -120,6 +124,8 @@ func New() *cobra.Command {
120124
cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)")
121125
cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes")
122126
cmd.Flags().Var(&f.output, "output", "type of output format")
127+
cmd.Flags().StringSliceVar(&f.exclude, "exclude", nil, "patterns to exclude from sync (can be specified multiple times)")
128+
cmd.Flags().StringSliceVar(&f.include, "include", nil, "patterns to include in sync (can be specified multiple times)")
123129

124130
// Wrapper for [root.MustWorkspaceClient] that disables loading authentication configuration from a bundle.
125131
mustWorkspaceClient := func(cmd *cobra.Command, args []string) error {

0 commit comments

Comments
 (0)