diff --git a/cmd/apps/deploy_bundle.go b/cmd/apps/deploy_bundle.go index 38e312e388..62ecd9b3a8 100644 --- a/cmd/apps/deploy_bundle.go +++ b/cmd/apps/deploy_bundle.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "time" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -39,12 +40,10 @@ func BundleDeployOverrideWithWrapper(wrapError ErrorWrapper) func(*cobra.Command var ( force bool skipValidation bool - skipTests bool ) deployCmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation") - deployCmd.Flags().BoolVar(&skipValidation, "skip-validation", false, "Skip project validation (build, typecheck, lint)") - deployCmd.Flags().BoolVar(&skipTests, "skip-tests", true, "Skip running tests during validation") + deployCmd.Flags().BoolVar(&skipValidation, "skip-validation", false, "Skip project validation entirely") makeArgsOptionalWithBundle(deployCmd, "deploy [APP_NAME]") @@ -53,7 +52,7 @@ func BundleDeployOverrideWithWrapper(wrapError ErrorWrapper) func(*cobra.Command if len(args) == 0 { b := root.TryConfigureBundle(cmd) if b != nil { - return runBundleDeploy(cmd, force, skipValidation, skipTests) + return runBundleDeploy(cmd, b, force, skipValidation) } } @@ -65,10 +64,14 @@ func BundleDeployOverrideWithWrapper(wrapError ErrorWrapper) func(*cobra.Command When run from a Databricks Apps project directory (containing databricks.yml) without an APP_NAME argument, this command runs an enhanced deployment pipeline: -1. Validates the project (build, typecheck, lint for Node.js projects) +1. Validates the project if needed (code changed or never validated) 2. Deploys the project to the workspace 3. Runs the app +Validation is automatically run when: +- No previous validation state exists +- Code has changed since last validation (checksum mismatch) + When an APP_NAME argument is provided (or when not in a project directory), creates an app deployment using the API directly. @@ -77,7 +80,7 @@ Arguments: When provided in a project directory, uses API deploy instead of project deploy. Examples: - # Deploy from a project directory (enhanced flow with validation) + # Deploy from a project directory (auto-validates if needed) databricks apps deploy # Deploy from a specific target @@ -86,7 +89,7 @@ Examples: # Deploy a specific app using the API (even from a project directory) databricks apps deploy my-app - # Deploy from project with validation skip + # Skip validation entirely databricks apps deploy --skip-validation # Force deploy (override git branch validation) @@ -95,7 +98,7 @@ Examples: } // runBundleDeploy executes the enhanced deployment flow for project directories. -func runBundleDeploy(cmd *cobra.Command, force, skipValidation, skipTests bool) error { +func runBundleDeploy(cmd *cobra.Command, initialBundle *bundle.Bundle, force, skipValidation bool) error { ctx := cmd.Context() workDir, err := os.Getwd() @@ -103,28 +106,16 @@ func runBundleDeploy(cmd *cobra.Command, force, skipValidation, skipTests bool) return fmt.Errorf("failed to get working directory: %w", err) } - // Step 1: Validate project (unless skipped) - if !skipValidation { - validator := validation.GetProjectValidator(workDir) - if validator != nil { - opts := validation.ValidateOptions{ - SkipTests: skipTests, - } - result, err := validator.Validate(ctx, workDir, opts) - if err != nil { - return fmt.Errorf("validation error: %w", err) - } + // Get state directory from the initial bundle (before ProcessBundle reinitializes it) + stateDir := initialBundle.GetLocalStateDir(ctx) - if !result.Success { - if result.Details != nil { - cmdio.LogString(ctx, result.Details.Error()) - } - return errors.New("validation failed - fix errors before deploying") - } - cmdio.LogString(ctx, "✅ "+result.Message) - } else { - log.Debugf(ctx, "No validator found for project type, skipping validation") + // Step 1: Validate if needed (unless --skip-validation) + if !skipValidation { + if err := validateIfNeeded(cmd, workDir, stateDir); err != nil { + return err } + } else { + log.Debugf(ctx, "Skipping validation (--skip-validation)") } // Step 2: Deploy project @@ -157,6 +148,11 @@ func runBundleDeploy(cmd *cobra.Command, force, skipValidation, skipTests bool) return fmt.Errorf("failed to run app: %w. Run `databricks apps logs` to view logs", err) } + // Step 4: Update state to deployed + if err := updateStateToDeployed(stateDir); err != nil { + log.Warnf(ctx, "Failed to update state file: %v", err) + } + cmdio.LogString(ctx, "✔ Deployment complete!") return nil } @@ -207,3 +203,81 @@ func runBundleApp(ctx context.Context, b *bundle.Bundle, appKey string) error { return nil } + +// validateIfNeeded checks validation state and runs validation if needed. +// stateDir is the bundle's local state directory for storing validation state. +func validateIfNeeded(cmd *cobra.Command, workDir, stateDir string) error { + ctx := cmd.Context() + + state, err := validation.LoadState(stateDir) + if err != nil { + return fmt.Errorf("failed to load validation state: %w", err) + } + + currentChecksum, err := validation.ComputeChecksum(workDir) + if err != nil { + return fmt.Errorf("failed to compute checksum: %w", err) + } + + // Check if validation is needed + needsValidation := state == nil || currentChecksum != state.Checksum + if !needsValidation { + log.Debugf(ctx, "Validation state up-to-date (checksum: %s...)", state.Checksum[:12]) + return nil + } + + if state == nil { + log.Infof(ctx, "No previous validation state, running validation...") + } else { + log.Infof(ctx, "Code changed since last validation, re-validating...") + } + + // Run validation + opts := validation.ValidateOptions{} + validator := validation.GetProjectValidator(workDir) + if validator != nil { + result, err := validator.Validate(ctx, workDir, opts) + if err != nil { + return fmt.Errorf("validation error: %w", err) + } + + if !result.Success { + if result.Details != nil { + cmdio.LogString(ctx, result.Details.Error()) + } + return errors.New("validation failed - fix errors before deploying") + } + cmdio.LogString(ctx, "✅ "+result.Message) + } else { + log.Debugf(ctx, "No validator found for project type, skipping validation checks") + } + + // Save state + newState := &validation.State{ + State: validation.StateValidated, + ValidatedAt: time.Now().UTC(), + Checksum: currentChecksum, + } + if err := validation.SaveState(stateDir, newState); err != nil { + return fmt.Errorf("failed to save state: %w", err) + } + + return nil +} + +// updateStateToDeployed updates the state file to mark the project as deployed. +// stateDir is the bundle's local state directory. +func updateStateToDeployed(stateDir string) error { + state, err := validation.LoadState(stateDir) + if err != nil { + return err + } + + if state == nil { + // No state file, nothing to update + return nil + } + + state.State = validation.StateDeployed + return validation.SaveState(stateDir, state) +} diff --git a/cmd/apps/validate.go b/cmd/apps/validate.go index f6490d03ac..09cdbfe941 100644 --- a/cmd/apps/validate.go +++ b/cmd/apps/validate.go @@ -4,9 +4,12 @@ import ( "errors" "fmt" "os" + "time" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/apps/validation" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" "github.com/spf13/cobra" ) @@ -20,22 +23,21 @@ func newValidateCmd() *cobra.Command { This command detects the project type and runs the appropriate validation: - Node.js projects (package.json): runs npm install, build, typecheck, lint, and tests +After successful validation, a state file is saved that allows deployment +without re-running validation (unless code changes are detected). + Examples: # Validate the current directory databricks apps validate # Validate a specific directory - databricks apps validate --path ./my-app - - # Run a quick validation without tests - databricks apps validate --skip-tests`, + databricks apps validate --path ./my-app`, RunE: func(cmd *cobra.Command, args []string) error { return runValidate(cmd) }, } cmd.Flags().String("path", "", "Path to the project directory (defaults to current directory)") - cmd.Flags().Bool("skip-tests", false, "Skip running tests for faster validation") return cmd } @@ -53,31 +55,57 @@ func runValidate(cmd *cobra.Command) error { } } - // Get validation options - skipTests, _ := cmd.Flags().GetBool("skip-tests") - opts := validation.ValidateOptions{ - SkipTests: skipTests, + // Try to get bundle context for state storage + var stateDir string + b := root.TryConfigureBundle(cmd) + if b != nil { + stateDir = b.GetLocalStateDir(ctx) } + opts := validation.ValidateOptions{} + // Get validator for project type validator := validation.GetProjectValidator(projectPath) - if validator == nil { - return errors.New("no supported project type detected (looking for package.json)") + if validator != nil { + // Run validation + result, err := validator.Validate(ctx, projectPath, opts) + if err != nil { + return fmt.Errorf("validation error: %w", err) + } + + if !result.Success { + if result.Details != nil { + cmdio.LogString(ctx, result.Details.Error()) + } + return errors.New("validation failed") + } + cmdio.LogString(ctx, "✅ "+result.Message) + } else { + cmdio.LogString(ctx, "✅ No validator found for project type, skipping validation checks") + } + + // Save state only if we have a bundle context + if stateDir == "" { + log.Debugf(ctx, "No bundle context, skipping state save") + return nil } - // Run validation - result, err := validator.Validate(ctx, projectPath, opts) + // Compute checksum and save state + checksum, err := validation.ComputeChecksum(projectPath) if err != nil { - return fmt.Errorf("validation error: %w", err) + return fmt.Errorf("failed to compute checksum: %w", err) } - if !result.Success { - if result.Details != nil { - cmdio.LogString(ctx, result.Details.Error()) - } - return errors.New("validation failed") + state := &validation.State{ + State: validation.StateValidated, + ValidatedAt: time.Now().UTC(), + Checksum: checksum, + } + + if err := validation.SaveState(stateDir, state); err != nil { + return fmt.Errorf("failed to save state: %w", err) } - cmdio.LogString(ctx, "✅ "+result.Message) + cmdio.LogString(ctx, fmt.Sprintf("State saved (checksum: %s...)", checksum[:12])) return nil } diff --git a/libs/apps/validation/nodejs.go b/libs/apps/validation/nodejs.go index 0584d52b9a..7250bb1aa6 100644 --- a/libs/apps/validation/nodejs.go +++ b/libs/apps/validation/nodejs.go @@ -20,7 +20,7 @@ type validationStep struct { command string errorPrefix string displayName string - skipIf func(workDir string, opts ValidateOptions) bool // Optional: skip step if this returns true + skipIf func(workDir string) bool // optional: skip step if this returns true } func (v *ValidationNodeJs) Validate(ctx context.Context, workDir string, opts ValidateOptions) (*ValidateResult, error) { @@ -36,7 +36,7 @@ func (v *ValidationNodeJs) Validate(ctx context.Context, workDir string, opts Va command: "npm install", errorPrefix: "Failed to install dependencies", displayName: "Installing dependencies", - skipIf: func(workDir string, _ ValidateOptions) bool { return hasNodeModules(workDir) }, + skipIf: func(workDir string) bool { return hasNodeModules(workDir) }, }, { name: "generate", @@ -67,13 +67,12 @@ func (v *ValidationNodeJs) Validate(ctx context.Context, workDir string, opts Va command: "npm run test --if-present", errorPrefix: "Failed to run tests", displayName: "Running tests", - skipIf: func(_ string, opts ValidateOptions) bool { return opts.SkipTests }, }, } for _, step := range steps { // Check if step should be skipped - if step.skipIf != nil && step.skipIf(workDir, opts) { + if step.skipIf != nil && step.skipIf(workDir) { log.Debugf(ctx, "skipping %s (condition met)", step.name) cmdio.LogString(ctx, "⏭️ Skipped "+step.displayName) continue diff --git a/libs/apps/validation/state.go b/libs/apps/validation/state.go new file mode 100644 index 0000000000..d7263a4ae7 --- /dev/null +++ b/libs/apps/validation/state.go @@ -0,0 +1,230 @@ +package validation + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const StateFileName = "app_validation_state.json" + +// AppState represents the validation state of a Databricks App. +type AppState string + +const ( + StateValidated AppState = "validated" + StateDeployed AppState = "deployed" +) + +// State represents the validation state file contents. +type State struct { + State AppState `json:"state"` + ValidatedAt time.Time `json:"validated_at"` + Checksum string `json:"checksum"` +} + +// excludedPatterns contains patterns to exclude from checksum computation. +var excludedPatterns = []string{ + // Git + ".git/", + // Node.js + "node_modules/", + ".next/", + "dist/", + "build/", + "coverage/", + ".cache/", + // Python + "__pycache__/", + ".venv/", + "venv/", + ".env/", + "env/", + ".pytest_cache/", + ".mypy_cache/", + ".ruff_cache/", + "htmlcov/", + // Editor/OS + ".DS_Store", + ".idea/", + ".vscode/", + // Temp files + "*.log", + "*.tmp", + "*.temp", + "*.swp", + "*.swo", + // Databricks + ".databricks/", +} + +// excludedExtensions contains file extensions to exclude. +var excludedExtensions = []string{ + ".pyc", + ".pyo", +} + +// excludedSuffixes contains directory suffixes to exclude. +var excludedSuffixes = []string{ + ".egg-info/", +} + +// shouldExclude checks if a path should be excluded from checksum. +func shouldExclude(relPath string) bool { + // Normalize path separators + normalizedPath := filepath.ToSlash(relPath) + + for _, pattern := range excludedPatterns { + // Directory patterns (ending with /) + if strings.HasSuffix(pattern, "/") { + dir := strings.TrimSuffix(pattern, "/") + if normalizedPath == dir || strings.HasPrefix(normalizedPath, dir+"/") { + return true + } + } else if strings.HasPrefix(pattern, "*.") { + // Extension patterns + ext := strings.TrimPrefix(pattern, "*") + if strings.HasSuffix(normalizedPath, ext) { + return true + } + } else { + // Exact match + if normalizedPath == pattern || strings.HasSuffix(normalizedPath, "/"+pattern) { + return true + } + } + } + + for _, ext := range excludedExtensions { + if strings.HasSuffix(normalizedPath, ext) { + return true + } + } + + for _, suffix := range excludedSuffixes { + trimmed := strings.TrimSuffix(suffix, "/") + if strings.HasSuffix(normalizedPath, trimmed) || strings.Contains(normalizedPath, trimmed+"/") { + return true + } + } + + return false +} + +// ComputeChecksum computes a SHA256 checksum of all relevant files in workDir. +func ComputeChecksum(workDir string) (string, error) { + var files []string + + err := filepath.WalkDir(workDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(workDir, path) + if err != nil { + return err + } + + // Skip root + if relPath == "." { + return nil + } + + if shouldExclude(relPath) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + if !d.IsDir() { + files = append(files, relPath) + } + return nil + }) + if err != nil { + return "", fmt.Errorf("failed to walk directory: %w", err) + } + + // Sort for deterministic ordering + sort.Strings(files) + + h := sha256.New() + for _, relPath := range files { + fullPath := filepath.Join(workDir, relPath) + + // Include filename in hash + h.Write([]byte(relPath)) + + f, err := os.Open(fullPath) + if err != nil { + return "", fmt.Errorf("failed to open %s: %w", relPath, err) + } + + _, err = io.Copy(h, f) + f.Close() + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", relPath, err) + } + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +// LoadState reads the state file from stateDir. +// stateDir should be the bundle's local state directory (e.g., .databricks/bundle//). +func LoadState(stateDir string) (*State, error) { + statePath := filepath.Join(stateDir, StateFileName) + + data, err := os.ReadFile(statePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to read state file: %w", err) + } + + var state State + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("failed to parse state file (file may be corrupted, delete %s to reset): %w", statePath, err) + } + + return &state, nil +} + +// SaveState writes the state file to stateDir atomically. +// stateDir should be the bundle's local state directory (e.g., .databricks/bundle//). +func SaveState(stateDir string, state *State) error { + if err := os.MkdirAll(stateDir, 0o700); err != nil { + return fmt.Errorf("failed to create state directory: %w", err) + } + statePath := filepath.Join(stateDir, StateFileName) + + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal state: %w", err) + } + + // Write to temp file first for atomicity + tmpPath := statePath + ".tmp" + // Clean up any leftover temp file from previous failed attempt + os.Remove(tmpPath) + if err := os.WriteFile(tmpPath, data, 0o644); err != nil { + return fmt.Errorf("failed to write temp state file: %w", err) + } + + if err := os.Rename(tmpPath, statePath); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("failed to rename state file: %w", err) + } + + return nil +} diff --git a/libs/apps/validation/state_test.go b/libs/apps/validation/state_test.go new file mode 100644 index 0000000000..49a1a2a529 --- /dev/null +++ b/libs/apps/validation/state_test.go @@ -0,0 +1,196 @@ +package validation + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestShouldExclude(t *testing.T) { + tests := []struct { + path string + excluded bool + }{ + // Git + {".git", true}, + {".git/config", true}, + // Node.js + {"node_modules", true}, + {"node_modules/express/index.js", true}, + {".next", true}, + {"dist", true}, + {"build", true}, + // Python + {"__pycache__", true}, + {"__pycache__/module.pyc", true}, + {".venv", true}, + {"venv", true}, + {"file.pyc", true}, + {"file.pyo", true}, + {".pytest_cache", true}, + {".mypy_cache", true}, + {".ruff_cache", true}, + {"package.egg-info", true}, + {"package.egg-info/PKG-INFO", true}, + // Editor/OS + {".DS_Store", true}, + {"file.swp", true}, + {".idea", true}, + {".vscode", true}, + // Temp files + {"app.log", true}, + {"file.tmp", true}, + {"file.temp", true}, + // Databricks + {".databricks", true}, + {".databricks/bundle", true}, + {".databricks/bundle/dev/" + StateFileName, true}, + // Should NOT be excluded + {"src/main.js", false}, + {"package.json", false}, + {"README.md", false}, + {"app.py", false}, + {"databricks.yml", false}, + } + + for _, tc := range tests { + t.Run(tc.path, func(t *testing.T) { + assert.Equal(t, tc.excluded, shouldExclude(tc.path)) + }) + } +} + +func TestComputeChecksum(t *testing.T) { + dir := t.TempDir() + + // Create some files + require.NoError(t, os.WriteFile(filepath.Join(dir, "file1.txt"), []byte("content1"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "file2.txt"), []byte("content2"), 0o644)) + + checksum1, err := ComputeChecksum(dir) + require.NoError(t, err) + assert.Len(t, checksum1, 64) // SHA256 hex + + // Same content = same checksum + checksum2, err := ComputeChecksum(dir) + require.NoError(t, err) + assert.Equal(t, checksum1, checksum2) + + // Modify file = different checksum + require.NoError(t, os.WriteFile(filepath.Join(dir, "file1.txt"), []byte("modified"), 0o644)) + checksum3, err := ComputeChecksum(dir) + require.NoError(t, err) + assert.NotEqual(t, checksum1, checksum3) +} + +func TestComputeChecksumExcludesPatterns(t *testing.T) { + dir := t.TempDir() + + // Create a source file + require.NoError(t, os.WriteFile(filepath.Join(dir, "main.js"), []byte("code"), 0o644)) + + checksum1, err := ComputeChecksum(dir) + require.NoError(t, err) + + // Add excluded files - checksum should not change + require.NoError(t, os.MkdirAll(filepath.Join(dir, "node_modules"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "node_modules", "pkg.js"), []byte("dep"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".databricks", "bundle", "dev"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".databricks", "bundle", "dev", StateFileName), []byte("state"), 0o644)) + + checksum2, err := ComputeChecksum(dir) + require.NoError(t, err) + assert.Equal(t, checksum1, checksum2) +} + +func TestLoadStateMissing(t *testing.T) { + dir := t.TempDir() + + state, err := LoadState(dir) + require.NoError(t, err) + assert.Nil(t, state) +} + +func TestSaveAndLoadState(t *testing.T) { + dir := t.TempDir() + + now := time.Now().UTC().Truncate(time.Second) + original := &State{ + State: StateValidated, + ValidatedAt: now, + Checksum: "abc123", + } + + require.NoError(t, SaveState(dir, original)) + + loaded, err := LoadState(dir) + require.NoError(t, err) + require.NotNil(t, loaded) + + assert.Equal(t, StateValidated, loaded.State) + assert.Equal(t, "abc123", loaded.Checksum) + assert.Equal(t, now.Unix(), loaded.ValidatedAt.Unix()) +} + +func TestSaveStateAtomic(t *testing.T) { + dir := t.TempDir() + + state := &State{ + State: StateDeployed, + ValidatedAt: time.Now().UTC(), + Checksum: "xyz789", + } + + require.NoError(t, SaveState(dir, state)) + + // Verify no temp file left behind + _, err := os.Stat(filepath.Join(dir, StateFileName+".tmp")) + assert.True(t, os.IsNotExist(err)) + + // Verify state file exists + _, err = os.Stat(filepath.Join(dir, StateFileName)) + assert.NoError(t, err) +} + +func TestLoadStateCorrupted(t *testing.T) { + dir := t.TempDir() + statePath := filepath.Join(dir, StateFileName) + require.NoError(t, os.WriteFile(statePath, []byte("not valid json"), 0o644)) + + state, err := LoadState(dir) + require.Error(t, err) + assert.Nil(t, state) + assert.Contains(t, err.Error(), "corrupted") +} + +func TestComputeChecksumEmptyDir(t *testing.T) { + dir := t.TempDir() + + checksum, err := ComputeChecksum(dir) + require.NoError(t, err) + assert.Len(t, checksum, 64) // Should still produce valid checksum +} + +func TestSaveStateCleansTempFile(t *testing.T) { + dir := t.TempDir() + tmpPath := filepath.Join(dir, StateFileName+".tmp") + + // Create a leftover temp file + require.NoError(t, os.WriteFile(tmpPath, []byte("leftover"), 0o644)) + + state := &State{ + State: StateValidated, + ValidatedAt: time.Now().UTC(), + Checksum: "test123", + } + + require.NoError(t, SaveState(dir, state)) + + // Temp file should be gone + _, err := os.Stat(tmpPath) + assert.True(t, os.IsNotExist(err)) +} diff --git a/libs/apps/validation/validation.go b/libs/apps/validation/validation.go index 4cc45f5a1e..71a2141cad 100644 --- a/libs/apps/validation/validation.go +++ b/libs/apps/validation/validation.go @@ -52,9 +52,7 @@ func (vr *ValidateResult) String() string { } // ValidateOptions configures validation behavior. -type ValidateOptions struct { - SkipTests bool // Skip running tests for faster validation -} +type ValidateOptions struct{} // Validation defines the interface for project validation strategies. type Validation interface {