Move generate utils to bundle/generate package (#3088)

## Changes

* Move the downloader and the configuration conversion functions to
`bundle/generate`.
* Export necessary functions.
* Add a simple unit test.

## Why

While looking into #2528, I wanted to add a unit test to the downloader
and found it doesn't have any direct unit tests yet, and was colocated
with the command implementations. This consolidates the downloader and
the configuration conversion functions from `bundle/config/generate`
into `bundle/generate` and adds a simple unit test for the downloader. A
follow-up PR will address path normalization.

## Tests

Unit tests pass.

Author: pietern

bundle/config/generate/app.go bundle/generate/app.gobundle/config/generate/app.go renamed to bundle/generate/app.go

@@ -18,23 +18,23 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-type downloader struct {
+type Downloader struct {
 	files     map[string]string
 	w         *databricks.WorkspaceClient
 	sourceDir string
 	configDir string
 	basePath  string
 }
 
-func (n *downloader) MarkTaskForDownload(ctx context.Context, task *jobs.Task) error {
+func (n *Downloader) MarkTaskForDownload(ctx context.Context, task *jobs.Task) error {
 	if task.NotebookTask == nil {
 		return nil
 	}
 
 	return n.markNotebookForDownload(ctx, &task.NotebookTask.NotebookPath)
 }
 
-func (n *downloader) MarkPipelineLibraryForDownload(ctx context.Context, lib *pipelines.PipelineLibrary) error {
+func (n *Downloader) MarkPipelineLibraryForDownload(ctx context.Context, lib *pipelines.PipelineLibrary) error {
 	if lib.Notebook != nil {
 		return n.markNotebookForDownload(ctx, &lib.Notebook.Path)
 	}
@@ -46,7 +46,7 @@ func (n *downloader) MarkPipelineLibraryForDownload(ctx context.Context, lib *pi
 	return nil
 }
 
-func (n *downloader) markFileForDownload(ctx context.Context, filePath *string) error {
+func (n *Downloader) markFileForDownload(ctx context.Context, filePath *string) error {
 	_, err := n.w.Workspace.GetStatusByPath(ctx, *filePath)
 	if err != nil {
 		return err
@@ -66,7 +66,7 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string)
 	return nil
 }
 
-func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error {
+func (n *Downloader) MarkDirectoryForDownload(ctx context.Context, dirPath *string) error {
 	_, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath)
 	if err != nil {
 		return err
@@ -102,7 +102,7 @@ func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *stri
 	return nil
 }
 
-func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
+func (n *Downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
 	info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
 	if err != nil {
 		return err
@@ -123,7 +123,7 @@ func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *
 	return nil
 }
 
-func (n *downloader) relativePath(fullPath string) string {
+func (n *Downloader) relativePath(fullPath string) string {
 	basePath := path.Dir(fullPath)
 	if n.basePath != "" {
 		basePath = n.basePath
@@ -138,7 +138,7 @@ func (n *downloader) relativePath(fullPath string) string {
 	return relPath
 }
 
-func (n *downloader) FlushToDisk(ctx context.Context, force bool) error {
+func (n *Downloader) FlushToDisk(ctx context.Context, force bool) error {
 	// First check that all files can be written
 	for targetPath := range n.files {
 		info, err := os.Stat(targetPath)
@@ -185,8 +185,8 @@ func (n *downloader) FlushToDisk(ctx context.Context, force bool) error {
 	return errs.Wait()
 }
 
-func newDownloader(w *databricks.WorkspaceClient, sourceDir, configDir string) *downloader {
-	return &downloader{
+func NewDownloader(w *databricks.WorkspaceClient, sourceDir, configDir string) *Downloader {
+	return &Downloader{
 		files:     make(map[string]string),
 		w:         w,
 		sourceDir: sourceDir,

bundle/config/generate/dashboard.go bundle/generate/dashboard.gobundle/config/generate/dashboard.go renamed to bundle/generate/dashboard.go

@@ -0,0 +1,42 @@
+package generate
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloader_MarkFileReturnsRelativePath(t *testing.T) {
+	ctx := context.Background()
+	m := mocks.NewMockWorkspaceClient(t)
+
+	dir := "base/dir/doesnt/matter"
+	sourceDir := filepath.Join(dir, "source")
+	configDir := filepath.Join(dir, "config")
+	downloader := NewDownloader(m.WorkspaceClient, sourceDir, configDir)
+
+	var err error
+
+	// Test that the path is normalized to be relative to the config directory.
+	f1 := "/a/b/c"
+	m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(ctx, f1).Return(&workspace.ObjectInfo{
+		Path: f1,
+	}, nil)
+	err = downloader.markFileForDownload(ctx, &f1)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.FromSlash("../source/c"), f1)
+
+	// Test that the previous path doesn't influence the next path.
+	f2 := "/a/b/c/d"
+	m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(ctx, f2).Return(&workspace.ObjectInfo{
+		Path: f2,
+	}, nil)
+	err = downloader.markFileForDownload(ctx, &f2)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.FromSlash("../source/d"), f2)
+}

cmd/bundle/generate/utils.go bundle/generate/downloader.gocmd/bundle/generate/utils.go renamed to bundle/generate/downloader.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/databricks/cli/bundle/config/generate"
+	"github.com/databricks/cli/bundle/generate"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/dyn"
@@ -46,10 +46,10 @@ func NewGenerateAppCommand() *cobra.Command {
 			return err
 		}
 
-		downloader := newDownloader(w, sourceDir, configDir)
+		downloader := generate.NewDownloader(w, sourceDir, configDir)
 
 		sourceCodePath := app.DefaultSourceCodePath
-		err = downloader.markDirectoryForDownload(ctx, &sourceCodePath)
+		err = downloader.MarkDirectoryForDownload(ctx, &sourceCodePath)
 		if err != nil {
 			return err
 		}

bundle/generate/downloader_test.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/generate"
 	"github.com/databricks/cli/bundle/deploy/terraform"
+	"github.com/databricks/cli/bundle/generate"
 	"github.com/databricks/cli/bundle/phases"
 	"github.com/databricks/cli/bundle/render"
 	"github.com/databricks/cli/bundle/resources"

bundle/config/generate/job.go bundle/generate/job.gobundle/config/generate/job.go renamed to bundle/generate/job.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/databricks/cli/bundle/config/generate"
+	"github.com/databricks/cli/bundle/generate"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/dyn"
@@ -49,7 +49,7 @@ func NewGenerateJobCommand() *cobra.Command {
 			return err
 		}
 
-		downloader := newDownloader(w, sourceDir, configDir)
+		downloader := generate.NewDownloader(w, sourceDir, configDir)
 
 		// Don't download files if the job is using Git source
 		// When Git source is used, the job will be using the files from the Git repository

bundle/config/generate/pipeline.go bundle/generate/pipeline.gobundle/config/generate/pipeline.go renamed to bundle/generate/pipeline.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/databricks/cli/bundle/config/generate"
+	"github.com/databricks/cli/bundle/generate"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/dyn"
@@ -49,7 +49,7 @@ func NewGeneratePipelineCommand() *cobra.Command {
 			return err
 		}
 
-		downloader := newDownloader(w, sourceDir, configDir)
+		downloader := generate.NewDownloader(w, sourceDir, configDir)
 		for _, lib := range pipeline.Spec.Libraries {
 			err := downloader.MarkPipelineLibraryForDownload(ctx, &lib)
 			if err != nil {