feat: private repo support

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/lmittmann/tint v1.1.2
 	github.com/minio/minio-go/v7 v7.0.97
 	go.etcd.io/bbolt v1.4.3
+	golang.org/x/mod v0.31.0
 )
 
 require (
@@ -30,7 +31,6 @@ require (
 	github.com/stretchr/testify v1.11.1 // indirect
 	github.com/tinylib/msgp v1.3.0 // indirect
 	golang.org/x/crypto v0.44.0 // indirect
-	golang.org/x/mod v0.31.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/text v0.32.0 // indirect

internal/config/config.go

@@ -15,7 +15,8 @@ import (
 	"github.com/block/cachew/internal/jobscheduler"
 	"github.com/block/cachew/internal/logging"
 	"github.com/block/cachew/internal/strategy"
-	_ "github.com/block/cachew/internal/strategy/git" // Register git strategy
+	_ "github.com/block/cachew/internal/strategy/git"   // Register git strategy
+	_ "github.com/block/cachew/internal/strategy/gomod" // Register gomod strategy
 )
 
 type loggingMux struct {

internal/strategy/gomod_cacher.go → internal/strategy/gomod/cacher.go

@@ -1,4 +1,4 @@
-package strategy
+package gomod
 
 import (
 	"context"

internal/strategy/gomod/fetcher.go

@@ -0,0 +1,55 @@
+package gomod
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/alecthomas/errors"
+	"github.com/goproxy/goproxy"
+)
+
+type compositeFetcher struct {
+	publicFetcher  goproxy.Fetcher
+	privateFetcher goproxy.Fetcher
+	matcher        *ModulePathMatcher
+}
+
+func newCompositeFetcher(
+	publicFetcher goproxy.Fetcher,
+	privateFetcher goproxy.Fetcher,
+	patterns []string,
+) *compositeFetcher {
+	return &compositeFetcher{
+		publicFetcher:  publicFetcher,
+		privateFetcher: privateFetcher,
+		matcher:        NewModulePathMatcher(patterns),
+	}
+}
+
+func (c *compositeFetcher) Query(ctx context.Context, path, query string) (version string, t time.Time, err error) {
+	if c.matcher.IsPrivate(path) {
+		v, tm, err := c.privateFetcher.Query(ctx, path, query)
+		return v, tm, errors.Wrap(err, "private fetcher query")
+	}
+	v, tm, err := c.publicFetcher.Query(ctx, path, query)
+	return v, tm, errors.Wrap(err, "public fetcher query")
+}
+
+func (c *compositeFetcher) List(ctx context.Context, path string) (versions []string, err error) {
+	if c.matcher.IsPrivate(path) {
+		v, err := c.privateFetcher.List(ctx, path)
+		return v, errors.Wrap(err, "private fetcher list")
+	}
+	v, err := c.publicFetcher.List(ctx, path)
+	return v, errors.Wrap(err, "public fetcher list")
+}
+
+func (c *compositeFetcher) Download(ctx context.Context, path, version string) (info, mod, zip io.ReadSeekCloser, err error) {
+	if c.matcher.IsPrivate(path) {
+		i, m, z, err := c.privateFetcher.Download(ctx, path, version)
+		return i, m, z, errors.Wrap(err, "private fetcher download")
+	}
+	i, m, z, err := c.publicFetcher.Download(ctx, path, version)
+	return i, m, z, errors.Wrap(err, "public fetcher download")
+}

internal/strategy/gomod/gomod.go

@@ -0,0 +1,128 @@
+package gomod
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/alecthomas/errors"
+	"github.com/goproxy/goproxy"
+
+	"github.com/block/cachew/internal/cache"
+	"github.com/block/cachew/internal/gitclone"
+	"github.com/block/cachew/internal/jobscheduler"
+	"github.com/block/cachew/internal/logging"
+	"github.com/block/cachew/internal/strategy"
+)
+
+func init() {
+	strategy.Register("gomod", "Caches Go module proxy requests.", New)
+}
+
+type Config struct {
+	Proxy            string        `hcl:"proxy,optional" help:"Upstream Go module proxy URL (defaults to proxy.golang.org)" default:"https://proxy.golang.org"`
+	PrivatePaths     []string      `hcl:"private-paths,optional" help:"Module path patterns for private repositories"`
+	MirrorRoot       string        `hcl:"mirror-root,optional" help:"Directory to store git clones for private repos." default:""`
+	FetchInterval    time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream for private repos." default:"15m"`
+	RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks for private repos." default:"10s"`
+	CloneDepth       int           `hcl:"clone-depth,optional" help:"Depth for shallow clones of private repos. 0 means full clone." default:"0"`
+}
+
+type Strategy struct {
+	config       Config
+	cache        cache.Cache
+	logger       *slog.Logger
+	proxy        *url.URL
+	goproxy      *goproxy.Goproxy
+	cloneManager *gitclone.Manager // Manager for cloning private repositories
+}
+
+var _ strategy.Strategy = (*Strategy)(nil)
+
+func New(ctx context.Context, config Config, _ jobscheduler.Scheduler, cache cache.Cache, mux strategy.Mux) (*Strategy, error) {
+	parsedURL, err := url.Parse(config.Proxy)
+	if err != nil {
+		return nil, fmt.Errorf("invalid proxy URL: %w", err)
+	}
+
+	s := &Strategy{
+		config: config,
+		cache:  cache,
+		logger: logging.FromContext(ctx),
+		proxy:  parsedURL,
+	}
+
+	publicFetcher := &goproxy.GoFetcher{
+		Env: []string{
+			"GOPROXY=" + config.Proxy,
+			"GOSUMDB=off", // Disable checksum database validation in fetcher, to prevent unneccessary double validation
+		},
+	}
+
+	var fetcher goproxy.Fetcher = publicFetcher
+
+	if len(config.PrivatePaths) > 0 {
+		// Set default mirror root if not specified
+		mirrorRoot := config.MirrorRoot
+		if mirrorRoot == "" {
+			homeDir, err := os.UserHomeDir()
+			if err != nil {
+				return nil, errors.Wrap(err, "get user home directory")
+			}
+			mirrorRoot = filepath.Join(homeDir, ".cache", "cachew", "gomod-git-mirrors")
+		}
+
+		// Create gitclone manager for private repositories
+		cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{
+			RootDir:          mirrorRoot,
+			FetchInterval:    config.FetchInterval,
+			RefCheckInterval: config.RefCheckInterval,
+			CloneDepth:       config.CloneDepth,
+			GitConfig:        gitclone.DefaultGitTuningConfig(),
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "create clone manager for private repos")
+		}
+		s.cloneManager = cloneManager
+
+		// Discover existing clones
+		if err := cloneManager.DiscoverExisting(ctx); err != nil {
+			s.logger.WarnContext(ctx, "Failed to discover existing clones for private repos",
+				slog.String("error", err.Error()))
+		}
+
+		privateFetcher := newPrivateFetcher(s, cloneManager)
+		fetcher = newCompositeFetcher(publicFetcher, privateFetcher, config.PrivatePaths)
+
+		s.logger.InfoContext(ctx, "Configured private module support",
+			slog.Any("private_paths", config.PrivatePaths),
+			slog.String("mirror_root", mirrorRoot))
+	}
+
+	s.goproxy = &goproxy.Goproxy{
+		Logger:  s.logger,
+		Fetcher: fetcher,
+		Cacher: &goproxyCacher{
+			cache: cache,
+		},
+		ProxiedSumDBs: []string{
+			"sum.golang.org https://sum.golang.org",
+		},
+	}
+
+	s.logger.InfoContext(ctx, "Initialized Go module proxy strategy",
+		slog.String("proxy", s.proxy.String()))
+
+	mux.Handle("GET /gomod/{path...}", http.StripPrefix("/gomod", s.goproxy))
+
+	return s, nil
+}
+
+func (s *Strategy) String() string {
+	return "gomod:" + s.proxy.Host
+}

internal/strategy/gomod_test.go → internal/strategy/gomod/gomod_test.go

@@ -1,4 +1,4 @@
-package strategy_test
+package gomod_test
 
 import (
 	"archive/zip"
@@ -17,7 +17,7 @@ import (
 	"github.com/block/cachew/internal/cache"
 	"github.com/block/cachew/internal/jobscheduler"
 	"github.com/block/cachew/internal/logging"
-	"github.com/block/cachew/internal/strategy"
+	"github.com/block/cachew/internal/strategy/gomod"
 )
 
 type mockGoModServer struct {
@@ -58,25 +58,33 @@ func newMockGoModServer(t *testing.T) *mockGoModServer {
 	return m
 }
 
-func createModuleZip(t *testing.T, modulePath, version string) string {
-	t.Helper()
+func createModuleZip(modulePath, version string) string {
 	var buf bytes.Buffer
 	w := zip.NewWriter(&buf)
 
 	prefix := modulePath + "@" + version + "/"
 
 	f, err := w.Create(prefix + "go.mod")
-	assert.NoError(t, err)
+	if err != nil {
+		panic(err)
+	}
 	_, err = f.Write([]byte("module " + modulePath + "\n\ngo 1.21\n"))
-	assert.NoError(t, err)
+	if err != nil {
+		panic(err)
+	}
 
 	f2, err := w.Create(prefix + "main.go")
-	assert.NoError(t, err)
+	if err != nil {
+		panic(err)
+	}
 	_, err = f2.Write([]byte("package main\n\nfunc main() {}\n"))
-	assert.NoError(t, err)
+	if err != nil {
+		panic(err)
+	}
 
-	err = w.Close()
-	assert.NoError(t, err)
+	if err := w.Close(); err != nil {
+		panic(err)
+	}
 
 	return buf.String()
 }
@@ -129,7 +137,7 @@ func (m *mockGoModServer) handleRequest(w http.ResponseWriter, r *http.Request)
 				version := strings.TrimSuffix(versionPart, ".zip")
 				resp = mockResponse{
 					status:  http.StatusOK,
-					content: createModuleZip(m.t, modulePath, version),
+					content: createModuleZip(modulePath, version),
 				}
 				found = true
 			}
@@ -176,7 +184,7 @@ func setupGoModTest(t *testing.T) (*mockGoModServer, *http.ServeMux, context.Con
 	t.Cleanup(func() { _ = memCache.Close() })
 
 	mux := http.NewServeMux()
-	_, err = strategy.NewGoMod(ctx, strategy.GoModConfig{
+	_, err = gomod.New(ctx, gomod.Config{
 		Proxy: mock.server.URL,
 	}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux)
 	assert.NoError(t, err)

internal/strategy/gomod/matcher.go

@@ -0,0 +1,32 @@
+package gomod
+
+import (
+	"path"
+	"strings"
+)
+
+// ModulePathMatcher matches module paths against patterns.
+type ModulePathMatcher struct {
+	patterns []string
+}
+
+// NewModulePathMatcher creates a new matcher with the given patterns.
+func NewModulePathMatcher(patterns []string) *ModulePathMatcher {
+	return &ModulePathMatcher{patterns: patterns}
+}
+
+// IsPrivate checks if a module path matches any private pattern.
+func (m *ModulePathMatcher) IsPrivate(modulePath string) bool {
+	for _, pattern := range m.patterns {
+		matched, err := path.Match(pattern, modulePath)
+		if err == nil && matched {
+			return true
+		}
+
+		if strings.HasPrefix(modulePath, pattern+"/") || modulePath == pattern {
+			return true
+		}
+	}
+
+	return false
+}