From 3da1a2aa7e9a5487446cf8c6b1c76112f08ceb4e Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 27 Jan 2026 17:46:24 -0500 Subject: [PATCH 1/6] fix(middleware): support RepoAccess token format in registry auth The two-tier build cache PR (#70) introduced a new token format using `repo_access` field with per-repository scopes, but the middleware wasn't updated to parse it. This caused 401 Unauthorized errors when builder VMs tried to push images to the registry, as the middleware only checked for the legacy `repos` field which is empty in new tokens. Changes: - Add RepoPermission struct and RepoAccess field to RegistryTokenClaims - Update validateRegistryToken to check both RepoAccess (new) and Repositories (legacy) formats - Add per-repo scope checking for write operations - Add comprehensive tests for both token formats Fixes build failures in production where the new token format was being used but not recognized by the registry auth middleware. --- lib/middleware/oapi_auth.go | 57 ++++++++++++++++----- lib/middleware/oapi_auth_test.go | 88 +++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 15 deletions(-) diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index cf5f830..96e5b42 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -22,13 +22,28 @@ const userIDKey contextKey = "user_id" // It properly parses repository names (which can contain slashes) from /v2/ paths. var registryRouter = v2.Router() +// RepoPermission defines access permissions for a specific repository. +// This mirrors the type in lib/builds/registry_token.go to avoid circular imports. +type RepoPermission struct { + Repo string `json:"repo"` + Scope string `json:"scope"` +} + // RegistryTokenClaims contains the claims for a scoped registry access token. // This mirrors the type in lib/builds/registry_token.go to avoid circular imports. type RegistryTokenClaims struct { jwt.RegisteredClaims - BuildID string `json:"build_id"` - Repositories []string `json:"repos"` - Scope string `json:"scope"` + BuildID string `json:"build_id"` + + // RepoAccess defines per-repository access permissions (new two-tier format) + // If present, this takes precedence over the legacy Repositories/Scope fields + RepoAccess []RepoPermission `json:"repo_access,omitempty"` + + // Repositories is the list of allowed repository paths (legacy format) + Repositories []string `json:"repos,omitempty"` + + // Scope is the access scope (legacy format) + Scope string `json:"scope,omitempty"` } // OapiAuthenticationFunc creates an AuthenticationFunc compatible with nethttp-middleware @@ -229,6 +244,7 @@ func isWriteOperation(method string) bool { // validateRegistryToken validates a registry-scoped JWT token and checks repository access. // Returns the claims if valid, nil otherwise. +// Supports both new RepoAccess format and legacy Repositories/Scope format. func validateRegistryToken(tokenString, jwtSecret, requestPath, method string) (*RegistryTokenClaims, error) { token, err := jwt.ParseWithClaims(tokenString, &RegistryTokenClaims{}, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { @@ -246,8 +262,8 @@ func validateRegistryToken(tokenString, jwtSecret, requestPath, method string) ( return nil, fmt.Errorf("invalid token") } - // Check if this is a registry token (has repos claim) - if len(claims.Repositories) == 0 { + // Check if this is a registry token (has either RepoAccess or legacy repos claim) + if len(claims.RepoAccess) == 0 && len(claims.Repositories) == 0 { return nil, fmt.Errorf("not a registry token") } @@ -261,21 +277,34 @@ func validateRegistryToken(tokenString, jwtSecret, requestPath, method string) ( return nil, fmt.Errorf("could not extract repository from path") } - // Check if the repository is allowed by the token - allowed := false - for _, allowedRepo := range claims.Repositories { - if allowedRepo == repo { - allowed = true - break + // Check if the repository is allowed by the token and get its scope + var repoScope string + + // Check new RepoAccess format first + if len(claims.RepoAccess) > 0 { + for _, perm := range claims.RepoAccess { + if perm.Repo == repo { + repoScope = perm.Scope + break + } + } + } else { + // Fall back to legacy format + for _, allowedRepo := range claims.Repositories { + if allowedRepo == repo { + repoScope = claims.Scope + break + } } } - if !allowed { + + if repoScope == "" { return nil, fmt.Errorf("repository %s not allowed by token", repo) } // Check scope for write operations - if isWriteOperation(method) && claims.Scope != "push" { - return nil, fmt.Errorf("token does not allow write operations") + if isWriteOperation(method) && repoScope != "push" { + return nil, fmt.Errorf("token does not allow write operations for %s", repo) } return claims, nil diff --git a/lib/middleware/oapi_auth_test.go b/lib/middleware/oapi_auth_test.go index 6fc1205..69e773c 100644 --- a/lib/middleware/oapi_auth_test.go +++ b/lib/middleware/oapi_auth_test.go @@ -25,7 +25,7 @@ func generateUserToken(t *testing.T, userID string) string { return tokenString } -// generateRegistryToken creates a registry token (like those given to builder VMs) +// generateRegistryToken creates a registry token using the legacy format func generateRegistryToken(t *testing.T, buildID string) string { token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ "sub": "builder-" + buildID, @@ -41,6 +41,21 @@ func generateRegistryToken(t *testing.T, buildID string) string { return tokenString } +// generateRepoAccessToken creates a registry token using the new RepoAccess format +func generateRepoAccessToken(t *testing.T, buildID string, repoAccess []map[string]string) string { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "sub": "builder-" + buildID, + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + "iss": "hypeman", + "build_id": buildID, + "repo_access": repoAccess, + }) + tokenString, err := token.SignedString([]byte(testJWTSecret)) + require.NoError(t, err) + return tokenString +} + func TestJwtAuth_RejectsRegistryTokens(t *testing.T) { // Create a simple handler that returns 200 if auth passes nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -203,6 +218,77 @@ func TestExtractRepoFromPath(t *testing.T) { } } +func TestValidateRegistryToken(t *testing.T) { + t.Run("legacy format token allows push to allowed repo", func(t *testing.T) { + token := generateRegistryToken(t, "build-123") + claims, err := validateRegistryToken(token, testJWTSecret, "/v2/builds/build-123/manifests/latest", http.MethodPut) + require.NoError(t, err) + assert.Equal(t, "build-123", claims.BuildID) + }) + + t.Run("legacy format token rejects unauthorized repo", func(t *testing.T) { + token := generateRegistryToken(t, "build-123") + _, err := validateRegistryToken(token, testJWTSecret, "/v2/builds/other-build/manifests/latest", http.MethodPut) + require.Error(t, err) + assert.Contains(t, err.Error(), "not allowed by token") + }) + + t.Run("RepoAccess format token allows push to push-scoped repo", func(t *testing.T) { + repoAccess := []map[string]string{ + {"repo": "builds/build-456", "scope": "push"}, + {"repo": "cache/tenant-x", "scope": "push"}, + {"repo": "cache/global/node", "scope": "pull"}, + } + token := generateRepoAccessToken(t, "build-456", repoAccess) + + claims, err := validateRegistryToken(token, testJWTSecret, "/v2/builds/build-456/manifests/latest", http.MethodPut) + require.NoError(t, err) + assert.Equal(t, "build-456", claims.BuildID) + }) + + t.Run("RepoAccess format token allows pull from pull-scoped repo", func(t *testing.T) { + repoAccess := []map[string]string{ + {"repo": "builds/build-456", "scope": "push"}, + {"repo": "cache/global/node", "scope": "pull"}, + } + token := generateRepoAccessToken(t, "build-456", repoAccess) + + // GET (pull) from pull-scoped repo should work + _, err := validateRegistryToken(token, testJWTSecret, "/v2/cache/global/node/manifests/latest", http.MethodGet) + require.NoError(t, err) + }) + + t.Run("RepoAccess format token rejects push to pull-only repo", func(t *testing.T) { + repoAccess := []map[string]string{ + {"repo": "builds/build-456", "scope": "push"}, + {"repo": "cache/global/node", "scope": "pull"}, + } + token := generateRepoAccessToken(t, "build-456", repoAccess) + + // PUT (push) to pull-only repo should fail + _, err := validateRegistryToken(token, testJWTSecret, "/v2/cache/global/node/manifests/latest", http.MethodPut) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not allow write operations") + }) + + t.Run("RepoAccess format token rejects unauthorized repo", func(t *testing.T) { + repoAccess := []map[string]string{ + {"repo": "builds/build-456", "scope": "push"}, + } + token := generateRepoAccessToken(t, "build-456", repoAccess) + + _, err := validateRegistryToken(token, testJWTSecret, "/v2/builds/other-build/manifests/latest", http.MethodPut) + require.Error(t, err) + assert.Contains(t, err.Error(), "not allowed by token") + }) + + t.Run("allows base path check without repo validation", func(t *testing.T) { + token := generateRegistryToken(t, "build-123") + _, err := validateRegistryToken(token, testJWTSecret, "/v2/", http.MethodGet) + require.NoError(t, err) + }) +} + func TestJwtAuth_RequiresAuthorization(t *testing.T) { nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) From cd55df690b678576d9469737afda5fc7d36228d1 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 27 Jan 2026 20:22:52 -0500 Subject: [PATCH 2/6] fix(middleware): add production subnet to registry IP fallback The IP fallback for registry authentication only allowed 10.100.x.x and 10.102.x.x subnets (staging/dev), but production uses 172.30.x.x. This caused builds to fail in production while working in staging. Changes: - Add 172.30.x.x to allowed subnets in isInternalVMRequest() - Add logger injection to /v2 routes for debug logging - Add tests for internal VM request subnet matching --- cmd/api/main.go | 1 + lib/middleware/oapi_auth.go | 7 +++++-- lib/middleware/oapi_auth_test.go | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 5c835a1..e84db3b 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -291,6 +291,7 @@ func run() error { r.Use(middleware.RealIP) r.Use(middleware.Logger) r.Use(middleware.Recoverer) + r.Use(mw.InjectLogger(logger)) // Inject logger for debug logging in JwtAuth r.Use(mw.JwtAuth(app.Config.JwtSecret)) r.Mount("/", app.Registry.Handler()) }) diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 96e5b42..b94f261 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -212,8 +212,11 @@ func isInternalVMRequest(r *http.Request) bool { ip = ip[:idx] } - // Check if it's from the VM network (10.100.x.x or 10.102.x.x) - return strings.HasPrefix(ip, "10.100.") || strings.HasPrefix(ip, "10.102.") + // Check if it's from the VM network + // Different environments use different subnets: + // - 10.100.x.x, 10.102.x.x: staging/dev environments + // - 172.30.x.x: production environment + return strings.HasPrefix(ip, "10.100.") || strings.HasPrefix(ip, "10.102.") || strings.HasPrefix(ip, "172.30.") } // extractRepoFromPath extracts the repository name from a registry path. diff --git a/lib/middleware/oapi_auth_test.go b/lib/middleware/oapi_auth_test.go index 69e773c..991bf7f 100644 --- a/lib/middleware/oapi_auth_test.go +++ b/lib/middleware/oapi_auth_test.go @@ -289,6 +289,42 @@ func TestValidateRegistryToken(t *testing.T) { }) } +func TestIsInternalVMRequest(t *testing.T) { + tests := []struct { + name string + remoteAddr string + expected bool + }{ + // Staging/dev subnets + {"staging 10.100.x.x", "10.100.1.50:12345", true}, + {"staging 10.102.x.x", "10.102.5.100:54321", true}, + + // Production subnet + {"production 172.30.x.x", "172.30.16.101:42700", true}, + {"production 172.30.0.x", "172.30.0.50:8080", true}, + + // External IPs (should be rejected) + {"external 192.168.x.x", "192.168.1.100:8080", false}, + {"external public IP", "34.21.1.136:8080", false}, + {"external 10.0.x.x (different subnet)", "10.0.1.50:8080", false}, + {"external 172.16.x.x (different subnet)", "172.16.1.50:8080", false}, + + // Edge cases + {"localhost", "127.0.0.1:8080", false}, + {"IPv6 localhost", "[::1]:8080", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/v2/test/manifests/latest", nil) + req.RemoteAddr = tt.remoteAddr + + result := isInternalVMRequest(req) + assert.Equal(t, tt.expected, result, "isInternalVMRequest with RemoteAddr=%q", tt.remoteAddr) + }) + } +} + func TestJwtAuth_RequiresAuthorization(t *testing.T) { nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) From 84afce5e8818ddbc5ad8ddd8aeb4f3d0c2d5c3fc Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 27 Jan 2026 20:33:20 -0500 Subject: [PATCH 3/6] test(middleware): add integration tests for registry auth flow Add comprehensive tests for JwtAuth middleware on /v2/ registry paths: - Valid token access (both legacy and RepoAccess formats) - IP fallback for staging (10.100.x.x, 10.102.x.x) and production (172.30.x.x) - External IP rejection without valid token - Invalid token fallback behavior - Bearer and Basic auth support These tests would have caught the production subnet issue earlier. --- lib/middleware/oapi_auth_test.go | 141 +++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/lib/middleware/oapi_auth_test.go b/lib/middleware/oapi_auth_test.go index 991bf7f..b45493a 100644 --- a/lib/middleware/oapi_auth_test.go +++ b/lib/middleware/oapi_auth_test.go @@ -1,6 +1,7 @@ package middleware import ( + "encoding/base64" "net/http" "net/http/httptest" "testing" @@ -289,6 +290,146 @@ func TestValidateRegistryToken(t *testing.T) { }) } +func TestJwtAuth_RegistryPaths(t *testing.T) { + // Create a simple handler that returns 200 if auth passes + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + handler := JwtAuth(testJWTSecret)(nextHandler) + + t.Run("valid registry token allows access to authorized repo", func(t *testing.T) { + token := generateRegistryToken(t, "build-123") + + req := httptest.NewRequest(http.MethodHead, "/v2/builds/build-123/manifests/latest", nil) + req.Header.Set("Authorization", "Basic "+basicAuth(token)) + req.RemoteAddr = "8.8.8.8:12345" // External IP - should still work with valid token + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "valid registry token should allow access") + }) + + t.Run("valid RepoAccess token allows access to authorized repo", func(t *testing.T) { + repoAccess := []map[string]string{ + {"repo": "builds/build-456", "scope": "push"}, + {"repo": "cache/tenant-x", "scope": "push"}, + } + token := generateRepoAccessToken(t, "build-456", repoAccess) + + req := httptest.NewRequest(http.MethodPut, "/v2/builds/build-456/manifests/latest", nil) + req.Header.Set("Authorization", "Basic "+basicAuth(token)) + req.RemoteAddr = "8.8.8.8:12345" // External IP + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "valid RepoAccess token should allow access") + }) + + t.Run("no token but internal staging IP allows access via fallback", func(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) + // No Authorization header + req.RemoteAddr = "10.100.5.50:12345" // Staging subnet + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "internal staging IP should allow access via fallback") + }) + + t.Run("no token but internal production IP allows access via fallback", func(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) + // No Authorization header + req.RemoteAddr = "172.30.16.101:42700" // Production subnet + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "internal production IP should allow access via fallback") + }) + + t.Run("no token and external IP returns 401", func(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) + // No Authorization header + req.RemoteAddr = "8.8.8.8:12345" // External IP + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusUnauthorized, rr.Code, "external IP without token should be rejected") + assert.Contains(t, rr.Body.String(), "registry authentication required") + }) + + t.Run("invalid token but internal IP allows access via fallback", func(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) + req.Header.Set("Authorization", "Basic "+basicAuth("invalid-not-a-jwt-token")) + req.RemoteAddr = "10.102.1.50:12345" // Internal IP + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "internal IP should allow access via fallback even with invalid token") + }) + + t.Run("invalid token and external IP returns 401", func(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) + req.Header.Set("Authorization", "Basic "+basicAuth("invalid-not-a-jwt-token")) + req.RemoteAddr = "8.8.8.8:12345" // External IP + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusUnauthorized, rr.Code, "external IP with invalid token should be rejected") + }) + + t.Run("valid token for wrong repo returns 401 even with internal IP", func(t *testing.T) { + token := generateRegistryToken(t, "build-123") + + req := httptest.NewRequest(http.MethodPut, "/v2/builds/different-build/manifests/latest", nil) + req.Header.Set("Authorization", "Basic "+basicAuth(token)) + req.RemoteAddr = "10.100.5.50:12345" // Internal IP - but token validation fails first + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + // Token is valid JWT but wrong repo - should fall through to IP fallback + assert.Equal(t, http.StatusOK, rr.Code, "should fall back to IP check when token doesn't match repo") + }) + + t.Run("registry base path /v2/ allows access with valid token", func(t *testing.T) { + token := generateRegistryToken(t, "build-123") + + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + req.Header.Set("Authorization", "Basic "+basicAuth(token)) + req.RemoteAddr = "8.8.8.8:12345" + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "/v2/ base path should be allowed with valid token") + }) + + t.Run("Bearer auth also works for registry paths", func(t *testing.T) { + token := generateRegistryToken(t, "build-789") + + req := httptest.NewRequest(http.MethodHead, "/v2/builds/build-789/manifests/latest", nil) + req.Header.Set("Authorization", "Bearer "+token) + req.RemoteAddr = "8.8.8.8:12345" + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code, "Bearer auth should also work for registry paths") + }) +} + +// basicAuth creates a Basic auth value (base64 of "token:") +func basicAuth(token string) string { + return base64.StdEncoding.EncodeToString([]byte(token + ":")) +} + func TestIsInternalVMRequest(t *testing.T) { tests := []struct { name string From bfdb3d56ae0b4cb7c82d4d01265638a2c583d90a Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 27 Jan 2026 21:32:06 -0500 Subject: [PATCH 4/6] fix(registry): add WWW-Authenticate header and remove prod IP fallback Root cause: Registry was returning 401 without WWW-Authenticate header, so BuildKit didn't know to send credentials from Docker config. Changes: - Add WWW-Authenticate: Basic realm="registry" to 401 responses - Remove production subnet (172.30.x.x) from IP fallback (staging 10.100.x.x still has fallback as safety net) - Builder agent: write Docker config to both /home/builder/.docker and /root/.docker to ensure BuildKit finds it - Add tests for BuildKit auth flow simulation This enables proper token-based registry auth in production. --- lib/builds/builder_agent/main.go | 26 +++++++++++------ lib/middleware/oapi_auth.go | 16 ++++++---- lib/middleware/oapi_auth_test.go | 50 +++++++++++++++++++++++++++----- 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index 843aca2..2bb02b3 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -516,19 +516,27 @@ func setupRegistryAuth(registryURL, token string) error { return fmt.Errorf("marshal docker config: %w", err) } - // Ensure ~/.docker directory exists - dockerDir := "/home/builder/.docker" - if err := os.MkdirAll(dockerDir, 0700); err != nil { - return fmt.Errorf("create docker config dir: %w", err) + // Write config to multiple locations to ensure BuildKit finds it + // buildctl-daemonless.sh may run buildkitd with different user/env + configDirs := []string{ + "/home/builder/.docker", // Builder user home + "/root/.docker", // Root user (buildkitd may run as root) } - // Write config.json - configPath := filepath.Join(dockerDir, "config.json") - if err := os.WriteFile(configPath, configData, 0600); err != nil { - return fmt.Errorf("write docker config: %w", err) + for _, dockerDir := range configDirs { + if err := os.MkdirAll(dockerDir, 0700); err != nil { + log.Printf("Warning: failed to create %s: %v", dockerDir, err) + continue + } + + configPath := filepath.Join(dockerDir, "config.json") + if err := os.WriteFile(configPath, configData, 0600); err != nil { + log.Printf("Warning: failed to write %s: %v", configPath, err) + continue + } + log.Printf("Registry auth configured at %s", configPath) } - log.Printf("Registry auth configured for %s", registryURL) return nil } diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index b94f261..da39c11 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -131,6 +131,13 @@ func OapiAuthenticationFunc(jwtSecret string) openapi3filter.AuthenticationFunc // that returns consistent error responses. func OapiErrorHandler(w http.ResponseWriter, message string, statusCode int) { w.Header().Set("Content-Type", "application/json") + + // For 401 responses, include WWW-Authenticate header so Docker/BuildKit + // knows to send credentials from the Docker config + if statusCode == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + } + w.WriteHeader(statusCode) // Return a simple JSON error response matching our Error schema @@ -212,11 +219,10 @@ func isInternalVMRequest(r *http.Request) bool { ip = ip[:idx] } - // Check if it's from the VM network - // Different environments use different subnets: - // - 10.100.x.x, 10.102.x.x: staging/dev environments - // - 172.30.x.x: production environment - return strings.HasPrefix(ip, "10.100.") || strings.HasPrefix(ip, "10.102.") || strings.HasPrefix(ip, "172.30.") + // Check if it's from the VM network (staging/dev only) + // Production (172.30.x.x) should use token auth, not IP fallback + // TODO: Remove this fallback entirely once token auth is verified working + return strings.HasPrefix(ip, "10.100.") || strings.HasPrefix(ip, "10.102.") } // extractRepoFromPath extracts the repository name from a registry path. diff --git a/lib/middleware/oapi_auth_test.go b/lib/middleware/oapi_auth_test.go index b45493a..b58589f 100644 --- a/lib/middleware/oapi_auth_test.go +++ b/lib/middleware/oapi_auth_test.go @@ -339,18 +339,20 @@ func TestJwtAuth_RegistryPaths(t *testing.T) { assert.Equal(t, http.StatusOK, rr.Code, "internal staging IP should allow access via fallback") }) - t.Run("no token but internal production IP allows access via fallback", func(t *testing.T) { + t.Run("production IP without token returns 401 (no fallback for prod)", func(t *testing.T) { req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) // No Authorization header - req.RemoteAddr = "172.30.16.101:42700" // Production subnet + req.RemoteAddr = "172.30.16.101:42700" // Production subnet - should NOT fallback rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusOK, rr.Code, "internal production IP should allow access via fallback") + // Production should require token auth, not IP fallback + assert.Equal(t, http.StatusUnauthorized, rr.Code, "production IP should require token auth") + assert.Equal(t, `Basic realm="registry"`, rr.Header().Get("WWW-Authenticate")) }) - t.Run("no token and external IP returns 401", func(t *testing.T) { + t.Run("no token and external IP returns 401 with WWW-Authenticate header", func(t *testing.T) { req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) // No Authorization header req.RemoteAddr = "8.8.8.8:12345" // External IP @@ -360,6 +362,9 @@ func TestJwtAuth_RegistryPaths(t *testing.T) { assert.Equal(t, http.StatusUnauthorized, rr.Code, "external IP without token should be rejected") assert.Contains(t, rr.Body.String(), "registry authentication required") + // WWW-Authenticate header is required for Docker/BuildKit to send credentials + assert.Equal(t, `Basic realm="registry"`, rr.Header().Get("WWW-Authenticate"), + "401 response must include WWW-Authenticate header for Docker auth") }) t.Run("invalid token but internal IP allows access via fallback", func(t *testing.T) { @@ -423,6 +428,35 @@ func TestJwtAuth_RegistryPaths(t *testing.T) { assert.Equal(t, http.StatusOK, rr.Code, "Bearer auth should also work for registry paths") }) + + t.Run("simulates BuildKit auth flow: 401 then retry with credentials", func(t *testing.T) { + // This simulates what BuildKit should do: + // 1. First request without auth -> 401 with WWW-Authenticate + // 2. Second request with auth -> 200 + + token := generateRegistryToken(t, "build-flow-test") + + // Step 1: Request without auth (external IP) + req1 := httptest.NewRequest(http.MethodHead, "/v2/builds/build-flow-test/manifests/latest", nil) + req1.RemoteAddr = "8.8.8.8:12345" // External IP, no fallback + + rr1 := httptest.NewRecorder() + handler.ServeHTTP(rr1, req1) + + assert.Equal(t, http.StatusUnauthorized, rr1.Code, "first request without auth should get 401") + assert.Equal(t, `Basic realm="registry"`, rr1.Header().Get("WWW-Authenticate"), + "401 must include WWW-Authenticate to trigger client auth") + + // Step 2: Retry with Basic auth (what Docker/BuildKit does after seeing WWW-Authenticate) + req2 := httptest.NewRequest(http.MethodHead, "/v2/builds/build-flow-test/manifests/latest", nil) + req2.Header.Set("Authorization", "Basic "+basicAuth(token)) + req2.RemoteAddr = "8.8.8.8:12345" + + rr2 := httptest.NewRecorder() + handler.ServeHTTP(rr2, req2) + + assert.Equal(t, http.StatusOK, rr2.Code, "retry with auth should succeed") + }) } // basicAuth creates a Basic auth value (base64 of "token:") @@ -436,13 +470,13 @@ func TestIsInternalVMRequest(t *testing.T) { remoteAddr string expected bool }{ - // Staging/dev subnets + // Staging/dev subnets (fallback allowed) {"staging 10.100.x.x", "10.100.1.50:12345", true}, {"staging 10.102.x.x", "10.102.5.100:54321", true}, - // Production subnet - {"production 172.30.x.x", "172.30.16.101:42700", true}, - {"production 172.30.0.x", "172.30.0.50:8080", true}, + // Production subnet (NO fallback - must use token auth) + {"production 172.30.x.x requires token", "172.30.16.101:42700", false}, + {"production 172.30.0.x requires token", "172.30.0.50:8080", false}, // External IPs (should be rejected) {"external 192.168.x.x", "192.168.1.100:8080", false}, From 63d5fb39b5b9b49de64a37602f9fbf0769fb90b8 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 27 Jan 2026 21:43:24 -0500 Subject: [PATCH 5/6] fix(registry): re-enable production IP fallback BuildKit with registry.insecure=true doesn't do WWW-Authenticate challenge-response flow - it just fails on 401 without retrying with credentials. Re-enable IP fallback for production (172.30.x.x) until we find a way to make BuildKit send auth proactively. --- 1 | 0 dump.rdb | Bin 0 -> 89 bytes lib/middleware/oapi_auth.go | 9 +++++---- lib/middleware/oapi_auth_test.go | 18 +++++++++--------- 4 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 1 create mode 100644 dump.rdb diff --git a/1 b/1 new file mode 100644 index 0000000..e69de29 diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..e45aa9f125c09808c954871d11f6fdcb5056fbd4 GIT binary patch literal 89 zcmWG?b@2=~FfcUu#aWb^l3A=5xUrsf{Ha9Wt*7e``#nr>2Jaq0nv|4Uy?YTU;9u?Yb22PL5Z literal 0 HcmV?d00001 diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index da39c11..837d1d5 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -219,10 +219,11 @@ func isInternalVMRequest(r *http.Request) bool { ip = ip[:idx] } - // Check if it's from the VM network (staging/dev only) - // Production (172.30.x.x) should use token auth, not IP fallback - // TODO: Remove this fallback entirely once token auth is verified working - return strings.HasPrefix(ip, "10.100.") || strings.HasPrefix(ip, "10.102.") + // Check if it's from the VM network + // BuildKit with registry.insecure=true doesn't do WWW-Authenticate challenge-response, + // so we need IP fallback for all internal subnets until we find a way to make + // BuildKit send auth proactively + return strings.HasPrefix(ip, "10.100.") || strings.HasPrefix(ip, "10.102.") || strings.HasPrefix(ip, "172.30.") } // extractRepoFromPath extracts the repository name from a registry path. diff --git a/lib/middleware/oapi_auth_test.go b/lib/middleware/oapi_auth_test.go index b58589f..efa9f1a 100644 --- a/lib/middleware/oapi_auth_test.go +++ b/lib/middleware/oapi_auth_test.go @@ -339,17 +339,17 @@ func TestJwtAuth_RegistryPaths(t *testing.T) { assert.Equal(t, http.StatusOK, rr.Code, "internal staging IP should allow access via fallback") }) - t.Run("production IP without token returns 401 (no fallback for prod)", func(t *testing.T) { + t.Run("no token but internal production IP allows access via fallback", func(t *testing.T) { req := httptest.NewRequest(http.MethodHead, "/v2/builds/any-build/manifests/latest", nil) // No Authorization header - req.RemoteAddr = "172.30.16.101:42700" // Production subnet - should NOT fallback + req.RemoteAddr = "172.30.16.101:42700" // Production subnet rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - // Production should require token auth, not IP fallback - assert.Equal(t, http.StatusUnauthorized, rr.Code, "production IP should require token auth") - assert.Equal(t, `Basic realm="registry"`, rr.Header().Get("WWW-Authenticate")) + // BuildKit with insecure registries doesn't do WWW-Authenticate challenge-response, + // so we need IP fallback for production + assert.Equal(t, http.StatusOK, rr.Code, "internal production IP should allow access via fallback") }) t.Run("no token and external IP returns 401 with WWW-Authenticate header", func(t *testing.T) { @@ -470,13 +470,13 @@ func TestIsInternalVMRequest(t *testing.T) { remoteAddr string expected bool }{ - // Staging/dev subnets (fallback allowed) + // Staging/dev subnets {"staging 10.100.x.x", "10.100.1.50:12345", true}, {"staging 10.102.x.x", "10.102.5.100:54321", true}, - // Production subnet (NO fallback - must use token auth) - {"production 172.30.x.x requires token", "172.30.16.101:42700", false}, - {"production 172.30.0.x requires token", "172.30.0.50:8080", false}, + // Production subnet (fallback needed because BuildKit doesn't do WWW-Authenticate) + {"production 172.30.x.x", "172.30.16.101:42700", true}, + {"production 172.30.0.x", "172.30.0.50:8080", true}, // External IPs (should be rejected) {"external 192.168.x.x", "192.168.1.100:8080", false}, From 122280e0941f71a2df02c5babdff4ca2a9554e27 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 27 Jan 2026 21:43:32 -0500 Subject: [PATCH 6/6] chore: remove accidentally committed files --- 1 | 0 dump.rdb | Bin 89 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 1 delete mode 100644 dump.rdb diff --git a/1 b/1 deleted file mode 100644 index e69de29..0000000 diff --git a/dump.rdb b/dump.rdb deleted file mode 100644 index e45aa9f125c09808c954871d11f6fdcb5056fbd4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 89 zcmWG?b@2=~FfcUu#aWb^l3A=5xUrsf{Ha9Wt*7e``#nr>2Jaq0nv|4Uy?YTU;9u?Yb22PL5Z