Move from callbacks to webhooks for repos handling - again - on top of v0.6.136 this time (#2492)

* Bring back docs on local development

* Improve local dev docs

* Agent task for moving from callbacks to webhooks

* Move from callback to webhooks, again

Author: ZIJ

agent-tasks/webhook-based-repo-management.md

@@ -242,6 +242,7 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 
 		githubApiGroup := apiGroup.Group("/github")
 		githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi)
+		githubApiGroup.POST("/resync", controllers.ResyncGithubInstallationApi)
 
 		vcsApiGroup := apiGroup.Group("/connections")
 		vcsApiGroup.GET("/:id", controllers.GetVCSConnection)

backend/bootstrap/main.go

@@ -73,6 +73,24 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 				c.String(http.StatusAccepted, "Failed to handle webhook event.")
 				return
 			}
+		} else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" {
+			if err := handleInstallationUpsertEvent(c.Request.Context(), gh, event, appId64); err != nil {
+				slog.Error("Failed to handle installation upsert event", "error", err)
+				c.String(http.StatusAccepted, "Failed to handle webhook event.")
+				return
+			}
+		}
+	case *github.InstallationRepositoriesEvent:
+		slog.Info("Processing InstallationRepositoriesEvent",
+			"action", event.GetAction(),
+			"installationId", event.Installation.GetID(),
+			"added", len(event.RepositoriesAdded),
+			"removed", len(event.RepositoriesRemoved),
+		)
+		if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil {
+			slog.Error("Failed to handle installation repositories event", "error", err)
+			c.String(http.StatusAccepted, "Failed to handle webhook event.")
+			return
 		}
 	case *github.PushEvent:
 		slog.Info("Processing PushEvent",

backend/controllers/github.go

@@ -8,7 +8,10 @@ import (
 
 	"github.com/diggerhq/digger/backend/middleware"
 	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
+	ci_github "github.com/diggerhq/digger/libs/ci/github"
 	"github.com/gin-gonic/gin"
+	"github.com/google/go-github/v61/github"
 	"gorm.io/gorm"
 )
 
@@ -85,3 +88,84 @@ func LinkGithubInstallationToOrgApi(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{"status": "Successfully created Github installation link"})
 	return
 }
+
+func ResyncGithubInstallationApi(c *gin.Context) {
+	type ResyncInstallationRequest struct {
+		InstallationId string `json:"installation_id"`
+	}
+
+	var request ResyncInstallationRequest
+	if err := c.BindJSON(&request); err != nil {
+		slog.Error("Error binding JSON for resync", "error", err)
+		c.JSON(http.StatusBadRequest, gin.H{"status": "Invalid request format"})
+		return
+	}
+
+	installationId, err := strconv.ParseInt(request.InstallationId, 10, 64)
+	if err != nil {
+		slog.Error("Failed to convert InstallationId to int64", "installationId", request.InstallationId, "error", err)
+		c.JSON(http.StatusBadRequest, gin.H{"status": "installationID should be a valid integer"})
+		return
+	}
+
+	link, err := models.DB.GetGithubAppInstallationLink(installationId)
+	if err != nil {
+		slog.Error("Could not get installation link for resync", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not get installation link"})
+		return
+	}
+	if link == nil {
+		slog.Warn("Installation link not found for resync", "installationId", installationId)
+		c.JSON(http.StatusNotFound, gin.H{"status": "Installation link not found"})
+		return
+	}
+
+	// Get appId from an existing installation record
+	var installationRecord models.GithubAppInstallation
+	if err := models.DB.GormDB.Where("github_installation_id = ?", installationId).Order("updated_at desc").First(&installationRecord).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Warn("No installation records found for resync", "installationId", installationId)
+			c.JSON(http.StatusNotFound, gin.H{"status": "No installation records found"})
+			return
+		}
+		slog.Error("Failed to fetch installation record for resync", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not fetch installation records"})
+		return
+	}
+
+	appId := installationRecord.GithubAppId
+	ghProvider := utils.DiggerGithubRealClientProvider{}
+
+	client, _, err := ghProvider.Get(appId, installationId)
+	if err != nil {
+		slog.Error("Failed to create GitHub client for resync", "installationId", installationId, "appId", appId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to create GitHub client"})
+		return
+	}
+
+	repos, err := ci_github.ListGithubRepos(client)
+	if err != nil {
+		slog.Error("Failed to list repos for resync", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to list repos for resync"})
+		return
+	}
+
+	// Build synthetic InstallationEvent and call upsert handler
+	installationPayload := &github.Installation{
+		ID:    github.Int64(installationId),
+		AppID: github.Int64(appId),
+	}
+	resyncEvent := &github.InstallationEvent{
+		Installation: installationPayload,
+		Repositories: repos,
+	}
+
+	if err := handleInstallationUpsertEvent(c.Request.Context(), ghProvider, resyncEvent, appId); err != nil {
+		slog.Error("Resync failed", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Resync failed"})
+		return
+	}
+
+	slog.Info("Resync completed", "installationId", installationId, "repoCount", len(repos))
+	c.JSON(http.StatusOK, gin.H{"status": "Resync completed", "repoCount": len(repos)})
+}

backend/controllers/github_api.go

@@ -5,12 +5,9 @@ import (
 	"log/slog"
 	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/backend/segment"
-	"github.com/diggerhq/digger/backend/utils"
-	"github.com/diggerhq/digger/libs/ci/github"
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 )
@@ -28,29 +25,17 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 		c.String(http.StatusBadRequest, "installation_id parameter for github app is empty")
 		return
 	}
-	//setupAction := c.Request.URL.Query()["setup_action"][0]
+
+	// Code parameter is optional - GitHub doesn't always send it on re-authorization flows
 	codeParams, codeExists := c.Request.URL.Query()["code"]
-	if !codeExists || len(codeParams) == 0 {
-		slog.Error("There was no code in the url query parameters")
-		c.String(http.StatusBadRequest, "could not find the code query parameter for github app")
-		return
-	}
-	code := codeParams[0]
-	if len(code) < 1 {
-		slog.Error("Code parameter is empty")
-		c.String(http.StatusBadRequest, "code parameter for github app is empty")
-		return
+	code := ""
+	if codeExists && len(codeParams) > 0 && len(codeParams[0]) > 0 {
+		code = codeParams[0]
 	}
-	appId := c.Request.URL.Query().Get("state")
 
-	slog.Info("Processing GitHub app callback", "installationId", installationId, "appId", appId)
+	appId := c.Request.URL.Query().Get("state")
 
-	clientId, clientSecret, _, _, err := d.GithubClientProvider.FetchCredentials(appId)
-	if err != nil {
-		slog.Error("Could not fetch credentials for GitHub app", "appId", appId, "error", err)
-		c.String(http.StatusInternalServerError, "could not find credentials for github app")
-		return
-	}
+	slog.Info("Processing GitHub app callback", "installationId", installationId, "appId", appId, "hasCode", code != "")
 
 	installationId64, err := strconv.ParseInt(installationId, 10, 64)
 	if err != nil {
@@ -62,38 +47,47 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 		return
 	}
 
-	slog.Debug("Validating GitHub callback", "installationId", installationId64, "clientId", clientId)
+	// vcsOwner is used for analytics; we'll populate it if we can validate via OAuth
+	var vcsOwner string
 
-	result, installation, err := validateGithubCallback(d.GithubClientProvider, clientId, clientSecret, code, installationId64)
-	if !result {
-		slog.Error("Failed to validate installation ID",
-			"installationId", installationId64,
-			"error", err,
-		)
-		c.String(http.StatusInternalServerError, "Failed to validate installation_id.")
-		return
-	}
+	// If we have a code parameter, validate the callback via OAuth
+	// This provides additional security by confirming the user authorized the installation
+	if code != "" {
+		clientId, clientSecret, _, _, err := d.GithubClientProvider.FetchCredentials(appId)
+		if err != nil {
+			slog.Error("Could not fetch credentials for GitHub app", "appId", appId, "error", err)
+			c.String(http.StatusInternalServerError, "could not find credentials for github app")
+			return
+		}
 
-	// TODO: Lookup org in GithubAppInstallation by installationID if found use that installationID otherwise
-	// create a new org for this installationID
-	// retrieve org for current orgID
-	installationIdInt64, err := strconv.ParseInt(installationId, 10, 64)
-	if err != nil {
-		slog.Error("Failed to parse installation ID as int64",
-			"installationId", installationId,
-			"error", err,
+		slog.Debug("Validating GitHub callback", "installationId", installationId64, "clientId", clientId)
+
+		result, installation, err := validateGithubCallback(d.GithubClientProvider, clientId, clientSecret, code, installationId64)
+		if !result {
+			slog.Error("Failed to validate installation ID",
+				"installationId", installationId64,
+				"error", err,
+			)
+			c.String(http.StatusInternalServerError, "Failed to validate installation_id.")
+			return
+		}
+
+		if installation != nil && installation.Account != nil && installation.Account.Login != nil {
+			vcsOwner = *installation.Account.Login
+		}
+	} else {
+		slog.Info("No code parameter provided, skipping OAuth validation (repos will sync via webhook)",
+			"installationId", installationId64,
 		)
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "installationId could not be parsed"})
-		return
 	}
 
-	slog.Debug("Looking up GitHub app installation link", "installationId", installationIdInt64)
+	slog.Debug("Looking up GitHub app installation link", "installationId", installationId64)
 
 	var link *models.GithubAppInstallationLink
-	link, err = models.DB.GetGithubAppInstallationLink(installationIdInt64)
+	link, err = models.DB.GetGithubAppInstallationLink(installationId64)
 	if err != nil {
 		slog.Error("Error getting GitHub app installation link",
-			"installationId", installationIdInt64,
+			"installationId", installationId64,
 			"error", err,
 		)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting github app link"})
@@ -153,14 +147,10 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 	org := link.Organisation
 	orgId := link.OrganisationId
 
-	var vcsOwner string = ""
-	if installation.Account.Login != nil {
-		vcsOwner = *installation.Account.Login
-	}
-	// we have multiple repos here, we don't really want to send an track event for each repo, so we just send the vcs owner
+	// Track the installation event for analytics
 	segment.Track(*org, vcsOwner, "", "github", "vcs_repo_installed", map[string]string{})
 
-	// create a github installation link (org ID matched to installation ID)
+	// Ensure the installation link exists (idempotent operation)
 	_, err = models.DB.CreateGithubInstallationLink(org, installationId64)
 	if err != nil {
 		slog.Error("Error creating GitHub installation link",
@@ -172,120 +162,9 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 		return
 	}
 
-	slog.Debug("Getting GitHub client",
-		"appId", *installation.AppID,
-		"installationId", installationId64,
-	)
-
-	client, _, err := d.GithubClientProvider.Get(*installation.AppID, installationId64)
-	if err != nil {
-		slog.Error("Error retrieving GitHub client",
-			"appId", *installation.AppID,
-			"installationId", installationId64,
-			"error", err,
-		)
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error fetching organisation"})
-		return
-	}
-
-	// we get repos accessible to this installation
-	slog.Debug("Listing repositories for installation", "installationId", installationId64)
-
-	repos, err := github.ListGithubRepos(client)
-	if err != nil {
-		slog.Error("Failed to list existing repositories",
-			"installationId", installationId64,
-			"error", err,
-		)
-		c.String(http.StatusInternalServerError, "Failed to list existing repos: %v", err)
-		return
-	}
-
-	// resets all existing installations (soft delete)
-	slog.Debug("Resetting existing GitHub installations",
-		"installationId", installationId,
-	)
-
-	var AppInstallation models.GithubAppInstallation
-	err = models.DB.GormDB.Model(&AppInstallation).Where("github_installation_id=?", installationId).Update("status", models.GithubAppInstallDeleted).Error
-	if err != nil {
-		slog.Error("Failed to update GitHub installations",
-			"installationId", installationId,
-			"error", err,
-		)
-		c.String(http.StatusInternalServerError, "Failed to update github installations: %v", err)
-		return
-	}
-
-	// reset all existing repos (soft delete)
-	slog.Debug("Soft deleting existing repositories",
-		"orgId", orgId,
-	)
-
-	var ExistingRepos []models.Repo
-	err = models.DB.GormDB.Delete(ExistingRepos, "organisation_id=?", orgId).Error
-	if err != nil {
-		slog.Error("Could not delete repositories",
-			"orgId", orgId,
-			"error", err,
-		)
-		c.String(http.StatusInternalServerError, "could not delete repos: %v", err)
-		return
-	}
-
-	// here we mark repos that are available one by one
-	slog.Info("Adding repositories to organization",
-		"orgId", orgId,
-		"repoCount", len(repos),
-	)
-
-	for i, repo := range repos {
-		repoFullName := *repo.FullName
-		repoOwner := strings.Split(*repo.FullName, "/")[0]
-		repoName := *repo.Name
-		repoUrl := fmt.Sprintf("https://%v/%v", utils.GetGithubHostname(), repoFullName)
-
-		slog.Debug("Processing repository",
-			"index", i+1,
-			"repoFullName", repoFullName,
-			"repoOwner", repoOwner,
-			"repoName", repoName,
-		)
-
-		_, err := models.DB.GithubRepoAdded(
-			installationId64,
-			*installation.AppID,
-			*installation.Account.Login,
-			*installation.Account.ID,
-			repoFullName,
-		)
-		if err != nil {
-			slog.Error("Error recording GitHub repository",
-				"repoFullName", repoFullName,
-				"error", err,
-			)
-			c.String(http.StatusInternalServerError, "github repos added error: %v", err)
-			return
-		}
-
-		cloneUrl := *repo.CloneURL
-		defaultBranch := *repo.DefaultBranch
-
-		_, _, err = createOrGetDiggerRepoForGithubRepo(repoFullName, repoOwner, repoName, repoUrl, installationId64, *installation.AppID, defaultBranch, cloneUrl)
-		if err != nil {
-			slog.Error("Error creating or getting Digger repo",
-				"repoFullName", repoFullName,
-				"error", err,
-			)
-			c.String(http.StatusInternalServerError, "createOrGetDiggerRepoForGithubRepo error: %v", err)
-			return
-		}
-	}
-
-	slog.Info("GitHub app callback processed successfully",
+	slog.Info("GitHub app callback processed",
 		"installationId", installationId64,
 		"orgId", orgId,
-		"repoCount", len(repos),
 	)
 
 	c.HTML(http.StatusOK, "github_success.tmpl", gin.H{})