pulumi support in digger (#1790)

* pulumi support in digger

Author: motatoes

action.yml

@@ -65,6 +65,10 @@ inputs:
     description: Setup OpenToFu
     required: false
     default: 'false'
+  setup-pulumi:
+    description: Setup Pulumi
+    required: false
+    default: 'false'
   terragrunt-version:
     description: Terragrunt version
     required: false
@@ -73,6 +77,11 @@ inputs:
     description: OpenTofu version
     required: false
     default: v1.6.1
+  pulumi-version:
+    description: Pulumi version
+    required: false
+    default: v3.3.0
+
   setup-terraform:
     description: Setup terraform
     required: false
@@ -272,6 +281,12 @@ runs:
         tofu_wrapper: false
       if: inputs.setup-opentofu == 'true'
 
+    - name: Setup Pulumi
+      uses: pulumi/actions@v4
+      with:
+        tofu_version: ${{ inputs.pulumi-version }}
+      if: inputs.setup-pulumi == 'true'
+
     - name: Setup Checkov
       run: |
         python3 -m venv .venv

backend/controllers/projects.go

@@ -11,8 +11,8 @@ import (
 	"github.com/diggerhq/digger/libs/ci"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	"github.com/diggerhq/digger/libs/digger_config"
+	"github.com/diggerhq/digger/libs/iac_utils"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
-	"github.com/diggerhq/digger/libs/terraform_utils"
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
 	"log"
@@ -317,12 +317,13 @@ func RunHistoryForProject(c *gin.Context) {
 }
 
 type SetJobStatusRequest struct {
-	Status          string                                  `json:"status"`
-	Timestamp       time.Time                               `json:"timestamp"`
-	JobSummary      *terraform_utils.TerraformSummary       `json:"job_summary"`
-	Footprint       *terraform_utils.TerraformPlanFootprint `json:"job_plan_footprint"`
-	PrCommentUrl    string                                  `json:"pr_comment_url"`
-	TerraformOutput string                                  `json:"terraform_output"`
+	Status          string                      `json:"status"`
+	Timestamp       time.Time                   `json:"timestamp"`
+	JobSummary      *iac_utils.IacSummary       `json:"job_summary"`
+	Footprint       *iac_utils.IacPlanFootprint `json:"job_plan_footprint"`
+	PrCommentUrl    string                      `json:"pr_comment_url"`
+	TerraformOutput string                      `json:"terraform_output"`
+
 }
 
 func (d DiggerController) SetJobStatusForProject(c *gin.Context) {

backend/services/spec.go

@@ -106,7 +106,7 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
 	})
 	hasDuplicates := len(justNames) != len(lo.Uniq(justNames))
 	if hasDuplicates {
-		return nil, fmt.Errorf("could not load variables due to duplicates: %v", err)
+		return nil, fmt.Errorf("could not load variables due to duplicates")
 	}
 
 	batch := job.Batch

cli/pkg/digger/digger.go

@@ -23,7 +23,7 @@ import (
 	utils "github.com/diggerhq/digger/cli/pkg/utils"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	config "github.com/diggerhq/digger/libs/digger_config"
-	"github.com/diggerhq/digger/libs/terraform_utils"
+	"github.com/diggerhq/digger/libs/iac_utils"
 
 	"github.com/dominikbraun/graph"
 )
@@ -141,7 +141,9 @@ func RunJobs(jobs []orchestrator.Job, prService ci.PullRequestService, orgServic
 			terraformOutput = exectorResults[0].TerraformOutput
 		}
 		prNumber := *currentJob.PullRequestNumber
-		batchResult, err := backendApi.ReportProjectJobStatus(repoNameForBackendReporting, projectNameForBackendReporting, jobId, "succeeded", time.Now(), &summary, "", jobPrCommentUrl, terraformOutput)
+
+		iacUtils := iac_utils.GetIacUtilsIacType(currentJob.IacType())
+		batchResult, err := backendApi.ReportProjectJobStatus(repoNameForBackendReporting, projectNameForBackendReporting, jobId, "succeeded", time.Now(), &summary, "", jobPrCommentUrl, terraformOutput, iacUtils)
 		if err != nil {
 			log.Printf("error reporting Job status: %v.\n", err)
 			return false, false, fmt.Errorf("error while running command: %v", err)
@@ -211,13 +213,20 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 	}
 
 	var terraformExecutor execution.TerraformExecutor
+	var iacUtils iac_utils.IacUtils
 	projectPath := path.Join(workingDir, job.ProjectDir)
 	if job.Terragrunt {
 		terraformExecutor = execution.Terragrunt{WorkingDir: projectPath}
+		iacUtils = iac_utils.TerraformUtils{}
 	} else if job.OpenTofu {
 		terraformExecutor = execution.OpenTofu{WorkingDir: projectPath, Workspace: job.ProjectWorkspace}
+		iacUtils = iac_utils.TerraformUtils{}
+	} else if job.Pulumi {
+		terraformExecutor = execution.Pulumi{WorkingDir: projectPath, Stack: job.ProjectWorkspace}
+		iacUtils = iac_utils.PulumiUtils{}
 	} else {
 		terraformExecutor = execution.Terraform{WorkingDir: projectPath, Workspace: job.ProjectWorkspace}
+		iacUtils = iac_utils.TerraformUtils{}
 	}
 
 	commandRunner := execution.CommandRunner{}
@@ -244,6 +253,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 			Reporter:          reporter,
 			PlanStorage:       planStorage,
 			PlanPathProvider:  planPathProvider,
+			IacUtils:          iacUtils,
 		},
 	}
 	executor := diggerExecutor.Executor.(execution.DiggerExecutor)
@@ -289,7 +299,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 					planPolicyFormatter = coreutils.AsComment(summary)
 				}
 
-				planSummary, err := terraform_utils.GetTfSummarizePlan(planJsonOutput)
+				planSummary, err := iacUtils.GetSummarizePlan(planJsonOutput)
 				if err != nil {
 					log.Printf("Failed to summarize plan. %v", err)
 				}
@@ -588,6 +598,8 @@ func RunJob(
 			terraformExecutor = execution.Terragrunt{WorkingDir: projectPath}
 		} else if job.OpenTofu {
 			terraformExecutor = execution.OpenTofu{WorkingDir: projectPath, Workspace: job.ProjectWorkspace}
+		} else if job.Pulumi {
+			terraformExecutor = execution.Pulumi{WorkingDir: projectPath, Stack: job.ProjectWorkspace}
 		} else {
 			terraformExecutor = execution.Terraform{WorkingDir: projectPath, Workspace: job.ProjectWorkspace}
 		}

cli/pkg/digger/digger_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/diggerhq/digger/libs/ci"
 	"github.com/diggerhq/digger/libs/execution"
+	"github.com/diggerhq/digger/libs/iac_utils"
 	orchestrator "github.com/diggerhq/digger/libs/scheduler"
 	"os"
 	"sort"
@@ -55,13 +56,13 @@ func (m *MockTerraformExecutor) Destroy(params []string, envs map[string]string)
 	return "", "", nil
 }
 
-func (m *MockTerraformExecutor) Show(params []string, envs map[string]string) (string, string, error) {
+func (m *MockTerraformExecutor) Show(params []string, envs map[string]string, planJsonFilePath string) (string, string, error) {
 	nonEmptyTerraformPlanJson := "{\"format_version\":\"1.1\",\"terraform_version\":\"1.4.6\",\"planned_values\":{\"root_module\":{\"resources\":[{\"address\":\"null_resource.test\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"test\",\"provider_name\":\"registry.terraform.io/hashicorp/null\",\"schema_version\":0,\"values\":{\"id\":\"7587790946951100994\",\"triggers\":null},\"sensitive_values\":{}},{\"address\":\"null_resource.testx\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"testx\",\"provider_name\":\"registry.terraform.io/hashicorp/null\",\"schema_version\":0,\"values\":{\"triggers\":null},\"sensitive_values\":{}}]}},\"resource_changes\":[{\"address\":\"null_resource.test\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"test\",\"provider_name\":\"registry.terraform.io/hashicorp/null\",\"change\":{\"actions\":[\"no-op\"],\"before\":{\"id\":\"7587790946951100994\",\"triggers\":null},\"after\":{\"id\":\"7587790946951100994\",\"triggers\":null},\"after_unknown\":{},\"before_sensitive\":{},\"after_sensitive\":{}}},{\"address\":\"null_resource.testx\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"testx\",\"provider_name\":\"registry.terraform.io/hashicorp/null\",\"change\":{\"actions\":[\"create\"],\"before\":null,\"after\":{\"triggers\":null},\"after_unknown\":{\"id\":true},\"before_sensitive\":false,\"after_sensitive\":{}}}],\"prior_state\":{\"format_version\":\"1.0\",\"terraform_version\":\"1.4.6\",\"values\":{\"root_module\":{\"resources\":[{\"address\":\"null_resource.test\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"test\",\"provider_name\":\"registry.terraform.io/hashicorp/null\",\"schema_version\":0,\"values\":{\"id\":\"7587790946951100994\",\"triggers\":null},\"sensitive_values\":{}}]}}},\"configuration\":{\"provider_config\":{\"null\":{\"name\":\"null\",\"full_name\":\"registry.terraform.io/hashicorp/null\"}},\"root_module\":{\"resources\":[{\"address\":\"null_resource.test\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"test\",\"provider_config_key\":\"null\",\"schema_version\":0},{\"address\":\"null_resource.testx\",\"mode\":\"managed\",\"type\":\"null_resource\",\"name\":\"testx\",\"provider_config_key\":\"null\",\"schema_version\":0}]}}}\n"
 	m.Commands = append(m.Commands, RunInfo{"Show", strings.Join(params, " "), time.Now()})
 	return nonEmptyTerraformPlanJson, "", nil
 }
 
-func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string) (bool, string, string, error) {
+func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string, planJsonFilePath string) (bool, string, string, error) {
 	m.Commands = append(m.Commands, RunInfo{"Plan", strings.Join(params, " "), time.Now()})
 	return true, "", "", nil
 }
@@ -279,13 +280,14 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) {
 		Reporter:          reporter,
 		PlanStorage:       planStorage,
 		PlanPathProvider:  planPathProvider,
+		IacUtils:          iac_utils.TerraformUtils{},
 	}
 
 	executor.Apply()
 
 	commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage, planPathProvider)
 
-	assert.Equal(t, []string{"RetrievePlan plan", "Init ", "Apply -lock-timeout=3m", "PublishComment 1 <details ><summary>Apply output</summary>\n\n```terraform\n\n```\n</details>", "Run   echo"}, commandStrings)
+	assert.Equal(t, []string{"RetrievePlan plan", "Init ", "Apply ", "PublishComment 1 <details ><summary>Apply output</summary>\n\n```terraform\n\n```\n</details>", "Run   echo"}, commandStrings)
 }
 
 func TestCorrectCommandExecutionWhenDestroying(t *testing.T) {
@@ -368,6 +370,7 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) {
 		Reporter:          reporter,
 		PlanStorage:       planStorage,
 		PlanPathProvider:  planPathProvider,
+		IacUtils:          iac_utils.TerraformUtils{},
 	}
 
 	os.WriteFile(planPathProvider.LocalPlanFilePath(), []byte{123}, 0644)
@@ -377,7 +380,7 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) {
 
 	commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage, planPathProvider)
 
-	assert.Equal(t, []string{"Init ", "Plan -out plan -lock-timeout=3m", "Show -no-color -json plan", "StorePlanFile plan", "Run   echo"}, commandStrings)
+	assert.Equal(t, []string{"Init ", "Plan ", "Show ", "StorePlanFile plan", "Run   echo"}, commandStrings)
 }
 
 func allCommandsInOrderWithParams(terraformExecutor *MockTerraformExecutor, commandRunner *MockCommandRunner, prManager *MockPRManager, lock *MockProjectLock, planStorage *MockPlanStorage, planPathProvider *MockPlanPathProvider) []string {

cli/pkg/github/github.go

@@ -148,6 +148,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 			ProjectWorkspace:  projectConfig.Workspace,
 			Terragrunt:        projectConfig.Terragrunt,
 			OpenTofu:          projectConfig.OpenTofu,
+			Pulumi:            projectConfig.Pulumi,
 			Commands:          []string{command},
 			ApplyStage:        scheduler.ToConfigStage(workflow.Apply),
 			PlanStage:         scheduler.ToConfigStage(workflow.Plan),
@@ -180,6 +181,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 				ProjectWorkspace:   projectConfig.Workspace,
 				Terragrunt:         projectConfig.Terragrunt,
 				OpenTofu:           projectConfig.OpenTofu,
+				Pulumi:             projectConfig.Pulumi,
 				Commands:           []string{"digger drift-detect"},
 				ApplyStage:         scheduler.ToConfigStage(workflow.Apply),
 				PlanStage:          scheduler.ToConfigStage(workflow.Plan),

cli/pkg/spec/spec.go

@@ -17,7 +17,7 @@ import (
 
 func reportError(spec spec.Spec, backendApi backend2.Api, message string, err error) {
 	log.Printf(message)
-	_, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "", "")
+	_, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "", "", nil)
 	if reportingError != nil {
 		usage.ReportErrorAndExit(spec.VCS.RepoOwner, fmt.Sprintf("Failed to run commands. %v", err), 5)
 	}
@@ -131,7 +131,7 @@ func RunSpec(
 	jobs := []scheduler.Job{job}
 
 	fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
-	_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "", "")
+	_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "", "", nil)
 	if err != nil {
 		message := fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err)
 		reportError(spec, backendApi, message, err)
@@ -152,7 +152,7 @@ func RunSpec(
 	reportTerraformOutput := spec.Reporter.ReportTerraformOutput
 	allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, true, reportTerraformOutput, commentId, currentDir)
 	if !allAppliesSuccess || err != nil {
-		serializedBatch, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "", "")
+		serializedBatch, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "", "", nil)
 		if reportingError != nil {
 			message := fmt.Sprintf("Failed run commands. %s", err)
 			reportError(spec, backendApi, message, err)

ee/drift/controllers/ci_jobs.go

@@ -7,17 +7,17 @@ import (
 
 	"github.com/diggerhq/digger/ee/drift/dbmodels"
 	"github.com/diggerhq/digger/ee/drift/model"
-	"github.com/diggerhq/digger/libs/terraform_utils"
+	"github.com/diggerhq/digger/libs/iac_utils"
 	"github.com/gin-gonic/gin"
 )
 
 type SetJobStatusRequest struct {
-	Status          string                                  `json:"status"`
-	Timestamp       time.Time                               `json:"timestamp"`
-	JobSummary      *terraform_utils.TerraformSummary       `json:"job_summary"`
-	Footprint       *terraform_utils.TerraformPlanFootprint `json:"job_plan_footprint"`
-	PrCommentUrl    string                                  `json:"pr_comment_url"`
-	TerraformOutput string                                  `json:"terraform_output"`
+	Status          string                      `json:"status"`
+	Timestamp       time.Time                   `json:"timestamp"`
+	JobSummary      *iac_utils.IacSummary       `json:"job_summary"`
+	Footprint       *iac_utils.IacPlanFootprint `json:"job_plan_footprint"`
+	PrCommentUrl    string                      `json:"pr_comment_url"`
+	TerraformOutput string                      `json:"terraform_output"`
 }
 
 func (mc MainController) SetJobStatusForProject(c *gin.Context) {

libs/backendapi/backend.go

@@ -1,15 +1,15 @@
 package backendapi
 
 import (
+	"github.com/diggerhq/digger/libs/iac_utils"
 	"github.com/diggerhq/digger/libs/scheduler"
-	"github.com/diggerhq/digger/libs/terraform_utils"
 	"time"
 )
 
 type Api interface {
 	ReportProject(repo string, projectName string, configuration string) error
 	ReportProjectRun(repo string, projectName string, startedAt time.Time, endedAt time.Time, status string, command string, output string) error
-	ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error)
+	ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *iac_utils.IacSummary, planJson string, PrCommentUrl string, terraformOutput string, iacUtils iac_utils.IacUtils) (*scheduler.SerializedBatch, error)
 	UploadJobArtefact(zipLocation string) (*int, *string, error)
 	DownloadJobArtefact(downloadTo string) (*string, error)
 }

libs/backendapi/diggerapi.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/diggerhq/digger/libs/iac_utils"
 	"github.com/diggerhq/digger/libs/scheduler"
-	"github.com/diggerhq/digger/libs/terraform_utils"
 	"io"
 	"log"
 	"mime"
@@ -29,7 +29,7 @@ func (n NoopApi) ReportProjectRun(namespace string, projectName string, startedA
 	return nil
 }
 
-func (n NoopApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
+func (n NoopApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *iac_utils.IacSummary, planJson string, PrCommentUrl string, terraformOutput string, iacUtils iac_utils.IacUtils) (*scheduler.SerializedBatch, error) {
 	return nil, nil
 }
 
@@ -129,14 +129,14 @@ func (d DiggerApi) ReportProjectRun(namespace string, projectName string, starte
 	return nil
 }
 
-func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
+func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *iac_utils.IacSummary, planJson string, PrCommentUrl string, terraformOutput string, iacUtils iac_utils.IacUtils) (*scheduler.SerializedBatch, error) {
 	u, err := url.Parse(d.DiggerHost)
 	if err != nil {
 		log.Fatalf("Not able to parse digger cloud url: %v", err)
 	}
 
 	var planSummaryJson interface{}
-	var planFootprint *terraform_utils.TerraformPlanFootprint = &terraform_utils.TerraformPlanFootprint{}
+	var planFootprint = &iac_utils.IacPlanFootprint{}
 	if summary == nil {
 		log.Printf("Warning: nil passed to plan result, sending empty")
 		planSummaryJson = nil
@@ -145,7 +145,7 @@ func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId
 		planSummary := summary
 		planSummaryJson = planSummary.ToJson()
 		if planJson != "" {
-			planFootprint, err = terraform_utils.GetPlanFootprint(planJson)
+			planFootprint, err = iacUtils.GetPlanFootprint(planJson)
 			if err != nil {
 				log.Printf("Error, could not get footprint from json plan: %v", err)
 			}