support filtered regex (#1978)

motatoes · web-flow · commit d73a9e9688f7 · 2025-06-13T20:09:17.000-07:00
* support masking of sensitive values
diff --git a/cli/pkg/digger/digger_test.go b/cli/pkg/digger/digger_test.go
@@ -63,7 +63,7 @@ func (m *MockTerraformExecutor) Show(params []string, envs map[string]string, pl
 	return nonEmptyTerraformPlanJson, "", nil
 }
 
-func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string, planJsonFilePath string) (bool, string, string, error) {
+func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string, planJsonFilePath string, s *string) (bool, string, string, error) {
 	m.Commands = append(m.Commands, RunInfo{"Plan", strings.Join(params, " "), time.Now()})
 	return true, "", "", nil
 }
diff --git a/docs/ce/howto/masking-sensitive-values.mdx b/docs/ce/howto/masking-sensitive-values.mdx
@@ -0,0 +1,27 @@
+---
+title: "Masking sensitive values"
+---
+
+You can use a regular expression to ensure that senstivie outputs are masked from plan outputs and comments by digger.
+This can be done at the workflow level as follows:
+
+```
+workflows:
+  default:
+    plan:
+      filter_regex: "((?i)filterme:\\s\"?)[^\"]*"
+      steps:
+        - init
+        - plan
+```
+
+Currently this is only supported at the plan step. The filter_regex is a regular expression and it will mask any occurence of
+that expression in the logs and the comment, replacing it with `<REDACTED>`. For example the following terraform:
+
+```
+output "filterme" {
+  value = "example_secret: filterme: topsecret"
+}
+```
+
+![](/images/configuration/masking-sensitive-values.png)
diff --git a/docs/images/configuration/masking-sensitive-values.png b/docs/images/configuration/masking-sensitive-values.png
diff --git a/docs/mint.json b/docs/mint.json
@@ -92,6 +92,7 @@
         "ce/howto/generate-projects",
         "ce/howto/group-plans-by-source",
         "ce/howto/include-exclude-patterns",
+        "ce/howto/masking-sensitive-values",
         "ce/howto/multiacc-aws",
         "ce/howto/policy-overrides",
         "ce/howto/project-level-roles",
diff --git a/libs/digger_config/config.go b/libs/digger_config/config.go
@@ -94,7 +94,8 @@ type Step struct {
 }
 
 type Stage struct {
-	Steps []Step
+	Steps       []Step
+	FilterRegex *string
 }
 
 func defaultWorkflow() *Workflow {
diff --git a/libs/digger_config/converters.go b/libs/digger_config/converters.go
@@ -120,7 +120,9 @@ func copyTerraformEnvConfig(terraformEnvConfig *TerraformEnvConfigYaml) *Terrafo
 }
 
 func copyStage(stage *StageYaml) *Stage {
-	result := Stage{}
+	result := Stage{
+		FilterRegex: stage.FilterRegex,
+	}
 	result.Steps = make([]Step, len(stage.Steps))
 
 	for i, s := range stage.Steps {
diff --git a/libs/digger_config/digger_config.go b/libs/digger_config/digger_config.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/samber/lo"
@@ -546,6 +547,31 @@ func ValidateDiggerConfigYaml(configYaml *DiggerConfigYaml, fileName string) err
 		}
 	}
 
+	if configYaml.Workflows != nil {
+		for _, workflow := range configYaml.Workflows {
+			if workflow == nil {
+				continue
+			}
+			if workflow.Plan != nil && workflow.Plan.FilterRegex != nil {
+				_, err := regexp.Compile(*workflow.Plan.FilterRegex)
+				if err != nil {
+					slog.Error("invalid regex for plan filter",
+						"regex", *workflow.Plan.FilterRegex,
+						"error", err)
+					return fmt.Errorf("regex for plan filter is invalid: %v", err)
+				}
+			}
+			if workflow.Apply != nil && workflow.Apply.FilterRegex != nil {
+				_, err := regexp.Compile(*workflow.Apply.FilterRegex)
+				if err != nil {
+					slog.Error("invalid regex for apply filter",
+						"regex", *workflow.Apply.FilterRegex,
+						"error", err)
+					return fmt.Errorf("regex for apply filter is invalid: %v", err)
+				}
+			}
+		}
+	}
 	if configYaml.GenerateProjectsConfig != nil {
 		if configYaml.GenerateProjectsConfig.Include != "" &&
 			configYaml.GenerateProjectsConfig.Exclude != "" &&
diff --git a/libs/digger_config/digger_config_test.go b/libs/digger_config/digger_config_test.go
@@ -513,10 +513,12 @@ projects:
   workflow: my_custom_workflow
 workflows:
   my_custom_workflow:
-    steps:
-      - run: echo "run"
-      - init: terraform init
-      - plan: terraform plan
+    plan:
+      filter_regex: "myregex"
+      steps:
+        - run: echo "run"
+        - init: 
+        - plan:
 `
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
@@ -527,6 +529,8 @@ workflows:
 	assert.Equal(t, "my_custom_workflow", dg.Projects[0].Workflow)
 	_, ok := dg.Workflows["my_custom_workflow"]
 	assert.True(t, ok)
+	r := dg.Workflows["my_custom_workflow"].Plan.FilterRegex
+	assert.Equal(t, "myregex", *r)
 }
 
 func TestDiggerConfigCustomWorkflowMissingParams(t *testing.T) {
diff --git a/libs/digger_config/yaml.go b/libs/digger_config/yaml.go
@@ -78,7 +78,8 @@ func (s *StageYaml) ToCoreStage() Stage {
 }
 
 type StageYaml struct {
-	Steps []StepYaml `yaml:"steps"`
+	FilterRegex *string    `yaml:"filter_regex,omitempty"`
+	Steps       []StepYaml `yaml:"steps"`
 }
 
 type StepYaml struct {
diff --git a/libs/execution/execution.go b/libs/execution/execution.go
@@ -234,7 +234,7 @@ func (d DiggerExecutor) Plan() (*iac_utils.IacSummary, bool, bool, string, strin
 			// TODO remove those only for pulumi project
 			planArgs = append(planArgs, step.ExtraArgs...)
 
-			_, stdout, stderr, err := d.TerraformExecutor.Plan(planArgs, d.CommandEnvVars, d.PlanPathProvider.LocalPlanFilePath())
+			_, stdout, stderr, err := d.TerraformExecutor.Plan(planArgs, d.CommandEnvVars, d.PlanPathProvider.LocalPlanFilePath(), d.PlanStage.FilterRegex)
 			if err != nil {
 				return nil, false, false, "", "", fmt.Errorf("error executing plan: %v", err)
 			}
diff --git a/libs/execution/io_writer_filtered.go b/libs/execution/io_writer_filtered.go
@@ -0,0 +1,52 @@
+package execution
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+)
+
+// FilteringWriter wraps an io.Writer and filters sensitive content
+type FilteringWriter struct {
+	writer  io.Writer
+	buffer  *bytes.Buffer
+	pattern *regexp.Regexp
+}
+
+func NewFilteringWriter(w io.Writer, buf *bytes.Buffer, pattern *regexp.Regexp) *FilteringWriter {
+	return &FilteringWriter{
+		writer:  w,
+		buffer:  buf,
+		pattern: pattern,
+	}
+}
+
+func (fw *FilteringWriter) Write(p []byte) (n int, err error) {
+	// Filter the content
+
+	var filtered []byte
+	if fw.pattern == nil {
+		filtered = p
+	} else {
+		filtered = fw.pattern.ReplaceAll(p, []byte("<REDACTED>"))
+	}
+
+	if fw.writer != nil {
+		fmt.Printf("filtered: %v\n", filtered)
+		_, err = fw.writer.Write(filtered)
+		if err != nil {
+			return 0, err
+		}
+	}
+	if fw.buffer != nil {
+		fmt.Printf("filtered: %v\n", filtered)
+		_, err = fw.buffer.Write(filtered)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	// Return original length to maintain compatibility
+	return len(p), err
+}
diff --git a/libs/execution/io_writer_filtered_test.go b/libs/execution/io_writer_filtered_test.go
@@ -0,0 +1,28 @@
+package execution
+
+import (
+	"bytes"
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+func TestNewFilteringWriter(t *testing.T) {
+	var buf bytes.Buffer = bytes.Buffer{}
+	pattern := regexp.MustCompile("sensitive")
+	writer := NewFilteringWriter(nil, &buf, pattern)
+
+	if writer.buffer != &buf {
+		t.Errorf("Expected buffer to be %v, got %v", &buf, writer.buffer)
+	}
+	if writer.pattern != pattern {
+		t.Errorf("Expected pattern to be %v, got %v", pattern, writer.pattern)
+	}
+
+	writer.Write([]byte("sensitive"))
+	assert.Equal(t, "<REDACTED>", buf.String())
+
+	writer.Write([]byte("output"))
+	assert.Equal(t, "<REDACTED>output", buf.String())
+
+}
diff --git a/libs/execution/opentofu.go b/libs/execution/opentofu.go
@@ -18,7 +18,7 @@ type OpenTofu struct {
 func (tf OpenTofu) Init(params []string, envs map[string]string) (string, string, error) {
 	params = append(params, "-input=false")
 	params = append(params, "-no-color")
-	stdout, stderr, _, err := tf.runOpentofuCommand("init", true, envs, params...)
+	stdout, stderr, _, err := tf.runOpentofuCommand("init", true, envs, nil, params...)
 	return stdout, stderr, err
 }
 
@@ -37,11 +37,11 @@ func (tf OpenTofu) Apply(params []string, plan *string, envs map[string]string)
 	if plan != nil {
 		params = append(params, *plan)
 	}
-	stdout, stderr, _, err := tf.runOpentofuCommand("apply", true, envs, params...)
+	stdout, stderr, _, err := tf.runOpentofuCommand("apply", true, envs, nil, params...)
 	return stdout, stderr, err
 }
 
-func (tf OpenTofu) Plan(params []string, envs map[string]string, planArtefactFilePath string) (bool, string, string, error) {
+func (tf OpenTofu) Plan(params []string, envs map[string]string, planArtefactFilePath string, filterRegex *string) (bool, string, string, error) {
 	if tf.Workspace != "default" {
 		err := tf.switchToWorkspace(envs)
 		if err != nil {
@@ -56,7 +56,7 @@ func (tf OpenTofu) Plan(params []string, envs map[string]string, planArtefactFil
 	}
 	params = append(params, "-lock-timeout=3m")
 	params = append(append(append(params, "-input=false"), "-no-color"), "-detailed-exitcode")
-	stdout, stderr, statusCode, err := tf.runOpentofuCommand("plan", true, envs, params...)
+	stdout, stderr, statusCode, err := tf.runOpentofuCommand("plan", true, envs, filterRegex, params...)
 	if err != nil && statusCode != 2 {
 		return false, "", "", err
 	}
@@ -65,7 +65,7 @@ func (tf OpenTofu) Plan(params []string, envs map[string]string, planArtefactFil
 
 func (tf OpenTofu) Show(params []string, envs map[string]string, planArtefactFilePath string) (string, string, error) {
 	params = append(params, []string{"-no-color", "-json", planArtefactFilePath}...)
-	stdout, stderr, _, err := tf.runOpentofuCommand("show", false, envs, params...)
+	stdout, stderr, _, err := tf.runOpentofuCommand("show", false, envs, nil, params...)
 	if err != nil {
 		return "", "", err
 	}
@@ -83,33 +83,33 @@ func (tf OpenTofu) Destroy(params []string, envs map[string]string) (string, str
 		}
 	}
 	params = append(append(append(params, "-input=false"), "-no-color"), "-auto-approve")
-	stdout, stderr, _, err := tf.runOpentofuCommand("destroy", true, envs, params...)
+	stdout, stderr, _, err := tf.runOpentofuCommand("destroy", true, envs, nil, params...)
 	return stdout, stderr, err
 }
 
 func (tf OpenTofu) switchToWorkspace(envs map[string]string) error {
-	workspaces, _, _, err := tf.runOpentofuCommand("workspace", false, envs, "list")
+	workspaces, _, _, err := tf.runOpentofuCommand("workspace", false, envs, nil, "list")
 	if err != nil {
 		return err
 	}
 	workspaces = tf.formatOpentofuWorkspaces(workspaces)
 	if strings.Contains(workspaces, tf.Workspace) {
 		slog.Debug("Selecting existing workspace", "workspace", tf.Workspace)
-		_, _, _, err := tf.runOpentofuCommand("workspace", true, envs, "select", tf.Workspace)
+		_, _, _, err := tf.runOpentofuCommand("workspace", true, envs, nil, "select", tf.Workspace)
 		if err != nil {
 			return err
 		}
 	} else {
 		slog.Debug("Creating new workspace", "workspace", tf.Workspace)
-		_, _, _, err := tf.runOpentofuCommand("workspace", true, envs, "new", tf.Workspace)
+		_, _, _, err := tf.runOpentofuCommand("workspace", true, envs, nil, "new", tf.Workspace)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (tf OpenTofu) runOpentofuCommand(command string, printOutputToStdout bool, envs map[string]string, arg ...string) (string, string, int, error) {
+func (tf OpenTofu) runOpentofuCommand(command string, printOutputToStdout bool, envs map[string]string, filterRegex *string, arg ...string) (string, string, int, error) {
 	args := []string{command}
 	args = append(args, arg...)
 
@@ -122,14 +122,19 @@ func (tf OpenTofu) runOpentofuCommand(command string, printOutputToStdout bool,
 		}
 	}
 
+	regEx, err := stringToRegex(filterRegex)
+	if err != nil {
+		return "", "", 0, err
+	}
+
 	var mwout, mwerr io.Writer
 	var stdout, stderr bytes.Buffer
 	if printOutputToStdout {
-		mwout = io.MultiWriter(os.Stdout, &stdout)
-		mwerr = io.MultiWriter(os.Stderr, &stderr)
+		mwout = NewFilteringWriter(os.Stdout, &stdout, regEx)
+		mwerr = NewFilteringWriter(os.Stderr, &stderr, regEx)
 	} else {
-		mwout = io.Writer(&stdout)
-		mwerr = io.Writer(&stderr)
+		mwout = NewFilteringWriter(nil, &stdout, regEx)
+		mwerr = NewFilteringWriter(nil, &stderr, regEx)
 	}
 
 	cmd := exec.Command("tofu", expandedArgs...)
@@ -151,7 +156,7 @@ func (tf OpenTofu) runOpentofuCommand(command string, printOutputToStdout bool,
 	cmd.Stdout = mwout
 	cmd.Stderr = mwerr
 
-	err := cmd.Run()
+	err = cmd.Run()
 
 	// terraform plan can return 2 if there are changes to be applied, so we don't want to fail in that case
 	if err != nil && cmd.ProcessState.ExitCode() != 2 {
diff --git a/libs/execution/opentofu_test.go b/libs/execution/opentofu_test.go
@@ -20,7 +20,7 @@ func TestExecuteTofuPlan(t *testing.T) {
 
 	tf := OpenTofu{WorkingDir: dir, Workspace: "dev"}
 	tf.Init([]string{}, map[string]string{})
-	_, _, _, err := tf.Plan([]string{}, map[string]string{}, "")
+	_, _, _, err := tf.Plan([]string{}, map[string]string{}, "", nil)
 	assert.NoError(t, err)
 }
 
@@ -37,7 +37,7 @@ func TestExecuteTofuApply(t *testing.T) {
 
 	tf := OpenTofu{WorkingDir: dir, Workspace: "dev"}
 	tf.Init([]string{}, map[string]string{})
-	_, _, _, err := tf.Plan([]string{}, map[string]string{}, "")
+	_, _, _, err := tf.Plan([]string{}, map[string]string{}, "", nil)
 	assert.NoError(t, err)
 }
 
@@ -56,7 +56,7 @@ func TestExecuteTofuApplyDefaultWorkspace(t *testing.T) {
 	tf.Init([]string{}, map[string]string{})
 	var planArgs []string
 	planArgs = append(planArgs, "-out", "plan.tfplan")
-	tf.Plan(planArgs, map[string]string{}, "")
+	tf.Plan(planArgs, map[string]string{}, "", nil)
 	plan := "plan.tfplan"
 	_, _, err := tf.Apply([]string{}, &plan, map[string]string{})
 	assert.NoError(t, err)
diff --git a/libs/execution/pulumi.go b/libs/execution/pulumi.go
@@ -35,7 +35,7 @@ func (pl Pulumi) Apply(params []string, plan *string, envs map[string]string) (s
 	return stdout, stderr, err
 }
 
-func (pl Pulumi) Plan(params []string, envs map[string]string, planArtefactFilePath string) (bool, string, string, error) {
+func (pl Pulumi) Plan(params []string, envs map[string]string, planArtefactFilePath string, s *string) (bool, string, string, error) {
 	pl.selectStack()
 	params = append(params, []string{"--save-plan", planArtefactFilePath}...)
 	stdout, stderr, statusCode, err := pl.runPululmiCommand("preview", true, envs, params...)
diff --git a/libs/execution/terragrunt.go b/libs/execution/terragrunt.go
diff --git a/libs/execution/tf.go b/libs/execution/tf.go
diff --git a/libs/execution/tf_test.go b/libs/execution/tf_test.go
diff --git a/libs/execution/utils.go b/libs/execution/utils.go
diff --git a/libs/scheduler/jobs.go b/libs/scheduler/jobs.go
diff --git a/libs/scheduler/json_models.go b/libs/scheduler/json_models.go

Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,7 @@ func (m *MockTerraformExecutor) Show(params []string, envs map[string]string, pl`
`63`	`63`	`return nonEmptyTerraformPlanJson, "", nil`
`64`	`64`	`}`
`65`	`65`
`66`		`-func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string, planJsonFilePath string) (bool, string, string, error) {`
	`66`	`+func (m MockTerraformExecutor) Plan(params []string, envs map[string]string, planJsonFilePath string, s string) (bool, string, string, error) {`
`67`	`67`	`m.Commands = append(m.Commands, RunInfo{"Plan", strings.Join(params, " "), time.Now()})`
`68`	`68`	`return true, "", "", nil`
`69`	`69`	`}`
Original file line number	Diff line number	Diff line change
`@@ -94,7 +94,8 @@ type Step struct {`
`94`	`94`	`}`
`95`	`95`
`96`	`96`	`type Stage struct {`
`97`		`- Steps []Step`
	`97`	`+ Steps []Step`
	`98`	`+ FilterRegex *string`
`98`	`99`	`}`
`99`	`100`
`100`	`101`	`func defaultWorkflow() *Workflow {`
Original file line number	Diff line number	Diff line change
`@@ -120,7 +120,9 @@ func copyTerraformEnvConfig(terraformEnvConfig TerraformEnvConfigYaml) Terrafo`
`120`	`120`	`}`
`121`	`121`
`122`	`122`	`func copyStage(stage StageYaml) Stage {`
`123`		`- result := Stage{}`
	`123`	`+ result := Stage{`
	`124`	`+ FilterRegex: stage.FilterRegex,`
	`125`	`+ }`
`124`	`126`	`result.Steps = make([]Step, len(stage.Steps))`
`125`	`127`
`126`	`128`	`for i, s := range stage.Steps {`
Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,8 @@ func (s *StageYaml) ToCoreStage() Stage {`
`78`	`78`	`}`
`79`	`79`
`80`	`80`	`type StageYaml struct {`
`81`		- Steps []StepYaml `yaml:"steps"`
	`81`	+ FilterRegex *string `yaml:"filter_regex,omitempty"`
	`82`	+ Steps []StepYaml `yaml:"steps"`
`82`	`83`	`}`
`83`	`84`
`84`	`85`	`type StepYaml struct {`
Original file line number	Diff line number	Diff line change
`@@ -234,7 +234,7 @@ func (d DiggerExecutor) Plan() (*iac_utils.IacSummary, bool, bool, string, strin`
`234`	`234`	`// TODO remove those only for pulumi project`
`235`	`235`	`planArgs = append(planArgs, step.ExtraArgs...)`
`236`	`236`
`237`		`- _, stdout, stderr, err := d.TerraformExecutor.Plan(planArgs, d.CommandEnvVars, d.PlanPathProvider.LocalPlanFilePath())`
	`237`	`+ _, stdout, stderr, err := d.TerraformExecutor.Plan(planArgs, d.CommandEnvVars, d.PlanPathProvider.LocalPlanFilePath(), d.PlanStage.FilterRegex)`
`238`	`238`	`if err != nil {`
`239`	`239`	`return nil, false, false, "", "", fmt.Errorf("error executing plan: %v", err)`
`240`	`240`	`}`