From 908575611e66a649a9affef69f9227c861e72e2b Mon Sep 17 00:00:00 2001 From: scme0 Date: Fri, 25 Jul 2025 11:09:13 +0200 Subject: [PATCH 1/5] fix? --- util/exec/exec.go | 207 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 202 insertions(+), 5 deletions(-) diff --git a/util/exec/exec.go b/util/exec/exec.go index 198c04cb482d6..0df69aeba7f9b 100644 --- a/util/exec/exec.go +++ b/util/exec/exec.go @@ -1,27 +1,35 @@ package exec import ( + "bytes" + "errors" "fmt" "os" "os/exec" "strconv" "strings" + "syscall" "time" "unicode" "github.com/argoproj/gitops-engine/pkg/utils/tracing" - argoexec "github.com/argoproj/pkg/exec" + "github.com/sirupsen/logrus" "github.com/argoproj/argo-cd/v3/util/log" + "github.com/argoproj/argo-cd/v3/util/rand" ) -var timeout time.Duration +var ( + timeout time.Duration + fatalTimeout time.Duration + Unredacted = Redact(nil) +) type ExecRunOpts struct { // Redactor redacts tokens from the output Redactor func(text string) string // TimeoutBehavior configures what to do in case of timeout - TimeoutBehavior argoexec.TimeoutBehavior + TimeoutBehavior TimeoutBehavior // SkipErrorLogging determines whether to skip logging of execution errors (rc > 0) SkipErrorLogging bool // CaptureStderr determines whether to capture stderr in addition to stdout @@ -38,6 +46,10 @@ func initTimeout() { if err != nil { timeout = 90 * time.Second } + fatalTimeout, err = time.ParseDuration(os.Getenv("ARGOCD_EXEC_FATAL_TIMEOUT")) + if err != nil { + fatalTimeout = 10 * time.Second + } } func Run(cmd *exec.Cmd) (string, error) { @@ -50,7 +62,7 @@ func RunWithRedactor(cmd *exec.Cmd, redactor func(text string) string) (string, } func RunWithExecRunOpts(cmd *exec.Cmd, opts ExecRunOpts) (string, error) { - cmdOpts := argoexec.CmdOpts{Timeout: timeout, Redactor: opts.Redactor, TimeoutBehavior: opts.TimeoutBehavior, SkipErrorLogging: opts.SkipErrorLogging} + cmdOpts := CmdOpts{Timeout: timeout, FatalTimeout: fatalTimeout, Redactor: opts.Redactor, TimeoutBehavior: opts.TimeoutBehavior, SkipErrorLogging: opts.SkipErrorLogging} span := tracing.NewLoggingTracer(log.NewLogrusLogger(log.NewWithCurrentConfig())).StartSpan(fmt.Sprintf("exec %v", cmd.Args[0])) span.SetBaggageItem("dir", cmd.Dir) if cmdOpts.Redactor != nil { @@ -59,7 +71,7 @@ func RunWithExecRunOpts(cmd *exec.Cmd, opts ExecRunOpts) (string, error) { span.SetBaggageItem("args", fmt.Sprintf("%v", cmd.Args)) } defer span.Finish() - return argoexec.RunCommandExt(cmd, cmdOpts) + return RunCommandExt(cmd, cmdOpts) } // GetCommandArgsToLog represents the given command in a way that we can copy-and-paste into a terminal @@ -88,3 +100,188 @@ func GetCommandArgsToLog(cmd *exec.Cmd) string { args := strings.Join(argsToLog, " ") return args } + +type CmdError struct { + Args string + Stderr string + Cause error +} + +func (ce *CmdError) Error() string { + res := fmt.Sprintf("`%v` failed %v", ce.Args, ce.Cause) + if ce.Stderr != "" { + res = fmt.Sprintf("%s: %s", res, ce.Stderr) + } + return res +} + +func (ce *CmdError) String() string { + return ce.Error() +} + +func newCmdError(args string, cause error, stderr string) *CmdError { + return &CmdError{Args: args, Stderr: stderr, Cause: cause} +} + +// TimeoutBehavior defines behavior for when the command takes longer than the passed in timeout to exit +// By default, SIGKILL is sent to the process and it is not waited upon +type TimeoutBehavior struct { + // Signal determines the signal to send to the process + Signal syscall.Signal + // ShouldWait determines whether to wait for the command to exit once timeout is reached + ShouldWait bool +} + +type CmdOpts struct { + // Timeout determines how long to wait for the command to exit + Timeout time.Duration + // FatalTimeout is the amount of additional time to wait after Timeout before fatal SIGKILL + FatalTimeout time.Duration + // Redactor redacts tokens from the output + Redactor func(text string) string + // TimeoutBehavior configures what to do in case of timeout + TimeoutBehavior TimeoutBehavior + // SkipErrorLogging defines whether to skip logging of execution errors (rc > 0) + SkipErrorLogging bool + // CaptureStderr defines whether to capture stderr in addition to stdout + CaptureStderr bool +} + +var DefaultCmdOpts = CmdOpts{ + Timeout: time.Duration(0), + FatalTimeout: time.Duration(0), + Redactor: Unredacted, + TimeoutBehavior: TimeoutBehavior{syscall.SIGKILL, false}, + SkipErrorLogging: false, + CaptureStderr: false, +} + +func Redact(items []string) func(text string) string { + return func(text string) string { + for _, item := range items { + text = strings.ReplaceAll(text, item, "******") + } + return text + } +} + +// RunCommandExt is a convenience function to run/log a command and return/log stderr in an error upon +// failure. +func RunCommandExt(cmd *exec.Cmd, opts CmdOpts) (string, error) { + execId, err := rand.RandHex(5) + if err != nil { + return "", err + } + logCtx := logrus.WithFields(logrus.Fields{"execID": execId}) + + redactor := DefaultCmdOpts.Redactor + if opts.Redactor != nil { + redactor = opts.Redactor + } + + // log in a way we can copy-and-paste into a terminal + args := strings.Join(cmd.Args, " ") + logCtx.WithFields(logrus.Fields{"dir": cmd.Dir}).Info(redactor(args)) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + start := time.Now() + err = cmd.Start() + if err != nil { + return "", err + } + + done := make(chan error) + go func() { done <- cmd.Wait() }() + + // Start timers for timeout + timeout := DefaultCmdOpts.Timeout + fatalTimeout := DefaultCmdOpts.FatalTimeout + + if opts.Timeout != time.Duration(0) { + timeout = opts.Timeout + } + + if opts.FatalTimeout != time.Duration(0) { + fatalTimeout = opts.FatalTimeout + } + + var timoutCh <-chan time.Time + if timeout != 0 { + timoutCh = time.NewTimer(timeout).C + } + + var fatalTimeoutCh <-chan time.Time + if fatalTimeout != 0 { + fatalTimeoutCh = time.NewTimer(timeout + fatalTimeout).C + } + + timeoutBehavior := DefaultCmdOpts.TimeoutBehavior + fatalTimeoutBehaviour := syscall.SIGKILL + if opts.TimeoutBehavior.Signal != syscall.Signal(0) { + timeoutBehavior = opts.TimeoutBehavior + } + + select { + // noinspection ALL + case <-timoutCh: + // send timeout signal + _ = cmd.Process.Signal(timeoutBehavior.Signal) + // wait on timeout signal and fallback to fatal timeout signal + if timeoutBehavior.ShouldWait { + select { + case <-done: + case <-fatalTimeoutCh: + // upgrades to SIGKILL if cmd does not respect SIGTERM + _ = cmd.Process.Signal(fatalTimeoutBehaviour) + // now original cmd should exit immediately after SIGKILL + <-done + // return error with a marker indicating that cmd exited only after fatal SIGKILL + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + err = newCmdError(redactor(args), fmt.Errorf("fatal timeout after %v", timeout+fatalTimeout), "") + logCtx.Error(err.Error()) + return strings.TrimSuffix(output, "\n"), err + } + } + // either did not wait for timeout or cmd did respect SIGTERM + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + err = newCmdError(redactor(args), fmt.Errorf("timeout after %v", timeout), "") + logCtx.Error(err.Error()) + return strings.TrimSuffix(output, "\n"), err + case err := <-done: + if err != nil { + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + err := newCmdError(redactor(args), errors.New(redactor(err.Error())), strings.TrimSpace(redactor(stderr.String()))) + if !opts.SkipErrorLogging { + logCtx.Error(err.Error()) + } + return strings.TrimSuffix(output, "\n"), err + } + } + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + + return strings.TrimSuffix(output, "\n"), nil +} + +func RunCommand(name string, opts CmdOpts, arg ...string) (string, error) { + return RunCommandExt(exec.Command(name, arg...), opts) +} From 655945740a2bf15123b161e453f85e41ca5ceed0 Mon Sep 17 00:00:00 2001 From: scme0 Date: Fri, 25 Jul 2025 11:26:15 +0200 Subject: [PATCH 2/5] fix --- test/e2e/fixture/applicationsets/utils/cmd.go | 2 +- test/e2e/fixture/cmd.go | 2 +- test/fixture/revision_metadata/author.go | 2 +- test/manifests_test.go | 2 +- util/exec/exec_test.go | 2 +- util/git/client.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/fixture/applicationsets/utils/cmd.go b/test/e2e/fixture/applicationsets/utils/cmd.go index 866bc72263b09..434e9454d7736 100644 --- a/test/e2e/fixture/applicationsets/utils/cmd.go +++ b/test/e2e/fixture/applicationsets/utils/cmd.go @@ -5,7 +5,7 @@ import ( "os/exec" "strings" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" ) func Run(workDir, name string, args ...string) (string, error) { diff --git a/test/e2e/fixture/cmd.go b/test/e2e/fixture/cmd.go index ca3af0daf1a2e..1d2e62d45ae41 100644 --- a/test/e2e/fixture/cmd.go +++ b/test/e2e/fixture/cmd.go @@ -5,7 +5,7 @@ import ( "os/exec" "strings" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" ) func Run(workDir, name string, args ...string) (string, error) { diff --git a/test/fixture/revision_metadata/author.go b/test/fixture/revision_metadata/author.go index d55de82303959..6bdd315967aac 100644 --- a/test/fixture/revision_metadata/author.go +++ b/test/fixture/revision_metadata/author.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" "github.com/argoproj/argo-cd/v3/util/errors" ) diff --git a/test/manifests_test.go b/test/manifests_test.go index c7a85fd794aad..8b6b7b60e9795 100644 --- a/test/manifests_test.go +++ b/test/manifests_test.go @@ -6,11 +6,11 @@ import ( "path/filepath" "testing" - argoexec "github.com/argoproj/pkg/exec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/argoproj/argo-cd/v3/test/fixture/test" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" ) func TestKustomizeVersion(t *testing.T) { diff --git a/util/exec/exec_test.go b/util/exec/exec_test.go index c41ee100f55ad..193d11a16f95f 100644 --- a/util/exec/exec_test.go +++ b/util/exec/exec_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/util/git/client.go b/util/git/client.go index f6ab375e50edf..8efa7ea41a7c9 100644 --- a/util/git/client.go +++ b/util/git/client.go @@ -19,7 +19,7 @@ import ( "github.com/Masterminds/semver/v3" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" "github.com/bmatcuk/doublestar/v4" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" From c07377bd4592f87ba61cbeca2fd6fa66f8987598 Mon Sep 17 00:00:00 2001 From: scme0 Date: Fri, 25 Jul 2025 11:42:46 +0200 Subject: [PATCH 3/5] fix --- util/rand/rand.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util/rand/rand.go b/util/rand/rand.go index 1e748bf93bb77..82603da176ada 100644 --- a/util/rand/rand.go +++ b/util/rand/rand.go @@ -2,6 +2,7 @@ package rand import ( "crypto/rand" + "encoding/hex" "fmt" "math/big" ) @@ -28,3 +29,12 @@ func StringFromCharset(n int, charset string) (string, error) { } return string(b), nil } + +// RandHex returns a cryptographically-secure pseudo-random alpha-numeric string of a given length +func RandHex(n int) (string, error) { + bytes := make([]byte, n/2+1) // we need one extra letter to discard + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes)[0:n], nil +} From b931415781eff895f0d0f2a9b3738fa80ba436c1 Mon Sep 17 00:00:00 2001 From: scme0 Date: Fri, 25 Jul 2025 12:08:13 +0200 Subject: [PATCH 4/5] try fix --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f338d3cd13802..f94240b4655d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG BASE_IMAGE=docker.io/library/ubuntu:25.04@sha256:10bb10bb062de665d4dc3e0ea36 # Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image # Also used as the image in CI jobs so needs all dependencies #################################################################################################### -FROM docker.io/library/golang:1.24.1@sha256:c5adecdb7b3f8c5ca3c88648a861882849cc8b02fed68ece31e25de88ad13418 AS builder +FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.24.1@sha256:c5adecdb7b3f8c5ca3c88648a861882849cc8b02fed68ece31e25de88ad13418 AS builder WORKDIR /tmp @@ -34,7 +34,7 @@ RUN ./install.sh helm && \ #################################################################################################### # Argo CD Base - used as the base for both the release and dev argocd images #################################################################################################### -FROM $BASE_IMAGE AS argocd-base +FROM --platform=$BUILDPLATFORM $BASE_IMAGE AS argocd-base LABEL org.opencontainers.image.source="https://github.com/argoproj/argo-cd" @@ -131,7 +131,7 @@ RUN GIT_COMMIT=$GIT_COMMIT \ #################################################################################################### # Final image #################################################################################################### -FROM argocd-base +FROM --platform=$BUILDPLATFORM argocd-base ENTRYPOINT ["/usr/bin/tini", "--"] COPY --from=argocd-build /go/src/github.com/argoproj/argo-cd/dist/argocd* /usr/local/bin/ From 22a16e8e7d4448f86d07f094837e3787af383260 Mon Sep 17 00:00:00 2001 From: scme0 Date: Fri, 25 Jul 2025 12:33:40 +0200 Subject: [PATCH 5/5] . --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f94240b4655d7..aa1d5f505e413 100644 --- a/Dockerfile +++ b/Dockerfile @@ -131,7 +131,7 @@ RUN GIT_COMMIT=$GIT_COMMIT \ #################################################################################################### # Final image #################################################################################################### -FROM --platform=$BUILDPLATFORM argocd-base +FROM argocd-base ENTRYPOINT ["/usr/bin/tini", "--"] COPY --from=argocd-build /go/src/github.com/argoproj/argo-cd/dist/argocd* /usr/local/bin/