diff --git a/Dockerfile b/Dockerfile index f338d3cd13802..aa1d5f505e413 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG BASE_IMAGE=docker.io/library/ubuntu:25.04@sha256:10bb10bb062de665d4dc3e0ea36 # Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image # Also used as the image in CI jobs so needs all dependencies #################################################################################################### -FROM docker.io/library/golang:1.24.1@sha256:c5adecdb7b3f8c5ca3c88648a861882849cc8b02fed68ece31e25de88ad13418 AS builder +FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.24.1@sha256:c5adecdb7b3f8c5ca3c88648a861882849cc8b02fed68ece31e25de88ad13418 AS builder WORKDIR /tmp @@ -34,7 +34,7 @@ RUN ./install.sh helm && \ #################################################################################################### # Argo CD Base - used as the base for both the release and dev argocd images #################################################################################################### -FROM $BASE_IMAGE AS argocd-base +FROM --platform=$BUILDPLATFORM $BASE_IMAGE AS argocd-base LABEL org.opencontainers.image.source="https://github.com/argoproj/argo-cd" diff --git a/test/e2e/fixture/applicationsets/utils/cmd.go b/test/e2e/fixture/applicationsets/utils/cmd.go index 866bc72263b09..434e9454d7736 100644 --- a/test/e2e/fixture/applicationsets/utils/cmd.go +++ b/test/e2e/fixture/applicationsets/utils/cmd.go @@ -5,7 +5,7 @@ import ( "os/exec" "strings" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" ) func Run(workDir, name string, args ...string) (string, error) { diff --git a/test/e2e/fixture/cmd.go b/test/e2e/fixture/cmd.go index ca3af0daf1a2e..1d2e62d45ae41 100644 --- a/test/e2e/fixture/cmd.go +++ b/test/e2e/fixture/cmd.go @@ -5,7 +5,7 @@ import ( "os/exec" "strings" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" ) func Run(workDir, name string, args ...string) (string, error) { diff --git a/test/fixture/revision_metadata/author.go b/test/fixture/revision_metadata/author.go index d55de82303959..6bdd315967aac 100644 --- a/test/fixture/revision_metadata/author.go +++ b/test/fixture/revision_metadata/author.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" "github.com/argoproj/argo-cd/v3/util/errors" ) diff --git a/test/manifests_test.go b/test/manifests_test.go index c7a85fd794aad..8b6b7b60e9795 100644 --- a/test/manifests_test.go +++ b/test/manifests_test.go @@ -6,11 +6,11 @@ import ( "path/filepath" "testing" - argoexec "github.com/argoproj/pkg/exec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/argoproj/argo-cd/v3/test/fixture/test" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" ) func TestKustomizeVersion(t *testing.T) { diff --git a/util/exec/exec.go b/util/exec/exec.go index 198c04cb482d6..0df69aeba7f9b 100644 --- a/util/exec/exec.go +++ b/util/exec/exec.go @@ -1,27 +1,35 @@ package exec import ( + "bytes" + "errors" "fmt" "os" "os/exec" "strconv" "strings" + "syscall" "time" "unicode" "github.com/argoproj/gitops-engine/pkg/utils/tracing" - argoexec "github.com/argoproj/pkg/exec" + "github.com/sirupsen/logrus" "github.com/argoproj/argo-cd/v3/util/log" + "github.com/argoproj/argo-cd/v3/util/rand" ) -var timeout time.Duration +var ( + timeout time.Duration + fatalTimeout time.Duration + Unredacted = Redact(nil) +) type ExecRunOpts struct { // Redactor redacts tokens from the output Redactor func(text string) string // TimeoutBehavior configures what to do in case of timeout - TimeoutBehavior argoexec.TimeoutBehavior + TimeoutBehavior TimeoutBehavior // SkipErrorLogging determines whether to skip logging of execution errors (rc > 0) SkipErrorLogging bool // CaptureStderr determines whether to capture stderr in addition to stdout @@ -38,6 +46,10 @@ func initTimeout() { if err != nil { timeout = 90 * time.Second } + fatalTimeout, err = time.ParseDuration(os.Getenv("ARGOCD_EXEC_FATAL_TIMEOUT")) + if err != nil { + fatalTimeout = 10 * time.Second + } } func Run(cmd *exec.Cmd) (string, error) { @@ -50,7 +62,7 @@ func RunWithRedactor(cmd *exec.Cmd, redactor func(text string) string) (string, } func RunWithExecRunOpts(cmd *exec.Cmd, opts ExecRunOpts) (string, error) { - cmdOpts := argoexec.CmdOpts{Timeout: timeout, Redactor: opts.Redactor, TimeoutBehavior: opts.TimeoutBehavior, SkipErrorLogging: opts.SkipErrorLogging} + cmdOpts := CmdOpts{Timeout: timeout, FatalTimeout: fatalTimeout, Redactor: opts.Redactor, TimeoutBehavior: opts.TimeoutBehavior, SkipErrorLogging: opts.SkipErrorLogging} span := tracing.NewLoggingTracer(log.NewLogrusLogger(log.NewWithCurrentConfig())).StartSpan(fmt.Sprintf("exec %v", cmd.Args[0])) span.SetBaggageItem("dir", cmd.Dir) if cmdOpts.Redactor != nil { @@ -59,7 +71,7 @@ func RunWithExecRunOpts(cmd *exec.Cmd, opts ExecRunOpts) (string, error) { span.SetBaggageItem("args", fmt.Sprintf("%v", cmd.Args)) } defer span.Finish() - return argoexec.RunCommandExt(cmd, cmdOpts) + return RunCommandExt(cmd, cmdOpts) } // GetCommandArgsToLog represents the given command in a way that we can copy-and-paste into a terminal @@ -88,3 +100,188 @@ func GetCommandArgsToLog(cmd *exec.Cmd) string { args := strings.Join(argsToLog, " ") return args } + +type CmdError struct { + Args string + Stderr string + Cause error +} + +func (ce *CmdError) Error() string { + res := fmt.Sprintf("`%v` failed %v", ce.Args, ce.Cause) + if ce.Stderr != "" { + res = fmt.Sprintf("%s: %s", res, ce.Stderr) + } + return res +} + +func (ce *CmdError) String() string { + return ce.Error() +} + +func newCmdError(args string, cause error, stderr string) *CmdError { + return &CmdError{Args: args, Stderr: stderr, Cause: cause} +} + +// TimeoutBehavior defines behavior for when the command takes longer than the passed in timeout to exit +// By default, SIGKILL is sent to the process and it is not waited upon +type TimeoutBehavior struct { + // Signal determines the signal to send to the process + Signal syscall.Signal + // ShouldWait determines whether to wait for the command to exit once timeout is reached + ShouldWait bool +} + +type CmdOpts struct { + // Timeout determines how long to wait for the command to exit + Timeout time.Duration + // FatalTimeout is the amount of additional time to wait after Timeout before fatal SIGKILL + FatalTimeout time.Duration + // Redactor redacts tokens from the output + Redactor func(text string) string + // TimeoutBehavior configures what to do in case of timeout + TimeoutBehavior TimeoutBehavior + // SkipErrorLogging defines whether to skip logging of execution errors (rc > 0) + SkipErrorLogging bool + // CaptureStderr defines whether to capture stderr in addition to stdout + CaptureStderr bool +} + +var DefaultCmdOpts = CmdOpts{ + Timeout: time.Duration(0), + FatalTimeout: time.Duration(0), + Redactor: Unredacted, + TimeoutBehavior: TimeoutBehavior{syscall.SIGKILL, false}, + SkipErrorLogging: false, + CaptureStderr: false, +} + +func Redact(items []string) func(text string) string { + return func(text string) string { + for _, item := range items { + text = strings.ReplaceAll(text, item, "******") + } + return text + } +} + +// RunCommandExt is a convenience function to run/log a command and return/log stderr in an error upon +// failure. +func RunCommandExt(cmd *exec.Cmd, opts CmdOpts) (string, error) { + execId, err := rand.RandHex(5) + if err != nil { + return "", err + } + logCtx := logrus.WithFields(logrus.Fields{"execID": execId}) + + redactor := DefaultCmdOpts.Redactor + if opts.Redactor != nil { + redactor = opts.Redactor + } + + // log in a way we can copy-and-paste into a terminal + args := strings.Join(cmd.Args, " ") + logCtx.WithFields(logrus.Fields{"dir": cmd.Dir}).Info(redactor(args)) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + start := time.Now() + err = cmd.Start() + if err != nil { + return "", err + } + + done := make(chan error) + go func() { done <- cmd.Wait() }() + + // Start timers for timeout + timeout := DefaultCmdOpts.Timeout + fatalTimeout := DefaultCmdOpts.FatalTimeout + + if opts.Timeout != time.Duration(0) { + timeout = opts.Timeout + } + + if opts.FatalTimeout != time.Duration(0) { + fatalTimeout = opts.FatalTimeout + } + + var timoutCh <-chan time.Time + if timeout != 0 { + timoutCh = time.NewTimer(timeout).C + } + + var fatalTimeoutCh <-chan time.Time + if fatalTimeout != 0 { + fatalTimeoutCh = time.NewTimer(timeout + fatalTimeout).C + } + + timeoutBehavior := DefaultCmdOpts.TimeoutBehavior + fatalTimeoutBehaviour := syscall.SIGKILL + if opts.TimeoutBehavior.Signal != syscall.Signal(0) { + timeoutBehavior = opts.TimeoutBehavior + } + + select { + // noinspection ALL + case <-timoutCh: + // send timeout signal + _ = cmd.Process.Signal(timeoutBehavior.Signal) + // wait on timeout signal and fallback to fatal timeout signal + if timeoutBehavior.ShouldWait { + select { + case <-done: + case <-fatalTimeoutCh: + // upgrades to SIGKILL if cmd does not respect SIGTERM + _ = cmd.Process.Signal(fatalTimeoutBehaviour) + // now original cmd should exit immediately after SIGKILL + <-done + // return error with a marker indicating that cmd exited only after fatal SIGKILL + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + err = newCmdError(redactor(args), fmt.Errorf("fatal timeout after %v", timeout+fatalTimeout), "") + logCtx.Error(err.Error()) + return strings.TrimSuffix(output, "\n"), err + } + } + // either did not wait for timeout or cmd did respect SIGTERM + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + err = newCmdError(redactor(args), fmt.Errorf("timeout after %v", timeout), "") + logCtx.Error(err.Error()) + return strings.TrimSuffix(output, "\n"), err + case err := <-done: + if err != nil { + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + err := newCmdError(redactor(args), errors.New(redactor(err.Error())), strings.TrimSpace(redactor(stderr.String()))) + if !opts.SkipErrorLogging { + logCtx.Error(err.Error()) + } + return strings.TrimSuffix(output, "\n"), err + } + } + output := stdout.String() + if opts.CaptureStderr { + output += stderr.String() + } + logCtx.WithFields(logrus.Fields{"duration": time.Since(start)}).Debug(redactor(output)) + + return strings.TrimSuffix(output, "\n"), nil +} + +func RunCommand(name string, opts CmdOpts, arg ...string) (string, error) { + return RunCommandExt(exec.Command(name, arg...), opts) +} diff --git a/util/exec/exec_test.go b/util/exec/exec_test.go index c41ee100f55ad..193d11a16f95f 100644 --- a/util/exec/exec_test.go +++ b/util/exec/exec_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/util/git/client.go b/util/git/client.go index f6ab375e50edf..8efa7ea41a7c9 100644 --- a/util/git/client.go +++ b/util/git/client.go @@ -19,7 +19,7 @@ import ( "github.com/Masterminds/semver/v3" - argoexec "github.com/argoproj/pkg/exec" + argoexec "github.com/argoproj/argo-cd/v3/util/exec" "github.com/bmatcuk/doublestar/v4" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" diff --git a/util/rand/rand.go b/util/rand/rand.go index 1e748bf93bb77..82603da176ada 100644 --- a/util/rand/rand.go +++ b/util/rand/rand.go @@ -2,6 +2,7 @@ package rand import ( "crypto/rand" + "encoding/hex" "fmt" "math/big" ) @@ -28,3 +29,12 @@ func StringFromCharset(n int, charset string) (string, error) { } return string(b), nil } + +// RandHex returns a cryptographically-secure pseudo-random alpha-numeric string of a given length +func RandHex(n int) (string, error) { + bytes := make([]byte, n/2+1) // we need one extra letter to discard + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes)[0:n], nil +}