✨ Various new features

.secrets.baseline

@@ -123,28 +123,28 @@
         "filename": "utils/config/service_configuration_test.go",
         "hashed_secret": "ddcec2f503a5d58f432a0beee3fb9544fa581f54",
         "is_verified": false,
-        "line_number": 33
+        "line_number": 35
       },
       {
         "type": "Secret Keyword",
         "filename": "utils/config/service_configuration_test.go",
         "hashed_secret": "7ca1cc114e7e5f955880bb96a5bf391b4dc20ab6",
         "is_verified": false,
-        "line_number": 497
+        "line_number": 533
       },
       {
         "type": "Secret Keyword",
         "filename": "utils/config/service_configuration_test.go",
         "hashed_secret": "11519c144be4850d95b34220a40030cbd5a36b57",
         "is_verified": false,
-        "line_number": 592
+        "line_number": 628
       },
       {
         "type": "Secret Keyword",
         "filename": "utils/config/service_configuration_test.go",
         "hashed_secret": "15fae91d8fa7f2c531c1cf3ddc745e1f4473c02d",
         "is_verified": false,
-        "line_number": 599
+        "line_number": 635
       }
     ],
     "utils/filesystem/filehash_test.go": [
@@ -272,5 +272,5 @@
       }
     ]
   },
-  "generated_at": "2025-07-31T17:26:46Z"
+  "generated_at": "2025-08-08T23:03:50Z"
 }

changes/20250808203905.feature

@@ -0,0 +1 @@
+:sparkles: [parallelisation] Added a way to register cancel functions to a close store to ensure everything is cancelled on close

changes/20250808204006.feature

@@ -0,0 +1 @@
+:sparkles: [commonerrors] Added a Join which is similar to errors.Join but following inline string convention

changes/20250808204222.feature

@@ -0,0 +1 @@
+:sparkles: [keyring] Added a module to store configuration in [system keyring service](https://github.com/zalando/go-keyring)

changes/20250808230944.feature

@@ -0,0 +1 @@
+:sparkles: `[config]` Added ability to read configuration from system keyring service

changes/20250811090744.bugfix

@@ -0,0 +1 @@
+:bug: [`maps`] Expand the `Flatten` function to support pointers

utils/commonerrors/errors.go

@@ -247,32 +247,56 @@ func isSpecialCase(target, specialErrorCase error, prefix string) bool {
 	return strings.TrimSuffix(target.Error(), underlyingErr.Error()) == prefix
 }
 
+// Join is similar to errors.Join but follows the common errors convention when printing
+func Join(errs ...error) error {
+	switch len(errs) {
+	case 0:
+		return nil
+	case 1:
+		return errs[0]
+	default:
+		err := multierror.Append(errs[0], errs[1:]...)
+		err.ErrorFormat = func(e []error) string {
+			builder := strings.Builder{}
+			_, _ = builder.WriteString(e[0].Error())
+			for i := range e[1:] {
+				if None(e[i+1], nil) {
+					_, _ = builder.WriteString(string(TypeReasonErrorSeparator))
+					_, _ = builder.WriteString(" ")
+					_, _ = builder.WriteString(e[i+1].Error())
+				}
+			}
+			return builder.String()
+		}
+		return err.ErrorOrNil()
+	}
+
+}
+
 // MarkAsFailure will tent an error as failure. It will retain its original error type but IsFailure should return true.
 func MarkAsFailure(err error) error {
 	if Any(err, nil, ErrFailed) {
 		return err
 	}
-	result := multierror.Append(err, ErrFailed)
-	result.ErrorFormat = func(e []error) string {
-		builder := strings.Builder{}
-		_, _ = builder.WriteString(failureStr)
-		for i := range e {
-			if None(e[i], nil, ErrFailed) {
-				_, _ = builder.WriteString(string(TypeReasonErrorSeparator))
-				_, _ = builder.WriteString(" ")
-				_, _ = builder.WriteString(e[i].Error())
-			}
-		}
-		return builder.String()
-	}
-	return result.ErrorOrNil()
+	return Join(ErrFailed, err)
 }
 
 // NewFailure creates a failure object.
 func NewFailure(msgFormat string, args ...any) error {
+	if len(args) == 0 {
+		return New(ErrFailed, msgFormat)
+	}
 	return Newf(ErrFailed, msgFormat, args...)
 }
 
+// NewWarningMessage creates a warning message.
+func NewWarningMessage(msgFormat string, args ...any) error {
+	if len(args) == 0 {
+		return New(ErrWarning, msgFormat)
+	}
+	return Newf(ErrWarning, msgFormat, args...)
+}
+
 // NewWarning will create a warning wrapper around an existing commonerror so that it can be easily recovered. If the
 // underlying error is not a commonerror then ok will be set to false
 func NewWarning(target error) (ok bool, err error) {
@@ -357,7 +381,7 @@ func WrapError(targetError, originalError error, msg string) error {
 		tErr = ErrUnknown
 	}
 	origErr := ConvertContextError(originalError)
-	if Any(origErr, ErrTimeout, ErrCancelled, ErrWarning, ErrFailed) {
+	if Any(origErr, ErrTimeout, ErrCancelled, ErrWarning, ErrFailed) || IsWarning(origErr) || IsFailure(origErr) {
 		tErr = origErr
 	}
 	if originalError == nil {

utils/commonerrors/errors_test.go

@@ -59,6 +59,14 @@ func TestIgnoreCorrespondTo(t *testing.T) {
 	assert.NoError(t, IgnoreCorrespondTo(ErrCondition, "condition"))
 }
 
+func TestJoin(t *testing.T) {
+	assert.True(t, Any(Join(ErrFailed, ErrMarshalling, ErrCancelled), ErrFailed))
+	assert.True(t, Any(Join(ErrFailed, ErrMarshalling, ErrCancelled), ErrCancelled))
+	assert.False(t, Any(Join(ErrFailed, ErrMarshalling, nil, ErrCancelled), nil))
+	assert.True(t, IsWarning(Join(ErrFailed, ErrMarshalling, NewWarningMessage(faker.Sentence()), ErrCancelled)))
+	require.NoError(t, Join(nil, nil, nil))
+}
+
 func TestContextErrorConversion(t *testing.T) {
 	defer goleak.VerifyNone(t)
 	task := func(ctx context.Context) {
@@ -126,6 +134,7 @@ func TestIsCommonError(t *testing.T) {
 
 func TestIsWarning(t *testing.T) {
 	assert.True(t, IsWarning(ErrWarning))
+	assert.True(t, IsWarning(NewWarningMessage(faker.Sentence())))
 	assert.False(t, IsWarning(ErrUnexpected))
 	assert.False(t, IsWarning(nil))
 	assert.True(t, IsWarning(fmt.Errorf("%w: i am a warning", ErrWarning)))

utils/config/service_configuration.go

@@ -6,6 +6,7 @@
 package config
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/ARM-software/golang-utils/utils/collection"
 	"github.com/ARM-software/golang-utils/utils/commonerrors"
 	"github.com/ARM-software/golang-utils/utils/field"
+	"github.com/ARM-software/golang-utils/utils/keyring"
 	"github.com/ARM-software/golang-utils/utils/reflection"
 )
 
@@ -35,6 +37,16 @@ func Load(envVarPrefix string, configurationToSet IServiceConfiguration, default
 	return LoadFromViper(viper.New(), envVarPrefix, configurationToSet, defaultConfiguration)
 }
 
+// LoadFromSystem is similar to Load but also fetches values from system's [keyring service](https://github.com/zalando/go-keyring).
+func LoadFromSystem(envVarPrefix string, configurationToSet IServiceConfiguration, defaultConfiguration IServiceConfiguration) error {
+	return LoadFromViperAndSystem(viper.New(), envVarPrefix, configurationToSet, defaultConfiguration)
+}
+
+// LoadFromViperAndSystem is the same as `LoadFromViper` but also fetches values from system's [keyring service](https://github.com/zalando/go-keyring).
+func LoadFromViperAndSystem(viperSession *viper.Viper, envVarPrefix string, configurationToSet IServiceConfiguration, defaultConfiguration IServiceConfiguration) error {
+	return LoadFromEnvironmentAndSystem(viperSession, envVarPrefix, configurationToSet, defaultConfiguration, "", true)
+}
+
 // LoadFromViper is the same as `Load` but instead of creating a new viper session, reuse the one provided.
 // Important note:
 // Viper's precedence order is maintained:
@@ -58,7 +70,22 @@ func LoadFromViper(viperSession *viper.Viper, envVarPrefix string, configuration
 // 5) key/value store
 // 6) default values (set via flag default values, or calls to `SetDefault` or via `defaultConfiguration` argument provided)
 // Nonetheless, when it comes to default values. It differs slightly from Viper as default values from the default Configuration (i.e. `defaultConfiguration` argument provided) will take precedence over defaults set via `SetDefault` or flags unless they are considered empty values according to `reflection.IsEmpty`.
-func LoadFromEnvironment(viperSession *viper.Viper, envVarPrefix string, configurationToSet IServiceConfiguration, defaultConfiguration IServiceConfiguration, configFile string) (err error) {
+func LoadFromEnvironment(viperSession *viper.Viper, envVarPrefix string, configurationToSet IServiceConfiguration, defaultConfiguration IServiceConfiguration, configFile string) error {
+	return LoadFromEnvironmentAndSystem(viperSession, envVarPrefix, configurationToSet, defaultConfiguration, configFile, false)
+}
+
+// LoadFromEnvironmentAndSystem is the same as `LoadFromEnvironment` but also gives the ability to load the configuration from system's [keyring service](https://github.com/zalando/go-keyring).
+// Important note:
+// Viper's precedence order is mostly maintained:
+// 1) values defined in keyring (if not empty and keyring is selected - this is the only difference from Viper)
+// 2) values set using explicit calls to `Set`
+// 3) flags
+// 4) environment (variables or `.env`)
+// 5) configuration file
+// 6) key/value store
+// 7) default values (set via flag default values, or calls to `SetDefault` or via `defaultConfiguration` argument provided)
+// Nonetheless, when it comes to default values. It differs slightly from Viper as default values from the default Configuration (i.e. `defaultConfiguration` argument provided) will take precedence over defaults set via `SetDefault` or flags unless they are considered empty values according to `reflection.IsEmpty`.
+func LoadFromEnvironmentAndSystem(viperSession *viper.Viper, envVarPrefix string, configurationToSet IServiceConfiguration, defaultConfiguration IServiceConfiguration, configFile string, useKeyring bool) (err error) {
 	// Load Defaults
 	var defaults map[string]interface{}
 	err = mapstructure.Decode(defaultConfiguration, &defaults)
@@ -91,6 +118,12 @@ func LoadFromEnvironment(viperSession *viper.Viper, envVarPrefix string, configu
 		err = commonerrors.WrapError(commonerrors.ErrMarshalling, err, "unable to fill configuration structure from the configuration session")
 		return
 	}
+	if useKeyring {
+		err = commonerrors.Ignore(keyring.FetchPointer[IServiceConfiguration](context.Background(), envVarPrefix, configurationToSet), commonerrors.ErrUnsupported)
+		if err != nil {
+			return
+		}
+	}
 	// Run validation
 	err = WrapValidationError(field.ToOptionalString(envVarPrefix), configurationToSet.Validate())
 	return

utils/config/service_configuration_test.go

@@ -5,6 +5,7 @@
 package config
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/rand"
@@ -22,6 +23,7 @@ import (
 
 	"github.com/ARM-software/golang-utils/utils/commonerrors"
 	"github.com/ARM-software/golang-utils/utils/commonerrors/errortest"
+	"github.com/ARM-software/golang-utils/utils/keyring"
 )
 
 var (
@@ -125,6 +127,7 @@ func TestServiceConfigurationLoad(t *testing.T) {
 	os.Clearenv()
 	configTest := &ConfigurationTest{}
 	defaults := DefaultConfiguration()
+	require.NoError(t, keyring.Clear(context.Background(), "test"))
 	err := Load("test", configTest, defaults)
 	// Some required values are missing.
 	require.Error(t, err)
@@ -171,6 +174,39 @@ func TestServiceConfigurationLoad(t *testing.T) {
 	assert.NotEqual(t, expectedDB, configTest.TestConfig.DB)
 	assert.True(t, configTest.TestConfig.Flag)
 	assert.False(t, configTest.TestConfig2.Flag)
+	t.Run("load from system", func(t *testing.T) {
+		configTest2 := &ConfigurationTest{}
+		err = LoadFromSystem("test", configTest2, defaults)
+		require.NoError(t, err)
+		require.NoError(t, configTest2.Validate())
+	})
+	t.Run("load from system", func(t *testing.T) {
+		configTest2 := &ConfigurationTest{}
+		err = Load("test", configTest2, defaults)
+		require.NoError(t, err)
+		require.NoError(t, configTest2.Validate())
+		assert.EqualExportedValues(t, configTest, configTest2)
+		configTest2.TestConfig2.Host = faker.URL()
+		configTest2.TestConfig2.User = faker.Name()
+		assert.NotEqual(t, configTest, configTest2)
+		err := keyring.Store[ConfigurationTest](context.Background(), "test", configTest2)
+		errortest.AssertError(t, err, nil, commonerrors.ErrUnsupported)
+		if commonerrors.Any(err, commonerrors.ErrUnsupported) {
+			t.Skip("keyring is not supported")
+		}
+		configTest3 := &ConfigurationTest{}
+		err = LoadFromSystem("test", configTest3, defaults)
+		require.NoError(t, err)
+		require.NoError(t, configTest3.Validate())
+		assert.EqualExportedValues(t, configTest2, configTest3)
+		assert.NotEqual(t, configTest, configTest3)
+		configTest4 := &ConfigurationTest{}
+		err = Load("test", configTest4, defaults)
+		require.NoError(t, err)
+		require.NoError(t, configTest4.Validate())
+		assert.EqualExportedValues(t, configTest, configTest4)
+		assert.NotEqual(t, configTest4, configTest3)
+	})
 }
 
 func TestServiceConfigurationLoad_Errors(t *testing.T) {

utils/go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/spf13/pflag v1.0.7
 	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.10.0
+	github.com/zalando/go-keyring v0.2.6
 	go.uber.org/atomic v1.11.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/mock v0.5.2
@@ -56,12 +57,14 @@ require (
 )
 
 require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/go-crypto v1.1.6 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect
 	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
+	github.com/danieljoos/wincred v1.2.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dmarkham/enumer v1.5.11 // indirect
 	github.com/ebitengine/purego v0.8.4 // indirect
@@ -71,6 +74,7 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/cabbie v1.0.2 // indirect
 	github.com/google/glazier v0.0.0-20211029225403-9f766cca891d // indirect

utils/go.sum

@@ -1,3 +1,5 @@
+al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho=
+al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
 bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwesoMFWAxPGd3UGjyY=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
@@ -31,6 +33,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/creachadair/staticfile v0.1.3/go.mod h1:a3qySzCIXEprDGxk6tSxSI+dBBdLzqeBOMhZ+o2d3pM=
 github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
 github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
+github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0=
+github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -93,6 +97,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0=
 github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
@@ -122,6 +128,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/logger v1.1.0/go.mod h1:w7O8nrRr0xufejBlQMI83MXqRusvREoJdaAxV+CoAB4=
 github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/winops v0.0.0-20210803215038-c8511b84de2b/go.mod h1:ShbX8v8clPm/3chw9zHVwtW3QhrFpL8mXOwNxClt4pg=
@@ -227,6 +235,8 @@ github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
 github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -243,6 +253,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s=
+github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=