✨ [logs] Added a rolling file logger

changes/20250723181707.bugfix

@@ -0,0 +1 @@
+:recycle: [logs] update slog utilities to use standard library slog rather than golang extension's implementation

changes/20250723182149.bugfix

@@ -0,0 +1 @@
+:recycle: [logs] Update ways to not close writers when loggers close

changes/20250723185203.feature

@@ -0,0 +1 @@
+:sparkles: [logs] Added a rolling file logger

utils/go.mod

@@ -5,6 +5,7 @@ go 1.23.0
 toolchain go1.24.1
 
 require (
+	github.com/DeRuina/timberjack v1.3.3
 	github.com/OneOfOne/xxhash v1.2.8
 	github.com/avast/retry-go/v4 v4.6.1
 	github.com/bmatcuk/doublestar/v3 v3.0.0
@@ -26,7 +27,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/go-retryablehttp v0.7.8
-	github.com/iamacarpet/go-win64api v0.0.0-20230324134531-ef6dbdd6db97
+	github.com/iamacarpet/go-win64api v0.0.0-20240507095429-873e84e85847
 	github.com/joho/godotenv v1.5.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
@@ -39,13 +40,12 @@ require (
 	github.com/spf13/pflag v1.0.7
 	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.10.0
-	github.com/zailic/slogr v0.0.2-alpha
 	go.uber.org/atomic v1.11.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/mock v0.5.2
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.40.0
-	golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6
+	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792
 	golang.org/x/mod v0.26.0
 	golang.org/x/net v0.42.0
 	golang.org/x/oauth2 v0.30.0
@@ -96,7 +96,7 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/tools v0.34.0 // indirect
+	golang.org/x/tools v0.35.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

utils/go.sum

@@ -2,6 +2,10 @@ bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwe
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DeRuina/timberjack v1.3.2 h1:ei548+RsKNhWNCoah4X0TKnR8VWRnbjsEMFZRBbSNdU=
+github.com/DeRuina/timberjack v1.3.2/go.mod h1:oIz9NBX34JRQd9U3ZuSEih49LuYlFnATRWiCfwbeiJ8=
+github.com/DeRuina/timberjack v1.3.3 h1:R+3oEvJP1wd8iYkGGsnq9e6pHGAXFqizPKReEMj0jvQ=
+github.com/DeRuina/timberjack v1.3.3/go.mod h1:pbNOvT1AIUxBqiH/ytL9nNsKrHUrDGJdz7T23y79RRU=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
@@ -52,6 +56,8 @@ github.com/evanphx/hclogr v0.2.0/go.mod h1:1jTbx9bMs6/CBoiPwzNobWb7BFkK/KF5XljvP
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -132,6 +138,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/iamacarpet/go-win64api v0.0.0-20210311141720-fe38760bed28/go.mod h1:oGJx9dz0Ny7HC7U55RZ0Smd6N9p3hXP/+hOFtuYrAxM=
 github.com/iamacarpet/go-win64api v0.0.0-20230324134531-ef6dbdd6db97 h1:VjwKCN2PMLlMKM2k9AW8QQsfmEH43ldlX+JGeWW9cEE=
 github.com/iamacarpet/go-win64api v0.0.0-20230324134531-ef6dbdd6db97/go.mod h1:B7zFQPAznj+ujXel5X+LUoK3LgY6VboCdVYHZNn7gpg=
+github.com/iamacarpet/go-win64api v0.0.0-20240507095429-873e84e85847 h1:cRHZFGwIDgQlr9abL/P93JXR7pYxzvf0xAIt0xzwrh0=
+github.com/iamacarpet/go-win64api v0.0.0-20240507095429-873e84e85847/go.mod h1:B7zFQPAznj+ujXel5X+LUoK3LgY6VboCdVYHZNn7gpg=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
@@ -235,8 +243,6 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/zailic/slogr v0.0.2-alpha h1:ZZ+96+AOnk4L9JoPkZ6aGbGXnn90/53A9zm9JcjYSYc=
-github.com/zailic/slogr v0.0.2-alpha/go.mod h1:cwplHb/RBT+83E4QzPcVtCs0Z/sAjmgMtC09XGm9SCU=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -254,6 +260,8 @@ golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
 golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
 golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
 golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -313,6 +321,7 @@ golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
 golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
+golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=

utils/logs/fifo_logger_test.go

@@ -12,13 +12,15 @@ import (
 	"github.com/go-faker/faker/v4"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 
 	"github.com/ARM-software/golang-utils/utils/commonerrors"
 	"github.com/ARM-software/golang-utils/utils/commonerrors/errortest"
 	"github.com/ARM-software/golang-utils/utils/parallelisation"
 )
 
 func TestFIFOLoggerLineIterator(t *testing.T) {
+	defer goleak.VerifyNone(t)
 	t.Run("logger tests", func(t *testing.T) {
 		loggers, err := NewFIFOLogger()
 		require.NoError(t, err)
@@ -48,6 +50,7 @@ func TestFIFOLoggerLineIterator(t *testing.T) {
 }
 
 func TestPlainFIFOLoggerLineIterator(t *testing.T) {
+	defer goleak.VerifyNone(t)
 	t.Run("logger tests", func(t *testing.T) {
 		loggers, err := NewPlainFIFOLogger()
 		require.NoError(t, err)

utils/logs/file_logger.go

@@ -6,9 +6,19 @@
 package logs
 
 import (
+	"fmt"
 	"io"
+	"log"
+	"time"
 
+	"github.com/DeRuina/timberjack"
 	"github.com/sirupsen/logrus"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+	"github.com/ARM-software/golang-utils/utils/parallelisation"
+	"github.com/ARM-software/golang-utils/utils/reflection"
+	"github.com/ARM-software/golang-utils/utils/safecast"
+	sizeUnits "github.com/ARM-software/golang-utils/utils/units/size"
 )
 
 // NewFileLogger creates a logger to a file.
@@ -29,3 +39,80 @@ func NewFileOnlyLogger(logFile string, loggerSource string) (loggers Loggers, er
 func CreateFileLogger(logFile string, loggerSource string) (loggers Loggers, err error) {
 	return NewFileLogger(logFile, loggerSource)
 }
+
+type FileLoggerOptions struct {
+	maxFileSize float64
+	maxAge      time.Duration
+	maxBackups  int
+}
+
+type FileLoggerOption func(*FileLoggerOptions) *FileLoggerOptions
+
+// WithMaxFileSize sets the maximum size in bytes of a log file before it gets rotated.
+func WithMaxFileSize(maxFileSize float64) FileLoggerOption {
+	return func(o *FileLoggerOptions) *FileLoggerOptions {
+		if o == nil {
+			return o
+		}
+		o.maxFileSize = maxFileSize
+		return o
+	}
+}
+
+// WithMaxAge sets the maximum duration old log files are retained.
+func WithMaxAge(maxAge time.Duration) FileLoggerOption {
+	return func(o *FileLoggerOptions) *FileLoggerOptions {
+		if o == nil {
+			return o
+		}
+		// This is necessary to avoid a Race Condition
+		if maxAge >= time.Minute {
+			o.maxAge = maxAge
+		}
+		return o
+	}
+}
+
+// WithMaxBackups sets the maximum number of old log files to retain.
+func WithMaxBackups(maxBackups int) FileLoggerOption {
+	return func(o *FileLoggerOptions) *FileLoggerOptions {
+		if o == nil {
+			return o
+		}
+		o.maxBackups = maxBackups
+		return o
+	}
+}
+
+// NewRollingFilesLogger creates a rolling file logger using [lumberjack](https://github.com/natefinch/lumberjack) under the bonnet.
+func NewRollingFilesLogger(logFile string, loggerSource string, options ...FileLoggerOption) (loggers Loggers, err error) {
+	opts := &FileLoggerOptions{
+		maxFileSize: 100 * sizeUnits.MiB,
+		maxAge:      24 * time.Hour,
+		maxBackups:  3,
+	}
+	for i := range options {
+		opts = options[i](opts)
+	}
+	if reflection.IsEmpty(logFile) {
+		err = commonerrors.New(commonerrors.ErrInvalidDestination, "missing file destination")
+		return
+	}
+	l := &timberjack.Logger{
+		Filename:   logFile,
+		MaxSize:    safecast.ToInt(opts.maxFileSize / sizeUnits.MiB),
+		MaxAge:     safecast.ToInt(opts.maxAge.Hours() / 24),
+		MaxBackups: opts.maxBackups,
+		LocalTime:  false,
+		Compress:   false,
+	}
+	closerStore := parallelisation.NewCloserStore(false)
+	closerStore.RegisterCloser(l)
+
+	loggers = &GenericLoggers{
+		Output:     log.New(l, fmt.Sprintf("[%v] Output: ", loggerSource), log.LstdFlags),
+		Error:      log.New(l, fmt.Sprintf("[%v] Error: ", loggerSource), log.LstdFlags),
+		closeStore: closerStore,
+	}
+	return
+}

utils/logs/file_logger_test.go

@@ -10,11 +10,13 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 
 	"github.com/ARM-software/golang-utils/utils/filesystem"
 )
 
 func TestFileLogger(t *testing.T) {
+	defer goleak.VerifyNone(t)
 	var tests = []struct {
 		loggerCreationFunc func(path string) (Loggers, error)
 	}{
@@ -24,10 +26,27 @@ func TestFileLogger(t *testing.T) {
 		{
 			loggerCreationFunc: func(path string) (Loggers, error) { return NewFileOnlyLogger(path, "Test") },
 		},
+		// FIXME uncomment when data race is fixed
+		// {
+		//	loggerCreationFunc: func(path string) (Loggers, error) {
+		//		return NewRollingFilesLogger(path, "Test", WithMaxFileSize(sizeUnits.MiB), WithMaxBackups(2), WithMaxAge(10*time.Minute))
+		//	},
+		// },
+		// {
+		//	loggerCreationFunc: func(path string) (Loggers, error) {
+		//		return NewRollingFilesLogger(path, "Test", WithMaxAge(time.Second))
+		//	},
+		// },
+		// {
+		//	loggerCreationFunc: func(path string) (Loggers, error) {
+		//		return NewRollingFilesLogger(path, "Test")
+		//	},
+		// },
 	}
 	for i := range tests {
 		test := tests[i]
 		t.Run(fmt.Sprintf("logger %v", i), func(t *testing.T) {
+			defer goleak.VerifyNone(t)
 			file, err := filesystem.TouchTempFileInTempDir("test-filelog-*.log")
 			require.NoError(t, err)
 

utils/logs/hclog_logger_test.go

@@ -9,18 +9,21 @@ import (
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 
 	"github.com/ARM-software/golang-utils/utils/logs/logstest"
 )
 
 func TestHclogLogger(t *testing.T) {
+	defer goleak.VerifyNone(t)
 	logger := hclog.New(nil)
 	loggers, err := NewHclogLogger(logger, "Test")
 	require.NoError(t, err)
 	testLog(t, loggers)
 }
 
 func TestHclogWrapper(t *testing.T) {
+	defer goleak.VerifyNone(t)
 	loggers, err := NewLogrLogger(logstest.NewTestLogger(t), "test")
 	require.NoError(t, err)
 	hcLogger, err := NewHclogWrapper(loggers)

utils/logs/json_logger.go

@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/ARM-software/golang-utils/utils/commonerrors"
+	"github.com/ARM-software/golang-utils/utils/parallelisation"
 )
 
 // JSONLoggers defines a JSON logger
@@ -22,6 +23,7 @@ type JSONLoggers struct {
 	loggerSource string
 	writer       WriterWithSource
 	zerologger   zerolog.Logger
+	closerStore  *parallelisation.CloserStore
 }
 
 func (l *JSONLoggers) SetLogSource(source string) error {
@@ -88,16 +90,31 @@ func (l *JSONLoggers) LogError(err ...interface{}) {
 func (l *JSONLoggers) Close() error {
 	l.mu.Lock()
 	defer l.mu.Unlock()
-	return l.writer.Close()
+	return l.closerStore.Close()
 }
 
 // NewJSONLogger creates a Json logger.
-func NewJSONLogger(writer WriterWithSource, loggerSource string, source string) (loggers Loggers, err error) {
+func NewJSONLogger(writer WriterWithSource, loggerSource string, source string) (Loggers, error) {
+	return newJSONLogger(true, writer, loggerSource, source)
+}
+
+// NewJSONLogger creates a Json logger. It is similar to NewJSONLogger but does not close the writer on Close().
+func NewJSONLoggerWithWriter(writer WriterWithSource, loggerSource string, source string) (Loggers, error) {
+	return newJSONLogger(false, writer, loggerSource, source)
+}
+
+func newJSONLogger(closeWriterOnClose bool, writer WriterWithSource, loggerSource string, source string) (loggers Loggers, err error) {
+	closeStore := parallelisation.NewCloserStore(false)
+	if closeWriterOnClose {
+		closeStore.RegisterCloser(writer)
+	}
+
 	zerroLogger := JSONLoggers{
 		source:       source,
 		loggerSource: loggerSource,
 		writer:       writer,
 		zerologger:   zerolog.New(writer),
+		closerStore:  closeStore,
 	}
 	err = zerroLogger.Check()
 	if err != nil {
@@ -112,8 +129,8 @@ func NewJSONLogger(writer WriterWithSource, loggerSource string, source string)
 	return
 }
 
-// NewJSONLoggerForSlowWriter creates a lock free, non blocking & thread safe logger
-// wrapped around slowWriter
+// NewJSONLoggerForSlowWriter creates a lock free, non-blocking & thread safe logger
+// wrapped around slowWriter. The slowWriter is closed on Close.
 //
 // params:
 // slowWriter : writer used to write data streams
@@ -126,3 +143,18 @@ func NewJSONLogger(writer WriterWithSource, loggerSource string, source string)
 func NewJSONLoggerForSlowWriter(slowWriter WriterWithSource, ringBufferSize int, pollInterval time.Duration, loggerSource string, source string, droppedMessagesLogger Loggers) (loggers Loggers, err error) {
 	return NewJSONLogger(NewDiodeWriterForSlowWriter(slowWriter, ringBufferSize, pollInterval, droppedMessagesLogger), loggerSource, source)
 }
+
+// NewJSONLoggerForSlowWriterWithoutClosingWriter creates a lock free, non-blocking & thread safe logger
+// wrapped around slowWriter. It is similar to NewJSONLoggerForSlowWriter but does not close the writer on Close().
+//
+// params:
+// slowWriter : writer used to write data streams
+// ringBufferSize : size of ring buffer used to receive messages
+// pollInterval : polling duration to check buffer content
+// loggerSource : logger application name
+// source : source string
+// droppedMessagesLogger : logger for dropped messages
+// If pollInterval is greater than 0, a poller is used otherwise a waiter is used.
+func NewJSONLoggerForSlowWriterWithoutClosingWriter(slowWriter WriterWithSource, ringBufferSize int, pollInterval time.Duration, loggerSource string, source string, droppedMessagesLogger Loggers) (loggers Loggers, err error) {
+	return NewJSONLogger(NewDiodeWriterForSlowWriterWithoutClosing(slowWriter, ringBufferSize, pollInterval, droppedMessagesLogger), loggerSource, source)
+}