feat: support human-readable data storage size units in config (#103)

Author: hedhyw

example.jlv.jsonc

@@ -65,5 +65,15 @@
     },
     // The maximum size of the file in bytes.
     // The rest of the file will be ignored.
-    "maxFileSizeBytes": 1073741824,
+    //
+    // It accepts the number of bytes:
+    //
+    //   "maxFileSizeBytes": 1073741824
+    //
+    // Or a text value with a unit:
+    //
+    //   "maxFileSizeBytes": "1000k"
+    //   "maxFileSizeBytes": "1.5m"
+    //   "maxFileSizeBytes": "1g"
+    "maxFileSizeBytes": "2g"
 }

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/charmbracelet/bubbles v0.20.0
 	github.com/charmbracelet/bubbletea v1.1.1
 	github.com/charmbracelet/lipgloss v0.13.0
+	github.com/docker/go-units v0.5.0
 	github.com/go-playground/validator/v10 v10.22.1
 	github.com/hedhyw/jsoncjson v1.1.0
 	github.com/hedhyw/semerr v0.6.7

go.sum

@@ -17,6 +17,8 @@ github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4c
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
 github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4=

internal/pkg/config/config.go

@@ -5,7 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 
+	units "github.com/docker/go-units"
 	"github.com/go-playground/validator/v10"
 	"github.com/hedhyw/jsoncjson"
 )
@@ -23,7 +25,7 @@ type Config struct {
 	CustomLevelMapping map[string]string `json:"customLevelMapping"`
 
 	// MaxFileSizeBytes is the maximum size of the file to load.
-	MaxFileSizeBytes int64 `json:"maxFileSizeBytes" validate:"min=1"`
+	MaxFileSizeBytes ByteSize `json:"maxFileSizeBytes" validate:"min=1"`
 }
 
 // FieldKind describes the type of the log field.
@@ -55,7 +57,7 @@ func GetDefaultConfig() *Config {
 	return &Config{
 		Path:               "default",
 		CustomLevelMapping: GetDefaultCustomLevelMapping(),
-		MaxFileSizeBytes:   1024 * 1024 * 1024,
+		MaxFileSizeBytes:   2 * units.GB,
 		Fields: []Field{{
 			Title:      "Time",
 			Kind:       FieldKindNumericTime,
@@ -149,3 +151,34 @@ func GetDefaultCustomLevelMapping() map[string]string {
 		"60": "fatal",
 	}
 }
+
+// ByteSize supports decoding from byte count or number with unit.
+//
+// Example: 1k, 1.5m, 1g, 1t, 1p.
+type ByteSize int
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (s *ByteSize) UnmarshalJSON(text []byte) error {
+	value := string(text)
+
+	valueUnquoted, err := strconv.Unquote(value)
+	if err == nil {
+		value = valueUnquoted
+	}
+
+	parsedBytes, err := strconv.Atoi(value)
+	if err == nil {
+		*s = ByteSize(parsedBytes)
+
+		return nil
+	}
+
+	size, err := units.FromHumanSize(value)
+	if err != nil {
+		return fmt.Errorf("parsing unit: %w", err)
+	}
+
+	*s = ByteSize(size)
+
+	return nil
+}

internal/pkg/config/config_test.go

@@ -11,8 +11,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/go-units"
 	"github.com/go-playground/validator/v10"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/hedhyw/json-log-viewer/internal/pkg/config"
 	"github.com/hedhyw/json-log-viewer/internal/pkg/tests"
@@ -142,7 +144,7 @@ func ExampleGetDefaultConfig() {
 	//     "50": "error",
 	//     "60": "fatal"
 	//   },
-	//   "maxFileSizeBytes": 1073741824
+	//   "maxFileSizeBytes": 2000000000
 	// }
 }
 
@@ -299,3 +301,76 @@ func TestValidateField(t *testing.T) {
 		})
 	}
 }
+
+func TestByteSize(t *testing.T) {
+	t.Parallel()
+
+	testCases := [...]struct {
+		Value    string
+		Expected config.ByteSize
+	}{{
+		Value:    `"1k"`,
+		Expected: units.KB,
+	}, {
+		Value:    `"1m"`,
+		Expected: units.MB,
+	}, {
+		Value:    `"1.5m"`,
+		Expected: units.MB * 1.5,
+	}, {
+		Value:    `"1g"`,
+		Expected: units.GB,
+	}, {
+		Value:    `"1t"`,
+		Expected: units.TB,
+	}, {
+		Value:    `"1p"`,
+		Expected: units.PB,
+	}, {
+		Value:    "1",
+		Expected: 1,
+	}, {
+		Value:    "12345",
+		Expected: 12345,
+	}}
+
+	for _, testCase := range testCases {
+		var actual config.ByteSize
+
+		err := json.Unmarshal([]byte(testCase.Value), &actual)
+		if assert.NoError(t, err, testCase.Value) {
+			assert.Equal(t, testCase.Expected, actual, testCase.Value)
+		}
+	}
+}
+
+func TestByteSizeParseFailed(t *testing.T) {
+	t.Parallel()
+
+	t.Run("invalid_number", func(t *testing.T) {
+		t.Parallel()
+
+		var value config.ByteSize
+
+		err := json.Unmarshal([]byte(`"123.123.123"`), &value)
+		require.Error(t, err)
+	})
+
+	t.Run("invalid_suffix", func(t *testing.T) {
+		t.Parallel()
+
+		var value config.ByteSize
+
+		err := json.Unmarshal([]byte(`"123X"`), &value)
+		require.Error(t, err)
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		t.Parallel()
+
+		var value config.ByteSize
+
+		err := json.Unmarshal([]byte(`""`), &value)
+		require.Error(t, err)
+	})
+}

internal/pkg/source/source.go

@@ -66,7 +66,7 @@ func File(name string, cfg *config.Config) (*Source, error) {
 	var err error
 
 	source := &Source{
-		maxSize: cfg.MaxFileSizeBytes,
+		maxSize: int64(cfg.MaxFileSizeBytes),
 		name:    name,
 	}
 
@@ -94,7 +94,7 @@ func Reader(input io.Reader, cfg *config.Config) (*Source, error) {
 	var err error
 
 	source := &Source{
-		maxSize: cfg.MaxFileSizeBytes,
+		maxSize: int64(cfg.MaxFileSizeBytes),
 	}
 
 	// We will write the as read to a temp file.  Seek against the temp file.