diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 8282f46..6dc262d 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -14,7 +14,7 @@ env: jobs: build: name: Upload Release Assets - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Install Go uses: actions/setup-go@v5 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a91bfee..978f679 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -80,8 +80,9 @@ jobs: curl -sLO https://github.com/vearutop/gocovdiff/releases/download/v1.4.2/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz && rm linux_amd64.tar.gz gocovdiff_hash=$(git hash-object ./gocovdiff) [ "$gocovdiff_hash" == "c37862c73a677e5a9c069470287823ab5bbf0244" ] || (echo "::error::unexpected hash for gocovdiff, possible tampering: $gocovdiff_hash" && exit 1) - git fetch origin master ${{ github.event.pull_request.base.sha }} - REP=$(./gocovdiff -mod github.com/$GITHUB_REPOSITORY -cov unit.coverprofile -gha-annotations gha-unit.txt -delta-cov-file delta-cov-unit.txt -target-delta-cov ${TARGET_DELTA_COV}) + # Fetch PR diff from GitHub API. + curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github.v3.diff" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }} > pull_request.diff + REP=$(./gocovdiff -diff pull_request.diff -mod github.com/$GITHUB_REPOSITORY -cov unit.coverprofile -gha-annotations gha-unit.txt -delta-cov-file delta-cov-unit.txt -target-delta-cov ${TARGET_DELTA_COV}) echo "${REP}" cat gha-unit.txt DIFF=$(test -e unit-base.txt && ./gocovdiff -mod github.com/$GITHUB_REPOSITORY -func-cov unit.txt -func-base-cov unit-base.txt || echo "Missing base coverage file") diff --git a/cmd/catp/README.md b/cmd/catp/README.md index 5bbbac9..82cf9d0 100644 --- a/cmd/catp/README.md +++ b/cmd/catp/README.md @@ -23,6 +23,7 @@ catp dev, go1.22rc1 CGO_ZSTD catp prints contents of files to STDOUT or dir/file output, while printing current progress status to STDERR. It can decompress data from .gz and .zst files. +Use dash (-) as PATH to read STDIN. Usage of catp: catp [OPTIONS] PATH ... @@ -37,7 +38,7 @@ catp [OPTIONS] PATH ... files will be written to out dir with original base names disables output flag -output string - output to file instead of STDOUT + output to file (can have .gz or .zst ext for compression) instead of STDOUT -parallel int number of parallel readers if multiple files are provided lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file) diff --git a/cmd/catp/catp/app.go b/cmd/catp/catp/app.go index f9b44c3..ca9b2f7 100644 --- a/cmd/catp/catp/app.go +++ b/cmd/catp/catp/app.go @@ -61,6 +61,32 @@ type runner struct { hasOptions bool options Options + + hasCompression bool +} + +// humanReadableBytes converts bytes to a human-readable string (TB, GB, MB, KB, or bytes). +func humanReadableBytes(bytes int64) string { + const ( + Byte = 1 + KByte = Byte * 1024 + MByte = KByte * 1024 + GByte = MByte * 1024 + TByte = GByte * 1024 + ) + + switch { + case bytes >= TByte: + return fmt.Sprintf("%.2f TB", float64(bytes)/float64(TByte)) + case bytes >= GByte: + return fmt.Sprintf("%.2f GB", float64(bytes)/float64(GByte)) + case bytes >= MByte: + return fmt.Sprintf("%.2f MB", float64(bytes)/float64(MByte)) + case bytes >= KByte: + return fmt.Sprintf("%.2f KB", float64(bytes)/float64(KByte)) + default: + return fmt.Sprintf("%d B", bytes) + } } // st renders Status as a string. @@ -98,17 +124,29 @@ func (r *runner) st(s progress.Status) string { } } else { if s.LinesCompleted != 0 { - res = fmt.Sprintf("%s: %.1f%% bytes read, %d lines processed, %.1f l/s, %.1f MB/s, elapsed %s, remaining %s", - s.Task, s.DonePercent, s.LinesCompleted, s.SpeedLPS, s.SpeedMBPS, - s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String()) + if r.totalBytes == -1 { // STDIN + res = fmt.Sprintf("%s read, %d lines processed, %.1f l/s, %.1f MB/s, elapsed %s", + humanReadableBytes(s.BytesCompleted), s.LinesCompleted, s.SpeedLPS, s.SpeedMBPS, + s.Elapsed.Round(10*time.Millisecond).String()) + } else { + res = fmt.Sprintf("%s: %.1f%% bytes read, %d lines processed, %.1f l/s, %.1f MB/s, elapsed %s, remaining %s", + s.Task, s.DonePercent, s.LinesCompleted, s.SpeedLPS, s.SpeedMBPS, + s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String()) + } } else { - res = fmt.Sprintf("%s: %.1f%% bytes read, %.1f MB/s, elapsed %s, remaining %s", - s.Task, s.DonePercent, s.SpeedMBPS, - s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String()) + if r.totalBytes == -1 { + res = fmt.Sprintf("%s read, %.1f MB/s, elapsed %s", + humanReadableBytes(s.BytesCompleted), s.SpeedMBPS, + s.Elapsed.Round(10*time.Millisecond).String()) + } else { + res = fmt.Sprintf("%s: %.1f%% bytes read, %.1f MB/s, elapsed %s, remaining %s", + s.Task, s.DonePercent, s.SpeedMBPS, + s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String()) + } } } - if currentBytesUncompressed > currentBytes { + if currentBytesUncompressed > currentBytes && r.hasCompression { lastBytesUncompressed := atomic.LoadInt64(&r.lastBytesUncompressed) lastStatusTime := atomic.LoadInt64(&r.lastStatusTime) now := time.Now().Unix() @@ -158,6 +196,7 @@ func (r *runner) scanFile(filename string, rd io.Reader, out io.Writer) { s.Buffer(make([]byte, 64*1024), 10*1024*1024) lines := 0 + buf := make([]byte, 64*1024) for s.Scan() { lines++ @@ -175,7 +214,8 @@ func (r *runner) scanFile(filename string, rd io.Reader, out io.Writer) { if r.hasOptions { if r.options.PrepareLine != nil { - line = r.options.PrepareLine(filename, lines, line) + buf = buf[:0] + line = r.options.PrepareLine(filename, lines, line, &buf) } if line == nil { @@ -265,26 +305,32 @@ func (r *runner) shouldWrite(line []byte) bool { } func (r *runner) cat(filename string) (err error) { //nolint:gocyclo - file, err := os.Open(filename) //nolint:gosec - if err != nil { - return err - } + var rd io.Reader - defer func() { - if clErr := file.Close(); clErr != nil && err == nil { - err = clErr + if filename == "-" { + rd = os.Stdin + } else { + file, err := os.Open(filename) //nolint:gosec + if err != nil { + return err } - }() - rd := io.Reader(file) + defer func() { + if clErr := file.Close(); clErr != nil && err == nil { + err = clErr + } + }() + + rd = io.Reader(file) + } if !r.noProgress { - cr := progress.NewCountingReader(file) + cr := progress.NewCountingReader(rd) cr.SetBytes(&r.currentBytes) cr.SetLines(nil) if r.parallel <= 1 { - cr = progress.NewCountingReader(file) + cr = progress.NewCountingReader(rd) cr.SetLines(nil) r.currentFile = cr r.currentTotal = r.sizes[filename] @@ -425,7 +471,8 @@ func (i *stringFlags) Set(value string) error { // Options allows behavior customisations. type Options struct { // PrepareLine is invoked for every line, if result is nil, line is skipped. - PrepareLine func(filename string, lineNr int, line []byte) []byte + // You can use buf to avoid allocations for a result, and change its capacity if needed. + PrepareLine func(filename string, lineNr int, line []byte, buf *[]byte) []byte } // Main is the entry point for catp CLI tool. @@ -437,14 +484,6 @@ func Main(options ...func(o *Options)) error { //nolint:funlen,cyclop,gocognit,g r := &runner{} - if len(options) > 0 { - r.hasOptions = true - - for _, opt := range options { - opt(&r.options) - } - } - flag.Var(&pass, "pass", "filter matching, may contain multiple AND patterns separated by ^,\n"+ "if filter matches, line is passed to the output (unless filtered out by -skip)\n"+ "each -pass value is added with OR logic,\n"+ @@ -476,7 +515,8 @@ func Main(options ...func(o *Options)) error { //nolint:funlen,cyclop,gocognit,g fmt.Println() fmt.Println("catp prints contents of files to STDOUT or dir/file output, \n" + "while printing current progress status to STDERR. \n" + - "It can decompress data from .gz and .zst files.") + "It can decompress data from .gz and .zst files.\n" + + "Use dash (-) as PATH to read STDIN.") fmt.Println() fmt.Println("Usage of catp:") fmt.Println("catp [OPTIONS] PATH ...") @@ -502,6 +542,14 @@ func Main(options ...func(o *Options)) error { //nolint:funlen,cyclop,gocognit,g defer pprof.StopCPUProfile() } + if len(options) > 0 { + r.hasOptions = true + + for _, opt := range options { + opt(&r.options) + } + } + if *output != "" && r.outDir == "" { //nolint:nestif fn := *output @@ -597,35 +645,51 @@ func Main(options ...func(o *Options)) error { //nolint:funlen,cyclop,gocognit,g var files []string - for _, f := range flag.Args() { - glob, err := filepath.Glob(f) - if err != nil { - return err - } + args := flag.Args() - for _, f := range glob { - alreadyThere := false + if len(args) == 1 && args[0] == "-" { + files = append(files, "-") // STDIN + } else { + for _, f := range args { + glob, err := filepath.Glob(f) + if err != nil { + return err + } - for _, e := range files { - if e == f { - alreadyThere = true + for _, f := range glob { + alreadyThere := false - break + for _, e := range files { + if e == f { + alreadyThere = true + + break + } } - } - if !alreadyThere { - files = append(files, f) + if !alreadyThere { + files = append(files, f) + } } } } for _, fn := range files { + if fn == "-" { + r.totalBytes = -1 + + continue + } + st, err := os.Stat(fn) if err != nil { return fmt.Errorf("failed to read file stats %s: %w", fn, err) } + if strings.HasSuffix(fn, ".zst") || strings.HasSuffix(fn, ".gz") { + r.hasCompression = true + } + r.totalBytes += st.Size() r.sizes[fn] = st.Size() } diff --git a/cmd/catp/default.pgo b/cmd/catp/default.pgo index 4d2e9eb..9f053fe 100644 Binary files a/cmd/catp/default.pgo and b/cmd/catp/default.pgo differ diff --git a/go.mod b/go.mod index ab01617..06fef44 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/bool64/progress go 1.22 require ( - github.com/DataDog/zstd v1.5.6 - github.com/bool64/dev v0.2.39 + github.com/DataDog/zstd v1.5.7 + github.com/bool64/dev v0.2.40 github.com/klauspost/compress v1.18.0 github.com/klauspost/pgzip v1.2.6 ) diff --git a/go.sum b/go.sum index 4b9c171..a79ea7a 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= -github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/bool64/dev v0.2.39 h1:kP8DnMGlWXhGYJEZE/J0l/gVBdbuhoPGL+MJG4QbofE= -github.com/bool64/dev v0.2.39/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/bool64/dev v0.2.40 h1:LUSD+Aq+WB3KwVntqXstevJ0wB12ig1bEgoG8ZafsZU= +github.com/bool64/dev v0.2.40/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=