Speed up 'make fmt' and 'make lint' by running it on changes only (#3021)

## Why

Major speed when working on a branch (one go file changed in a small
package):

This branch - 'make fmt':

```
% hyperfine 'make fmt'
Benchmark 1: make fmt
  Time (mean ± σ):     386.0 ms ±  98.0 ms    [User: 178.4 ms, System: 239.8 ms]
  Range (min … max):   345.5 ms … 664.6 ms    10 runs

  Warning: The first benchmarking run for this command was significantly slower than the rest (664.6 ms). This could be caused by (filesystem) caches that were not filled until after the first run. You should consider using the '--warmup' option to fill those caches before the actual benchmark. Alternatively, use the '--prepare' option to clear the caches before each timing run.
```

main branch:

```
% git checkout main
M       libs/git/config.go
Switched to branch 'main'
Your branch is up to date with 'origin/main'.
~/work/cli-side % hyperfine 'make fmt'
Benchmark 1: make fmt
  Time (mean ± σ):     11.304 s ±  0.568 s    [User: 4.183 s, System: 3.622 s]
  Range (min … max):   10.502 s … 12.154 s    10 runs
```

Author: denik

.github/workflows/push.yml

@@ -112,10 +112,10 @@ jobs:
         with:
           version: "0.9.1"
           args: "format --check"
-      - name: "make fmt: Python and Go formatting"
+      - name: "make fmtfull: Python and Go formatting"
         # This is already done by actions, but good to check that make command is working
         run: |
-          make fmt
+          make fmtfull
           git diff --exit-code
       - name: "make checks: custom checks outside of fmt and lint"
         run: |

Makefile

@@ -6,20 +6,27 @@ GOTESTSUM_FORMAT ?= pkgname-and-test-fails
 GOTESTSUM_CMD ?= go tool gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped --jsonfile test-output.json
 
 
-lint:
+lintfull:
 	golangci-lint run --fix
 
+lint:
+	./tools/lintdiff.py run --fix
+
 tidy:
 	@# not part of golangci-lint, apparently
 	go mod tidy
 
 lintcheck:
 	golangci-lint run ./...
 
-fmt:
+fmtfull:
 	ruff format -qn
 	golangci-lint fmt
 
+fmt:
+	ruff format -qn
+	./tools/lintdiff.py fmt
+
 ws:
 	./tools/validate_whitespace.py
 
@@ -78,4 +85,4 @@ generate:
 	[ ! -f .github/workflows/next-changelog.yml ] || rm .github/workflows/next-changelog.yml
 	pushd experimental/python && make codegen
 
-.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot schema integration integration-short acc-cover acc-showcover docs ws links checks
+.PHONY: lint lintfull tidy lintcheck fmt fmtfull test cover showcover build snapshot schema integration integration-short acc-cover acc-showcover docs ws links checks

tools/lintdiff.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Drop in replacement for golangci-lint that runs it only on changed packages.
+
+Changes are calculated as diff against main by default, use --ref or -H/--head to change this.
+"""
+
+import os
+import sys
+import argparse
+import subprocess
+
+
+def parse_lines(cmd):
+    # print("+ " + " ".join(cmd), file=sys.stderr, flush=True)
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8", check=True)
+    return [x.strip() for x in result.stdout.split("\n") if x.strip()]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ref", default="main", help="Reference to calculate diff against.")
+    parser.add_argument("-H", "--head", action="store_true", help="Shortcut for '--ref HEAD' - test uncommitted changes only")
+    parser.add_argument("args", nargs=argparse.REMAINDER, help="golangci-lint command and options")
+    args = parser.parse_args()
+
+    if not args.args:
+        args.args = ["run"]
+
+    if args.head:
+        args.ref = "HEAD"
+
+    gitroot = parse_lines(["git", "rev-parse", "--show-toplevel"])[0]
+    os.chdir(gitroot)
+
+    changed = parse_lines(["git", "diff", "--name-only", args.ref, "--", "."])
+
+    # We need to pass packages to golangci-lint, not individual files.
+    dirs = set()
+    for filename in changed:
+        if "/testdata/" in filename:
+            continue
+        if filename.endswith(".go"):
+            d = os.path.dirname(filename)
+            dirs.add(d)
+
+    if not dirs:
+        sys.exit(0)
+
+    dirs = ["./" + d for d in sorted(dirs)]
+
+    cmd = ["golangci-lint"] + args.args + dirs
+    print("+ " + " ".join(cmd), file=sys.stderr, flush=True)
+    os.execvp(cmd[0], cmd)
+
+
+if __name__ == "__main__":
+    main()