Upload arm64 builds to shared bucket

.github/workflows/ci.yml

@@ -17,6 +17,7 @@ permissions:
   actions: read
   contents: read
   packages: write
+  id-token: write
 
 concurrency: ci-${{ github.ref }}
 
@@ -25,12 +26,11 @@ defaults:
     shell: bash
 
 jobs:
-  ci:
+  prepare:
     runs-on: ubuntu-latest-4-cores
+    outputs:
+      plugins: ${{ steps.set-plugins.outputs.plugins }}
     steps:
-    - name: set PLUGINS from workflow inputs
-      if: ${{ inputs.plugins }}
-      run: echo "PLUGINS=${{ inputs.plugins }}" >> $GITHUB_ENV
     - uses: actions/checkout@v6
       with:
         fetch-depth: 0
@@ -45,15 +45,34 @@ jobs:
       with:
         go-version: '1.25'
         check-latest: true
-    - name: Calculate changed plugins and set PLUGINS env var from last successful commit push
-      if: ${{ inputs.plugins == '' }}
+    - name: Calculate plugins to build
+      id: set-plugins
       env:
         BASE_REF: ${{ steps.last_successful_commit_push.outputs.base }}
       run: |
-        val=`go run ./internal/cmd/changed-plugins .`
-        if [[ -n "${val}" && -z "${PLUGINS}" ]]; then
-          echo "PLUGINS=${val}" >> $GITHUB_ENV
+        if [[ -n "${{ inputs.plugins }}" ]]; then
+          PLUGINS="${{ inputs.plugins }}"
+        else
+          PLUGINS=`go run ./internal/cmd/changed-plugins .`
         fi
+        echo "plugins=${PLUGINS}" >> $GITHUB_OUTPUT
+        echo "Building plugins: ${PLUGINS}"
+    - name: Run Go tests
+      run: go test -race -count=1 ./...
+
+  build:
+    needs: prepare
+    if: needs.prepare.outputs.plugins != ''
+    runs-on: ubuntu-latest-4-cores
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+    - name: Install Go
+      uses: actions/setup-go@v6
+      with:
+        go-version: '1.25'
+        check-latest: true
     - name: Get buf version
       shell: bash
       run: |
@@ -77,8 +96,57 @@ jobs:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Test
+    - name: Build and test Docker images
+      env:
+        PLUGINS: ${{ needs.prepare.outputs.plugins }}
       run: make test
     - name: Push to GHCR
       if: github.repository == 'bufbuild/plugins'
+      env:
+        PLUGINS: ${{ needs.prepare.outputs.plugins }}
       run: make dockerpush
+
+  upload-arm64:
+    needs: [prepare, build]
+    if: github.repository == 'bufbuild/plugins' && github.ref == 'refs/heads/main' && needs.prepare.outputs.plugins != ''
+    environment: production
+    runs-on: ubuntu-24.04-arm
+    steps:
+    - uses: actions/checkout@v6
+    - name: Install Go
+      uses: actions/setup-go@v6
+      with:
+        go-version: '1.25'
+        check-latest: true
+    - name: Get buf version
+      shell: bash
+      run: |
+        echo BUF_VERSION=$(go list -m -f '{{.Version}}' github.com/bufbuild/buf | cut -c2-) >> $GITHUB_ENV
+    - uses: bufbuild/buf-action@v1
+      with:
+        setup_only: true
+    - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
+    - name: Login to Docker Hub
+      uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+    - name: Build and test ARM64 Docker images
+      env:
+        PLUGINS: ${{ needs.prepare.outputs.plugins }}
+      run: make test
+    - name: Auth To GCP
+      uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
+      with:
+        workload_identity_provider: projects/491113660045/locations/global/workloadIdentityPools/plugins-workload-pool/providers/plugins-workload-provider
+        service_account: buf-plugins-1-bufbuild-plugins@buf-plugins-1.iam.gserviceaccount.com
+    - name: Setup gcloud
+      uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
+    - name: Create ARM64 plugin zips
+      env:
+        PLUGINS: ${{ needs.prepare.outputs.plugins }}
+      run: go run ./cmd/create-plugin-zips -dir . -org "bufbuild" -out builds
+    - name: Upload ARM64 plugins to GCS
+      run: gsutil -m rsync -r builds gs://buf-plugins/arm64

cmd/create-plugin-zips/main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+
+	"buf.build/go/interrupt"
+
+	"github.com/bufbuild/plugins/internal/docker"
+	"github.com/bufbuild/plugins/internal/plugin"
+	"github.com/bufbuild/plugins/internal/release"
+)
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatalf("failed to create plugin zips: %v", err)
+	}
+}
+
+func run() error {
+	var (
+		dir    = flag.String("dir", ".", "Directory path to plugins")
+		org    = flag.String("org", "bufbuild", "Docker Organization (without registry)")
+		outDir = flag.String("out", "downloads", "Output directory for plugin zips")
+	)
+	flag.Parse()
+
+	ctx := interrupt.Handle(context.Background())
+	basedir := *dir
+
+	plugins, err := plugin.FindAll(basedir)
+	if err != nil {
+		return err
+	}
+	includedPlugins, err := plugin.FilterByPluginsEnv(plugins, os.Getenv("PLUGINS"))
+	if err != nil {
+		return err
+	}
+	if len(includedPlugins) == 0 {
+		log.Printf("no plugins to process")
+		return nil
+	}
+
+	// Create output directory
+	if err := os.MkdirAll(*outDir, 0755); err != nil {
+		return fmt.Errorf("failed to create output directory: %w", err)
+	}
+
+	for _, includedPlugin := range includedPlugins {
+		if err := createPluginZip(ctx, includedPlugin, *org, *outDir); err != nil {
+			log.Printf(
+				"failed to process plugin %s:%s: %v",
+				includedPlugin.Name,
+				includedPlugin.PluginVersion,
+				err,
+			)
+			return err
+		}
+		log.Printf("created zip for plugin %s:%s", includedPlugin.Name, includedPlugin.PluginVersion)
+	}
+	return nil
+}
+
+func createPluginZip(ctx context.Context, plugin *plugin.Plugin, dockerOrg string, outDir string) error {
+	// Get image name from already-built local image
+	imageName := docker.ImageName(plugin, dockerOrg)
+
+	// Get the image ID
+	imageID, err := getImageID(ctx, imageName)
+	if err != nil {
+		return fmt.Errorf("failed to get image ID for %s: %w (image must be built first)", imageName, err)
+	}
+
+	// Create zip file
+	_, err = release.CreatePluginZip(ctx, outDir, plugin, imageID)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getImageID(ctx context.Context, imageName string) (string, error) {
+	cmd := exec.CommandContext(ctx, "docker", "inspect", "--format={{.Id}}", imageName)
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return string(output[:len(output)-1]), nil // trim newline
+}

internal/cmd/release/main.go

@@ -1,15 +1,12 @@
 package main
 
 import (
-	"archive/zip"
 	"cmp"
-	"compress/flate"
 	"context"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"io/fs"
 	"log"
 	"os"
@@ -403,79 +400,13 @@ func createPluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin,
 	if err := pullImage(ctx, registryImage); err != nil {
 		return "", err
 	}
-	zipName := pluginZipName(plugin)
-	pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName)))
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		if err := os.RemoveAll(pluginTempDir); err != nil {
-			log.Printf("failed to remove %q: %v", pluginTempDir, err)
-		}
-	}()
-	if err := saveImageToDir(ctx, imageID, pluginTempDir); err != nil {
-		return "", err
-	}
-	log.Printf("creating %s", zipName)
-	zipFile := filepath.Join(basedir, zipName)
-	zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
-			log.Printf("failed to close: %v", err)
-		}
-	}()
-	zw := zip.NewWriter(zf)
-	zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) {
-		return flate.NewWriter(w, flate.BestCompression)
-	})
-	if err := addFileToZip(zw, plugin.Path); err != nil {
-		return "", err
-	}
-	if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil {
-		return "", err
-	}
-	if err := zw.Close(); err != nil {
-		return "", err
-	}
-	if err := zf.Close(); err != nil {
-		return "", err
-	}
-	digest, err := release.CalculateDigest(zipFile)
+	digest, err := release.CreatePluginZip(ctx, basedir, plugin, imageID)
 	if err != nil {
 		return "", err
 	}
 	return digest, nil
 }
 
-func addFileToZip(zipWriter *zip.Writer, path string) error {
-	w, err := zipWriter.Create(filepath.Base(path))
-	if err != nil {
-		return err
-	}
-	r, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := r.Close(); err != nil {
-			log.Printf("failed to close: %v", err)
-		}
-	}()
-	if _, err := io.Copy(w, r); err != nil {
-		return err
-	}
-	return nil
-}
-
-func saveImageToDir(ctx context.Context, imageRef string, dir string) error {
-	cmd := dockerCmd(ctx, "save", imageRef, "-o", "image.tar")
-	cmd.Dir = dir
-	return cmd.Run()
-}
-
 func createPluginReleases(dir string, plugins []release.PluginRelease) error {
 	f, err := os.OpenFile(filepath.Join(dir, release.PluginReleasesFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
@@ -527,7 +458,7 @@ func calculateNextRelease(now time.Time, latestRelease *github.RepositoryRelease
 }
 
 func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string) string {
-	zipName := pluginZipName(plugin)
+	zipName := release.PluginZipName(plugin)
 	return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, zipName)
 }
 
@@ -541,11 +472,6 @@ func (c *command) pluginReleasesURL(releaseName string) string {
 	)
 }
 
-func pluginZipName(plugin *plugin.Plugin) string {
-	identity := plugin.Identity
-	return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion)
-}
-
 func fetchRegistryImageAndImageID(plugin *plugin.Plugin) (string, string, error) {
 	identity := plugin.Identity
 	imageName := fmt.Sprintf("ghcr.io/%s/plugins-%s-%s:%s", release.GithubOwnerBufbuild, identity.Owner(), identity.Plugin(), plugin.PluginVersion)