From d0770e9df5a29ce3367cbbbeb679b0fbbaa46c82 Mon Sep 17 00:00:00 2001 From: Rubens Farias Date: Mon, 19 Jan 2026 15:15:14 -0500 Subject: [PATCH 1/2] do the version where all the arm64 ones are included in the release manifest in github packages --- .github/workflows/ci.yml | 45 ++++++++++++---- .github/workflows/upload.yml | 9 +++- cmd/download-plugins/main.go | 27 ++++++++++ internal/cmd/release/main.go | 101 +++++++++++++++++++---------------- internal/docker/name.go | 12 ++++- internal/release/release.go | 1 + 6 files changed, 136 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5125f2163..f76a71b77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,12 +25,11 @@ defaults: shell: bash jobs: - ci: + prepare: runs-on: ubuntu-latest-4-cores + outputs: + plugins: ${{ steps.set-plugins.outputs.plugins }} steps: - - name: set PLUGINS from workflow inputs - if: ${{ inputs.plugins }} - run: echo "PLUGINS=${{ inputs.plugins }}" >> $GITHUB_ENV - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -45,15 +44,37 @@ jobs: with: go-version: '1.25' check-latest: true - - name: Calculate changed plugins and set PLUGINS env var from last successful commit push - if: ${{ inputs.plugins == '' }} + - name: Calculate plugins to build + id: set-plugins env: BASE_REF: ${{ steps.last_successful_commit_push.outputs.base }} run: | - val=`go run ./internal/cmd/changed-plugins .` - if [[ -n "${val}" && -z "${PLUGINS}" ]]; then - echo "PLUGINS=${val}" >> $GITHUB_ENV + if [[ -n "${{ inputs.plugins }}" ]]; then + PLUGINS="${{ inputs.plugins }}" + else + PLUGINS=`go run ./internal/cmd/changed-plugins .` fi + echo "plugins=${PLUGINS}" >> $GITHUB_OUTPUT + echo "Building plugins: ${PLUGINS}" + - name: Run Go tests + run: go test -race -count=1 ./... + + build: + needs: prepare + if: needs.prepare.outputs.plugins != '' + strategy: + matrix: + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest-4-cores' }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: '1.25' + check-latest: true - name: Get buf version shell: bash run: | @@ -77,8 +98,12 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Test + - name: Build and test Docker images + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} run: make test - name: Push to GHCR if: github.repository == 'bufbuild/plugins' + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} run: make dockerpush diff --git a/.github/workflows/upload.yml b/.github/workflows/upload.yml index a2f67e5f2..bfc1432f2 100644 --- a/.github/workflows/upload.yml +++ b/.github/workflows/upload.yml @@ -48,9 +48,14 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - go run ./cmd/download-plugins -since ${{ inputs.since }} downloads + go run ./cmd/download-plugins -all-archs -since ${{ inputs.since }} downloads - name: Upload To Release Bucket - run: gsutil -m rsync -r downloads gs://buf-plugins + run: | + mkdir -p downloads/amd64 downloads/arm64 + find downloads -maxdepth 1 -name "*-arm64.zip" -exec mv {} downloads/arm64/ \; + find downloads -maxdepth 1 -name "*.zip" ! -name "*-arm64.zip" -exec mv {} downloads/amd64/ \; + gsutil -m rsync -r downloads/amd64 gs://buf-plugins + gsutil -m rsync -r downloads/arm64 gs://buf-plugins/arm64 - name: Generate Github Token id: generate_issues_token uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 diff --git a/cmd/download-plugins/main.go b/cmd/download-plugins/main.go index 967890128..8b57d9dab 100644 --- a/cmd/download-plugins/main.go +++ b/cmd/download-plugins/main.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "strings" "time" @@ -35,10 +36,12 @@ func run() error { minisignPublicKey string releaseTag string since time.Duration + allArchs bool ) flag.StringVar(&minisignPublicKey, "minisign-public-key", "", "path to minisign public key file (default: bufbuild/plugins public key)") flag.StringVar(&releaseTag, "release-tag", "", "release to download (default: latest release)") flag.DurationVar(&since, "since", 0, "only download plugins created/modified since this time") + flag.BoolVar(&allArchs, "all-archs", false, "download plugins for all architectures instead of just current runtime arch") flag.Parse() if len(flag.Args()) != 1 { @@ -99,6 +102,30 @@ func run() error { } for _, pluginRelease := range pluginReleases.Releases { + // Filter by architecture unless -all-archs is specified + if !allArchs { + // If Arch field is set, use it for filtering + if pluginRelease.Arch != "" { + if pluginRelease.Arch != runtime.GOARCH { + continue + } + } else { + // Fallback for older releases without Arch field + // We historically had non-suffixed amd64 images, and then added arm64 images with the -arm64 + // suffix for the arch. Thus we need a bit of a conditional check to choose which ones we're doing. + if runtime.GOARCH == "amd64" { + if !strings.HasSuffix(pluginRelease.URL, ".zip") || strings.HasSuffix(pluginRelease.URL, "-arm64.zip") { + continue + } + } else { + archSuffix := fmt.Sprintf("-%s.zip", runtime.GOARCH) + if !strings.HasSuffix(pluginRelease.URL, archSuffix) { + continue + } + } + } + } + // Filter out plugins which aren't specified in PLUGINS env var if includePlugins != nil { var matched bool diff --git a/internal/cmd/release/main.go b/internal/cmd/release/main.go index 99493c724..cc3ebaeba 100644 --- a/internal/cmd/release/main.go +++ b/internal/cmd/release/main.go @@ -28,6 +28,7 @@ import ( "github.com/google/go-github/v72/github" "golang.org/x/mod/semver" + "github.com/bufbuild/plugins/internal/docker" "github.com/bufbuild/plugins/internal/plugin" "github.com/bufbuild/plugins/internal/release" ) @@ -186,46 +187,52 @@ func (c *command) calculateNewReleasePlugins(ctx context.Context, currentRelease if err != nil { return err } - registryImage, imageID, err := fetchRegistryImageAndImageID(plugin) - if err != nil { - return err - } - identity := plugin.Identity - if registryImage == "" || imageID == "" { - log.Printf("unable to detect registry image and image ID for plugin %s/%s:%s", identity.Owner(), identity.Plugin(), plugin.PluginVersion) - return nil - } - key := pluginNameVersion{name: identity.Owner() + "/" + identity.Plugin(), version: plugin.PluginVersion} - pluginRelease := pluginNameVersionToRelease[key] - // Found existing release - only rebuild if changed image digest or buf.plugin.yaml digest - if pluginRelease.ImageID != imageID || pluginRelease.PluginYAMLDigest != pluginYamlDigest { - downloadURL := c.pluginDownloadURL(plugin, releaseName) - zipDigest, err := createPluginZip(ctx, tmpDir, plugin, registryImage, imageID) + + // Process both amd64 and arm64 + for _, arch := range []string{"amd64", "arm64"} { + registryImage, imageID, err := fetchRegistryImageAndImageID(plugin, arch) if err != nil { - return err + log.Printf("unable to fetch %s for plugin %s/%s:%s: %v", arch, plugin.Identity.Owner(), plugin.Identity.Plugin(), plugin.PluginVersion, err) + continue } - status := release.StatusUpdated - if pluginRelease.ImageID == "" { - status = release.StatusNew + identity := plugin.Identity + if registryImage == "" || imageID == "" { + log.Printf("unable to detect registry image and image ID for %s plugin %s/%s:%s", arch, identity.Owner(), identity.Plugin(), plugin.PluginVersion) + continue + } + key := pluginNameVersion{name: identity.Owner() + "/" + identity.Plugin(), version: plugin.PluginVersion} + pluginRelease := pluginNameVersionToRelease[key] + // Found existing release - only rebuild if changed image digest or buf.plugin.yaml digest + if pluginRelease.ImageID != imageID || pluginRelease.PluginYAMLDigest != pluginYamlDigest { + downloadURL := c.pluginDownloadURL(plugin, releaseName, arch) + zipDigest, err := createPluginZip(ctx, tmpDir, plugin, registryImage, imageID, arch) + if err != nil { + return err + } + status := release.StatusUpdated + if pluginRelease.ImageID == "" { + status = release.StatusNew + } + newPlugins = append(newPlugins, release.PluginRelease{ + PluginName: fmt.Sprintf("%s/%s", identity.Owner(), identity.Plugin()), + PluginVersion: plugin.PluginVersion, + PluginZipDigest: zipDigest, + PluginYAMLDigest: pluginYamlDigest, + RegistryImage: registryImage, + ImageID: imageID, + ReleaseTag: releaseName, + URL: downloadURL, + LastUpdated: now, + Arch: arch, + Status: status, + Dependencies: pluginDependencies(plugin), + }) + } else { + log.Printf("plugin %s:%s (%s) unchanged", pluginRelease.PluginName, pluginRelease.PluginVersion, arch) + pluginRelease.Status = release.StatusExisting + pluginRelease.Dependencies = pluginDependencies(plugin) + existingPlugins = append(existingPlugins, pluginRelease) } - newPlugins = append(newPlugins, release.PluginRelease{ - PluginName: fmt.Sprintf("%s/%s", identity.Owner(), identity.Plugin()), - PluginVersion: plugin.PluginVersion, - PluginZipDigest: zipDigest, - PluginYAMLDigest: pluginYamlDigest, - RegistryImage: registryImage, - ImageID: imageID, - ReleaseTag: releaseName, - URL: downloadURL, - LastUpdated: now, - Status: status, - Dependencies: pluginDependencies(plugin), - }) - } else { - log.Printf("plugin %s:%s unchanged", pluginRelease.PluginName, pluginRelease.PluginVersion) - pluginRelease.Status = release.StatusExisting - pluginRelease.Dependencies = pluginDependencies(plugin) - existingPlugins = append(existingPlugins, pluginRelease) } return nil }); err != nil { @@ -399,11 +406,11 @@ func signPluginReleases(dir string, privateKey minisign.PrivateKey) error { return nil } -func createPluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, registryImage string, imageID string) (string, error) { +func createPluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, registryImage string, imageID string, arch string) (string, error) { if err := pullImage(ctx, registryImage); err != nil { return "", err } - zipName := pluginZipName(plugin) + zipName := pluginZipName(plugin, arch) pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName))) if err != nil { return "", err @@ -526,8 +533,8 @@ func calculateNextRelease(now time.Time, latestRelease *github.RepositoryRelease return releaseName, nil } -func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string) string { - zipName := pluginZipName(plugin) +func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string, arch string) string { + zipName := pluginZipName(plugin, arch) return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, zipName) } @@ -541,14 +548,16 @@ func (c *command) pluginReleasesURL(releaseName string) string { ) } -func pluginZipName(plugin *plugin.Plugin) string { +func pluginZipName(plugin *plugin.Plugin, arch string) string { identity := plugin.Identity - return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) + if arch == "amd64" { + return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) + } + return fmt.Sprintf("%s-%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion, arch) } -func fetchRegistryImageAndImageID(plugin *plugin.Plugin) (string, string, error) { - identity := plugin.Identity - imageName := fmt.Sprintf("ghcr.io/%s/plugins-%s-%s:%s", release.GithubOwnerBufbuild, identity.Owner(), identity.Plugin(), plugin.PluginVersion) +func fetchRegistryImageAndImageID(plugin *plugin.Plugin, arch string) (string, string, error) { + imageName := docker.ImageNameForArch(plugin, fmt.Sprintf("ghcr.io/%s", release.GithubOwnerBufbuild), arch) parsedName, err := name.ParseReference(imageName) if err != nil { return "", "", err diff --git a/internal/docker/name.go b/internal/docker/name.go index bc43a0601..f21a6543e 100644 --- a/internal/docker/name.go +++ b/internal/docker/name.go @@ -2,12 +2,22 @@ package docker import ( "fmt" + "runtime" "github.com/bufbuild/plugins/internal/plugin" ) // ImageName returns the name of the plugin's tagged image in the given organization. func ImageName(plugin *plugin.Plugin, org string) string { + return ImageNameForArch(plugin, org, runtime.GOARCH) +} + +// ImageNameForArch returns the name of the plugin's tagged image in the given organization for a specific architecture. +func ImageNameForArch(plugin *plugin.Plugin, org string, arch string) string { identity := plugin.Identity - return fmt.Sprintf("%s/plugins-%s-%s:%s", org, identity.Owner(), identity.Plugin(), plugin.PluginVersion) + prefix := "plugins" + if arch == "arm64" { + prefix = "plugins-arm64" + } + return fmt.Sprintf("%s/%s-%s-%s:%s", org, prefix, identity.Owner(), identity.Plugin(), plugin.PluginVersion) } diff --git a/internal/release/release.go b/internal/release/release.go index eea37da54..5aa6d60eb 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -40,6 +40,7 @@ type PluginRelease struct { ReleaseTag string `json:"release_tag"` // GitHub release tag - i.e. 20221121.1 URL string `json:"url"` // URL to GitHub release zip file for the plugin - i.e. https://github.com/bufbuild/plugins/releases/download/20221121.1/bufbuild-connect-go-v1.1.0.zip LastUpdated time.Time `json:"last_updated"` + Arch string `json:"arch"` // architecture - amd64 or arm64 Status Status `json:"-"` Dependencies []string `json:"dependencies,omitempty"` // direct dependencies on other plugins } From d52ad4ab85bf9de1e10b8fc0c380656a5c5a3db3 Mon Sep 17 00:00:00 2001 From: Rubens Farias Date: Mon, 19 Jan 2026 16:52:01 -0500 Subject: [PATCH 2/2] ok actually make no changes to release.yaml and just upload stuff straight to gcs --- .github/workflows/ci.yml | 51 +++++++++- .github/workflows/upload.yml | 9 +- cmd/create-plugin-zips/main.go | 94 ++++++++++++++++++ cmd/download-plugins/main.go | 27 ------ internal/cmd/release/main.go | 169 +++++++++------------------------ internal/docker/name.go | 12 +-- internal/release/release.go | 1 - internal/release/zip.go | 99 +++++++++++++++++++ 8 files changed, 286 insertions(+), 176 deletions(-) create mode 100644 cmd/create-plugin-zips/main.go create mode 100644 internal/release/zip.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f76a71b77..00a844c75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ permissions: actions: read contents: read packages: write + id-token: write concurrency: ci-${{ github.ref }} @@ -62,10 +63,7 @@ jobs: build: needs: prepare if: needs.prepare.outputs.plugins != '' - strategy: - matrix: - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest-4-cores' }} + runs-on: ubuntu-latest-4-cores steps: - uses: actions/checkout@v6 with: @@ -107,3 +105,48 @@ jobs: env: PLUGINS: ${{ needs.prepare.outputs.plugins }} run: make dockerpush + + upload-arm64: + needs: [prepare, build] + if: github.repository == 'bufbuild/plugins' && github.ref == 'refs/heads/main' && needs.prepare.outputs.plugins != '' + environment: production + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v6 + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: '1.25' + check-latest: true + - name: Get buf version + shell: bash + run: | + echo BUF_VERSION=$(go list -m -f '{{.Version}}' github.com/bufbuild/buf | cut -c2-) >> $GITHUB_ENV + - uses: bufbuild/buf-action@v1 + with: + setup_only: true + - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + - name: Login to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + - name: Build and test ARM64 Docker images + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} + run: make test + - name: Auth To GCP + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + workload_identity_provider: projects/491113660045/locations/global/workloadIdentityPools/plugins-workload-pool/providers/plugins-workload-provider + service_account: buf-plugins-1-bufbuild-plugins@buf-plugins-1.iam.gserviceaccount.com + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Create ARM64 plugin zips + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} + run: go run ./cmd/create-plugin-zips -dir . -org "bufbuild" -out builds + - name: Upload ARM64 plugins to GCS + run: gsutil -m rsync -r builds gs://buf-plugins/arm64 diff --git a/.github/workflows/upload.yml b/.github/workflows/upload.yml index bfc1432f2..a2f67e5f2 100644 --- a/.github/workflows/upload.yml +++ b/.github/workflows/upload.yml @@ -48,14 +48,9 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - go run ./cmd/download-plugins -all-archs -since ${{ inputs.since }} downloads + go run ./cmd/download-plugins -since ${{ inputs.since }} downloads - name: Upload To Release Bucket - run: | - mkdir -p downloads/amd64 downloads/arm64 - find downloads -maxdepth 1 -name "*-arm64.zip" -exec mv {} downloads/arm64/ \; - find downloads -maxdepth 1 -name "*.zip" ! -name "*-arm64.zip" -exec mv {} downloads/amd64/ \; - gsutil -m rsync -r downloads/amd64 gs://buf-plugins - gsutil -m rsync -r downloads/arm64 gs://buf-plugins/arm64 + run: gsutil -m rsync -r downloads gs://buf-plugins - name: Generate Github Token id: generate_issues_token uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 diff --git a/cmd/create-plugin-zips/main.go b/cmd/create-plugin-zips/main.go new file mode 100644 index 000000000..4fd9f1bb2 --- /dev/null +++ b/cmd/create-plugin-zips/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/exec" + + "buf.build/go/interrupt" + + "github.com/bufbuild/plugins/internal/docker" + "github.com/bufbuild/plugins/internal/plugin" + "github.com/bufbuild/plugins/internal/release" +) + +func main() { + if err := run(); err != nil { + log.Fatalf("failed to create plugin zips: %v", err) + } +} + +func run() error { + var ( + dir = flag.String("dir", ".", "Directory path to plugins") + org = flag.String("org", "bufbuild", "Docker Organization (without registry)") + outDir = flag.String("out", "downloads", "Output directory for plugin zips") + ) + flag.Parse() + + ctx := interrupt.Handle(context.Background()) + basedir := *dir + + plugins, err := plugin.FindAll(basedir) + if err != nil { + return err + } + includedPlugins, err := plugin.FilterByPluginsEnv(plugins, os.Getenv("PLUGINS")) + if err != nil { + return err + } + if len(includedPlugins) == 0 { + log.Printf("no plugins to process") + return nil + } + + // Create output directory + if err := os.MkdirAll(*outDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + for _, includedPlugin := range includedPlugins { + if err := createPluginZip(ctx, includedPlugin, *org, *outDir); err != nil { + log.Printf( + "failed to process plugin %s:%s: %v", + includedPlugin.Name, + includedPlugin.PluginVersion, + err, + ) + return err + } + log.Printf("created zip for plugin %s:%s", includedPlugin.Name, includedPlugin.PluginVersion) + } + return nil +} + +func createPluginZip(ctx context.Context, plugin *plugin.Plugin, dockerOrg string, outDir string) error { + // Get image name from already-built local image + imageName := docker.ImageName(plugin, dockerOrg) + + // Get the image ID + imageID, err := getImageID(ctx, imageName) + if err != nil { + return fmt.Errorf("failed to get image ID for %s: %w (image must be built first)", imageName, err) + } + + // Create zip file + _, err = release.CreatePluginZip(ctx, outDir, plugin, imageID) + if err != nil { + return err + } + + return nil +} + +func getImageID(ctx context.Context, imageName string) (string, error) { + cmd := exec.CommandContext(ctx, "docker", "inspect", "--format={{.Id}}", imageName) + output, err := cmd.Output() + if err != nil { + return "", err + } + return string(output[:len(output)-1]), nil // trim newline +} diff --git a/cmd/download-plugins/main.go b/cmd/download-plugins/main.go index 8b57d9dab..967890128 100644 --- a/cmd/download-plugins/main.go +++ b/cmd/download-plugins/main.go @@ -13,7 +13,6 @@ import ( "net/http" "os" "path/filepath" - "runtime" "strings" "time" @@ -36,12 +35,10 @@ func run() error { minisignPublicKey string releaseTag string since time.Duration - allArchs bool ) flag.StringVar(&minisignPublicKey, "minisign-public-key", "", "path to minisign public key file (default: bufbuild/plugins public key)") flag.StringVar(&releaseTag, "release-tag", "", "release to download (default: latest release)") flag.DurationVar(&since, "since", 0, "only download plugins created/modified since this time") - flag.BoolVar(&allArchs, "all-archs", false, "download plugins for all architectures instead of just current runtime arch") flag.Parse() if len(flag.Args()) != 1 { @@ -102,30 +99,6 @@ func run() error { } for _, pluginRelease := range pluginReleases.Releases { - // Filter by architecture unless -all-archs is specified - if !allArchs { - // If Arch field is set, use it for filtering - if pluginRelease.Arch != "" { - if pluginRelease.Arch != runtime.GOARCH { - continue - } - } else { - // Fallback for older releases without Arch field - // We historically had non-suffixed amd64 images, and then added arm64 images with the -arm64 - // suffix for the arch. Thus we need a bit of a conditional check to choose which ones we're doing. - if runtime.GOARCH == "amd64" { - if !strings.HasSuffix(pluginRelease.URL, ".zip") || strings.HasSuffix(pluginRelease.URL, "-arm64.zip") { - continue - } - } else { - archSuffix := fmt.Sprintf("-%s.zip", runtime.GOARCH) - if !strings.HasSuffix(pluginRelease.URL, archSuffix) { - continue - } - } - } - } - // Filter out plugins which aren't specified in PLUGINS env var if includePlugins != nil { var matched bool diff --git a/internal/cmd/release/main.go b/internal/cmd/release/main.go index cc3ebaeba..570c55815 100644 --- a/internal/cmd/release/main.go +++ b/internal/cmd/release/main.go @@ -1,15 +1,12 @@ package main import ( - "archive/zip" "cmp" - "compress/flate" "context" "encoding/json" "errors" "flag" "fmt" - "io" "io/fs" "log" "os" @@ -28,7 +25,6 @@ import ( "github.com/google/go-github/v72/github" "golang.org/x/mod/semver" - "github.com/bufbuild/plugins/internal/docker" "github.com/bufbuild/plugins/internal/plugin" "github.com/bufbuild/plugins/internal/release" ) @@ -187,52 +183,46 @@ func (c *command) calculateNewReleasePlugins(ctx context.Context, currentRelease if err != nil { return err } - - // Process both amd64 and arm64 - for _, arch := range []string{"amd64", "arm64"} { - registryImage, imageID, err := fetchRegistryImageAndImageID(plugin, arch) + registryImage, imageID, err := fetchRegistryImageAndImageID(plugin) + if err != nil { + return err + } + identity := plugin.Identity + if registryImage == "" || imageID == "" { + log.Printf("unable to detect registry image and image ID for plugin %s/%s:%s", identity.Owner(), identity.Plugin(), plugin.PluginVersion) + return nil + } + key := pluginNameVersion{name: identity.Owner() + "/" + identity.Plugin(), version: plugin.PluginVersion} + pluginRelease := pluginNameVersionToRelease[key] + // Found existing release - only rebuild if changed image digest or buf.plugin.yaml digest + if pluginRelease.ImageID != imageID || pluginRelease.PluginYAMLDigest != pluginYamlDigest { + downloadURL := c.pluginDownloadURL(plugin, releaseName) + zipDigest, err := createPluginZip(ctx, tmpDir, plugin, registryImage, imageID) if err != nil { - log.Printf("unable to fetch %s for plugin %s/%s:%s: %v", arch, plugin.Identity.Owner(), plugin.Identity.Plugin(), plugin.PluginVersion, err) - continue - } - identity := plugin.Identity - if registryImage == "" || imageID == "" { - log.Printf("unable to detect registry image and image ID for %s plugin %s/%s:%s", arch, identity.Owner(), identity.Plugin(), plugin.PluginVersion) - continue + return err } - key := pluginNameVersion{name: identity.Owner() + "/" + identity.Plugin(), version: plugin.PluginVersion} - pluginRelease := pluginNameVersionToRelease[key] - // Found existing release - only rebuild if changed image digest or buf.plugin.yaml digest - if pluginRelease.ImageID != imageID || pluginRelease.PluginYAMLDigest != pluginYamlDigest { - downloadURL := c.pluginDownloadURL(plugin, releaseName, arch) - zipDigest, err := createPluginZip(ctx, tmpDir, plugin, registryImage, imageID, arch) - if err != nil { - return err - } - status := release.StatusUpdated - if pluginRelease.ImageID == "" { - status = release.StatusNew - } - newPlugins = append(newPlugins, release.PluginRelease{ - PluginName: fmt.Sprintf("%s/%s", identity.Owner(), identity.Plugin()), - PluginVersion: plugin.PluginVersion, - PluginZipDigest: zipDigest, - PluginYAMLDigest: pluginYamlDigest, - RegistryImage: registryImage, - ImageID: imageID, - ReleaseTag: releaseName, - URL: downloadURL, - LastUpdated: now, - Arch: arch, - Status: status, - Dependencies: pluginDependencies(plugin), - }) - } else { - log.Printf("plugin %s:%s (%s) unchanged", pluginRelease.PluginName, pluginRelease.PluginVersion, arch) - pluginRelease.Status = release.StatusExisting - pluginRelease.Dependencies = pluginDependencies(plugin) - existingPlugins = append(existingPlugins, pluginRelease) + status := release.StatusUpdated + if pluginRelease.ImageID == "" { + status = release.StatusNew } + newPlugins = append(newPlugins, release.PluginRelease{ + PluginName: fmt.Sprintf("%s/%s", identity.Owner(), identity.Plugin()), + PluginVersion: plugin.PluginVersion, + PluginZipDigest: zipDigest, + PluginYAMLDigest: pluginYamlDigest, + RegistryImage: registryImage, + ImageID: imageID, + ReleaseTag: releaseName, + URL: downloadURL, + LastUpdated: now, + Status: status, + Dependencies: pluginDependencies(plugin), + }) + } else { + log.Printf("plugin %s:%s unchanged", pluginRelease.PluginName, pluginRelease.PluginVersion) + pluginRelease.Status = release.StatusExisting + pluginRelease.Dependencies = pluginDependencies(plugin) + existingPlugins = append(existingPlugins, pluginRelease) } return nil }); err != nil { @@ -406,83 +396,17 @@ func signPluginReleases(dir string, privateKey minisign.PrivateKey) error { return nil } -func createPluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, registryImage string, imageID string, arch string) (string, error) { +func createPluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, registryImage string, imageID string) (string, error) { if err := pullImage(ctx, registryImage); err != nil { return "", err } - zipName := pluginZipName(plugin, arch) - pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName))) - if err != nil { - return "", err - } - defer func() { - if err := os.RemoveAll(pluginTempDir); err != nil { - log.Printf("failed to remove %q: %v", pluginTempDir, err) - } - }() - if err := saveImageToDir(ctx, imageID, pluginTempDir); err != nil { - return "", err - } - log.Printf("creating %s", zipName) - zipFile := filepath.Join(basedir, zipName) - zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) - if err != nil { - return "", err - } - defer func() { - if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - log.Printf("failed to close: %v", err) - } - }() - zw := zip.NewWriter(zf) - zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(w, flate.BestCompression) - }) - if err := addFileToZip(zw, plugin.Path); err != nil { - return "", err - } - if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil { - return "", err - } - if err := zw.Close(); err != nil { - return "", err - } - if err := zf.Close(); err != nil { - return "", err - } - digest, err := release.CalculateDigest(zipFile) + digest, err := release.CreatePluginZip(ctx, basedir, plugin, imageID) if err != nil { return "", err } return digest, nil } -func addFileToZip(zipWriter *zip.Writer, path string) error { - w, err := zipWriter.Create(filepath.Base(path)) - if err != nil { - return err - } - r, err := os.Open(path) - if err != nil { - return err - } - defer func() { - if err := r.Close(); err != nil { - log.Printf("failed to close: %v", err) - } - }() - if _, err := io.Copy(w, r); err != nil { - return err - } - return nil -} - -func saveImageToDir(ctx context.Context, imageRef string, dir string) error { - cmd := dockerCmd(ctx, "save", imageRef, "-o", "image.tar") - cmd.Dir = dir - return cmd.Run() -} - func createPluginReleases(dir string, plugins []release.PluginRelease) error { f, err := os.OpenFile(filepath.Join(dir, release.PluginReleasesFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { @@ -533,8 +457,8 @@ func calculateNextRelease(now time.Time, latestRelease *github.RepositoryRelease return releaseName, nil } -func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string, arch string) string { - zipName := pluginZipName(plugin, arch) +func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string) string { + zipName := release.PluginZipName(plugin) return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, zipName) } @@ -548,16 +472,9 @@ func (c *command) pluginReleasesURL(releaseName string) string { ) } -func pluginZipName(plugin *plugin.Plugin, arch string) string { +func fetchRegistryImageAndImageID(plugin *plugin.Plugin) (string, string, error) { identity := plugin.Identity - if arch == "amd64" { - return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) - } - return fmt.Sprintf("%s-%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion, arch) -} - -func fetchRegistryImageAndImageID(plugin *plugin.Plugin, arch string) (string, string, error) { - imageName := docker.ImageNameForArch(plugin, fmt.Sprintf("ghcr.io/%s", release.GithubOwnerBufbuild), arch) + imageName := fmt.Sprintf("ghcr.io/%s/plugins-%s-%s:%s", release.GithubOwnerBufbuild, identity.Owner(), identity.Plugin(), plugin.PluginVersion) parsedName, err := name.ParseReference(imageName) if err != nil { return "", "", err diff --git a/internal/docker/name.go b/internal/docker/name.go index f21a6543e..bc43a0601 100644 --- a/internal/docker/name.go +++ b/internal/docker/name.go @@ -2,22 +2,12 @@ package docker import ( "fmt" - "runtime" "github.com/bufbuild/plugins/internal/plugin" ) // ImageName returns the name of the plugin's tagged image in the given organization. func ImageName(plugin *plugin.Plugin, org string) string { - return ImageNameForArch(plugin, org, runtime.GOARCH) -} - -// ImageNameForArch returns the name of the plugin's tagged image in the given organization for a specific architecture. -func ImageNameForArch(plugin *plugin.Plugin, org string, arch string) string { identity := plugin.Identity - prefix := "plugins" - if arch == "arm64" { - prefix = "plugins-arm64" - } - return fmt.Sprintf("%s/%s-%s-%s:%s", org, prefix, identity.Owner(), identity.Plugin(), plugin.PluginVersion) + return fmt.Sprintf("%s/plugins-%s-%s:%s", org, identity.Owner(), identity.Plugin(), plugin.PluginVersion) } diff --git a/internal/release/release.go b/internal/release/release.go index 5aa6d60eb..eea37da54 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -40,7 +40,6 @@ type PluginRelease struct { ReleaseTag string `json:"release_tag"` // GitHub release tag - i.e. 20221121.1 URL string `json:"url"` // URL to GitHub release zip file for the plugin - i.e. https://github.com/bufbuild/plugins/releases/download/20221121.1/bufbuild-connect-go-v1.1.0.zip LastUpdated time.Time `json:"last_updated"` - Arch string `json:"arch"` // architecture - amd64 or arm64 Status Status `json:"-"` Dependencies []string `json:"dependencies,omitempty"` // direct dependencies on other plugins } diff --git a/internal/release/zip.go b/internal/release/zip.go new file mode 100644 index 000000000..695723367 --- /dev/null +++ b/internal/release/zip.go @@ -0,0 +1,99 @@ +package release + +import ( + "archive/zip" + "compress/flate" + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/bufbuild/plugins/internal/plugin" +) + +func addFileToZip(zipWriter *zip.Writer, path string) error { + w, err := zipWriter.Create(filepath.Base(path)) + if err != nil { + return err + } + r, err := os.Open(path) + if err != nil { + return err + } + defer func() { + if err := r.Close(); err != nil { + log.Printf("failed to close: %v", err) + } + }() + if _, err := io.Copy(w, r); err != nil { + return err + } + return nil +} + +func saveImageToDir(ctx context.Context, imageRef string, dir string) error { + cmd := exec.CommandContext(ctx, "docker", "save", imageRef, "-o", "image.tar") + cmd.Dir = dir + return cmd.Run() +} + +func PluginZipName(plugin *plugin.Plugin) string { + identity := plugin.Identity + return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) +} + +// CreatePluginZip creates a plugin zip file containing the buf.plugin.yaml and Docker image. +// Returns the path to the created zip file and a digest. +func CreatePluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, imageID string) (string, error) { + zipName := PluginZipName(plugin) + pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName))) + if err != nil { + return "", err + } + defer func() { + if err := os.RemoveAll(pluginTempDir); err != nil { + log.Printf("failed to remove %q: %v", pluginTempDir, err) + } + }() + if err := saveImageToDir(ctx, imageID, pluginTempDir); err != nil { + return "", err + } + log.Printf("creating %s", zipName) + zipFile := filepath.Join(basedir, zipName) + zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + return "", err + } + defer func() { + if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + log.Printf("failed to close: %v", err) + } + }() + zw := zip.NewWriter(zf) + zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(w, flate.BestCompression) + }) + if err := addFileToZip(zw, plugin.Path); err != nil { + return "", err + } + if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil { + return "", err + } + if err := zw.Close(); err != nil { + return "", err + } + if err := zf.Close(); err != nil { + return "", err + } + + digest, err := CalculateDigest(zipFile) + if err != nil { + return "", err + } + return digest, nil +}