diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5125f2163..00a844c75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ permissions: actions: read contents: read packages: write + id-token: write concurrency: ci-${{ github.ref }} @@ -25,12 +26,11 @@ defaults: shell: bash jobs: - ci: + prepare: runs-on: ubuntu-latest-4-cores + outputs: + plugins: ${{ steps.set-plugins.outputs.plugins }} steps: - - name: set PLUGINS from workflow inputs - if: ${{ inputs.plugins }} - run: echo "PLUGINS=${{ inputs.plugins }}" >> $GITHUB_ENV - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -45,15 +45,34 @@ jobs: with: go-version: '1.25' check-latest: true - - name: Calculate changed plugins and set PLUGINS env var from last successful commit push - if: ${{ inputs.plugins == '' }} + - name: Calculate plugins to build + id: set-plugins env: BASE_REF: ${{ steps.last_successful_commit_push.outputs.base }} run: | - val=`go run ./internal/cmd/changed-plugins .` - if [[ -n "${val}" && -z "${PLUGINS}" ]]; then - echo "PLUGINS=${val}" >> $GITHUB_ENV + if [[ -n "${{ inputs.plugins }}" ]]; then + PLUGINS="${{ inputs.plugins }}" + else + PLUGINS=`go run ./internal/cmd/changed-plugins .` fi + echo "plugins=${PLUGINS}" >> $GITHUB_OUTPUT + echo "Building plugins: ${PLUGINS}" + - name: Run Go tests + run: go test -race -count=1 ./... + + build: + needs: prepare + if: needs.prepare.outputs.plugins != '' + runs-on: ubuntu-latest-4-cores + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: '1.25' + check-latest: true - name: Get buf version shell: bash run: | @@ -77,8 +96,57 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Test + - name: Build and test Docker images + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} run: make test - name: Push to GHCR if: github.repository == 'bufbuild/plugins' + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} run: make dockerpush + + upload-arm64: + needs: [prepare, build] + if: github.repository == 'bufbuild/plugins' && github.ref == 'refs/heads/main' && needs.prepare.outputs.plugins != '' + environment: production + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v6 + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: '1.25' + check-latest: true + - name: Get buf version + shell: bash + run: | + echo BUF_VERSION=$(go list -m -f '{{.Version}}' github.com/bufbuild/buf | cut -c2-) >> $GITHUB_ENV + - uses: bufbuild/buf-action@v1 + with: + setup_only: true + - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 + - name: Login to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + - name: Build and test ARM64 Docker images + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} + run: make test + - name: Auth To GCP + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + workload_identity_provider: projects/491113660045/locations/global/workloadIdentityPools/plugins-workload-pool/providers/plugins-workload-provider + service_account: buf-plugins-1-bufbuild-plugins@buf-plugins-1.iam.gserviceaccount.com + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Create ARM64 plugin zips + env: + PLUGINS: ${{ needs.prepare.outputs.plugins }} + run: go run ./cmd/create-plugin-zips -dir . -org "bufbuild" -out builds + - name: Upload ARM64 plugins to GCS + run: gsutil -m rsync -r builds gs://buf-plugins/arm64 diff --git a/cmd/create-plugin-zips/main.go b/cmd/create-plugin-zips/main.go new file mode 100644 index 000000000..4fd9f1bb2 --- /dev/null +++ b/cmd/create-plugin-zips/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/exec" + + "buf.build/go/interrupt" + + "github.com/bufbuild/plugins/internal/docker" + "github.com/bufbuild/plugins/internal/plugin" + "github.com/bufbuild/plugins/internal/release" +) + +func main() { + if err := run(); err != nil { + log.Fatalf("failed to create plugin zips: %v", err) + } +} + +func run() error { + var ( + dir = flag.String("dir", ".", "Directory path to plugins") + org = flag.String("org", "bufbuild", "Docker Organization (without registry)") + outDir = flag.String("out", "downloads", "Output directory for plugin zips") + ) + flag.Parse() + + ctx := interrupt.Handle(context.Background()) + basedir := *dir + + plugins, err := plugin.FindAll(basedir) + if err != nil { + return err + } + includedPlugins, err := plugin.FilterByPluginsEnv(plugins, os.Getenv("PLUGINS")) + if err != nil { + return err + } + if len(includedPlugins) == 0 { + log.Printf("no plugins to process") + return nil + } + + // Create output directory + if err := os.MkdirAll(*outDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + for _, includedPlugin := range includedPlugins { + if err := createPluginZip(ctx, includedPlugin, *org, *outDir); err != nil { + log.Printf( + "failed to process plugin %s:%s: %v", + includedPlugin.Name, + includedPlugin.PluginVersion, + err, + ) + return err + } + log.Printf("created zip for plugin %s:%s", includedPlugin.Name, includedPlugin.PluginVersion) + } + return nil +} + +func createPluginZip(ctx context.Context, plugin *plugin.Plugin, dockerOrg string, outDir string) error { + // Get image name from already-built local image + imageName := docker.ImageName(plugin, dockerOrg) + + // Get the image ID + imageID, err := getImageID(ctx, imageName) + if err != nil { + return fmt.Errorf("failed to get image ID for %s: %w (image must be built first)", imageName, err) + } + + // Create zip file + _, err = release.CreatePluginZip(ctx, outDir, plugin, imageID) + if err != nil { + return err + } + + return nil +} + +func getImageID(ctx context.Context, imageName string) (string, error) { + cmd := exec.CommandContext(ctx, "docker", "inspect", "--format={{.Id}}", imageName) + output, err := cmd.Output() + if err != nil { + return "", err + } + return string(output[:len(output)-1]), nil // trim newline +} diff --git a/internal/cmd/release/main.go b/internal/cmd/release/main.go index 99493c724..570c55815 100644 --- a/internal/cmd/release/main.go +++ b/internal/cmd/release/main.go @@ -1,15 +1,12 @@ package main import ( - "archive/zip" "cmp" - "compress/flate" "context" "encoding/json" "errors" "flag" "fmt" - "io" "io/fs" "log" "os" @@ -403,79 +400,13 @@ func createPluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, if err := pullImage(ctx, registryImage); err != nil { return "", err } - zipName := pluginZipName(plugin) - pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName))) - if err != nil { - return "", err - } - defer func() { - if err := os.RemoveAll(pluginTempDir); err != nil { - log.Printf("failed to remove %q: %v", pluginTempDir, err) - } - }() - if err := saveImageToDir(ctx, imageID, pluginTempDir); err != nil { - return "", err - } - log.Printf("creating %s", zipName) - zipFile := filepath.Join(basedir, zipName) - zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) - if err != nil { - return "", err - } - defer func() { - if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - log.Printf("failed to close: %v", err) - } - }() - zw := zip.NewWriter(zf) - zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(w, flate.BestCompression) - }) - if err := addFileToZip(zw, plugin.Path); err != nil { - return "", err - } - if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil { - return "", err - } - if err := zw.Close(); err != nil { - return "", err - } - if err := zf.Close(); err != nil { - return "", err - } - digest, err := release.CalculateDigest(zipFile) + digest, err := release.CreatePluginZip(ctx, basedir, plugin, imageID) if err != nil { return "", err } return digest, nil } -func addFileToZip(zipWriter *zip.Writer, path string) error { - w, err := zipWriter.Create(filepath.Base(path)) - if err != nil { - return err - } - r, err := os.Open(path) - if err != nil { - return err - } - defer func() { - if err := r.Close(); err != nil { - log.Printf("failed to close: %v", err) - } - }() - if _, err := io.Copy(w, r); err != nil { - return err - } - return nil -} - -func saveImageToDir(ctx context.Context, imageRef string, dir string) error { - cmd := dockerCmd(ctx, "save", imageRef, "-o", "image.tar") - cmd.Dir = dir - return cmd.Run() -} - func createPluginReleases(dir string, plugins []release.PluginRelease) error { f, err := os.OpenFile(filepath.Join(dir, release.PluginReleasesFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { @@ -527,7 +458,7 @@ func calculateNextRelease(now time.Time, latestRelease *github.RepositoryRelease } func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string) string { - zipName := pluginZipName(plugin) + zipName := release.PluginZipName(plugin) return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, zipName) } @@ -541,11 +472,6 @@ func (c *command) pluginReleasesURL(releaseName string) string { ) } -func pluginZipName(plugin *plugin.Plugin) string { - identity := plugin.Identity - return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) -} - func fetchRegistryImageAndImageID(plugin *plugin.Plugin) (string, string, error) { identity := plugin.Identity imageName := fmt.Sprintf("ghcr.io/%s/plugins-%s-%s:%s", release.GithubOwnerBufbuild, identity.Owner(), identity.Plugin(), plugin.PluginVersion) diff --git a/internal/release/zip.go b/internal/release/zip.go new file mode 100644 index 000000000..695723367 --- /dev/null +++ b/internal/release/zip.go @@ -0,0 +1,99 @@ +package release + +import ( + "archive/zip" + "compress/flate" + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/bufbuild/plugins/internal/plugin" +) + +func addFileToZip(zipWriter *zip.Writer, path string) error { + w, err := zipWriter.Create(filepath.Base(path)) + if err != nil { + return err + } + r, err := os.Open(path) + if err != nil { + return err + } + defer func() { + if err := r.Close(); err != nil { + log.Printf("failed to close: %v", err) + } + }() + if _, err := io.Copy(w, r); err != nil { + return err + } + return nil +} + +func saveImageToDir(ctx context.Context, imageRef string, dir string) error { + cmd := exec.CommandContext(ctx, "docker", "save", imageRef, "-o", "image.tar") + cmd.Dir = dir + return cmd.Run() +} + +func PluginZipName(plugin *plugin.Plugin) string { + identity := plugin.Identity + return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) +} + +// CreatePluginZip creates a plugin zip file containing the buf.plugin.yaml and Docker image. +// Returns the path to the created zip file and a digest. +func CreatePluginZip(ctx context.Context, basedir string, plugin *plugin.Plugin, imageID string) (string, error) { + zipName := PluginZipName(plugin) + pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName))) + if err != nil { + return "", err + } + defer func() { + if err := os.RemoveAll(pluginTempDir); err != nil { + log.Printf("failed to remove %q: %v", pluginTempDir, err) + } + }() + if err := saveImageToDir(ctx, imageID, pluginTempDir); err != nil { + return "", err + } + log.Printf("creating %s", zipName) + zipFile := filepath.Join(basedir, zipName) + zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + return "", err + } + defer func() { + if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + log.Printf("failed to close: %v", err) + } + }() + zw := zip.NewWriter(zf) + zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(w, flate.BestCompression) + }) + if err := addFileToZip(zw, plugin.Path); err != nil { + return "", err + } + if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil { + return "", err + } + if err := zw.Close(); err != nil { + return "", err + } + if err := zf.Close(); err != nil { + return "", err + } + + digest, err := CalculateDigest(zipFile) + if err != nil { + return "", err + } + return digest, nil +}