[build-and-test] Use buildbuddy remote execution for build and test actions. (#1406)

Summary: Uses buildbuddy remote execution to run build and test bazel
commands. Uses Github "environments" to access the authorize access to
the BB_IO_API_KEY secret.

Type of change: /kind test-infra.

Test Plan: Tested on my fork. Had @vihangm create a PR, saw that it
required me to approve the environment before it would run.

Signed-off-by: James Bartlett <jamesbartlett@pixielabs.ai>

Author: JamesMBartlett

.bazelrc

@@ -228,19 +228,19 @@ coverage --define PL_COVERAGE=true
 coverage --copt -DPL_COVERAGE
 coverage --test_tag_filters=-requires_root,-requires_bpf,-no_coverage,-disabled,-no_gcc
 
+
+try-import %workspace%/bes.bazelrc
 # jenkins.bazelrc is copied from ci/jenkins.bazelrc by Jenkins workers during the build.
 # The intention is to avoid polluting configurations of bazel for developers.
 try-import %workspace%/jenkins.bazelrc
 # github.bazelrc is copied from ci/github/bazelrc by the github action workers during the build.
 try-import %workspace%/github.bazelrc
 
-# Put your own configurations into user.bazelrc, which is ignored by git.
-try-import %workspace%/user.bazelrc
-
 # Import a machine specific bazelrc. This can be used to enable caching.
 try-import /etc/bazelrc
 
-try-import %workspace%/bes.bazelrc
+# Put your own configurations into user.bazelrc, which is ignored by git.
+try-import %workspace%/user.bazelrc
 
 # Tensorflow requires this option
 common --experimental_repo_remote_exec
@@ -263,5 +263,6 @@ build:remote --remote_retries=5
 build:remote --spawn_strategy=remote,local
 build:remote --experimental_remote_cache_compression
 build:remote --jobs=100
+test:remote --jobs=100
 build:remote --nolegacy_important_outputs
 build:remote --build_metadata=VISIBILITY=PUBLIC

.github/actions/bazelrc/action.yaml

@@ -8,6 +8,12 @@ inputs:
   dev:
     description: 'Whether to use DEV or CI settings for the bazelrc. defaults to dev'
     default: 'true'
+  use_remote_exec:
+    description: 'Use buildbuddy remote execution'
+    default: 'false'
+  BB_API_KEY:
+    description: 'API key to use for buildbuddy if `use_remote_exec`'
+    default: ''
 runs:
   using: "composite"
   steps:
@@ -35,3 +41,11 @@ runs:
     run: |
       echo "build --build_metadata=ROLE=CI" >> github.bazelrc
     shell: bash
+  - name: Add remote execution
+    if: inputs.use_remote_exec == 'true'
+    env:
+      BB_API_KEY: ${{ inputs.BB_API_KEY }}
+    run: |
+      echo "build:remote --remote_header=x-buildbuddy-api-key=$BB_API_KEY" >> github.bazelrc
+      echo "build --config=remote" >> github.bazelrc
+    shell: bash

.github/actions/env_protected_pr/action.yaml

@@ -0,0 +1,53 @@
+---
+name: env-protected-pr
+description: Setups actions that run on pull_request_target events, protected by github's deployment environments.
+outputs:
+  env-name:
+    description: 'name of deployment environment to use'
+    value: ${{ steps.output.outputs.env-name }}
+  ref:
+    description: 'ref to checkout'
+    value: ${{ steps.output.outputs.ref }}
+runs:
+  using: "composite"
+  steps:
+  - name: Not pull_request_target
+    # yamllint disable rule:indentation
+    if: github.event_name != 'pull_request_target'
+    shell: bash
+    run:
+      echo "" > env_name
+      echo "${{ github.ref }}" >> ref
+  - name: Member pull_request_target
+    if: >-
+      github.event_name == 'pull_request_target' &&
+      (
+        github.event.pull_request.author_association == 'OWNER' ||
+        github.event.pull_request.author_association == 'MEMBER'
+      )
+    # yamllint enable rule:indentation
+    shell: bash
+    run: |
+      echo "" > env_name
+      echo "${{ github.event.pull_request.head.sha  }}" >> ref
+  - name: Require external environment authorization.
+    # yamllint disable rule:indentation
+    if: >-
+      github.event_name == 'pull_request_target' &&
+      !(
+        github.event.pull_request.author_association == 'OWNER' ||
+        github.event.pull_request.author_association == 'MEMBER'
+      )
+    # yamllint enable rule:indentation
+    shell: bash
+    run: |
+      echo "pr-actions-approval" > env_name
+      echo "${{ github.event.pull_request.head.sha  }}" >> ref
+  - name: Set Output
+    id: output
+    shell: bash
+    run: |
+      echo "env: $(cat env_name)"
+      echo "ref: $(cat ref)"
+      echo "env-name=$(cat env_name)" >> $GITHUB_OUTPUT
+      echo "ref=$(cat ref)" >> $GITHUB_OUTPUT

.github/workflows/build_and_test.yaml

@@ -1,7 +1,7 @@
 ---
 name: build-and-test
 on:
-  pull_request:
+  pull_request_target:
   push:
     branches:
     - 'main'
@@ -11,26 +11,43 @@ on:
 permissions:
   contents: read
 jobs:
+  env-protect-setup:
+    runs-on: ubuntu-latest
+    outputs:
+      env-name: ${{ steps.output.outputs.env-name }}
+      ref: ${{ steps.output.outputs.ref }}
+    steps:
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3  # v3.5.0
+    - id: output
+      uses: ./.github/actions/env_protected_pr
+  authorize:
+    runs-on: ubuntu-latest
+    needs: env-protect-setup
+    environment: ${{ needs.env-protect-setup.outputs.env-name }}
+    steps:
+    - run: echo "Authorized"
   get-dev-image:
+    needs: authorize
     uses: ./.github/workflows/get_image.yaml
     with:
       image-base-name: "dev_image_with_extras"
   clang-tidy:
-    needs: get-dev-image
-    runs-on: [self-hosted, nokvm]
+    runs-on: ubuntu-latest-16-cores
+    needs: [authorize, env-protect-setup, get-dev-image]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
-      volumes:
-      - /etc/bazelrc:/etc/bazelrc
-      options: --cpus 16
     steps:
     - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3  # v3.5.0
       with:
         fetch-depth: 0
+        ref: ${{ needs.env-protect-setup.outputs.ref }}
     - name: Add pwd to git safe dir
       run: git config --global --add safe.directory `pwd`
     - name: get bazel config
       uses: ./.github/actions/bazelrc
+      with:
+        use_remote_exec: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
     - name: Save Diff Info
       run: ./ci/save_diff_info.sh
     - name: Run Clang Tidy
@@ -45,24 +62,23 @@ jobs:
       # yamllint enable rule:indentation
   code-coverage:
     if: github.event_name == 'push'
-    needs: get-dev-image
-    runs-on: [self-hosted, nokvm]
+    needs: [authorize, env-protect-setup, get-dev-image]
+    runs-on: ubuntu-latest-16-cores
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
-      volumes:
-      - /etc/bazelrc:/etc/bazelrc
-      # Needs to be priviledged to enable IPV6
-      options: --cpus 16 --privileged
     steps:
     - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3  # v3.5.0
       with:
         fetch-depth: 0
+        ref: ${{ needs.env-protect-setup.outputs.ref }}
     - name: Add pwd to git safe dir
       run: git config --global --add safe.directory `pwd`
     - name: get bazel config
       uses: ./.github/actions/bazelrc
       with:
         dev: 'false'
+        use_remote_exec: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
     - name: Collect and upload coverage
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -71,23 +87,24 @@ jobs:
         sysctl -w net.ipv6.conf.lo.disable_ipv6=0
         ./ci/collect_coverage.sh -u -b main -c "$(git rev-parse HEAD)" -r pixie-io/pixie
   generate-matrix:
-    needs: get-dev-image
-    runs-on: [self-hosted, nokvm]
+    needs: [authorize, env-protect-setup, get-dev-image]
+    runs-on: ubuntu-latest-16-cores
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
-      volumes:
-      - /etc/bazelrc:/etc/bazelrc
-      options: --cpus 16
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
     - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3  # v3.5.0
       with:
         fetch-depth: 0
+        ref: ${{ needs.env-protect-setup.outputs.ref }}
     - name: Add pwd to git safe dir
       run: git config --global --add safe.directory `pwd`
     - name: get bazel config
       uses: ./.github/actions/bazelrc
+      with:
+        use_remote_exec: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
     - name: Set matrix
       id: set-matrix
       shell: bash
@@ -103,22 +120,19 @@ jobs:
           bazel_buildables_*
           bazel_tests_*
   build-and-test:
-    needs: [get-dev-image, generate-matrix]
-    runs-on:
-    - self-hosted
-    - ${{ matrix.runner }}
+    needs: [authorize, env-protect-setup, get-dev-image, generate-matrix]
+    runs-on: ubuntu-latest-16-cores
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
-      volumes:
-      - /etc/bazelrc:/etc/bazelrc
-      - /var/run/docker.sock:/var/run/docker.sock
       options: --privileged
     if: ${{ needs.generate-matrix.outputs.matrix && (toJson(fromJson(needs.generate-matrix.outputs.matrix)) != '[]') }}
     strategy:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
       fail-fast: false
     steps:
     - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3  # v3.5.0
+      with:
+        ref: ${{ needs.env-protect-setup.outputs.ref }}
     - name: Add pwd to git safe dir
       run: git config --global --add safe.directory `pwd`
     - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a  # v3.0.2
@@ -127,11 +141,15 @@ jobs:
       uses: ./.github/actions/bazelrc
       with:
         dev: 'true'
+        use_remote_exec: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
     - name: get ci bazel config
       if: github.event_name == 'push'
       uses: ./.github/actions/bazelrc
       with:
         dev: 'false'
+        use_remote_exec: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
     - name: Build ${{ matrix.name }}
       shell: bash
       # yamllint disable rule:indentation

ci/github/matrix.sh

@@ -58,7 +58,7 @@ elif [[ "${event_name}" == "schedule" ]]; then
   build_deps_flags+=("-a" "-b")
   extra_bazel_args+=("--runs_per_test=${nightly_regression_test_iterations}")
   kernel_versions=( "${all_kernel_versions[@]}" )
-elif [[ "${event_name}" == "pull_request" ]]; then
+elif [[ "${event_name}" == "pull_request_target" ]] || [[ "${event_name}" == "pull_request" ]]; then
   # Ignore bazel dependency tracking and run all targets if #ci:ignore-deps is in the commit message.
   if check_tag '#ci:ignore-deps'; then
     echo "Found #ci:ignore-deps tag. Building all targets" >&2