[ci] Add autoscaling to executors and make API_KEY external secret (#2188)

Summary: This adds autoscaling and moves the secret to be externally
specified.

Relevant Issues: N/A

Type of change: /kind infra

Test Plan: Check this CI run and see if the executors scale up.

Signed-off-by: Vihang Mehta <vihang@gimletlabs.ai>

Author: vihangm

k8s/devinfra/buildbuddy-executor/README.md

@@ -4,10 +4,15 @@ Bazel RBE with Buildbuddy
 To install buildbuddy executor to a k8s cluster, run the following from the workspace root.
 
 ```bash
+export BB_EXECUTOR_API_KEY=""
+export NAMESPACE=""
+
 helm repo add buildbuddy https://helm.buildbuddy.io
 
-BB_EXECUTOR_API_KEY=<api_key> \
+kubectl create namespace "${NAMESPACE}" || true
+kubectl create secret -n "${NAMESPACE}" generic buildbuddy-executor-api-key --from-literal=api-key="${BB_EXECUTOR_API_KEY}" || true
+
 IMAGE_TAG="$(grep DOCKER_IMAGE_TAG "docker.properties" | cut -d= -f2)" \
-envsubst < k8s/devinfra/buildbuddy-executor/values.yaml | \
-helm upgrade --install -f - buildbuddy buildbuddy/buildbuddy-executor --create-namespace -n pixie-executors
+envsubst '$IMAGE_TAG' < k8s/devinfra/buildbuddy-executor/values.yaml | \
+helm upgrade --install -f - buildbuddy buildbuddy/buildbuddy-executor -n "${NAMESPACE}"
 ```

k8s/devinfra/buildbuddy-executor/values.yaml

@@ -1,22 +1,37 @@
 ---
-replicas: 32
+replicas: 16
+
+autoscaler:
+  enabled: true
+  minReplicas: 8
+  maxReplicas: 48
+  metrics:
+  - type: ContainerResource
+    containerResource:
+      name: cpu
+      container: buildbuddy-executor
+      target:
+        # These executors idle at a very low CPU usage, so any utilization should trigger some scaling.
+        type: Utilization
+        averageUtilization: 20
 
 resources:
   limits:
     memory: 8Gi
     cpu: null
   requests:
-    cpu: null
+    cpu: 4
 config:
+  server_type: buildbuddy-executor
   executor:
     app_target: "grpcs://remote.buildbuddy.io:443"
-    api_key: $BB_EXECUTOR_API_KEY
+    api_key: "${API_KEY}"
     default_isolation_type: none
     docker_socket: ""
     enable_podman: false
     enable_oci: false
     enable_bare_runner: true
-    local_cache_size_bytes: 375000000000  # 375GB
+    local_cache_size_bytes: 20000000000  # 20GB
     millicpu: 4000
 
 extraInitContainers:
@@ -41,7 +56,7 @@ image:
 customExecutorCommand:
 - /bin/sh
 - -c
-- 'sysctl -w net.ipv6.conf.lo.disable_ipv6=0 && /bb-executor/executor --server_type=buildbuddy-executor'
+- 'sysctl -w net.ipv6.conf.lo.disable_ipv6=0 && /bb-executor/executor'
 
 poolName: '"$IMAGE_TAG"'
 
@@ -59,6 +74,12 @@ extraVolumes:
   emptyDir: {}
 - name: tmp
   emptyDir: {}
+extraEnvVars:
+- name: API_KEY
+  valueFrom:
+    secretKeyRef:
+      name: buildbuddy-executor-api-key
+      key: api-key
 
 containerSecurityContext:
   privileged: true