Support KAS

Dockerfile

@@ -10,6 +10,7 @@ ENV GITLAB_VERSION=${VERSION} \
     GITLAB_SHELL_VERSION=14.45.5 \
     GITLAB_PAGES_VERSION=18.8.4 \
     GITALY_SERVER_VERSION=18.8.4 \
+    GITLAB_AGENT_VERSION=18.8.4 \
     GITLAB_USER="git" \
     GITLAB_HOME="/home/git" \
     GITLAB_LOG_DIR="/var/log/gitlab" \
@@ -21,6 +22,7 @@ ENV GITLAB_VERSION=${VERSION} \
 ENV GITLAB_INSTALL_DIR="${GITLAB_HOME}/gitlab" \
     GITLAB_SHELL_INSTALL_DIR="${GITLAB_HOME}/gitlab-shell" \
     GITLAB_GITALY_INSTALL_DIR="${GITLAB_HOME}/gitaly" \
+    GITLAB_AGENT_INSTALL_DIR="${GITLAB_HOME}/gitlab-agent" \
     GITLAB_DATA_DIR="${GITLAB_HOME}/data" \
     GITLAB_BUILD_DIR="${GITLAB_CACHE_DIR}/build" \
     GITLAB_RUNTIME_DIR="${GITLAB_CACHE_DIR}/runtime"

README.md

@@ -54,6 +54,7 @@
     - [Piwik](#piwik)
     - [Feature flags](#feature-flags)
     - [Exposing ssh port in dockerized gitlab-ce](docs/exposing-ssh-port.md)
+    - [Gitlab KAS](#gitlab-kas)
     - [Available Configuration Parameters](#available-configuration-parameters)
 - [Maintenance](#maintenance)
     - [Creating Backups](#creating-backups)
@@ -914,6 +915,80 @@ Configuring gitlab::feature_flags...
 ...
 ````
 
+#### Gitlab KAS
+
+GitLab agent server for Kubernetes (KAS) is disabled by default, but you can enable it by setting configuration parameter [`GITLAB_KAS_ENABLED`](#gitlab_kas_enabled) to true.  
+By default, built-in `gitlab-kas` is also enabled once you enable KAS feature. But you can use an external installation of KAS by setting internal URL for the GitLab backend. Corresponding configuration parameter is [`GITLAB_KAS_INTERNAL`](#gitlab_kas_internal).  
+You can specify user-facing URL by setting [`GITLAB_KAS_EXTERNAL`](#gitlab_kas_external). If you set up proxy URL, use `GITLAB_KAS_PROXY`.
+
+You can specify custom secret file by setting [`GITLAB_KAS_SECRET`](#gitlab_kas_secret). This secret file will be generated if they don't exist.
+
+Here is an example settings for kubernetes rc.yml:
+
+```yaml
+spec:
+  containers:
+  - name: gitlab
+  image: sameersbn/gitlab:latest
+  env:
+  - name: GITLAB_KAS_ENABLED
+    value: "true"
+  - name: GITLAB_AGENT_BUILTIN_KAS_ENABLED
+    value: "true"
+  - name: GITLAB_KAS_EXTERNAL
+    value: wss://gitlab.example.com/gitlab/-/kubernetes-agent/
+  - name: GITLAB_KAS_INTERNAL
+    value: grpc://127.0.0.1:8153
+  - name: GITLAB_KAS_PROXY
+    value: https://gitlab.example.com/gitlab/-/kubernetes-agent/k8s-proxy/
+  - name: OWN_PRIVATE_API_URL
+    value: grpc://127.0.0.1:8155
+```
+
+and for docker-compose.yml:
+
+```yaml
+services:
+  gitlab:
+    image: sameersbn/gitlab:latest
+  environment:
+    - GITLAB_KAS_ENABLED=true
+    - GITLAB_AGENT_BUILTIN_KAS_ENABLED=true
+    - GITLAB_KAS_EXTERNAL=wss://gitlab.example.com/gitlab/-/kubernetes-agent/
+    - GITLAB_KAS_INTERNAL=grpc://127.0.0.1:8153
+    - GITLAB_KAS_PROXY=https://gitlab.example.com/gitlab/-/kubernetes-agent/k8s-proxy/
+    - OWN_PRIVATE_API_URL=grpc://127.0.0.1:8155
+```
+
+or in another style:
+
+```yaml
+services:
+  gitlab:
+    image: sameersbn/gitlab:latest
+  environment:
+    GITLAB_KAS_ENABLED: "true"
+    GITLAB_AGENT_BUILTIN_KAS_ENABLED: "true"
+    GITLAB_KAS_EXTERNAL: wss://gitlab.example.com/gitlab/-/kubernetes-agent/
+    GITLAB_KAS_INTERNAL: grpc://127.0.0.1:8153
+    GITLAB_KAS_PROXY: https://gitlab.example.com/gitlab/-/kubernetes-agent/k8s-proxy/
+    OWN_PRIVATE_API_URL: grpc://127.0.0.1:8155
+```
+
+#### Built-in GitLab-Agent KAS
+
+To control whether launch built-in `gitlab-kas` on container startup or not, you can use configuration parameter [`GITLAB_AGENT_BUILTIN_KAS_ENABLED`](#gitlab_agent_builtin_kas_enabled).
+
+You can specify custom secret file by setting [`GITLAB_AGENT_KAS_API_LISTEN_AUTHENTICATION_SECRET_FILE`](#gitlab_agent_kas_api_listen_authentication_secret_file) and [`GITLAB_AGENT_KAS_PRIVATE_API_LISTEN_AUTHENTICATION_SECRET_FILE`](#gitlab_agent_kas_private_api_listen_authentication_secret_file). These secret files also be generated if they don't exist.  
+Authentication secret file will be set to same value of `GITLAB_AGENT_KAS_API_LISTEN_AUTHENTICATION_SECRET_FILE` but you can overwrite it by setting [`GITLAB_AGENT_KAS_GITLAB_AUTHENTICATION_SECRET_FILE`](#gitlab_agent_kas_gitlab_authentication_secret_file).
+
+Built-in KAS communicates to redis. The host and ports are set using `REDIS_HOST` and `REDIS_PORT`.  
+You can specify the password file path in `GITLAB_AGENT_KAS_REDIS_PASSWORD_FILE`, but please do not set the parameter. We still do not support password authentication for Redis. The password file should contain the redis authentication password, but this is not currently done because there is no way to specify the redis password. So please let this parameter empty. See [sameersbn/gitlab#1026](https://github.com/sameersbn/docker-gitlab/pull/1026)
+
+Also note that KAS requires that environment variable `OWN_PRIVATE_API_URL` is set (e.g. `OWN_PRIVATE_API_URL=grpc://127.0.0.1:8155`). If not, the KAS service will keep restarting.
+
+See [official documentation](https://docs.gitlab.com/ee/administration/clusters/kas.html) for more detail.
+
 #### Available Configuration Parameters
 
 *Please refer the docker run command options for the `--env-file` flag where you can specify all required environment variables in a single file. This will save you from writing a potentially long docker run command. Alternatively you can use docker-compose. docker-compose users and Docker Swarm mode users can also use the [secrets and config file options](#docker-secrets-and-configs)*
@@ -1236,6 +1311,52 @@ Default Google key file. Defaults to `$GITLAB_OBJECT_STORE_CONNECTION_GOOGLE_JSO
 
 Cron notation for the GitLab pipeline schedule worker. Defaults to `'19 * * * *'`
 
+##### `GITLAB_KAS_ENABLED`
+
+Enable/Disable GitLab agent server for Kubernetes (KAS). See details on [official documentation](https://docs.gitlab.com/ee/administration/clusters/kas.html). Defaults to `false`
+
+##### `GITLAB_KAS_SECRET`
+
+File that contains the secret key for verifying access for GitLab KAS. This value will be used for `production.gitlab_kas.secret_file` in gitlab.yml. Defaults to `${GITLAB_AGENT_KAS_GITLAB_AUTHENTICATION_SECRET_FILE}`
+
+##### `GITLAB_KAS_EXTERNAL`
+
+User-facing URL for the in-cluster agent. Defaults to `"wss://kas.example.com"`
+
+##### `GITLAB_KAS_INTERNAL`
+
+Internal URL for the GitLab backend. Defaults to `"grpc://localhost:8153"`
+
+##### `GITLAB_KAS_PROXY`
+
+The URL to the Kubernetes API proxy (used by GitLab users). No default.
+
+##### `GITLAB_AGENT_BUILTIN_KAS_ENABLED`
+
+Control startup behavior of built-in KAS. `autostart` value in supervisor configuration for KAS will be set to this value. Default to [`GITLAB_KAS_ENABLED`](#gitlab_kas_enabled)
+
+##### `GITLAB_AGENT_KAS_WEBSOCKET_TOKEN_SECRET_FILE`
+
+Websocket token secret file. Default to `${GITLAB_INSTALL_DIR}/.gitlab_kas_websocket_token_secret`
+
+##### `GITLAB_AGENT_KAS_GITLAB_AUTHENTICATION_SECRET_FILE`
+
+An authentication secret file used to connect to gitlab from KAS. Defaults to `${GITLAB_AGENT_KAS_API_LISTEN_AUTHENTICATION_SECRET_FILE}`.
+
+##### `GITLAB_AGENT_KAS_API_LISTEN_AUTHENTICATION_SECRET_FILE`
+
+An authentication secret file to verify JWT token, for built-in KAS API. If not exist, an secret file will be generated on startup. Defaults to `${GITLAB_INSTALL_DIR}/.gitlab_kas_api_secret`
+
+##### `GITLAB_AGENT_KAS_PRIVATE_API_LISTEN_AUTHENTICATION_SECRET_FILE`
+
+An authentication secret file to verify JWT token, for built-in KAS internal API. If not exists, an secret file will be generated on startup. This is not "required", so please leave blank if you don't need it. No default.
+
+##### `GITLAB_AGENT_KAS_REDIS_PASSWORD_FILE`
+
+Path for the file that contains redis password to be used by built-in KAS. This is not "required", so please leave blank if you don't need it. No default.
+
+NOTE: We currently do not support password authentication between gitlab and redis. See [sameersbn/gitlab#1026](https://github.com/sameersbn/docker-gitlab/pull/1026)
+
 ##### `GITLAB_LFS_ENABLED`
 
 Enable/Disable Git LFS support. Defaults to `true`.

assets/build/install.sh

@@ -5,10 +5,12 @@ GITLAB_CLONE_URL=https://gitlab.com/gitlab-org/gitlab-foss.git
 GITLAB_SHELL_URL=https://gitlab.com/gitlab-org/gitlab-shell/-/archive/v${GITLAB_SHELL_VERSION}/gitlab-shell-v${GITLAB_SHELL_VERSION}.tar.bz2
 GITLAB_PAGES_URL=https://gitlab.com/gitlab-org/gitlab-pages.git
 GITLAB_GITALY_URL=https://gitlab.com/gitlab-org/gitaly.git
+GITLAB_AGENT_URL=https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent.git
 
 GITLAB_WORKHORSE_BUILD_DIR=${GITLAB_INSTALL_DIR}/workhorse
 GITLAB_PAGES_BUILD_DIR=/tmp/gitlab-pages
 GITLAB_GITALY_BUILD_DIR=/tmp/gitaly
+GITLAB_AGENT_BUILD_DIR=/tmp/gitlab-agent
 
 RUBY_SRC_URL=https://cache.ruby-lang.org/pub/ruby/${RUBY_VERSION%.*}/ruby-${RUBY_VERSION}.tar.gz
 
@@ -171,6 +173,18 @@ make -C ${GITLAB_GITALY_BUILD_DIR} git GIT_PREFIX=/usr/local
 # clean up
 rm -rf ${GITLAB_GITALY_BUILD_DIR}
 
+# download gitlab-agent (KAS)
+echo "Downloading gitlab-agent v.${GITLAB_AGENT_VERSION}..."
+git clone -q -b v${GITLAB_AGENT_VERSION} --depth 1 ${GITLAB_AGENT_URL} ${GITLAB_AGENT_BUILD_DIR}
+
+# install gitlab-agent (KAS)
+mkdir -p "${GITLAB_AGENT_INSTALL_DIR}"
+make -C ${GITLAB_AGENT_BUILD_DIR} kas TARGET_DIRECTORY=/usr/local/bin
+chown -R ${GITLAB_USER}: ${GITLAB_AGENT_INSTALL_DIR}
+
+# clean up
+rm -rf ${GITLAB_AGENT_BUILD_DIR}
+
 # remove go
 go clean --modcache
 rm -rf ${GITLAB_BUILD_DIR}/go${GOLANG_VERSION}.linux-amd64.tar.gz ${GOROOT}
@@ -410,6 +424,20 @@ stdout_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
 stderr_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
 EOF
 
+# configure superisord to start gitlab-agent (KAS)
+cat > /etc/supervisor/conf.d/gitlab-kas.conf <<EOF
+[program:gitlab_kas]
+priority=5
+directory=${GITLAB_AGENT_INSTALL_DIR}
+environment=HOME=${GITLAB_HOME}
+command=/usr/local/bin/kas --configuration-file="${GITLAB_AGENT_INSTALL_DIR}/gitlab-kas_config.yaml"
+user=git
+autostart={{GITLAB_AGENT_BUILTIN_KAS_ENABLED}}
+autorestart=true
+stdout_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
+stderr_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log
+EOF
+
 # configure supervisord to start mail_room
 cat > /etc/supervisor/conf.d/mail_room.conf <<EOF
 [program:mail_room]

assets/runtime/config/gitlab-agent/gitlab-kas_config.yaml

@@ -0,0 +1,94 @@
+# This is a configuration file for kas that contains the default values for the settings.
+# It DOES NOT contain all the possible configuration knobs.
+# The source of truth is kascfg.proto.
+# It contains all the fields and documentation them.
+# If you are looking for a setting, start from the ConfigurationFile message in:
+# - the proto file kascfg.proto.
+# - the generated documentation in kascfg_proto_docs.md.
+# Correctness of this file is enforced by a unit test in kascfg_defaults_test.go.
+
+agent:
+  configuration:
+    max_configuration_file_size: 131072
+    poll_period: 300s
+  info_cache_error_ttl: 60s
+  info_cache_ttl: 300s
+  kubernetes_api:
+    allowed_agent_cache_error_ttl: 10s
+    allowed_agent_cache_ttl: 60s
+    listen:
+      address: :8154
+      listen_grace_period: 5s
+      network: tcp
+      shutdown_grace_period: 3600s
+    url_path_prefix: {{GITLAB_RELATIVE_URL_ROOT}}/-/kubernetes-agent/k8s-proxy/
+    websocket_token_secret_file: {{GITLAB_AGENT_KAS_WEBSOCKET_TOKEN_SECRET_FILE}}
+  listen:
+    address: 127.0.0.1:8150
+    connections_per_token_per_minute: 40000
+    listen_grace_period: 5s
+    max_connection_age: 7200s
+    network: tcp
+    websocket: true
+  receptive_agent:
+    poll_period: 60s
+  redis_conn_info_gc: 600s
+  redis_conn_info_refresh: 240s
+  redis_conn_info_ttl: 300s
+api:
+  listen:
+    address: 127.0.0.1:8153
+    listen_grace_period: 5s
+    max_connection_age: 7200s
+    network: tcp
+    authentication_secret_file: {{GITLAB_AGENT_KAS_API_LISTEN_AUTHENTICATION_SECRET_FILE}}  # required
+gitaly:
+  global_api_rate_limit:
+    bucket_size: 70
+    refill_rate_per_second: 30
+  per_server_api_rate_limit:
+    bucket_size: 40
+    refill_rate_per_second: 15
+gitlab:
+  address: http://127.0.0.1:8080{{GITLAB_RELATIVE_URL_ROOT}}
+  authentication_secret_file: {{GITLAB_AGENT_KAS_GITLAB_AUTHENTICATION_SECRET_FILE}} # required
+  api_rate_limit:
+    bucket_size: 250
+    refill_rate_per_second: 50
+observability:
+  event_reporting_period: 300s
+  google_profiler: {}
+  listen:
+    address: 127.0.0.1:8151
+    network: tcp
+  liveness_probe:
+    url_path: /liveness
+  logging:
+    level: debug
+    grpc_level: debug
+  prometheus:
+    url_path: /metrics
+  readiness_probe:
+    url_path: /readiness
+  sentry: {}
+  usage_reporting_period: 10s
+private_api:
+  listen:
+    address: :8155
+    listen_grace_period: 5s
+    max_connection_age: 7200s
+    network: tcp
+    authentication_secret_file: {{GITLAB_AGENT_KAS_PRIVATE_API_LISTEN_AUTHENTICATION_SECRET_FILE}}
+redis:
+  dial_timeout: 5s
+  key_prefix: gitlab-kas
+  network: tcp
+  write_timeout: 3s
+  server:
+    address: "{{REDIS_HOST}}:{{REDIS_PORT}}" # required
+workspaces:
+  listen:
+    address: :8160
+    listen_grace_period: 5s
+    network: tcp
+    shutdown_grace_period: 3600s

assets/runtime/config/gitlabhq/gitlab.yml

@@ -1174,6 +1174,22 @@ production: &base
     # Default is '.gitlab_workhorse_secret' relative to Rails.root (i.e. root of the GitLab app).
     # secret_file: /home/git/gitlab/.gitlab_workhorse_secret
 
+  gitlab_kas:
+    enabled: {{GITLAB_KAS_ENABLED}}
+    # File that contains the secret key for verifying access for gitlab-kas.
+    # Default is '.gitlab_kas_secret' relative to Rails.root (i.e. root of the GitLab app).
+    secret_file: {{GITLAB_KAS_SECRET}} # /home/git/gitlab/.gitlab_kas_secret
+
+    # The URL to the external KAS API (used by the Kubernetes agents)
+    external_url: {{GITLAB_KAS_EXTERNAL}} # wss://kas.example.com
+
+    # The URL to the internal KAS API (used by the GitLab backend)
+    internal_url: {{GITLAB_KAS_INTERNAL}} # grpc://localhost:8153
+
+    # The URL to the Kubernetes API proxy (used by GitLab users)
+    external_k8s_proxy_url: {{GITLAB_KAS_PROXY}} # https://localhost:8154 # default: nil
+
+
   ## GitLab Elasticsearch settings
   elasticsearch:
     indexer_path: {{GITLAB_HOME}}/gitlab-elasticsearch-indexer/
@@ -1358,7 +1374,7 @@ test:
         region: us-east-1
 
   gitlab:
-    host: localhost
+    host: 127.0.0.1
     port: 80
 
     content_security_policy:

assets/runtime/config/nginx/gitlab

@@ -84,6 +84,54 @@ server {
     proxy_pass http://gitlab-workhorse;
   }
 
+  #start-builtin-kas
+  location {{GITLAB_RELATIVE_URL_ROOT}}/-/kubernetes-agent/ {
+    client_max_body_size 0;
+    gzip off;
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      300;
+    proxy_connect_timeout   300;
+    proxy_redirect          off;
+    proxy_buffering         {{NGINX_PROXY_BUFFERING}};
+
+    proxy_http_version 1.1;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{NGINX_X_FORWARDED_PROTO}};
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab;
+
+    proxy_pass http://127.0.0.1:8150;
+  }
+
+  location {{GITLAB_RELATIVE_URL_ROOT}}/-/kubernetes-agent/k8s-proxy/ {
+    client_max_body_size 0;
+    gzip off;
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      300;
+    proxy_connect_timeout   300;
+    proxy_redirect          off;
+    proxy_buffering         {{NGINX_PROXY_BUFFERING}};
+
+    proxy_http_version 1.1;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{NGINX_X_FORWARDED_PROTO}};
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab;
+
+    proxy_pass http://127.0.0.1:8154;
+  }
+  #end-builtin-kas
+
   error_page 404 /404.html;
   error_page 422 /422.html;
   error_page 500 /500.html;