[linux headers] Update build to support >= 6.3 kernels. Add 6.6 headers to fix ARM Container OS issue (#2036)

ddelnano · web-flow · commit 6ec8dcf04c3c · 2025-05-15T11:53:11.000-07:00
Summary: Update linux_headers_image to support kernels >= 6.3. Add 6.6 headers to fix ARM Container OS issue #2035 reported an issue where the socket tracer fails to run on a openSUSE MicroOS instance running a 6.11 kernel. This change updates our header building process to support >= 6.3 kernels since [upstream](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=05e96e96a315fa49faca4da2aedd1761a218b616) broke the source building process from non git trees (binary builds still work outside of git trees). > Going forward, building source packages is only possible in a git-managed tree. Building binary packages does not require git. This also adds 6.6.x linux headers to the build. A recent bug report identified that recent ARM Container OS clusters fail to start the socket tracer without the 6.6.x headers. COS doesn't have a distro provided linux headers package, which means these installs are dependent on the pre-packaged headers. Relevant Issues: #2035 Type of change: /kind compatibility Test Plan: Verified the following - [x] Ran full build without upload - [x] Built x86_64 and arm64 kernels pre and post 6.3 - [x] That the previous dockerfile and pre 6.3 dockerfile is same as before ``` $ git diff main:tools/docker/linux_headers_image/Dockerfile tools/docker/linux_headers_image/Dockerfile.bionic $ ``` --------- Signed-off-by: Dom Del Nano <ddelnano@gmail.com>
diff --git a/src/stirling/BUILD.bazel b/src/stirling/BUILD.bazel
@@ -68,8 +68,8 @@ pl_cc_library(
 # Used by stirling runtime to install linux headers on hosts having no Linux headers pre-installed.
 #
 # Note that each linux header tarball is ~15MB compressed and ~230MB uncompressed.
-# Trimmed versions are ~8MB compressed and ~110MB uncompressed.
-# These are all trimmed versions, so we have 8 * 12 = 96 MB of headers.
+# Trimmed versions are ~8-11MB (8.6 MB avg) compressed and ~45MB uncompressed.
+# These are all trimmed versions, so we have 9MB * 29 = ~250 MB of headers.
 stirling_linux_headers_tarballs = select({
     "@platforms//cpu:aarch64": ["@linux_headers_merged_arm64_tar_gz//file:file"],
     "@platforms//cpu:x86_64": ["@linux_headers_merged_x86_64_tar_gz//file:file"],
diff --git a/tools/docker/Makefile b/tools/docker/Makefile
@@ -73,40 +73,44 @@ SYSROOT_CREATOR_IMAGE_TAG := sysroot-creator-$(SYSROOT_REV)
 ## Linux image parameters
 LINUX_HEADER_BUILD_DIR := $(BUILD_DIR)/linux_headers
 LINUX_HEADER_ASSETS_BUILD_DIR := $(LINUX_HEADER_BUILD_DIR)/assets
-LINUX_KERNEL_VERSIONS := 4.14.304 \
+LINUX_KERNEL_VERSIONS := 4.14.309 \
 	4.15.18 \
 	4.16.18 \
 	4.17.19 \
 	4.18.20 \
-	4.19.271 \
+	4.19.325 \
 	4.20.17 \
 	5.0.21 \
 	5.1.21 \
 	5.2.21 \
 	5.3.18 \
-	5.4.230 \
+	5.4.293 \
 	5.5.19 \
 	5.6.19 \
 	5.7.19 \
 	5.8.18 \
 	5.9.16 \
-	5.10.165 \
+	5.10.237 \
 	5.11.22 \
 	5.12.19 \
 	5.13.19 \
 	5.14.21 \
-	5.15.90 \
+	5.15.181 \
 	5.16.20 \
 	5.17.15 \
 	5.18.19 \
 	5.19.17 \
 	6.0.19 \
-	6.1.8
+	6.1.137 \
+	6.6.89
+
+LINUX_HEADER_TEMPLATE := linux-headers-%.tar.gz
+LINUX_HEADER_X86_64_TARGETS = $(addprefix $(LINUX_HEADER_ASSETS_BUILD_DIR)/, \
+			          $(patsubst %,$(subst x86_64,%,$(LINUX_HEADER_TEMPLATE)), $(addprefix x86_64-,$(LINUX_KERNEL_VERSIONS))))
+
+LINUX_HEADER_ARM64_TARGETS = $(addprefix $(LINUX_HEADER_ASSETS_BUILD_DIR)/, \
+			         $(patsubst %,$(subst arm64,%,$(LINUX_HEADER_TEMPLATE)), $(addprefix arm64-,$(LINUX_KERNEL_VERSIONS))))
 
-LINUX_HEADER_X86_64_TEMPLATE := linux-headers-x86_64-%.tar.gz
-LINUX_HEADER_ARM64_TEMPLATE := linux-headers-arm64-%.tar.gz
-LINUX_HEADER_X86_64_TARGETS = $(addprefix $(LINUX_HEADER_ASSETS_BUILD_DIR)/, $(patsubst %,$(LINUX_HEADER_X86_64_TEMPLATE), $(LINUX_KERNEL_VERSIONS)))
-LINUX_HEADER_ARM64_TARGETS = $(addprefix $(LINUX_HEADER_ASSETS_BUILD_DIR)/, $(patsubst %,$(LINUX_HEADER_ARM64_TEMPLATE), $(LINUX_KERNEL_VERSIONS)))
 LINUX_HEADERS_X86_64_MERGED_FILE := $(LINUX_HEADER_BUILD_DIR)/linux-headers-merged-x86_64-$(LINUX_HEADERS_REV).tar.gz
 LINUX_HEADERS_ARM64_MERGED_FILE := $(LINUX_HEADER_BUILD_DIR)/linux-headers-merged-arm64-$(LINUX_HEADERS_REV).tar.gz
 LINUX_HEADERS_GS_PATH := gs://pixie-dev-public/linux-headers/$(LINUX_HEADERS_REV)
@@ -205,26 +209,34 @@ upload_graalvm_archive: build_graalvm_archive_image
 ##############################################
 # Linux Headers Build
 ##############################################
-$(LINUX_HEADER_ASSETS_BUILD_DIR)/linux-headers-x86_64-%.tar.gz: linux_headers_image/Dockerfile linux_headers_image/x86_64_config
-ifndef LINUX_HEADERS_REV
-	$(error set LINUX_HEADERS_REV to a valid value)
-endif
-	@mkdir -p $(@D)
-	linux_headers_image_tag="gcr.io/pixie-oss/pixie-dev-public:$*-$(LINUX_HEADERS_REV)"; \
-	$(DOCKER) build --build-arg KERN_VERSION=$* --build-arg ARCH=x86_64 linux_headers_image -t "$${linux_headers_image_tag}" && \
-	$(DOCKER) run --rm -v $(PWD)/$(LINUX_HEADER_ASSETS_BUILD_DIR):/output "$${linux_headers_image_tag}"
-
-$(LINUX_HEADER_ASSETS_BUILD_DIR)/linux-headers-arm64-%.tar.gz: linux_headers_image/Dockerfile linux_headers_image/arm64_config
+LINUX_HEADERS_BASE_PRE_v6_3 := ubuntu:18.04@sha256:152dc042452c496007f07ca9127571cb9c29697f42acbfad72324b2bb2e43c98
+LINUX_HEADERS_BASE_POST_v6_3 := ubuntu:20.04@sha256:8feb4d8ca5354def3d8fce243717141ce31e2c428701f6682bd2fafe15388214
+$(LINUX_HEADER_ASSETS_BUILD_DIR)/linux-headers-%.tar.gz: linux_headers_image/Dockerfile
 ifndef LINUX_HEADERS_REV
 	$(error set LINUX_HEADERS_REV to a valid value)
 endif
 	@mkdir -p $(@D)
-	linux_headers_image_tag="gcr.io/pixie-oss/pixie-dev-public:$*-$(LINUX_HEADERS_REV)"; \
-	$(DOCKER) build --build-arg KERN_VERSION=$* \
-		--build-arg ARCH=arm64 \
-		--build-arg CROSS_COMPILE=aarch64-linux-gnu- \
-		linux_headers_image -t "$${linux_headers_image_tag}" && \
-	$(DOCKER) run --rm -v $(PWD)/$(LINUX_HEADER_ASSETS_BUILD_DIR):/output "$${linux_headers_image_tag}"
+	# Linux 6.3 and later has differences in the build process.
+	# We need to use a different base image for these versions.
+	ARCH=$$(echo $* | cut -d- -f1); \
+	KVER=$$(echo $* | cut -d- -f2-); \
+	KERN_MAJ=$$(echo $$KVER | cut -d. -f1); \
+	KERN_MIN=$$(echo $$KVER | cut -d. -f2); \
+	BASE_IMAGE=$$(echo $(LINUX_HEADERS_BASE_POST_v6_3)); \
+	if [ $${KERN_MAJ} -lt 6 ] || { [ $${KERN_MAJ} -le 6 ] && [ $${KERN_MIN} -lt 3 ]; }; then \
+		BASE_IMAGE=$$(echo $(LINUX_HEADERS_BASE_PRE_v6_3)); \
+	fi; \
+	CONFIG_FILE=$$(echo linux_headers_image/"$$ARCH"_config); \
+	if [ ! -f $$CONFIG_FILE ]; then \
+		echo "Unsupported ARCH=$$ARCH. Missing $$CONFIG_FILE"; exit 1; \
+	fi; \
+	linux_headers_image_tag="gcr.io/pixie-oss/pixie-dev-public:$$KVER-$(LINUX_HEADERS_REV)"; \
+	$(DOCKER) build --build-arg BASE_IMAGE=$${BASE_IMAGE} linux_headers_image \
+		-t "$${linux_headers_image_tag}" && \
+	$(DOCKER) run --rm --env ARCH=$$ARCH \
+	              --env KERN_VERSION=$$KVER \
+		      $$( [ "$$ARCH" = "arm64" ] && echo "--env CROSS_COMPILE=aarch64-linux-gnu-" ) \
+		      -v $(PWD)/$(LINUX_HEADER_ASSETS_BUILD_DIR):/output "$${linux_headers_image_tag}"
 
 $(LINUX_HEADERS_X86_64_MERGED_FILE): $(LINUX_HEADER_X86_64_TARGETS)
 	tar -czf $@ -C $(LINUX_HEADER_ASSETS_BUILD_DIR) $(^F)
diff --git a/tools/docker/linux_headers_image/Dockerfile b/tools/docker/linux_headers_image/Dockerfile
@@ -14,9 +14,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-FROM ubuntu:18.04@sha256:8aa9c2798215f99544d1ce7439ea9c3a6dfd82de607da1cec3a8a2fae005931b
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
 
 # Install required packages
+ENV DEBIAN_FRONTEND=noninteractive
+ENV KERN_VERSION=${KERN_VERSION}
+ENV ARCH=${ARCH}
+ENV CROSS_COMPILE=${CROSS_COMPILE}
+
 RUN apt-get update
 RUN apt-get upgrade -y -q
 RUN apt-get install -y -q build-essential \
@@ -32,47 +38,18 @@ RUN apt-get install -y -q build-essential \
   binutils-aarch64-linux-gnu \
   gcc-aarch64-linux-gnu \
   dwarves \
+  debhelper \
   python3
 
-ARG KERN_VERSION
-
-# Download Linux sources
-WORKDIR /px/src
-RUN KERN_MAJ="$(echo "${KERN_VERSION}" | cut -d'.' -f1)"; \
-    wget -nv http://mirrors.edge.kernel.org/pub/linux/kernel/v${KERN_MAJ}.x/linux-${KERN_VERSION}.tar.gz
-RUN tar zxf linux-${KERN_VERSION}.tar.gz
-
 WORKDIR /configs
 ADD x86_64_config /configs/x86_64
 ADD arm64_config /configs/arm64
 
-ARG ARCH
-ARG CROSS_COMPILE
-
-# Build Linux kernel
-WORKDIR /px/src/linux-${KERN_VERSION}
-RUN cp /configs/${ARCH} .config
-RUN make ARCH=${ARCH} olddefconfig
-RUN make ARCH=${ARCH} clean
-RUN make ARCH=${ARCH} -j $(nproc) deb-pkg LOCALVERSION=-pl
-
-# Extract headers into a tarball
 WORKDIR /px
-RUN DEB_ARCH=$(echo ${ARCH} | sed 's/x86_64/amd64/g'); dpkg -x src/linux-headers-${KERN_VERSION}-pl_${KERN_VERSION}-pl-1_${DEB_ARCH}.deb .
 
-# Remove broken symlinks
-RUN find usr/src/linux-headers-${KERN_VERSION}-pl -xtype l -exec rm {} +
+ADD build_linux_headers.sh /px/build_linux_headers.sh
 
-# Remove uneeded files to reduce size
-# Keep only:
-# - usr/src/linux-headers-x.x.x-pl/include
-# - usr/src/linux-headers-x.x.x-pl/arch/${ARCH}
-# This reduces the size by a little over 2x.
-RUN rm -rf usr/share
-RUN find usr/src/linux-headers-${KERN_VERSION}-pl -maxdepth 1 -mindepth 1 ! -name include ! -name arch -type d \
-    -exec rm -rf {} +
-RUN find usr/src/linux-headers-${KERN_VERSION}-pl/arch -maxdepth 1 -mindepth 1 ! -name $(echo ${ARCH} | sed 's/x86_64/x86/g') -type d -exec rm -rf {} +
-RUN tar zcf linux-headers-${ARCH}-${KERN_VERSION}.tar.gz usr
+RUN chmod +x /px/build_linux_headers.sh
 
 VOLUME /output
-CMD ["sh", "-c", "cp linux-headers-*.tar.gz /output/"]
+CMD ["sh", "-c", "/px/build_linux_headers.sh /px ${KERN_VERSION} ${ARCH} /output"]
diff --git a/tools/docker/linux_headers_image/build_linux_headers.sh b/tools/docker/linux_headers_image/build_linux_headers.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+usage() {
+    echo "Usage: $0 [workspace] [kernel_version] [arch] [output_dir]"
+    echo "Example: $0 /px 5.4.0-42-generic x86_64 /output"
+    echo "If cross-compiling, set the CROSS_COMPILE environment variable to the cross-compiler prefix."
+}
+
+if [ "$#" -ne 4 ]; then
+    echo "Invalid number of arguments. Expected 4, but received $# ($*)"
+    usage
+    exit 1
+fi
+
+WORKSPACE=$1
+KERN_VERSION=$2
+ARCH=$3
+OUTPUT_DIR=$4
+
+if [ -z "${WORKSPACE}" ] || [ -z "${KERN_VERSION}" ] || [ -z "${ARCH}" ] || [ -z "${OUTPUT_DIR}" ]; then
+    usage
+    exit 1
+fi
+
+if [ "${ARCH}" != "$(uname -m)" ]; then
+    if [ -z "${CROSS_COMPILE}" ]; then
+        echo "CROSS_COMPILE is not set. Please set it to the cross-compiler prefix."
+        exit 1
+    fi
+fi
+
+mkdir -p "${WORKSPACE}"/src
+pushd "${WORKSPACE}"/src || exit
+
+KERN_MAJ=$(echo "${KERN_VERSION}" | cut -d'.' -f1);
+KERN_MIN=$(echo "${KERN_VERSION}" | cut -d'.' -f2);
+wget -nv http://mirrors.edge.kernel.org/pub/linux/kernel/v"${KERN_MAJ}".x/linux-"${KERN_VERSION}".tar.gz
+
+tar zxf linux-"${KERN_VERSION}".tar.gz
+
+pushd linux-"${KERN_VERSION}" || exit
+
+cp /configs/"${ARCH}" .config
+make ARCH="${ARCH}" olddefconfig
+make ARCH="${ARCH}" clean
+
+LOCALVERSION="-pl"
+
+DEB_ARCH="${ARCH//x86_64/amd64}"
+# binary builds are required for non git trees after linux v6.3 (inclusive).
+# The .deb file suffix is also different.
+TARGET='bindeb-pkg'
+DEB_SUFFIX="-1_${DEB_ARCH}.deb"
+if [ "${KERN_MAJ}" -lt 6 ] || { [ "${KERN_MAJ}" -le 6 ] && [ "${KERN_MIN}" -lt 3 ]; }; then
+    TARGET='deb-pkg'
+    DEB_SUFFIX="${LOCALVERSION}-1_${DEB_ARCH}.deb"
+fi
+echo "Building ${TARGET} for ${KERN_VERSION}${LOCALVERSION} (${ARCH})"
+
+make ARCH="${ARCH}" -j "$(nproc)" "${TARGET}" LOCALVERSION="${LOCALVERSION}"
+
+popd || exit
+popd || exit
+
+# Extract headers into a tarball
+dpkg -x src/linux-headers-"${KERN_VERSION}${LOCALVERSION}_${KERN_VERSION}${DEB_SUFFIX}" .
+
+# Remove broken symlinks
+find usr/src/linux-headers-"${KERN_VERSION}${LOCALVERSION}" -xtype l -exec rm {} +
+
+# Remove uneeded files to reduce size
+# Keep only:
+# - usr/src/linux-headers-x.x.x-pl/include
+# - usr/src/linux-headers-x.x.x-pl/arch/${ARCH}
+# This reduces the size by a little over 2x.
+rm -rf usr/share
+find usr/src/linux-headers-"${KERN_VERSION}${LOCALVERSION}" -maxdepth 1 -mindepth 1 ! -name include ! -name arch -type d \
+    -exec rm -rf {} +
+find usr/src/linux-headers-"${KERN_VERSION}${LOCALVERSION}"/arch -maxdepth 1 -mindepth 1 ! -name "${ARCH//x86_64/x86}" -type d -exec rm -rf {} +
+
+tar zcf linux-headers-"${ARCH}"-"${KERN_VERSION}".tar.gz usr
+
+cp linux-headers-*.tar.gz "${OUTPUT_DIR}"/
