From 1146033ce683a4657e46b56c9f87e11194a71357 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 13 Aug 2025 19:36:59 +0000 Subject: [PATCH 1/6] Fix golangci-lint issues Signed-off-by: Dom Del Nano --- .../pkg/metrics/data_loss_handler.go | 2 +- .../pkg/metrics/prometheus_recorder.go | 2 +- src/shared/artifacts/manifest/query.go | 8 +- src/shared/services/election/election.go | 8 +- src/stirling/kernel_headers_faq.md | 113 ++++++++++++++++++ src/utils/artifacts/versions_gen/main.go | 8 +- 6 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 src/stirling/kernel_headers_faq.md diff --git a/src/e2e_test/perf_tool/pkg/metrics/data_loss_handler.go b/src/e2e_test/perf_tool/pkg/metrics/data_loss_handler.go index 061022100bc..c7ca747b6ab 100644 --- a/src/e2e_test/perf_tool/pkg/metrics/data_loss_handler.go +++ b/src/e2e_test/perf_tool/pkg/metrics/data_loss_handler.go @@ -84,7 +84,7 @@ func (h *dataLossHandler) HandleDone(ctx context.Context) error { func (h *dataLossHandler) sendRows(ctx context.Context) error { ts := h.lastTimestamp - if (ts == time.Time{}) { + if (ts.Equal(time.Time{})) { ts = time.Now() } percentRow := newResultRow() diff --git a/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go b/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go index 9c9d1f5cca0..19d08b1b0a9 100644 --- a/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go +++ b/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go @@ -126,7 +126,7 @@ func (r *prometheusRecorderImpl) scrapeFunc(fw *forwarder) func() error { continue } // Only collect counters and gauges for now. - if !(mf.GetType() == io_prometheus_client.MetricType_COUNTER || mf.GetType() == io_prometheus_client.MetricType_GAUGE) { + if mf.GetType() != io_prometheus_client.MetricType_COUNTER && mf.GetType() != io_prometheus_client.MetricType_GAUGE { continue } for _, m := range mf.GetMetric() { diff --git a/src/shared/artifacts/manifest/query.go b/src/shared/artifacts/manifest/query.go index ac9e00d4827..ba676418c70 100644 --- a/src/shared/artifacts/manifest/query.go +++ b/src/shared/artifacts/manifest/query.go @@ -34,8 +34,8 @@ var ErrArtifactSetNotFound = errors.New("no artifact set with given artifact nam var ErrArtifactNotFound = errors.New("no artifact with given version") // GetArtifact returns a an artifact with the given artifact name and version string. -func (m *ArtifactManifest) GetArtifact(name string, version string) (*versionspb.Artifact, error) { - as, ok := m.sets[name] +func (a *ArtifactManifest) GetArtifact(name string, version string) (*versionspb.Artifact, error) { + as, ok := a.sets[name] if !ok { return nil, ErrArtifactSetNotFound } @@ -50,8 +50,8 @@ func (m *ArtifactManifest) GetArtifact(name string, version string) (*versionspb type ArtifactFilter func(*versionspb.Artifact) bool // ListArtifacts returns artifacts with the given name, in version sorted order (newest versions first). It returns up to `limit` artifacts. -func (m *ArtifactManifest) ListArtifacts(name string, limit int64, filters ...ArtifactFilter) ([]*versionspb.Artifact, error) { - as, ok := m.sets[name] +func (a *ArtifactManifest) ListArtifacts(name string, limit int64, filters ...ArtifactFilter) ([]*versionspb.Artifact, error) { + as, ok := a.sets[name] if !ok { return nil, ErrArtifactSetNotFound } diff --git a/src/shared/services/election/election.go b/src/shared/services/election/election.go index 41156058483..d55939a0bdc 100644 --- a/src/shared/services/election/election.go +++ b/src/shared/services/election/election.go @@ -55,7 +55,7 @@ type K8sLeaderElectionMgr struct { } // NewK8sLeaderElectionMgr creates a K8sLeaderElectionMgr. -func NewK8sLeaderElectionMgr(electionNamespace string, expectedMaxSkewMS, renewDeadlineMS time.Duration, electionName string) (*K8sLeaderElectionMgr, error) { +func NewK8sLeaderElectionMgr(electionNamespace string, expectedMaxSkew, renewDuration time.Duration, electionName string) (*K8sLeaderElectionMgr, error) { if electionNamespace == "" { return nil, errors.New("namespace must be specified for leader election") } @@ -63,9 +63,9 @@ func NewK8sLeaderElectionMgr(electionNamespace string, expectedMaxSkewMS, renewD return &K8sLeaderElectionMgr{ namespace: electionNamespace, name: electionName, - leaseDuration: expectedMaxSkewMS + renewDeadlineMS, - renewDeadline: renewDeadlineMS, - retryPeriod: renewDeadlineMS / 4, + leaseDuration: expectedMaxSkew + renewDuration, + renewDeadline: renewDuration, + retryPeriod: renewDuration / 4, kubeConfig: "", }, nil } diff --git a/src/stirling/kernel_headers_faq.md b/src/stirling/kernel_headers_faq.md new file mode 100644 index 00000000000..0ec3b91ad5b --- /dev/null +++ b/src/stirling/kernel_headers_faq.md @@ -0,0 +1,113 @@ +# Pixie Recommends Installing Your Distro’s Linux Kernel Headers + +Pixie uses eBPF programs; some are compiled **at runtime** (via BCC/LLVM) against the node’s running kernel. That compilation needs the kernel’s C headers that match the exact kernel build (including vendor backports). Mismatched or missing headers can lead to compile failures or silently incorrect offsets in tracers. + +--- + +## TL;DR + +- **If Pixie deploys and its tracers start successfully, you can ignore the kernel-headers warning.** It’s informational. +- **If you hit *any* Pixie issues or suspect a bug, first install your distro’s kernel headers on every node and re-test.** This resolves a large class of tracer initialization problems on vendor-patched kernels. +- **Note:** Some minimal/immutable distros don’t ship a headers package (notably **Google Container-Optimized OS / COS**). See guidance below. + +> **Screenshot of the warning in `px`** +> +> Replace the URL below with the image from PR **#2250**. +> +> `![Missing kernel headers — informational warning](INSERT_IMAGE_URL_FROM_PR_2250)` + +--- + +## Why headers matter + +- **Runtime compilation (BCC):** Pixie compiles certain eBPF programs on the node. BCC needs kernel headers to resolve structure layouts, constants, and feature guards accurately. Without a match, compilation may fail or produce incorrect code. +- **Vendor patches/backports:** Distro kernels often diverge from upstream. Only the **distro’s header package** reflects those exact changes; generic or prepackaged headers are best-effort and can be wrong. + +--- + +## What you’ll see when headers are missing + +- A **warning** during deploy/validation that headers weren’t detected. +- In agent logs (e.g., `vizier-pem`), messages about missing `/lib/modules/$(uname -r)/{build,source}` and attempts to fall back to bundled headers. +- Tracers that rely on BCC may fail to initialize (socket/HTTP/DNS tracing most commonly). + +--- + +## Install instructions (per distro) + +> Ensure the installed header version **matches `uname -r`**. If a kernel update is pulled in, **reboot** so the running kernel matches the headers. + +- **Debian/Ubuntu** + ```bash + sudo apt-get update + sudo apt-get install -y linux-headers-$(uname -r) + ``` + +- **RHEL / Rocky / Alma / Fedora** + ```bash + # DNF-based + sudo dnf install -y kernel-devel-$(uname -r) + # YUM-based + sudo yum install -y kernel-devel-$(uname -r) + # (Some environments also need: kernel-headers-$(uname -r)) + ``` + +- **Amazon Linux** + ```bash + AL2 + sudo yum install -y kernel-devel-$(uname -r) + + # AL2023 + sudo dnf install -y kernel-devel + # If this pulled a newer kernel, reboot so uname -r matches. + ``` + +- **openSUSE** + ```bash + sudo zypper install -y kernel-default-devel + ``` + +## Distros without kernel-headers packages (notable: Google COS) + +Some immutable/minimal distros **don’t provide a headers package or a host package manager**: + +- **Google Container-Optimized OS (COS):** Root filesystem is read-only; there’s no `apt`/`dnf` on the host. COS’s `toolbox` is for ephemeral debugging and not for installing kernel headers on the node. + **Options:** + - Use **Ubuntu** (or other header-capable) node images on **GKE Standard** when you need BCC runtime compilation. + - For **Autopilot** (COS-based), consider running Pixie on **Standard** for debugging, or use a custom image/variant that includes headers where feasible. + - Pixie will try **prepackaged headers** as a fallback, but these are not guaranteed to match vendor-patched kernels. + - If available in your environment, loading **in-kernel headers** via `kheaders` (`/sys/kernel/kheaders.tar.xz`) can help, but support varies by vendor. + +Other minimal/immutable OSes (e.g., Fedora CoreOS, Bottlerocket, MicroOS) have similar constraints—prefer images/variants that include headers or build them into your node image pipeline. + +--- + +## Troubleshooting checklist + +1. **Match versions:** `uname -r` must match the headers you installed. +2. **Reboot after updates:** If the package manager updated the kernel, reboot nodes. +3. **Check presence:** Ensure `/lib/modules/$(uname -r)/build` and `.../include` exist. +4. **Review logs:** Look at `vizier-pem` logs for BCC compile/load failures. +5. **Try headers first:** If you see tracer init failures, install headers and re-test before deeper debugging. + +--- + +## FAQ + +**Is BTF/CO-RE enough to avoid headers?** +Not universally. Pixie still uses BCC in parts of the data path, which needs headers at runtime. BTF helps CO-RE workflows, but does not remove the need for matching headers where BCC is used. + +**What breaks if I skip headers?** +Most commonly, the **socket/HTTP/DNS tracers** fail to initialize. You might still see some metrics, but protocol-level visibility will be limited. + +**Why prefer distro headers over Pixie’s packaged headers?** +Because **distro kernels are not vanilla**—they include backports and config deltas. Distro header packages are the authoritative match; bundled headers are a convenience, not a guarantee. + +--- + +## Related references + +- Pixie PR: **#2250** — CLI warning message and docs link + https://github.com/pixie-io/pixie/pull/2250 +- Pixie Issue: **#2051** — Recommend installing distro headers; examples across distros + https://github.com/pixie-io/pixie/issues/2051 diff --git a/src/utils/artifacts/versions_gen/main.go b/src/utils/artifacts/versions_gen/main.go index c2ca776aee4..e9a4d916039 100644 --- a/src/utils/artifacts/versions_gen/main.go +++ b/src/utils/artifacts/versions_gen/main.go @@ -47,12 +47,12 @@ func init() { } func availableArtifacts(artifactName string) []vpb.ArtifactType { - switch { - case artifactName == "cli": + switch artifactName { + case "cli": return []vpb.ArtifactType{vpb.AT_LINUX_AMD64, vpb.AT_DARWIN_AMD64} - case artifactName == "vizier": + case "vizier": return []vpb.ArtifactType{vpb.AT_CONTAINER_SET_LINUX_AMD64, vpb.AT_CONTAINER_SET_YAMLS, vpb.AT_CONTAINER_SET_TEMPLATE_YAMLS} - case artifactName == "operator": + case "operator": return []vpb.ArtifactType{vpb.AT_CONTAINER_SET_LINUX_AMD64, vpb.AT_CONTAINER_SET_TEMPLATE_YAMLS} default: panic(fmt.Sprintf("Unknown artifact type: %s", artifactName)) From 189e51124f9fad1c97f5d793e14dd65162eb6de4 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 13 Aug 2025 19:37:18 +0000 Subject: [PATCH 2/6] Upgrade minikube, skaffold, gh cli, sops, trivy Signed-off-by: Dom Del Nano --- .../px_dev_extras/attributes/linux.rb | 20 +++++++++---------- .../px_dev_extras/attributes/mac_os_x.rb | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb b/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb index 5591cc0d8e6..41168ee694f 100644 --- a/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb +++ b/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb @@ -33,9 +33,9 @@ '6c9234d0b2b024bf0e7c845fc092339b51b94e5addeee9612a7219cfd2a7b731' default['gh']['download_path'] = - 'https://github.com/cli/cli/releases/download/v2.26.1/gh_2.26.1_linux_amd64.tar.gz' + 'https://github.com/cli/cli/releases/download/v2.76.1/gh_2.76.1_linux_amd64.tar.gz' default['gh']['sha256'] = - '4e1f98c10eb0af2c3bfe6849c10328a2a2446a9ff4294db0dfdb20b7d16eaecc' + '18367ca38b4462889ae38fba6a18c53a4c2818b6af309bbe53d0810bb06036e9' default['helm']['download_path'] = 'https://get.helm.sh/helm-v3.11.3-linux-amd64.tar.gz' @@ -58,9 +58,9 @@ 'd6a6dbf82ae9a1a7f9fbc8d85c224617a17337afa4284aaca6b0556a7347609d' default['minikube']['download_path'] = - 'https://github.com/kubernetes/minikube/releases/download/v1.30.1/minikube-linux-amd64' + 'https://github.com/kubernetes/minikube/releases/download/v1.36.0/minikube-linux-amd64' default['minikube']['sha256'] = - 'e53d9e8c31f4c5f683182f5323d3527aa0725f713945c6d081cf71aa548ab388' + 'cddeab5ab86ab98e4900afac9d62384dae0941498dfbe712ae0c8868250bc3d7' default['opm']['download_path'] = 'https://github.com/operator-framework/operator-registry/releases/download/v1.26.4/linux-amd64-opm' @@ -73,19 +73,19 @@ '8a94b84542d21b8785847f4cccc8a6da4c7be5e16d4b1a2d0a5f7ec5532faec0' default['skaffold']['download_path'] = - 'https://storage.googleapis.com/skaffold/releases/v2.0.4/skaffold-linux-amd64' + 'https://storage.googleapis.com/skaffold/releases/v2.16.1/skaffold-linux-amd64' default['skaffold']['sha256'] = - 'f1c54245b97073f39b209dd4b769802701c9901ceaa95774e7507d95bbd1d4c0' + '1cbeea85aa14ba603dbc2bbdfa7bfde5644d7988beed0fdc0fd1c67298d4cf67' default['sops']['download_path'] = - 'https://github.com/mozilla/sops/releases/download/3.3.1/sops-3.3.1.linux' + 'https://github.com/mozilla/sops/releases/download/v3.10.2/sops-v3.10.2.linux.amd64' default['sops']['sha256'] = - '6eacdd01b68fd140eb71bbca233bea897cccb75dbf9e00a02e648b2f9a8a6939' + '79b0f844237bd4b0446e4dc884dbc1765fc7dedc3968f743d5949c6f2e701739' default['trivy']['download_path'] = - 'https://github.com/aquasecurity/trivy/releases/download/v0.39.0/trivy_0.39.0_Linux-64bit.tar.gz' + 'https://github.com/aquasecurity/trivy/releases/download/v0.64.1/trivy_0.64.1_Linux-64bit.tar.gz' default['trivy']['sha256'] = - '77cfe88b6b803909982e2dd41149c042fd9e6a58e1f55103f27e7bab2d2fb6d2' + '1a09d86667b3885a8783d1877c9abc8061b2b4e9b403941b22cbd82f10d275a8' default['yq']['download_path'] = 'https://github.com/mikefarah/yq/releases/download/v4.30.8/yq_linux_amd64' diff --git a/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb b/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb index f681e5aeca7..dea948cc8a6 100644 --- a/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb +++ b/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb @@ -34,9 +34,9 @@ '869f4d8acaa1feb11ce76b2204c5476b8a04d9451216adde6b18e2ef2f978794' default['gh']['download_path'] = - 'https://github.com/cli/cli/releases/download/v2.26.1/gh_2.26.1_macOS_amd64.tar.gz' + 'https://github.com/cli/cli/releases/download/v2.76.1/gh_2.76.1_macOS_amd64.tar.gz' default['gh']['sha256'] = - 'ef398ece1f31d033df6374458f7a87500ccdbdc9964170db04b6a5f707632417' + '0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5' default['helm']['download_path'] = 'https://get.helm.sh/helm-v3.11.3-darwin-amd64.tar.gz' @@ -59,9 +59,9 @@ 'eaf2792d9731c911da671a6145eebd5ba136c20446adb542e7b1463ffe868388' default['minikube']['download_path'] = - 'https://github.com/kubernetes/minikube/releases/download/v1.30.1/minikube-darwin-amd64' + 'https://github.com/kubernetes/minikube/releases/download/v1.36.0/minikube-darwin-amd64' default['minikube']['sha256'] = - 'b5938a8772c5565b5d0b795938c367c5190bf65bb51fc55fb2417cb4e1d04ef1' + 'a7e3da0db4041b2f845ca37af592424a9cbe34087ac922220b1e3abc4e1976ea' default['opm']['download_path'] = 'https://github.com/operator-framework/operator-registry/releases/download/v1.26.4/darwin-amd64-opm' @@ -74,19 +74,19 @@ '8666031111138e2e79ff7d1e42888b23b793b856bc2d9c9dccbb1e2d2cccb5cf' default['skaffold']['download_path'] = - 'https://storage.googleapis.com/skaffold/releases/v2.0.4/skaffold-darwin-amd64' + 'https://storage.googleapis.com/skaffold/releases/v2.16.1/skaffold-darwin-amd64' default['skaffold']['sha256'] = - 'd0956712db4d2dd8084ffe297bf645ec92506b87db5f61c0f5e24c7fd99bf0a3' + 'ed4c6cd0c82f48908db6bc1210b2f609bb5672b340ad1fbaa092ed2c6acedeb5' default['sops']['download_path'] = - 'https://github.com/mozilla/sops/releases/download/3.3.1/sops-3.3.1.darwin' + 'https://github.com/mozilla/sops/releases/download/v3.10.2/sops-v3.10.2.darwin.amd64' default['sops']['sha256'] = - '09bb5920ae609bdf041b74843e2d8211a7059847b21729fadfbd3c3e33e67d26' + 'dece9b0131af5ced0f8c278a53c0cf06a4f0d1d70a177c0979f6d111654397ce' default['trivy']['download_path'] = - 'https://github.com/aquasecurity/trivy/releases/download/v0.39.0/trivy_0.39.0_macOS-64bit.tar.gz' + 'https://github.com/aquasecurity/trivy/releases/download/v0.64.1/trivy_0.64.1_macOS-64bit.tar.gz' default['trivy']['sha256'] = - 'e0e6831395310452a65cae8dcb142fb743a05b27b0698177e8fad93b24490e19' + '107a874b41c1f0a48849f859b756f500d8be06f2d2b8956a046a97ae38088bf6' default['yq']['download_path'] = 'https://github.com/mikefarah/yq/releases/download/v4.30.8/yq_darwin_amd64' From 15da16d335b65cb59857471b946172d3af569617 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 13 Aug 2025 19:13:12 +0000 Subject: [PATCH 3/6] Upgrade golangci-lint Signed-off-by: Dom Del Nano --- tools/chef/cookbooks/px_dev/attributes/linux.rb | 4 ++-- tools/chef/cookbooks/px_dev/attributes/mac_os_x.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/chef/cookbooks/px_dev/attributes/linux.rb b/tools/chef/cookbooks/px_dev/attributes/linux.rb index ea92f2935ba..2087ba5dbfb 100644 --- a/tools/chef/cookbooks/px_dev/attributes/linux.rb +++ b/tools/chef/cookbooks/px_dev/attributes/linux.rb @@ -38,9 +38,9 @@ 'bbca37cc395c974ffa4893ee35819ad23ebb27426df87af92e93a9ec66ef8712' default['golangci-lint']['download_path'] = - 'https://github.com/golangci/golangci-lint/releases/download/v2.0.2/golangci-lint-2.0.2-linux-amd64.tar.gz' + 'https://github.com/golangci/golangci-lint/releases/download/v2.3.1/golangci-lint-2.3.1-linux-amd64.tar.gz' default['golangci-lint']['sha256'] = - '89cc8a7810dc63b9a37900da03e37c3601caf46d42265d774e0f1a5d883d53e2' + '57c3a4c56b8363f7f1f695ac957b0bb474aca00c490bc5e949669a6c2ba5c432' default['nodejs']['download_path'] = 'https://nodejs.org/dist/v18.16.0/node-v18.16.0-linux-x64.tar.xz' diff --git a/tools/chef/cookbooks/px_dev/attributes/mac_os_x.rb b/tools/chef/cookbooks/px_dev/attributes/mac_os_x.rb index aa6a8d4d8c3..f4b429f74ba 100644 --- a/tools/chef/cookbooks/px_dev/attributes/mac_os_x.rb +++ b/tools/chef/cookbooks/px_dev/attributes/mac_os_x.rb @@ -39,9 +39,9 @@ '4a8d7a32052f223e71faab424a69430455b27b3fff5f4e651f9d97c3e51a8746' default['golangci-lint']['download_path'] = - 'https://github.com/golangci/golangci-lint/releases/download/v2.0.2/golangci-lint-2.0.2-darwin-amd64.tar.gz' + 'https://github.com/golangci/golangci-lint/releases/download/v2.3.1/golangci-lint-2.3.1-darwin-amd64.tar.gz' default['golangci-lint']['sha256'] = - 'a88cbdc86b483fe44e90bf2dcc3fec2af8c754116e6edf0aa6592cac5baa7a0e' + '2f4412ffa77cb58ab3a7d065e1fff01ab714bb2b6fa06ad16ed793b6fc8d534d' default['nodejs']['download_path'] = 'https://nodejs.org/dist/v18.16.0/node-v18.16.0-darwin-x64.tar.gz' From 78f47e74476d0a27df2edd9ef0712e5a9e787fc8 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 13 Aug 2025 19:13:37 +0000 Subject: [PATCH 4/6] Upgrade kubectl to latest version Signed-off-by: Dom Del Nano --- tools/chef/cookbooks/px_dev_extras/attributes/linux.rb | 4 ++-- tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb b/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb index 41168ee694f..abb58c3669b 100644 --- a/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb +++ b/tools/chef/cookbooks/px_dev_extras/attributes/linux.rb @@ -43,9 +43,9 @@ 'ca2d5d40d4cdfb9a3a6205dd803b5bc8def00bd2f13e5526c127e9b667974a89' default['kubectl']['download_path'] = - 'https://storage.googleapis.com/kubernetes-release/release/v1.26.0/bin/linux/amd64/kubectl' + 'https://dl.k8s.io/release/v1.33.3/bin/linux/amd64/kubectl' default['kubectl']['sha256'] = - 'b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae' + '2fcf65c64f352742dc253a25a7c95617c2aba79843d1b74e585c69fe4884afb0' default['kustomize']['download_path'] = 'https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.0.3/kustomize_v5.0.3_linux_amd64.tar.gz' diff --git a/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb b/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb index dea948cc8a6..84cc19c046a 100644 --- a/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb +++ b/tools/chef/cookbooks/px_dev_extras/attributes/mac_os_x.rb @@ -44,9 +44,9 @@ '9d029df37664b50e427442a600e4e065fa75fd74dac996c831ac68359654b2c4' default['kubectl']['download_path'] = - 'https://storage.googleapis.com/kubernetes-release/release/v1.26.0/bin/darwin/amd64/kubectl' + 'https://dl.k8s.io/release/v1.33.3/bin/darwin/amd64/kubectl' default['kubectl']['sha256'] = - 'be9dc0782a7b257d9cfd66b76f91081e80f57742f61e12cd29068b213ee48abc' + '9652b55a58e84454196a7b9009f6d990d3961e2bd4bd03f64111d959282b46b1' default['kustomize']['download_path'] = 'https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.0.3/kustomize_v5.0.3_darwin_amd64.tar.gz' From 754c64cd5030d359ad659d326dbb2c7b9380dd35 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 13 Aug 2025 19:36:28 +0000 Subject: [PATCH 5/6] Build new images Signed-off-by: Dom Del Nano --- docker.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker.properties b/docker.properties index e54e1ae7e8e..82ea19c1351 100644 --- a/docker.properties +++ b/docker.properties @@ -1,4 +1,4 @@ -DOCKER_IMAGE_TAG=202508081946 -LINTER_IMAGE_DIGEST=85a928e7607ef9c0c78f7aa421b04e328d5085bb1698bcf7eec1dcb6d609e3c8 -DEV_IMAGE_DIGEST=4a116347d80d330e978d57552a9a49258fd615912cd8f94398a2281bb7e362eb -DEV_IMAGE_WITH_EXTRAS_DIGEST=5b60b6fa4a98265da41a8ec4ec363bf946fd1b5d7b977d46bd6c6d941e183816 +DOCKER_IMAGE_TAG=202508131912 +LINTER_IMAGE_DIGEST=db3238ae3ab3f0fe307aef8920a29b5f0df808248c16a1650baa228c4cefbd4c +DEV_IMAGE_DIGEST=42c7f00b68db0835c266c5aceb6f67ec7e43342336f95218af14e19858e08854 +DEV_IMAGE_WITH_EXTRAS_DIGEST=bbcd6dc6d269231163be9782d42bdf2b2855a34ab384a853fa67e13e946948ec From 11431069fe96306ae6fdfd52cb6dd7f42b341756 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 13 Aug 2025 19:42:02 +0000 Subject: [PATCH 6/6] Remove irrelevant file Signed-off-by: Dom Del Nano --- src/stirling/kernel_headers_faq.md | 113 ----------------------------- 1 file changed, 113 deletions(-) delete mode 100644 src/stirling/kernel_headers_faq.md diff --git a/src/stirling/kernel_headers_faq.md b/src/stirling/kernel_headers_faq.md deleted file mode 100644 index 0ec3b91ad5b..00000000000 --- a/src/stirling/kernel_headers_faq.md +++ /dev/null @@ -1,113 +0,0 @@ -# Pixie Recommends Installing Your Distro’s Linux Kernel Headers - -Pixie uses eBPF programs; some are compiled **at runtime** (via BCC/LLVM) against the node’s running kernel. That compilation needs the kernel’s C headers that match the exact kernel build (including vendor backports). Mismatched or missing headers can lead to compile failures or silently incorrect offsets in tracers. - ---- - -## TL;DR - -- **If Pixie deploys and its tracers start successfully, you can ignore the kernel-headers warning.** It’s informational. -- **If you hit *any* Pixie issues or suspect a bug, first install your distro’s kernel headers on every node and re-test.** This resolves a large class of tracer initialization problems on vendor-patched kernels. -- **Note:** Some minimal/immutable distros don’t ship a headers package (notably **Google Container-Optimized OS / COS**). See guidance below. - -> **Screenshot of the warning in `px`** -> -> Replace the URL below with the image from PR **#2250**. -> -> `![Missing kernel headers — informational warning](INSERT_IMAGE_URL_FROM_PR_2250)` - ---- - -## Why headers matter - -- **Runtime compilation (BCC):** Pixie compiles certain eBPF programs on the node. BCC needs kernel headers to resolve structure layouts, constants, and feature guards accurately. Without a match, compilation may fail or produce incorrect code. -- **Vendor patches/backports:** Distro kernels often diverge from upstream. Only the **distro’s header package** reflects those exact changes; generic or prepackaged headers are best-effort and can be wrong. - ---- - -## What you’ll see when headers are missing - -- A **warning** during deploy/validation that headers weren’t detected. -- In agent logs (e.g., `vizier-pem`), messages about missing `/lib/modules/$(uname -r)/{build,source}` and attempts to fall back to bundled headers. -- Tracers that rely on BCC may fail to initialize (socket/HTTP/DNS tracing most commonly). - ---- - -## Install instructions (per distro) - -> Ensure the installed header version **matches `uname -r`**. If a kernel update is pulled in, **reboot** so the running kernel matches the headers. - -- **Debian/Ubuntu** - ```bash - sudo apt-get update - sudo apt-get install -y linux-headers-$(uname -r) - ``` - -- **RHEL / Rocky / Alma / Fedora** - ```bash - # DNF-based - sudo dnf install -y kernel-devel-$(uname -r) - # YUM-based - sudo yum install -y kernel-devel-$(uname -r) - # (Some environments also need: kernel-headers-$(uname -r)) - ``` - -- **Amazon Linux** - ```bash - AL2 - sudo yum install -y kernel-devel-$(uname -r) - - # AL2023 - sudo dnf install -y kernel-devel - # If this pulled a newer kernel, reboot so uname -r matches. - ``` - -- **openSUSE** - ```bash - sudo zypper install -y kernel-default-devel - ``` - -## Distros without kernel-headers packages (notable: Google COS) - -Some immutable/minimal distros **don’t provide a headers package or a host package manager**: - -- **Google Container-Optimized OS (COS):** Root filesystem is read-only; there’s no `apt`/`dnf` on the host. COS’s `toolbox` is for ephemeral debugging and not for installing kernel headers on the node. - **Options:** - - Use **Ubuntu** (or other header-capable) node images on **GKE Standard** when you need BCC runtime compilation. - - For **Autopilot** (COS-based), consider running Pixie on **Standard** for debugging, or use a custom image/variant that includes headers where feasible. - - Pixie will try **prepackaged headers** as a fallback, but these are not guaranteed to match vendor-patched kernels. - - If available in your environment, loading **in-kernel headers** via `kheaders` (`/sys/kernel/kheaders.tar.xz`) can help, but support varies by vendor. - -Other minimal/immutable OSes (e.g., Fedora CoreOS, Bottlerocket, MicroOS) have similar constraints—prefer images/variants that include headers or build them into your node image pipeline. - ---- - -## Troubleshooting checklist - -1. **Match versions:** `uname -r` must match the headers you installed. -2. **Reboot after updates:** If the package manager updated the kernel, reboot nodes. -3. **Check presence:** Ensure `/lib/modules/$(uname -r)/build` and `.../include` exist. -4. **Review logs:** Look at `vizier-pem` logs for BCC compile/load failures. -5. **Try headers first:** If you see tracer init failures, install headers and re-test before deeper debugging. - ---- - -## FAQ - -**Is BTF/CO-RE enough to avoid headers?** -Not universally. Pixie still uses BCC in parts of the data path, which needs headers at runtime. BTF helps CO-RE workflows, but does not remove the need for matching headers where BCC is used. - -**What breaks if I skip headers?** -Most commonly, the **socket/HTTP/DNS tracers** fail to initialize. You might still see some metrics, but protocol-level visibility will be limited. - -**Why prefer distro headers over Pixie’s packaged headers?** -Because **distro kernels are not vanilla**—they include backports and config deltas. Distro header packages are the authoritative match; bundled headers are a convenience, not a guarantee. - ---- - -## Related references - -- Pixie PR: **#2250** — CLI warning message and docs link - https://github.com/pixie-io/pixie/pull/2250 -- Pixie Issue: **#2051** — Recommend installing distro headers; examples across distros - https://github.com/pixie-io/pixie/issues/2051