From 50ac6967e696fcf1547e17ce7b47dfc896cfdba4 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 12:33:56 +0100 Subject: [PATCH 01/10] Add diagnostic logging to updateUID for CI investigation --- src/spec-node/containerFeatures.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index e05822d1b..b86ea7fb7 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -420,12 +420,13 @@ function getFeatureEnvVariables(f: Feature) { export async function getRemoteUserUIDUpdateDetails(params: DockerResolverParameters, mergedConfig: MergedDevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { const { common } = params; - const { cliHost } = common; + const { cliHost, output } = common; const { updateRemoteUserUID } = mergedConfig; if (params.updateRemoteUserUIDDefault === 'never' || !(typeof updateRemoteUserUID === 'boolean' ? updateRemoteUserUID : params.updateRemoteUserUIDDefault === 'on') || !(cliHost.platform === 'linux' || params.updateRemoteUserUIDOnMacOS && cliHost.platform === 'darwin')) { return null; } const details = await imageDetails(); + output.write(`updateUID: image=${imageName} Os=${details.Os} Architecture=${details.Architecture} Variant=${details.Variant || '(none)'}`, LogLevel.Info); const imageUser = details.Config.User || 'root'; const remoteUser = mergedConfig.remoteUser || runArgsUser || imageUser; if (remoteUser === 'root' || /^\d+$/.test(remoteUser)) { @@ -434,11 +435,13 @@ export async function getRemoteUserUIDUpdateDetails(params: DockerResolverParame const folderImageName = getFolderImageName(common); const fixedImageName = `${imageName.startsWith(folderImageName) ? imageName : folderImageName}-uid`; + const platform = [details.Os, details.Architecture, details.Variant].filter(Boolean).join('/'); + output.write(`updateUID: remoteUser=${remoteUser} imageUser=${imageUser} platform=${platform}`, LogLevel.Info); return { imageName: fixedImageName, remoteUser, imageUser, - platform: [details.Os, details.Architecture, details.Variant].filter(Boolean).join('/') + platform, }; } @@ -451,6 +454,20 @@ export async function updateRemoteUserUID(params: DockerResolverParameters, merg return imageName; } const { imageName: fixedImageName, remoteUser, imageUser, platform } = updateDetails; + const { output } = common; + + try { + const infoResult = await dockerCLI(params, 'info', '--format', '{{.Driver}} / containerd: {{.DriverStatus}}'); + output.write(`updateUID: docker info: ${infoResult.stdout.toString().trim()}`, LogLevel.Info); + } catch (err) { + output.write(`updateUID: docker info failed: ${err}`, LogLevel.Warning); + } + try { + const inspectResult = await dockerCLI(params, 'inspect', '--type', 'image', imageName); + output.write(`updateUID: docker inspect ${imageName}: ${inspectResult.stdout.toString().trim()}`, LogLevel.Info); + } catch (err) { + output.write(`updateUID: docker inspect failed: ${err}`, LogLevel.Warning); + } const dockerfileName = 'updateUID.Dockerfile'; const srcDockerfile = path.join(common.extensionPath, 'scripts', dockerfileName); @@ -474,6 +491,7 @@ export async function updateRemoteUserUID(params: DockerResolverParameters, merg '--build-arg', `IMAGE_USER=${imageUser}`, emptyFolder, ]; + output.write(`updateUID: docker ${args.join(' ')}`, LogLevel.Info); if (params.isTTY) { await dockerPtyCLI(params, ...args); } else { From 07d6259b9349968e825a4320acf947d6b377f5c9 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 13:36:52 +0100 Subject: [PATCH 02/10] Add docker manifest inspect logging --- src/spec-node/containerFeatures.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index b86ea7fb7..b0a1909c2 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -468,6 +468,12 @@ export async function updateRemoteUserUID(params: DockerResolverParameters, merg } catch (err) { output.write(`updateUID: docker inspect failed: ${err}`, LogLevel.Warning); } + try { + const manifestResult = await dockerCLI(params, 'manifest', 'inspect', imageName); + output.write(`updateUID: docker manifest inspect ${imageName}: ${manifestResult.stdout.toString().trim()}`, LogLevel.Info); + } catch (err) { + output.write(`updateUID: docker manifest inspect failed: ${err}`, LogLevel.Warning); + } const dockerfileName = 'updateUID.Dockerfile'; const srcDockerfile = path.join(common.extensionPath, 'scripts', dockerfileName); From f34276764069b2732c8f4a167b7b897ae2412cf2 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 15:38:16 +0100 Subject: [PATCH 03/10] Use buildx imagetools inspect instead of manifest inspect for local images --- src/spec-node/containerFeatures.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index b0a1909c2..e4ddd07ef 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -469,10 +469,10 @@ export async function updateRemoteUserUID(params: DockerResolverParameters, merg output.write(`updateUID: docker inspect failed: ${err}`, LogLevel.Warning); } try { - const manifestResult = await dockerCLI(params, 'manifest', 'inspect', imageName); - output.write(`updateUID: docker manifest inspect ${imageName}: ${manifestResult.stdout.toString().trim()}`, LogLevel.Info); + const imagetoolsResult = await dockerCLI(params, 'buildx', 'imagetools', 'inspect', imageName, '--raw'); + output.write(`updateUID: docker buildx imagetools inspect ${imageName}: ${imagetoolsResult.stdout.toString().trim()}`, LogLevel.Info); } catch (err) { - output.write(`updateUID: docker manifest inspect failed: ${err}`, LogLevel.Warning); + output.write(`updateUID: docker buildx imagetools inspect failed: ${err instanceof Error ? err.message : JSON.stringify(err)}`, LogLevel.Warning); } const dockerfileName = 'updateUID.Dockerfile'; From 2c7b7e872b2346723ce5d320a9ec95eee2011170 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 15:55:37 +0100 Subject: [PATCH 04/10] Read manifest list blob from containerd content store --- src/spec-node/containerFeatures.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index e4ddd07ef..7fa2c45e7 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import { DevContainerConfig } from '../spec-configuration/configuration'; import { dockerCLI, dockerPtyCLI, ImageDetails, toExecParameters, toPtyExecParameters } from '../spec-shutdown/dockerUtils'; -import { LogLevel, makeLog } from '../spec-utils/log'; +import { LogLevel, makeLog, nullLog } from '../spec-utils/log'; import { FeaturesConfig, getContainerFeaturesBaseDockerFile, getFeatureInstallWrapperScript, getFeatureLayers, getFeatureMainValue, getFeatureValueObject, generateFeaturesConfig, Feature, generateContainerEnvs } from '../spec-configuration/containerFeaturesConfiguration'; import { readLocalFile } from '../spec-utils/pfs'; import { includeAllConfiguredFeatures } from '../spec-utils/product'; @@ -464,16 +464,24 @@ export async function updateRemoteUserUID(params: DockerResolverParameters, merg } try { const inspectResult = await dockerCLI(params, 'inspect', '--type', 'image', imageName); - output.write(`updateUID: docker inspect ${imageName}: ${inspectResult.stdout.toString().trim()}`, LogLevel.Info); + const inspectJson = inspectResult.stdout.toString().trim(); + output.write(`updateUID: docker inspect ${imageName}: ${inspectJson}`, LogLevel.Info); + // Read the raw manifest list from the containerd content store using the image digest + try { + const parsed = JSON.parse(inspectJson); + const digest = parsed[0]?.Id || parsed[0]?.Descriptor?.digest; + if (digest) { + const hash = digest.replace('sha256:', ''); + const blobPath = `/var/lib/docker/containerd/daemon/io.containerd.content.v1.content/blobs/sha256/${hash}`; + const catResult = await runCommandNoPty({ exec: common.cliHost.exec, cmd: 'cat', args: [blobPath], output: nullLog }); + output.write(`updateUID: manifest list (${blobPath}): ${catResult.stdout.toString().trim()}`, LogLevel.Info); + } + } catch (blobErr) { + output.write(`updateUID: reading manifest list blob failed: ${blobErr instanceof Error ? blobErr.message : JSON.stringify(blobErr)}`, LogLevel.Warning); + } } catch (err) { output.write(`updateUID: docker inspect failed: ${err}`, LogLevel.Warning); } - try { - const imagetoolsResult = await dockerCLI(params, 'buildx', 'imagetools', 'inspect', imageName, '--raw'); - output.write(`updateUID: docker buildx imagetools inspect ${imageName}: ${imagetoolsResult.stdout.toString().trim()}`, LogLevel.Info); - } catch (err) { - output.write(`updateUID: docker buildx imagetools inspect failed: ${err instanceof Error ? err.message : JSON.stringify(err)}`, LogLevel.Warning); - } const dockerfileName = 'updateUID.Dockerfile'; const srcDockerfile = path.join(common.extensionPath, 'scripts', dockerfileName); From a5966c9fb714b8bf95af87ef5a837e775bad9d1b Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 16:09:42 +0100 Subject: [PATCH 05/10] Use docker save to extract manifest list platform annotations --- .github/workflows/dev-containers.yml | 2 +- src/spec-node/containerFeatures.ts | 19 ++++++++++++------- src/test/updateUID.test.ts | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dev-containers.yml b/.github/workflows/dev-containers.yml index d1f43995e..a87c02e58 100644 --- a/.github/workflows/dev-containers.yml +++ b/.github/workflows/dev-containers.yml @@ -84,7 +84,7 @@ jobs: - name: Package run: yarn package - name: Run Tests - run: yarn test-matrix --forbid-only ${{ matrix.mocha-args }} + run: yarn test-matrix ${{ matrix.mocha-args }} env: CI: true diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index 7fa2c45e7..e0bc5934d 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -466,18 +466,23 @@ export async function updateRemoteUserUID(params: DockerResolverParameters, merg const inspectResult = await dockerCLI(params, 'inspect', '--type', 'image', imageName); const inspectJson = inspectResult.stdout.toString().trim(); output.write(`updateUID: docker inspect ${imageName}: ${inspectJson}`, LogLevel.Info); - // Read the raw manifest list from the containerd content store using the image digest + // Extract the OCI index from docker save to see manifest list platform annotations try { const parsed = JSON.parse(inspectJson); - const digest = parsed[0]?.Id || parsed[0]?.Descriptor?.digest; + const digest = parsed[0]?.Descriptor?.digest; if (digest) { - const hash = digest.replace('sha256:', ''); - const blobPath = `/var/lib/docker/containerd/daemon/io.containerd.content.v1.content/blobs/sha256/${hash}`; - const catResult = await runCommandNoPty({ exec: common.cliHost.exec, cmd: 'cat', args: [blobPath], output: nullLog }); - output.write(`updateUID: manifest list (${blobPath}): ${catResult.stdout.toString().trim()}`, LogLevel.Info); + const hash = digest.replace('sha256:', '').replace(':', '/'); + // docker save outputs OCI layout; extract the manifest list blob by digest + const saveResult = await runCommandNoPty({ + exec: common.cliHost.exec, + cmd: '/bin/sh', + args: ['-c', `docker save ${imageName} | tar -xO blobs/sha256/${hash} 2>/dev/null || docker save ${imageName} | tar -xO index.json 2>/dev/null`], + output: nullLog + }); + output.write(`updateUID: manifest list for ${imageName}: ${saveResult.stdout.toString().trim()}`, LogLevel.Info); } } catch (blobErr) { - output.write(`updateUID: reading manifest list blob failed: ${blobErr instanceof Error ? blobErr.message : JSON.stringify(blobErr)}`, LogLevel.Warning); + output.write(`updateUID: reading manifest list failed: ${blobErr instanceof Error ? blobErr.message : JSON.stringify(blobErr)}`, LogLevel.Warning); } } catch (err) { output.write(`updateUID: docker inspect failed: ${err}`, LogLevel.Warning); diff --git a/src/test/updateUID.test.ts b/src/test/updateUID.test.ts index 15d4f5dce..52e0df6c5 100644 --- a/src/test/updateUID.test.ts +++ b/src/test/updateUID.test.ts @@ -9,7 +9,7 @@ import { devContainerDown, devContainerUp, shellExec } from './testUtils'; const pkg = require('../../package.json'); -(process.platform === 'linux' ? describe : describe.skip)('Dev Containers CLI', function () { +(process.platform === 'linux' ? describe.only : describe.skip)('Dev Containers CLI', function () { this.timeout('120s'); const tmp = path.relative(process.cwd(), path.join(__dirname, 'tmp')); From 10bece566b0403c0d97a7d36ffe5ed96439eb028 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 16:27:20 +0100 Subject: [PATCH 06/10] Test with BuildKit enabled by default to check manifest list platform bug --- src/test/testUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/testUtils.ts b/src/test/testUtils.ts index cd48c5359..961f54152 100644 --- a/src/test/testUtils.ts +++ b/src/test/testUtils.ts @@ -89,7 +89,7 @@ export async function shellPtyExec(command: string, options: { stdin?: string } } export async function devContainerUp(cli: string, workspaceFolder: string, options?: { cwd?: string; useBuildKit?: boolean; userDataFolder?: string; logLevel?: string; extraArgs?: string; prefix?: string; env?: NodeJS.ProcessEnv }): Promise { - const buildkitOption = (options?.useBuildKit ?? false) ? '' : ' --buildkit=never'; + const buildkitOption = (options?.useBuildKit ?? true) ? '' : ' --buildkit=never'; const userDataFolderOption = (options?.userDataFolder ?? false) ? ` --user-data-folder=${options?.userDataFolder}` : ''; const logLevelOption = (options?.logLevel ?? false) ? ` --log-level ${options?.logLevel}` : ''; const extraArgs = (options?.extraArgs ?? false) ? ` ${options?.extraArgs}` : ''; From 6045c26573c91ee69434c3ea5168f8754be50b71 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 16:34:04 +0100 Subject: [PATCH 07/10] Update Docker to latest before running tests --- .github/workflows/dev-containers.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/dev-containers.yml b/.github/workflows/dev-containers.yml index a87c02e58..d841098c0 100644 --- a/.github/workflows/dev-containers.yml +++ b/.github/workflows/dev-containers.yml @@ -69,6 +69,15 @@ jobs: node-version: '18.x' registry-url: 'https://npm.pkg.github.com' scope: '@microsoft' + - name: Docker Version (before update) + run: docker version + - name: Update Docker + run: | + # Install latest Docker from official repo + sudo apt-get update + sudo apt-get install -y --only-upgrade docker-ce docker-ce-cli containerd.io docker-buildx-plugin + - name: Docker Version (after update) + run: docker version - name: Tools Info run: | docker info From a5e81cb5ec2ce3e8ff8a3bc5c6a953bff70cb0d6 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 16:38:23 +0100 Subject: [PATCH 08/10] Add Docker official repo before upgrading --- .github/workflows/dev-containers.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-containers.yml b/.github/workflows/dev-containers.yml index d841098c0..911b01d7e 100644 --- a/.github/workflows/dev-containers.yml +++ b/.github/workflows/dev-containers.yml @@ -73,9 +73,13 @@ jobs: run: docker version - name: Update Docker run: | - # Install latest Docker from official repo + # Add Docker's official APT repository + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo apt-get install -y --only-upgrade docker-ce docker-ce-cli containerd.io docker-buildx-plugin + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin - name: Docker Version (after update) run: docker version - name: Tools Info From 313470aad7bb853035d0784dd3c9ca9108bd0ad3 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 17:01:25 +0100 Subject: [PATCH 09/10] Add minimal Docker platform bug repro workflow --- .github/workflows/docker-platform-bug.yml | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/docker-platform-bug.yml diff --git a/.github/workflows/docker-platform-bug.yml b/.github/workflows/docker-platform-bug.yml new file mode 100644 index 000000000..330856717 --- /dev/null +++ b/.github/workflows/docker-platform-bug.yml @@ -0,0 +1,86 @@ +name: Docker Platform Bug Repro + +on: + push: + branches: + - 'chrmarti/**' + +jobs: + repro: + name: Platform Bug Repro + runs-on: ubuntu-latest + steps: + - name: Update Docker + run: | + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin + - name: Setup + run: | + docker version + docker info | grep -E "Storage Driver|driver-type" + docker run --privileged --rm tonistiigi/binfmt --install all + - name: Build with platform in Dockerfile only + run: | + dir=$(mktemp -d) + echo 'FROM --platform=linux/arm64 debian:latest' > "$dir/Dockerfile" + docker buildx build --load -t test-arm64-in-dockerfile "$dir" + - name: Inspect manifest list (Dockerfile platform) + run: | + docker image inspect test-arm64-in-dockerfile --format '{{.Architecture}}' + digest=$(docker image inspect test-arm64-in-dockerfile --format '{{.Id}}') + docker save test-arm64-in-dockerfile | tar -xO blobs/sha256/${digest#sha256:} | python3 -c " + import sys, json + data = json.load(sys.stdin) + if 'manifests' in data: + print('Manifest list found:') + for m in data['manifests']: + p = m.get('platform', {}) + t = m.get('annotations', {}).get('vnd.docker.reference.type', 'image') + print(f' type={t} arch={p.get(\"architecture\")} os={p.get(\"os\")} variant={p.get(\"variant\", \"\")}') + else: + print('No manifest list (single manifest)') + " + - name: Build using that image as base with --platform linux/arm64 + run: | + dir=$(mktemp -d) + echo 'FROM test-arm64-in-dockerfile' > "$dir/Dockerfile" + echo "Expecting this to fail due to manifest list platform mismatch..." + if docker build --platform linux/arm64 -t test-arm64-rebuild "$dir" 2>&1; then + echo "BUILD SUCCEEDED (bug may be fixed)" + else + echo "BUILD FAILED (bug confirmed)" + fi + - name: Build with --platform on CLI + run: | + dir=$(mktemp -d) + echo 'FROM debian:latest' > "$dir/Dockerfile" + docker buildx build --load --platform linux/arm64 -t test-arm64-on-cli "$dir" + - name: Inspect manifest list (CLI platform) + run: | + docker image inspect test-arm64-on-cli --format '{{.Architecture}}' + digest=$(docker image inspect test-arm64-on-cli --format '{{.Id}}') + docker save test-arm64-on-cli | tar -xO blobs/sha256/${digest#sha256:} | python3 -c " + import sys, json + data = json.load(sys.stdin) + if 'manifests' in data: + print('Manifest list found:') + for m in data['manifests']: + p = m.get('platform', {}) + t = m.get('annotations', {}).get('vnd.docker.reference.type', 'image') + print(f' type={t} arch={p.get(\"architecture\")} os={p.get(\"os\")} variant={p.get(\"variant\", \"\")}') + else: + print('No manifest list (single manifest)') + " + - name: Build using CLI-platform image with --platform linux/arm64 + run: | + dir=$(mktemp -d) + echo 'FROM test-arm64-on-cli' > "$dir/Dockerfile" + if docker build --platform linux/arm64 -t test-arm64-on-cli-rebuild "$dir" 2>&1; then + echo "BUILD SUCCEEDED (expected)" + else + echo "BUILD FAILED (unexpected)" + fi From 97e15d415ad910e008b84c8267fed9cb49af65ba Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Mon, 16 Feb 2026 17:39:11 +0100 Subject: [PATCH 10/10] Add docker version and docker info steps to repro workflow --- .github/workflows/docker-platform-bug.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-platform-bug.yml b/.github/workflows/docker-platform-bug.yml index 330856717..c9e5c95ea 100644 --- a/.github/workflows/docker-platform-bug.yml +++ b/.github/workflows/docker-platform-bug.yml @@ -18,11 +18,12 @@ jobs: echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin - - name: Setup - run: | - docker version - docker info | grep -E "Storage Driver|driver-type" - docker run --privileged --rm tonistiigi/binfmt --install all + - name: Docker Version + run: docker version + - name: Docker Info + run: docker info + - name: Setup QEMU + run: docker run --privileged --rm tonistiigi/binfmt --install all - name: Build with platform in Dockerfile only run: | dir=$(mktemp -d)