From 96470a6e2c32487652d631033a3f679cd86c74da Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Fri, 20 Feb 2026 06:15:45 -0800 Subject: [PATCH] Update common Docker engineering infrastructure with latest --- eng/docker-tools/CHANGELOG.md | 76 +++++++++++++++++++ .../templates/jobs/build-images.yml | 2 - eng/docker-tools/templates/jobs/publish.yml | 2 - .../templates/stages/build-and-test.yml | 6 +- .../stages/dotnet/publish-config-nonprod.yml | 62 ++++++++------- .../stages/dotnet/publish-config-prod.yml | 62 ++++++++------- .../stages/setup-service-connections.yml | 36 ++++++++- .../templates/steps/copy-base-images.yml | 4 - .../templates/variables/docker-images.yml | 2 +- 9 files changed, 174 insertions(+), 78 deletions(-) create mode 100644 eng/docker-tools/CHANGELOG.md diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md new file mode 100644 index 00000000..7903acd8 --- /dev/null +++ b/eng/docker-tools/CHANGELOG.md @@ -0,0 +1,76 @@ +# Docker Tools / ImageBuilder Changelog + +All breaking changes and new features in `eng/docker-tools` will be documented in this file. + +--- + +## 2026-02-19: Separate Registry Endpoints from Authentication + +- Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945) +- Issue: [#1914](https://github.com/dotnet/docker-tools/issues/1914) + +Authentication details (`serviceConnection`, `resourceGroup`, `subscription`) have been moved from individual registry endpoints into a centralized `RegistryAuthentication` list. +This fixes an issue where ACR authentication could fail when multiple service connections existed for the same registry. + +**Before:** Each registry endpoint embedded its own authentication: + +```yaml +publishConfig: + BuildRegistry: + server: $(acr.server) + repoPrefix: "my-prefix/" + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(serviceConnectionName) + id: $(serviceConnection.id) + clientId: $(serviceConnection.clientId) + tenantId: $(tenant) + PublishRegistry: + server: $(acr.server) + repoPrefix: "publish/" + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(publishServiceConnectionName) + id: $(publishServiceConnection.id) + clientId: $(publishServiceConnection.clientId) + tenantId: $(tenant) +``` + +**After:** Registry endpoints only contain `server` and `repoPrefix`. Authentication is centralized: + +```yaml +publishConfig: + BuildRegistry: + server: $(acr.server) + repoPrefix: "my-prefix/" + PublishRegistry: + server: $(acr.server) + repoPrefix: "publish/" + RegistryAuthentication: + - server: $(acr.server) + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(serviceConnectionName) + id: $(serviceConnection.id) + clientId: $(serviceConnection.clientId) + tenantId: $(tenant) +``` + +How to update: +- Update any publishConfig parameters to match the new structure. + - Multiple registries can share authentication. If two registries use the same ACR server, only one entry is needed in `RegistryAuthentication`. + - The new structure should match [ImageBuilder's Configuration Model](https://github.com/dotnet/docker-tools/tree/a82572386854f15af441c50c6efa698a627e9f2b/src/ImageBuilder/Configuration). +- Update service connection setup (if using `setup-service-connections.yml`): + - The template now supports looking up service connections from `publishConfig.RegistryAuthentication` + - Use the new `usesRegistries` parameter to specify which registries need auth setup: + ```yaml + - template: eng/docker-tools/templates/stages/setup-service-connections.yml + parameters: + publishConfig: ${{ variables.publishConfig }} + usesRegistries: + - $(buildRegistry.server) + - $(publishRegistry.server) + ``` diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index db9b4f7f..025d3c8d 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -91,8 +91,6 @@ jobs: --architecture $(architecture) --retry --digests-out-var 'builtImages' - --acr-subscription '${{ parameters.publishConfig.BuildRegistry.subscription }}' - --acr-resource-group '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}' $(manifestVariables) $(imageBuilderBuildArgs) - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index 98e29f4a..b86ec1ee 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -97,8 +97,6 @@ jobs: internalProjectName: ${{ parameters.internalProjectName }} args: >- copyAcrImages - '${{ parameters.publishConfig.BuildRegistry.subscription }}' - '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}' '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}' '${{ parameters.publishConfig.BuildRegistry.server }}' --os-type '*' diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml index 3f7118ab..1e39a998 100644 --- a/eng/docker-tools/templates/stages/build-and-test.yml +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -3,11 +3,13 @@ parameters: testMatrixType: platformVersionedOs buildMatrixCustomBuildLegGroupArgs: "" testMatrixCustomBuildLegGroupArgs: "" - customCopyBaseImagesInitSteps: [] - customGenerateMatrixInitSteps: [] # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. customInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before copy-base-images runs. + customCopyBaseImagesInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before matrix generation runs. + customGenerateMatrixInitSteps: [] # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml index 424f3aaf..6f3e4995 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml @@ -53,35 +53,44 @@ stages: InternalMirrorRegistry: server: $(acr-staging-test.server) repoPrefix: $(internalMirrorRepoPrefix) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - serviceConnection: - name: $(internal-mirror-test.serviceConnectionName) - id: $(internal-mirror-test.serviceConnection.id) - clientId: $(internal-mirror-test.serviceConnection.clientId) - tenantId: $(testTenant) PublicMirrorRegistry: server: $(public-mirror.server) repoPrefix: $(publicMirrorRepoPrefix) - resourceGroup: $(public-mirror.resourceGroup) - subscription: $(public-mirror.subscription) - serviceConnection: - name: $(public-mirror.serviceConnectionName) - id: $(public-mirror.serviceConnection.id) - tenantId: $(public-mirror.serviceConnection.tenantId) - clientId: $(public-mirror.serviceConnection.clientId) BuildRegistry: server: $(acr-staging-test.server) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" - serviceConnection: - name: $(build-test.serviceConnectionName) - id: $(build-test.serviceConnection.id) - clientId: $(build-test.serviceConnection.clientId) - tenantId: $(testTenant) + + PublishRegistry: + server: $(acr-test.server) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + + RegistryAuthentication: + - server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + - server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + - server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) cleanServiceConnection: name: $(clean-test.serviceConnectionName) @@ -94,14 +103,3 @@ stages: id: $(test-nonprod.serviceConnection.id) clientId: $(test-nonprod.serviceConnection.clientId) tenantId: $(testTenant) - - PublishRegistry: - server: $(acr-test.server) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - repoPrefix: "${{ parameters.publishRepoPrefix }}" - serviceConnection: - name: $(publish-test.serviceConnectionName) - id: $(publish-test.serviceConnection.id) - clientId: $(publish-test.serviceConnection.clientId) - tenantId: $(testTenant) diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml index 7f9a4e00..24746b3a 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml @@ -53,35 +53,44 @@ stages: InternalMirrorRegistry: server: $(acr-staging.server) repoPrefix: $(internalMirrorRepoPrefix) - resourceGroup: $(acr-staging.resourceGroup) - subscription: $(acr-staging.subscription) - serviceConnection: - name: $(internal-mirror.serviceConnectionName) - id: $(internal-mirror.serviceConnection.id) - clientId: $(internal-mirror.serviceConnection.clientId) - tenantId: $(internal-mirror.serviceConnection.tenantId) PublicMirrorRegistry: server: $(public-mirror.server) repoPrefix: $(publicMirrorRepoPrefix) - resourceGroup: $(public-mirror.resourceGroup) - subscription: $(public-mirror.subscription) - serviceConnection: - name: $(public-mirror.serviceConnectionName) - id: $(public-mirror.serviceConnection.id) - tenantId: $(public-mirror.serviceConnection.tenantId) - clientId: $(public-mirror.serviceConnection.clientId) BuildRegistry: server: $(acr-staging.server) - resourceGroup: $(acr-staging.resourceGroup) - subscription: $(acr-staging.subscription) repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" - serviceConnection: - name: $(build.serviceConnectionName) - id: $(build.serviceConnection.id) - clientId: $(build.serviceConnection.clientId) - tenantId: $(build.serviceConnection.tenantId) + + PublishRegistry: + server: $(acr.server) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + + RegistryAuthentication: + - server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + - server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + - server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) cleanServiceConnection: name: $(clean.serviceConnectionName) @@ -94,14 +103,3 @@ stages: id: $(test.serviceConnection.id) clientId: $(test.serviceConnection.clientId) tenantId: $(test.serviceConnection.tenantId) - - PublishRegistry: - server: $(acr.server) - resourceGroup: $(acr.resourceGroup) - subscription: $(acr.subscription) - repoPrefix: "${{ parameters.publishRepoPrefix }}" - serviceConnection: - name: $(publish.serviceConnectionName) - id: $(publish.serviceConnection.id) - clientId: $(publish.serviceConnection.clientId) - tenantId: $(publish.serviceConnection.tenantId) diff --git a/eng/docker-tools/templates/stages/setup-service-connections.yml b/eng/docker-tools/templates/stages/setup-service-connections.yml index 2ef74e90..405bc703 100644 --- a/eng/docker-tools/templates/stages/setup-service-connections.yml +++ b/eng/docker-tools/templates/stages/setup-service-connections.yml @@ -3,6 +3,10 @@ # it is declared in this stage's parameters, even if your pipeline has already # been granted access to the service connection. This stage also does not need # to complete before the service connection is used. +# +# There are two ways to specify service connections: +# - Pass `serviceConnections` directly (list of {name: string} objects) +# - Pass `publishConfig` + `registries` to look up auth from RegistryAuthentication parameters: - name: pool type: object @@ -10,14 +14,26 @@ parameters: name: $(default1ESInternalPoolName) image: $(default1ESInternalPoolImage) os: linux -# serviceConnections object shape: -# - name: string + +# Explicit list of service connections to initialize +# Shape: [{ name: string }] - name: serviceConnections type: object default: [] -stages: +# List of registry servers that need authentication. These will be looked up in +# publishConfig.RegistryAuthentication. +# Make sure to provide the publishConfig parameter. +- name: usesRegistries + type: object + default: [] +# Look up service connections from publishConfig based on registries +# The publish configuration containing RegistryAuthentication entries. +- name: publishConfig + type: object + default: {} +stages: - stage: SetupServiceConnectionsStage displayName: Setup service connections jobs: @@ -27,6 +43,8 @@ stages: pool: ${{ parameters.pool }} steps: - checkout: none + + # Direct service connections list - ${{ each serviceConnection in parameters.serviceConnections }}: - task: AzureCLI@2 displayName: Setup ${{ serviceConnection.name }} @@ -36,3 +54,15 @@ stages: scriptLocation: inlineScript inlineScript: | az account show + + # Setup registry service connections + - ${{ if gt(length(parameters.usesRegistries), 0) }}: + - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}: + - ${{ if containsValue(parameters.usesRegistries, auth.server) }}: + - task: AzureCLI@2 + displayName: Setup ${{ auth.serviceConnection.name }} + inputs: + azureSubscription: ${{ auth.serviceConnection.name }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: az account show diff --git a/eng/docker-tools/templates/steps/copy-base-images.yml b/eng/docker-tools/templates/steps/copy-base-images.yml index 0e9e09f6..6664c8f9 100644 --- a/eng/docker-tools/templates/steps/copy-base-images.yml +++ b/eng/docker-tools/templates/steps/copy-base-images.yml @@ -3,8 +3,6 @@ parameters: type: object default: server: "" - subscription: "" - resourceGroup: "" repoPrefix: "" - name: additionalOptions type: string @@ -29,8 +27,6 @@ steps: # error args: >- copyBaseImages - '${{ parameters.acr.subscription }}' - '${{ parameters.acr.resourceGroup }}' $(dockerHubRegistryCreds) $(customCopyBaseImagesArgs) --repo-prefix '${{ parameters.acr.repoPrefix }}' diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index b9327035..1b4eb7cb 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2887966 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2908146 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner