[main] Update common Docker engineering infrastructure with latest

eng/docker-tools/CHANGELOG.md

@@ -0,0 +1,76 @@
+# Docker Tools / ImageBuilder Changelog
+
+All breaking changes and new features in `eng/docker-tools` will be documented in this file.
+
+---
+
+## 2026-02-19: Separate Registry Endpoints from Authentication
+
+- Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945)
+- Issue: [#1914](https://github.com/dotnet/docker-tools/issues/1914)
+
+Authentication details (`serviceConnection`, `resourceGroup`, `subscription`) have been moved from individual registry endpoints into a centralized `RegistryAuthentication` list.
+This fixes an issue where ACR authentication could fail when multiple service connections existed for the same registry.
+
+**Before:** Each registry endpoint embedded its own authentication:
+
+```yaml
+publishConfig:
+  BuildRegistry:
+    server: $(acr.server)
+    repoPrefix: "my-prefix/"
+    resourceGroup: $(resourceGroup)
+    subscription: $(subscription)
+    serviceConnection:
+      name: $(serviceConnectionName)
+      id: $(serviceConnection.id)
+      clientId: $(serviceConnection.clientId)
+      tenantId: $(tenant)
+  PublishRegistry:
+    server: $(acr.server)
+    repoPrefix: "publish/"
+    resourceGroup: $(resourceGroup)
+    subscription: $(subscription)
+    serviceConnection:
+      name: $(publishServiceConnectionName)
+      id: $(publishServiceConnection.id)
+      clientId: $(publishServiceConnection.clientId)
+      tenantId: $(tenant)
+```
+
+**After:** Registry endpoints only contain `server` and `repoPrefix`. Authentication is centralized:
+
+```yaml
+publishConfig:
+  BuildRegistry:
+    server: $(acr.server)
+    repoPrefix: "my-prefix/"
+  PublishRegistry:
+    server: $(acr.server)
+    repoPrefix: "publish/"
+  RegistryAuthentication:
+    - server: $(acr.server)
+      resourceGroup: $(resourceGroup)
+      subscription: $(subscription)
+      serviceConnection:
+        name: $(serviceConnectionName)
+        id: $(serviceConnection.id)
+        clientId: $(serviceConnection.clientId)
+        tenantId: $(tenant)
+```
+
+How to update:
+- Update any publishConfig parameters to match the new structure.
+    - Multiple registries can share authentication. If two registries use the same ACR server, only one entry is needed in `RegistryAuthentication`.
+    - The new structure should match [ImageBuilder's Configuration Model](https://github.com/dotnet/docker-tools/tree/a82572386854f15af441c50c6efa698a627e9f2b/src/ImageBuilder/Configuration).
+- Update service connection setup (if using `setup-service-connections.yml`):
+   - The template now supports looking up service connections from `publishConfig.RegistryAuthentication`
+   - Use the new `usesRegistries` parameter to specify which registries need auth setup:
+     ```yaml
+     - template: eng/docker-tools/templates/stages/setup-service-connections.yml
+       parameters:
+         publishConfig: ${{ variables.publishConfig }}
+         usesRegistries:
+           - $(buildRegistry.server)
+           - $(publishRegistry.server)
+     ```

eng/docker-tools/templates/jobs/build-images.yml

@@ -91,8 +91,6 @@ jobs:
         --architecture $(architecture)
         --retry
         --digests-out-var 'builtImages'
-        --acr-subscription '${{ parameters.publishConfig.BuildRegistry.subscription }}'
-        --acr-resource-group '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}'
         $(manifestVariables)
         $(imageBuilderBuildArgs)
   - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self

eng/docker-tools/templates/jobs/publish.yml

@@ -97,8 +97,6 @@ jobs:
       internalProjectName: ${{ parameters.internalProjectName }}
       args: >-
         copyAcrImages
-        '${{ parameters.publishConfig.BuildRegistry.subscription }}'
-        '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}'
         '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}'
         '${{ parameters.publishConfig.BuildRegistry.server }}'
         --os-type '*'

eng/docker-tools/templates/stages/build-and-test.yml

@@ -3,11 +3,13 @@ parameters:
   testMatrixType: platformVersionedOs
   buildMatrixCustomBuildLegGroupArgs: ""
   testMatrixCustomBuildLegGroupArgs: ""
-  customCopyBaseImagesInitSteps: []
-  customGenerateMatrixInitSteps: []
   # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
   # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
   customInitSteps: []
+  # Custom steps that run after ImageBuilder is set up but before copy-base-images runs.
+  customCopyBaseImagesInitSteps: []
+  # Custom steps that run after ImageBuilder is set up but before matrix generation runs.
+  customGenerateMatrixInitSteps: []
   # Custom steps that run after ImageBuilder is set up but before the build starts.
   # Use for build-specific initialization (e.g., setting variables, additional setup).
   customBuildInitSteps: []

eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml

@@ -53,35 +53,44 @@ stages:
       InternalMirrorRegistry:
         server: $(acr-staging-test.server)
         repoPrefix: $(internalMirrorRepoPrefix)
-        resourceGroup: $(testResourceGroup)
-        subscription: $(testSubscription)
-        serviceConnection:
-          name: $(internal-mirror-test.serviceConnectionName)
-          id: $(internal-mirror-test.serviceConnection.id)
-          clientId: $(internal-mirror-test.serviceConnection.clientId)
-          tenantId: $(testTenant)
 
       PublicMirrorRegistry:
         server: $(public-mirror.server)
         repoPrefix: $(publicMirrorRepoPrefix)
-        resourceGroup: $(public-mirror.resourceGroup)
-        subscription: $(public-mirror.subscription)
-        serviceConnection:
-          name: $(public-mirror.serviceConnectionName)
-          id: $(public-mirror.serviceConnection.id)
-          tenantId: $(public-mirror.serviceConnection.tenantId)
-          clientId: $(public-mirror.serviceConnection.clientId)
 
       BuildRegistry:
         server: $(acr-staging-test.server)
-        resourceGroup: $(testResourceGroup)
-        subscription: $(testSubscription)
         repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/"
-        serviceConnection:
-          name: $(build-test.serviceConnectionName)
-          id: $(build-test.serviceConnection.id)
-          clientId: $(build-test.serviceConnection.clientId)
-          tenantId: $(testTenant)
+
+      PublishRegistry:
+        server: $(acr-test.server)
+        repoPrefix: "${{ parameters.publishRepoPrefix }}"
+
+      RegistryAuthentication:
+        - server: $(acr-staging-test.server)
+          resourceGroup: $(testResourceGroup)
+          subscription: $(testSubscription)
+          serviceConnection:
+            name: $(build-test.serviceConnectionName)
+            id: $(build-test.serviceConnection.id)
+            clientId: $(build-test.serviceConnection.clientId)
+            tenantId: $(testTenant)
+        - server: $(public-mirror.server)
+          resourceGroup: $(public-mirror.resourceGroup)
+          subscription: $(public-mirror.subscription)
+          serviceConnection:
+            name: $(public-mirror.serviceConnectionName)
+            id: $(public-mirror.serviceConnection.id)
+            tenantId: $(public-mirror.serviceConnection.tenantId)
+            clientId: $(public-mirror.serviceConnection.clientId)
+        - server: $(acr-test.server)
+          resourceGroup: $(testResourceGroup)
+          subscription: $(testSubscription)
+          serviceConnection:
+            name: $(publish-test.serviceConnectionName)
+            id: $(publish-test.serviceConnection.id)
+            clientId: $(publish-test.serviceConnection.clientId)
+            tenantId: $(testTenant)
 
       cleanServiceConnection:
         name: $(clean-test.serviceConnectionName)
@@ -94,14 +103,3 @@ stages:
         id: $(test-nonprod.serviceConnection.id)
         clientId: $(test-nonprod.serviceConnection.clientId)
         tenantId: $(testTenant)
-
-      PublishRegistry:
-        server: $(acr-test.server)
-        resourceGroup: $(testResourceGroup)
-        subscription: $(testSubscription)
-        repoPrefix: "${{ parameters.publishRepoPrefix }}"
-        serviceConnection:
-          name: $(publish-test.serviceConnectionName)
-          id: $(publish-test.serviceConnection.id)
-          clientId: $(publish-test.serviceConnection.clientId)
-          tenantId: $(testTenant)

eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml

@@ -53,35 +53,44 @@ stages:
       InternalMirrorRegistry:
         server: $(acr-staging.server)
         repoPrefix: $(internalMirrorRepoPrefix)
-        resourceGroup: $(acr-staging.resourceGroup)
-        subscription: $(acr-staging.subscription)
-        serviceConnection:
-          name: $(internal-mirror.serviceConnectionName)
-          id: $(internal-mirror.serviceConnection.id)
-          clientId: $(internal-mirror.serviceConnection.clientId)
-          tenantId: $(internal-mirror.serviceConnection.tenantId)
 
       PublicMirrorRegistry:
         server: $(public-mirror.server)
         repoPrefix: $(publicMirrorRepoPrefix)
-        resourceGroup: $(public-mirror.resourceGroup)
-        subscription: $(public-mirror.subscription)
-        serviceConnection:
-          name: $(public-mirror.serviceConnectionName)
-          id: $(public-mirror.serviceConnection.id)
-          tenantId: $(public-mirror.serviceConnection.tenantId)
-          clientId: $(public-mirror.serviceConnection.clientId)
 
       BuildRegistry:
         server: $(acr-staging.server)
-        resourceGroup: $(acr-staging.resourceGroup)
-        subscription: $(acr-staging.subscription)
         repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/"
-        serviceConnection:
-          name: $(build.serviceConnectionName)
-          id: $(build.serviceConnection.id)
-          clientId: $(build.serviceConnection.clientId)
-          tenantId: $(build.serviceConnection.tenantId)
+
+      PublishRegistry:
+        server: $(acr.server)
+        repoPrefix: "${{ parameters.publishRepoPrefix }}"
+
+      RegistryAuthentication:
+        - server: $(acr-staging.server)
+          resourceGroup: $(acr-staging.resourceGroup)
+          subscription: $(acr-staging.subscription)
+          serviceConnection:
+            name: $(build.serviceConnectionName)
+            id: $(build.serviceConnection.id)
+            clientId: $(build.serviceConnection.clientId)
+            tenantId: $(build.serviceConnection.tenantId)
+        - server: $(public-mirror.server)
+          resourceGroup: $(public-mirror.resourceGroup)
+          subscription: $(public-mirror.subscription)
+          serviceConnection:
+            name: $(public-mirror.serviceConnectionName)
+            id: $(public-mirror.serviceConnection.id)
+            tenantId: $(public-mirror.serviceConnection.tenantId)
+            clientId: $(public-mirror.serviceConnection.clientId)
+        - server: $(acr.server)
+          resourceGroup: $(acr.resourceGroup)
+          subscription: $(acr.subscription)
+          serviceConnection:
+            name: $(publish.serviceConnectionName)
+            id: $(publish.serviceConnection.id)
+            clientId: $(publish.serviceConnection.clientId)
+            tenantId: $(publish.serviceConnection.tenantId)
 
       cleanServiceConnection:
         name: $(clean.serviceConnectionName)
@@ -94,14 +103,3 @@ stages:
         id: $(test.serviceConnection.id)
         clientId: $(test.serviceConnection.clientId)
         tenantId: $(test.serviceConnection.tenantId)
-
-      PublishRegistry:
-        server: $(acr.server)
-        resourceGroup: $(acr.resourceGroup)
-        subscription: $(acr.subscription)
-        repoPrefix: "${{ parameters.publishRepoPrefix }}"
-        serviceConnection:
-          name: $(publish.serviceConnectionName)
-          id: $(publish.serviceConnection.id)
-          clientId: $(publish.serviceConnection.clientId)
-          tenantId: $(publish.serviceConnection.tenantId)

eng/docker-tools/templates/stages/setup-service-connections.yml

@@ -3,21 +3,37 @@
 # it is declared in this stage's parameters, even if your pipeline has already
 # been granted access to the service connection. This stage also does not need
 # to complete before the service connection is used.
+#
+# There are two ways to specify service connections:
+# - Pass `serviceConnections` directly (list of {name: string} objects)
+# - Pass `publishConfig` + `registries` to look up auth from RegistryAuthentication
 parameters:
 - name: pool
   type: object
   default:
     name: $(default1ESInternalPoolName)
     image: $(default1ESInternalPoolImage)
     os: linux
-# serviceConnections object shape:
-# - name: string
+
+# Explicit list of service connections to initialize
+# Shape: [{ name: string }]
 - name: serviceConnections
   type: object
   default: []
 
-stages:
+# List of registry servers that need authentication. These will be looked up in
+# publishConfig.RegistryAuthentication.
+# Make sure to provide the publishConfig parameter.
+- name: usesRegistries
+  type: object
+  default: []
+# Look up service connections from publishConfig based on registries
+# The publish configuration containing RegistryAuthentication entries.
+- name: publishConfig
+  type: object
+  default: {}
 
+stages:
 - stage: SetupServiceConnectionsStage
   displayName: Setup service connections
   jobs:
@@ -27,6 +43,8 @@ stages:
     pool: ${{ parameters.pool }}
     steps:
     - checkout: none
+
+    # Direct service connections list
     - ${{ each serviceConnection in parameters.serviceConnections }}:
       - task: AzureCLI@2
         displayName: Setup ${{ serviceConnection.name }}
@@ -36,3 +54,15 @@ stages:
           scriptLocation: inlineScript
           inlineScript: |
             az account show
+
+    # Setup registry service connections
+    - ${{ if gt(length(parameters.usesRegistries), 0) }}:
+      - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}:
+        - ${{ if containsValue(parameters.usesRegistries, auth.server) }}:
+          - task: AzureCLI@2
+            displayName: Setup ${{ auth.serviceConnection.name }}
+            inputs:
+              azureSubscription: ${{ auth.serviceConnection.name }}
+              scriptType: pscore
+              scriptLocation: inlineScript
+              inlineScript: az account show

eng/docker-tools/templates/steps/copy-base-images.yml

@@ -3,8 +3,6 @@ parameters:
   type: object
   default:
     server: ""
-    subscription: ""
-    resourceGroup: ""
     repoPrefix: ""
 - name: additionalOptions
   type: string
@@ -29,8 +27,6 @@ steps:
     # error
     args: >-
       copyBaseImages
-      '${{ parameters.acr.subscription }}'
-      '${{ parameters.acr.resourceGroup }}'
       $(dockerHubRegistryCreds)
       $(customCopyBaseImagesArgs)
       --repo-prefix '${{ parameters.acr.repoPrefix }}'

eng/docker-tools/templates/variables/docker-images.yml

@@ -1,5 +1,5 @@
 variables:
-  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2887966
+  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2908146
   imageNames.imageBuilder: $(imageNames.imageBuilderName)
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
   imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner