From f3289f4a68d09d38257fed8da7b3b62130753397 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 23 May 2025 13:25:00 +0200 Subject: [PATCH 1/2] feat(cli): metadata can live in a separate file When CDK apps grow extremely large (think 10-100 stacks, 1000-10000 constructs), all metadata together begins to exceed 512MB, the maximum string size in NodeJS. People usually deal with this by disabling metadata, but they shouldn't have to. In addition, even for manifests that don't exceed 512MB the extremely large size of the single JSON object slows down its writing and reading every time, even if the metadata doesn't need to be accessed. An effective solution is to write the metadata of an artifact to a separate file. This PR introduces the ability for that into the Cloud Assembly schema, and updates the CLI to read from both sources if available. Relates to https://github.com/aws/aws-cdk/pull/34480. --- .../lib/cloud-assembly/schema.ts | 14 +++++++++++++ .../toolkit-lib/lib/api/utilities/index.ts | 1 + .../lib/api/utilities/manifest-reading.ts | 21 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 packages/@aws-cdk/toolkit-lib/lib/api/utilities/index.ts create mode 100644 packages/@aws-cdk/toolkit-lib/lib/api/utilities/manifest-reading.ts diff --git a/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts b/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts index e593328e9..d02b39e5e 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts +++ b/packages/@aws-cdk/cloud-assembly-schema/lib/cloud-assembly/schema.ts @@ -81,10 +81,24 @@ export interface ArtifactManifest { /** * Associated metadata. * + * Metadata can be stored directly in the assembly manifest, as well as in a + * separate file (see `additionalMetadataFile`). It should prefer to be stored + * in the additional file, as that will reduce the size of the assembly + * manifest in cases of a lot of metdata (which CDK does emit by default). + * * @default - no metadata. */ readonly metadata?: { [path: string]: MetadataEntry[] }; + /** + * A file with additional metadata entries. + * + * The schema of this file is exactly the same as the type of the `metadata` field. + * + * @default - no additional metadata + */ + readonly additionalMetadataFile?: string; + /** * IDs of artifacts that must be deployed before this artifact. * diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/utilities/index.ts b/packages/@aws-cdk/toolkit-lib/lib/api/utilities/index.ts new file mode 100644 index 000000000..77fb751a2 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/lib/api/utilities/index.ts @@ -0,0 +1 @@ +export * from './manifest-reading'; diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/utilities/manifest-reading.ts b/packages/@aws-cdk/toolkit-lib/lib/api/utilities/manifest-reading.ts new file mode 100644 index 000000000..6adbcfa26 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/lib/api/utilities/manifest-reading.ts @@ -0,0 +1,21 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import type { ArtifactManifest, MetadataEntry } from '@aws-cdk/cloud-assembly-schema'; + +/** + * Read the metadata for the given artifact + * + * You must use this instead of accessing `ArtifactManifest.metadata` + * directly; this can also deal with the case of where the metadata + * has been written to a file. + */ +export function readArtifactMetadata(assemblyDirectory: string, x: ArtifactManifest): Record { + const ret = {}; + if (x.additionalMetadataFile) { + Object.assign(ret, JSON.parse(fs.readFileSync(path.join(assemblyDirectory, x.additionalMetadataFile), 'utf-8'))); + } + // FIXME: Conflicting paths + // FIXME: Rewrite stack tags + Object.assign(ret, x.metadata); + return ret; +} From 1820699e9c232c0d8494448660d9029f79fcbc11 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 3 Jun 2025 13:27:07 +0000 Subject: [PATCH 2/2] chore: self mutation Signed-off-by: github-actions --- .../cloud-assembly-schema/schema/cloud-assembly.schema.json | 6 +++++- packages/@aws-cdk/cloud-assembly-schema/schema/version.json | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json b/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json index 855b0ba0b..ce3360b1c 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json +++ b/packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.schema.json @@ -49,7 +49,7 @@ "type": "string" }, "metadata": { - "description": "Associated metadata. (Default - no metadata.)", + "description": "Associated metadata.\n\nMetadata can be stored directly in the assembly manifest, as well as in a\nseparate file (see `additionalMetadataFile`). It should prefer to be stored\nin the additional file, as that will reduce the size of the assembly\nmanifest in cases of a lot of metdata (which CDK does emit by default). (Default - no metadata.)", "type": "object", "additionalProperties": { "type": "array", @@ -58,6 +58,10 @@ } } }, + "additionalMetadataFile": { + "description": "A file with additional metadata entries.\n\nThe schema of this file is exactly the same as the type of the `metadata` field. (Default - no additional metadata)", + "type": "string" + }, "dependencies": { "description": "IDs of artifacts that must be deployed before this artifact. (Default - no dependencies.)", "type": "array", diff --git a/packages/@aws-cdk/cloud-assembly-schema/schema/version.json b/packages/@aws-cdk/cloud-assembly-schema/schema/version.json index 8caae8e30..0d6f8054c 100644 --- a/packages/@aws-cdk/cloud-assembly-schema/schema/version.json +++ b/packages/@aws-cdk/cloud-assembly-schema/schema/version.json @@ -1,4 +1,4 @@ { - "schemaHash": "96958a4c40e0a00e850f0c14dd6e9c0fc8db0b075f1f155cea41ab198c0514be", - "revision": 43 + "schemaHash": "e03b716ea6f59e089369ce279c03a194917456d466994d434e5f9f366b745f12", + "revision": 44 } \ No newline at end of file