(tree) Added new forest data and summary formats for incremental summaries (#26002)

Added new formats for the forest summarizer's data and summaries -
`ForestFormatVersion.v2` and `ForestSummaryFormatVersion.v3`
respectively. They both require `minVersionForCollab` to be 2.74.0.
- `ForestFormatVersion.v2` - Incremental encoding of the forest's data
is supported from this version onwards.
- `ForestSummaryFormatVersion.v3` - Incremental summary in forest is
supported from this version onwards. Also, the top-level forest content
is stored under a summary blob with key "contents". This is the same key
where incremental chunk contents are stored making the summary tree
consistent.

Incremental summaries will require these two versions to be enabled,
i.e., it is supported from `minVersionForCollab` 2.74.0 onwards.
Enabling incremental summaries is a cross-client breaking change, so
this will ensure that clients can read these new formats when it is
enabled.
Added validation during forest summarizer load that the top-level
content is present in the correct blob key as per the summary format
version.


[AB#41865](https://dev.azure.com/fluidframework/235294da-091d-4c29-84fc-cdfc3d90890b/_workitems/edit/41865)

Author: agarwal-navin

packages/dds/tree/src/codec/codec.ts

@@ -465,8 +465,14 @@ export function withSchemaValidation<
  * If the need arises, they might be added in the future.
  *
  * @privateRemarks
- * Entries in these enums should document the user facing impact of opting into a particular version.
- * For example, document if there is an encoding efficiency improvement of oping into that version or newer.
+ * The entries in these enums should document the following:
+ * - The user facing impact of opting into a particular version. This will help customers decide if they want to opt into
+ * a new version. For example, document if there is an encoding efficiency improvement of oping into that version or newer.
+ * - Any new data formats that are introduced in that version. This will help developers tell which data formats a given
+ * version will write. For example, document if a new summary or encoding format is added in a version.
+ * - Whether the above features or data formats introduced in a version are enabled by default or require the
+ * {@link minVersionForCollab} option to be set to that particular version.
+ *
  * Versions with no notable impact can be omitted.
  *
  * This scheme assumes a single version will always be enough to communicate compatibility.
@@ -505,38 +511,40 @@ export const FluidClientVersion = {
 	 * Fluid Framework Client 2.43 and newer.
 	 * @remarks
 	 * New formats introduced in 2.43:
-	 * - SchemaFormatVersion.v2
-	 * - MessageFormatVersion.v4
-	 * - EditManagerFormatVersion.v4
-	 * - Sequence format version 3
+	 * - SchemaFormatVersion.v2 - written when minVersionForCollab \>= 2.43
+	 * - MessageFormatVersion.v4 - written when minVersionForCollab \>= 2.43
+	 * - EditManagerFormatVersion.v4 - written when minVersionForCollab \>= 2.43
+	 * - sequence-field/formatV3 - written when minVersionForCollab \>= 2.43
 	 */
 	v2_43: "2.43.0",
 
 	/**
 	 * Fluid Framework Client 2.52 and newer.
 	 * @remarks
 	 * New formats introduced in 2.52:
-	 * - DetachedFieldIndexFormatVersion.v2
+	 * - DetachedFieldIndexFormatVersion.v2 - written when minVersionForCollab \>= 2.52
 	 */
 	v2_52: "2.52.0",
 
 	/**
 	 * Fluid Framework Client 2.73 and newer.
 	 * @remarks
 	 * New formats introduced in 2.73:
-	 * - FieldBatchFormatVersion v2
+	 * - FieldBatchFormatVersion.v2 - written when minVersionForCollab \>= 2.73
 	 */
 	v2_73: "2.73.0",
 
 	/**
 	 * Fluid Framework Client 2.74 and newer.
 	 * @remarks
 	 * New formats introduced in 2.74:
-	 * - SharedTreeSummaryFormatVersion v2
-	 * - DetachedFieldIndexSummaryFormatVersion v2
-	 * - SchemaSummaryFormatVersion v2
-	 * - EditManagerSummaryFormatVersion v2
-	 * - ForestSummaryFormatVersion v2
+	 * - SharedTreeSummaryFormatVersion.v2 - written by default
+	 * - DetachedFieldIndexSummaryFormatVersion.v2 - written by default
+	 * - SchemaSummaryFormatVersion.v2 - written by default
+	 * - EditManagerSummaryFormatVersion.v2 - written by default
+	 * - ForestSummaryFormatVersion.v2 - written by default
+	 * - ForestFormatVersion.v2 - written when minVersionForCollab \>= 2.74
+	 * - ForestSummaryFormatVersion.v3 - written when minVersionForCollab \>= 2.74
 	 */
 	v2_74: "2.74.0",
 } as const satisfies Record<string, MinimumVersionForCollab>;

packages/dds/tree/src/feature-libraries/forest-summary/codec.ts

@@ -5,48 +5,60 @@
 
 import { assert, oob } from "@fluidframework/core-utils/internal";
 import type { MinimumVersionForCollab } from "@fluidframework/runtime-definitions/internal";
+import {
+	getConfigForMinVersionForCollab,
+	lowestMinVersionForCollab,
+} from "@fluidframework/runtime-utils/internal";
 
 import {
 	type CodecTree,
 	type CodecWriteOptions,
+	FluidClientVersion,
 	type IJsonCodec,
 	makeVersionedValidatedCodec,
 } from "../../codec/index.js";
 import type { FieldKey, ITreeCursorSynchronous } from "../../core/index.js";
 import type { FieldBatchCodec, FieldBatchEncodingContext } from "../chunked-forest/index.js";
 
-import { FormatV1, ForestFormatVersion } from "./format.js";
+import {
+	ForestFormatVersion,
+	validVersions,
+	type Format,
+	FormatCommon,
+} from "./formatCommon.js";
 import { brand } from "../../util/index.js";
 
 /**
  * Uses field cursors
  */
 export type FieldSet = ReadonlyMap<FieldKey, ITreeCursorSynchronous>;
-export type ForestCodec = IJsonCodec<FieldSet, FormatV1, FormatV1, FieldBatchEncodingContext>;
+export type ForestCodec = IJsonCodec<FieldSet, Format, Format, FieldBatchEncodingContext>;
 
 /**
  * Convert a MinimumVersionForCollab to a ForestFormatVersion.
  * @param clientVersion - The MinimumVersionForCollab to convert.
  * @returns The ForestFormatVersion that corresponds to the provided MinimumVersionForCollab.
  */
-function clientVersionToForestSummaryFormatVersion(
+export function clientVersionToForestFormatVersion(
 	clientVersion: MinimumVersionForCollab,
 ): ForestFormatVersion {
-	// Currently, forest summary codec only writes in version 1.
-	return brand(ForestFormatVersion.v1);
+	return brand(
+		getConfigForMinVersionForCollab(clientVersion, {
+			[lowestMinVersionForCollab]: ForestFormatVersion.v1,
+			[FluidClientVersion.v2_74]: ForestFormatVersion.v2,
+		}),
+	);
 }
 
 export function makeForestSummarizerCodec(
 	options: CodecWriteOptions,
 	fieldBatchCodec: FieldBatchCodec,
 ): ForestCodec {
 	const inner = fieldBatchCodec;
-	// TODO: AB#41865
-	// This needs to be updated to support multiple versions.
-	// The second version will be used to enable incremental summarization.
-	const writeVersion = clientVersionToForestSummaryFormatVersion(options.minVersionForCollab);
-	return makeVersionedValidatedCodec(options, new Set([ForestFormatVersion.v1]), FormatV1, {
-		encode: (data: FieldSet, context: FieldBatchEncodingContext): FormatV1 => {
+	const writeVersion = clientVersionToForestFormatVersion(options.minVersionForCollab);
+	const formatSchema = FormatCommon(writeVersion);
+	return makeVersionedValidatedCodec(options, validVersions, formatSchema, {
+		encode: (data: FieldSet, context: FieldBatchEncodingContext): Format => {
 			const keys: FieldKey[] = [];
 			const fields: ITreeCursorSynchronous[] = [];
 			for (const [key, value] of data) {
@@ -55,7 +67,7 @@ export function makeForestSummarizerCodec(
 			}
 			return { keys, fields: inner.encode(fields, context), version: writeVersion };
 		},
-		decode: (data: FormatV1, context: FieldBatchEncodingContext): FieldSet => {
+		decode: (data: Format, context: FieldBatchEncodingContext): FieldSet => {
 			const out: Map<FieldKey, ITreeCursorSynchronous> = new Map();
 			const fields = inner.decode(data.fields, context);
 			assert(data.keys.length === fields.length, 0x891 /* mismatched lengths */);
@@ -70,5 +82,5 @@ export function makeForestSummarizerCodec(
 export function getCodecTreeForForestFormat(
 	clientVersion: MinimumVersionForCollab,
 ): CodecTree {
-	return { name: "Forest", version: clientVersionToForestSummaryFormatVersion(clientVersion) };
+	return { name: "Forest", version: clientVersionToForestFormatVersion(clientVersion) };
 }

packages/dds/tree/src/feature-libraries/forest-summary/forestSummarizer.ts

@@ -42,19 +42,26 @@ import {
 	type IncrementalEncodingPolicy,
 } from "../chunked-forest/index.js";
 
-import { type ForestCodec, makeForestSummarizerCodec } from "./codec.js";
+import {
+	clientVersionToForestFormatVersion,
+	type ForestCodec,
+	makeForestSummarizerCodec,
+} from "./codec.js";
 import {
 	ForestIncrementalSummaryBehavior,
 	ForestIncrementalSummaryBuilder,
 } from "./incrementalSummaryBuilder.js";
 import {
-	forestSummaryContentKey,
-	forestSummaryKey,
 	minVersionToForestSummaryFormatVersion,
-	supportedForestSummaryFormatVersions,
-	type ForestSummaryFormatVersion,
+	getForestRootSummaryContentKey,
 } from "./summaryTypes.js";
 import { TreeCompressionStrategy } from "../treeCompressionUtils.js";
+import { ForestFormatVersion } from "./formatCommon.js";
+import {
+	ForestSummaryFormatVersion,
+	forestSummaryKey,
+	supportedForestSummaryFormatVersions,
+} from "./summaryFormatCommon.js";
 
 /**
  * Provides methods for summarizing and loading a forest.
@@ -66,6 +73,7 @@ export class ForestSummarizer
 	private readonly codec: ForestCodec;
 
 	private readonly incrementalSummaryBuilder: ForestIncrementalSummaryBuilder;
+	private readonly forestRootSummaryContentKey: string;
 
 	/**
 	 * @param encoderContext - The schema if provided here must be mutated by the caller to keep it up to date.
@@ -87,11 +95,25 @@ export class ForestSummarizer
 			true /* supportPreVersioningFormat */,
 		);
 
-		// TODO: this should take in CodecWriteOptions, and use it to pick the write version.
 		this.codec = makeForestSummarizerCodec(options, fieldBatchCodec);
+
+		const forestFormatWriteVersion = clientVersionToForestFormatVersion(
+			options.minVersionForCollab,
+		);
+		const summaryFormatWriteVersion = minVersionToForestSummaryFormatVersion(
+			options.minVersionForCollab,
+		);
+		this.forestRootSummaryContentKey = getForestRootSummaryContentKey(
+			summaryFormatWriteVersion,
+		);
+
+		// Incremental summary is supported from ForestFormatVersion.v2 and ForestSummaryFormatVersion.v3 onwards.
+		const enableIncrementalSummary =
+			forestFormatWriteVersion >= ForestFormatVersion.v2 &&
+			summaryFormatWriteVersion >= ForestSummaryFormatVersion.v3 &&
+			encoderContext.encodeType === TreeCompressionStrategy.CompressedIncremental;
 		this.incrementalSummaryBuilder = new ForestIncrementalSummaryBuilder(
-			encoderContext.encodeType ===
-				TreeCompressionStrategy.CompressedIncremental /* enableIncrementalSummary */,
+			enableIncrementalSummary,
 			(cursor: ITreeCursorSynchronous) => this.forest.chunkField(cursor),
 			shouldEncodeIncrementally,
 			initialSequenceNumber,
@@ -153,23 +175,26 @@ export class ForestSummarizer
 
 		this.incrementalSummaryBuilder.completeSummary({
 			incrementalSummaryContext,
-			forestSummaryContent: stringify(encoded),
+			forestSummaryRootContent: stringify(encoded),
+			forestSummaryRootContentKey: this.forestRootSummaryContentKey,
 			builder,
 		});
 	}
 
 	protected async loadInternal(
 		services: IChannelStorageService,
 		parse: SummaryElementParser,
+		version: ForestSummaryFormatVersion | undefined,
 	): Promise<void> {
-		// The contents of the top-level forest must be present under a summary blob named `forestSummaryContentKey`.
+		// Get the key of the summary blob where the top-level forest content is stored based on the summary format version.
 		// If the summary was generated as `ForestIncrementalSummaryBehavior.SingleBlob`, this blob will contain all
 		// of forest's contents.
 		// If the summary was generated as `ForestIncrementalSummaryBehavior.Incremental`, this blob will contain only
 		// the top-level forest node's contents.
 		// The contents of the incremental chunks will be in separate tree nodes and will be read later during decoding.
+		const forestSummaryRootContentKey = getForestRootSummaryContentKey(version);
 		assert(
-			await services.contains(forestSummaryContentKey),
+			await services.contains(forestSummaryRootContentKey),
 			0xc21 /* Forest summary content missing in snapshot */,
 		);
 
@@ -184,7 +209,7 @@ export class ForestSummarizer
 		// TODO: this code is parsing data without an optional validator, this should be defined in a typebox schema as part of the
 		// forest summary format.
 		const fields = this.codec.decode(
-			await readAndParseSnapshotBlob(forestSummaryContentKey, services, parse),
+			await readAndParseSnapshotBlob(forestSummaryRootContentKey, services, parse),
 			{
 				...this.encoderContext,
 				incrementalEncoderDecoder: this.incrementalSummaryBuilder,

…ature-libraries/forest-summary/format.ts …libraries/forest-summary/formatCommon.tspackages/dds/tree/src/feature-libraries/forest-summary/format.ts renamed to packages/dds/tree/src/feature-libraries/forest-summary/formatCommon.ts packages/dds/tree/src/feature-libraries/forest-summary/format.ts renamed to packages/dds/tree/src/feature-libraries/forest-summary/formatCommon.ts

@@ -6,21 +6,25 @@
 import { type Static, Type } from "@sinclair/typebox";
 
 import { schemaFormatV1 } from "../../core/index.js";
-import { brand, type Brand } from "../../util/index.js";
+import type { Brand } from "../../util/index.js";
 import { EncodedFieldBatch } from "../chunked-forest/index.js";
 
 /**
  * The format version for the forest.
  */
 export const ForestFormatVersion = {
 	v1: 1,
+	/** This format supports incremental encoding */
+	v2: 2,
 } as const;
 export type ForestFormatVersion = Brand<
 	(typeof ForestFormatVersion)[keyof typeof ForestFormatVersion],
 	"ForestFormatVersion"
 >;
 
-const FormatGeneric = (
+export const validVersions = new Set([...Object.values(ForestFormatVersion)]);
+
+export const FormatCommon = (
 	version: ForestFormatVersion,
 	// Return type is intentionally derived.
 	// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
@@ -33,6 +37,4 @@ const FormatGeneric = (
 		},
 		{ additionalProperties: false },
 	);
-
-export const FormatV1 = FormatGeneric(brand<ForestFormatVersion>(ForestFormatVersion.v1));
-export type FormatV1 = Static<typeof FormatV1>;
+export type Format = Static<ReturnType<typeof FormatCommon>>;

packages/dds/tree/src/feature-libraries/forest-summary/formatV1.ts

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { Static } from "@sinclair/typebox";
+
+import { brand } from "../../util/index.js";
+import { FormatCommon, ForestFormatVersion } from "./formatCommon.js";
+
+export const FormatV1 = FormatCommon(brand<ForestFormatVersion>(ForestFormatVersion.v1));
+export type FormatV1 = Static<typeof FormatV1>;

packages/dds/tree/src/feature-libraries/forest-summary/formatV2.ts

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { Static } from "@sinclair/typebox";
+
+import { brand } from "../../util/index.js";
+import { FormatCommon, ForestFormatVersion } from "./formatCommon.js";
+
+export const FormatV2 = FormatCommon(brand<ForestFormatVersion>(ForestFormatVersion.v2));
+export type FormatV2 = Static<typeof FormatV2>;