From 6d2eff1379281c84139d49e2cbb0ac31e619ace6 Mon Sep 17 00:00:00 2001 From: "Craig Macomber (Microsoft)" <42876482+CraigMacomber@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:19:21 -0800 Subject: [PATCH 01/20] tree2: Make chunked forest stop depending on default-schema (#18681) ## Description Make chunked forest stop depending on default-schema: it should work with any schema. --- .../chunked-forest/codec/compressedEncode.ts | 3 +++ .../chunked-forest/codec/schemaBasedEncoding.ts | 13 +++---------- .../codec/compressedEncode.spec.ts | 16 ++++++++++++++++ .../chunked-forest/codec/nodeShape.spec.ts | 6 ++++++ .../codec/schemaBasedEncoding.spec.ts | 8 ++++++++ 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/compressedEncode.ts b/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/compressedEncode.ts index 76401bfae0a7..0174f5df2b60 100644 --- a/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/compressedEncode.ts +++ b/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/compressedEncode.ts @@ -12,8 +12,10 @@ import { TreeNodeSchemaIdentifier, Value, forEachNode, + FieldKindIdentifier, } from "../../../core"; import { fail, getOrCreate } from "../../../util"; +import { type FieldKind } from "../../modular-schema"; import { BufferFormat as BufferFormatGeneric, Shape as ShapeGeneric, @@ -417,6 +419,7 @@ export class EncoderCache implements TreeShaper, FieldShaper { public constructor( private readonly treeEncoder: TreeShapePolicy, private readonly fieldEncoder: FieldShapePolicy, + public readonly fieldShapes: ReadonlyMap, ) {} public shapeFromTree(schemaName: TreeNodeSchemaIdentifier): NodeEncoder { diff --git a/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts b/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts index d5876a1897f9..bbf2901951ad 100644 --- a/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts +++ b/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts @@ -11,9 +11,8 @@ import { TreeNodeSchemaIdentifier, ValueSchema, } from "../../../core"; -import { FieldKind, FullSchemaPolicy, Multiplicity } from "../../modular-schema"; +import { FullSchemaPolicy, Multiplicity } from "../../modular-schema"; import { fail } from "../../../util"; -import { fieldKinds } from "../../default-schema"; import { EncodedChunk, EncodedValueShape } from "./format"; import { EncoderCache, @@ -47,17 +46,11 @@ export function buildCache(schema: StoredSchemaCollection, policy: FullSchemaPol treeShaper(schema, policy, fieldHandler, schemaName), (treeHandler: TreeShaper, field: TreeFieldStoredSchema) => fieldShaper(treeHandler, field, cache), + policy.fieldKinds, ); return cache; } -export function getFieldKind(fieldSchema: TreeFieldStoredSchema): FieldKind { - // TODO: - // This module currently is assuming use of defaultFieldKinds. - // The field kinds should instead come from a view schema registry thats provided somewhere. - return fieldKinds.get(fieldSchema.kind.identifier) ?? fail("missing field kind"); -} - /** * Selects shapes to use to encode fields. */ @@ -66,7 +59,7 @@ export function fieldShaper( field: TreeFieldStoredSchema, cache: EncoderCache, ): FieldEncoder { - const kind = getFieldKind(field); + const kind = cache.fieldShapes.get(field.kind.identifier) ?? fail("missing FieldKind"); const type = oneFromSet(field.types); const nodeEncoder = type !== undefined ? treeHandler.shapeFromTree(type) : anyNodeEncoder; // eslint-disable-next-line unicorn/prefer-ternary diff --git a/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/compressedEncode.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/compressedEncode.spec.ts index 7ab717241d44..c5eac0f39b9d 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/compressedEncode.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/compressedEncode.spec.ts @@ -51,6 +51,8 @@ import { import { brand } from "../../../../util"; import { typeboxValidator } from "../../../../external-utilities"; import { cursorForJsonableTreeField } from "../../../../feature-libraries"; +// eslint-disable-next-line import/no-internal-modules +import { fieldKinds } from "../../../../feature-libraries/default-schema"; import { checkFieldEncode, checkNodeEncode } from "./checkEncode"; const anyNodeShape = new NodeShape(undefined, undefined, [], anyFieldEncoder); @@ -70,6 +72,7 @@ describe("compressedEncode", () => { anyNodeShape, (treeShaper: TreeShaper, field: TreeFieldStoredSchema): FieldEncoder => anyFieldEncoder, + fieldKinds, ); const codec = makeCompressedCodec({ jsonValidator: typeboxValidator }, cache); const result = codec.encode(input); @@ -109,6 +112,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => anyNodeShape, () => fail(), + fieldKinds, ); const buffer = checkNodeEncode(anyNodeEncoder, cache, { type: brand("foo") }); assert.deepEqual(buffer, [anyNodeShape, new IdentifierToken("foo"), false, []]); @@ -119,6 +123,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(InlineArrayShape.empty, cache, []); assert(compareArrays(buffer, [])); @@ -128,6 +133,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const shape = new InlineArrayShape(1, asNodesEncoder(onlyTypeShape)); const buffer = checkFieldEncode(shape, cache, [{ type: brand("foo") }]); @@ -138,6 +144,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const shape = new InlineArrayShape(2, asNodesEncoder(onlyTypeShape)); const buffer = checkFieldEncode(shape, cache, [ @@ -151,6 +158,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const shapeInner = new InlineArrayShape(2, asNodesEncoder(onlyTypeShape)); const shapeOuter = new InlineArrayShape(2, shapeInner); @@ -174,6 +182,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(new NestedArrayShape(onlyTypeShape), cache, []); assert.deepEqual(buffer, [0]); @@ -183,6 +192,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const shape = new NestedArrayShape(onlyTypeShape); const buffer = checkFieldEncode(shape, cache, [{ type: brand("foo") }]); @@ -193,6 +203,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const shape = new NestedArrayShape(onlyTypeShape); const buffer = checkFieldEncode(shape, cache, [ @@ -207,6 +218,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(new NestedArrayShape(constantFooShape), cache, [ { type: brand("foo") }, @@ -218,6 +230,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(new NestedArrayShape(constantFooShape), cache, [ { type: brand("foo") }, @@ -233,6 +246,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(anyFieldEncoder, cache, []); // For size purposes, this should remain true @@ -250,6 +264,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => onlyTypeShape, () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(anyFieldEncoder, cache, [{ type: brand("foo") }]); // Should use anyNodeEncoder, which will lookup the shape from cache: @@ -260,6 +275,7 @@ describe("compressedEncode", () => { const cache = new EncoderCache( () => onlyTypeShape, () => fail(), + fieldKinds, ); const buffer = checkFieldEncode(anyFieldEncoder, cache, [ { type: brand("foo") }, diff --git a/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/nodeShape.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/nodeShape.spec.ts index 28611bbcea72..83c176ffeb02 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/nodeShape.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/nodeShape.spec.ts @@ -22,6 +22,8 @@ import { } from "../../../../feature-libraries/chunked-forest/codec/chunkEncodingGeneric"; import { JsonableTree } from "../../../../core"; import { brand } from "../../../../util"; +// eslint-disable-next-line import/no-internal-modules +import { fieldKinds } from "../../../../feature-libraries/default-schema"; import { checkNodeEncode } from "./checkEncode"; describe("nodeShape", () => { @@ -35,6 +37,7 @@ describe("nodeShape", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const buffer = checkNodeEncode(shape, cache, { @@ -51,6 +54,7 @@ describe("nodeShape", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const encodedChunk = checkNodeEncode(shape, cache, { @@ -64,6 +68,7 @@ describe("nodeShape", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const fieldShapeLocal = cache.nestedArray( @@ -98,6 +103,7 @@ describe("nodeShape", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); // Shape which encodes to nothing. diff --git a/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncoding.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncoding.spec.ts index 5bed5e4b3a3c..4a438a6a972d 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncoding.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncoding.spec.ts @@ -42,6 +42,8 @@ import { } from "../../../testTrees"; import { typeboxValidator } from "../../../../external-utilities"; import { leaf, SchemaBuilder } from "../../../../domains"; +// eslint-disable-next-line import/no-internal-modules +import { fieldKinds } from "../../../../feature-libraries/default-schema"; import { checkFieldEncode, checkNodeEncode } from "./checkEncode"; const anyNodeShape = new NodeShape(undefined, undefined, [], anyFieldEncoder); @@ -60,6 +62,7 @@ describe("schemaBasedEncoding", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const log: string[] = []; const shape = fieldShaper( @@ -86,6 +89,7 @@ describe("schemaBasedEncoding", () => { const cache = new EncoderCache( () => anyNodeShape, () => fail(), + fieldKinds, ); const log: string[] = []; const shape = fieldShaper( @@ -108,6 +112,7 @@ describe("schemaBasedEncoding", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const log: string[] = []; const shape = fieldShaper( @@ -138,6 +143,7 @@ describe("schemaBasedEncoding", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const shape = treeShaper( library, @@ -153,6 +159,7 @@ describe("schemaBasedEncoding", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const log: TreeFieldStoredSchema[] = []; const shape = treeShaper( @@ -188,6 +195,7 @@ describe("schemaBasedEncoding", () => { const cache = new EncoderCache( () => fail(), () => fail(), + fieldKinds, ); const log: TreeFieldStoredSchema[] = []; const shape = treeShaper( From 898760b7c62c6bc155cbd82894e5ab052c5ef92c Mon Sep 17 00:00:00 2001 From: daesunp <138815173+daesunp@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:00:22 -0800 Subject: [PATCH 02/20] move multiplicity to out of modular-schema (#18680) ## Description This PR moves the `Multiplicity` enum out from `modular-schema` to the parent directory `feature-libraries`, to make chunkTree stop depending on `Multiplicity`, to help mitigate circular dependency issues. --- .../chunked-forest/chunkTree.ts | 3 +- .../codec/schemaBasedEncoding.ts | 3 +- .../feature-libraries/contextuallyTyped.ts | 3 +- .../default-schema/defaultFieldKinds.ts | 2 +- .../dds/tree2/src/feature-libraries/index.ts | 3 +- .../modular-schema/comparison.ts | 3 +- .../modular-schema/fieldKind.ts | 52 +----------------- .../modular-schema/genericFieldKind.ts | 3 +- .../feature-libraries/modular-schema/index.ts | 2 +- .../src/feature-libraries/multiplicity.ts | 54 +++++++++++++++++++ .../schema-aware/schemaAware.ts | 2 +- .../typed-schema/schemaCollection.ts | 2 +- .../dds/tree2/src/test/counterFieldKind.ts | 2 +- .../modular-schema/basicRebasers.ts | 2 +- 14 files changed, 73 insertions(+), 63 deletions(-) create mode 100644 experimental/dds/tree2/src/feature-libraries/multiplicity.ts diff --git a/experimental/dds/tree2/src/feature-libraries/chunked-forest/chunkTree.ts b/experimental/dds/tree2/src/feature-libraries/chunked-forest/chunkTree.ts index 67ddd61c62e6..d46d1ecb8905 100644 --- a/experimental/dds/tree2/src/feature-libraries/chunked-forest/chunkTree.ts +++ b/experimental/dds/tree2/src/feature-libraries/chunked-forest/chunkTree.ts @@ -17,8 +17,9 @@ import { TreeStoredSchema, StoredSchemaCollection, } from "../../core"; -import { FullSchemaPolicy, Multiplicity } from "../modular-schema"; +import { FullSchemaPolicy } from "../modular-schema"; import { fail } from "../../util"; +import { Multiplicity } from "../multiplicity"; import { TreeChunk, tryGetChunk } from "./chunk"; import { BasicChunk } from "./basicChunk"; import { FieldShape, TreeShape, UniformChunk } from "./uniformChunk"; diff --git a/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts b/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts index bbf2901951ad..e3313c23e940 100644 --- a/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts +++ b/experimental/dds/tree2/src/feature-libraries/chunked-forest/codec/schemaBasedEncoding.ts @@ -11,8 +11,9 @@ import { TreeNodeSchemaIdentifier, ValueSchema, } from "../../../core"; -import { FullSchemaPolicy, Multiplicity } from "../../modular-schema"; +import { FullSchemaPolicy } from "../../modular-schema"; import { fail } from "../../../util"; +import { Multiplicity } from "../../multiplicity"; import { EncodedChunk, EncodedValueShape } from "./format"; import { EncoderCache, diff --git a/experimental/dds/tree2/src/feature-libraries/contextuallyTyped.ts b/experimental/dds/tree2/src/feature-libraries/contextuallyTyped.ts index 2a3804f69313..0e595f651049 100644 --- a/experimental/dds/tree2/src/feature-libraries/contextuallyTyped.ts +++ b/experimental/dds/tree2/src/feature-libraries/contextuallyTyped.ts @@ -23,7 +23,7 @@ import { // This module currently is assuming use of default-field-kinds. // The field kinds should instead come from a view schema registry thats provided somewhere. import { fieldKinds } from "./default-schema"; -import { FieldKind, Multiplicity } from "./modular-schema"; +import { FieldKind } from "./modular-schema"; import { AllowedTypes, TreeFieldSchema, @@ -38,6 +38,7 @@ import { } from "./schema-aware"; import { isFluidHandle, allowsValue } from "./valueUtilities"; import { TreeDataContext } from "./fieldGenerator"; +import { Multiplicity } from "./multiplicity"; /** * This library defines a tree data format that can infer its types from context. diff --git a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts index 2e04d7b7347e..cb8999428a0d 100644 --- a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts +++ b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts @@ -14,7 +14,6 @@ import { import { fail } from "../../util"; import { FieldKind, - Multiplicity, allowsTreeSchemaIdentifierSuperset, ToDelta, FieldChangeHandler, @@ -29,6 +28,7 @@ import { optionalChangeHandler, optionalFieldEditor, } from "../optional-field"; +import { Multiplicity } from "../multiplicity"; /** * ChangeHandler that only handles no-op / identity changes. diff --git a/experimental/dds/tree2/src/feature-libraries/index.ts b/experimental/dds/tree2/src/feature-libraries/index.ts index a4499e4795b6..02f16b926c28 100644 --- a/experimental/dds/tree2/src/feature-libraries/index.ts +++ b/experimental/dds/tree2/src/feature-libraries/index.ts @@ -114,7 +114,6 @@ export { CrossFieldManager, CrossFieldTarget, FieldKind, - Multiplicity, FullSchemaPolicy, allowsRepoSuperset, GenericChangeset, @@ -126,6 +125,8 @@ export { RelevantRemovedRootsFromChild, } from "./modular-schema"; +export { Multiplicity } from "./multiplicity"; + export { TreeNodeSchema, AllowedTypes, diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/comparison.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/comparison.ts index bd97918df791..7c90e531f154 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/comparison.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/comparison.ts @@ -13,7 +13,8 @@ import { TreeStoredSchema, storedEmptyFieldSchema, } from "../../core"; -import { FullSchemaPolicy, Multiplicity, withEditor } from "./fieldKind"; +import { Multiplicity } from "../multiplicity"; +import { FullSchemaPolicy, withEditor } from "./fieldKind"; /** * @returns true iff `superset` is a superset of `original`. diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldKind.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldKind.ts index d2b59f9bf4d9..dc9ae363bee7 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldKind.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldKind.ts @@ -11,6 +11,7 @@ import { FieldKindSpecifier, TreeTypeSet, } from "../../core"; +import { Multiplicity } from "../multiplicity"; import { isNeverField } from "./comparison"; import { FieldChangeHandler, FieldEditor } from "./fieldChangeHandler"; @@ -146,54 +147,3 @@ export interface FullSchemaPolicy { */ readonly fieldKinds: ReadonlyMap; } - -/** - * Describes how a particular field functions. - * - * This determine its reading and editing APIs, multiplicity, and what merge resolution policies it will use. - * @alpha - */ -export enum Multiplicity { - /** - * Exactly one item. - */ - Single, - /** - * 0 or 1 items. - */ - Optional, - /** - * 0 or more items. - */ - Sequence, - /** - * Exactly 0 items. - * - * Using Forbidden makes what types are listed for allowed in a field irrelevant - * since the field will never have values in it. - * - * Using Forbidden is equivalent to picking a kind that permits empty (like sequence or optional) - * and having no allowed types (or only never types). - * Because of this, its possible to express everything constraint wise without Forbidden, - * but using Forbidden can be more semantically clear than optional with no allowed types. - * - * For view schema, this can be useful if you need to: - * - run a specific out of schema handler when a field is present, - * but otherwise are ignoring or tolerating (ex: via extra fields) unmentioned fields. - * - prevent a specific field from being used as an extra field - * (perhaps for some past of future compatibility reason) - * - keep a field in a schema for metadata purposes - * (ex: for improved error messaging, error handling or documentation) - * that is not used in this specific version of the schema (ex: to document what it was or will be used for). - * - * For stored schema, this can be useful if you need to: - * - have a field which can have its schema updated to Optional or Sequence of any type. - * - to exclude a field from extra fields - * - for the schema system to use as a default for fields which aren't declared - * (ex: when updating a field that did not exist into one that does) - * - * @privateRemarks - * See storedEmptyFieldSchema for a constant, reusable field using Forbidden. - */ - Forbidden, -} diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts index 051c9eaeba7b..b9acbc9f04a0 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts @@ -13,6 +13,7 @@ import { TaggedChange, } from "../../core"; import { fail, IdAllocator } from "../../util"; +import { Multiplicity } from "../multiplicity"; import { CrossFieldManager } from "./crossFieldQueries"; import { FieldChangeHandler, @@ -23,7 +24,7 @@ import { RelevantRemovedRootsFromChild, NodeChangePruner, } from "./fieldChangeHandler"; -import { FieldKindWithEditor, Multiplicity } from "./fieldKind"; +import { FieldKindWithEditor } from "./fieldKind"; import { makeGenericChangeCodec } from "./genericFieldKindCodecs"; import { GenericChange, GenericChangeset } from "./genericFieldKindTypes"; import { NodeChangeset } from "./modularChangeTypes"; diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts index b4a60bee6329..c66ee221b84e 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts @@ -20,7 +20,7 @@ export { setInCrossFieldMap, } from "./crossFieldQueries"; export { ChangesetLocalIdSchema, EncodedChangeAtomId } from "./modularChangeFormat"; -export { FieldKind, FullSchemaPolicy, Multiplicity, FieldKindWithEditor } from "./fieldKind"; +export { FieldKind, FullSchemaPolicy, FieldKindWithEditor } from "./fieldKind"; export { FieldChangeHandler, FieldChangeRebaser, diff --git a/experimental/dds/tree2/src/feature-libraries/multiplicity.ts b/experimental/dds/tree2/src/feature-libraries/multiplicity.ts new file mode 100644 index 000000000000..d91ba9804d72 --- /dev/null +++ b/experimental/dds/tree2/src/feature-libraries/multiplicity.ts @@ -0,0 +1,54 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ +/** + * Describes how a particular field functions. + * + * This determine its reading and editing APIs, multiplicity, and what merge resolution policies it will use. + * @alpha + */ +export enum Multiplicity { + /** + * Exactly one item. + */ + Single, + /** + * 0 or 1 items. + */ + Optional, + /** + * 0 or more items. + */ + Sequence, + /** + * Exactly 0 items. + * + * Using Forbidden makes what types are listed for allowed in a field irrelevant + * since the field will never have values in it. + * + * Using Forbidden is equivalent to picking a kind that permits empty (like sequence or optional) + * and having no allowed types (or only never types). + * Because of this, its possible to express everything constraint wise without Forbidden, + * but using Forbidden can be more semantically clear than optional with no allowed types. + * + * For view schema, this can be useful if you need to: + * - run a specific out of schema handler when a field is present, + * but otherwise are ignoring or tolerating (ex: via extra fields) unmentioned fields. + * - prevent a specific field from being used as an extra field + * (perhaps for some past of future compatibility reason) + * - keep a field in a schema for metadata purposes + * (ex: for improved error messaging, error handling or documentation) + * that is not used in this specific version of the schema (ex: to document what it was or will be used for). + * + * For stored schema, this can be useful if you need to: + * - have a field which can have its schema updated to Optional or Sequence of any type. + * - to exclude a field from extra fields + * - for the schema system to use as a default for fields which aren't declared + * (ex: when updating a field that did not exist into one that does) + * + * @privateRemarks + * See storedEmptyFieldSchema for a constant, reusable field using Forbidden. + */ + Forbidden, +} diff --git a/experimental/dds/tree2/src/feature-libraries/schema-aware/schemaAware.ts b/experimental/dds/tree2/src/feature-libraries/schema-aware/schemaAware.ts index 9d3b1f67b722..5c74cb5f6ea0 100644 --- a/experimental/dds/tree2/src/feature-libraries/schema-aware/schemaAware.ts +++ b/experimental/dds/tree2/src/feature-libraries/schema-aware/schemaAware.ts @@ -5,7 +5,6 @@ import { TreeNodeSchemaIdentifier, TreeValue, ValueSchema } from "../../core"; import { ContextuallyTypedNodeData, typeNameSymbol, valueSymbol } from "../contextuallyTyped"; -import { Multiplicity } from "../modular-schema"; import { TreeFieldSchema, TreeNodeSchema, @@ -19,6 +18,7 @@ import { LazyItem, } from "../typed-schema"; import { Assume, FlattenKeys, _InlineTrick } from "../../util"; +import { Multiplicity } from "../multiplicity"; /** * Empty Object for use in type computations that should contribute no fields when `&`ed with another type. diff --git a/experimental/dds/tree2/src/feature-libraries/typed-schema/schemaCollection.ts b/experimental/dds/tree2/src/feature-libraries/typed-schema/schemaCollection.ts index bf681f8e71c7..299796e33954 100644 --- a/experimental/dds/tree2/src/feature-libraries/typed-schema/schemaCollection.ts +++ b/experimental/dds/tree2/src/feature-libraries/typed-schema/schemaCollection.ts @@ -5,9 +5,9 @@ import { assert } from "@fluidframework/core-utils"; import { Adapters, TreeAdapter, TreeNodeSchemaIdentifier } from "../../core"; -import { Multiplicity } from "../modular-schema"; import { capitalize, fail, requireAssignableTo } from "../../util"; import { defaultSchemaPolicy, FieldKinds } from "../default-schema"; +import { Multiplicity } from "../multiplicity"; import { TreeFieldSchema, TreeNodeSchema, diff --git a/experimental/dds/tree2/src/test/counterFieldKind.ts b/experimental/dds/tree2/src/test/counterFieldKind.ts index 2339002036fa..1cd02f2f2e16 100644 --- a/experimental/dds/tree2/src/test/counterFieldKind.ts +++ b/experimental/dds/tree2/src/test/counterFieldKind.ts @@ -8,12 +8,12 @@ import { ICodecFamily, makeCodecFamily, makeValueCodec } from "../codec"; import { FieldChangeHandler, FieldChangeRebaser, + Multiplicity, cursorForJsonableTreeNode, } from "../feature-libraries"; // This is imported directly to implement an example of a field kind. import { FieldKindWithEditor, - Multiplicity, referenceFreeFieldChangeRebaser, // eslint-disable-next-line import/no-internal-modules } from "../feature-libraries/modular-schema"; diff --git a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/basicRebasers.ts b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/basicRebasers.ts index 9c37419c8696..51e039cdedd3 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/basicRebasers.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/basicRebasers.ts @@ -9,7 +9,6 @@ import { FieldChangeHandler, FieldChangeRebaser, FieldKindWithEditor, - Multiplicity, referenceFreeFieldChangeRebaser, // eslint-disable-next-line import/no-internal-modules } from "../../../feature-libraries/modular-schema"; @@ -17,6 +16,7 @@ import { Mutable, fail } from "../../../util"; import { makeCodecFamily, makeValueCodec } from "../../../codec"; import { singleJsonCursor } from "../../../domains"; import { DeltaFieldChanges, makeDetachedNodeId } from "../../../core"; +import { Multiplicity } from "../../../feature-libraries"; /** * Picks the last value written. From bd8a566ca5b97c6ecb15e9f0493d13cae89eeafc Mon Sep 17 00:00:00 2001 From: "Craig Macomber (Microsoft)" <42876482+CraigMacomber@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:31:05 -0800 Subject: [PATCH 03/20] tree2: Better ambiguous input errors (#18683) ## Description When an insertable tree is ambiguous, provide a better error message. --- experimental/dds/tree2/package.json | 2 +- .../dds/tree2/src/simple-tree/toMapTree.ts | 23 +++++++++++++++++-- .../src/test/simple-tree/toMapTree.spec.ts | 13 +++++++++++ pnpm-lock.yaml | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/experimental/dds/tree2/package.json b/experimental/dds/tree2/package.json index 36f235f9fc53..d43fba8c3608 100644 --- a/experimental/dds/tree2/package.json +++ b/experimental/dds/tree2/package.json @@ -78,6 +78,7 @@ "@fluidframework/runtime-definitions": "workspace:~", "@fluidframework/runtime-utils": "workspace:~", "@fluidframework/shared-object-base": "workspace:~", + "@fluidframework/telemetry-utils": "workspace:~", "@sinclair/typebox": "^0.29.4", "@ungap/structured-clone": "^1.2.0", "sorted-btree": "^1.8.0", @@ -95,7 +96,6 @@ "@fluidframework/container-loader": "workspace:~", "@fluidframework/eslint-config-fluid": "^3.1.0", "@fluidframework/mocha-test-setup": "workspace:~", - "@fluidframework/telemetry-utils": "workspace:~", "@fluidframework/test-runtime-utils": "workspace:~", "@fluidframework/test-utils": "workspace:~", "@microsoft/api-extractor": "^7.38.3", diff --git a/experimental/dds/tree2/src/simple-tree/toMapTree.ts b/experimental/dds/tree2/src/simple-tree/toMapTree.ts index cbe79343c514..a1ef0f1d4fb7 100644 --- a/experimental/dds/tree2/src/simple-tree/toMapTree.ts +++ b/experimental/dds/tree2/src/simple-tree/toMapTree.ts @@ -5,6 +5,7 @@ import { IFluidHandle } from "@fluidframework/core-interfaces"; import { assert } from "@fluidframework/core-utils"; +import { UsageError } from "@fluidframework/telemetry-utils"; import { type FieldKey, @@ -325,9 +326,13 @@ function getType( ): TreeNodeSchemaIdentifier { const possibleTypes = getPossibleTypes(context, typeSet, data as ContextuallyTypedNodeData); assert(possibleTypes.length !== 0, "data is incompatible with all types allowed by the schema"); - assert( + checkInput( possibleTypes.length === 1, - "data is compatible with more than one type allowed by the schema", + () => + `The provided data is compatible with more than one type allowed by the schema. +The set of possible types is ${JSON.stringify([...possibleTypes], undefined)}. +Explicitly construct an unhydrated node of the desired type to disambiguate. +For class-based schema, this can be done by replacing an expression like "{foo: 1}" with "new MySchema({foo: 1})".`, ); return possibleTypes[0]; } @@ -335,3 +340,17 @@ function getType( function getSchema(context: TreeDataContext, type: TreeNodeSchemaIdentifier): TreeNodeStoredSchema { return context.schema.nodeSchema.get(type) ?? fail("Requested type does not exist in schema."); } + +/** + * An invalid tree has been provided, presumably by the user of this package. + * Throw and an error that properly preserves the message (unlike asserts which will get hard to read short codes intended for package internal logic errors). + */ +function invalidInput(message: string): never { + throw new UsageError(message); +} + +function checkInput(condition: boolean, message: string | (() => string)): asserts condition { + if (!condition) { + invalidInput(typeof message === "string" ? message : message()); + } +} diff --git a/experimental/dds/tree2/src/test/simple-tree/toMapTree.spec.ts b/experimental/dds/tree2/src/test/simple-tree/toMapTree.spec.ts index d7550ef1a73a..b1baab6fe2bb 100644 --- a/experimental/dds/tree2/src/test/simple-tree/toMapTree.spec.ts +++ b/experimental/dds/tree2/src/test/simple-tree/toMapTree.spec.ts @@ -12,6 +12,7 @@ import { SchemaBuilder, leaf } from "../../domains"; // eslint-disable-next-line import/no-internal-modules import { nodeDataToMapTree } from "../../simple-tree/toMapTree"; import { brand } from "../../util"; +import { FieldKinds, SchemaBuilderBase } from "../../feature-libraries"; describe("toMapTree", () => { it("string", () => { @@ -374,6 +375,18 @@ describe("toMapTree", () => { assert.deepEqual(actual, expected); }); + it("ambagious unions", () => { + const schemaBuilder = new SchemaBuilderBase(FieldKinds.required, { scope: "test" }); + const a = schemaBuilder.object("a", {}); + const b = schemaBuilder.object("b", {}); + const schema = schemaBuilder.intoSchema([a, b]); + + assert.throws( + () => nodeDataToMapTree({}, { schema }, schema.rootFieldSchema.types), + /\["test.a","test.b"]/, + ); + }); + // Our data serialization format does not support certain numeric values. // These tests are intended to verify the mapping behaviors for those values. describe("Incompatible numeric value handling", () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4206d218b323..95e769b5624d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5891,6 +5891,7 @@ importers: '@fluidframework/runtime-definitions': link:../../../packages/runtime/runtime-definitions '@fluidframework/runtime-utils': link:../../../packages/runtime/runtime-utils '@fluidframework/shared-object-base': link:../../../packages/dds/shared-object-base + '@fluidframework/telemetry-utils': link:../../../packages/utils/telemetry-utils '@sinclair/typebox': 0.29.6 '@ungap/structured-clone': 1.2.0 sorted-btree: 1.8.1 @@ -5907,7 +5908,6 @@ importers: '@fluidframework/container-loader': link:../../../packages/loader/container-loader '@fluidframework/eslint-config-fluid': 3.1.0_loebgezstcsvd2poh2d55fifke '@fluidframework/mocha-test-setup': link:../../../packages/test/mocha-test-setup - '@fluidframework/telemetry-utils': link:../../../packages/utils/telemetry-utils '@fluidframework/test-runtime-utils': link:../../../packages/runtime/test-runtime-utils '@fluidframework/test-utils': link:../../../packages/test/test-utils '@microsoft/api-extractor': 7.38.3_veevzrg6trzjzkr6gaha4thdmm_@types+node@18.19.1 From c82e8a127d73911b93a6b83aa958d294addeb7dc Mon Sep 17 00:00:00 2001 From: Joshua Smithrud <54606601+Josmithr@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:26:38 -0800 Subject: [PATCH 04/20] test(api-markdown-documenter): Update test suite for newer api-extractor-model and to capture some missing test cases (#18685) The key change here is updating `simple-suite-test.json`. This is a sample test report which the end-to-end tests use as input. The updates include: - Updating to a more recent version of the api-extractor schema and regenerating the contents against it - You will see (among other things) some changes to how signature contents are tokenized and displayed. - Updating the sample suite to update the release tags on a couple of items under `TestNamespace` - Makes 1 `@alpha`, 1 `@beta`, to ensure nested items are trimmed correctly via the changes made in #18663 --- .../default-config/simple-suite-test.html | 4 +- ...aceextendingotherinterfaces-interface.html | 2 +- .../testnamespace-namespace.html | 17 +- .../testinterface-interface.html | 29 +++- .../flat-config/simple-suite-test.html | 154 +++--------------- ...aceextendingotherinterfaces-interface.html | 2 +- .../testmappedtype-typealias.html | 2 +- .../testnamespace-namespace.html | 12 ++ .../testnamespace-testconst-variable.html | 26 --- ...testnamespace-testinterface-interface.html | 95 ----------- ...e-testinterfacemethod-methodsignature.html | 26 --- ...stinterfaceproperty-propertysignature.html | 26 --- .../typealias-typealias.html | 2 +- .../default-config/simple-suite-test.md | 4 +- ...rfaceextendingotherinterfaces-interface.md | 2 +- .../testnamespace-namespace.md | 16 +- .../testnamespace/testinterface-interface.md | 26 +-- .../flat-config/simple-suite-test.md | 72 ++------ ...rfaceextendingotherinterfaces-interface.md | 2 +- .../testmappedtype-typealias.md | 2 +- .../testnamespace-namespace.md | 12 +- .../testnamespace-testconst-variable.md | 9 - .../testnamespace-testinterface-interface.md | 23 --- ...ace-testinterfacemethod-methodsignature.md | 9 - ...testinterfaceproperty-propertysignature.md | 9 - .../simple-suite-test/typealias-typealias.md | 2 +- .../src/test/test-data/simple-suite-test.json | 92 ++++++++--- 27 files changed, 196 insertions(+), 481 deletions(-) delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.html delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.html delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.html delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.html delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.md delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.md delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.md delete mode 100644 tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.md diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test.html index 91a79f3223b9..7f4f2c423c16 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test.html @@ -606,7 +606,7 @@

Signature

- export declare type TestMappedType = { + export type TestMappedType = {
[K in TestEnum]: boolean;
@@ -636,7 +636,7 @@

Signature

- export declare type TypeAlias = string; + export type TypeAlias = string;
diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html index 7916dcdfee1d..40e42c2c5177 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html @@ -27,7 +27,7 @@

export interface TestInterfaceExtendingOtherInterfaces extends TestInterface, TestMappedType, TestInterfaceWithTypeParameter<number>

- Extends: TestInterface, TestMappedType, TestInterfaceWithTypeParameter + Extends: TestInterface, TestMappedType, TestInterfaceWithTypeParameter<number>

diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.html index 15dbb7fe5c6f..8a9b6bdd2373 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.html @@ -78,6 +78,9 @@

Interface + + Alerts + Description @@ -88,6 +91,9 @@

TestInterface + + ALPHA + Test interface @@ -219,6 +225,9 @@

Variable + + Alerts + Modifiers @@ -232,6 +241,9 @@

TestConst + + BETA + readonly @@ -468,13 +480,16 @@

- TestConst + TestConst (BETA)

Test Constant

+
+ WARNING: This API is provided as a beta preview and may change without notice. Use at your own risk. +

Signature diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.html index 113411ded02d..10989404476f 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.html @@ -7,7 +7,7 @@

- TestInterface + TestInterface (ALPHA)

@@ -19,6 +19,9 @@

Test interface

+
+ WARNING: This API is provided as an alpha preview and may change without notice. Use at your own risk. +

Signature @@ -27,7 +30,7 @@

interface TestInterface extends TestInterfaceWithTypeParameter<TestEnum>

- Extends: TestInterfaceWithTypeParameter<TestEnum + Extends: TestInterfaceWithTypeParameter<TestEnum>

@@ -40,6 +43,9 @@

Property + + Alerts + Type @@ -53,6 +59,9 @@

testInterfaceProperty + + ALPHA + boolean @@ -73,6 +82,9 @@

Method + + Alerts + Return Type @@ -86,6 +98,9 @@

testInterfaceMethod() + + ALPHA + void @@ -102,13 +117,16 @@

- testInterfaceProperty + testInterfaceProperty (ALPHA)

Test interface property

+
+ WARNING: This API is provided as an alpha preview and may change without notice. Use at your own risk. +

Signature @@ -125,13 +143,16 @@

- testInterfaceMethod + testInterfaceMethod (ALPHA)

Test interface method

+
+ WARNING: This API is provided as an alpha preview and may change without notice. Use at your own risk. +

Signature diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html index ae24af3a0668..8214efa9d2c4 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html @@ -876,7 +876,7 @@

export interface TestInterfaceExtendingOtherInterfaces extends TestInterface, TestMappedType, TestInterfaceWithTypeParameter<number>

- Extends: TestInterface, TestMappedType, TestInterfaceWithTypeParameter + Extends: TestInterface, TestMappedType, TestInterfaceWithTypeParameter<number>

@@ -2531,7 +2531,7 @@

Signature

- export declare type TestMappedType = { + export type TestMappedType = {
[K in TestEnum]: boolean;
@@ -2561,7 +2561,7 @@

Signature

- export declare type TypeAlias = string; + export type TypeAlias = string;
@@ -2878,6 +2878,9 @@

Interface + + Alerts + Description @@ -2888,6 +2891,9 @@

TestInterface + + ALPHA + Test interface @@ -3019,6 +3025,9 @@

Variable + + Alerts + Modifiers @@ -3032,6 +3041,9 @@

TestConst + + BETA + readonly @@ -3073,137 +3085,6 @@

Interface Details

-
-

- TestInterface -

-
-

- Test interface -

-
-
-
- Signature -
- - interface TestInterface extends TestInterfaceWithTypeParameter<TestEnum> - -

- Extends: TestInterfaceWithTypeParameter<TestEnum -

-
-
-
- Properties -
- - - - - - - - - - - - - - - -
- Property - - Type - - Description -
- testInterfaceProperty - - boolean - - Test interface property -
-
-
-
- Methods -
- - - - - - - - - - - - - - - -
- Method - - Return Type - - Description -
- testInterfaceMethod() - - void - - Test interface method -
-
-
-
- Property Details -
-
-
- testInterfaceProperty -
-
-

- Test interface property -

-
-
- - Signature - - testInterfaceProperty: boolean; - -
-
-
-
-
- Method Details -
-
-
- testInterfaceMethod -
-
-

- Test interface method -

-
-
- - Signature - - testInterfaceMethod(): void; - -
-
-
-

@@ -3679,13 +3560,16 @@

- TestConst + TestConst (BETA)

Test Constant

+
+ WARNING: This API is provided as a beta preview and may change without notice. Use at your own risk. +
Signature diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html index 9363752c64ac..ebc9bf733d21 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.html @@ -21,7 +21,7 @@

export interface TestInterfaceExtendingOtherInterfaces extends TestInterface, TestMappedType, TestInterfaceWithTypeParameter<number>

- Extends: TestInterface, TestMappedType, TestInterfaceWithTypeParameter + Extends: TestInterface, TestMappedType, TestInterfaceWithTypeParameter<number>

diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.html index 691ec5f9c952..b26d231177d5 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.html @@ -18,7 +18,7 @@

Signature

- export declare type TestMappedType = { + export type TestMappedType = {
[K in TestEnum]: boolean;
diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html index c0d56da1652e..9a99f41926cd 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html @@ -72,6 +72,9 @@

Interface + + Alerts + Description @@ -82,6 +85,9 @@

TestInterface + + ALPHA + Test interface @@ -213,6 +219,9 @@

Variable + + Alerts + Modifiers @@ -226,6 +235,9 @@

TestConst + + BETA + readonly diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.html deleted file mode 100644 index abc0b2c83a12..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - -
-

- TestConst -

-
-

- Test Constant -

-
-
-

- Signature -

- - TestConst = "Hello world!" - -
-
- - diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.html deleted file mode 100644 index 5b111521dafd..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.html +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - -
-

- TestInterface -

-
-

- Test interface -

-
-
-

- Signature -

- - interface TestInterface extends TestInterfaceWithTypeParameter<TestEnum> - -

- Extends: TestInterfaceWithTypeParameter<TestEnum -

-
-
-

- Properties -

- - - - - - - - - - - - - - - -
- Property - - Type - - Description -
- testInterfaceProperty - - boolean - - Test interface property -
-
-
-

- Methods -

- - - - - - - - - - - - - - - -
- Method - - Return Type - - Description -
- testInterfaceMethod() - - void - - Test interface method -
-
-
- - diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.html deleted file mode 100644 index 998f9d612086..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - -
-

- testInterfaceMethod -

-
-

- Test interface method -

-
-
-

- Signature -

- - testInterfaceMethod(): void; - -
-
- - diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.html deleted file mode 100644 index adb168527a9e..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - -
-

- testInterfaceProperty -

-
-

- Test interface property -

-
-
-

- Signature -

- - testInterfaceProperty: boolean; - -
-
- - diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.html index f3d0e74c0065..be0d3041e421 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.html @@ -18,7 +18,7 @@

Signature

- export declare type TypeAlias = string; + export type TypeAlias = string;
diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test.md index d6e6625ae8c0..d0a37061e7f7 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test.md @@ -186,7 +186,7 @@ Test Mapped Type, using [TestEnum](./simple-suite-test#testenum-enum) #### Signature {#testmappedtype-signature} ```typescript -export declare type TestMappedType = { +export type TestMappedType = { [K in TestEnum]: boolean; }; ``` @@ -202,7 +202,7 @@ Test Type-Alias #### Signature {#typealias-signature} ```typescript -export declare type TypeAlias = string; +export type TypeAlias = string; ``` #### Remarks {#typealias-remarks} diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md index ae60aae5b7bc..f200b32da394 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md @@ -12,7 +12,7 @@ Test interface that extends other interfaces export interface TestInterfaceExtendingOtherInterfaces extends TestInterface, TestMappedType, TestInterfaceWithTypeParameter ``` -**Extends:** [TestInterface](./simple-suite-test/testinterface-interface), [TestMappedType](./simple-suite-test#testmappedtype-typealias), [TestInterfaceWithTypeParameter](./simple-suite-test/testinterfacewithtypeparameter-interface) +**Extends:** [TestInterface](./simple-suite-test/testinterface-interface), [TestMappedType](./simple-suite-test#testmappedtype-typealias), [TestInterfaceWithTypeParameter](./simple-suite-test/testinterfacewithtypeparameter-interface)<number> ## Remarks {#testinterfaceextendingotherinterfaces-remarks} diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.md index 11750a448de1..f2f39081dea1 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace-namespace.md @@ -32,9 +32,9 @@ const bar = foo ## Interfaces -| Interface | Description | -| --- | --- | -| [TestInterface](./simple-suite-test/testnamespace/testinterface-interface) | Test interface | +| Interface | Alerts | Description | +| --- | --- | --- | +| [TestInterface](./simple-suite-test/testnamespace/testinterface-interface) | `ALPHA` | Test interface | ## Classes @@ -62,9 +62,9 @@ const bar = foo ## Variables -| Variable | Modifiers | Description | -| --- | --- | --- | -| [TestConst](./simple-suite-test/testnamespace-namespace#testconst-variable) | `readonly` | Test Constant | +| Variable | Alerts | Modifiers | Description | +| --- | --- | --- | --- | +| [TestConst](./simple-suite-test/testnamespace-namespace#testconst-variable) | `BETA` | `readonly` | Test Constant | ## Namespaces @@ -153,10 +153,12 @@ An Error ## Variable Details -### TestConst {#testconst-variable} +### TestConst (BETA) {#testconst-variable} Test Constant +**WARNING: This API is provided as a beta preview and may change without notice. Use at your own risk.** + #### Signature {#testconst-signature} ```typescript diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.md index 9fa36ebab185..b66e1fabeee6 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/default-config/simple-suite-test/testnamespace/testinterface-interface.md @@ -1,37 +1,41 @@ -# TestInterface +# TestInterface (ALPHA) [Packages](./) > [simple-suite-test](./simple-suite-test) > [TestNamespace](./simple-suite-test/testnamespace-namespace) > [TestInterface](./simple-suite-test/testnamespace/testinterface-interface) Test interface +**WARNING: This API is provided as an alpha preview and may change without notice. Use at your own risk.** + ## Signature {#testinterface-signature} ```typescript interface TestInterface extends TestInterfaceWithTypeParameter ``` -**Extends:** [TestInterfaceWithTypeParameter](./simple-suite-test/testinterfacewithtypeparameter-interface)<[TestEnum](./simple-suite-test/testnamespace-namespace#testenum-enum) +**Extends:** [TestInterfaceWithTypeParameter](./simple-suite-test/testinterfacewithtypeparameter-interface)<[TestEnum](./simple-suite-test/testnamespace-namespace#testenum-enum)> ## Properties -| Property | Type | Description | -| --- | --- | --- | -| [testInterfaceProperty](./simple-suite-test/testnamespace/testinterface-interface#testinterfaceproperty-propertysignature) | boolean | Test interface property | +| Property | Alerts | Type | Description | +| --- | --- | --- | --- | +| [testInterfaceProperty](./simple-suite-test/testnamespace/testinterface-interface#testinterfaceproperty-propertysignature) | `ALPHA` | boolean | Test interface property | ## Methods -| Method | Return Type | Description | -| --- | --- | --- | -| [testInterfaceMethod()](./simple-suite-test/testnamespace/testinterface-interface#testinterfacemethod-methodsignature) | void | Test interface method | +| Method | Alerts | Return Type | Description | +| --- | --- | --- | --- | +| [testInterfaceMethod()](./simple-suite-test/testnamespace/testinterface-interface#testinterfacemethod-methodsignature) | `ALPHA` | void | Test interface method | ## Property Details -### testInterfaceProperty {#testinterfaceproperty-propertysignature} +### testInterfaceProperty (ALPHA) {#testinterfaceproperty-propertysignature} Test interface property +**WARNING: This API is provided as an alpha preview and may change without notice. Use at your own risk.** + #### Signature {#testinterfaceproperty-signature} ```typescript @@ -40,10 +44,12 @@ testInterfaceProperty: boolean; ## Method Details -### testInterfaceMethod {#testinterfacemethod-methodsignature} +### testInterfaceMethod (ALPHA) {#testinterfacemethod-methodsignature} Test interface method +**WARNING: This API is provided as an alpha preview and may change without notice. Use at your own risk.** + #### Signature {#testinterfacemethod-signature} ```typescript diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md index bd2010bc00b7..9027543fcd81 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md @@ -273,7 +273,7 @@ Test interface that extends other interfaces export interface TestInterfaceExtendingOtherInterfaces extends TestInterface, TestMappedType, TestInterfaceWithTypeParameter ``` -**Extends:** [TestInterface](docs/simple-suite-test#testinterface-interface), [TestMappedType](docs/simple-suite-test#testmappedtype-typealias), [TestInterfaceWithTypeParameter](docs/simple-suite-test#testinterfacewithtypeparameter-interface) +**Extends:** [TestInterface](docs/simple-suite-test#testinterface-interface), [TestMappedType](docs/simple-suite-test#testmappedtype-typealias), [TestInterfaceWithTypeParameter](docs/simple-suite-test#testinterfacewithtypeparameter-interface)<number> ### Remarks {#testinterfaceextendingotherinterfaces-remarks} @@ -845,7 +845,7 @@ Test Mapped Type, using [TestEnum](docs/simple-suite-test#testenum-enum) ### Signature {#testmappedtype-signature} ```typescript -export declare type TestMappedType = { +export type TestMappedType = { [K in TestEnum]: boolean; }; ``` @@ -861,7 +861,7 @@ Test Type-Alias ### Signature {#typealias-signature} ```typescript -export declare type TypeAlias = string; +export type TypeAlias = string; ``` ### Remarks {#typealias-remarks} @@ -1007,9 +1007,9 @@ const bar = foo ### Interfaces -| Interface | Description | -| --- | --- | -| [TestInterface](docs/simple-suite-test#testnamespace-testinterface-interface) | Test interface | +| Interface | Alerts | Description | +| --- | --- | --- | +| [TestInterface](docs/simple-suite-test#testnamespace-testinterface-interface) | `ALPHA` | Test interface | ### Classes @@ -1037,9 +1037,9 @@ const bar = foo ### Variables -| Variable | Modifiers | Description | -| --- | --- | --- | -| [TestConst](docs/simple-suite-test#testnamespace-testconst-variable) | `readonly` | Test Constant | +| Variable | Alerts | Modifiers | Description | +| --- | --- | --- | --- | +| [TestConst](docs/simple-suite-test#testnamespace-testconst-variable) | `BETA` | `readonly` | Test Constant | ### Namespaces @@ -1049,56 +1049,6 @@ const bar = foo ### Interface Details -#### TestInterface {#testnamespace-testinterface-interface} - -Test interface - -##### Signature {#testinterface-signature} - -```typescript -interface TestInterface extends TestInterfaceWithTypeParameter -``` - -**Extends:** [TestInterfaceWithTypeParameter](docs/simple-suite-test#testinterfacewithtypeparameter-interface)<[TestEnum](docs/simple-suite-test#testnamespace-testenum-enum) - -##### Properties - -| Property | Type | Description | -| --- | --- | --- | -| [testInterfaceProperty](docs/simple-suite-test#testnamespace-testinterface-testinterfaceproperty-propertysignature) | boolean | Test interface property | - -##### Methods - -| Method | Return Type | Description | -| --- | --- | --- | -| [testInterfaceMethod()](docs/simple-suite-test#testnamespace-testinterface-testinterfacemethod-methodsignature) | void | Test interface method | - -##### Property Details - -###### testInterfaceProperty {#testnamespace-testinterface-testinterfaceproperty-propertysignature} - -Test interface property - - -**Signature** - -```typescript -testInterfaceProperty: boolean; -``` - -##### Method Details - -###### testInterfaceMethod {#testnamespace-testinterface-testinterfacemethod-methodsignature} - -Test interface method - - -**Signature** - -```typescript -testInterfaceMethod(): void; -``` - ### Class Details #### TestClass {#testnamespace-testclass-class} @@ -1281,10 +1231,12 @@ An Error ### Variable Details -#### TestConst {#testnamespace-testconst-variable} +#### TestConst (BETA) {#testnamespace-testconst-variable} Test Constant +**WARNING: This API is provided as a beta preview and may change without notice. Use at your own risk.** + ##### Signature {#testconst-signature} ```typescript diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md index 4e4a9b59816b..deeacf7039b6 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testinterfaceextendingotherinterfaces-interface.md @@ -8,7 +8,7 @@ Test interface that extends other interfaces export interface TestInterfaceExtendingOtherInterfaces extends TestInterface, TestMappedType, TestInterfaceWithTypeParameter ``` -**Extends:** [TestInterface](docs/simple-suite-test/testinterface-interface), [TestMappedType](docs/simple-suite-test/testmappedtype-typealias), [TestInterfaceWithTypeParameter](docs/simple-suite-test/testinterfacewithtypeparameter-interface) +**Extends:** [TestInterface](docs/simple-suite-test/testinterface-interface), [TestMappedType](docs/simple-suite-test/testmappedtype-typealias), [TestInterfaceWithTypeParameter](docs/simple-suite-test/testinterfacewithtypeparameter-interface)<number> ### Remarks {#testinterfaceextendingotherinterfaces-remarks} diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.md index 1e61c3da6929..3e7aa14d519f 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testmappedtype-typealias.md @@ -5,7 +5,7 @@ Test Mapped Type, using [TestEnum](docs/simple-suite-test/testenum-enum) ### Signature {#testmappedtype-signature} ```typescript -export declare type TestMappedType = { +export type TestMappedType = { [K in TestEnum]: boolean; }; ``` diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md index cc6347da9889..a6baffdf5f1a 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md @@ -28,9 +28,9 @@ const bar = foo ### Interfaces -| Interface | Description | -| --- | --- | -| [TestInterface](docs/simple-suite-test/testnamespace-testinterface-interface) | Test interface | +| Interface | Alerts | Description | +| --- | --- | --- | +| [TestInterface](docs/simple-suite-test/testnamespace-testinterface-interface) | `ALPHA` | Test interface | ### Classes @@ -58,9 +58,9 @@ const bar = foo ### Variables -| Variable | Modifiers | Description | -| --- | --- | --- | -| [TestConst](docs/simple-suite-test/testnamespace-testconst-variable) | `readonly` | Test Constant | +| Variable | Alerts | Modifiers | Description | +| --- | --- | --- | --- | +| [TestConst](docs/simple-suite-test/testnamespace-testconst-variable) | `BETA` | `readonly` | Test Constant | ### Namespaces diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.md deleted file mode 100644 index 36b31c8f259b..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testconst-variable.md +++ /dev/null @@ -1,9 +0,0 @@ -## TestConst - -Test Constant - -### Signature {#testconst-signature} - -```typescript -TestConst = "Hello world!" -``` diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.md deleted file mode 100644 index c4034c8d3c57..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-interface.md +++ /dev/null @@ -1,23 +0,0 @@ -## TestInterface - -Test interface - -### Signature {#testinterface-signature} - -```typescript -interface TestInterface extends TestInterfaceWithTypeParameter -``` - -**Extends:** [TestInterfaceWithTypeParameter](docs/simple-suite-test/testinterfacewithtypeparameter-interface)<[TestEnum](docs/simple-suite-test/testnamespace-testenum-enum) - -### Properties - -| Property | Type | Description | -| --- | --- | --- | -| [testInterfaceProperty](docs/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature) | boolean | Test interface property | - -### Methods - -| Method | Return Type | Description | -| --- | --- | --- | -| [testInterfaceMethod()](docs/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature) | void | Test interface method | diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.md deleted file mode 100644 index 4e81acf1a9ca..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfacemethod-methodsignature.md +++ /dev/null @@ -1,9 +0,0 @@ -## testInterfaceMethod - -Test interface method - -### Signature {#testinterfacemethod-signature} - -```typescript -testInterfaceMethod(): void; -``` diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.md deleted file mode 100644 index b645118353a8..000000000000 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-testinterface-testinterfaceproperty-propertysignature.md +++ /dev/null @@ -1,9 +0,0 @@ -## testInterfaceProperty - -Test interface property - -### Signature {#testinterfaceproperty-signature} - -```typescript -testInterfaceProperty: boolean; -``` diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.md index fcbd14a58f1a..3b9f39b40567 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/typealias-typealias.md @@ -5,7 +5,7 @@ Test Type-Alias ### Signature {#typealias-signature} ```typescript -export declare type TypeAlias = string; +export type TypeAlias = string; ``` ### Remarks {#typealias-remarks} diff --git a/tools/api-markdown-documenter/src/test/test-data/simple-suite-test.json b/tools/api-markdown-documenter/src/test/test-data/simple-suite-test.json index d69dcb77fa61..ebc97fdb2cc2 100644 --- a/tools/api-markdown-documenter/src/test/test-data/simple-suite-test.json +++ b/tools/api-markdown-documenter/src/test/test-data/simple-suite-test.json @@ -1,8 +1,8 @@ { "metadata": { "toolPackage": "@microsoft/api-extractor", - "toolVersion": "7.28.3", - "schemaVersion": 1009, + "toolVersion": "7.38.3", + "schemaVersion": 1011, "oldestForwardsCompatibleVersion": 1001, "tsdocConfig": { "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json", @@ -182,7 +182,9 @@ "text": "export declare abstract class TestAbstractClass " } ], + "fileUrlPath": "dist/TestClass.d.ts", "releaseTag": "Public", + "isAbstract": true, "name": "TestAbstractClass", "preserveMemberOrder": false, "members": [ @@ -263,7 +265,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": false + "isProtected": false, + "isAbstract": true }, { "kind": "Property", @@ -293,7 +296,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": true + "isProtected": true, + "isAbstract": false }, { "kind": "Method", @@ -323,6 +327,7 @@ "overloadIndex": 1, "parameters": [], "isOptional": false, + "isAbstract": true, "name": "publicAbstractMethod" }, { @@ -353,6 +358,7 @@ "overloadIndex": 1, "parameters": [], "isOptional": false, + "isAbstract": false, "name": "sealedMethod" }, { @@ -383,6 +389,7 @@ "overloadIndex": 1, "parameters": [], "isOptional": false, + "isAbstract": false, "name": "virtualMethod" } ], @@ -407,6 +414,7 @@ "text": " " } ], + "fileUrlPath": "dist/TestClass.d.ts", "releaseTag": "Public", "typeParameters": [ { @@ -432,6 +440,7 @@ } } ], + "isAbstract": false, "name": "TestClass", "preserveMemberOrder": false, "members": [ @@ -544,7 +553,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": false + "isProtected": false, + "isAbstract": false }, { "kind": "Method", @@ -574,6 +584,7 @@ "overloadIndex": 1, "parameters": [], "isOptional": false, + "isAbstract": false, "name": "publicAbstractMethod" }, { @@ -603,7 +614,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": false + "isProtected": false, + "isAbstract": false }, { "kind": "Property", @@ -632,7 +644,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": false + "isProtected": false, + "isAbstract": false }, { "kind": "Method", @@ -679,6 +692,7 @@ } ], "isOptional": false, + "isAbstract": false, "name": "testClassMethod" }, { @@ -708,7 +722,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": false + "isProtected": false, + "isAbstract": false }, { "kind": "Method", @@ -755,6 +770,7 @@ } ], "isOptional": false, + "isAbstract": false, "name": "testClassStaticMethod" }, { @@ -784,7 +800,8 @@ "endIndex": 2 }, "isStatic": true, - "isProtected": false + "isProtected": false, + "isAbstract": false }, { "kind": "Method", @@ -814,6 +831,7 @@ "overloadIndex": 1, "parameters": [], "isOptional": false, + "isAbstract": false, "name": "virtualMethod" } ], @@ -837,6 +855,7 @@ "text": "42" } ], + "fileUrlPath": "dist/TestConst.d.ts", "initializerTokenRange": { "startIndex": 1, "endIndex": 2 @@ -863,6 +882,7 @@ "text": "\"I have a `@deprecated` tag with an empty comment block.\"" } ], + "fileUrlPath": "dist/TestConst.d.ts", "initializerTokenRange": { "startIndex": 1, "endIndex": 2 @@ -885,6 +905,7 @@ "text": "export interface TestEmptyInterface " } ], + "fileUrlPath": "dist/TestInterface.d.ts", "releaseTag": "Public", "name": "TestEmptyInterface", "preserveMemberOrder": false, @@ -901,6 +922,7 @@ "text": "export declare enum TestEnum " } ], + "fileUrlPath": "dist/TestEnum.d.ts", "releaseTag": "Public", "name": "TestEnum", "preserveMemberOrder": false, @@ -1004,6 +1026,7 @@ "text": ";" } ], + "fileUrlPath": "dist/TestFunction.d.ts", "returnTypeTokenRange": { "startIndex": 5, "endIndex": 6 @@ -1070,6 +1093,7 @@ "text": ";" } ], + "fileUrlPath": "dist/TestFunction.d.ts", "returnTypeTokenRange": { "startIndex": 1, "endIndex": 4 @@ -1111,6 +1135,7 @@ "text": ";" } ], + "fileUrlPath": "dist/TestFunction.d.ts", "returnTypeTokenRange": { "startIndex": 1, "endIndex": 5 @@ -1143,6 +1168,7 @@ "text": ";" } ], + "fileUrlPath": "dist/TestFunction.d.ts", "returnTypeTokenRange": { "startIndex": 1, "endIndex": 3 @@ -1162,6 +1188,7 @@ "text": "export interface TestInterface " } ], + "fileUrlPath": "dist/TestInterface.d.ts", "releaseTag": "Public", "name": "TestInterface", "preserveMemberOrder": false, @@ -1457,9 +1484,14 @@ }, { "kind": "Content", - "text": " " + "text": "" + }, + { + "kind": "Content", + "text": " " } ], + "fileUrlPath": "dist/TestInterface.d.ts", "releaseTag": "Public", "name": "TestInterfaceExtendingOtherInterfaces", "preserveMemberOrder": false, @@ -1521,7 +1553,7 @@ }, { "startIndex": 5, - "endIndex": 6 + "endIndex": 7 } ] }, @@ -1535,6 +1567,7 @@ "text": "export interface TestInterfaceWithIndexSignature " } ], + "fileUrlPath": "dist/TestInterface.d.ts", "releaseTag": "Public", "name": "TestInterfaceWithIndexSignature", "preserveMemberOrder": false, @@ -1596,6 +1629,7 @@ "text": "export interface TestInterfaceWithTypeParameter " } ], + "fileUrlPath": "dist/TestInterface.d.ts", "releaseTag": "Public", "typeParameters": [ { @@ -1650,7 +1684,7 @@ "excerptTokens": [ { "kind": "Content", - "text": "export declare type TestMappedType = " + "text": "export type TestMappedType = " }, { "kind": "Content", @@ -1670,6 +1704,7 @@ "text": ";" } ], + "fileUrlPath": "dist/TestType.d.ts", "releaseTag": "Public", "name": "TestMappedType", "typeTokenRange": { @@ -1682,6 +1717,7 @@ "canonicalReference": "simple-suite-test!TestModule:namespace", "docComment": "", "excerptTokens": [], + "fileUrlPath": "dist/index.d.ts", "releaseTag": "None", "name": "TestModule", "preserveMemberOrder": false, @@ -1700,6 +1736,7 @@ "text": "2" } ], + "fileUrlPath": "dist/TestModule.d.ts", "initializerTokenRange": { "startIndex": 1, "endIndex": 2 @@ -1724,6 +1761,7 @@ "text": "export declare namespace TestNamespace " } ], + "fileUrlPath": "dist/TestNamespace.d.ts", "releaseTag": "Public", "name": "TestNamespace", "preserveMemberOrder": false, @@ -1731,7 +1769,7 @@ { "kind": "Class", "canonicalReference": "simple-suite-test!TestNamespace.TestClass:class", - "docComment": "/**\n * Test class\n *\n * @public\n */\n", + "docComment": "/**\n * Test class\n */\n", "excerptTokens": [ { "kind": "Content", @@ -1739,6 +1777,7 @@ } ], "releaseTag": "Public", + "isAbstract": false, "name": "TestClass", "preserveMemberOrder": false, "members": [ @@ -1824,6 +1863,7 @@ } ], "isOptional": false, + "isAbstract": false, "name": "testClassMethod" }, { @@ -1853,7 +1893,8 @@ "endIndex": 2 }, "isStatic": false, - "isProtected": false + "isProtected": false, + "isAbstract": false } ], "implementsTokenRanges": [] @@ -1861,7 +1902,7 @@ { "kind": "Variable", "canonicalReference": "simple-suite-test!TestNamespace.TestConst:var", - "docComment": "/**\n * Test Constant\n *\n * @public\n */\n", + "docComment": "/**\n * Test Constant\n *\n * @beta\n */\n", "excerptTokens": [ { "kind": "Content", @@ -1877,7 +1918,7 @@ "endIndex": 2 }, "isReadonly": true, - "releaseTag": "Public", + "releaseTag": "Beta", "name": "TestConst", "variableTypeTokenRange": { "startIndex": 0, @@ -1989,7 +2030,7 @@ { "kind": "Interface", "canonicalReference": "simple-suite-test!TestNamespace.TestInterface:interface", - "docComment": "/**\n * Test interface\n *\n * @public\n */\n", + "docComment": "/**\n * Test interface\n *\n * @alpha\n */\n", "excerptTokens": [ { "kind": "Content", @@ -2011,10 +2052,14 @@ }, { "kind": "Content", - "text": "> " + "text": ">" + }, + { + "kind": "Content", + "text": " " } ], - "releaseTag": "Public", + "releaseTag": "Alpha", "name": "TestInterface", "preserveMemberOrder": false, "members": [ @@ -2041,7 +2086,7 @@ "startIndex": 1, "endIndex": 2 }, - "releaseTag": "Public", + "releaseTag": "Alpha", "overloadIndex": 1, "parameters": [], "name": "testInterfaceMethod" @@ -2066,7 +2111,7 @@ ], "isReadonly": false, "isOptional": false, - "releaseTag": "Public", + "releaseTag": "Alpha", "name": "testInterfaceProperty", "propertyTypeTokenRange": { "startIndex": 1, @@ -2077,7 +2122,7 @@ "extendsTokenRanges": [ { "startIndex": 1, - "endIndex": 4 + "endIndex": 5 } ] }, @@ -2130,7 +2175,7 @@ "excerptTokens": [ { "kind": "Content", - "text": "export declare type TypeAlias = " + "text": "export type TypeAlias = " }, { "kind": "Content", @@ -2141,6 +2186,7 @@ "text": ";" } ], + "fileUrlPath": "dist/TestType.d.ts", "releaseTag": "Public", "name": "TypeAlias", "typeTokenRange": { From f3bed2f4ecc93183795ff23c16ac28a76749bef3 Mon Sep 17 00:00:00 2001 From: daesunp <138815173+daesunp@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:29:41 -0800 Subject: [PATCH 05/20] Chunk encoding in SharedTree insert changesets (#18415) ## Description This PR updates the encoding of trees used in sharedTree changesets --- .../dds/tree2/api-report/tree2.api.md | 5 +- .../default-schema/defaultChangeFamily.ts | 19 +- .../modular-schema/modularChangeFamily.ts | 45 +- .../modular-schema/modularChangeTypes.ts | 4 +- .../defaultChangeFamily.spec.ts | 43 + .../modularChangeFamily.spec.ts | 33 +- .../modularChangeFamilyIntegration.spec.ts | 6 +- .../src/test/shared-tree/editing.spec.ts | 5 +- .../snapshots/files/complete-3x3-final.json | 2142 +++++++++++++++-- .../files/concurrent-inserts-tree2.json | 120 +- .../files/concurrent-inserts-tree3.json | 216 +- ...insert-and-delete-tree-0-after-insert.json | 24 +- .../move-across-fields-tree-0-final.json | 79 +- .../files/nested-sequence-change-final.json | 44 +- 14 files changed, 2521 insertions(+), 264 deletions(-) diff --git a/experimental/dds/tree2/api-report/tree2.api.md b/experimental/dds/tree2/api-report/tree2.api.md index 78d1a3d5705b..4ce1b9166b93 100644 --- a/experimental/dds/tree2/api-report/tree2.api.md +++ b/experimental/dds/tree2/api-report/tree2.api.md @@ -828,7 +828,10 @@ export interface IDefaultEditBuilder { // (undocumented) optionalField(field: FieldUpPath): OptionalFieldEditBuilder; // (undocumented) - sequenceField(field: FieldUpPath): SequenceFieldEditBuilder; + sequenceField(field: FieldUpPath, shapeInfo?: { + schema: StoredSchemaCollection; + policy: FullSchemaPolicy; + }): SequenceFieldEditBuilder; // (undocumented) valueField(field: FieldUpPath): ValueFieldEditBuilder; } diff --git a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts index 3a7d7559bf29..b6ddc984a98f 100644 --- a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts +++ b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts @@ -16,6 +16,7 @@ import { topDownPath, TaggedChange, DeltaRoot, + StoredSchemaCollection, } from "../../core"; import { brand, isReadonlyArray } from "../../util"; import { @@ -24,6 +25,7 @@ import { FieldChangeset, ModularChangeset, FieldEditDescription, + FullSchemaPolicy, intoDelta as intoModularDelta, } from "../modular-schema"; import { fieldKinds, optional, sequence, required as valueFieldKind } from "./defaultFieldKinds"; @@ -87,11 +89,19 @@ export interface IDefaultEditBuilder { /** * @param field - the sequence field which is being edited under the parent node + * + * @param shapeInfo - optional shape information used for schema based chunk encoding. + * TODO: The 'shapeInfo' parameter is a temporary solution enabling schema-based chunk encoding within this function. + * This parameter should be removed once the encoded format is eventually separated out. + * * @returns An object with methods to edit the given field of the given parent. * The returned object can be used (i.e., have its methods called) multiple times but its lifetime * is bounded by the lifetime of this edit builder. */ - sequenceField(field: FieldUpPath): SequenceFieldEditBuilder; + sequenceField( + field: FieldUpPath, + shapeInfo?: { schema: StoredSchemaCollection; policy: FullSchemaPolicy }, + ): SequenceFieldEditBuilder; /** * Moves a subsequence from one sequence field to another sequence field. @@ -249,7 +259,10 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild } } - public sequenceField(field: FieldUpPath): SequenceFieldEditBuilder { + public sequenceField( + field: FieldUpPath, + shapeInfo?: { schema: StoredSchemaCollection; policy: FullSchemaPolicy }, + ): SequenceFieldEditBuilder { return { insert: (index: number, newContent: ITreeCursor | readonly ITreeCursor[]): void => { const content = isReadonlyArray(newContent) ? newContent : [newContent]; @@ -259,7 +272,7 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild } const firstId = this.modularBuilder.generateId(length); - const build = this.modularBuilder.buildTrees(firstId, content); + const build = this.modularBuilder.buildTrees(firstId, content, shapeInfo); const change: FieldChangeset = brand( sequence.changeHandler.editor.insert(index, length, firstId), ); diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts index ac2a353abe9c..1c07345ebbd9 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts @@ -24,7 +24,6 @@ import { RevisionInfo, revisionMetadataSourceFromInfo, ChangeAtomIdMap, - JsonableTree, makeDetachedNodeId, ITreeCursor, emptyDelta, @@ -32,6 +31,9 @@ import { DeltaFieldChanges, DeltaDetachedNodeBuild, DeltaRoot, + ITreeCursorSynchronous, + mapCursorField, + StoredSchemaCollection, } from "../../core"; import { brand, @@ -45,8 +47,15 @@ import { isReadonlyArray, Mutable, } from "../../util"; -import { cursorForJsonableTreeNode, jsonableTreeFromCursor } from "../treeTextCursor"; import { MemoizedIdRangeAllocator } from "../memoizedIdRangeAllocator"; +import { + EncodedChunk, + chunkTree, + decode, + defaultChunkPolicy, + schemaCompressedEncode, + uncompressedEncode, +} from "../chunked-forest"; import { CrossFieldManager, CrossFieldMap, @@ -61,7 +70,7 @@ import { NodeExistenceState, RebaseRevisionMetadata, } from "./fieldChangeHandler"; -import { FieldKind, FieldKindWithEditor, withEditor } from "./fieldKind"; +import { FieldKind, FieldKindWithEditor, FullSchemaPolicy, withEditor } from "./fieldKind"; import { convertGenericChange, genericFieldKind, newGenericChangeset } from "./genericFieldKind"; import { GenericChangeset } from "./genericFieldKindTypes"; import { makeModularChangeCodecFamily } from "./modularChangeCodecs"; @@ -198,7 +207,7 @@ export class ModularChangeFamily crossFieldTable.invalidatedFields.size === 0, 0x59b /* Should not need more than one amend pass. */, ); - const allBuilds: ChangeAtomIdMap = new Map(); + const allBuilds: ChangeAtomIdMap = new Map(); for (const taggedChange of changes) { const revision = revisionFromTaggedChange(taggedChange); const change = taggedChange.change; @@ -789,9 +798,12 @@ export function intoDelta( if (change.builds && change.builds.size > 0) { const builds: DeltaDetachedNodeBuild[] = []; forEachInNestedMap(change.builds, (tree, major, minor) => { + const cursor = decode(tree).cursor(); + assert(cursor.getFieldLength() === 1, "each encoded chunk should only contain 1 node."); + cursor.enterNode(0); builds.push({ id: makeDetachedNodeId(major ?? revision, minor), - trees: [cursorForJsonableTreeNode(tree)], + trees: [cursor], }); }); rootDelta.build = builds; @@ -1053,7 +1065,7 @@ function makeModularChangeset( maxId: number = -1, revisions: readonly RevisionInfo[] | undefined = undefined, constraintViolationCount: number | undefined = undefined, - builds?: ChangeAtomIdMap, + builds?: ChangeAtomIdMap, ): ModularChangeset { const changeset: Mutable = { fieldChanges: changes ?? new Map() }; if (revisions !== undefined && revisions.length > 0) { @@ -1105,18 +1117,33 @@ export class ModularEditBuilder extends EditBuilder { public buildTrees( firstId: ChangesetLocalId, newContent: ITreeCursor | readonly ITreeCursor[], + shapeInfo?: { schema: StoredSchemaCollection; policy: FullSchemaPolicy }, ): GlobalEditDescription { const content = isReadonlyArray(newContent) ? newContent : [newContent]; const length = content.length; if (length === 0) { return { type: "global" }; } - const builds: ChangeAtomIdMap = new Map(); + const builds: ChangeAtomIdMap = new Map(); const innerMap = new Map(); builds.set(undefined, innerMap); let id = firstId; + + const fieldCursors = content.map((cursor) => + chunkTree(cursor as ITreeCursorSynchronous, defaultChunkPolicy).cursor(), + ); + const nodeCursors = fieldCursors + .map((fieldCursor) => mapCursorField(fieldCursor, (c) => c)) + .flat(); + const encodedTrees = + shapeInfo !== undefined + ? nodeCursors.map((cursor) => + schemaCompressedEncode(shapeInfo.schema, shapeInfo.policy, cursor), + ) + : nodeCursors.map(uncompressedEncode); + // TODO:YA6307 adopt more efficient representation, likely based on contiguous runs of IDs - for (const tree of content.map(jsonableTreeFromCursor)) { + for (const tree of encodedTrees) { assert(!innerMap.has(id), "Unexpected duplicate build ID"); innerMap.set(id, tree); id = brand((id as number) + 1); @@ -1228,7 +1255,7 @@ export interface FieldEditDescription { */ export interface GlobalEditDescription { type: "global"; - builds?: ChangeAtomIdMap; + builds?: ChangeAtomIdMap; } /** diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeTypes.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeTypes.ts index 00e932d33bae..5b9964397229 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeTypes.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeTypes.ts @@ -8,11 +8,11 @@ import { ChangesetLocalId, FieldKey, FieldKindIdentifier, - JsonableTree, RevisionInfo, RevisionTag, } from "../../core"; import { Brand } from "../../util"; +import { EncodedChunk } from "../chunked-forest"; /** * @alpha @@ -32,7 +32,7 @@ export interface ModularChangeset extends HasFieldChanges { fieldChanges: FieldChangeMap; constraintViolationCount?: number; // TODO:YA6307 adopt more efficient representation, likely based on contiguous runs of IDs - readonly builds?: ChangeAtomIdMap; + readonly builds?: ChangeAtomIdMap; } /** diff --git a/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultChangeFamily.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultChangeFamily.spec.ts index 96bb2d5696ee..f8b8163749ec 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultChangeFamily.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultChangeFamily.spec.ts @@ -17,6 +17,7 @@ import { UpPath, applyDelta, makeDetachedFieldIndex, + ChangesetLocalId, DeltaRoot, } from "../../../core"; import { leaf, jsonObject } from "../../../domains"; @@ -24,14 +25,19 @@ import { DefaultChangeFamily, DefaultChangeset, DefaultEditBuilder, + ModularChangeset, buildForest, + cursorForJsonableTreeField, cursorForJsonableTreeNode, + defaultSchemaPolicy, intoDelta, jsonableTreeFromCursor, + schemaCompressedEncode, } from "../../../feature-libraries"; import { brand } from "../../../util"; import { assertDeltaEqual } from "../../utils"; import { noopValidator } from "../../../codec"; +import { testTrees } from "../../testTrees"; const defaultChangeFamily = new DefaultChangeFamily({ jsonValidator: noopValidator }); const family = defaultChangeFamily; @@ -359,6 +365,43 @@ describe("DefaultEditBuilder", () => { expectForest(forest, expected); }); + describe("encodes insert ops using schema based encoding", () => { + for (const { name, treeFactory, schemaData } of testTrees) { + it(name, () => { + const tree = treeFactory(); + const changes: ModularChangeset[] = []; + const changeReceiver = (change: ModularChangeset) => changes.push(change); + const builder = new DefaultEditBuilder(defaultChangeFamily, changeReceiver); + builder + .sequenceField( + { parent: undefined, field: rootKey }, + { + schema: schemaData, + policy: defaultSchemaPolicy, + }, + ) + .insert( + 0, + tree.map((node) => cursorForJsonableTreeNode(node)), + ); + + for (let index = 0; index < tree.length; index++) { + const treeField = tree.length === 1 ? tree : [tree[index]]; + const expectedOp = schemaCompressedEncode( + schemaData, + defaultSchemaPolicy, + cursorForJsonableTreeField(treeField), + ); + + const changesetId: ChangesetLocalId = brand(index); + const encodedOp = changes[0].builds?.get(undefined)?.get(changesetId); + + assert.deepEqual(encodedOp, expectedOp); + } + }); + } + }); + it("Can delete a root node", () => { const { builder, forest } = initializeEditableForest(nodeX); builder.sequenceField({ parent: undefined, field: rootKey }).delete(0, 1); diff --git a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts index dcd6e08cc03e..6afa4943bf63 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts @@ -14,6 +14,10 @@ import { FieldChange, ModularChangeset, FieldKindWithEditor, + chunkTree, + defaultChunkPolicy, + uncompressedEncode, + EncodedChunk, } from "../../../feature-libraries"; import { makeAnonChange, @@ -27,6 +31,7 @@ import { assertIsRevisionTag, deltaForSet, revisionMetadataSourceFromInfo, + ITreeCursorSynchronous, DeltaFieldChanges, DeltaRoot, } from "../../../core"; @@ -333,11 +338,15 @@ describe("ModularChangeFamily", () => { it("prioritizes earlier build entries when faced with duplicates", () => { const change1: ModularChangeset = { fieldChanges: new Map(), - builds: new Map([[undefined, new Map([[brand(0), singleJsonCursor(1)]])]]), + builds: new Map([ + [undefined, new Map([[brand(0), encodedChunkFromCursor(singleJsonCursor(1))]])], + ]), }; const change2: ModularChangeset = { fieldChanges: new Map(), - builds: new Map([[undefined, new Map([[brand(0), singleJsonCursor(2)]])]]), + builds: new Map([ + [undefined, new Map([[brand(0), encodedChunkFromCursor(singleJsonCursor(2))]])], + ]), }; assert.deepEqual( family.compose([makeAnonChange(change1), makeAnonChange(change2)]), @@ -535,8 +544,8 @@ describe("ModularChangeFamily", () => { { fieldChanges: new Map([]), builds: new Map([ - [undefined, new Map([[brand(0), node1]])], - [tag3, new Map([[brand(0), node1]])], + [undefined, new Map([[brand(0), encodedChunkFromCursor(node1)]])], + [tag3, new Map([[brand(0), encodedChunkFromCursor(node1)]])], ]), }, tag1, @@ -546,8 +555,8 @@ describe("ModularChangeFamily", () => { { fieldChanges: new Map([]), builds: new Map([ - [undefined, new Map([[brand(2), node1]])], - [tag3, new Map([[brand(2), node1]])], + [undefined, new Map([[brand(2), encodedChunkFromCursor(node1)]])], + [tag3, new Map([[brand(2), encodedChunkFromCursor(node1)]])], ]), revisions: [{ revision: tag2 }], }, @@ -561,13 +570,13 @@ describe("ModularChangeFamily", () => { const expected: ModularChangeset = { fieldChanges: new Map(), builds: new Map([ - [tag1, new Map([[brand(0), node1]])], - [tag2, new Map([[brand(2), node1]])], + [tag1, new Map([[brand(0), encodedChunkFromCursor(node1)]])], + [tag2, new Map([[brand(2), encodedChunkFromCursor(node1)]])], [ tag3, new Map([ - [brand(0), node1], - [brand(2), node1], + [brand(0), encodedChunkFromCursor(node1)], + [brand(2), encodedChunkFromCursor(node1)], ]), ], ]), @@ -734,3 +743,7 @@ describe("ModularChangeFamily", () => { assert.deepEqual(changes, [expectedChange]); }); }); + +function encodedChunkFromCursor(cursor: ITreeCursorSynchronous): EncodedChunk { + return uncompressedEncode(chunkTree(cursor, defaultChunkPolicy).cursor()); +} diff --git a/experimental/dds/tree2/src/test/feature-libraries/modularChangeFamilyIntegration.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/modularChangeFamilyIntegration.spec.ts index 935f739040ee..51b6829d970c 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/modularChangeFamilyIntegration.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/modularChangeFamilyIntegration.spec.ts @@ -29,7 +29,7 @@ import { } from "../../feature-libraries"; import { brand, IdAllocator, idAllocatorFromMaxId, Mutable } from "../../util"; -import { defaultRevisionMetadataFromChanges, testChangeReceiver } from "../utils"; +import { assertDeltaEqual, defaultRevisionMetadataFromChanges, testChangeReceiver } from "../utils"; import { intoDelta, ModularChangeFamily, @@ -244,7 +244,7 @@ describe("ModularChangeFamily integration", () => { }; const delta = intoDelta(makeAnonChange(composed), family.fieldKinds); - assert.deepEqual(delta, expected); + assertDeltaEqual(delta, expected); }); it("cross-field move and inverse with nested changes", () => { @@ -309,7 +309,7 @@ describe("ModularChangeFamily integration", () => { ], ]), }; - assert.deepEqual(actual, expected); + assertDeltaEqual(actual, expected); }); it("two cross-field moves of same node", () => { diff --git a/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts b/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts index d3f6ab025e5a..f924d6cb522d 100644 --- a/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts +++ b/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts @@ -207,10 +207,7 @@ describe("Editing", () => { insert(tree2, 1, "C"); tree3.editor .sequenceField(rootField) - .insert( - 0, - cursorForJsonableTreeNode({ type: jsonObject.name, fields: { foo: [] } }), - ); + .insert(0, cursorForJsonableTreeNode({ type: jsonObject.name })); const aEditor = tree3.editor.sequenceField({ parent: rootNode, field: brand("foo") }); aEditor.insert(0, cursorForJsonableTreeNode({ type: leaf.string.name, value: "a" })); diff --git a/experimental/dds/tree2/src/test/snapshots/files/complete-3x3-final.json b/experimental/dds/tree2/src/test/snapshots/files/complete-3x3-final.json index 50c1953bb045..d9cc0ce9c119 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/complete-3x3-final.json +++ b/experimental/dds/tree2/src/test/snapshots/files/complete-3x3-final.json @@ -41,7 +41,27 @@ [ 0, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -89,8 +109,28 @@ [ 1, { - "type": "com.fluidframework.leaf.string", - "value": "1" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "1", + [] + ] + ] } ] ] @@ -141,8 +181,28 @@ [ 2, { - "type": "com.fluidframework.leaf.string", - "value": "2" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "2", + [] + ] + ] } ] ] @@ -193,8 +253,28 @@ [ 3, { - "type": "com.fluidframework.leaf.string", - "value": "3" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "3", + [] + ] + ] } ] ] @@ -242,8 +322,28 @@ [ 4, { - "type": "com.fluidframework.leaf.string", - "value": "4" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "4", + [] + ] + ] } ] ] @@ -294,8 +394,28 @@ [ 5, { - "type": "com.fluidframework.leaf.string", - "value": "5" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "5", + [] + ] + ] } ] ] @@ -346,8 +466,28 @@ [ 6, { - "type": "com.fluidframework.leaf.string", - "value": "6" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "6", + [] + ] + ] } ] ] @@ -395,8 +535,28 @@ [ 7, { - "type": "com.fluidframework.leaf.string", - "value": "7" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "7", + [] + ] + ] } ] ] @@ -447,8 +607,28 @@ [ 8, { - "type": "com.fluidframework.leaf.string", - "value": "8" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "8", + [] + ] + ] } ] ] @@ -499,8 +679,28 @@ [ 9, { - "type": "com.fluidframework.leaf.string", - "value": "9" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "9", + [] + ] + ] } ] ] @@ -538,7 +738,27 @@ [ 10, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -586,8 +806,28 @@ [ 11, { - "type": "com.fluidframework.leaf.string", - "value": "10" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "10", + [] + ] + ] } ] ] @@ -638,8 +878,28 @@ [ 12, { - "type": "com.fluidframework.leaf.string", - "value": "11" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "11", + [] + ] + ] } ] ] @@ -690,8 +950,28 @@ [ 13, { - "type": "com.fluidframework.leaf.string", - "value": "12" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "12", + [] + ] + ] } ] ] @@ -739,8 +1019,28 @@ [ 14, { - "type": "com.fluidframework.leaf.string", - "value": "13" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "13", + [] + ] + ] } ] ] @@ -791,8 +1091,28 @@ [ 15, { - "type": "com.fluidframework.leaf.string", - "value": "14" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "14", + [] + ] + ] } ] ] @@ -843,8 +1163,28 @@ [ 16, { - "type": "com.fluidframework.leaf.string", - "value": "15" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "15", + [] + ] + ] } ] ] @@ -892,8 +1232,28 @@ [ 17, { - "type": "com.fluidframework.leaf.string", - "value": "16" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "16", + [] + ] + ] } ] ] @@ -944,8 +1304,28 @@ [ 18, { - "type": "com.fluidframework.leaf.string", - "value": "17" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "17", + [] + ] + ] } ] ] @@ -996,8 +1376,28 @@ [ 19, { - "type": "com.fluidframework.leaf.string", - "value": "18" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "18", + [] + ] + ] } ] ] @@ -1035,7 +1435,27 @@ [ 20, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -1083,8 +1503,28 @@ [ 21, { - "type": "com.fluidframework.leaf.string", - "value": "19" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "19", + [] + ] + ] } ] ] @@ -1135,8 +1575,28 @@ [ 22, { - "type": "com.fluidframework.leaf.string", - "value": "20" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "20", + [] + ] + ] } ] ] @@ -1187,8 +1647,28 @@ [ 23, { - "type": "com.fluidframework.leaf.string", - "value": "21" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "21", + [] + ] + ] } ] ] @@ -1236,8 +1716,28 @@ [ 24, { - "type": "com.fluidframework.leaf.string", - "value": "22" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "22", + [] + ] + ] } ] ] @@ -1288,8 +1788,28 @@ [ 25, { - "type": "com.fluidframework.leaf.string", - "value": "23" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "23", + [] + ] + ] } ] ] @@ -1340,8 +1860,28 @@ [ 26, { - "type": "com.fluidframework.leaf.string", - "value": "24" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "24", + [] + ] + ] } ] ] @@ -1389,8 +1929,28 @@ [ 27, { - "type": "com.fluidframework.leaf.string", - "value": "25" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "25", + [] + ] + ] } ] ] @@ -1441,8 +2001,28 @@ [ 28, { - "type": "com.fluidframework.leaf.string", - "value": "26" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "26", + [] + ] + ] } ] ] @@ -1493,8 +2073,28 @@ [ 29, { - "type": "com.fluidframework.leaf.string", - "value": "27" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "27", + [] + ] + ] } ] ] @@ -1529,7 +2129,27 @@ [ 30, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -1577,8 +2197,28 @@ [ 31, { - "type": "com.fluidframework.leaf.string", - "value": "28" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "28", + [] + ] + ] } ] ] @@ -1629,8 +2269,28 @@ [ 32, { - "type": "com.fluidframework.leaf.string", - "value": "29" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "29", + [] + ] + ] } ] ] @@ -1681,8 +2341,28 @@ [ 33, { - "type": "com.fluidframework.leaf.string", - "value": "30" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "30", + [] + ] + ] } ] ] @@ -1730,8 +2410,28 @@ [ 34, { - "type": "com.fluidframework.leaf.string", - "value": "31" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "31", + [] + ] + ] } ] ] @@ -1782,8 +2482,28 @@ [ 35, { - "type": "com.fluidframework.leaf.string", - "value": "32" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "32", + [] + ] + ] } ] ] @@ -1834,8 +2554,28 @@ [ 36, { - "type": "com.fluidframework.leaf.string", - "value": "33" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "33", + [] + ] + ] } ] ] @@ -1883,8 +2623,28 @@ [ 37, { - "type": "com.fluidframework.leaf.string", - "value": "34" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "34", + [] + ] + ] } ] ] @@ -1935,8 +2695,28 @@ [ 38, { - "type": "com.fluidframework.leaf.string", - "value": "35" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "35", + [] + ] + ] } ] ] @@ -1987,8 +2767,28 @@ [ 39, { - "type": "com.fluidframework.leaf.string", - "value": "36" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "36", + [] + ] + ] } ] ] @@ -2026,7 +2826,27 @@ [ 40, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -2074,8 +2894,28 @@ [ 41, { - "type": "com.fluidframework.leaf.string", - "value": "37" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "37", + [] + ] + ] } ] ] @@ -2126,8 +2966,28 @@ [ 42, { - "type": "com.fluidframework.leaf.string", - "value": "38" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "38", + [] + ] + ] } ] ] @@ -2178,8 +3038,28 @@ [ 43, { - "type": "com.fluidframework.leaf.string", - "value": "39" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "39", + [] + ] + ] } ] ] @@ -2227,8 +3107,28 @@ [ 44, { - "type": "com.fluidframework.leaf.string", - "value": "40" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "40", + [] + ] + ] } ] ] @@ -2279,8 +3179,28 @@ [ 45, { - "type": "com.fluidframework.leaf.string", - "value": "41" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "41", + [] + ] + ] } ] ] @@ -2331,8 +3251,28 @@ [ 46, { - "type": "com.fluidframework.leaf.string", - "value": "42" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "42", + [] + ] + ] } ] ] @@ -2380,8 +3320,28 @@ [ 47, { - "type": "com.fluidframework.leaf.string", - "value": "43" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "43", + [] + ] + ] } ] ] @@ -2432,8 +3392,28 @@ [ 48, { - "type": "com.fluidframework.leaf.string", - "value": "44" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "44", + [] + ] + ] } ] ] @@ -2484,8 +3464,28 @@ [ 49, { - "type": "com.fluidframework.leaf.string", - "value": "45" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "45", + [] + ] + ] } ] ] @@ -2523,7 +3523,27 @@ [ 50, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -2571,8 +3591,28 @@ [ 51, { - "type": "com.fluidframework.leaf.string", - "value": "46" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "46", + [] + ] + ] } ] ] @@ -2623,8 +3663,28 @@ [ 52, { - "type": "com.fluidframework.leaf.string", - "value": "47" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "47", + [] + ] + ] } ] ] @@ -2675,8 +3735,28 @@ [ 53, { - "type": "com.fluidframework.leaf.string", - "value": "48" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "48", + [] + ] + ] } ] ] @@ -2724,8 +3804,28 @@ [ 54, { - "type": "com.fluidframework.leaf.string", - "value": "49" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "49", + [] + ] + ] } ] ] @@ -2776,8 +3876,28 @@ [ 55, { - "type": "com.fluidframework.leaf.string", - "value": "50" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "50", + [] + ] + ] } ] ] @@ -2828,8 +3948,28 @@ [ 56, { - "type": "com.fluidframework.leaf.string", - "value": "51" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "51", + [] + ] + ] } ] ] @@ -2877,8 +4017,28 @@ [ 57, { - "type": "com.fluidframework.leaf.string", - "value": "52" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "52", + [] + ] + ] } ] ] @@ -2929,8 +4089,28 @@ [ 58, { - "type": "com.fluidframework.leaf.string", - "value": "53" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "53", + [] + ] + ] } ] ] @@ -2981,8 +4161,28 @@ [ 59, { - "type": "com.fluidframework.leaf.string", - "value": "54" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "54", + [] + ] + ] } ] ] @@ -3017,7 +4217,27 @@ [ 60, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -3065,8 +4285,28 @@ [ 61, { - "type": "com.fluidframework.leaf.string", - "value": "55" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "55", + [] + ] + ] } ] ] @@ -3117,8 +4357,28 @@ [ 62, { - "type": "com.fluidframework.leaf.string", - "value": "56" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "56", + [] + ] + ] } ] ] @@ -3169,8 +4429,28 @@ [ 63, { - "type": "com.fluidframework.leaf.string", - "value": "57" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "57", + [] + ] + ] } ] ] @@ -3218,8 +4498,28 @@ [ 64, { - "type": "com.fluidframework.leaf.string", - "value": "58" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "58", + [] + ] + ] } ] ] @@ -3270,8 +4570,28 @@ [ 65, { - "type": "com.fluidframework.leaf.string", - "value": "59" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "59", + [] + ] + ] } ] ] @@ -3322,8 +4642,28 @@ [ 66, { - "type": "com.fluidframework.leaf.string", - "value": "60" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "60", + [] + ] + ] } ] ] @@ -3371,8 +4711,28 @@ [ 67, { - "type": "com.fluidframework.leaf.string", - "value": "61" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "61", + [] + ] + ] } ] ] @@ -3423,8 +4783,28 @@ [ 68, { - "type": "com.fluidframework.leaf.string", - "value": "62" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "62", + [] + ] + ] } ] ] @@ -3475,8 +4855,28 @@ [ 69, { - "type": "com.fluidframework.leaf.string", - "value": "63" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "63", + [] + ] + ] } ] ] @@ -3514,7 +4914,27 @@ [ 70, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -3562,8 +4982,28 @@ [ 71, { - "type": "com.fluidframework.leaf.string", - "value": "64" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "64", + [] + ] + ] } ] ] @@ -3614,8 +5054,28 @@ [ 72, { - "type": "com.fluidframework.leaf.string", - "value": "65" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "65", + [] + ] + ] } ] ] @@ -3666,8 +5126,28 @@ [ 73, { - "type": "com.fluidframework.leaf.string", - "value": "66" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "66", + [] + ] + ] } ] ] @@ -3715,8 +5195,28 @@ [ 74, { - "type": "com.fluidframework.leaf.string", - "value": "67" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "67", + [] + ] + ] } ] ] @@ -3767,8 +5267,28 @@ [ 75, { - "type": "com.fluidframework.leaf.string", - "value": "68" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "68", + [] + ] + ] } ] ] @@ -3819,8 +5339,28 @@ [ 76, { - "type": "com.fluidframework.leaf.string", - "value": "69" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "69", + [] + ] + ] } ] ] @@ -3868,8 +5408,28 @@ [ 77, { - "type": "com.fluidframework.leaf.string", - "value": "70" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "70", + [] + ] + ] } ] ] @@ -3920,8 +5480,28 @@ [ 78, { - "type": "com.fluidframework.leaf.string", - "value": "71" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "71", + [] + ] + ] } ] ] @@ -3972,8 +5552,28 @@ [ 79, { - "type": "com.fluidframework.leaf.string", - "value": "72" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "72", + [] + ] + ] } ] ] @@ -4011,7 +5611,27 @@ [ 80, { - "type": "test trees.TestInner" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "test trees.TestInner", + false, + [] + ] + ] } ] ] @@ -4059,8 +5679,28 @@ [ 81, { - "type": "com.fluidframework.leaf.string", - "value": "73" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "73", + [] + ] + ] } ] ] @@ -4111,8 +5751,28 @@ [ 82, { - "type": "com.fluidframework.leaf.string", - "value": "74" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "74", + [] + ] + ] } ] ] @@ -4163,8 +5823,28 @@ [ 83, { - "type": "com.fluidframework.leaf.string", - "value": "75" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "75", + [] + ] + ] } ] ] @@ -4212,8 +5892,28 @@ [ 84, { - "type": "com.fluidframework.leaf.string", - "value": "76" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "76", + [] + ] + ] } ] ] @@ -4264,8 +5964,28 @@ [ 85, { - "type": "com.fluidframework.leaf.string", - "value": "77" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "77", + [] + ] + ] } ] ] @@ -4316,8 +6036,28 @@ [ 86, { - "type": "com.fluidframework.leaf.string", - "value": "78" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "78", + [] + ] + ] } ] ] @@ -4365,8 +6105,28 @@ [ 87, { - "type": "com.fluidframework.leaf.string", - "value": "79" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "79", + [] + ] + ] } ] ] @@ -4417,8 +6177,28 @@ [ 88, { - "type": "com.fluidframework.leaf.string", - "value": "80" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "80", + [] + ] + ] } ] ] @@ -4469,8 +6249,28 @@ [ 89, { - "type": "com.fluidframework.leaf.string", - "value": "81" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "81", + [] + ] + ] } ] ] diff --git a/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree2.json b/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree2.json index bb69a0329190..cd90a7167c03 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree2.json +++ b/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree2.json @@ -41,8 +41,28 @@ [ 0, { - "type": "com.fluidframework.leaf.string", - "value": "y" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "y", + [] + ] + ] } ] ] @@ -77,8 +97,28 @@ [ 1, { - "type": "com.fluidframework.leaf.string", - "value": "x" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "x", + [] + ] + ] } ] ] @@ -116,15 +156,55 @@ [ 0, { - "type": "com.fluidframework.leaf.string", - "value": "a" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "a", + [] + ] + ] } ], [ 1, { - "type": "com.fluidframework.leaf.string", - "value": "c" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "c", + [] + ] + ] } ] ] @@ -162,8 +242,28 @@ [ 2, { - "type": "com.fluidframework.leaf.string", - "value": "b" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "b", + [] + ] + ] } ] ] diff --git a/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree3.json b/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree3.json index 7f29f5798e78..1613c65801ba 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree3.json +++ b/experimental/dds/tree2/src/test/snapshots/files/concurrent-inserts-tree3.json @@ -41,8 +41,28 @@ [ 0, { - "type": "com.fluidframework.leaf.string", - "value": "y" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "y", + [] + ] + ] } ] ] @@ -77,8 +97,28 @@ [ 1, { - "type": "com.fluidframework.leaf.string", - "value": "x" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "x", + [] + ] + ] } ] ] @@ -116,15 +156,55 @@ [ 0, { - "type": "com.fluidframework.leaf.string", - "value": "a" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "a", + [] + ] + ] } ], [ 1, { - "type": "com.fluidframework.leaf.string", - "value": "c" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "c", + [] + ] + ] } ] ] @@ -162,8 +242,28 @@ [ 2, { - "type": "com.fluidframework.leaf.string", - "value": "b" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "b", + [] + ] + ] } ] ] @@ -198,8 +298,28 @@ [ 2, { - "type": "com.fluidframework.leaf.string", - "value": "z" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "z", + [] + ] + ] } ] ] @@ -237,15 +357,55 @@ [ 0, { - "type": "com.fluidframework.leaf.string", - "value": "d" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "d", + [] + ] + ] } ], [ 1, { - "type": "com.fluidframework.leaf.string", - "value": "e" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "e", + [] + ] + ] } ] ] @@ -283,8 +443,28 @@ [ 2, { - "type": "com.fluidframework.leaf.string", - "value": "f" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "f", + [] + ] + ] } ] ] diff --git a/experimental/dds/tree2/src/test/snapshots/files/insert-and-delete-tree-0-after-insert.json b/experimental/dds/tree2/src/test/snapshots/files/insert-and-delete-tree-0-after-insert.json index 14fa46b4c7a3..f69831606051 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/insert-and-delete-tree-0-after-insert.json +++ b/experimental/dds/tree2/src/test/snapshots/files/insert-and-delete-tree-0-after-insert.json @@ -41,8 +41,28 @@ [ 0, { - "type": "com.fluidframework.leaf.string", - "value": "42" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.string", + true, + "42", + [] + ] + ] } ] ] diff --git a/experimental/dds/tree2/src/test/snapshots/files/move-across-fields-tree-0-final.json b/experimental/dds/tree2/src/test/snapshots/files/move-across-fields-tree-0-final.json index 0ceb9b5cd578..65b81538721d 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/move-across-fields-tree-0-final.json +++ b/experimental/dds/tree2/src/test/snapshots/files/move-across-fields-tree-0-final.json @@ -41,37 +41,58 @@ [ 0, { - "type": "Node", - "fields": { - "foo": [ - { - "type": "Node", - "value": "a" - }, - { - "type": "Node", - "value": "b" - }, - { - "type": "Node", - "value": "c" - } - ], - "bar": [ - { - "type": "Node", - "value": "d" - }, - { - "type": "Node", - "value": "e" - }, - { - "type": "Node", - "value": "f" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "Node", + false, + [ + "foo", + [ + "Node", + true, + "a", + [], + "Node", + true, + "b", + [], + "Node", + true, + "c", + [] + ], + "bar", + [ + "Node", + true, + "d", + [], + "Node", + true, + "e", + [], + "Node", + true, + "f", + [] + ] + ] ] - } + ] } ] ] diff --git a/experimental/dds/tree2/src/test/snapshots/files/nested-sequence-change-final.json b/experimental/dds/tree2/src/test/snapshots/files/nested-sequence-change-final.json index af1ac05eea4b..31f1b44156ae 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/nested-sequence-change-final.json +++ b/experimental/dds/tree2/src/test/snapshots/files/nested-sequence-change-final.json @@ -62,13 +62,53 @@ [ 0, { - "type": "has-sequence-map.SeqMap" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "has-sequence-map.SeqMap", + false, + [] + ] + ] } ], [ 1, { - "type": "has-sequence-map.SeqMap" + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "has-sequence-map.SeqMap", + false, + [] + ] + ] } ] ] From ec1307589e69016ba4f6f3a1d34763e2fa4826a7 Mon Sep 17 00:00:00 2001 From: Mark Fields Date: Wed, 6 Dec 2023 18:27:14 -0800 Subject: [PATCH 06/20] GC: Two-Stage GC that runs Tombstone then Sweep, instead of only one or the other (#18559) This change updates GC configuration and logic to transition unreferenced objects to TombstoneReady (and next GC will mark as Tombstone) then after a configurable delay, transitions to SweepReady (and next GC will delete). Context: Based on feedback from our 1P early adopters of GC, it's desirable to see an object be Tombstoned first, then later swept. For a document being actively used that hits a GC-impacting bug, this will cause the bug to manifest as a Tombstone error first, giving some reaction time (operationally) before the data is permanently lost. --- .../api-report/container-runtime.api.md | 1 + .../src/gc/garbageCollection.md | 28 +-- .../src/gc/garbageCollection.ts | 46 ++-- .../container-runtime/src/gc/gcConfigs.ts | 20 +- .../container-runtime/src/gc/gcDefinitions.ts | 22 +- .../container-runtime/src/gc/gcTelemetry.ts | 20 +- .../src/gc/gcUnreferencedStateTracker.ts | 83 ++++++-- .../runtime/container-runtime/src/gc/index.ts | 1 + .../src/test/gc/garbageCollection.spec.ts | 152 +++++++------- .../src/test/gc/gcConfigs.spec.ts | 36 ++++ .../src/test/gc/gcTelemetry.spec.ts | 93 +++++++-- .../gc/gcUnreferencedStateTracker.spec.ts | 197 ++++++++++++------ .../src/validateAssertionError.ts | 6 +- .../src/test/gc/gcInactiveNodes.spec.ts | 2 +- .../test/gc/gcSweepAttachmentBlobs.spec.ts | 5 +- .../src/test/gc/gcSweepDataStores.spec.ts | 5 +- .../test/gc/gcSweepUnreferencePhases.spec.ts | 133 ++++++------ .../src/test/gc/gcTestSummaryUtils.ts | 40 +++- .../gc/gcTombstoneAttachmentBlobs.spec.ts | 2 +- .../src/test/gc/gcTombstoneDataStores.spec.ts | 25 ++- .../gc/gcTombstoneUnreferencePhases.spec.ts | 143 ------------- .../src/test/gc/gcTrailingOps.spec.ts | 4 +- .../test-service-load/src/optionsMatrix.ts | 3 +- .../api-report/telemetry-utils.api.md | 3 + packages/utils/telemetry-utils/src/error.ts | 19 ++ packages/utils/telemetry-utils/src/index.ts | 1 + 26 files changed, 635 insertions(+), 455 deletions(-) delete mode 100644 packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneUnreferencePhases.spec.ts diff --git a/packages/runtime/container-runtime/api-report/container-runtime.api.md b/packages/runtime/container-runtime/api-report/container-runtime.api.md index 638aa115b402..e2a72b6ff3da 100644 --- a/packages/runtime/container-runtime/api-report/container-runtime.api.md +++ b/packages/runtime/container-runtime/api-report/container-runtime.api.md @@ -442,6 +442,7 @@ export interface IGCRuntimeOptions { gcAllowed?: boolean; runFullGC?: boolean; sessionExpiryTimeoutMs?: number; + sweepGracePeriodMs?: number; } // @alpha diff --git a/packages/runtime/container-runtime/src/gc/garbageCollection.md b/packages/runtime/container-runtime/src/gc/garbageCollection.md index 9d2c4555a8e3..9acf599d53f3 100644 --- a/packages/runtime/container-runtime/src/gc/garbageCollection.md +++ b/packages/runtime/container-runtime/src/gc/garbageCollection.md @@ -50,16 +50,19 @@ Mark phase is enabled by default for a container. It is enabled during creation ### Sweep phase -In this phase, the GC algorithm identifies all Fluid objects that have been unreferenced for a specific amount of time (typically 30-40 days) and deletes them. -Objects are only swept once the GC system is sure that they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it. -The Fluid Runtime enforces a maximum session length (configurable) in order to guarantee an object is safe to delete after sufficient time has elapsed. +In this phase, the GC algorithm deletes any Fluid object that has been unreferenced for a sufficient time to guarantee +they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it again. +The Fluid Runtime enforces a maximum session length (configurable) in order to guarantee all in-memory objects are cleared before +it concludes an object is safe to delete. -GC sweep phase has not been enabled by default yet. A "soft" version of Sweep called "Tombstone Mode" is enabled by default -as part of the Mark Phase when Sweep is disabled. In this mode, any object that GC determines is ready to be deleted is -marked as a "Tombstone", which triggers certain logging events and/or behavior changes if/when that Tombstoned object is -accessed by the application. +GC sweep phase runs in two stages: -Tombstone is intended for use by early adopters of GC and is documented in more detail [here](./gcEarlyAdoption.md). +- The first stage is the "Tombstone" stage, where objects are marked as Tombstones, meaning GC believes they will + never be referenced again and are safe to delete. They are not yet deleted at this point, but any attempt to + load them will fail. This way, there's a chance to recover a Tombstoned object in case we detect it's still being used. +- The second stage is the "Sweep" or "Delete" stage, where the objects are fully deleted. + This occurs after a configurable delay called the "Sweep Grace Period", to give time for application teams + to monitor for Tombstone-related errors and react before delete occurs. ## GC Configuration @@ -94,12 +97,11 @@ covered in the [Advanced Configuration](./gcEarlyAdoption.md#more-advanced-confi ### Enabling Sweep Phase -To enable Sweep Phase for new documents, you must set the `gcSweepGeneration` GC Option to a number, e.g. 0 to start. -The full semantics of this GC Option are discussed [here](./gcEarlyAdoption.md#more-about-gcsweepgeneration-and-gctombstonegeneration). -Note that this will disabled Tombstone Mode. +The Tombstone stage of Sweep is enabled by default. -A full treatment of Tombstone and Sweep configuration can be found in -[this companion document geared towards early adopters of GC](./gcEarlyAdoption.md). +To enable the Delete Stage for new documents, you must set the `gcSweepGeneration` GC Option to a number, e.g. 0 to start. +This generation number is persisted, and any document where the persisted value matches the current value will have +Sweep enabled. ### More Advanced Configuration diff --git a/packages/runtime/container-runtime/src/gc/garbageCollection.ts b/packages/runtime/container-runtime/src/gc/garbageCollection.ts index 26174fd51d10..addf3bd47390 100644 --- a/packages/runtime/container-runtime/src/gc/garbageCollection.ts +++ b/packages/runtime/container-runtime/src/gc/garbageCollection.ts @@ -418,6 +418,7 @@ export class GarbageCollector implements IGarbageCollector { this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, + this.configs.sweepGracePeriodMs, ), ); } @@ -573,7 +574,7 @@ export class GarbageCollector implements IGarbageCollector { // 3. Run the Mark phase. // It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept. - const sweepReadyNodeIds = this.runMarkPhase( + const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase( gcResult, allReferencedNodeIds, currentReferenceTimestampMs, @@ -581,7 +582,11 @@ export class GarbageCollector implements IGarbageCollector { // 4. Run the Sweep phase. // It will delete sweep ready nodes and return a list of deleted node ids. - const deletedNodeIds = this.runSweepPhase(gcResult, sweepReadyNodeIds); + const deletedNodeIds = this.runSweepPhase( + gcResult, + tombstoneReadyNodeIds, + sweepReadyNodeIds, + ); this.gcDataFromLastRun = cloneGCData( gcData, @@ -610,13 +615,13 @@ export class GarbageCollector implements IGarbageCollector { * @param gcResult - The result of the GC run on the gcData. * @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run. * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp. - * @returns A list of sweep ready nodes, i.e., nodes that ready to be deleted. + * @returns The sets of TombstoneReady and SweepReady nodes, i.e., nodes that ready to be tombstoned or deleted. */ private runMarkPhase( gcResult: IGCResult, allReferencedNodeIds: string[], currentReferenceTimestampMs: number, - ): Set { + ): { tombstoneReadyNodeIds: Set; sweepReadyNodeIds: Set } { // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any. for (const nodeId of allReferencedNodeIds) { const nodeStateTracker = this.unreferencedNodesState.get(nodeId); @@ -629,6 +634,7 @@ export class GarbageCollector implements IGarbageCollector { } // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them. + const tombstoneReadyNodeIds: Set = new Set(); const sweepReadyNodeIds: Set = new Set(); for (const nodeId of gcResult.deletedNodeIds) { const nodeStateTracker = this.unreferencedNodesState.get(nodeId); @@ -640,6 +646,7 @@ export class GarbageCollector implements IGarbageCollector { this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, + this.configs.sweepGracePeriodMs, ), ); } else { @@ -647,7 +654,10 @@ export class GarbageCollector implements IGarbageCollector { // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward. nodeStateTracker.updateTracking(currentReferenceTimestampMs); - // If a node is sweep ready, store it so it can be returned. + // If a node is tombstone or sweep ready, store it so it can be returned. + if (nodeStateTracker.state === UnreferencedState.TombstoneReady) { + tombstoneReadyNodeIds.add(nodeId); + } if (nodeStateTracker.state === UnreferencedState.SweepReady) { sweepReadyNodeIds.add(nodeId); } @@ -657,7 +667,7 @@ export class GarbageCollector implements IGarbageCollector { // 3. Call the runtime to update referenced nodes in this run. this.runtime.updateUsedRoutes(gcResult.referencedNodeIds); - return sweepReadyNodeIds; + return { tombstoneReadyNodeIds, sweepReadyNodeIds }; } /** @@ -666,20 +676,20 @@ export class GarbageCollector implements IGarbageCollector { * 2. Clears tracking for deleted nodes. * * @param gcResult - The result of the GC run on the gcData. + * @param tombstoneReadyNodes - List of nodes that are tombstone ready. * @param sweepReadyNodes - List of nodes that are sweep ready. - * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp. - * @param logger - The logger to be used to log any telemetry. * @returns A list of nodes that have been deleted. */ - private runSweepPhase(gcResult: IGCResult, sweepReadyNodes: Set): string[] { + private runSweepPhase( + gcResult: IGCResult, + tombstoneReadyNodes: Set, + sweepReadyNodes: Set, + ): string[] { /** - * Currently, there are 3 modes for sweep: - * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready. - * Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted. - * Sweep mode - Sweep ready modes are deleted. + * Under "Test Mode", Unreferenced nodes are immediately deleted without waiting for them to be sweep ready. * - * These modes serve as staging for applications that want to enable sweep by providing an incremental - * way to test and validate sweep works as expected. + * Otherwise, depending on how long it's been since the node was unreferenced, it will either be + * marked as Tombstone, or deleted by Sweep. */ if (this.configs.testMode) { // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted. @@ -688,11 +698,9 @@ export class GarbageCollector implements IGarbageCollector { } if (this.configs.tombstoneMode) { - this.tombstones = Array.from(sweepReadyNodes); - // If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios - // involving access to "deleted" data without actually deleting the data from summaries. + this.tombstones = Array.from(tombstoneReadyNodes); + // If we are running in GC tombstone mode, update tombstoned routes. this.runtime.updateTombstonedRoutes(this.tombstones); - return []; } if (!this.configs.shouldRunSweep) { diff --git a/packages/runtime/container-runtime/src/gc/gcConfigs.ts b/packages/runtime/container-runtime/src/gc/gcConfigs.ts index c7739a08cfd8..abce13151c2a 100644 --- a/packages/runtime/container-runtime/src/gc/gcConfigs.ts +++ b/packages/runtime/container-runtime/src/gc/gcConfigs.ts @@ -3,7 +3,11 @@ * Licensed under the MIT License. */ -import { MonitoringContext, UsageError } from "@fluidframework/telemetry-utils"; +import { + MonitoringContext, + UsageError, + validatePrecondition, +} from "@fluidframework/telemetry-utils"; import { IContainerRuntimeMetadata } from "../summary"; import { nextGCVersion, @@ -27,6 +31,7 @@ import { throwOnTombstoneLoadOverrideKey, throwOnTombstoneUsageKey, gcThrowOnTombstoneLoadOptionName, + defaultSweepGracePeriodMs, } from "./gcDefinitions"; import { getGCVersion, shouldAllowGcSweep, shouldAllowGcTombstoneEnforcement } from "./gcHelpers"; @@ -159,11 +164,17 @@ export function generateGCConfigs( // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted. const testMode = mc.config.getBoolean(gcTestModeKey) ?? createParams.gcOptions.runGCInTestMode === true; - // Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled - // via feature flags. - const tombstoneMode = !shouldRunSweep && mc.config.getBoolean(disableTombstoneKey) !== true; + // Whether we are running in tombstone mode. If disabled, tombstone data will not be written to or read from snapshots, + // and objects will not be marked as tombstoned even if they pass to the "TombstoneReady" state during the session. + const tombstoneMode = mc.config.getBoolean(disableTombstoneKey) !== true; const runFullGC = createParams.gcOptions.runFullGC; + const sweepGracePeriodMs = + createParams.gcOptions.sweepGracePeriodMs ?? defaultSweepGracePeriodMs; + validatePrecondition(sweepGracePeriodMs >= 0, "sweepGracePeriodMs must be non-negative", { + sweepGracePeriodMs, + }); + const throwOnInactiveLoad: boolean | undefined = createParams.gcOptions.throwOnInactiveLoad; const tombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement( createParams.metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */, @@ -193,6 +204,7 @@ export function generateGCConfigs( tombstoneMode, sessionExpiryTimeoutMs, sweepTimeoutMs, + sweepGracePeriodMs, inactiveTimeoutMs, persistedGcFeatureMatrix, gcVersionInBaseSnapshot, diff --git a/packages/runtime/container-runtime/src/gc/gcDefinitions.ts b/packages/runtime/container-runtime/src/gc/gcDefinitions.ts index 6930ea3cbe7d..3d7cabe4850e 100644 --- a/packages/runtime/container-runtime/src/gc/gcDefinitions.ts +++ b/packages/runtime/container-runtime/src/gc/gcDefinitions.ts @@ -89,6 +89,7 @@ export const maxSnapshotCacheExpiryMs = 5 * oneDayMs; export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days +export const defaultSweepGracePeriodMs = 1 * oneDayMs; // 1 day /** * @see IGCMetadata.gcFeatureMatrix @@ -354,6 +355,13 @@ export interface IGCRuntimeOptions { */ sessionExpiryTimeoutMs?: number; + /** + * Delay between when Tombstone should run and when the object should be deleted. + * This grace period gives a chance to intervene to recover if needed, before Sweep deletes the object. + * If not present, a default (non-zero) value will be used. + */ + sweepGracePeriodMs?: number; + /** * Allows additional GC options to be passed. */ @@ -392,15 +400,21 @@ export interface IGarbageCollectorConfigs { readonly sessionExpiryTimeoutMs: number | undefined; /** The time after which an unreferenced node is ready to be swept. */ readonly sweepTimeoutMs: number | undefined; + /** + * The delay between tombstone and sweep. Not persisted, so concurrent sessions may use different values. + * Sweep is implemented in an eventually-consistent way so this is acceptable. + */ + readonly sweepGracePeriodMs: number; /** The time after which an unreferenced node is inactive. */ readonly inactiveTimeoutMs: number; /** Tracks whether GC should run in test mode. In this mode, unreferenced objects are deleted immediately. */ readonly testMode: boolean; /** - * Tracks whether GC should run in tombstone mode. In this mode, sweep ready objects are marked as tombstones. + * Tracks whether GC should run in tombstone mode. In this mode, objects are marked as tombstones as a step along the + * way before they are fully deleted. * In interactive (non-summarizer) clients, tombstone objects behave as if they are deleted, i.e., access to them - * is not allowed. However, these objects can be accessed after referencing them first. It is used as a staging - * step for sweep where accidental sweep ready objects can be recovered. + * is not allowed. However, these objects can be accessed after referencing them first. It is used as a "warning" + * step before sweep, where objects wrongly marked as unreferenced can be recovered. */ readonly tombstoneMode: boolean; /** @see GCFeatureMatrix. */ @@ -425,6 +439,8 @@ export const UnreferencedState = { Active: "Active", /** The node is inactive, i.e., it should not become referenced. */ Inactive: "Inactive", + /** The node is ready to be tombstoned */ + TombstoneReady: "TombstoneReady", /** The node is ready to be deleted by the sweep phase. */ SweepReady: "SweepReady", } as const; diff --git a/packages/runtime/container-runtime/src/gc/gcTelemetry.ts b/packages/runtime/container-runtime/src/gc/gcTelemetry.ts index 70930395f15f..7140f4a011a5 100644 --- a/packages/runtime/container-runtime/src/gc/gcTelemetry.ts +++ b/packages/runtime/container-runtime/src/gc/gcTelemetry.ts @@ -142,6 +142,21 @@ export class GCTelemetryTracker { const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id); const nodeType = this.getNodeType(nodeUsageProps.id); + const timeout = (() => { + switch (nodeStateTracker?.state) { + case UnreferencedState.Inactive: + return this.configs.inactiveTimeoutMs; + case UnreferencedState.TombstoneReady: + return this.configs.sweepTimeoutMs; + case UnreferencedState.SweepReady: + return ( + this.configs.sweepTimeoutMs && + this.configs.sweepTimeoutMs + this.configs.sweepGracePeriodMs + ); + default: + return undefined; + } + })(); const { usageType, currentReferenceTimestampMs, @@ -159,10 +174,7 @@ export class GCTelemetryTracker { ? nodeUsageProps.currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs : -1, - timeout: - nodeStateTracker?.state === UnreferencedState.Inactive - ? this.configs.inactiveTimeoutMs - : this.configs.sweepTimeoutMs, + timeout, ...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }), ...propsToLog, ...this.createContainerMetadata, diff --git a/packages/runtime/container-runtime/src/gc/gcUnreferencedStateTracker.ts b/packages/runtime/container-runtime/src/gc/gcUnreferencedStateTracker.ts index e4eb86260e57..35276dcdd26c 100644 --- a/packages/runtime/container-runtime/src/gc/gcUnreferencedStateTracker.ts +++ b/packages/runtime/container-runtime/src/gc/gcUnreferencedStateTracker.ts @@ -4,6 +4,7 @@ */ import { assert, Timer } from "@fluidframework/core-utils"; +import { validatePrecondition } from "@fluidframework/telemetry-utils"; import { UnreferencedState } from "./gcDefinitions"; /** A wrapper around common-utils Timer that requires the timeout when calling start/restart */ @@ -26,7 +27,7 @@ class TimerWithNoDefaultTimeout extends Timer { /** * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can - * be deleted by the sweep phase. + * be tombstoned or deleted by the sweep phase. */ export class UnreferencedStateTracker { private _state: UnreferencedState = UnreferencedState.Active; @@ -36,6 +37,8 @@ export class UnreferencedStateTracker { /** Timer to indicate when an unreferenced object is considered Inactive */ private readonly inactiveTimer: TimerWithNoDefaultTimeout; + /** Timer to indicate when an unreferenced object is Tombstone-Ready */ + private readonly tombstoneTimer: TimerWithNoDefaultTimeout; /** Timer to indicate when an unreferenced object is Sweep-Ready */ private readonly sweepTimer: TimerWithNoDefaultTimeout; @@ -45,32 +48,49 @@ export class UnreferencedStateTracker { private readonly inactiveTimeoutMs: number, /** The current reference timestamp used to track how long this node has been unreferenced for. */ currentReferenceTimestampMs: number, - /** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */ - private readonly sweepTimeoutMs: number | undefined, + /** The time after which node transitions to TombstoneReady state; undefined if session expiry is disabled. */ + private readonly tombstoneTimeoutMs: number | undefined, + /** The delay from TombstoneReady to SweepReady (only applies if tombstoneTimeoutMs is defined) */ + private readonly sweepGracePeriodMs: number, ) { - if (this.sweepTimeoutMs !== undefined) { - assert( - this.inactiveTimeoutMs <= this.sweepTimeoutMs, - 0x3b0 /* inactive timeout must not be greater than the sweep timeout */, - ); - } + validatePrecondition( + this.tombstoneTimeoutMs === undefined || + this.tombstoneTimeoutMs >= this.inactiveTimeoutMs, + "inactiveTimeoutMs must not be greater than the tombstoneTimeoutMs", + ); this.sweepTimer = new TimerWithNoDefaultTimeout(() => { this._state = UnreferencedState.SweepReady; assert( - !this.inactiveTimer.hasTimer, - 0x3b1 /* inactiveTimer still running after sweepTimer fired! */, + !this.inactiveTimer.hasTimer && !this.tombstoneTimer.hasTimer, + "inactiveTimer or tombstoneTimer still running after sweepTimer fired!", ); }); + this.tombstoneTimer = new TimerWithNoDefaultTimeout(() => { + this._state = UnreferencedState.TombstoneReady; + assert( + !this.inactiveTimer.hasTimer, + "inactiveTimer still running after tombstoneTimer fired!", + ); // aka 0x3b1 + + if (this.sweepGracePeriodMs > 0) { + // After the node becomes tombstone ready, start the sweep timer after which the node will be ready for sweep. + this.sweepTimer.restart(this.sweepGracePeriodMs); + } else { + this._state = UnreferencedState.SweepReady; + } + }); + this.inactiveTimer = new TimerWithNoDefaultTimeout(() => { this._state = UnreferencedState.Inactive; - // After the node becomes inactive, start the sweep timer after which the node will be ready for sweep. - if (this.sweepTimeoutMs !== undefined) { - this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs); + // After the node becomes inactive, start the tombstone timer after which the node will be ready for tombstone. + if (this.tombstoneTimeoutMs !== undefined) { + this.tombstoneTimer.restart(this.tombstoneTimeoutMs - this.inactiveTimeoutMs); } }); + this.updateTracking(currentReferenceTimestampMs); } @@ -78,21 +98,39 @@ export class UnreferencedStateTracker { public updateTracking(currentReferenceTimestampMs: number) { const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs; - // If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady. - if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) { + // Below we will set the appropriate timer (or none). Any running timers are superceded by the new currentReferenceTimestampMs + this.clearTimers(); + + // If the node has been unreferenced long enough, update the state to SweepReady. + if ( + this.tombstoneTimeoutMs !== undefined && + unreferencedDurationMs >= this.tombstoneTimeoutMs + this.sweepGracePeriodMs + ) { this._state = UnreferencedState.SweepReady; - this.clearTimers(); return; } - // If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive. - // Also, start a timer for the sweep timeout. + // If the node has been unreferenced long enough, update the state to TombstoneReady. + // Also, start a timer for the remainder of the sweep delay. + if ( + this.tombstoneTimeoutMs !== undefined && + unreferencedDurationMs >= this.tombstoneTimeoutMs + ) { + this._state = UnreferencedState.TombstoneReady; + + this.sweepTimer.restart( + this.tombstoneTimeoutMs + this.sweepGracePeriodMs - unreferencedDurationMs, + ); + return; + } + + // If the node has been unreferenced for long enough, update the state to inactive. + // Also, start a timer for the remainder of the tombstone timeout. if (unreferencedDurationMs >= this.inactiveTimeoutMs) { this._state = UnreferencedState.Inactive; - this.inactiveTimer.clear(); - if (this.sweepTimeoutMs !== undefined) { - this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs); + if (this.tombstoneTimeoutMs !== undefined) { + this.tombstoneTimer.restart(this.tombstoneTimeoutMs - unreferencedDurationMs); } return; } @@ -103,6 +141,7 @@ export class UnreferencedStateTracker { private clearTimers() { this.inactiveTimer.clear(); + this.tombstoneTimer.clear(); this.sweepTimer.clear(); } diff --git a/packages/runtime/container-runtime/src/gc/index.ts b/packages/runtime/container-runtime/src/gc/index.ts index 70f8d0962d12..9d7204878bc7 100644 --- a/packages/runtime/container-runtime/src/gc/index.ts +++ b/packages/runtime/container-runtime/src/gc/index.ts @@ -7,6 +7,7 @@ export { GarbageCollector } from "./garbageCollection"; export { nextGCVersion, defaultInactiveTimeoutMs, + defaultSweepGracePeriodMs, defaultSessionExpiryDurationMs, GCNodeType, gcTestModeKey, diff --git a/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts b/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts index 315cda32286f..bdeef60de283 100644 --- a/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts +++ b/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts @@ -48,6 +48,9 @@ import { IGarbageCollectionSnapshotData, IGCStats, IGCRuntimeOptions, + UnreferencedStateTracker, + UnreferencedState, + defaultSweepGracePeriodMs, } from "../../gc"; import { dataStoreAttributesBlobName, @@ -70,11 +73,13 @@ type GcWithPrivates = IGarbageCollector & { readonly baseSnapshotDataP: Promise; readonly tombstones: string[]; readonly deletedNodes: Set; + readonly unreferencedNodesState: Map; }; describe("Garbage Collection Tests", () => { const defaultSnapshotCacheExpiryMs = 5 * 24 * 60 * 60 * 1000; - const sweepTimeoutMs = defaultSessionExpiryDurationMs + defaultSnapshotCacheExpiryMs + oneDayMs; + const defaultSweepTimeoutMs = + defaultSessionExpiryDurationMs + defaultSnapshotCacheExpiryMs + oneDayMs; // Nodes in the reference graph. const nodes: string[] = ["/node1", "/node2", "/node3", "/node4", "/node5", "/node6"]; const testPkgPath = ["testPkg"]; @@ -245,10 +250,11 @@ describe("Garbage Collection Tests", () => { const summarizerContainerTests = ( timeout: number, - mode: "inactive" | "sweep", + mode: "inactive" | "tombstone" | "sweep", revivedEventName: string, changedEventName: string, loadedEventName: string, + sweepGracePeriodMsOverride?: number, ) => { // Validates that no unexpected event has been fired. function validateNoEvents() { @@ -262,12 +268,21 @@ describe("Garbage Collection Tests", () => { ); } + const sweepGracePeriodMs = sweepGracePeriodMsOverride ?? defaultSweepGracePeriodMs; + const createGCOverride = ( baseSnapshot?: ISnapshotTree, gcBlobsMap?: Map, ) => { - return createGarbageCollector({ baseSnapshot }, gcBlobsMap, { - sweepTimeoutMs: mode === "sweep" ? timeout : undefined, + const sweepTimeoutMs = + mode === "tombstone" + ? timeout + : mode === "sweep" + ? timeout - sweepGracePeriodMs + : undefined; + const gcOptions = { sweepGracePeriodMs }; + return createGarbageCollector({ baseSnapshot, gcOptions }, gcBlobsMap, { + sweepTimeoutMs, }); }; @@ -702,9 +717,30 @@ describe("Garbage Collection Tests", () => { ); }); - describe("SweepReady events (summarizer container)", () => { + describe("TombstoneReady events (summarizer container)", () => { + summarizerContainerTests( + defaultSweepTimeoutMs, + "tombstone", + "GarbageCollector:TombstoneReadyObject_Revived", + "GarbageCollector:TombstoneReadyObject_Changed", + "GarbageCollector:TombstoneReadyObject_Loaded", + ); + }); + + describe("SweepReady events - No sweepGracePeriodMs (summarizer container)", () => { summarizerContainerTests( - sweepTimeoutMs, + defaultSweepTimeoutMs, + "sweep", // Jump straight to SweepReady given 0 delay + "GarbageCollector:SweepReadyObject_Revived", + "GarbageCollector:SweepReadyObject_Changed", + "GarbageCollector:SweepReadyObject_Loaded", + 0 /* sweepGracePeriodMsOverride */, + ); + }); + + describe("SweepReady events - with sweepGracePeriodMs delay (summarizer container)", () => { + summarizerContainerTests( + defaultSweepTimeoutMs + defaultSweepGracePeriodMs, "sweep", "GarbageCollector:SweepReadyObject_Revived", "GarbageCollector:SweepReadyObject_Changed", @@ -849,12 +885,30 @@ describe("Garbage Collection Tests", () => { }); }); - it("generates both inactive and sweep ready events when nodes are used after time out", async () => { + it("Unreferenced nodes transition through Inactive, TombstoneReady and SweepReady states", async () => { const inactiveTimeoutMs = 500; injectedSettings["Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs"] = inactiveTimeoutMs; - const garbageCollector = createGarbageCollector({}); + const garbageCollector = createGarbageCollector({}) as GcWithPrivates; + + function validateUnreferencedStates( + expectedUnreferencedStates: Record, + ) { + // Base assumption is that all 6 nodes are still referenced (no state tracker aka 'undefined') + // Then update this with the given expected unreferenced states + const expectedStates = Object.assign( + [undefined, undefined, undefined, undefined, undefined, undefined], + expectedUnreferencedStates, + ); + for (const [id, state] of expectedStates.entries()) { + assert.equal( + garbageCollector.unreferencedNodesState.get(nodes[id])?.state, + state, + `node ${id} should be ${state ?? "referenced"}`, + ); + } + } // Remove node 2's reference from node 1. This should make node 2 and node 3 unreferenced. defaultGCData.gcNodes[nodes[1]] = []; @@ -862,63 +916,18 @@ describe("Garbage Collection Tests", () => { // Advance the clock to trigger inactive timeout and validate that we get inactive events. clock.tick(inactiveTimeoutMs + 1); - await mockNodeChangesAndRunGC(garbageCollector); - mockLogger.assertMatch( - [ - { - eventName: "GarbageCollector:InactiveObject_Loaded", - timeout: inactiveTimeoutMs, - ...tagCodeArtifacts({ id: nodes[2] }), - }, - { - eventName: "GarbageCollector:InactiveObject_Changed", - timeout: inactiveTimeoutMs, - ...tagCodeArtifacts({ id: nodes[2] }), - }, - { - eventName: "GarbageCollector:InactiveObject_Loaded", - timeout: inactiveTimeoutMs, - ...tagCodeArtifacts({ id: nodes[3] }), - }, - { - eventName: "GarbageCollector:InactiveObject_Changed", - timeout: inactiveTimeoutMs, - ...tagCodeArtifacts({ id: nodes[3] }), - }, - ], - "inactive events not generated as expected", - true /* inlineDetailsProp */, - ); + await garbageCollector.collectGarbage({}); + validateUnreferencedStates({ 2: "Inactive", 3: "Inactive" }); - // Advance the clock to trigger sweep timeout and validate that we get sweep ready events. - clock.tick(sweepTimeoutMs - inactiveTimeoutMs); - await mockNodeChangesAndRunGC(garbageCollector); - mockLogger.assertMatch( - [ - { - eventName: "GarbageCollector:SweepReadyObject_Loaded", - timeout: sweepTimeoutMs, - ...tagCodeArtifacts({ id: nodes[2] }), - }, - { - eventName: "GarbageCollector:SweepReadyObject_Changed", - timeout: sweepTimeoutMs, - ...tagCodeArtifacts({ id: nodes[2] }), - }, - { - eventName: "GarbageCollector:SweepReadyObject_Loaded", - timeout: sweepTimeoutMs, - ...tagCodeArtifacts({ id: nodes[3] }), - }, - { - eventName: "GarbageCollector:SweepReadyObject_Changed", - timeout: sweepTimeoutMs, - ...tagCodeArtifacts({ id: nodes[3] }), - }, - ], - "sweep ready events not generated as expected", - true /* inlineDetailsProp */, - ); + // Advance the clock to trigger sweepTimeoutMs and validate that we get tombstone ready events. + clock.tick(defaultSweepTimeoutMs - inactiveTimeoutMs); + await garbageCollector.collectGarbage({}); + validateUnreferencedStates({ 2: "TombstoneReady", 3: "TombstoneReady" }); + + // Advance the clock the sweep delay and validate that we get sweep ready events. + clock.tick(defaultSweepGracePeriodMs); + await garbageCollector.collectGarbage({}); + validateUnreferencedStates({ 2: "SweepReady", 3: "SweepReady" }); }); }); @@ -1627,7 +1636,7 @@ describe("Garbage Collection Tests", () => { // This means this node should time out as soon as its data is loaded. const node3GCDetails: IGarbageCollectionSummaryDetailsLegacy = { gcData: { gcNodes: { "/": [] } }, - unrefTimestamp: Date.now() - sweepTimeoutMs * 100, + unrefTimestamp: Date.now() - defaultSweepTimeoutMs * 100, }; const node3Snapshot = getDummySnapshotTree(); const gcBlobId = "node3GCDetails"; @@ -1647,7 +1656,7 @@ describe("Garbage Collection Tests", () => { [attributesBlobId, {}], ]); const garbageCollector = createGarbageCollector({ baseSnapshot }, gcBlobMap, { - sweepTimeoutMs, + sweepTimeoutMs: defaultSweepTimeoutMs, }) as GcWithPrivates; // GC state and tombstone state should be discarded but deleted nodes should be read from base snapshot. @@ -1696,7 +1705,10 @@ describe("Garbage Collection Tests", () => { deletedAttachmentBlobCount: 0, }; - const gcOptions: IGCRuntimeOptions = sweepEnabled ? { gcSweepGeneration: 1 } : {}; + const sweepGracePeriodMs = 0; // Skip TombstoneReady for these tests and go straight to SweepReady + const gcOptions: IGCRuntimeOptions = sweepEnabled + ? { gcSweepGeneration: 1, sweepGracePeriodMs } + : { sweepGracePeriodMs }; garbageCollector = createGarbageCollector({ gcOptions }); }); @@ -1795,7 +1807,7 @@ describe("Garbage Collection Tests", () => { ); // Advance the clock past sweep timeout so that unreferenced nodes are deleted. - clock.tick(sweepTimeoutMs + 1); + clock.tick(defaultSweepTimeoutMs + 1); // There should be 2 deleted nodes and data stores. There shouldn't be any nodes whose // reference state updated. @@ -1822,7 +1834,7 @@ describe("Garbage Collection Tests", () => { ); // Advance the clock past sweep timeout so that unreferenced nodes are deleted. - clock.tick(sweepTimeoutMs + 1); + clock.tick(defaultSweepTimeoutMs + 1); // There should be 2 deleted nodes and data stores. There shouldn't be any nodes whose // reference state updated. @@ -1857,7 +1869,7 @@ describe("Garbage Collection Tests", () => { assert.deepStrictEqual(gcStats, expectedStats, "Incorrect GC stats 2"); // Advance the clock past sweep timeout again so that unreferenced node is deleted. - clock.tick(sweepTimeoutMs + 1); + clock.tick(defaultSweepTimeoutMs + 1); // No nodes are updated since the last run. // There should be 1 more deleted node / data store. diff --git a/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts b/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts index 2ffbf68f3b0b..0fb1f9026a36 100644 --- a/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts +++ b/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts @@ -35,6 +35,7 @@ import { runGCKey, runSweepKey, defaultInactiveTimeoutMs, + defaultSweepGracePeriodMs, gcTestModeKey, nextGCVersion, stableGCVersion, @@ -826,6 +827,41 @@ describe("Garbage Collection configurations", () => { ); }); }); + describe("sweepGracePeriodMs", () => { + const testCases: { + option: number | undefined; + expectedResult: number; + }[] = [ + { option: 123, expectedResult: 123 }, + { option: 0, expectedResult: 0 }, + { option: undefined, expectedResult: defaultSweepGracePeriodMs }, + ]; + testCases.forEach((testCase) => { + it(`Test Case ${JSON.stringify(testCase)}`, () => { + gc = createGcWithPrivateMembers( + {} /* metadata */, + { + sweepGracePeriodMs: testCase.option, + }, + ); + assert.equal(gc.configs.sweepGracePeriodMs, testCase.expectedResult); + }); + }); + it("sweepGracePeriodMs must be non-negative", () => { + assert.throws( + () => { + gc = createGcWithPrivateMembers( + {} /* metadata */, + { + sweepGracePeriodMs: -1, + }, + ); + }, + (e: IErrorBase) => e.errorType === "usageError", + "sweepGracePeriodMs must be non-negative", + ); + }); + }); describe("testMode", () => { const testCases: { setting?: boolean; diff --git a/packages/runtime/container-runtime/src/test/gc/gcTelemetry.spec.ts b/packages/runtime/container-runtime/src/test/gc/gcTelemetry.spec.ts index 5b164428aafe..04f91af894d9 100644 --- a/packages/runtime/container-runtime/src/test/gc/gcTelemetry.spec.ts +++ b/packages/runtime/container-runtime/src/test/gc/gcTelemetry.spec.ts @@ -46,6 +46,7 @@ describe("GC Telemetry Tracker", () => { let mockLogger: MockLogger; let mc: MonitoringContext; let clock: SinonFakeTimers; + let sweepGracePeriodMs = 1000; // Default case for these tests let unreferencedNodesState: Map = new Map(); let telemetryTracker: GCTelemetryTracker; @@ -81,6 +82,7 @@ describe("GC Telemetry Tracker", () => { inactiveTimeoutMs, sessionExpiryTimeoutMs: defaultSessionExpiryDurationMs, sweepTimeoutMs: enableSweep ? sweepTimeoutMs : undefined, + sweepGracePeriodMs, tombstoneEnforcementAllowed: false, throwOnTombstoneLoad: false, throwOnTombstoneUsage: false, @@ -114,6 +116,7 @@ describe("GC Telemetry Tracker", () => { inactiveTimeoutMs, Date.now(), sweepTimeoutMs, + sweepGracePeriodMs, ), ); }); @@ -183,6 +186,7 @@ describe("GC Telemetry Tracker", () => { clock.reset(); mockLogger.clear(); injectedSettings = {}; + sweepGracePeriodMs = 1000; // Default case for these tests }); after(() => { @@ -193,8 +197,8 @@ describe("GC Telemetry Tracker", () => { const clientTypeTests = (isSummarizerClient: boolean) => { /** * Asserts that the events are as expected based on whether its a summarizer client or not. In non-summarizer - * clients, only "InactiveObject_Loaded" and "SweepReadyObject_Loaded" events are logged. "Changed" and "Revived" - * events are not logged. + * clients, only "InactiveObject_Loaded", "TombstoneReadyObject_Loaded" and "SweepReadyObject_Loaded" events are logged. + * "_Changed" and "_Revived" events are not logged. */ function assertMatchEvents( events: Omit[], @@ -220,7 +224,7 @@ describe("GC Telemetry Tracker", () => { mockLogger.assertMatchNone(unexpectedEvents, message, true /* inlineDetailsProp */); } - it("generates inactive and sweep ready events when nodes are used after time out", async () => { + it("generates inactive, tombstone ready, and sweep ready events when nodes are used after time out", async () => { telemetryTracker = createTelemetryTracker(true /* enable Sweep */, isSummarizerClient); // Mark nodes 2 and 3 as unreferenced. markNodesUnreferenced([nodes[2], nodes[3]]); @@ -255,33 +259,63 @@ describe("GC Telemetry Tracker", () => { "inactive events not as expected", ); - // Advance the clock to trigger sweep timeout and validate that sweep ready events are as expected. + // Advance the clock to trigger sweep timeout and validate that TombstoneReady events are as expected. clock.tick(sweepTimeoutMs - inactiveTimeoutMs); mockNodeChanges(nodes); await simulateGCToTriggerEvents(isSummarizerClient); assertMatchEvents( [ { - eventName: "GarbageCollector:SweepReadyObject_Loaded", + eventName: "GarbageCollector:TombstoneReadyObject_Loaded", timeout: sweepTimeoutMs, ...tagCodeArtifacts({ id: nodes[2] }), }, { - eventName: "GarbageCollector:SweepReadyObject_Changed", + eventName: "GarbageCollector:TombstoneReadyObject_Changed", timeout: sweepTimeoutMs, ...tagCodeArtifacts({ id: nodes[2] }), }, { - eventName: "GarbageCollector:SweepReadyObject_Loaded", + eventName: "GarbageCollector:TombstoneReadyObject_Loaded", timeout: sweepTimeoutMs, ...tagCodeArtifacts({ id: nodes[3] }), }, { - eventName: "GarbageCollector:SweepReadyObject_Changed", + eventName: "GarbageCollector:TombstoneReadyObject_Changed", timeout: sweepTimeoutMs, ...tagCodeArtifacts({ id: nodes[3] }), }, ], + "tombstone ready events not as expected", + ); + + // Advance the clock by the delay and validate that SweepReady events are as expected. + clock.tick(sweepGracePeriodMs); + mockNodeChanges(nodes); + await simulateGCToTriggerEvents(isSummarizerClient); + assertMatchEvents( + [ + { + eventName: "GarbageCollector:SweepReadyObject_Loaded", + timeout: sweepTimeoutMs + sweepGracePeriodMs, + ...tagCodeArtifacts({ id: nodes[2] }), + }, + { + eventName: "GarbageCollector:SweepReadyObject_Changed", + timeout: sweepTimeoutMs + sweepGracePeriodMs, + ...tagCodeArtifacts({ id: nodes[2] }), + }, + { + eventName: "GarbageCollector:SweepReadyObject_Loaded", + timeout: sweepTimeoutMs + sweepGracePeriodMs, + ...tagCodeArtifacts({ id: nodes[3] }), + }, + { + eventName: "GarbageCollector:SweepReadyObject_Changed", + timeout: sweepTimeoutMs + sweepGracePeriodMs, + ...tagCodeArtifacts({ id: nodes[3] }), + }, + ], "sweep ready events not as expected", ); }); @@ -306,13 +340,14 @@ describe("GC Telemetry Tracker", () => { ); }); - /** Tests that validate either inactive or sweep events are logged as expected. */ - const inactiveOrSweepEventTests = ( + /** Tests that validate either the relevant events are logged as expected. */ + const unreferencedPhasesEventTests = ( timeout: number, - mode: "inactive" | "sweep", + mode: "inactive" | "tombstone" | "sweep", revivedEventName: string, changedEventName: string, loadedEventName: string, + sweepGracePeriodMsOverride?: number, ) => { // Validates that no unexpected event has been fired. function validateNoEvents() { @@ -327,8 +362,11 @@ describe("GC Telemetry Tracker", () => { } beforeEach(() => { + if (sweepGracePeriodMsOverride !== undefined) { + sweepGracePeriodMs = sweepGracePeriodMsOverride; + } telemetryTracker = createTelemetryTracker( - mode === "sweep" ? true : false /* enableSweep */, + mode !== "inactive" /* enableSweep */, isSummarizerClient, ); }); @@ -351,7 +389,7 @@ describe("GC Telemetry Tracker", () => { validateNoEvents(); }); - it("generates events for nodes that are used after inactive / sweep ready", async () => { + it("generates events for nodes that are used after state changes", async () => { // Mark nodes 1 and 2 as unreferenced. markNodesUnreferenced([nodes[1], nodes[2]]); @@ -453,7 +491,7 @@ describe("GC Telemetry Tracker", () => { // This test is only relevant for summarizer client because it does not log changed events if the node is revived. if (isSummarizerClient) { - it("generates only revived event in summarizer when an inactive node is updated and revived", async () => { + it("generates only revived event in summarizer when a node is updated and revived", async () => { // Mark node 2 as unreferenced. markNodesUnreferenced([nodes[2]]); @@ -501,7 +539,7 @@ describe("GC Telemetry Tracker", () => { }; describe("Inactive events", () => { - inactiveOrSweepEventTests( + unreferencedPhasesEventTests( inactiveTimeoutMs, "inactive", "GarbageCollector:InactiveObject_Revived", @@ -510,9 +548,30 @@ describe("GC Telemetry Tracker", () => { ); }); - describe("SweepReady events", () => { - inactiveOrSweepEventTests( + describe("TombstoneReady events", () => { + unreferencedPhasesEventTests( + sweepTimeoutMs, + "tombstone", + "GarbageCollector:TombstoneReadyObject_Revived", + "GarbageCollector:TombstoneReadyObject_Changed", + "GarbageCollector:TombstoneReadyObject_Loaded", + ); + }); + + describe("SweepReady events (with no delay)", () => { + unreferencedPhasesEventTests( sweepTimeoutMs, + "sweep", // Jump straight to SweepReady given 0 delay + "GarbageCollector:SweepReadyObject_Revived", + "GarbageCollector:SweepReadyObject_Changed", + "GarbageCollector:SweepReadyObject_Loaded", + 0 /* sweepGracePeriodMsOverride */, + ); + }); + + describe("SweepReady events", () => { + unreferencedPhasesEventTests( + sweepTimeoutMs + sweepGracePeriodMs, "sweep", "GarbageCollector:SweepReadyObject_Revived", "GarbageCollector:SweepReadyObject_Changed", diff --git a/packages/runtime/container-runtime/src/test/gc/gcUnreferencedStateTracker.spec.ts b/packages/runtime/container-runtime/src/test/gc/gcUnreferencedStateTracker.spec.ts index 04d7a1fa1d2d..328e53c35234 100644 --- a/packages/runtime/container-runtime/src/test/gc/gcUnreferencedStateTracker.spec.ts +++ b/packages/runtime/container-runtime/src/test/gc/gcUnreferencedStateTracker.spec.ts @@ -7,6 +7,26 @@ import { strict as assert } from "assert"; import { SinonFakeTimers, SinonSpy, useFakeTimers, spy } from "sinon"; import { UnreferencedState, UnreferencedStateTracker } from "../../gc"; +/** Schema for steps taken to test unreferenced state progression / tracking */ +type Steps = [ + { + /** Start time (used as both local time and currentReferenceTimestampMs) */ + time: number; + /** Expected initial state */ + state: UnreferencedState; + /** Configured sweepGracePeriodMs - defaults to 10ms for these tests */ + sweepGracePeriodMs?: number; + }, + ...{ + /** Local time of the next step */ + time: number; + /** If defined, call updateTracking with this as currentReferenceTimestampMs */ + updateWith?: number; + /** Expected new state (after calling updateTracking if applicable) */ + state: UnreferencedState; + }[], +]; + describe("Garbage Collection Tests", () => { let clock: SinonFakeTimers; @@ -28,6 +48,7 @@ describe("Garbage Collection Tests", () => { afterEach(() => { tracker.stopTracking(); }); + /** * During the lifetime of an unreferenced object, its state is tracked and updated in two ways. * Timers are set to trigger transitioning to the next state, and updateTracking is also called @@ -35,110 +56,139 @@ describe("Garbage Collection Tests", () => { * These tests specify how to advance the clock (to hit the timers) and also when to call updateTracking, * checking that the expected state transitions occur as specified */ - function runTestCase(testCase: { - start: [number, UnreferencedState]; - steps: [number, number | undefined, UnreferencedState][]; - }) { - const [startTimestamp, startState] = testCase.start; + function runTestCase(allSteps: Steps) { + const [start, ...steps] = allSteps; + clock.tick(start.time); + tracker = new UnreferencedStateTracker( 0 /* unreferencedTimestampMs */, 10 /* inactiveTimeoutMs */, - startTimestamp /* currentReferenceTimestampMs */, + start.time /* currentReferenceTimestampMs */, 20 /* sweepTimeoutMs */, + start.sweepGracePeriodMs ?? 10 /* sweepGracePeriodMs */, ); - assert.equal(tracker.state, startState, `Wrong starting state`); - let lastTime = startTimestamp; - testCase.steps.forEach( - ([localTime, currentReferenceTimestampMs, expectedState], index) => { - assert( - localTime > lastTime, - "INVALID TEST CASE: steps must move forward in time, following start", - ); - const delta = localTime - lastTime; - clock.tick(delta); - lastTime = localTime; + assert.equal(tracker.state, start.state, `Wrong starting state`); + steps.forEach(({ time: advanceClockTo, updateWith, state: expectedState }, index) => { + assert( + advanceClockTo > clock.now, + "INVALID TEST CASE: steps must move forward in time, following start", + ); + clock.tick(advanceClockTo - clock.now); - if (currentReferenceTimestampMs !== undefined) { - tracker.updateTracking(currentReferenceTimestampMs); - } + if (updateWith !== undefined) { + tracker.updateTracking(updateWith); + } - assert.equal(tracker.state, expectedState, `Wrong state at step ${index}`); - }, - ); + assert.equal(tracker.state, expectedState, `Wrong state at step ${index + 1}`); // 0-indexed including start + }); } /** * Test cases to run through above function runTestCase - * Each test case specifies: - * - * `name`: Name of the test - * - * `start`: Starting value for currentReferenceTimestampMs and expected starting state - * - * `steps`: Each step gives: - * - * - The timestamp to advance to - * - * - The currentReferenceTimestampMs to pass to updateTracking (or skip if undefined) - * - * - The expected state at that time (after calling updateTracking if applicable) * - * In all cases: unreferencedTimestampMs = 0, inactiveTimeoutMs = 10, sweepTimeoutMs = 20 + * In all cases: + * - unreferencedTimestampMs = 0 + * - inactiveTimeoutMs = 10 + * - sweepTimeoutMs = 20 + * - sweepGracePeriodMs defaults to 10 (so sweep at 30) */ const testCases: { name: string; - start: [number, UnreferencedState]; - steps: [number, number | undefined, UnreferencedState][]; + steps: Steps; }[] = [ { name: "No calls to updateTracking", - start: [0, "Active"], steps: [ - [3, undefined, "Active"], - [5, undefined, "Active"], - [12, undefined, "Inactive"], - [15, undefined, "Inactive"], - [25, undefined, "SweepReady"], + { time: 0, state: "Active" }, + { time: 3, state: "Active" }, + { time: 5, state: "Active" }, + { time: 12, state: "Inactive" }, + { time: 15, state: "Inactive" }, + { time: 25, state: "TombstoneReady" }, + { time: 35, state: "SweepReady" }, + ], + }, + { + name: "No calls to updateTracking - sweepGracePeriodMs 0 (no Tombstone phase)", + steps: [ + { time: 0, state: "Active", sweepGracePeriodMs: 0 }, + { time: 3, state: "Active" }, + { time: 5, state: "Active" }, + { time: 12, state: "Inactive" }, + { time: 19, state: "Inactive" }, + { time: 20, state: "SweepReady" }, + { time: 21, state: "SweepReady" }, + ], + }, + { + name: "Skip to SweepReady", + steps: [ + { time: 0, state: "Active" }, + { time: 5, state: "Active" }, + { time: 35, state: "SweepReady" }, + ], + }, + { + name: "Skip to SweepReady - sweepGracePeriodMs 0 (no Tombstone phase)", + steps: [ + { time: 0, state: "Active", sweepGracePeriodMs: 0 }, + { time: 5, state: "Active" }, + { time: 20, state: "SweepReady" }, + ], + }, + { + name: "Skip to SweepReady (via updateTracking) - sweepGracePeriodMs 0 (no Tombstone phase)", + steps: [ + { time: 0, state: "Active", sweepGracePeriodMs: 0 }, + { time: 5, state: "Active" }, + { time: 20, updateWith: 20, state: "SweepReady" }, ], }, { name: "Call update, but triggered via timers", - start: [0, "Active"], steps: [ - [3, 2, "Active"], - [5, 5, "Active"], - [12, 9, "Inactive"], // Timer will have fired even though server time hasn't passed threshold - [15, 15, "Inactive"], - [25, undefined, "SweepReady"], + { time: 0, state: "Active" }, + { time: 3, updateWith: 2, state: "Active" }, + { time: 5, updateWith: 5, state: "Active" }, + { time: 12, updateWith: 9, state: "Inactive" }, // Timer will have fired even though server time hasn't passed threshold + { time: 17, updateWith: 15, state: "Inactive" }, // No-op, timer already fired ], }, { name: "currentReferenceTimestampMs jumps ahead", - start: [0, "Active"], steps: [ - [5, undefined, "Active"], - [10, undefined, "Inactive"], - [11, 20, "SweepReady"], // Shouldn't be physically possible, but supported in API + { time: 0, state: "Active" }, + { time: 5, state: "Active" }, + { time: 10, state: "Inactive" }, + { time: 11, updateWith: 20, state: "TombstoneReady" }, // Shouldn't be physically possible, but supported in API ], }, { name: "Start Inactive", - start: [12, "Inactive"], steps: [ - [15, undefined, "Inactive"], - [20, undefined, "SweepReady"], + { time: 12, state: "Inactive" }, + { time: 15, state: "Inactive" }, + { time: 20, state: "TombstoneReady" }, + { time: 35, state: "SweepReady" }, + ], + }, + { + name: "Start TombstoneReady", + steps: [ + { time: 22, state: "TombstoneReady" }, + { time: 25, state: "TombstoneReady" }, + { time: 35, state: "SweepReady" }, ], }, { name: "Start SweepReady", - start: [22, "SweepReady"], - steps: [], + steps: [{ time: 32, state: "SweepReady" }], }, ]; testCases.forEach((testCase) => { it(testCase.name, () => { - runTestCase(testCase); + runTestCase(testCase.steps); }); }); @@ -148,15 +198,26 @@ describe("Garbage Collection Tests", () => { 3 /* inactiveTimeoutMs */, 11 /* currentReferenceTimestampMs */, 7 /* sweepTimeoutMs */, + 15 /* sweepGracePeriodMs */, ); assert.equal(tracker.state, UnreferencedState.Active, "Should start as Active"); - clock.tick(5); - assert.equal(tracker.state, UnreferencedState.Inactive, "Should be Inactive 5ms later"); + clock.tick(2); + assert.equal( + tracker.state, + UnreferencedState.Inactive, + "Should be Inactive 2ms later (at 13)", + ); tracker.updateTracking(17); + assert.equal( + tracker.state, + UnreferencedState.TombstoneReady, + "Should be TombstoneReady after currentReferenceTimestampMs=17", + ); + clock.tick(15); assert.equal( tracker.state, UnreferencedState.SweepReady, - "Should be SweepReady after currentReferenceTimestampMs=17", + "Should be SweepReady 15ms later", ); }); it("Timers can't be crossed", () => { @@ -165,6 +226,7 @@ describe("Garbage Collection Tests", () => { 10 /* inactiveTimeoutMs */, 0 /* currentReferenceTimestampMs */, 12 /* sweepTimeoutMs */, + 0 /* sweepGracePeriodMs */, ); assert.equal(tracker.state, UnreferencedState.Active, "Should start as Active"); tracker.updateTracking(10); @@ -188,15 +250,15 @@ describe("Garbage Collection Tests", () => { 20 /* inactiveTimeoutMs */, 5 /* currentReferenceTimestampMs */, undefined /* sweepTimeoutMs */, + 0 /* sweepGracePeriodMs */, ); assert.equal(tracker.state, UnreferencedState.Active, "Should start as Active"); const timerClearSpy: SinonSpy = spy((tracker as any).inactiveTimer, "clear"); // At T10 we had 15 to go based on server timestamps, so Timer is set to 25 clock.tick(6); // at T16 (9 to go) tracker.updateTracking(15); // Simulate processing a more-recent Summary (reference time 15 at T16). Pulls in timer to 21 (5 to go) - assert.equal( - timerClearSpy.callCount, - 1, + assert( + timerClearSpy.callCount > 0, "Expected underlying Timer to clear and reset to support shorter timeout", ); clock.tick(5); @@ -208,6 +270,7 @@ describe("Garbage Collection Tests", () => { 10 /* inactiveTimeoutMs */, 0 /* currentReferenceTimestampMs */, undefined /* sweepTimeoutMs */, + 0 /* sweepGracePeriodMs */, ); assert.equal(tracker.state, UnreferencedState.Active, "Should start as Active"); clock.tick(5); // at T5, 5 to go diff --git a/packages/runtime/test-runtime-utils/src/validateAssertionError.ts b/packages/runtime/test-runtime-utils/src/validateAssertionError.ts index e88914d858b4..907a925bf3dc 100644 --- a/packages/runtime/test-runtime-utils/src/validateAssertionError.ts +++ b/packages/runtime/test-runtime-utils/src/validateAssertionError.ts @@ -32,7 +32,11 @@ export function validateAssertionError(error: Error, expectedErrorMsg: string | ) { // This throws an Error instead of an AssertionError because AssertionError would require a dependency on the // node assert library, which we don't want to do for this library because it's used in the browser. - throw new Error(`Unexpected assertion thrown: ${error.message} ('${mappedMsg}')`); + const message = + shortCodeMap[error.message] === undefined + ? `Unexpected assertion thrown\nActual: ${error.message}\nExpected: ${expectedErrorMsg}` + : `Unexpected assertion thrown\nActual: ${error.message} ('${mappedMsg}')\nExpected: ${expectedErrorMsg}`; + throw new Error(message); } return true; } diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcInactiveNodes.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcInactiveNodes.spec.ts index e0229fc7b9a8..f8aad0bb29c4 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcInactiveNodes.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcInactiveNodes.spec.ts @@ -458,7 +458,7 @@ describeCompat("GC inactive nodes tests", "NoCompat", (getTestObjectProvider) => assert.equal( inactiveError?.underlyingResponseHeaders?.[InactiveResponseHeaderKey], true, - "Inactive error from handle.get should include the tombstone flag", + "Inactive error from handle.get should include the inactive flag", ); } mockLogger.assertMatch( diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcSweepAttachmentBlobs.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcSweepAttachmentBlobs.spec.ts index 7a55044d793e..f53efaaaa53a 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcSweepAttachmentBlobs.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcSweepAttachmentBlobs.spec.ts @@ -38,7 +38,10 @@ import { describeCompat("GC attachment blob sweep tests", "NoCompat", (getTestObjectProvider) => { const sweepTimeoutMs = 200; const settings = {}; - const gcOptions: IGCRuntimeOptions = { inactiveTimeoutMs: 0 }; + const gcOptions: IGCRuntimeOptions = { + inactiveTimeoutMs: 0, + sweepGracePeriodMs: 0, // Skip Tombstone, these tests focus on Sweep + }; const testContainerConfig: ITestContainerConfig = { runtimeOptions: { summaryOptions: { diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcSweepDataStores.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcSweepDataStores.spec.ts index 176a66015fd0..ca3f3b56ae08 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcSweepDataStores.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcSweepDataStores.spec.ts @@ -46,7 +46,10 @@ describeCompat("GC data store sweep tests", "NoCompat", (getTestObjectProvider) ); const settings = {}; - const gcOptions: IGCRuntimeOptions = { inactiveTimeoutMs: 0 }; + const gcOptions: IGCRuntimeOptions = { + inactiveTimeoutMs: 0, + sweepGracePeriodMs: 0, // Skip Tombstone, these tests focus on Sweep + }; const testContainerConfig: ITestContainerConfig = { runtimeOptions: { summaryOptions: { diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcSweepUnreferencePhases.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcSweepUnreferencePhases.spec.ts index a97b03a07095..9bc57056be6c 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcSweepUnreferencePhases.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcSweepUnreferencePhases.spec.ts @@ -18,11 +18,10 @@ import { TestDataObjectType, } from "@fluid-private/test-version-utils"; import { IGCRuntimeOptions } from "@fluidframework/container-runtime"; -import { stringToBuffer } from "@fluid-internal/client-utils"; import { delay } from "@fluidframework/core-utils"; import { gcTreeKey } from "@fluidframework/runtime-definitions"; import { ISummaryTree, SummaryType } from "@fluidframework/protocol-definitions"; -import { IContainer, LoaderHeader } from "@fluidframework/container-definitions"; +import { IContainer } from "@fluidframework/container-definitions"; import { getGCStateFromSummary, getGCDeletedStateFromSummary, @@ -30,14 +29,19 @@ import { } from "./gcTestSummaryUtils.js"; /** - * Validates that an unreferenced datastore and blob goes through all the GC sweep phases without overlapping. + * Validates that an unreferenced datastore goes through all the GC phases without overlapping. */ -describeCompat("GC sweep unreference phases", "NoCompat", (getTestObjectProvider) => { - const inactiveTimeoutMs = 100; - const sweepTimeoutMs = 200; +describeCompat("GC unreference phases", "NoCompat", (getTestObjectProvider) => { + // Since these tests depend on these timing windows, they should not be run against drivers talking over the network + // (see this.skip() call below) + const sweepTimeoutMs = 200; // Tombstone at 200ms + const sweepGracePeriodMs = 100; // Sweep at 300ms const settings = {}; - const gcOptions: IGCRuntimeOptions = { inactiveTimeoutMs }; + const gcOptions: IGCRuntimeOptions = { + inactiveTimeoutMs: sweepTimeoutMs / 2, // Required to avoid an error + sweepGracePeriodMs, + }; const testContainerConfig: ITestContainerConfig = { runtimeOptions: { summaryOptions: { @@ -72,8 +76,10 @@ describeCompat("GC sweep unreference phases", "NoCompat", (getTestObjectProvider beforeEach(async function () { provider = getTestObjectProvider({ syncSummarizer: true }); + // These tests validate the GC state in summary generated by the container runtime. They do not care // about the snapshot that is downloaded from the server. So, it doesn't need to run against real services. + // Additionally, they depend on tight timing windows. So, they should not be run against drivers talking over the network. if (provider.driver.type !== "local") { this.skip(); } @@ -84,14 +90,15 @@ describeCompat("GC sweep unreference phases", "NoCompat", (getTestObjectProvider settings["Fluid.GarbageCollection.TestOverride.SweepTimeoutMs"] = sweepTimeoutMs; }); - it("GC nodes go from referenced to unreferenced to inactive to sweep ready to swept", async () => { + it("Unreferenced objects follow the sequence [unreferenced, tombstoned, deleted]", async () => { const mainContainer = await provider.makeTestContainer(testContainerConfig); const mainDataStore = (await mainContainer.getEntryPoint()) as ITestDataObject; await waitForContainerConnection(mainContainer); - const { container, summarizer } = await loadSummarizer(mainContainer); + const { container: nonInteractiveContainer, summarizer } = + await loadSummarizer(mainContainer); - // create datastore and blob + // create datastore const dataStore = await mainDataStore._context.containerRuntime.createDataStore(TestDataObjectType); const dataStoreHandle = dataStore.entryPoint; @@ -99,24 +106,17 @@ describeCompat("GC sweep unreference phases", "NoCompat", (getTestObjectProvider const dataObject = (await dataStoreHandle.get()) as ITestDataObject; const dataStoreId = dataObject._context.id; const ddsHandle = dataObject._root.handle; - const blobContents = "Blob contents"; - const blobHandle = await mainDataStore._runtime.uploadBlob( - stringToBuffer(blobContents, "utf-8"), - ); - // store datastore and blob handles + // store datastore handles mainDataStore._root.set("dataStore", dataStoreHandle); - mainDataStore._root.set("blob", blobHandle); - // unreference datastore and blob handles + // unreference datastore handles mainDataStore._root.delete("dataStore"); - mainDataStore._root.delete("blob"); - // Summarize and verify datastore and blob are unreferenced and not tombstoned + // Summarize and verify datastore are unreferenced and not tombstoned await provider.ensureSynchronized(); - const summaryTree1 = (await summarizeNow(summarizer)).summaryTree; - // GC graph check - const gcState = getGCStateFromSummary(summaryTree1); + let summaryTree = (await summarizeNow(summarizer)).summaryTree; + const gcState = getGCStateFromSummary(summaryTree); assert(gcState !== undefined, "Expected GC state to be generated"); assert( gcState.gcNodes[dataStoreHandle.absolutePath] !== undefined, @@ -126,83 +126,82 @@ describeCompat("GC sweep unreference phases", "NoCompat", (getTestObjectProvider gcState.gcNodes[dataStoreHandle.absolutePath].unreferencedTimestampMs !== undefined, "Data Store should be unreferenced", ); - assert( - gcState.gcNodes[blobHandle.absolutePath] !== undefined, - "Blob should exist on gc graph", - ); - assert( - gcState.gcNodes[blobHandle.absolutePath].unreferencedTimestampMs !== undefined, - "Blob should be unreferenced", - ); - // GC Tombstone check - const tombstoneState1 = getGCTombstoneStateFromSummary(summaryTree1); - assert(tombstoneState1 === undefined, "Nothing should be tombstoned"); - // GC Sweep check - const deletedState1 = getGCDeletedStateFromSummary(summaryTree1); - assert(deletedState1 === undefined, "Nothing should be swept"); + let tombstoneState = getGCTombstoneStateFromSummary(summaryTree); + assert(tombstoneState === undefined, "Nothing should be tombstoned"); + let deletedState = getGCDeletedStateFromSummary(summaryTree); + assert(deletedState === undefined, "Nothing should be swept"); // Summary check assert( - await isDataStoreInSummaryTree(summaryTree1, dataStoreId), + await isDataStoreInSummaryTree(summaryTree, dataStoreId), "Data Store should be in the summary!", ); - // Wait inactive timeout - await delay(inactiveTimeoutMs); - // Summarize and verify datastore and blob are unreferenced and not tombstoned - // Functionally being inactive should have no effect on datastores + // Wait half the time to Tombstone state. Nothing should change + await delay(sweepTimeoutMs / 2); + // Summarize and verify datastore is unreferenced but not tombstoned mainDataStore._root.set("send", "op"); await provider.ensureSynchronized(); - const summaryTree2 = (await summarizeNow(summarizer)).summaryTree; + summaryTree = (await summarizeNow(summarizer)).summaryTree; // GC state is a handle meaning it is the same as before, meaning nothing is tombstoned. - assert( - summaryTree2.tree[gcTreeKey].type === SummaryType.Handle, - "GC tree should not have changed", + assert.equal( + summaryTree.tree[gcTreeKey].type, + SummaryType.Handle, + "GC tree should not have changed (indicated by incremental summary using the SummaryType.Handle)", ); + + // Wait the other half of sweepTimeoutMs, triggering Tombstone + await delay(sweepTimeoutMs / 2); + mainDataStore._root.set("send", "op2"); + await provider.ensureSynchronized(); + summaryTree = (await summarizeNow(summarizer)).summaryTree; + + const rootGCTree = summaryTree.tree[gcTreeKey]; + assert.equal(rootGCTree?.type, SummaryType.Tree, "GC data should be a tree"); + tombstoneState = getGCTombstoneStateFromSummary(summaryTree); + // After sweepTimeoutMs the object should be tombstoned. + assert(tombstoneState !== undefined, "Should have tombstone state"); assert( - await isDataStoreInSummaryTree(summaryTree2, dataStoreId), - "Data Store should be in the summary!", + tombstoneState.includes(dataStoreHandle.absolutePath), + "Datastore should be tombstoned", ); - // Wait sweep timeout - await delay(sweepTimeoutMs); + // Wait sweepGracePeriodMs, triggering Sweep + await delay(sweepGracePeriodMs); mainDataStore._root.set("send", "op2"); await provider.ensureSynchronized(); - const summary3 = await summarizeNow(summarizer); - const summaryTree3 = summary3.summaryTree; + const summaryWithObjectDeleted = await summarizeNow(summarizer); + summaryTree = summaryWithObjectDeleted.summaryTree; // GC graph check - const gcState3 = getGCStateFromSummary(summaryTree3); + const gcState3 = getGCStateFromSummary(summaryTree); assert(gcState3 !== undefined, "Expected GC state to be generated"); assert( !(dataStoreHandle.absolutePath in gcState3.gcNodes), "Data Store should not exist on gc graph", ); // GC Tombstone check - const tombstoneState3 = getGCTombstoneStateFromSummary(summaryTree3); - assert(tombstoneState3 === undefined, "Nothing should be tombstoned"); + tombstoneState = getGCTombstoneStateFromSummary(summaryTree); + assert(tombstoneState === undefined, "Nothing should be tombstoned"); // GC Sweep check - const deletedState3 = getGCDeletedStateFromSummary(summaryTree3); - assert(deletedState3 !== undefined, "Should have sweep state"); - assert(deletedState3.includes(dataStoreHandle.absolutePath), "Data Store should be swept"); - assert(deletedState3.includes(ddsHandle.absolutePath), "DDS should be swept"); - assert(deletedState3.length === 2, "Nothing else should have been swept"); + deletedState = getGCDeletedStateFromSummary(summaryTree); + assert(deletedState !== undefined, "Should have sweep state"); + assert(deletedState.includes(dataStoreHandle.absolutePath), "Data Store should be swept"); + assert(deletedState.includes(ddsHandle.absolutePath), "DDS should be swept"); + assert(deletedState.length === 2, "Nothing else should have been swept"); // Summary check assert( - !(await isDataStoreInSummaryTree(summaryTree3, dataStoreId)), + !(await isDataStoreInSummaryTree(summaryTree, dataStoreId)), "Data Store should not be in the summary!", ); - await provider.loadTestContainer(testContainerConfig, { - [LoaderHeader.version]: summary3.summaryVersion, - }); - container.close(); + nonInteractiveContainer.close(); const { summarizer: remoteSummarizer } = await loadSummarizer( mainContainer, - summary3.summaryVersion, + summaryWithObjectDeleted.summaryVersion, ); - const summaryTree4 = (await summarizeNow(remoteSummarizer)).summaryTree; + summaryTree = (await summarizeNow(remoteSummarizer)).summaryTree; assert( - !(await isDataStoreInSummaryTree(summaryTree4, dataStoreId)), + !(await isDataStoreInSummaryTree(summaryTree, dataStoreId)), "Data Store should not be in the summary!", ); }); diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcTestSummaryUtils.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcTestSummaryUtils.ts index c8cceffb387d..0f40a9cd9e04 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcTestSummaryUtils.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcTestSummaryUtils.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. */ -import { assert } from "@fluidframework/core-utils"; +import { strict as assert } from "assert"; import { ISummaryTree, SummaryType } from "@fluidframework/protocol-definitions"; import { gcBlobPrefix, @@ -31,7 +31,11 @@ export function getGCStateFromSummary( if (rootGCTree === undefined) { return undefined; } - assert(rootGCTree.type === SummaryType.Tree, `GC data should be a tree`); + assert.equal( + rootGCTree.type, + SummaryType.Tree, + "getGCStateFromSummary: GC data should be a tree", + ); let rootGCState: IGarbageCollectionState = { gcNodes: {} }; for (const key of Object.keys(rootGCTree.tree)) { @@ -41,8 +45,12 @@ export function getGCStateFromSummary( } const gcBlob = rootGCTree.tree[key]; - assert(gcBlob !== undefined, "GC state not available"); - assert(gcBlob.type === SummaryType.Blob, "GC state is not a blob"); + assert(gcBlob !== undefined, "getGCStateFromSummary: GC state not available"); + assert.equal( + gcBlob.type, + SummaryType.Blob, + "getGCStateFromSummary: GC state is not a blob", + ); const gcState = JSON.parse(gcBlob.content as string) as IGarbageCollectionState; // Merge the GC state of this blob into the root GC state. rootGCState = concatGarbageCollectionStates(rootGCState, gcState); @@ -62,13 +70,21 @@ export function getGCTombstoneStateFromSummary(summaryTree: ISummaryTree): strin return undefined; } - assert(rootGCTree.type === SummaryType.Tree, "GC data should be a tree"); + assert.equal( + rootGCTree.type, + SummaryType.Tree, + "getGCTombstoneStateFromSummary: GC data should be a tree", + ); const tombstoneBlob = rootGCTree.tree[gcTombstoneBlobKey]; if (tombstoneBlob === undefined) { return undefined; } - assert(tombstoneBlob.type === SummaryType.Blob, "Tombstone state is not a blob"); + assert.equal( + tombstoneBlob.type, + SummaryType.Blob, + "getGCTombstoneStateFromSummary: Tombstone state is not a blob", + ); return JSON.parse(tombstoneBlob.content as string) as string[]; } @@ -84,13 +100,21 @@ export function getGCDeletedStateFromSummary(summaryTree: ISummaryTree): string[ return undefined; } - assert(rootGCTree.type === SummaryType.Tree, "GC data should be a tree"); + assert.equal( + rootGCTree.type, + SummaryType.Tree, + "getGCDeletedStateFromSummary: GC data should be a tree", + ); const sweepBlob = rootGCTree.tree[gcDeletedBlobKey]; if (sweepBlob === undefined) { return undefined; } - assert(sweepBlob.type === SummaryType.Blob, "Sweep state is not a blob"); + assert.equal( + sweepBlob.type, + SummaryType.Blob, + "getGCDeletedStateFromSummary: Sweep state is not a blob", + ); return JSON.parse(sweepBlob.content as string) as string[]; } diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneAttachmentBlobs.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneAttachmentBlobs.spec.ts index 1e703676feb1..91fb78479c2b 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneAttachmentBlobs.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneAttachmentBlobs.spec.ts @@ -263,7 +263,7 @@ describeCompat("GC attachment blob tombstone tests", "NoCompat", (getTestObjectP clientType: "interactive", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Revived", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Revived", clientType: "noninteractive/summarizer", }, ], diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneDataStores.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneDataStores.spec.ts index e7afeb226d82..11b4cdd2c404 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneDataStores.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneDataStores.spec.ts @@ -43,7 +43,7 @@ import { SharedMap } from "@fluidframework/map"; import { getGCStateFromSummary, getGCTombstoneStateFromSummary } from "./gcTestSummaryUtils.js"; /** - * These tests validate that SweepReady data stores are correctly marked as tombstones. Tombstones should be added + * These tests validate that TombstoneReady data stores are correctly marked as tombstones. Tombstones should be added * to the summary and changing them (sending / receiving ops, loading, etc.) is not allowed. */ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvider) => { @@ -767,7 +767,7 @@ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvid clientType: "noninteractive/summarizer", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Revived", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Revived", clientType: "noninteractive/summarizer", }, ], @@ -1043,11 +1043,11 @@ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvid clientType: "noninteractive/summarizer", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Revived", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Revived", clientType: "noninteractive/summarizer", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Revived", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Revived", clientType: "noninteractive/summarizer", }, ], @@ -1196,7 +1196,8 @@ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvid const summary2 = await summarizeNow(summarizer); assert.throws( () => getGCStateFromSummary(summary2.summaryTree), - (e: Error) => validateAssertionError(e, "GC state is not a blob"), + (e: Error) => + validateAssertionError(e, "getGCStateFromSummary: GC state is not a blob"), ); const tombstoneState = getGCTombstoneStateFromSummary(summary2.summaryTree); assert( @@ -1208,7 +1209,11 @@ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvid const summary3 = await summarizeNow(summarizer); assert.throws( () => getGCTombstoneStateFromSummary(summary3.summaryTree), - (e: Error) => validateAssertionError(e, "GC data should be a tree"), + (e: Error) => + validateAssertionError( + e, + "getGCTombstoneStateFromSummary: GC data should be a tree", + ), ); }); @@ -1388,11 +1393,11 @@ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvid clientType: "interactive", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Changed", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Changed", clientType: "noninteractive/summarizer", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Loaded", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Loaded", clientType: "noninteractive/summarizer", }, { @@ -1524,11 +1529,11 @@ describeCompat("GC data store tombstone tests", "NoCompat", (getTestObjectProvid clientType: "interactive", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Changed", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Changed", clientType: "noninteractive/summarizer", }, { - eventName: "fluid:telemetry:Summarizer:Running:SweepReadyObject_Loaded", + eventName: "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Loaded", clientType: "noninteractive/summarizer", }, ], diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneUnreferencePhases.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneUnreferencePhases.spec.ts deleted file mode 100644 index 98644f5e74e9..000000000000 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcTombstoneUnreferencePhases.spec.ts +++ /dev/null @@ -1,143 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -import { strict as assert } from "assert"; -import { - createSummarizer, - ITestContainerConfig, - ITestObjectProvider, - mockConfigProvider, - summarizeNow, - waitForContainerConnection, -} from "@fluidframework/test-utils"; -import { - describeCompat, - ITestDataObject, - TestDataObjectType, -} from "@fluid-private/test-version-utils"; -import { IGCRuntimeOptions } from "@fluidframework/container-runtime"; -import { stringToBuffer } from "@fluid-internal/client-utils"; -import { delay } from "@fluidframework/core-utils"; -import { gcTreeKey } from "@fluidframework/runtime-definitions"; -import { SummaryType } from "@fluidframework/protocol-definitions"; -import { getGCStateFromSummary, getGCTombstoneStateFromSummary } from "./gcTestSummaryUtils.js"; - -/** - * Validates that an unreferenced datastore and blob goes through all the GC phases without overlapping. - */ -describeCompat("GC unreference phases", "NoCompat", (getTestObjectProvider) => { - const inactiveTimeoutMs = 100; - const sweepTimeoutMs = 200; - - const settings = {}; - const gcOptions: IGCRuntimeOptions = { inactiveTimeoutMs }; - const testContainerConfig: ITestContainerConfig = { - runtimeOptions: { - summaryOptions: { - summaryConfigOverrides: { - state: "disabled", - }, - }, - gcOptions, - }, - loaderProps: { configProvider: mockConfigProvider(settings) }, - }; - - let provider: ITestObjectProvider; - - beforeEach(async function () { - provider = getTestObjectProvider({ syncSummarizer: true }); - // These tests validate the GC state in summary generated by the container runtime. They do not care - // about the snapshot that is downloaded from the server. So, it doesn't need to run against real services. - if (provider.driver.type !== "local") { - this.skip(); - } - - settings["Fluid.GarbageCollection.ThrowOnTombstoneUsage"] = true; - settings["Fluid.GarbageCollection.TestOverride.SweepTimeoutMs"] = sweepTimeoutMs; - }); - - it("GC nodes go from referenced to unreferenced to inactive to sweep ready to tombstone", async () => { - const mainContainer = await provider.makeTestContainer(testContainerConfig); - const mainDataStore = (await mainContainer.getEntryPoint()) as ITestDataObject; - await waitForContainerConnection(mainContainer); - - const { summarizer } = await createSummarizer(provider, mainContainer, { - runtimeOptions: { gcOptions }, - loaderProps: { configProvider: mockConfigProvider(settings) }, - }); - - // create datastore and blob - const dataStore = - await mainDataStore._context.containerRuntime.createDataStore(TestDataObjectType); - const dataStoreHandle = dataStore.entryPoint; - assert(dataStoreHandle !== undefined, "Expected a handle when creating a datastore"); - const blobContents = "Blob contents"; - const blobHandle = await mainDataStore._runtime.uploadBlob( - stringToBuffer(blobContents, "utf-8"), - ); - - // store datastore and blob handles - mainDataStore._root.set("dataStore", dataStoreHandle); - mainDataStore._root.set("blob", blobHandle); - - // unreference datastore and blob handles - mainDataStore._root.delete("dataStore"); - mainDataStore._root.delete("blob"); - - // Summarize and verify datastore and blob are unreferenced and not tombstoned - await provider.ensureSynchronized(); - let summaryTree = (await summarizeNow(summarizer)).summaryTree; - const gcState = getGCStateFromSummary(summaryTree); - assert(gcState !== undefined, "Expected GC state to be generated"); - assert( - gcState.gcNodes[dataStoreHandle.absolutePath] !== undefined, - "Datastore should exist on gc graph", - ); - assert( - gcState.gcNodes[dataStoreHandle.absolutePath].unreferencedTimestampMs !== undefined, - "Datastore should be unreferenced", - ); - assert( - gcState.gcNodes[blobHandle.absolutePath] !== undefined, - "Blob should exist on gc graph", - ); - assert( - gcState.gcNodes[blobHandle.absolutePath].unreferencedTimestampMs !== undefined, - "Blob should be unreferenced", - ); - let tombstoneState = getGCTombstoneStateFromSummary(summaryTree); - assert(tombstoneState === undefined, "Nothing should be tombstoned"); - - // Wait inactive timeout - await delay(inactiveTimeoutMs); - // Summarize and verify datastore and blob are unreferenced and not tombstoned - // Functionally being inactive should have no effect on datastores - mainDataStore._root.set("send", "op"); - await provider.ensureSynchronized(); - summaryTree = (await summarizeNow(summarizer)).summaryTree; - // GC state is a handle meaning it is the same as before, meaning nothing is tombstoned. - assert( - summaryTree.tree[gcTreeKey].type === SummaryType.Handle, - "GC tree should not have changed", - ); - - // Wait sweep timeout - await delay(sweepTimeoutMs); - mainDataStore._root.set("send", "op2"); - await provider.ensureSynchronized(); - summaryTree = (await summarizeNow(summarizer)).summaryTree; - const rootGCTree = summaryTree.tree[gcTreeKey]; - assert(rootGCTree?.type === SummaryType.Tree, `GC data should be a tree`); - tombstoneState = getGCTombstoneStateFromSummary(summaryTree); - // After sweep timeout the datastore and blob should be tombstoned. - assert(tombstoneState !== undefined, "Should have tombstone state"); - assert( - tombstoneState.includes(dataStoreHandle.absolutePath), - "Datastore should be tombstoned", - ); - assert(tombstoneState.includes(blobHandle.absolutePath), "Blob should be tombstoned"); - }); -}); diff --git a/packages/test/test-end-to-end-tests/src/test/gc/gcTrailingOps.spec.ts b/packages/test/test-end-to-end-tests/src/test/gc/gcTrailingOps.spec.ts index e43455730d7a..09da4f1dd105 100644 --- a/packages/test/test-end-to-end-tests/src/test/gc/gcTrailingOps.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/gc/gcTrailingOps.spec.ts @@ -147,11 +147,11 @@ describeCompat("GC trailing ops tests", "NoCompat", (getTestObjectProvider) => { ? [ { eventName: - "fluid:telemetry:Summarizer:Running:SweepReadyObject_Revived", + "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Revived", }, { eventName: - "fluid:telemetry:Summarizer:Running:SweepReadyObject_Revived", + "fluid:telemetry:Summarizer:Running:TombstoneReadyObject_Revived", }, ] : [], diff --git a/packages/test/test-service-load/src/optionsMatrix.ts b/packages/test/test-service-load/src/optionsMatrix.ts index 7d77ee36c6d9..29670738a664 100644 --- a/packages/test/test-service-load/src/optionsMatrix.ts +++ b/packages/test/test-service-load/src/optionsMatrix.ts @@ -66,7 +66,8 @@ const gcOptionsMatrix: OptionsMatrix = { disableGC: booleanCases, gcAllowed: booleanCases, runFullGC: booleanCases, - sessionExpiryTimeoutMs: [undefined], // Don't want coverage here + sessionExpiryTimeoutMs: [undefined], // Don't want sessions to expire at a fixed time + sweepGracePeriodMs: [undefined], // Don't need coverage here, GC sweep is tested separately }; const summaryOptionsMatrix: OptionsMatrix = { diff --git a/packages/utils/telemetry-utils/api-report/telemetry-utils.api.md b/packages/utils/telemetry-utils/api-report/telemetry-utils.api.md index 254c705a6b92..c8e07fd32298 100644 --- a/packages/utils/telemetry-utils/api-report/telemetry-utils.api.md +++ b/packages/utils/telemetry-utils/api-report/telemetry-utils.api.md @@ -439,6 +439,9 @@ export class UsageError extends LoggingError implements IUsageError, IFluidError readonly errorType: "usageError"; } +// @internal +export function validatePrecondition(condition: boolean, message: string, props?: ITelemetryBaseProperties): asserts condition; + // @internal export function wrapError(innerError: unknown, newErrorFn: (message: string) => T): T; diff --git a/packages/utils/telemetry-utils/src/error.ts b/packages/utils/telemetry-utils/src/error.ts index a4ba8ad8fe02..cb4b5fbe40ad 100644 --- a/packages/utils/telemetry-utils/src/error.ts +++ b/packages/utils/telemetry-utils/src/error.ts @@ -21,6 +21,25 @@ import { } from "./errorLogging"; import { IFluidErrorBase } from "./fluidErrorBase"; +/** + * Throws a UsageError with the given message if the condition is not met. + * Use this API when `false` indicates a precondition is not met on a public API (for any FF layer). + * + * @param condition - The condition that should be true, if the condition is false a UsageError will be thrown. + * @param message - The message to include in the error when the condition does not hold. + * @param props - Telemetry props to include on the error when the condition does not hold. + * @internal + */ +export function validatePrecondition( + condition: boolean, + message: string, + props?: ITelemetryBaseProperties, +): asserts condition { + if (!condition) { + throw new UsageError(message, props); + } +} + /** * Generic wrapper for an unrecognized/uncategorized error object * diff --git a/packages/utils/telemetry-utils/src/index.ts b/packages/utils/telemetry-utils/src/index.ts index 8b18ba94ac1a..c6a121d7a2b1 100644 --- a/packages/utils/telemetry-utils/src/index.ts +++ b/packages/utils/telemetry-utils/src/index.ts @@ -16,6 +16,7 @@ export { extractSafePropertiesFromMessage, GenericError, UsageError, + validatePrecondition, } from "./error"; export { extractLogSafeErrorProperties, From 25305eab002b2d83d64fe6d1781cf03939bd98d0 Mon Sep 17 00:00:00 2001 From: dhr-verma <137836091+dhr-verma@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:41:17 -0800 Subject: [PATCH 07/20] Log sanitization during service start config dump (#18670) ## Description This PR adds a feature to log the entire service config during start of the service. During the logging, we want to redact any sensitive information from the logged config while not altering the config object itself. To redact secrets, we use `fast-redact`. We add a new optional parameter `secretsList`. The `ConfigDumper` object has a default list. If no list is provided, the default list will be used for redaction in case the values are found in the config. In case the user provides a list, it will be appended to the default list. The logging is done only if `config.configDumpEnabled` is set to true. ## Reviewer Guidance 1. Whether to use the `fast-redact` (https://www.npmjs.com/package/fast-redact) library or write our own recursive function to redact secrets before logging. 2. Creating a deep copy of the config object before logging to prevent the original config from getting corrupted. ### Unit test coverage 1) Ensure that the values in `secretNamesToRedactInConfigDump` are correctly redacted. 2) Ensure that the redacted object is not the same as the original config object if a value is redacted. 3) Ensure that the redacted object is not the same as the original config object if no value is redacted. 4) Should not throw an error if `secretNamesToRedactInConfigDump` values are not present in `nconf` 5) Should not throw an error if `secretNamesToRedactInConfigDump` contains duplicate values 6) Should redact keys inside nested objects in `secretNamesToRedactInConfigDump` ### FRS changes I plan to add the `config` object under the frs-shared configmap. This way it will be easy to maintain access every service. Additionally, I will update the shell extension files to have a list of default secrets to redact. This can be changed as per our needs. The review process is outlined on [this wiki page](https://github.com/microsoft/FluidFramework/wiki/PR-Guidelines#guidelines). --- .../templates/gitrest-configmap.yaml | 4 + .../templates/historian-configmap.yaml | 4 + server/charts/historian/values.yaml | 18 +++ server/gitrest/packages/gitrest/config.json | 11 ++ .../historian/packages/historian/config.json | 11 ++ .../templates/fluid-configmap.yaml | 4 + .../kubernetes/routerlicious/values.yaml | 10 ++ .../packages/routerlicious/config/config.json | 11 ++ .../packages/services-shared/package.json | 1 + .../services-shared/src/configDumper.ts | 62 +++++++++ .../packages/services-shared/src/index.ts | 1 + .../packages/services-shared/src/runner.ts | 12 ++ .../src/test/configDumper.spec.ts | 128 ++++++++++++++++++ server/routerlicious/pnpm-lock.yaml | 7 + 14 files changed, 284 insertions(+) create mode 100644 server/routerlicious/packages/services-shared/src/configDumper.ts create mode 100644 server/routerlicious/packages/services-shared/src/test/configDumper.spec.ts diff --git a/server/charts/historian/templates/gitrest-configmap.yaml b/server/charts/historian/templates/gitrest-configmap.yaml index 857690155a36..9a150056eb88 100644 --- a/server/charts/historian/templates/gitrest-configmap.yaml +++ b/server/charts/historian/templates/gitrest-configmap.yaml @@ -23,6 +23,10 @@ data: "enableSanitization": {{ .Values.lumberjack.options.enableSanitization }} } }, + "config": { + "configDumpEnabled": {{ .Values.gitrest.config.configDumpEnabled }}, + "secretNamesToRedactInConfigDump": {{ .Values.gitrest.config.secretNamesToRedactInConfigDump }} + }, "requestSizeLimit": "1gb", "storageDir": { "baseDir": "/home/node/documents", diff --git a/server/charts/historian/templates/historian-configmap.yaml b/server/charts/historian/templates/historian-configmap.yaml index 5b13a20f03fc..bc886f6788c9 100644 --- a/server/charts/historian/templates/historian-configmap.yaml +++ b/server/charts/historian/templates/historian-configmap.yaml @@ -23,6 +23,10 @@ data: "enableSanitization": {{ .Values.lumberjack.options.enableSanitization }} } }, + "config": { + "configDumpEnabled": {{ .Values.historian.config.configDumpEnabled }}, + "secretNamesToRedactInConfigDump": {{ .Values.historian.config.secretNamesToRedactInConfigDump }} + }, "requestSizeLimit": "1gb", "riddler": "{{ .Values.historian.riddler }}", "ignoreEphemeralFlag": {{ .Values.historian.ignoreEphemeralFlag }}, diff --git a/server/charts/historian/values.yaml b/server/charts/historian/values.yaml index c7333e186da1..d7c6d666c56b 100644 --- a/server/charts/historian/values.yaml +++ b/server/charts/historian/values.yaml @@ -35,6 +35,15 @@ historian: system: httpServer: connectionTimeoutMs: 0 + config: + configDumpEnabled: false + secretNamesToRedactInConfigDump: + - mongo.globalDbEndpoint + - mongo.operationsDbEndpoint + - redis.pass + - redisForTenantCache.pass + - redis2.pass + - redisForThrottling.pass gitrest: name: gitrest @@ -70,6 +79,15 @@ gitrest: system: httpServer: connectionTimeoutMs: 0 + config: + configDumpEnabled: false + secretNamesToRedactInConfigDump: + - mongo.globalDbEndpoint + - mongo.operationsDbEndpoint + - redis.pass + - redisForTenantCache.pass + - redis2.pass + - redisForThrottling.pass gitssh: name: gitssh diff --git a/server/gitrest/packages/gitrest/config.json b/server/gitrest/packages/gitrest/config.json index 21f6d4522d09..f9eeea7d20c5 100644 --- a/server/gitrest/packages/gitrest/config.json +++ b/server/gitrest/packages/gitrest/config.json @@ -6,6 +6,17 @@ "level": "info", "timestamp": true }, + "config": { + "configDumpEnabled": false, + "secretNamesToRedactInConfigDump": [ + "mongo.globalDbEndpoint", + "mongo.operationsDbEndpoint", + "redis.pass", + "redisForTenantCache.pass", + "redis2.pass", + "redisForThrottling.pass" + ] + }, "requestSizeLimit": "1gb", "storageDir": { "baseDir": "/home/node/documents", diff --git a/server/historian/packages/historian/config.json b/server/historian/packages/historian/config.json index 868d2f77d3f9..7754dc9d9511 100644 --- a/server/historian/packages/historian/config.json +++ b/server/historian/packages/historian/config.json @@ -6,6 +6,17 @@ "level": "info", "timestamp": true }, + "config": { + "configDumpEnabled": false, + "secretNamesToRedactInConfigDump": [ + "mongo.globalDbEndpoint", + "mongo.operationsDbEndpoint", + "redis.pass", + "redisForTenantCache.pass", + "redis2.pass", + "redisForThrottling.pass" + ] + }, "riddler": "http://riddler:5000", "alfred": "http://alfred:3000", "requestSizeLimit": "1gb", diff --git a/server/routerlicious/kubernetes/routerlicious/templates/fluid-configmap.yaml b/server/routerlicious/kubernetes/routerlicious/templates/fluid-configmap.yaml index 37e51dad7ccc..7eb338306678 100644 --- a/server/routerlicious/kubernetes/routerlicious/templates/fluid-configmap.yaml +++ b/server/routerlicious/kubernetes/routerlicious/templates/fluid-configmap.yaml @@ -19,6 +19,10 @@ data: "timestamp": false, "label": "winston" }, + "config": { + "configDumpEnabled": {{ .Values.config.configDumpEnabled }}, + "secretNamesToRedactInConfigDump": {{ .Values.config.secretNamesToRedactInConfigDump }} + }, "lumberjack": { "options": { "enableGlobalTelemetryContext": {{ .Values.lumberjack.options.enableGlobalTelemetryContext }}, diff --git a/server/routerlicious/kubernetes/routerlicious/values.yaml b/server/routerlicious/kubernetes/routerlicious/values.yaml index 6e7ea77a1a13..5d3b456b2174 100644 --- a/server/routerlicious/kubernetes/routerlicious/values.yaml +++ b/server/routerlicious/kubernetes/routerlicious/values.yaml @@ -169,3 +169,13 @@ lumberjack: options: enableGlobalTelemetryContext: true enableSanitization: false + +config: + configDumpEnabled: false + secretNamesToRedactInConfigDump: + - mongo.globalDbEndpoint + - mongo.operationsDbEndpoint + - redis.pass + - redisForTenantCache.pass + - redis2.pass + - redisForThrottling.pass diff --git a/server/routerlicious/packages/routerlicious/config/config.json b/server/routerlicious/packages/routerlicious/config/config.json index c6629df4f3ef..d0e08d167116 100644 --- a/server/routerlicious/packages/routerlicious/config/config.json +++ b/server/routerlicious/packages/routerlicious/config/config.json @@ -7,6 +7,17 @@ "timestamp": true, "label": "winston" }, + "config": { + "configDumpEnabled": false, + "secretNamesToRedactInConfigDump": [ + "mongo.globalDbEndpoint", + "mongo.operationsDbEndpoint", + "redis.pass", + "redisForTenantCache.pass", + "redis2.pass", + "redisForThrottling.pass" + ] + }, "lumberjack": { "options": { "enableGlobalTelemetryContext": true, diff --git a/server/routerlicious/packages/services-shared/package.json b/server/routerlicious/packages/services-shared/package.json index a966733c70cf..61a0c3870175 100644 --- a/server/routerlicious/packages/services-shared/package.json +++ b/server/routerlicious/packages/services-shared/package.json @@ -64,6 +64,7 @@ "body-parser": "^1.17.1", "debug": "^4.3.4", "events": "^3.1.0", + "fast-redact": "^3.3.0", "ioredis": "^5.2.3", "lodash": "^4.17.21", "nconf": "^0.12.0", diff --git a/server/routerlicious/packages/services-shared/src/configDumper.ts b/server/routerlicious/packages/services-shared/src/configDumper.ts new file mode 100644 index 000000000000..dbf266f66419 --- /dev/null +++ b/server/routerlicious/packages/services-shared/src/configDumper.ts @@ -0,0 +1,62 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import fastRedact from "fast-redact"; +import { ILogger } from "@fluidframework/server-services-core"; +import { Lumberjack } from "@fluidframework/server-services-telemetry"; +const errorSanitizationMessage = "FluidREDACTED"; + +export class ConfigDumper { + private readonly config: Record; + private readonly secretNamesToRedactInConfigDump = [ + "mongo.globalDbEndpoint", + "mongo.operationsDbEndpoint", + "redis.pass", + "redisForTenantCache.pass", + "redis2.pass", + "redisForThrottling.pass", + ]; + private readonly logger: ILogger | undefined; + + constructor( + config: Record, + logger?: ILogger, + secretNamesToRedactInConfigDump?: string[], + ) { + // Create a deep copy of the config so that we can redact values without affecting the original config. + this.config = JSON.parse(JSON.stringify(config)); + if (secretNamesToRedactInConfigDump !== undefined) { + this.secretNamesToRedactInConfigDump = this.secretNamesToRedactInConfigDump.concat( + secretNamesToRedactInConfigDump, + ); + } + // Ensure unique redaction keys. + this.secretNamesToRedactInConfigDump = Array.from( + new Set(this.secretNamesToRedactInConfigDump), + ); + this.logger = logger; + } + + public getConfig(): Record { + return this.config; + } + + public dumpConfig() { + const redactJsonKeys = fastRedact({ + paths: this.secretNamesToRedactInConfigDump, + censor: errorSanitizationMessage, + serialize: false, + }); + + try { + redactJsonKeys(this.config); + this.logger?.info(`Service config: ${JSON.stringify(this.config)}`); + Lumberjack.info(`Service config`, this.config); + } catch (err) { + this.logger?.error(`Log sanitization failed.`, err); + Lumberjack.error(`Log sanitization failed.`, undefined, err); + } + } +} diff --git a/server/routerlicious/packages/services-shared/src/index.ts b/server/routerlicious/packages/services-shared/src/index.ts index 583a31cc7a64..d82622924e5c 100644 --- a/server/routerlicious/packages/services-shared/src/index.ts +++ b/server/routerlicious/packages/services-shared/src/index.ts @@ -34,3 +34,4 @@ export { } from "./webServer"; export { WholeSummaryReadGitManager } from "./wholeSummaryReadGitManager"; export { WholeSummaryWriteGitManager } from "./wholeSummaryWriteGitManager"; +export { ConfigDumper } from "./configDumper"; diff --git a/server/routerlicious/packages/services-shared/src/runner.ts b/server/routerlicious/packages/services-shared/src/runner.ts index e1dfd266760e..42839c348dde 100644 --- a/server/routerlicious/packages/services-shared/src/runner.ts +++ b/server/routerlicious/packages/services-shared/src/runner.ts @@ -16,6 +16,7 @@ import { LumberEventName, CommonProperties, } from "@fluidframework/server-services-telemetry"; +import { ConfigDumper } from "./configDumper"; /** * Uses the provided factories to create and execute a runner. @@ -107,6 +108,17 @@ export function runService( .use("memory") : configOrPath; + const configDumpEnabled = (config.get("config:configDumpEnabled") as boolean) ?? false; + if (configDumpEnabled) { + const secretNamesToRedactInConfigDump = + (config.get("config:secretNamesToRedactInConfigDump") as string[]) ?? undefined; + const configDumper = new ConfigDumper( + config.get(), + logger, + secretNamesToRedactInConfigDump, + ); + configDumper.dumpConfig(); + } const waitInMs = waitBeforeExitInMs ?? 1000; const runnerMetric = Lumberjack.newLumberMetric(LumberEventName.RunService); const runningP = run(config, resourceFactory, runnerFactory, logger); diff --git a/server/routerlicious/packages/services-shared/src/test/configDumper.spec.ts b/server/routerlicious/packages/services-shared/src/test/configDumper.spec.ts new file mode 100644 index 000000000000..7cc35acf0955 --- /dev/null +++ b/server/routerlicious/packages/services-shared/src/test/configDumper.spec.ts @@ -0,0 +1,128 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import assert from "assert"; +import { ConfigDumper } from "../configDumper"; + +describe("ConfigDumper", () => { + describe("dumpConfig", () => { + it("should redact values in secretNamesToRedactInConfigDump", () => { + const nconf = { + key1: "value1", + key2: "value2", + }; + + const secretNamesToRedactInConfigDump = ["key1"]; + const configDumper = new ConfigDumper( + nconf, + undefined, + secretNamesToRedactInConfigDump, + ); + configDumper.dumpConfig(); + const redactedConfig = configDumper.getConfig(); + + assert.strictEqual(redactedConfig.key1, "FluidREDACTED"); + assert.strictEqual(redactedConfig.key2, "value2"); + }); + + it("redacted object should not be the same be as the config object if a value is redacted", () => { + const nconf = { + key1: "value1", + key2: "value2", + }; + + const secretNamesToRedactInConfigDump = ["key1"]; + const configDumper = new ConfigDumper( + nconf, + undefined, + secretNamesToRedactInConfigDump, + ); + configDumper.dumpConfig(); + const redactedConfig = configDumper.getConfig(); + + assert.notDeepStrictEqual(nconf, redactedConfig); + }); + + it("redacted object should not be the same be as the config object if no value is redacted", () => { + const nconf = { + key1: "value1", + key2: "value2", + }; + + const secretNamesToRedactInConfigDump = []; + const configDumper = new ConfigDumper( + nconf, + undefined, + secretNamesToRedactInConfigDump, + ); + configDumper.dumpConfig(); + const redactedConfig = configDumper.getConfig(); + + assert.deepStrictEqual(nconf, redactedConfig); + assert.notStrictEqual(nconf, redactedConfig); + }); + + it("should not throw an error if secretNamesToRedactInConfigDump values are not present in nconf", () => { + const nconf = { + key1: "value1", + key2: "value2", + }; + + const secretNamesToRedactInConfigDump = ["key3"]; + const configDumper = new ConfigDumper( + nconf, + undefined, + secretNamesToRedactInConfigDump, + ); + + assert.doesNotThrow(() => { + configDumper.dumpConfig(); + }); + }); + + it("should not throw an error if secretNamesToRedactInConfigDump contains duplicate values", () => { + const nconf = { + key1: "value1", + key2: "value2", + }; + + const secretNamesToRedactInConfigDump = ["key1", "key1"]; + const configDumper = new ConfigDumper( + nconf, + undefined, + secretNamesToRedactInConfigDump, + ); + + assert.doesNotThrow(() => { + configDumper.dumpConfig(); + }); + }); + + it("should redact keys inside nested objects in secretNamesToRedactInConfigDump", () => { + const nconf = { + key1: "value1", + key2: "value2", + nested: { + key3: "nestedValue1", + key4: "nestedValue2", + }, + }; + + const secretNamesToRedactInConfigDump = ["nested.key3"]; + const configDumper = new ConfigDumper( + nconf, + undefined, + secretNamesToRedactInConfigDump, + ); + configDumper.dumpConfig(); + const redactedConfig = configDumper.getConfig(); + + assert.strictEqual(redactedConfig.key1, "value1"); + assert.strictEqual(redactedConfig.key2, "value2"); + assert.strictEqual(redactedConfig.nested.key3, "FluidREDACTED"); + assert.strictEqual(redactedConfig.nested.key4, "nestedValue2"); + }); + }); +}); diff --git a/server/routerlicious/pnpm-lock.yaml b/server/routerlicious/pnpm-lock.yaml index 2d62a9a5a3cd..bec442eaaf87 100644 --- a/server/routerlicious/pnpm-lock.yaml +++ b/server/routerlicious/pnpm-lock.yaml @@ -1072,6 +1072,7 @@ importers: eslint: ~8.27.0 events: ^3.1.0 express: ^4.17.3 + fast-redact: ^3.3.0 ioredis: ^5.2.3 lodash: ^4.17.21 mocha: ^10.2.0 @@ -1101,6 +1102,7 @@ importers: body-parser: 1.20.2 debug: 4.3.4 events: 3.3.0 + fast-redact: 3.3.0 ioredis: 5.3.2 lodash: 4.17.21 nconf: 0.12.0 @@ -11272,6 +11274,11 @@ packages: resolution: {integrity: sha512-Ue0LwpDYErFbmNnZSF0UH6eImUwDmogUO1jyE+JbN2gsQz/jICm1Ve7t9QT0rNSsfJt+Hs4/S3GnsDVjL4HVrw==} dev: true + /fast-redact/3.3.0: + resolution: {integrity: sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==} + engines: {node: '>=6'} + dev: false + /fast-xml-parser/4.2.5: resolution: {integrity: sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==} hasBin: true From d8b0a8e8e3bbe022fcfbe49d4dfb3732772d8b28 Mon Sep 17 00:00:00 2001 From: yann-achard-MS <97201204+yann-achard-MS@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:45:56 -0800 Subject: [PATCH 08/20] test(tree2): fix test harness for exhaustive sequence editing (#18692) --- experimental/dds/tree2/src/test/shared-tree/editing.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts b/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts index f924d6cb522d..e82b75284737 100644 --- a/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts +++ b/experimental/dds/tree2/src/test/shared-tree/editing.spec.ts @@ -1845,6 +1845,8 @@ describe("Editing", () => { // Represented as an integer (0: removed, 1: present) to facilitate summing. // Used to compute the index of the next node to remove. const present = makeArray(nbPeers, () => makeArray(nbNodes, () => 1)); + // Same as `present` but for `tree` branch. + const presentOnTree = makeArray(nbNodes, () => 1); // The number of remaining undos available for each peer. const undoQueues: number[][] = makeArray(nbPeers, () => []); @@ -1880,13 +1882,15 @@ describe("Editing", () => { unreachableCase(step); } tree.merge(peer, false); + presentOnTree[affectedNode] = presence; // We only let peers with a higher index learn of this edit. // This breaks the symmetry between scenarios where the permutation of actions is the same // except for which peer does which set of actions. // It also helps simulate different peers learning of the same edit at different times. for (let downhillPeer = iPeer + 1; downhillPeer < nbPeers; downhillPeer++) { peers[downhillPeer].rebaseOnto(tree); - present[downhillPeer][affectedNode] = presence; + // The peer should now be in the same state as `tree`. + present[downhillPeer] = [...presentOnTree]; } present[iPeer][affectedNode] = presence; } From ea326458fe2ce16d746149ecc220447beb86cdd8 Mon Sep 17 00:00:00 2001 From: yann-achard-MS <97201204+yann-achard-MS@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:46:07 -0800 Subject: [PATCH 09/20] fix(tree2): revertibles should work after rebasing a branch (#18691) --- experimental/dds/tree2/src/shared-tree-core/branch.ts | 8 ++++++++ .../tree2/src/test/shared-tree-core/branch.spec.ts | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/experimental/dds/tree2/src/shared-tree-core/branch.ts b/experimental/dds/tree2/src/shared-tree-core/branch.ts index 24b239679769..32d6ef6946be 100644 --- a/experimental/dds/tree2/src/shared-tree-core/branch.ts +++ b/experimental/dds/tree2/src/shared-tree-core/branch.ts @@ -468,6 +468,14 @@ export class SharedTreeBranch exten const { newSourceHead, commits } = rebaseResult; const { deletedSourceCommits, targetCommits, sourceCommits } = commits; + // It's possible that the target branch already contained some of the commits that + // were on this branch. When that's the case, we adopt the commit objects from the target branch. + // Because of that, we need to make sure that any revertibles that were based on the old commit objects + // now point to the new object that were adopted from the target branch. + for (const targetCommit of targetCommits) { + this.updateRevertibleCommit(targetCommit); + } + const newCommits = targetCommits.concat(sourceCommits); const changeEvent = { type: "replace", diff --git a/experimental/dds/tree2/src/test/shared-tree-core/branch.spec.ts b/experimental/dds/tree2/src/test/shared-tree-core/branch.spec.ts index a728e8a0e244..bbe89af3fff4 100644 --- a/experimental/dds/tree2/src/test/shared-tree-core/branch.spec.ts +++ b/experimental/dds/tree2/src/test/shared-tree-core/branch.spec.ts @@ -22,6 +22,7 @@ import { } from "../../feature-libraries"; import { brand, fail } from "../../util"; import { noopValidator } from "../../codec"; +import { createTestUndoRedoStacks } from "../utils"; const defaultChangeFamily = new DefaultChangeFamily({ jsonValidator: noopValidator }); @@ -157,6 +158,7 @@ describe("Branches", () => { // Create a parent branch and a child fork const parent = create(); const child = parent.fork(); + const stacks = createTestUndoRedoStacks(child); // Apply a change to the parent const tagParent = change(parent); // Apply a change to the child @@ -170,6 +172,15 @@ describe("Branches", () => { child.rebaseOnto(parent); assertBased(child, parent); assertHistory(child, tagParent, tagChild, tagParent2, tagChild2); + + // It should still be possible to revert the the child branch's revertibles + assert.equal(stacks.undoStack.length, 2); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + stacks.undoStack.pop()!.revert(); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + stacks.undoStack.pop()!.revert(); + + stacks.unsubscribe(); }); it("emit a change event after each change", () => { From dd05c2e51c1583e97d93be73ae0abe1f7ba30d94 Mon Sep 17 00:00:00 2001 From: Joshua Smithrud <54606601+Josmithr@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:11:08 -0800 Subject: [PATCH 10/20] build: Enable trimmed d.ts rollup generation by default (#18696) Enables d.ts trimmed rollup generation by default across the repo. Makes no changes to how the rollups are consumed, just adds generation to the build. --- azure/packages/azure-client/api-extractor.json | 3 --- azure/packages/azure-service-utils/api-extractor.json | 5 +---- common/build/build-common/api-extractor-base.json | 2 +- experimental/dds/tree2/api-extractor.json | 3 --- packages/common/client-utils/api-extractor.json | 3 --- packages/common/container-definitions/api-extractor.json | 3 --- packages/common/core-interfaces/api-extractor.json | 3 --- packages/common/core-utils/api-extractor.json | 3 --- packages/common/driver-definitions/api-extractor.json | 3 --- packages/dds/cell/api-extractor.json | 3 --- packages/dds/counter/api-extractor.json | 3 --- packages/dds/ink/api-extractor.json | 3 --- packages/dds/map/api-extractor.json | 3 --- packages/dds/matrix/api-extractor.json | 3 --- packages/dds/merge-tree/api-extractor.json | 3 --- packages/dds/ordered-collection/api-extractor.json | 3 --- packages/dds/pact-map/api-extractor.json | 3 --- packages/dds/register-collection/api-extractor.json | 3 --- packages/dds/sequence/api-extractor.json | 3 --- packages/dds/shared-object-base/api-extractor.json | 3 --- packages/dds/shared-summary-block/api-extractor.json | 3 --- packages/dds/task-manager/api-extractor.json | 3 --- packages/dds/test-dds-utils/api-extractor.json | 3 --- packages/drivers/debugger/api-extractor.json | 3 --- packages/drivers/driver-base/api-extractor.json | 3 --- packages/drivers/driver-web-cache/api-extractor.json | 3 --- packages/drivers/file-driver/api-extractor.json | 3 --- .../drivers/fluidapp-odsp-urlResolver/api-extractor.json | 3 --- packages/drivers/local-driver/api-extractor.json | 3 --- packages/drivers/odsp-driver-definitions/api-extractor.json | 3 --- packages/drivers/odsp-driver/api-extractor.json | 3 --- packages/drivers/odsp-urlResolver/api-extractor.json | 3 --- packages/drivers/replay-driver/api-extractor.json | 3 --- packages/drivers/routerlicious-driver/api-extractor.json | 3 --- .../drivers/routerlicious-urlResolver/api-extractor.json | 3 --- packages/drivers/tinylicious-driver/api-extractor.json | 3 --- packages/framework/agent-scheduler/api-extractor.json | 3 --- packages/framework/aqueduct/api-extractor.json | 3 --- packages/framework/attributor/api-extractor.json | 3 --- .../client-logger/app-insights-logger/api-extractor.json | 3 --- packages/framework/data-object-base/api-extractor.json | 3 --- packages/framework/dds-interceptions/api-extractor.json | 5 +---- packages/framework/fluid-framework/api-extractor.json | 5 +---- packages/framework/fluid-static/api-extractor.json | 3 --- packages/framework/oldest-client-observer/api-extractor.json | 3 --- packages/framework/request-handler/api-extractor.json | 3 --- packages/framework/synthesize/api-extractor.json | 3 --- packages/framework/tinylicious-client/api-extractor.json | 3 --- packages/framework/undo-redo/api-extractor.json | 3 --- packages/framework/view-adapters/api-extractor.json | 3 --- packages/framework/view-interfaces/api-extractor.json | 3 --- packages/loader/container-loader/api-extractor.json | 3 --- packages/loader/driver-utils/api-extractor.json | 3 --- .../loader/location-redirection-utils/api-extractor.json | 5 +---- packages/loader/test-loader-utils/api-extractor.json | 3 --- .../runtime/container-runtime-definitions/api-extractor.json | 3 --- packages/runtime/container-runtime/api-extractor.json | 3 --- packages/runtime/datastore-definitions/api-extractor.json | 3 --- packages/runtime/datastore/api-extractor.json | 3 --- packages/runtime/runtime-definitions/api-extractor.json | 3 --- packages/runtime/runtime-utils/api-extractor.json | 3 --- packages/runtime/test-runtime-utils/api-extractor.json | 3 --- packages/service-clients/odsp-client/api-extractor.json | 3 --- packages/test/mocha-test-setup/api-extractor.json | 3 --- packages/test/stochastic-test-utils/api-extractor.json | 3 --- packages/test/test-driver-definitions/api-extractor.json | 3 --- packages/test/test-drivers/api-extractor.json | 3 --- packages/test/test-pairwise-generator/api-extractor.json | 3 --- packages/test/test-utils/api-extractor.json | 3 --- packages/test/test-version-utils/api-extractor.json | 3 --- packages/tools/devtools/devtools-core/api-extractor.json | 3 --- packages/tools/devtools/devtools-view/api-extractor.json | 5 +---- packages/tools/devtools/devtools/api-extractor.json | 5 +---- packages/tools/replay-tool/api-extractor.json | 5 +---- packages/utils/odsp-doclib-utils/api-extractor.json | 3 --- packages/utils/telemetry-utils/api-extractor.json | 3 --- packages/utils/tool-utils/api-extractor.json | 3 --- tools/api-markdown-documenter/api-extractor.json | 5 +---- 78 files changed, 9 insertions(+), 240 deletions(-) diff --git a/azure/packages/azure-client/api-extractor.json b/azure/packages/azure-client/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/azure/packages/azure-client/api-extractor.json +++ b/azure/packages/azure-client/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/azure/packages/azure-service-utils/api-extractor.json b/azure/packages/azure-service-utils/api-extractor.json index 3cd83b2483bb..34b78596056c 100644 --- a/azure/packages/azure-service-utils/api-extractor.json +++ b/azure/packages/azure-service-utils/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../../common/build/build-common/api-extractor-base.json" } diff --git a/common/build/build-common/api-extractor-base.json b/common/build/build-common/api-extractor-base.json index 60a3bbc8ff10..693b983142b0 100644 --- a/common/build/build-common/api-extractor-base.json +++ b/common/build/build-common/api-extractor-base.json @@ -35,7 +35,7 @@ * Configures how the .d.ts rollup file will be generated. */ "dtsRollup": { - "enabled": false, + "enabled": true, "alphaTrimmedFilePath": "/dist/-alpha.d.ts", "betaTrimmedFilePath": "/dist/-beta.d.ts", "publicTrimmedFilePath": "/dist/-public.d.ts", diff --git a/experimental/dds/tree2/api-extractor.json b/experimental/dds/tree2/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/experimental/dds/tree2/api-extractor.json +++ b/experimental/dds/tree2/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/common/client-utils/api-extractor.json b/packages/common/client-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/common/client-utils/api-extractor.json +++ b/packages/common/client-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/common/container-definitions/api-extractor.json b/packages/common/container-definitions/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/common/container-definitions/api-extractor.json +++ b/packages/common/container-definitions/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/common/core-interfaces/api-extractor.json b/packages/common/core-interfaces/api-extractor.json index 457d8f530f12..58230b469fc5 100644 --- a/packages/common/core-interfaces/api-extractor.json +++ b/packages/common/core-interfaces/api-extractor.json @@ -8,8 +8,5 @@ "logLevel": "none" } } - }, - "dtsRollup": { - "enabled": true } } diff --git a/packages/common/core-utils/api-extractor.json b/packages/common/core-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/common/core-utils/api-extractor.json +++ b/packages/common/core-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/common/driver-definitions/api-extractor.json b/packages/common/driver-definitions/api-extractor.json index 457d8f530f12..58230b469fc5 100644 --- a/packages/common/driver-definitions/api-extractor.json +++ b/packages/common/driver-definitions/api-extractor.json @@ -8,8 +8,5 @@ "logLevel": "none" } } - }, - "dtsRollup": { - "enabled": true } } diff --git a/packages/dds/cell/api-extractor.json b/packages/dds/cell/api-extractor.json index 457d8f530f12..58230b469fc5 100644 --- a/packages/dds/cell/api-extractor.json +++ b/packages/dds/cell/api-extractor.json @@ -8,8 +8,5 @@ "logLevel": "none" } } - }, - "dtsRollup": { - "enabled": true } } diff --git a/packages/dds/counter/api-extractor.json b/packages/dds/counter/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/counter/api-extractor.json +++ b/packages/dds/counter/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/ink/api-extractor.json b/packages/dds/ink/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/ink/api-extractor.json +++ b/packages/dds/ink/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/map/api-extractor.json b/packages/dds/map/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/map/api-extractor.json +++ b/packages/dds/map/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/matrix/api-extractor.json b/packages/dds/matrix/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/matrix/api-extractor.json +++ b/packages/dds/matrix/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/merge-tree/api-extractor.json b/packages/dds/merge-tree/api-extractor.json index b9cd3178bf72..84c7489b5a91 100644 --- a/packages/dds/merge-tree/api-extractor.json +++ b/packages/dds/merge-tree/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Fix violations and remove this rule override diff --git a/packages/dds/ordered-collection/api-extractor.json b/packages/dds/ordered-collection/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/ordered-collection/api-extractor.json +++ b/packages/dds/ordered-collection/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/pact-map/api-extractor.json b/packages/dds/pact-map/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/pact-map/api-extractor.json +++ b/packages/dds/pact-map/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/register-collection/api-extractor.json b/packages/dds/register-collection/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/register-collection/api-extractor.json +++ b/packages/dds/register-collection/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/sequence/api-extractor.json b/packages/dds/sequence/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/sequence/api-extractor.json +++ b/packages/dds/sequence/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/shared-object-base/api-extractor.json b/packages/dds/shared-object-base/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/shared-object-base/api-extractor.json +++ b/packages/dds/shared-object-base/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/shared-summary-block/api-extractor.json b/packages/dds/shared-summary-block/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/shared-summary-block/api-extractor.json +++ b/packages/dds/shared-summary-block/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/task-manager/api-extractor.json b/packages/dds/task-manager/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/task-manager/api-extractor.json +++ b/packages/dds/task-manager/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/dds/test-dds-utils/api-extractor.json b/packages/dds/test-dds-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/dds/test-dds-utils/api-extractor.json +++ b/packages/dds/test-dds-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/debugger/api-extractor.json b/packages/drivers/debugger/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/debugger/api-extractor.json +++ b/packages/drivers/debugger/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/driver-base/api-extractor.json b/packages/drivers/driver-base/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/driver-base/api-extractor.json +++ b/packages/drivers/driver-base/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/driver-web-cache/api-extractor.json b/packages/drivers/driver-web-cache/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/driver-web-cache/api-extractor.json +++ b/packages/drivers/driver-web-cache/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/file-driver/api-extractor.json b/packages/drivers/file-driver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/file-driver/api-extractor.json +++ b/packages/drivers/file-driver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/fluidapp-odsp-urlResolver/api-extractor.json b/packages/drivers/fluidapp-odsp-urlResolver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/fluidapp-odsp-urlResolver/api-extractor.json +++ b/packages/drivers/fluidapp-odsp-urlResolver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/local-driver/api-extractor.json b/packages/drivers/local-driver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/local-driver/api-extractor.json +++ b/packages/drivers/local-driver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/odsp-driver-definitions/api-extractor.json b/packages/drivers/odsp-driver-definitions/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/odsp-driver-definitions/api-extractor.json +++ b/packages/drivers/odsp-driver-definitions/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/odsp-driver/api-extractor.json b/packages/drivers/odsp-driver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/odsp-driver/api-extractor.json +++ b/packages/drivers/odsp-driver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/odsp-urlResolver/api-extractor.json b/packages/drivers/odsp-urlResolver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/odsp-urlResolver/api-extractor.json +++ b/packages/drivers/odsp-urlResolver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/replay-driver/api-extractor.json b/packages/drivers/replay-driver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/replay-driver/api-extractor.json +++ b/packages/drivers/replay-driver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/routerlicious-driver/api-extractor.json b/packages/drivers/routerlicious-driver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/routerlicious-driver/api-extractor.json +++ b/packages/drivers/routerlicious-driver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/routerlicious-urlResolver/api-extractor.json b/packages/drivers/routerlicious-urlResolver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/routerlicious-urlResolver/api-extractor.json +++ b/packages/drivers/routerlicious-urlResolver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/drivers/tinylicious-driver/api-extractor.json b/packages/drivers/tinylicious-driver/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/drivers/tinylicious-driver/api-extractor.json +++ b/packages/drivers/tinylicious-driver/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/agent-scheduler/api-extractor.json b/packages/framework/agent-scheduler/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/agent-scheduler/api-extractor.json +++ b/packages/framework/agent-scheduler/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/aqueduct/api-extractor.json b/packages/framework/aqueduct/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/aqueduct/api-extractor.json +++ b/packages/framework/aqueduct/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/attributor/api-extractor.json b/packages/framework/attributor/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/attributor/api-extractor.json +++ b/packages/framework/attributor/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/client-logger/app-insights-logger/api-extractor.json b/packages/framework/client-logger/app-insights-logger/api-extractor.json index 40764e08e832..fec4322f1134 100644 --- a/packages/framework/client-logger/app-insights-logger/api-extractor.json +++ b/packages/framework/client-logger/app-insights-logger/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { // The following overrides are workarounds for API-Extractor incorrectly running analysis on our application // insights dependency. diff --git a/packages/framework/data-object-base/api-extractor.json b/packages/framework/data-object-base/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/data-object-base/api-extractor.json +++ b/packages/framework/data-object-base/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/dds-interceptions/api-extractor.json b/packages/framework/dds-interceptions/api-extractor.json index 3cd83b2483bb..34b78596056c 100644 --- a/packages/framework/dds-interceptions/api-extractor.json +++ b/packages/framework/dds-interceptions/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../../common/build/build-common/api-extractor-base.json" } diff --git a/packages/framework/fluid-framework/api-extractor.json b/packages/framework/fluid-framework/api-extractor.json index 3cd83b2483bb..34b78596056c 100644 --- a/packages/framework/fluid-framework/api-extractor.json +++ b/packages/framework/fluid-framework/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../../common/build/build-common/api-extractor-base.json" } diff --git a/packages/framework/fluid-static/api-extractor.json b/packages/framework/fluid-static/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/fluid-static/api-extractor.json +++ b/packages/framework/fluid-static/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/oldest-client-observer/api-extractor.json b/packages/framework/oldest-client-observer/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/oldest-client-observer/api-extractor.json +++ b/packages/framework/oldest-client-observer/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/request-handler/api-extractor.json b/packages/framework/request-handler/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/request-handler/api-extractor.json +++ b/packages/framework/request-handler/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/synthesize/api-extractor.json b/packages/framework/synthesize/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/synthesize/api-extractor.json +++ b/packages/framework/synthesize/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/tinylicious-client/api-extractor.json b/packages/framework/tinylicious-client/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/tinylicious-client/api-extractor.json +++ b/packages/framework/tinylicious-client/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/undo-redo/api-extractor.json b/packages/framework/undo-redo/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/undo-redo/api-extractor.json +++ b/packages/framework/undo-redo/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/view-adapters/api-extractor.json b/packages/framework/view-adapters/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/view-adapters/api-extractor.json +++ b/packages/framework/view-adapters/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/framework/view-interfaces/api-extractor.json b/packages/framework/view-interfaces/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/framework/view-interfaces/api-extractor.json +++ b/packages/framework/view-interfaces/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/loader/container-loader/api-extractor.json b/packages/loader/container-loader/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/loader/container-loader/api-extractor.json +++ b/packages/loader/container-loader/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/loader/driver-utils/api-extractor.json b/packages/loader/driver-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/loader/driver-utils/api-extractor.json +++ b/packages/loader/driver-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/loader/location-redirection-utils/api-extractor.json b/packages/loader/location-redirection-utils/api-extractor.json index 3cd83b2483bb..34b78596056c 100644 --- a/packages/loader/location-redirection-utils/api-extractor.json +++ b/packages/loader/location-redirection-utils/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../../common/build/build-common/api-extractor-base.json" } diff --git a/packages/loader/test-loader-utils/api-extractor.json b/packages/loader/test-loader-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/loader/test-loader-utils/api-extractor.json +++ b/packages/loader/test-loader-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/container-runtime-definitions/api-extractor.json b/packages/runtime/container-runtime-definitions/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/container-runtime-definitions/api-extractor.json +++ b/packages/runtime/container-runtime-definitions/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/container-runtime/api-extractor.json b/packages/runtime/container-runtime/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/container-runtime/api-extractor.json +++ b/packages/runtime/container-runtime/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/datastore-definitions/api-extractor.json b/packages/runtime/datastore-definitions/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/datastore-definitions/api-extractor.json +++ b/packages/runtime/datastore-definitions/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/datastore/api-extractor.json b/packages/runtime/datastore/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/datastore/api-extractor.json +++ b/packages/runtime/datastore/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/runtime-definitions/api-extractor.json b/packages/runtime/runtime-definitions/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/runtime-definitions/api-extractor.json +++ b/packages/runtime/runtime-definitions/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/runtime-utils/api-extractor.json b/packages/runtime/runtime-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/runtime-utils/api-extractor.json +++ b/packages/runtime/runtime-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/runtime/test-runtime-utils/api-extractor.json b/packages/runtime/test-runtime-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/runtime/test-runtime-utils/api-extractor.json +++ b/packages/runtime/test-runtime-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/service-clients/odsp-client/api-extractor.json b/packages/service-clients/odsp-client/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/service-clients/odsp-client/api-extractor.json +++ b/packages/service-clients/odsp-client/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/mocha-test-setup/api-extractor.json b/packages/test/mocha-test-setup/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/mocha-test-setup/api-extractor.json +++ b/packages/test/mocha-test-setup/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/stochastic-test-utils/api-extractor.json b/packages/test/stochastic-test-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/stochastic-test-utils/api-extractor.json +++ b/packages/test/stochastic-test-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/test-driver-definitions/api-extractor.json b/packages/test/test-driver-definitions/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/test-driver-definitions/api-extractor.json +++ b/packages/test/test-driver-definitions/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/test-drivers/api-extractor.json b/packages/test/test-drivers/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/test-drivers/api-extractor.json +++ b/packages/test/test-drivers/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/test-pairwise-generator/api-extractor.json b/packages/test/test-pairwise-generator/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/test-pairwise-generator/api-extractor.json +++ b/packages/test/test-pairwise-generator/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/test-utils/api-extractor.json b/packages/test/test-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/test-utils/api-extractor.json +++ b/packages/test/test-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/test/test-version-utils/api-extractor.json b/packages/test/test-version-utils/api-extractor.json index e38853e5d61e..58230b469fc5 100644 --- a/packages/test/test-version-utils/api-extractor.json +++ b/packages/test/test-version-utils/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, "messages": { "extractorMessageReporting": { // TODO: Add missing documentation and remove this rule override diff --git a/packages/tools/devtools/devtools-core/api-extractor.json b/packages/tools/devtools/devtools-core/api-extractor.json index ab7fadc2c07d..b56fc5bdcea1 100644 --- a/packages/tools/devtools/devtools-core/api-extractor.json +++ b/packages/tools/devtools/devtools-core/api-extractor.json @@ -1,9 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "../../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - }, // TODO: Fix violations and remove these rule overrides "messages": { "extractorMessageReporting": { diff --git a/packages/tools/devtools/devtools-view/api-extractor.json b/packages/tools/devtools/devtools-view/api-extractor.json index 60d6e12d947c..1c52bcc6f1bb 100644 --- a/packages/tools/devtools/devtools-view/api-extractor.json +++ b/packages/tools/devtools/devtools-view/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../../../common/build/build-common/api-extractor-base.json" } diff --git a/packages/tools/devtools/devtools/api-extractor.json b/packages/tools/devtools/devtools/api-extractor.json index 60d6e12d947c..1c52bcc6f1bb 100644 --- a/packages/tools/devtools/devtools/api-extractor.json +++ b/packages/tools/devtools/devtools/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../../../common/build/build-common/api-extractor-base.json" } diff --git a/packages/tools/replay-tool/api-extractor.json b/packages/tools/replay-tool/api-extractor.json index 360d7ef44ea1..55b185cd45ad 100644 --- a/packages/tools/replay-tool/api-extractor.json +++ b/packages/tools/replay-tool/api-extractor.json @@ -1,8 +1,5 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", "extends": "@fluidframework/build-common/api-extractor-common.json", - "mainEntryPointFilePath": "/dist/replayTool.d.ts", - "dtsRollup": { - "enabled": true - } + "mainEntryPointFilePath": "/dist/replayTool.d.ts" } diff --git a/packages/utils/odsp-doclib-utils/api-extractor.json b/packages/utils/odsp-doclib-utils/api-extractor.json index 457d8f530f12..58230b469fc5 100644 --- a/packages/utils/odsp-doclib-utils/api-extractor.json +++ b/packages/utils/odsp-doclib-utils/api-extractor.json @@ -8,8 +8,5 @@ "logLevel": "none" } } - }, - "dtsRollup": { - "enabled": true } } diff --git a/packages/utils/telemetry-utils/api-extractor.json b/packages/utils/telemetry-utils/api-extractor.json index 457d8f530f12..58230b469fc5 100644 --- a/packages/utils/telemetry-utils/api-extractor.json +++ b/packages/utils/telemetry-utils/api-extractor.json @@ -8,8 +8,5 @@ "logLevel": "none" } } - }, - "dtsRollup": { - "enabled": true } } diff --git a/packages/utils/tool-utils/api-extractor.json b/packages/utils/tool-utils/api-extractor.json index 457d8f530f12..58230b469fc5 100644 --- a/packages/utils/tool-utils/api-extractor.json +++ b/packages/utils/tool-utils/api-extractor.json @@ -8,8 +8,5 @@ "logLevel": "none" } } - }, - "dtsRollup": { - "enabled": true } } diff --git a/tools/api-markdown-documenter/api-extractor.json b/tools/api-markdown-documenter/api-extractor.json index d44e848ec2e0..3158b833a46d 100644 --- a/tools/api-markdown-documenter/api-extractor.json +++ b/tools/api-markdown-documenter/api-extractor.json @@ -1,7 +1,4 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "extends": "../../common/build/build-common/api-extractor-base.json", - "dtsRollup": { - "enabled": true - } + "extends": "../../common/build/build-common/api-extractor-base.json" } From 427186fed58f71cc9f7f419f44b32fd83ba5ca93 Mon Sep 17 00:00:00 2001 From: justus-camp <102552460+justus-camp@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:26:05 -0800 Subject: [PATCH 11/20] Add a no-op RevisionTag codec (#18627) ## Description Adds a no-op RevisionTag codec with the intent of supplying the runtime's IdCompressor in a subsequent PR to translate between Id spaces. ## Reviewer Guidance The review process is outlined on [this wiki page](https://github.com/microsoft/FluidFramework/wiki/PR-Guidelines#guidelines). --- experimental/dds/tree2/src/core/index.ts | 2 + .../dds/tree2/src/core/rebase/index.ts | 2 + .../dds/tree2/src/core/rebase/types.ts | 8 ++- .../modular-schema/fieldChangeHandler.ts | 6 +- .../feature-libraries/modular-schema/index.ts | 6 +- .../modular-schema/modularChangeCodecs.ts | 59 +++++++++++++++-- .../modular-schema/modularChangeFamily.ts | 7 +- .../modular-schema/modularChangeFormat.ts | 4 +- .../optional-field/optionalFieldCodecs.ts | 65 +++++++++++++------ .../sequence-field/sequenceFieldCodecs.ts | 65 +++++++++++++------ .../dds/tree2/src/feature-libraries/utils.ts | 35 ++++++++++ .../src/shared-tree-core/editManagerCodecs.ts | 16 +++-- .../src/shared-tree-core/editManagerFormat.ts | 15 ++++- .../shared-tree-core/editManagerSummarizer.ts | 5 +- .../dds/tree2/src/shared-tree-core/index.ts | 9 ++- .../src/shared-tree-core/messageCodecs.ts | 6 +- .../src/shared-tree-core/messageFormat.ts | 4 +- .../src/shared-tree-core/revisionTagCodecs.ts | 16 +++++ .../src/shared-tree-core/sharedTreeCore.ts | 5 +- .../editManagerCodecs.spec.ts | 12 ++-- .../modular-schema/genericFieldKind.spec.ts | 7 +- .../optionalFieldChangeCodecs.spec.ts | 6 +- .../sequenceFieldCodecs.spec.ts | 6 +- .../src/test/shared-tree-core/message.spec.ts | 4 +- .../files/optional-field-scenarios-final.json | 4 +- 25 files changed, 300 insertions(+), 74 deletions(-) create mode 100644 experimental/dds/tree2/src/feature-libraries/utils.ts create mode 100644 experimental/dds/tree2/src/shared-tree-core/revisionTagCodecs.ts diff --git a/experimental/dds/tree2/src/core/index.ts b/experimental/dds/tree2/src/core/index.ts index 1d5a18b478a6..978fdf560c0a 100644 --- a/experimental/dds/tree2/src/core/index.ts +++ b/experimental/dds/tree2/src/core/index.ts @@ -171,6 +171,8 @@ export { RevisionMetadataSource, revisionMetadataSourceFromInfo, RevisionInfo, + EncodedRevisionTag, + EncodedChangeAtomId, } from "./rebase"; export { diff --git a/experimental/dds/tree2/src/core/rebase/index.ts b/experimental/dds/tree2/src/core/rebase/index.ts index 63c62dbf2fde..83093f9d0b3d 100644 --- a/experimental/dds/tree2/src/core/rebase/index.ts +++ b/experimental/dds/tree2/src/core/rebase/index.ts @@ -12,6 +12,8 @@ export { GraphCommit, RevisionTag, RevisionTagSchema, + EncodedRevisionTag, + EncodedChangeAtomId, ChangesetLocalId, ChangeAtomId, ChangeAtomIdMap, diff --git a/experimental/dds/tree2/src/core/rebase/types.ts b/experimental/dds/tree2/src/core/rebase/types.ts index 5ac5f6cb0aa7..28717d03517e 100644 --- a/experimental/dds/tree2/src/core/rebase/types.ts +++ b/experimental/dds/tree2/src/core/rebase/types.ts @@ -21,7 +21,8 @@ export const SessionIdSchema = brandedStringType(); */ // TODO: These can be compressed by an `IdCompressor` in the future export type RevisionTag = StableId; -export const RevisionTagSchema = brandedStringType(); +export type EncodedRevisionTag = Brand; +export const RevisionTagSchema = brandedStringType(); /** * An ID which is unique within a revision of a `ModularChangeset`. @@ -50,6 +51,11 @@ export interface ChangeAtomId { readonly localId: ChangesetLocalId; } +export interface EncodedChangeAtomId { + readonly revision?: EncodedRevisionTag; + readonly localId: ChangesetLocalId; +} + /** * @alpha */ diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts index 8419d6a006cf..553bc5446a4a 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts @@ -10,6 +10,7 @@ import { DeltaFieldMap, DeltaFieldChanges, DeltaDetachedNodeId, + EncodedRevisionTag, } from "../../core"; import { fail, IdAllocator, Invariant } from "../../util"; import { ICodecFamily, IJsonCodec } from "../../codec"; @@ -27,7 +28,10 @@ export interface FieldChangeHandler< > { _typeCheck?: Invariant; readonly rebaser: FieldChangeRebaser; - readonly codecsFactory: (childCodec: IJsonCodec) => ICodecFamily; + readonly codecsFactory: ( + childCodec: IJsonCodec, + revisionTagCodec: IJsonCodec, + ) => ICodecFamily; readonly editor: TEditor; intoDelta( change: TaggedChange, diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts index c66ee221b84e..d1f06f07bccf 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/index.ts @@ -19,7 +19,11 @@ export { CrossFieldTarget, setInCrossFieldMap, } from "./crossFieldQueries"; -export { ChangesetLocalIdSchema, EncodedChangeAtomId } from "./modularChangeFormat"; +export { + ChangesetLocalIdSchema, + EncodedChangeAtomId, + EncodedRevisionInfo, +} from "./modularChangeFormat"; export { FieldKind, FullSchemaPolicy, FieldKindWithEditor } from "./fieldKind"; export { FieldChangeHandler, diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeCodecs.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeCodecs.ts index cc6b0a0e3c60..a5c03edc5b5e 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeCodecs.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeCodecs.ts @@ -7,6 +7,7 @@ import { TAnySchema } from "@sinclair/typebox"; import { assert } from "@fluidframework/core-utils"; import { ChangesetLocalId, + EncodedRevisionTag, FieldKey, FieldKindIdentifier, RevisionInfo, @@ -42,10 +43,12 @@ import { EncodedFieldChangeMap, EncodedModularChangeset, EncodedNodeChangeset, + EncodedRevisionInfo, } from "./modularChangeFormat"; function makeV0Codec( fieldKinds: ReadonlyMap, + revisionTagCodec: IJsonCodec, { jsonValidator: validator }: ICodecOptions, ): IJsonCodec { const nodeChangesetCodec: IJsonCodec = { @@ -55,7 +58,9 @@ function makeV0Codec( }; const getMapEntry = (field: FieldKindWithEditor) => { - const codec = field.changeHandler.codecsFactory(nodeChangesetCodec).resolve(0); + const codec = field.changeHandler + .codecsFactory(nodeChangesetCodec, revisionTagCodec) + .resolve(0); return { codec, compiledSchema: codec.json.encodedSchema @@ -167,7 +172,7 @@ function makeV0Codec( // `undefined` does not round-trip through JSON strings, so it needs special handling. // Most entries will have an undefined revision due to the revision information being inherited from the `ModularChangeset`. // We therefore optimize for the common case by omitting the revision when it is undefined. - r !== undefined ? [r, i, t] : [i, t], + r !== undefined ? [revisionTagCodec.encode(r), i, t] : [i, t], ); return encoded.length === 0 ? undefined : encoded; } @@ -177,16 +182,57 @@ function makeV0Codec( return undefined; } const list: [RevisionTag | undefined, ChangesetLocalId, any][] = encoded.map((tuple) => - tuple.length === 3 ? tuple : [undefined, ...tuple], + tuple.length === 3 + ? [revisionTagCodec.decode(tuple[0]), tuple[1], tuple[2]] + : [undefined, ...tuple], ); return nestedMapFromFlatList(list); } + function encodeRevisionInfos(revisions: readonly RevisionInfo[]): EncodedRevisionInfo[] { + const encodedRevisions = []; + for (const revision of revisions) { + const encodedRevision: Mutable = { + revision: revisionTagCodec.encode(revision.revision), + }; + + if (revision.rollbackOf !== undefined) { + encodedRevision.rollbackOf = revisionTagCodec.encode(revision.rollbackOf); + } + + encodedRevisions.push(encodedRevision); + } + + return encodedRevisions; + } + + function decodeRevisionInfos(revisions: readonly EncodedRevisionInfo[]): RevisionInfo[] { + const decodedRevisions = []; + for (const revision of revisions) { + const decodedRevision: Mutable = { + revision: revisionTagCodec.decode(revision.revision), + }; + + if (revision.rollbackOf !== undefined) { + decodedRevision.rollbackOf = revisionTagCodec.decode(revision.rollbackOf); + } + + decodedRevisions.push(decodedRevision); + } + + return decodedRevisions; + } + return { encode: (change) => { return { maxId: change.maxId, - revisions: change.revisions as readonly RevisionInfo[] & JsonCompatibleReadOnly, + revisions: + change.revisions === undefined + ? change.revisions + : (encodeRevisionInfos( + change.revisions, + ) as unknown as readonly RevisionInfo[] & JsonCompatibleReadOnly), changes: encodeFieldChangesForJson(change.fieldChanges), builds: encodeBuilds(change.builds), }; @@ -200,7 +246,7 @@ function makeV0Codec( decoded.builds = decodeBuilds(encodedChange.builds); } if (encodedChange.revisions !== undefined) { - decoded.revisions = encodedChange.revisions; + decoded.revisions = decodeRevisionInfos(encodedChange.revisions); } if (encodedChange.maxId !== undefined) { decoded.maxId = encodedChange.maxId; @@ -213,7 +259,8 @@ function makeV0Codec( export function makeModularChangeCodecFamily( fieldKinds: ReadonlyMap, + revisionTagCodec: IJsonCodec, options: ICodecOptions, ): ICodecFamily { - return makeCodecFamily([[0, makeV0Codec(fieldKinds, options)]]); + return makeCodecFamily([[0, makeV0Codec(fieldKinds, revisionTagCodec, options)]]); } diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts index 1c07345ebbd9..b58125338add 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts @@ -35,6 +35,7 @@ import { mapCursorField, StoredSchemaCollection, } from "../../core"; +import { RevisionTagCodec } from "../../shared-tree-core"; import { brand, forEachInNestedMap, @@ -98,7 +99,11 @@ export class ModularChangeFamily public readonly fieldKinds: ReadonlyMap, codecOptions: ICodecOptions, ) { - this.codecs = makeModularChangeCodecFamily(this.fieldKinds, codecOptions); + this.codecs = makeModularChangeCodecFamily( + this.fieldKinds, + new RevisionTagCodec(), + codecOptions, + ); } public get rebaser(): ChangeRebaser { diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFormat.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFormat.ts index 69c9d3514ce7..4d650e256936 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFormat.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFormat.ts @@ -102,7 +102,7 @@ export const EncodedNodeChangeset = Type.Object( */ export type EncodedNodeChangeset = Static; -const EncodedRevisionInfo = Type.Object( +export const EncodedRevisionInfo = Type.Object( { revision: Type.Readonly(RevisionTagSchema), rollbackOf: Type.ReadonlyOptional(RevisionTagSchema), @@ -110,6 +110,8 @@ const EncodedRevisionInfo = Type.Object( noAdditionalProps, ); +export type EncodedRevisionInfo = Static; + // TODO:YA6307 adopt more efficient encoding, likely based on contiguous runs of IDs export const EncodedBuilds = Type.Array( Type.Union([ diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts index f743115d0332..e1df17a9a7f9 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts @@ -4,7 +4,10 @@ */ import { TAnySchema, Type } from "@sinclair/typebox"; +import { decodeChangeAtomId, encodeChangeAtomId } from "../utils"; import { ICodecFamily, IJsonCodec, makeCodecFamily, unitCodec } from "../../codec"; +import { EncodedRevisionTag, RevisionTag } from "../../core"; +import { Mutable } from "../../util"; import type { NodeChangeset } from "../modular-schema"; import type { OptionalChangeset, RegisterId } from "./optionalFieldChangeTypes"; import { @@ -17,35 +20,55 @@ export const noChangeCodecFamily: ICodecFamily<0> = makeCodecFamily([[0, unitCod export const makeOptionalFieldCodecFamily = ( childCodec: IJsonCodec, -): ICodecFamily => makeCodecFamily([[0, makeOptionalFieldCodec(childCodec)]]); + revisionTagCodec: IJsonCodec, +): ICodecFamily => + makeCodecFamily([[0, makeOptionalFieldCodec(childCodec, revisionTagCodec)]]); -const registerIdCodec: IJsonCodec = { - encode: (registerId: RegisterId) => { - if (registerId === "self") { - return 0; - } +function makeRegisterIdCodec( + revisionTagCodec: IJsonCodec, +): IJsonCodec { + return { + encode: (registerId: RegisterId) => { + if (registerId === "self") { + return 0; + } + + const encodedRegisterId: EncodedRegisterId = { localId: registerId.localId }; + if (registerId.revision !== undefined) { + encodedRegisterId.revision = revisionTagCodec.encode(registerId.revision); + } + + return encodedRegisterId; + }, + decode: (registerId: EncodedRegisterId) => { + if (registerId === 0) { + return "self"; + } - return { ...registerId }; - }, - decode: (registerId: EncodedRegisterId) => { - if (registerId === 0) { - return "self"; - } + const decodedRegisterId: Mutable = { localId: registerId.localId }; + if (registerId.revision !== undefined) { + decodedRegisterId.revision = revisionTagCodec.decode(registerId.revision); + } - return { ...registerId }; - }, -}; + return decodedRegisterId; + }, + encodedSchema: EncodedRegisterId, + }; +} function makeOptionalFieldCodec( childCodec: IJsonCodec, + revisionTagCodec: IJsonCodec, ): IJsonCodec> { + const registerIdCodec = makeRegisterIdCodec(revisionTagCodec); + return { encode: (change: OptionalChangeset) => { const encoded: EncodedOptionalChangeset = {}; if (change.build.length > 0) { const builds: EncodedBuild[] = []; for (const build of change.build) { - builds.push([build.id, build.set]); + builds.push([encodeChangeAtomId(revisionTagCodec, build.id), build.set]); } encoded.b = builds; } @@ -87,10 +110,12 @@ function makeOptionalFieldCodec( ) ?? []; const decoded: OptionalChangeset = { build: - encoded.b?.map(([id, set]) => ({ - id, - set, - })) ?? [], + encoded.b?.map(([id, set]) => { + return { + id: decodeChangeAtomId(revisionTagCodec, id), + set, + }; + }) ?? [], moves, childChanges: encoded.c?.map(([id, encodedChange]) => [ diff --git a/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldCodecs.ts b/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldCodecs.ts index d01ce8a906f2..45b6627ab606 100644 --- a/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldCodecs.ts +++ b/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldCodecs.ts @@ -7,6 +7,8 @@ import { unreachableCase } from "@fluidframework/core-utils"; import { TAnySchema, Type } from "@sinclair/typebox"; import { JsonCompatibleReadOnly, Mutable, fail } from "../../util"; import { DiscriminatedUnionDispatcher, IJsonCodec, makeCodecFamily } from "../../codec"; +import { EncodedRevisionTag, RevisionTag } from "../../core"; +import { decodeChangeAtomId, encodeChangeAtomId } from "../utils"; import { Attach, AttachAndDetach, @@ -24,11 +26,13 @@ import { import { Changeset as ChangesetSchema, Encoded } from "./format"; import { isNoopMark } from "./utils"; -export const sequenceFieldChangeCodecFactory = (childCodec: IJsonCodec) => - makeCodecFamily>([[0, makeV0Codec(childCodec)]]); - +export const sequenceFieldChangeCodecFactory = ( + childCodec: IJsonCodec, + revisionTagCodec: IJsonCodec, +) => makeCodecFamily>([[0, makeV0Codec(childCodec, revisionTagCodec)]]); function makeV0Codec( childCodec: IJsonCodec, + revisionTagCodec: IJsonCodec, ): IJsonCodec> { const markEffectCodec: IJsonCodec = { encode(effect: MarkEffect): Encoded.MarkEffect { @@ -37,25 +41,48 @@ function makeV0Codec( case "MoveIn": return { moveIn: { - finalEndpoint: effect.finalEndpoint, + finalEndpoint: + effect.finalEndpoint === undefined + ? undefined + : encodeChangeAtomId(revisionTagCodec, effect.finalEndpoint), id: effect.id, }, }; case "Insert": - return { insert: { revision: effect.revision, id: effect.id } }; + return { + insert: { + revision: + effect.revision === undefined + ? undefined + : revisionTagCodec.encode(effect.revision), + id: effect.id, + }, + }; case "Delete": return { delete: { - revision: effect.revision, - detachIdOverride: effect.detachIdOverride, + revision: + effect.revision === undefined + ? undefined + : revisionTagCodec.encode(effect.revision), + detachIdOverride: + effect.detachIdOverride === undefined + ? undefined + : encodeChangeAtomId(revisionTagCodec, effect.detachIdOverride), id: effect.id, }, }; case "MoveOut": return { moveOut: { - finalEndpoint: effect.finalEndpoint, - detachIdOverride: effect.detachIdOverride, + finalEndpoint: + effect.finalEndpoint === undefined + ? undefined + : encodeChangeAtomId(revisionTagCodec, effect.finalEndpoint), + detachIdOverride: + effect.detachIdOverride === undefined + ? undefined + : encodeChangeAtomId(revisionTagCodec, effect.detachIdOverride), id: effect.id, }, }; @@ -90,7 +117,7 @@ function makeV0Codec( id, }; if (finalEndpoint !== undefined) { - mark.finalEndpoint = finalEndpoint; + mark.finalEndpoint = decodeChangeAtomId(revisionTagCodec, finalEndpoint); } return mark; }, @@ -101,7 +128,7 @@ function makeV0Codec( id, }; if (revision !== undefined) { - mark.revision = revision; + mark.revision = revisionTagCodec.decode(revision); } return mark; }, @@ -112,10 +139,10 @@ function makeV0Codec( id, }; if (revision !== undefined) { - mark.revision = revision; + mark.revision = revisionTagCodec.decode(revision); } if (detachIdOverride !== undefined) { - mark.detachIdOverride = detachIdOverride; + mark.detachIdOverride = decodeChangeAtomId(revisionTagCodec, detachIdOverride); } return mark; }, @@ -126,10 +153,10 @@ function makeV0Codec( id, }; if (finalEndpoint !== undefined) { - mark.finalEndpoint = finalEndpoint; + mark.finalEndpoint = decodeChangeAtomId(revisionTagCodec, finalEndpoint); } if (detachIdOverride !== undefined) { - mark.detachIdOverride = detachIdOverride; + mark.detachIdOverride = decodeChangeAtomId(revisionTagCodec, detachIdOverride); } return mark; }, @@ -149,12 +176,12 @@ function makeV0Codec( adjacentCells: adjacentCells?.map(({ id, count }) => [id, count]), // eslint-disable-next-line @typescript-eslint/no-shadow lineage: lineage?.map(({ revision, id, count, offset }) => [ - revision, + revisionTagCodec.encode(revision), id, count, offset, ]), - revision, + revision: revision === undefined ? revision : revisionTagCodec.encode(revision), }; return encoded; }, @@ -166,7 +193,7 @@ function makeV0Codec( localId, }; if (revision !== undefined) { - decoded.revision = revision; + decoded.revision = revisionTagCodec.decode(revision); } if (adjacentCells !== undefined) { decoded.adjacentCells = adjacentCells.map(([id, count]) => ({ @@ -177,7 +204,7 @@ function makeV0Codec( if (lineage !== undefined) { // eslint-disable-next-line @typescript-eslint/no-shadow decoded.lineage = lineage.map(([revision, id, count, offset]) => ({ - revision, + revision: revisionTagCodec.decode(revision), id, count, offset, diff --git a/experimental/dds/tree2/src/feature-libraries/utils.ts b/experimental/dds/tree2/src/feature-libraries/utils.ts new file mode 100644 index 000000000000..aa9ed705aa2b --- /dev/null +++ b/experimental/dds/tree2/src/feature-libraries/utils.ts @@ -0,0 +1,35 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { IJsonCodec } from "../codec"; +import { RevisionTag, EncodedRevisionTag, ChangeAtomId, EncodedChangeAtomId } from "../core"; + +export function encodeChangeAtomId( + revisionTagCodec: IJsonCodec, + changeAtomId: ChangeAtomId, +): EncodedChangeAtomId { + if (changeAtomId.revision === undefined) { + return { localId: changeAtomId.localId }; + } + + return { + localId: changeAtomId.localId, + revision: revisionTagCodec.encode(changeAtomId.revision), + }; +} + +export function decodeChangeAtomId( + revisionTagCodec: IJsonCodec, + changeAtomId: EncodedChangeAtomId, +): ChangeAtomId { + if (changeAtomId.revision === undefined) { + return { localId: changeAtomId.localId }; + } + + return { + localId: changeAtomId.localId, + revision: revisionTagCodec.decode(changeAtomId.revision), + }; +} diff --git a/experimental/dds/tree2/src/shared-tree-core/editManagerCodecs.ts b/experimental/dds/tree2/src/shared-tree-core/editManagerCodecs.ts index ba4ce79a674f..0d467208db82 100644 --- a/experimental/dds/tree2/src/shared-tree-core/editManagerCodecs.ts +++ b/experimental/dds/tree2/src/shared-tree-core/editManagerCodecs.ts @@ -5,12 +5,14 @@ import { assert } from "@fluidframework/core-utils"; import { ICodecOptions, IJsonCodec, IMultiFormatCodec } from "../codec"; +import { EncodedRevisionTag, RevisionTag } from "../core"; import { JsonCompatibleReadOnly, JsonCompatibleReadOnlySchema, mapIterable } from "../util"; import { SummaryData } from "./editManager"; -import { Commit, EncodedEditManager } from "./editManagerFormat"; +import { Commit, EncodedCommit, EncodedEditManager } from "./editManagerFormat"; export function makeEditManagerCodec( changeCodec: IMultiFormatCodec, + revisionTagCodec: IJsonCodec, { jsonValidator: validator }: ICodecOptions, ): IJsonCodec> { const format = validator.compile( @@ -19,11 +21,13 @@ export function makeEditManagerCodec( const encodeCommit = >(commit: T) => ({ ...commit, + revision: revisionTagCodec.encode(commit.revision), change: changeCodec.json.encode(commit.change), }); - const decodeCommit = >(commit: T) => ({ + const decodeCommit = >(commit: T) => ({ ...commit, + revision: revisionTagCodec.decode(commit.revision), change: changeCodec.json.decode(commit.change), }); @@ -37,7 +41,7 @@ export function makeEditManagerCodec( ]), }; assert(format.check(json), 0x6cc /* Encoded schema should validate */); - return json; + return json as unknown as JsonCompatibleReadOnly; }, decode: (json) => { assert(format.check(json), 0x6cd /* Encoded schema should validate */); @@ -46,7 +50,11 @@ export function makeEditManagerCodec( branches: new Map( mapIterable(json.branches, ([sessionId, branch]) => [ sessionId, - { ...branch, commits: branch.commits.map(decodeCommit) }, + { + ...branch, + base: revisionTagCodec.decode(branch.base), + commits: branch.commits.map(decodeCommit), + }, ]), ), }; diff --git a/experimental/dds/tree2/src/shared-tree-core/editManagerFormat.ts b/experimental/dds/tree2/src/shared-tree-core/editManagerFormat.ts index 50e5047e224a..81c9c6ab0fee 100644 --- a/experimental/dds/tree2/src/shared-tree-core/editManagerFormat.ts +++ b/experimental/dds/tree2/src/shared-tree-core/editManagerFormat.ts @@ -5,7 +5,13 @@ import { TSchema, Type, ObjectOptions } from "@sinclair/typebox"; import { Brand, brandedNumberType } from "../util"; -import { SessionId, SessionIdSchema, RevisionTag, RevisionTagSchema } from "../core"; +import { + SessionId, + SessionIdSchema, + RevisionTag, + RevisionTagSchema, + EncodedRevisionTag, +} from "../core"; /** * Contains a single change to the `SharedTree` and associated metadata. @@ -17,6 +23,13 @@ export interface Commit { readonly sessionId: SessionId; } +// eslint-disable-next-line @typescript-eslint/consistent-type-definitions +export type EncodedCommit = { + readonly revision: EncodedRevisionTag; + readonly change: TChangeset; + readonly sessionId: SessionId; +}; + const noAdditionalProps: ObjectOptions = { additionalProperties: false }; const CommitBase = (tChange: ChangeSchema) => diff --git a/experimental/dds/tree2/src/shared-tree-core/editManagerSummarizer.ts b/experimental/dds/tree2/src/shared-tree-core/editManagerSummarizer.ts index 2dbd11a3e61a..9d6297935f4d 100644 --- a/experimental/dds/tree2/src/shared-tree-core/editManagerSummarizer.ts +++ b/experimental/dds/tree2/src/shared-tree-core/editManagerSummarizer.ts @@ -13,7 +13,7 @@ import { } from "@fluidframework/runtime-definitions"; import { createSingleBlobSummary } from "@fluidframework/shared-object-base"; import { ICodecOptions, IJsonCodec } from "../codec"; -import { ChangeFamily, ChangeFamilyEditor } from "../core"; +import { ChangeFamily, ChangeFamilyEditor, EncodedRevisionTag, RevisionTag } from "../core"; import { JsonCompatibleReadOnly } from "../util"; import { Summarizable, SummaryElementParser, SummaryElementStringifier } from "./sharedTreeCore"; import { EditManager, SummaryData } from "./editManager"; @@ -41,10 +41,11 @@ export class EditManagerSummarizer implements Summarizable { TChangeset, ChangeFamily >, + revisionTagCodec: IJsonCodec, options: ICodecOptions, ) { const changesetCodec = this.editManager.changeFamily.codecs.resolve(formatVersion); - this.codec = makeEditManagerCodec(changesetCodec, options); + this.codec = makeEditManagerCodec(changesetCodec, revisionTagCodec, options); } public getAttachSummary( diff --git a/experimental/dds/tree2/src/shared-tree-core/index.ts b/experimental/dds/tree2/src/shared-tree-core/index.ts index c4339f79092c..baa314c725db 100644 --- a/experimental/dds/tree2/src/shared-tree-core/index.ts +++ b/experimental/dds/tree2/src/shared-tree-core/index.ts @@ -22,4 +22,11 @@ export { TransactionStack } from "./transactionStack"; export { makeEditManagerCodec } from "./editManagerCodecs"; export { EditManagerSummarizer } from "./editManagerSummarizer"; export { EditManager, minimumPossibleSequenceNumber, SummaryData } from "./editManager"; -export { Commit, SeqNumber, SequencedCommit, SummarySessionBranch } from "./editManagerFormat"; +export { + Commit, + SeqNumber, + SequencedCommit, + SummarySessionBranch, + EncodedCommit, +} from "./editManagerFormat"; +export { RevisionTagCodec } from "./revisionTagCodecs"; diff --git a/experimental/dds/tree2/src/shared-tree-core/messageCodecs.ts b/experimental/dds/tree2/src/shared-tree-core/messageCodecs.ts index ba94a3dfd68e..d6c0549af80e 100644 --- a/experimental/dds/tree2/src/shared-tree-core/messageCodecs.ts +++ b/experimental/dds/tree2/src/shared-tree-core/messageCodecs.ts @@ -6,11 +6,13 @@ import { TAnySchema, Type } from "@sinclair/typebox"; import { JsonCompatibleReadOnly } from "../util"; import { ICodecOptions, IJsonCodec, withSchemaValidation } from "../codec"; +import { EncodedRevisionTag, RevisionTag } from "../core"; import { DecodedMessage } from "./messageTypes"; import { Message } from "./messageFormat"; export function makeMessageCodec( changesetCodec: IJsonCodec, + revisionTagCodec: IJsonCodec, options: ICodecOptions, ): IJsonCodec> { return withSchemaValidation, TAnySchema>( @@ -18,7 +20,7 @@ export function makeMessageCodec( { encode: ({ commit, sessionId }: DecodedMessage) => { const message: Message = { - revision: commit.revision, + revision: revisionTagCodec.encode(commit.revision), originatorId: sessionId, changeset: changesetCodec.encode(commit.change), }; @@ -28,7 +30,7 @@ export function makeMessageCodec( const { revision, originatorId, changeset } = encoded as unknown as Message; return { commit: { - revision, + revision: revisionTagCodec.decode(revision), change: changesetCodec.decode(changeset), }, sessionId: originatorId, diff --git a/experimental/dds/tree2/src/shared-tree-core/messageFormat.ts b/experimental/dds/tree2/src/shared-tree-core/messageFormat.ts index f6a4040d4900..ed29b27098e0 100644 --- a/experimental/dds/tree2/src/shared-tree-core/messageFormat.ts +++ b/experimental/dds/tree2/src/shared-tree-core/messageFormat.ts @@ -5,7 +5,7 @@ import { Type, TSchema } from "@sinclair/typebox"; import { JsonCompatibleReadOnly } from "../util"; -import { RevisionTag, RevisionTagSchema, SessionIdSchema } from "../core"; +import { EncodedRevisionTag, RevisionTagSchema, SessionIdSchema } from "../core"; /** * The format of messages that SharedTree sends and receives. @@ -14,7 +14,7 @@ export interface Message { /** * The revision tag for the change in this message */ - readonly revision: RevisionTag; + readonly revision: EncodedRevisionTag; /** * The stable ID that identifies the originator of the message. */ diff --git a/experimental/dds/tree2/src/shared-tree-core/revisionTagCodecs.ts b/experimental/dds/tree2/src/shared-tree-core/revisionTagCodecs.ts new file mode 100644 index 000000000000..bfca5437d1ee --- /dev/null +++ b/experimental/dds/tree2/src/shared-tree-core/revisionTagCodecs.ts @@ -0,0 +1,16 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { IJsonCodec } from "../codec"; +import { EncodedRevisionTag, RevisionTag } from "../core"; + +export class RevisionTagCodec implements IJsonCodec { + public encode(tag: RevisionTag) { + return tag as unknown as EncodedRevisionTag; + } + public decode(tag: EncodedRevisionTag) { + return tag as unknown as RevisionTag; + } +} diff --git a/experimental/dds/tree2/src/shared-tree-core/sharedTreeCore.ts b/experimental/dds/tree2/src/shared-tree-core/sharedTreeCore.ts index 1b02a78ef37e..ab320abf35ad 100644 --- a/experimental/dds/tree2/src/shared-tree-core/sharedTreeCore.ts +++ b/experimental/dds/tree2/src/shared-tree-core/sharedTreeCore.ts @@ -26,6 +26,7 @@ import { EditManager, minimumPossibleSequenceNumber } from "./editManager"; import { SeqNumber } from "./editManagerFormat"; import { DecodedMessage } from "./messageTypes"; import { makeMessageCodec } from "./messageCodecs"; +import { RevisionTagCodec } from "./revisionTagCodecs"; // TODO: How should the format version be determined? const formatVersion = 0; @@ -122,8 +123,9 @@ export class SharedTreeCore extends } }); + const revisionTagCodec = new RevisionTagCodec(); this.summarizables = [ - new EditManagerSummarizer(this.editManager, options), + new EditManagerSummarizer(this.editManager, revisionTagCodec, options), ...summarizables, ]; assert( @@ -133,6 +135,7 @@ export class SharedTreeCore extends this.messageCodec = makeMessageCodec( changeFamily.codecs.resolve(formatVersion).json, + new RevisionTagCodec(), options, ); } diff --git a/experimental/dds/tree2/src/test/feature-libraries/editManagerCodecs.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/editManagerCodecs.spec.ts index 3c7eb153a0d4..60dcee73a382 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/editManagerCodecs.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/editManagerCodecs.spec.ts @@ -8,7 +8,7 @@ import { typeboxValidator } from "../../external-utilities"; import { mintRevisionTag } from "../../core"; import { TestChange } from "../testChange"; import { brand } from "../../util"; -import { SummaryData, makeEditManagerCodec } from "../../shared-tree-core"; +import { RevisionTagCodec, SummaryData, makeEditManagerCodec } from "../../shared-tree-core"; import { EncodingTestData, makeEncodingTestSuite } from "../utils"; const tags = Array.from({ length: 3 }, mintRevisionTag); @@ -145,9 +145,13 @@ const testCases: EncodingTestData, unknown> = { }; describe("EditManager codec", () => { - const codec = makeEditManagerCodec(withDefaultBinaryEncoding(TestChange.codec), { - jsonValidator: typeboxValidator, - }); + const codec = makeEditManagerCodec( + withDefaultBinaryEncoding(TestChange.codec), + new RevisionTagCodec(), + { + jsonValidator: typeboxValidator, + }, + ); makeEncodingTestSuite(makeCodecFamily([[0, codec]]), testCases); diff --git a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts index 3a1e6dd42425..d6ab2ef6dd3f 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts @@ -27,6 +27,7 @@ import { makeEncodingTestSuite, } from "../../utils"; import { IJsonCodec } from "../../../codec"; +import { RevisionTagCodec } from "../../../shared-tree-core"; import { singleJsonCursor } from "../../../domains"; // eslint-disable-next-line import/no-internal-modules import { RebaseRevisionMetadata } from "../../../feature-libraries/modular-schema"; @@ -420,7 +421,9 @@ describe("Generic FieldKind", () => { decode: unexpectedDelegate, }; - const leafCodec = valueHandler.codecsFactory(throwCodec).resolve(0).json; + const leafCodec = valueHandler + .codecsFactory(throwCodec, new RevisionTagCodec()) + .resolve(0).json; const childCodec: IJsonCodec = { encode: (nodeChange) => { const valueChange = valueChangeFromNodeChange(nodeChange); @@ -433,7 +436,7 @@ describe("Generic FieldKind", () => { }; makeEncodingTestSuite( - genericFieldKind.changeHandler.codecsFactory(childCodec), + genericFieldKind.changeHandler.codecsFactory(childCodec, new RevisionTagCodec()), encodingTestData, ); }); diff --git a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts index 399d82599ea0..040d17d72385 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts @@ -15,6 +15,7 @@ import { } from "../../../feature-libraries/optional-field"; import { IJsonCodec } from "../../../codec"; import { changesetForChild, testTree, testTreeCursor } from "../fieldKindTestUtils"; +import { RevisionTagCodec } from "../../../shared-tree-core"; const nodeChange1 = changesetForChild("nodeChange1"); @@ -75,7 +76,10 @@ describe("defaultFieldChangeCodecs", () => { ], }; - makeEncodingTestSuite(makeOptionalFieldCodecFamily(childCodec1), encodingTestData); + makeEncodingTestSuite( + makeOptionalFieldCodecFamily(childCodec1, new RevisionTagCodec()), + encodingTestData, + ); }); // TODO: test other kinds of changesets diff --git a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceFieldCodecs.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceFieldCodecs.spec.ts index c0a51fe8a997..6a01ef4c1ca1 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceFieldCodecs.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceFieldCodecs.spec.ts @@ -7,6 +7,7 @@ import { mintRevisionTag } from "../../../core"; import { SequenceField as SF } from "../../../feature-libraries"; // eslint-disable-next-line import/no-internal-modules import { Changeset } from "../../../feature-libraries/sequence-field"; +import { RevisionTagCodec } from "../../../shared-tree-core"; import { brand } from "../../../util"; import { TestChange } from "../../testChange"; import { EncodingTestData, makeEncodingTestSuite } from "../../utils"; @@ -28,5 +29,8 @@ const encodingTestData: EncodingTestData, unknown> = { }; describe("SequenceField encoding", () => { - makeEncodingTestSuite(SF.sequenceFieldChangeCodecFactory(TestChange.codec), encodingTestData); + makeEncodingTestSuite( + SF.sequenceFieldChangeCodecFactory(TestChange.codec, new RevisionTagCodec()), + encodingTestData, + ); }); diff --git a/experimental/dds/tree2/src/test/shared-tree-core/message.spec.ts b/experimental/dds/tree2/src/test/shared-tree-core/message.spec.ts index 375d54a943ac..63ff47b83d04 100644 --- a/experimental/dds/tree2/src/test/shared-tree-core/message.spec.ts +++ b/experimental/dds/tree2/src/test/shared-tree-core/message.spec.ts @@ -9,6 +9,8 @@ import { typeboxValidator } from "../../external-utilities"; import { makeMessageCodec } from "../../shared-tree-core/messageCodecs"; // eslint-disable-next-line import/no-internal-modules import { DecodedMessage } from "../../shared-tree-core/messageTypes"; +// eslint-disable-next-line import/no-internal-modules +import { RevisionTagCodec } from "../../shared-tree-core/revisionTagCodecs"; import { useDeterministicStableId } from "../../util"; import { TestChange } from "../testChange"; import { EncodingTestData, makeEncodingTestSuite } from "../utils"; @@ -98,7 +100,7 @@ const testCases = useDeterministicStableId(() => { }); describe("message codec", () => { - const codec = makeMessageCodec(TestChange.codec, { + const codec = makeMessageCodec(TestChange.codec, new RevisionTagCodec(), { jsonValidator: typeboxValidator, }); diff --git a/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json b/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json index 01304acb9b4d..6b4564f2d14f 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json +++ b/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json @@ -151,8 +151,8 @@ "c": [ [ { - "revision": "beefbeef-beef-4000-8000-000000000007", - "localId": 5 + "localId": 5, + "revision": "beefbeef-beef-4000-8000-000000000007" }, { "fieldChanges": [ From 7b1c0c3a13057b3c83f5ec08acedd9cc422fee25 Mon Sep 17 00:00:00 2001 From: yann-achard-MS <97201204+yann-achard-MS@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:37:11 -0800 Subject: [PATCH 12/20] test(tree2): update state-based axiom test harness to generate appropriate rebase metadata (#18694) --- .../modular-schema/modularChangeFamily.ts | 7 +- .../tree2/src/test/exhaustiveRebaserUtils.ts | 6 +- .../sequenceChangeRebaser.spec.ts | 12 +- .../feature-libraries/sequence-field/utils.ts | 9 +- .../tree2/src/test/rebaserAxiomaticTests.ts | 312 +++++++++++------- 5 files changed, 217 insertions(+), 129 deletions(-) diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts index b58125338add..42e55323867a 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/modularChangeFamily.ts @@ -859,7 +859,12 @@ function deltaFromNodeChange( /** * @alpha * @param revInfos - This should describe all revisions in the rebase path, even if not part of the current base changeset. - * @param baseRevisions - The set of revisions in the changeset being rebased over + * For example, when rebasing change B from a local branch [A, B, C] over a branch [X, Y], the `revInfos` must include + * the changes [A⁻¹ X, Y, A'] for each rebase step of B. + * @param baseRevisions - The set of revisions in the changeset being rebased over. + * For example, when rebasing change B from a local branch [A, B, C] over a branch [X, Y], the `baseRevisions` must include + * revisions [A⁻¹ X, Y, A'] if rebasing over the composition of all those changes, or + * revision [A⁻¹] for the first rebase, then [X], etc. if rebasing over edits individually. * @returns - RebaseRevisionMetadata to be passed to `FieldChangeRebaser.rebase`* */ export function rebaseRevisionMetadataFromInfo( diff --git a/experimental/dds/tree2/src/test/exhaustiveRebaserUtils.ts b/experimental/dds/tree2/src/test/exhaustiveRebaserUtils.ts index 2af622087e4e..7db32d13b92c 100644 --- a/experimental/dds/tree2/src/test/exhaustiveRebaserUtils.ts +++ b/experimental/dds/tree2/src/test/exhaustiveRebaserUtils.ts @@ -4,6 +4,8 @@ */ import { RevisionMetadataSource, RevisionTag, TaggedChange } from "../core"; +// eslint-disable-next-line import/no-internal-modules +import { RebaseRevisionMetadata } from "../feature-libraries/modular-schema"; /** * Given a state tree, constructs the sequence of edits which led to that state. @@ -46,7 +48,7 @@ export interface BoundFieldChangeRebaser { rebase( change: TChangeset, base: TaggedChange, - metadata?: RevisionMetadataSource, + metadata?: RebaseRevisionMetadata, ): TChangeset; /** * Rebase the provided change over the composition of a set of base changes. @@ -57,7 +59,7 @@ export interface BoundFieldChangeRebaser { * on further refactoring would be nice to remove. */ rebaseComposed( - metadata: RevisionMetadataSource, + metadata: RebaseRevisionMetadata, change: TChangeset, ...baseChanges: TaggedChange[] ): TChangeset; diff --git a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceChangeRebaser.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceChangeRebaser.spec.ts index bcba52da7261..1168bf935bc0 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceChangeRebaser.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/sequenceChangeRebaser.spec.ts @@ -488,15 +488,15 @@ const generateChildStates: ChildStateGenerator = funct describe.skip("SequenceField - State-based Rebaser Axioms", () => { runExhaustiveComposeRebaseSuite( - [{ content: { length: 4, numNodes: [1, 3], maxIndex: 2 } }], + [{ content: { length: 4, numNodes: [1], maxIndex: 2 } }], generateChildStates, { rebase, invert, - compose: (changes, metadata) => compose(changes), + compose: (changes, metadata) => compose(changes, metadata), rebaseComposed: (metadata, change, ...baseChanges) => { - const composedChanges = compose(baseChanges); - return rebase(change, makeAnonChange(composedChanges)); + const composedChanges = compose(baseChanges, metadata); + return rebase(change, makeAnonChange(composedChanges), metadata); }, assertEqual: (change1, change2) => { if (change1 === undefined && change2 === undefined) { @@ -514,7 +514,9 @@ describe.skip("SequenceField - State-based Rebaser Axioms", () => { }, }, { - groupSubSuites: true, + groupSubSuites: false, + numberOfEditsToVerifyAssociativity: 3, + skipRebaseOverCompose: true, }, ); }); diff --git a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/utils.ts b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/utils.ts index 482b4171f441..fcdc45d2d4b2 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/utils.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/utils.ts @@ -9,6 +9,7 @@ import { ChangesetLocalId, DeltaFieldChanges, RevisionInfo, + RevisionMetadataSource, RevisionTag, TaggedChange, makeAnonChange, @@ -38,7 +39,7 @@ export function composeNoVerify( export function compose( changes: TaggedChange[], - revInfos?: RevisionInfo[], + revInfos?: RevisionInfo[] | RevisionMetadataSource, childComposer?: (childChanges: TaggedChange[]) => TestChange, ): TestChangeset { return composeI(changes, childComposer ?? TestChange.compose, revInfos); @@ -71,7 +72,7 @@ export function shallowCompose( function composeI( changes: TaggedChange>[], composer: (childChanges: TaggedChange[]) => T, - revInfos?: RevisionInfo[], + revInfos?: RevisionInfo[] | RevisionMetadataSource, ): SF.Changeset { const moveEffects = SF.newCrossFieldTable(); const idAllocator = continuingAllocator(changes); @@ -81,7 +82,9 @@ function composeI( idAllocator, moveEffects, revInfos !== undefined - ? revisionMetadataSourceFromInfo(revInfos) + ? Array.isArray(revInfos) + ? revisionMetadataSourceFromInfo(revInfos) + : revInfos : defaultRevisionMetadataFromChanges(changes), ); diff --git a/experimental/dds/tree2/src/test/rebaserAxiomaticTests.ts b/experimental/dds/tree2/src/test/rebaserAxiomaticTests.ts index d41a9d408d62..191bb8112083 100644 --- a/experimental/dds/tree2/src/test/rebaserAxiomaticTests.ts +++ b/experimental/dds/tree2/src/test/rebaserAxiomaticTests.ts @@ -6,13 +6,16 @@ import { strict as assert } from "assert"; import { makeAnonChange, RevisionTag, tagChange, TaggedChange, tagRollbackInverse } from "../core"; import { fail } from "../util"; -import { defaultRevisionMetadataFromChanges } from "./utils"; +// eslint-disable-next-line import/no-internal-modules +import { rebaseRevisionMetadataFromInfo } from "../feature-libraries/modular-schema"; +import { defaultRevInfosFromChanges, defaultRevisionMetadataFromChanges } from "./utils"; import { FieldStateTree, generatePossibleSequenceOfEdits, ChildStateGenerator, BoundFieldChangeRebaser, makeIntentionMinter, + NamedChangeset, } from "./exhaustiveRebaserUtils"; interface ExhaustiveSuiteOptions { @@ -37,68 +40,39 @@ const defaultSuiteOptions: Required = { numberOfEditsToVerifyAssociativity: 4, }; +/** + * Rebases `change` over all edits in `rebasePath`. + * @param rebase - The rebase function to use. + * @param change - The change to rebase + * @param rebasePath - The edits to rebase over. + * Must contain all the prior edits from the branch that `change` comes from. + * For example, if `change` is B in branch [A, B, C], being rebased over edits [X, Y], + * then `rebasePath` must be [A⁻¹, X, Y, A']. + */ +function rebaseTagged( + rebase: BoundFieldChangeRebaser["rebase"], + change: TaggedChange, + rebasePath: TaggedChange[], +): TaggedChange { + let currChange = change; + const revisionInfo = defaultRevInfosFromChanges(rebasePath); + for (const base of rebasePath) { + const metadata = rebaseRevisionMetadataFromInfo(revisionInfo, [base.revision]); + currChange = tagChange(rebase(currChange.change, base, metadata), currChange.revision); + } + + return currChange; +} + export function runExhaustiveComposeRebaseSuite( initialStates: FieldStateTree[], generateChildStates: ChildStateGenerator, - { rebase, rebaseComposed, invert, compose, assertEqual }: BoundFieldChangeRebaser, + fieldRebaser: BoundFieldChangeRebaser, options?: ExhaustiveSuiteOptions, ) { - const assertDeepEqual = assertEqual ?? ((a, b) => assert.deepEqual(a, b)); + const assertDeepEqual = getDefaultedEqualityAssert(fieldRebaser); const definedOptions = { ...defaultSuiteOptions, ...options }; - function rebaseTagged( - change: TaggedChange, - ...baseChanges: TaggedChange[] - ): TaggedChange { - let currChange = change; - const metadata = defaultRevisionMetadataFromChanges([change, ...baseChanges]); - for (const base of baseChanges) { - currChange = tagChange(rebase(currChange.change, base, metadata), currChange.revision); - } - - return currChange; - } - - function verifyComposeAssociativity(edits: TaggedChange[]) { - const metadata = defaultRevisionMetadataFromChanges(edits); - const singlyComposed = makeAnonChange(compose(edits, metadata)); - const leftPartialCompositions: TaggedChange[] = [ - edits.at(0) ?? fail("Expected at least one edit"), - ]; - for (let i = 1; i < edits.length; i++) { - leftPartialCompositions.push( - makeAnonChange( - compose( - [ - leftPartialCompositions.at(-1) ?? fail("Expected at least one edit"), - edits[i], - ], - metadata, - ), - ), - ); - } - - const rightPartialCompositions: TaggedChange[] = [ - edits.at(-1) ?? fail("Expected at least one edit"), - ]; - for (let i = edits.length - 2; i >= 0; i--) { - rightPartialCompositions.push( - makeAnonChange( - compose( - [ - edits[i], - rightPartialCompositions.at(-1) ?? fail("Expected at least one edit"), - ], - metadata, - ), - ), - ); - } - - assertDeepEqual(leftPartialCompositions.at(-1), singlyComposed); - assertDeepEqual(rightPartialCompositions.at(-1), singlyComposed); - } // To limit combinatorial explosion, we test 'rebasing over a compose is equivalent to rebasing over the individual edits' // by: // - Rebasing a single edit over N sequential edits @@ -149,26 +123,10 @@ export function runExhaustiveComposeRebaseSuite( )}`; innerFixture(title, () => { - const editsToRebaseOver = namedEditsToRebaseOver.map( - ({ changeset }) => changeset, - ); - const rebaseWithoutCompose = rebaseTagged( - edit, - ...editsToRebaseOver, - ).change; - const metadata = defaultRevisionMetadataFromChanges([ - ...editsToRebaseOver, + rebaseOverSinglesVsRebaseOverCompositions( edit, - ]); - const rebaseWithCompose = rebaseComposed( - metadata, - edit.change, - ...editsToRebaseOver, - ); - - assertDeepEqual( - tagChange(rebaseWithCompose, undefined), - tagChange(rebaseWithoutCompose, undefined), + namedEditsToRebaseOver, + fieldRebaser, ); }); } @@ -211,54 +169,28 @@ export function runExhaustiveComposeRebaseSuite( const editToRebaseOver = namedEditToRebaseOver; const sourceEdits = namedSourceEdits.map(({ changeset }) => changeset); - const inverses = sourceEdits.map((change) => + const rollbacks = sourceEdits.map((change) => tagRollbackInverse( - invert(change), + fieldRebaser.invert(change), `rollback-${change.revision}` as RevisionTag, change.revision, ), ); - inverses.reverse(); + rollbacks.reverse(); - const rebasedEditsWithoutCompose: TaggedChange[] = []; - const rebasedEditsWithCompose: TaggedChange[] = []; - - for (let i = 0; i < sourceEdits.length; i++) { - const edit = sourceEdits[i]; - const editsToRebaseOver = [ - ...inverses.slice(sourceEdits.length - i), - editToRebaseOver, - ...rebasedEditsWithoutCompose, - ]; - rebasedEditsWithoutCompose.push( - rebaseTagged(edit, ...editsToRebaseOver), - ); - } + const rebasedEditsWithoutCompose = sandwichRebaseWithoutCompose( + sourceEdits, + rollbacks, + editToRebaseOver, + fieldRebaser, + ); - let currentComposedEdit = editToRebaseOver; - // This needs to be used to pass an updated RevisionMetadataSource to rebase. - const allTaggedEdits = [...inverses, editToRebaseOver]; - for (let i = 0; i < sourceEdits.length; i++) { - let metadata = defaultRevisionMetadataFromChanges(allTaggedEdits); - const edit = sourceEdits[i]; - const rebasedEdit = tagChange( - rebaseComposed(metadata, edit.change, currentComposedEdit), - edit.revision, - ); - rebasedEditsWithCompose.push(rebasedEdit); - allTaggedEdits.push(rebasedEdit); - metadata = defaultRevisionMetadataFromChanges(allTaggedEdits); - currentComposedEdit = makeAnonChange( - compose( - [ - inverses[sourceEdits.length - i - 1], - currentComposedEdit, - rebasedEdit, - ], - metadata, - ), - ); - } + const rebasedEditsWithCompose = sandwichRebaseWithCompose( + sourceEdits, + rollbacks, + editToRebaseOver, + fieldRebaser, + ); for (let i = 0; i < rebasedEditsWithoutCompose.length; i++) { assertDeepEqual( @@ -267,7 +199,13 @@ export function runExhaustiveComposeRebaseSuite( ); } - verifyComposeAssociativity(allTaggedEdits); + // TODO: consider testing the compose associativity with the `rebasedEditsWithoutCompose` as well. + const allTaggedEdits = [ + ...rollbacks, + editToRebaseOver, + ...rebasedEditsWithCompose, + ]; + verifyComposeAssociativity(allTaggedEdits, fieldRebaser); }); } } @@ -292,10 +230,148 @@ export function runExhaustiveComposeRebaseSuite( // That's covered some by "Composed sandwich rebase over single edit" innerFixture(title, () => { const edits = namedSourceEdits.map(({ changeset }) => changeset); - verifyComposeAssociativity(edits); + verifyComposeAssociativity(edits, fieldRebaser); }); } }); } }); } + +function sandwichRebaseWithCompose( + sourceEdits: TaggedChange[], + rollbacks: TaggedChange[], + editToRebaseOver: TaggedChange, + fieldRebaser: BoundFieldChangeRebaser, +): TaggedChange[] { + const rebasedEditsWithCompose: TaggedChange[] = []; + let compositionScope: TaggedChange[] = [editToRebaseOver]; + let currentComposedEdit = editToRebaseOver; + // This needs to be used to pass an updated RevisionMetadataSource to rebase. + for (let i = 0; i < sourceEdits.length; i++) { + const edit = sourceEdits[i]; + const rebasePath = [ + ...rollbacks.slice(sourceEdits.length - i), + editToRebaseOver, + ...sourceEdits.slice(0, i), + ]; + const rebaseMetadata = rebaseRevisionMetadataFromInfo( + defaultRevInfosFromChanges(rebasePath), + [editToRebaseOver.revision], + ); + const rebasedEdit = tagChange( + fieldRebaser.rebaseComposed(rebaseMetadata, edit.change, currentComposedEdit), + edit.revision, + ); + rebasedEditsWithCompose.push(rebasedEdit); + compositionScope = [ + rollbacks[sourceEdits.length - i - 1], + ...compositionScope, + rebasedEdit, + ]; + const composeMetadata = defaultRevisionMetadataFromChanges(compositionScope); + currentComposedEdit = makeAnonChange( + fieldRebaser.compose( + [rollbacks[sourceEdits.length - i - 1], currentComposedEdit, rebasedEdit], + composeMetadata, + ), + ); + } + return rebasedEditsWithCompose; +} + +function sandwichRebaseWithoutCompose( + sourceEdits: TaggedChange[], + rollbacks: TaggedChange[], + editToRebaseOver: TaggedChange, + fieldRebaser: BoundFieldChangeRebaser, +): TaggedChange[] { + const rebasedEditsWithoutCompose: TaggedChange[] = []; + for (let i = 0; i < sourceEdits.length; i++) { + const edit = sourceEdits[i]; + const rebasePath = [ + ...rollbacks.slice(sourceEdits.length - i), + editToRebaseOver, + ...rebasedEditsWithoutCompose, + ]; + rebasedEditsWithoutCompose.push(rebaseTagged(fieldRebaser.rebase, edit, rebasePath)); + } + return rebasedEditsWithoutCompose; +} + +function rebaseOverSinglesVsRebaseOverCompositions( + edit: TaggedChange, + namedEditsToRebaseOver: NamedChangeset[], + fieldRebaser: BoundFieldChangeRebaser, +) { + const editsToRebaseOver = namedEditsToRebaseOver.map(({ changeset }) => changeset); + + // Rebase over each base edit individually + const rebaseWithoutCompose = rebaseTagged(fieldRebaser.rebase, edit, editsToRebaseOver).change; + + // Rebase over the composition of base edits + const metadata = rebaseRevisionMetadataFromInfo( + defaultRevInfosFromChanges(editsToRebaseOver), + editsToRebaseOver.map(({ revision }) => revision), + ); + const rebaseWithCompose = fieldRebaser.rebaseComposed( + metadata, + edit.change, + ...editsToRebaseOver, + ); + + const assertDeepEqual = getDefaultedEqualityAssert(fieldRebaser); + assertDeepEqual( + tagChange(rebaseWithCompose, edit.revision), + tagChange(rebaseWithoutCompose, edit.revision), + ); +} + +function verifyComposeAssociativity( + edits: TaggedChange[], + fieldRebaser: BoundFieldChangeRebaser, +) { + const metadata = defaultRevisionMetadataFromChanges(edits); + const singlyComposed = makeAnonChange(fieldRebaser.compose(edits, metadata)); + const leftPartialCompositions: TaggedChange[] = [ + edits.at(0) ?? fail("Expected at least one edit"), + ]; + for (let i = 1; i < edits.length; i++) { + leftPartialCompositions.push( + makeAnonChange( + fieldRebaser.compose( + [ + leftPartialCompositions.at(-1) ?? fail("Expected at least one edit"), + edits[i], + ], + metadata, + ), + ), + ); + } + + const rightPartialCompositions: TaggedChange[] = [ + edits.at(-1) ?? fail("Expected at least one edit"), + ]; + for (let i = edits.length - 2; i >= 0; i--) { + rightPartialCompositions.push( + makeAnonChange( + fieldRebaser.compose( + [ + edits[i], + rightPartialCompositions.at(-1) ?? fail("Expected at least one edit"), + ], + metadata, + ), + ), + ); + } + + const assertDeepEqual = getDefaultedEqualityAssert(fieldRebaser); + assertDeepEqual(leftPartialCompositions.at(-1), singlyComposed); + assertDeepEqual(rightPartialCompositions.at(-1), singlyComposed); +} + +function getDefaultedEqualityAssert(fieldRebaser: BoundFieldChangeRebaser) { + return fieldRebaser.assertEqual ?? ((a, b) => assert.deepEqual(a, b)); +} From 0bc233809b1edf1934474cc460e4812227d87bc6 Mon Sep 17 00:00:00 2001 From: Abram Sanderson Date: Thu, 7 Dec 2023 12:39:19 -0800 Subject: [PATCH 13/20] doc(tree2): Add persisted format/compatibility docs (#18528) ## Description Adds a document describing tree2's strategy for managing persisted format, compatibility, and testing concerns. This document is largely a pared-down and updated version of #14954 to reflect the current state of things. --------- Co-authored-by: Abram Sanderson --- experimental/dds/tree2/docs/README.md | 1 + .../dds/tree2/docs/main/compatibility.md | 149 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 experimental/dds/tree2/docs/main/compatibility.md diff --git a/experimental/dds/tree2/docs/README.md b/experimental/dds/tree2/docs/README.md index c7613a547178..b3288bea19bc 100644 --- a/experimental/dds/tree2/docs/README.md +++ b/experimental/dds/tree2/docs/README.md @@ -19,3 +19,4 @@ List of technical/design documents (to be organized into appropriate sections at - [Undo](./main/undo.md) - [V1 Undo Example Flow](./main/v1-undo-example-flow.md) - [V1 Undo](./main/v1-undo.md) +- [Compatibility](./main/compatibility.md) diff --git a/experimental/dds/tree2/docs/main/compatibility.md b/experimental/dds/tree2/docs/main/compatibility.md new file mode 100644 index 000000000000..3926ef3409f8 --- /dev/null +++ b/experimental/dds/tree2/docs/main/compatibility.md @@ -0,0 +1,149 @@ +# Compatibility + +This document provides concrete guidelines and strategies for organizing code that impacts SharedTree's persisted format. + +## Prerequisites + +[This document](../../../../../packages/dds/SchemaVersioning.md) provides general "best practices" for working with persisted data within the Fluid Framework. +It's strongly recommended to read through and understand its rationale before continuing with this document, +as most of the concrete recommendations presented henceforth fall out of those best practices. + +## What State is Persisted? + +A DDS's persisted format encompasses the format it uses for its summaries as well as its ops (due to [trailing ops](../../../README.md)) +including transitively referenced structured blob data. + +Since documents are stored outside of Fluid control (i.e. no type of central data migration is possible), +DDSes necessarily commit to backwards compatibility of their format for all time. + +## Format Management + +The persisted format version should be a configuration option an application author can specify using `SharedTreeFactory`: +this ensures applications can control rollout of configuration changes which require code saturation of some prior version. +It also empowers the container author (rather than the host--if they differ) to control their data model. + +In the SharedTree MVP, there is currently no mechanism for safely changing the persisted format version. +However, it is feasible to add such a mechanism in the future, and specifying the persisted format explicity in configuration sets us up to easily do so. +One example of prior art in the space is `@fluid-experimental/tree`'s [format-breaking migration strategy](../../../tree/docs/Breaking-Change-Migration.md), +though we would likely want to make the mechanism usable across the Fluid Framework. + +## Code Organization + +Each part of SharedTree which contributes to the persisted format should define: + +1. Types defining the _in-memory format_ needed to load or work with its data +1. A set of versioned _persisted formats_ which encompass all supported formats in the current and past. +1. EITHER An `IJsonCodec` capable of transcoding between the in-memory format and a particular persisted format, OR an `ICodecFamily` of such `IJsonCodec`s (for different persisted format versions) + +Split the above components into files as is reasonable. +Current organizational standards put: + +- The in-memory format into a file ending in `Types.ts` +- The persisted format into a file ending in `Format.ts` +- Codecs into a file ending in `Codec.ts` / `Codecs.ts` + +Having consistent conventions for these files helps to make changes to persisted formats obvious at review time. +Schemas for primitive types which are used in persisted formats but don't intrinsically define formats (such as branded strings) can be defined where convenient. +Codec logic should generally be self-contained: all imports should either be of the form `import type`, or should import from another persisted format file. +Importing Fluid Framework libraries that have the same guarantees (e.g. `SummaryTreeBuilder`) is also acceptable. +Codecs should expose the minimal necessary set of types. +Encoding should take care to only include necessary object properties. **In particular, avoid constructs like object spread**. +Decoding should validate that the data is not malformed: see [encoding validation](#encoding-validation) for more details. + +With the exception of primitives, storage format types should never be exposed in the public API. + +> Note: due to API-extractor implementation details, the typebox schemas for primitive types _cannot_ share a name with the primitive type, +> as it exposes _both_ the value and the type exported under the same name, even if the export is specified via `export type`. +> For example, the typebox schema for `ChangesetLocalId` is named `ChangesetLocalIdSchema`. + +Using this structure, SharedTree will have access to a library of codecs capable of encoding/decoding between +the in-memory format and some persisted format. + +## Encoding Validation + +Validating that an encoded format thoroughly matches what the code expects and failing fast otherwise is valuable to reduce the risk of data corruption: +it's a lot easier to investigate and deploy fixes for documents when incompatible clients haven't also changed the contents of a file. + +For this reason, encoded data formats should declare JSON schema for the purpose of runtime validation. + +> In the near term, we're using [typebox](https://github.com/sinclairzx81/typebox) to declare these schemas. +> This choice is a matter of convenience: its API naturally matches the expressiveness of typescript types. + +Since format validation does incur runtime and bundle-size cost to obtain additional safety, +whether or not to perform it should ultimately be left as a policy choice to the user of shared-tree. +This choice will probably also be made in the shared-tree factory by providing a `JsonValidator`, +but it doesn't need to be persisted and can be changed at will +(there's no issue with collaboration between clients that have different policies around how much +of the persisted data should be validated). + +An out-of-the-box implementation of `JsonValidator` based on Typebox's JSON validator is provided, +but application authors may feel free to implement their own. + +## Test Strategy + +This section covers types of tests to include when adding new persisted configuration. + +There are a couple different dimensions to consider with respect to testing: + +1. SharedTree works correctly for all configurations it can be initialized in when collaborating with SharedTrees with similar configuration +1. SharedTree is compatible with clients using different source code versions of SharedTree (and the documents those clients may create) +1. Once supported, SharedTree can correctly execute document upgrade processes (changes to persisted configuration such as write format) + +### Configuration Unit Tests + +Each codec family should contain a suite of unit tests which verify the in-memory representation can be round-tripped through encoding and decoding. +When adding a new codec version, the test data for this suite should be augmented if existing data doesn't yield 100% code coverage on the new +codec version. + +If the persisted configuration impacts more than just the data encoding step, +appropriate unit tests should be added for whatever components that configuration impacts. +As a simple example, a persisted configuration flag which controls whether SharedTree stores attribution information +should have unit tests which verify processing ops of various sorts yield reasonable attribution on the parts of the tree they affect. + +Example: [experimental/dds/tree2/src/test/feature-libraries/editManagerCodecs.spec.ts](../../src/test/feature-libraries/editManagerCodecs.spec.ts) + +### Multiple-configuration Functional Tests + +Once SharedTree supports multiple persisted formats, we should modify a small set of functional acceptance tests +(e.g. `sharedTree.spec.ts`) to run for larger sets of configurations. +Using `generatePairwiseOptions` will help mitigate the combinatorial explosion concern. + +These tests in aggregate will verify that SharedTree works when initialized with some particular configuration +and collaborates with other SharedTree instances initialized with the same configuration. +They would reasonably detect basic defects in codecs or problems unrelated to backwards compatibility or any upgrade process. + +In the same vein, fuzz tests should cover a variety of valid configurations. + +### Snapshot Tests + +The last dimension of compatibility concerns direct or indirect collaboration between clients using different versions of SharedTree source code. +This is a vast area that could use more well-established framework testing support, but snapshot testing is a relatively effective category for +catching regressions. + +The idea behind snapshot testing is to verify a document produced using one version of the code is still usable using another version of the code. +It's typically implemented by writing some code to generate a set of fixed documents "from scratch," then source-controlling the serialized form +of those documents after summarization. +Since the serialized form of the documents correspond to documents produced by an older version of the code, this enables writing a test suite that verifies: + +1. The current version of the code serializes each document to exactly match how the older version of the code serialized each document. +1. The current version of the code is capable of loading documents written using older versions of the code. + +A few examples (which may not be exhaustive) of snapshot tests are: + +- [Legacy SharedTree](../../../../../experimental/dds/tree/src/test/Summary.tests.ts) +- [Sequence / SharedString](../../../sequence/src/test/snapshotVersion.spec.ts) +- [e2e Snapshot tests](../../../../test/snapshots/README.md) + +The first two examples generate their "from scratch" documents by directly calling DDS APIs on a newly created document. +The e2e snapshot tests accomplish "from scratch" generation by serializing the op stream alongside the snapshots and replaying it. +In addition to verifying serialized states line up between old and current version of the code, it can also be helpful to +verify equivalence at runtime, which typically gives more friendly error messages. + +> Aside: this approach is also a bit more flexible: it's possible that different snapshots can load to produce logically equivalent DDSes. +> Matrix is an example of this: it uses a pair of permutation vectors mapping logical indices (i.e. "row 5, column 3") to in-memory indices for the contents of cells. +> Thus, permuting both the permutation vectors and the cell data contained within a snapshot would not logically change its data. + +Snapshot tests are effective at catching changes which inadvertently modify the document format over time. + +Tree2's full-scale snapshot tests can be found at [experimental/dds/tree2/src/test/snapshots/summary.spec.ts](../../src/test/snapshots/summary.spec.ts), +with smaller-scale snapshot tests (e.g. snapshot testing just the SchemaIndex format) nearby. From 7d341a8fb5c1a4ddd240fa016a60a892b43b997b Mon Sep 17 00:00:00 2001 From: Abram Sanderson Date: Thu, 7 Dec 2023 13:19:19 -0800 Subject: [PATCH 14/20] Simplify OptionalField relevantRemovedTrees (#18623) ## Description Follow-up to #18600 to update the `relevantRemovedTrees` implementation. --------- Co-authored-by: Abram Sanderson Co-authored-by: yann-achard-MS <97201204+yann-achard-MS@users.noreply.github.com> --- .../optional-field/optionalField.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts index f761eef2d3fb..efa32612b112 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts @@ -636,14 +636,12 @@ function* relevantRemovedRoots( change: OptionalChangeset, relevantRemovedRootsFromChild: RelevantRemovedRootsFromChild, ): Iterable { - const dstToSrc = new RegisterMap(); const alreadyYieldedOrNewlyBuilt = new RegisterMap(); for (const { id } of change.build) { alreadyYieldedOrNewlyBuilt.set(id, true); } - for (const [src, dst] of change.moves) { - dstToSrc.set(dst, src); + for (const [src] of change.moves) { if (src !== "self" && !alreadyYieldedOrNewlyBuilt.has(src)) { alreadyYieldedOrNewlyBuilt.set(src, true); yield nodeIdFromChangeAtom(src); @@ -651,12 +649,11 @@ function* relevantRemovedRoots( } for (const [id, childChange] of change.childChanges) { - // Child changes are relevant unless they apply to the tree which existed in the starting context of + // Child changes make the tree they apply to relevant unless that tree existed in the starting context of // of this change. - const startingId = dstToSrc.get(id) ?? id; - if (startingId !== "self" && !alreadyYieldedOrNewlyBuilt.has(startingId)) { - alreadyYieldedOrNewlyBuilt.set(startingId, true); - yield nodeIdFromChangeAtom(startingId); + if (id !== "self" && !alreadyYieldedOrNewlyBuilt.has(id)) { + alreadyYieldedOrNewlyBuilt.set(id, true); + yield nodeIdFromChangeAtom(id); } yield* relevantRemovedRootsFromChild(childChange); } From 2245c0578e756c944caa5c22311eaafbb73452bb Mon Sep 17 00:00:00 2001 From: Mark Fields Date: Thu, 7 Dec 2023 14:40:00 -0800 Subject: [PATCH 15/20] GC: Switch default for throwOnTombstoneLoad config to true (#18651) We are confident in the reliability of the GC code and have worked with key 1P early adopters of GC to enable Tombstone Enforcement. It's time to enable it by default. --- .changeset/loose-comics-wonder.md | 9 ++++++ .../src/gc/garbageCollection.md | 3 +- .../container-runtime/src/gc/gcConfigs.ts | 5 ++- .../container-runtime/src/gc/gcDefinitions.ts | 8 +++-- .../runtime/container-runtime/src/gc/index.ts | 2 +- .../src/test/gc/garbageCollection.spec.ts | 3 +- .../src/test/gc/gcConfigs.spec.ts | 31 ++++++++++++------- 7 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 .changeset/loose-comics-wonder.md diff --git a/.changeset/loose-comics-wonder.md b/.changeset/loose-comics-wonder.md new file mode 100644 index 000000000000..6e09feaeba88 --- /dev/null +++ b/.changeset/loose-comics-wonder.md @@ -0,0 +1,9 @@ +--- +"@fluidframework/container-runtime": minor +--- + +GC: Tombstoned objects will fail to load by default + +Previously, by default Tombstoned objects would merely trigger informational logs, with an option via config +to also cause errors to be thrown on load. Now failure to load is the default, with an option to disable it if necessary. +This reflects the purpose of Tombstone stage which is to mimic the user experience of having objects deleted. diff --git a/packages/runtime/container-runtime/src/gc/garbageCollection.md b/packages/runtime/container-runtime/src/gc/garbageCollection.md index 9acf599d53f3..8cc4f6010038 100644 --- a/packages/runtime/container-runtime/src/gc/garbageCollection.md +++ b/packages/runtime/container-runtime/src/gc/garbageCollection.md @@ -70,7 +70,8 @@ The default configuration for GC today is: - GC Mark Phase is **enabled**, including Tombstone Mode - Session Expiry is **enabled** -- GC Sweep Phase is **disabled** +- The "Tombstone" stage of Sweep Phase is **enabled** (attempting to load a tombstoned object will fail) +- The "Delete" stage of Sweep Phase is **disabled** - Note: Once enabled, Sweep will only run for documents created from that point forward ### Techniques used for configuration diff --git a/packages/runtime/container-runtime/src/gc/gcConfigs.ts b/packages/runtime/container-runtime/src/gc/gcConfigs.ts index abce13151c2a..c44712ebf404 100644 --- a/packages/runtime/container-runtime/src/gc/gcConfigs.ts +++ b/packages/runtime/container-runtime/src/gc/gcConfigs.ts @@ -30,7 +30,7 @@ import { stableGCVersion, throwOnTombstoneLoadOverrideKey, throwOnTombstoneUsageKey, - gcThrowOnTombstoneLoadOptionName, + gcDisableThrowOnTombstoneLoadOptionName, defaultSweepGracePeriodMs, } from "./gcDefinitions"; import { getGCVersion, shouldAllowGcSweep, shouldAllowGcTombstoneEnforcement } from "./gcHelpers"; @@ -183,8 +183,7 @@ export function generateGCConfigs( const throwOnTombstoneLoadConfig = mc.config.getBoolean(throwOnTombstoneLoadOverrideKey) ?? - createParams.gcOptions[gcThrowOnTombstoneLoadOptionName] ?? - false; + createParams.gcOptions[gcDisableThrowOnTombstoneLoadOptionName] !== true; const throwOnTombstoneLoad = throwOnTombstoneLoadConfig && tombstoneEnforcementAllowed && diff --git a/packages/runtime/container-runtime/src/gc/gcDefinitions.ts b/packages/runtime/container-runtime/src/gc/gcDefinitions.ts index 3d7cabe4850e..217245a3ac4b 100644 --- a/packages/runtime/container-runtime/src/gc/gcDefinitions.ts +++ b/packages/runtime/container-runtime/src/gc/gcDefinitions.ts @@ -40,10 +40,12 @@ export const nextGCVersion: GCVersion = 4; export const gcTombstoneGenerationOptionName = "gcTombstoneGeneration"; /** - * This undocumented GC Option (on ContainerRuntime Options) allows an app to enable throwing an error when tombstone - * object is loaded (requested). + * This undocumented GC Option (on ContainerRuntime Options) allows an app to disable throwing an error when tombstone + * object is loaded (requested), merely logging a message instead. + * + * By default, attempting to load a Tombstoned object will result in an error. */ -export const gcThrowOnTombstoneLoadOptionName = "gcThrowOnTombstoneLoad"; +export const gcDisableThrowOnTombstoneLoadOptionName = "gcDisableThrowOnTombstoneLoad"; /** * This GC Option (on ContainerRuntime Options) allows an app to disable GC Sweep on old documents by incrementing this value. diff --git a/packages/runtime/container-runtime/src/gc/index.ts b/packages/runtime/container-runtime/src/gc/index.ts index 9d7204878bc7..5c2f3e1b931a 100644 --- a/packages/runtime/container-runtime/src/gc/index.ts +++ b/packages/runtime/container-runtime/src/gc/index.ts @@ -12,7 +12,7 @@ export { GCNodeType, gcTestModeKey, gcTombstoneGenerationOptionName, - gcThrowOnTombstoneLoadOptionName, + gcDisableThrowOnTombstoneLoadOptionName, gcSweepGenerationOptionName, GCFeatureMatrix, GCVersion, diff --git a/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts b/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts index bdeef60de283..87c299471ad7 100644 --- a/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts +++ b/packages/runtime/container-runtime/src/test/gc/garbageCollection.spec.ts @@ -5,7 +5,7 @@ import { strict as assert } from "assert"; import { SinonFakeTimers, useFakeTimers } from "sinon"; -import { ITelemetryBaseEvent } from "@fluidframework/core-interfaces"; +import { ITelemetryBaseEvent, ConfigTypes } from "@fluidframework/core-interfaces"; import { ICriticalContainerError } from "@fluidframework/container-definitions"; import { ISnapshotTree, SummaryType } from "@fluidframework/protocol-definitions"; import { @@ -20,7 +20,6 @@ import { } from "@fluidframework/runtime-definitions"; import { MockLogger, - ConfigTypes, mixinMonitoringContext, MonitoringContext, tagCodeArtifacts, diff --git a/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts b/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts index 0fb1f9026a36..9b9674d39b8f 100644 --- a/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts +++ b/packages/runtime/container-runtime/src/test/gc/gcConfigs.spec.ts @@ -43,7 +43,7 @@ import { gcTombstoneGenerationOptionName, gcSweepGenerationOptionName, throwOnTombstoneLoadOverrideKey, - gcThrowOnTombstoneLoadOptionName, + gcDisableThrowOnTombstoneLoadOptionName, GCVersion, runSessionExpiryKey, } from "../../gc"; @@ -890,35 +890,44 @@ describe("Garbage Collection configurations", () => { }); describe("throwOnTombstoneLoad", () => { - it("throwOnTombstoneLoad enabled", () => { + it("gcDisableThrowOnTombstoneLoad true", () => { gc = createGcWithPrivateMembers( { gcFeature: 0 }, - { [gcThrowOnTombstoneLoadOptionName]: true }, + { [gcDisableThrowOnTombstoneLoadOptionName]: true }, false /* isSummarizerClient */, ); - assert.equal(gc.configs.throwOnTombstoneLoad, true, "throwOnTombstoneLoad incorrect"); + assert.equal(gc.configs.throwOnTombstoneLoad, false, "throwOnTombstoneLoad incorrect"); }); - it("throwOnTombstoneLoad disabled", () => { + it("gcDisableThrowOnTombstoneLoad false", () => { gc = createGcWithPrivateMembers( { gcFeature: 0 }, - { [gcThrowOnTombstoneLoadOptionName]: false }, + { [gcDisableThrowOnTombstoneLoadOptionName]: false }, false /* isSummarizerClient */, ); - assert.equal(gc.configs.throwOnTombstoneLoad, false, "throwOnTombstoneLoad incorrect"); + assert.equal(gc.configs.throwOnTombstoneLoad, true, "throwOnTombstoneLoad incorrect"); }); - it("throwOnTombstoneLoad undefined", () => { + it("gcDisableThrowOnTombstoneLoad undefined", () => { gc = createGcWithPrivateMembers( { gcFeature: 0 }, undefined /* gcOptions */, false /* isSummarizerClient */, ); - assert.equal(gc.configs.throwOnTombstoneLoad, false, "throwOnTombstoneLoad incorrect"); + assert.equal(gc.configs.throwOnTombstoneLoad, true, "throwOnTombstoneLoad incorrect"); + }); + it("Old 'enable' option false (ignored)", () => { + const gcThrowOnTombstoneLoadOptionName_old = "gcThrowOnTombstoneLoad"; + gc = createGcWithPrivateMembers( + { gcFeature: 0 }, + { [gcThrowOnTombstoneLoadOptionName_old]: false }, + false /* isSummarizerClient */, + ); + assert.equal(gc.configs.throwOnTombstoneLoad, true, "throwOnTombstoneLoad incorrect"); }); it("throwOnTombstoneLoad enabled via override", () => { injectedSettings[throwOnTombstoneLoadOverrideKey] = true; gc = createGcWithPrivateMembers( { gcFeature: 0 }, - { [gcThrowOnTombstoneLoadOptionName]: false }, + { [gcDisableThrowOnTombstoneLoadOptionName]: true }, false /* isSummarizerClient */, ); assert.equal(gc.configs.throwOnTombstoneLoad, true, "throwOnTombstoneLoad incorrect"); @@ -927,7 +936,7 @@ describe("Garbage Collection configurations", () => { injectedSettings[throwOnTombstoneLoadOverrideKey] = false; gc = createGcWithPrivateMembers( { gcFeature: 0 }, - { [gcThrowOnTombstoneLoadOptionName]: true }, + { [gcDisableThrowOnTombstoneLoadOptionName]: false }, false /* isSummarizerClient */, ); assert.equal(gc.configs.throwOnTombstoneLoad, false, "throwOnTombstoneLoad incorrect"); From 1f5bfa2cc245a90f5204fd8091a79b0242639cae Mon Sep 17 00:00:00 2001 From: Joshua Smithrud <54606601+Josmithr@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:45:05 -0800 Subject: [PATCH 16/20] fix(api-markdown-documenter): Correctly filter child items based on release scope when rendering summary tables (#18687) #18663 Added the ability to filter what kinds of items should have documentation generated for them based on their release scope. It correctly implemented filtering where details rendering occurs (including when separate documents would be generated for those items), but did not correctly update rendering of top-level summary tables to filter out such items. This resulted in summary table entries with broken links to non-existent details entries. This PR addresses this issue. See updated test snapshots for examples of the issues now resolved. Also: - adds filter helpers to the public API via the `ApiItemUtilities` entrypoint. - Needed to ensure custom layout implementations can correctly filter child items - adds a transformation-level regression/unit test for the scenario. --- .../api-report/api-markdown-documenter.api.md | 8 + tools/api-markdown-documenter/package.json | 2 +- .../src/ApiItemUtilitiesModule.ts | 2 + .../ApiItemTransformUtilities.ts | 35 ++- .../TransformApiClass.ts | 6 +- .../TransformApiEntryPoint.ts | 7 +- .../TransformApiEnum.ts | 6 +- .../TransformApiInterface.ts | 6 +- .../TransformApiModuleLike.ts | 21 +- .../TransformApiNamespace.ts | 2 +- .../src/api-item-transforms/index.ts | 2 + .../test/Transformation.test.ts | 155 +++++++++- .../test/test-data/model-template.json | 5 +- .../test/test-data/test-namespace.json | 273 ++++++++++++++++++ .../flat-config/simple-suite-test.html | 52 ---- .../sparse-config/simple-suite-test.html | 28 -- .../testnamespace-namespace.html | 72 ----- .../flat-config/simple-suite-test.md | 9 - .../sparse-config/simple-suite-test.md | 2 - .../testnamespace-namespace.md | 12 - 20 files changed, 499 insertions(+), 206 deletions(-) create mode 100644 tools/api-markdown-documenter/src/api-item-transforms/test/test-data/test-namespace.json diff --git a/tools/api-markdown-documenter/api-report/api-markdown-documenter.api.md b/tools/api-markdown-documenter/api-report/api-markdown-documenter.api.md index d454ef3d39ad..8e0fbd923c1b 100644 --- a/tools/api-markdown-documenter/api-report/api-markdown-documenter.api.md +++ b/tools/api-markdown-documenter/api-report/api-markdown-documenter.api.md @@ -71,8 +71,10 @@ export interface ApiItemTransformationOptions { declare namespace ApiItemUtilities { export { doesItemRequireOwnDocument, + filterItems, getHeadingForApiItem, getLinkForApiItem, + shouldItemBeIncluded, getDefaultValueBlock, getDeprecatedBlock, getExampleBlocks, @@ -310,6 +312,9 @@ export interface FileSystemConfiguration { outputDirectoryPath: string; } +// @public +function filterItems(apiItems: readonly ApiItem[], config: Required): ApiItem[]; + // @public export function getApiItemTransformationConfigurationWithDefaults(inputOptions: ApiItemTransformationConfiguration): Required; @@ -587,6 +592,9 @@ export class SectionNode extends DocumentationParentNodeBase implements MultiLin readonly type = DocumentationNodeType.Section; } +// @public +function shouldItemBeIncluded(apiItem: ApiItem, config: Required): boolean; + // @public export interface SingleLineDocumentationNode extends DocumentationNode { readonly singleLine: true; diff --git a/tools/api-markdown-documenter/package.json b/tools/api-markdown-documenter/package.json index 76d771b67a85..deb4a221b9cc 100644 --- a/tools/api-markdown-documenter/package.json +++ b/tools/api-markdown-documenter/package.json @@ -1,6 +1,6 @@ { "name": "@fluid-tools/api-markdown-documenter", - "version": "0.11.2", + "version": "0.11.3", "description": "Processes .api.json files generated by API-Extractor and generates Markdown documentation from them.", "homepage": "https://fluidframework.com", "repository": { diff --git a/tools/api-markdown-documenter/src/ApiItemUtilitiesModule.ts b/tools/api-markdown-documenter/src/ApiItemUtilitiesModule.ts index 4a8f7a06d07e..12964d0decef 100644 --- a/tools/api-markdown-documenter/src/ApiItemUtilitiesModule.ts +++ b/tools/api-markdown-documenter/src/ApiItemUtilitiesModule.ts @@ -9,8 +9,10 @@ export { doesItemRequireOwnDocument, + filterItems, getHeadingForApiItem, getLinkForApiItem, + shouldItemBeIncluded, } from "./api-item-transforms"; export { getDefaultValueBlock, diff --git a/tools/api-markdown-documenter/src/api-item-transforms/ApiItemTransformUtilities.ts b/tools/api-markdown-documenter/src/api-item-transforms/ApiItemTransformUtilities.ts index 247fca1f8514..26d963341a49 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/ApiItemTransformUtilities.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/ApiItemTransformUtilities.ts @@ -463,7 +463,7 @@ function doesItemGenerateHierarchy( /** * Determines whether or not the specified API item should have documentation generated for it. * This is determined based on its release tag (or inherited release scope) compared to - * {@link ApiItemTransformationConfiguration.minimumReleaseLevel}. + * {@link DocumentationSuiteOptions.minimumReleaseLevel}. * * @remarks * @@ -499,6 +499,8 @@ function doesItemGenerateHierarchy( * } * } * ``` + * + * @public */ export function shouldItemBeIncluded( apiItem: ApiItem, @@ -521,3 +523,34 @@ export function shouldItemBeIncluded( return releaseTag >= (config.minimumReleaseLevel as ReleaseTag); } + +/** + * Filters and returns the provided list of `ApiItem`s to include only those desired by the user configuration. + * This is determined based on its release tag (or inherited release scope) compared to + * {@link DocumentationSuiteOptions.minimumReleaseLevel}. + * @param apiItem - The API item being queried. + * @param config - See {@link ApiItemTransformationConfiguration}. + * + * @public + */ +export function filterItems( + apiItems: readonly ApiItem[], + config: Required, +): ApiItem[] { + return apiItems.filter((member) => shouldItemBeIncluded(member, config)); +} + +/** + * Filters and returns the child members of the provided `apiItem` to include only those desired by the user configuration. + * This is determined based on its release tag (or inherited release scope) compared to + * {@link DocumentationSuiteOptions.minimumReleaseLevel}. + * @remarks See {@link shouldItemBeIncluded} for more details. + * @param apiItem - The API item being queried. + * @param config - See {@link ApiItemTransformationConfiguration}. + */ +export function filterChildMembers( + apiItem: ApiItem, + config: Required, +): ApiItem[] { + return filterItems(apiItem.members, config); +} diff --git a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiClass.ts b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiClass.ts index a7762caf7d3b..917793ffcce7 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiClass.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiClass.ts @@ -17,6 +17,7 @@ import { SectionNode } from "../../documentation-domain"; import { ApiModifier, filterByKind, isStatic } from "../../utilities"; import { ApiItemTransformationConfiguration } from "../configuration"; import { createChildDetailsSection, createMemberTables } from "../helpers"; +import { filterChildMembers } from "../ApiItemTransformUtilities"; /** * Default documentation transform for `Class` items. @@ -64,9 +65,8 @@ export function transformApiClass( ): SectionNode[] { const sections: SectionNode[] = []; - const hasAnyChildren = apiClass.members.length > 0; - - if (hasAnyChildren) { + const filteredChildren = filterChildMembers(apiClass, config); + if (filteredChildren.length > 0) { // Accumulate child items const constructors = filterByKind(apiClass.members, [ApiItemKind.Constructor]).map( (apiItem) => apiItem as ApiConstructor, diff --git a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEntryPoint.ts b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEntryPoint.ts index 1843ad253776..2e298c8e42c2 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEntryPoint.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEntryPoint.ts @@ -16,10 +16,5 @@ export function transformApiEntryPoint( config: Required, generateChildContent: (apiItem: ApiItem) => SectionNode[], ): SectionNode[] { - return transformApiModuleLike( - apiEntryPoint, - apiEntryPoint.members, - config, - generateChildContent, - ); + return transformApiModuleLike(apiEntryPoint, config, generateChildContent); } diff --git a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEnum.ts b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEnum.ts index e75689c4dff0..2cdc29f0ba74 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEnum.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiEnum.ts @@ -8,6 +8,7 @@ import { DocumentationNode, SectionNode } from "../../documentation-domain"; import { filterByKind } from "../../utilities"; import { ApiItemTransformationConfiguration } from "../configuration"; import { createMemberTables, wrapInSection } from "../helpers"; +import { filterChildMembers } from "../ApiItemTransformUtilities"; /** * Default documentation transform for `Enum` items. @@ -19,9 +20,8 @@ export function transformApiEnum( ): SectionNode[] { const sections: SectionNode[] = []; - const hasAnyChildren = apiEnum.members.length > 0; - - if (hasAnyChildren) { + const filteredChildren = filterChildMembers(apiEnum, config); + if (filteredChildren.length > 0) { // Accumulate child items const flags = filterByKind(apiEnum.members, [ApiItemKind.EnumMember]).map( (apiItem) => apiItem as ApiEnumMember, diff --git a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiInterface.ts b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiInterface.ts index ba91d3a9733b..0667ee3e3e87 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiInterface.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiInterface.ts @@ -17,6 +17,7 @@ import { SectionNode } from "../../documentation-domain"; import { filterByKind } from "../../utilities"; import { ApiItemTransformationConfiguration } from "../configuration"; import { createChildDetailsSection, createMemberTables } from "../helpers"; +import { filterChildMembers } from "../ApiItemTransformUtilities"; /** * Default documentation transform for `Interface` items. @@ -58,9 +59,8 @@ export function transformApiInterface( ): SectionNode[] { const childSections: SectionNode[] = []; - const hasAnyChildren = apiInterface.members.length > 0; - - if (hasAnyChildren) { + const filteredChildren = filterChildMembers(apiInterface, config); + if (filteredChildren.length > 0) { // Accumulate child items const constructSignatures = filterByKind(apiInterface.members, [ ApiItemKind.ConstructSignature, diff --git a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiModuleLike.ts b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiModuleLike.ts index 34475072d1e0..447d6af1190b 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiModuleLike.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiModuleLike.ts @@ -18,6 +18,7 @@ import { SectionNode } from "../../documentation-domain"; import { ApiModuleLike, filterByKind } from "../../utilities"; import { ApiItemTransformationConfiguration } from "../configuration"; import { createChildDetailsSection, createMemberTables } from "../helpers"; +import { filterItems } from "../ApiItemTransformUtilities"; /** * Default documentation transform for module-like API items (packages, namespaces). @@ -58,41 +59,39 @@ import { createChildDetailsSection, createMemberTables } from "../helpers"; */ export function transformApiModuleLike( apiItem: ApiModuleLike, - childItems: readonly ApiItem[], config: Required, generateChildContent: (apiItem: ApiItem) => SectionNode[], ): SectionNode[] { const children: SectionNode[] = []; - const hasAnyChildren = childItems.length > 0; - - if (hasAnyChildren) { + const filteredChildren = filterItems(apiItem.members, config); + if (filteredChildren.length > 0) { // Accumulate child items - const interfaces = filterByKind(childItems, [ApiItemKind.Interface]).map( + const interfaces = filterByKind(filteredChildren, [ApiItemKind.Interface]).map( (_apiItem) => _apiItem as ApiInterface, ); - const classes = filterByKind(childItems, [ApiItemKind.Class]).map( + const classes = filterByKind(filteredChildren, [ApiItemKind.Class]).map( (_apiItem) => _apiItem as ApiClass, ); - const namespaces = filterByKind(childItems, [ApiItemKind.Namespace]).map( + const namespaces = filterByKind(filteredChildren, [ApiItemKind.Namespace]).map( (_apiItem) => _apiItem as ApiNamespace, ); - const types = filterByKind(childItems, [ApiItemKind.TypeAlias]).map( + const types = filterByKind(filteredChildren, [ApiItemKind.TypeAlias]).map( (_apiItem) => _apiItem as ApiTypeAlias, ); - const functions = filterByKind(childItems, [ApiItemKind.Function]).map( + const functions = filterByKind(filteredChildren, [ApiItemKind.Function]).map( (_apiItem) => _apiItem as ApiFunction, ); - const enums = filterByKind(childItems, [ApiItemKind.Enum]).map( + const enums = filterByKind(filteredChildren, [ApiItemKind.Enum]).map( (_apiItem) => _apiItem as ApiEnum, ); - const variables = filterByKind(childItems, [ApiItemKind.Variable]).map( + const variables = filterByKind(filteredChildren, [ApiItemKind.Variable]).map( (_apiItem) => _apiItem as ApiVariable, ); diff --git a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiNamespace.ts b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiNamespace.ts index 903f00f6ded4..9464955a3232 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiNamespace.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/default-implementations/TransformApiNamespace.ts @@ -16,5 +16,5 @@ export function transformApiNamespace( config: Required, generateChildContent: (apiItem: ApiItem) => SectionNode[], ): SectionNode[] { - return transformApiModuleLike(apiNamespace, apiNamespace.members, config, generateChildContent); + return transformApiModuleLike(apiNamespace, config, generateChildContent); } diff --git a/tools/api-markdown-documenter/src/api-item-transforms/index.ts b/tools/api-markdown-documenter/src/api-item-transforms/index.ts index d37d7e28b51f..ea1c8d60d10e 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/index.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/index.ts @@ -9,8 +9,10 @@ export { doesItemRequireOwnDocument, + filterItems, getHeadingForApiItem, getLinkForApiItem, + shouldItemBeIncluded, } from "./ApiItemTransformUtilities"; export { type ApiItemTransformationConfiguration, diff --git a/tools/api-markdown-documenter/src/api-item-transforms/test/Transformation.test.ts b/tools/api-markdown-documenter/src/api-item-transforms/test/Transformation.test.ts index d0b0319408b5..39aed379aaa8 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/test/Transformation.test.ts +++ b/tools/api-markdown-documenter/src/api-item-transforms/test/Transformation.test.ts @@ -10,7 +10,9 @@ import { ApiItem, ApiItemKind, ApiModel, + ApiNamespace, ApiVariable, + ReleaseTag, } from "@microsoft/api-extractor-model"; import { expect } from "chai"; @@ -38,7 +40,7 @@ import { ApiItemTransformationConfiguration, getApiItemTransformationConfigurationWithDefaults, } from "../configuration"; -import { wrapInSection } from "../helpers"; +import { betaWarningSpan, wrapInSection } from "../helpers"; import { transformApiModel } from "../TransformApiModel"; /** @@ -366,6 +368,157 @@ describe("ApiItem to Documentation transformation tests", () => { expect(result).deep.equals(expected); }); + it("Transform Namespace with children at different release levels", () => { + const model = generateModel("test-namespace.json"); + const members = getApiItems(model); + const apiNamespace = findApiMember( + members, + "TestNamespace", + ApiItemKind.Namespace, + ) as ApiNamespace; + + const config = createConfig( + { + ...defaultPartialConfig, + minimumReleaseLevel: ReleaseTag.Beta, // Only include `@beta` and `@public` items in generated docs + }, + model, + ); + + const result = config.transformApiNamespace(apiNamespace, config, (childItem) => + apiItemToSections(childItem, config), + ); + + // Note: the namespace being processed includes 3 const variables: + // - foo (@public) + // - bar (@beta) + // - baz (@alpha) + // We expect docs to be generated for `foo` and `bar`, but not `baz`, since it's @alpha, and we are filtering those out per our config above. + // Also note that child items are listed alphabetically, so we expect `bar` before `foo`. + const expected: DocumentationNode[] = [ + // Summary section + wrapInSection([ParagraphNode.createFromPlainText("Test namespace")]), + + // Signature section + wrapInSection( + [ + FencedCodeBlockNode.createFromPlainText( + "export declare namespace TestNamespace", + "typescript", + ), + ], + { title: "Signature", id: "testnamespace-signature" }, + ), + + // Variables section + wrapInSection( + [ + new TableNode( + [ + // Table row for `bar` + new TableBodyRowNode([ + new TableBodyCellNode([ + LinkNode.createFromPlainText( + "bar", + "./test-package/testnamespace-namespace#bar-variable", + ), + ]), + new TableBodyCellNode([CodeSpanNode.createFromPlainText("BETA")]), // Alert + new TableBodyCellNode([ + CodeSpanNode.createFromPlainText("readonly"), + ]), // Modifier + TableBodyCellNode.Empty, // Description + ]), + // Table row for `foo` + new TableBodyRowNode([ + new TableBodyCellNode([ + LinkNode.createFromPlainText( + "foo", + "./test-package/testnamespace-namespace#foo-variable", + ), + ]), + TableBodyCellNode.Empty, // No alert for `@public` + new TableBodyCellNode([ + CodeSpanNode.createFromPlainText("readonly"), + ]), // Modifier + TableBodyCellNode.Empty, // Description + ]), + // No entry should be included for `baz` because it is `@alpha` + ], + new TableHeaderRowNode([ + TableHeaderCellNode.createFromPlainText("Variable"), + TableHeaderCellNode.createFromPlainText("Alerts"), + TableHeaderCellNode.createFromPlainText("Modifiers"), + TableHeaderCellNode.createFromPlainText("Description"), + ]), + ), + ], + { title: "Variables" }, + ), + + // Variables details section + wrapInSection( + [ + // Details for `bar` + wrapInSection( + [ + // Summary + wrapInSection([ParagraphNode.Empty]), // No summary docs on `bar` + // Beta warning + wrapInSection([betaWarningSpan]), + // Signature + wrapInSection( + [ + FencedCodeBlockNode.createFromPlainText( + 'bar = "bar"', + "typescript", + ), + ], + { + title: "Signature", + id: "bar-signature", + }, + ), + ], + { + title: "bar (BETA)", + id: "bar-variable", + }, + ), + // Details for `foo` + wrapInSection( + [ + // Summary + wrapInSection([ParagraphNode.Empty]), // No summary docs on `bar` + // Signature + wrapInSection( + [ + FencedCodeBlockNode.createFromPlainText( + 'foo = "foo"', + "typescript", + ), + ], + { + title: "Signature", + id: "foo-signature", + }, + ), + ], + { + title: "foo", + id: "foo-variable", + }, + ), + + // No entry should be included for `baz` because it is `@alpha` + ], + { title: "Variable Details" }, + ), + ]; + + expect(result).deep.equals(expected); + }); + it("Transform a Model with multiple entry-points", () => { const model = generateModel("multiple-entry-points.json"); const config = createConfig(defaultPartialConfig, model); diff --git a/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/model-template.json b/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/model-template.json index 43e5555da848..dad19026610f 100644 --- a/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/model-template.json +++ b/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/model-template.json @@ -170,7 +170,10 @@ "kind": "EntryPoint", "canonicalReference": "test-package!", "name": "", - "preserveMemberOrder": false + "preserveMemberOrder": false, + "members": [ + // Add members here as needed for your test case + ] } ] } diff --git a/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/test-namespace.json b/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/test-namespace.json new file mode 100644 index 000000000000..1c8ebd3967d7 --- /dev/null +++ b/tools/api-markdown-documenter/src/api-item-transforms/test/test-data/test-namespace.json @@ -0,0 +1,273 @@ +{ + "metadata": { + "toolPackage": "@microsoft/api-extractor", + "toolVersion": "7.38.3", + "schemaVersion": 1011, + "oldestForwardsCompatibleVersion": 1001, + "tsdocConfig": { + "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json", + "noStandardTags": true, + "tagDefinitions": [ + { + "tagName": "@alpha", + "syntaxKind": "modifier" + }, + { + "tagName": "@beta", + "syntaxKind": "modifier" + }, + { + "tagName": "@defaultValue", + "syntaxKind": "block" + }, + { + "tagName": "@decorator", + "syntaxKind": "block", + "allowMultiple": true + }, + { + "tagName": "@deprecated", + "syntaxKind": "block" + }, + { + "tagName": "@eventProperty", + "syntaxKind": "modifier" + }, + { + "tagName": "@example", + "syntaxKind": "block", + "allowMultiple": true + }, + { + "tagName": "@experimental", + "syntaxKind": "modifier" + }, + { + "tagName": "@inheritDoc", + "syntaxKind": "inline" + }, + { + "tagName": "@internal", + "syntaxKind": "modifier" + }, + { + "tagName": "@label", + "syntaxKind": "inline" + }, + { + "tagName": "@link", + "syntaxKind": "inline", + "allowMultiple": true + }, + { + "tagName": "@override", + "syntaxKind": "modifier" + }, + { + "tagName": "@packageDocumentation", + "syntaxKind": "modifier" + }, + { + "tagName": "@param", + "syntaxKind": "block", + "allowMultiple": true + }, + { + "tagName": "@privateRemarks", + "syntaxKind": "block" + }, + { + "tagName": "@public", + "syntaxKind": "modifier" + }, + { + "tagName": "@readonly", + "syntaxKind": "modifier" + }, + { + "tagName": "@remarks", + "syntaxKind": "block" + }, + { + "tagName": "@returns", + "syntaxKind": "block" + }, + { + "tagName": "@sealed", + "syntaxKind": "modifier" + }, + { + "tagName": "@see", + "syntaxKind": "block" + }, + { + "tagName": "@throws", + "syntaxKind": "block", + "allowMultiple": true + }, + { + "tagName": "@typeParam", + "syntaxKind": "block", + "allowMultiple": true + }, + { + "tagName": "@virtual", + "syntaxKind": "modifier" + }, + { + "tagName": "@betaDocumentation", + "syntaxKind": "modifier" + }, + { + "tagName": "@internalRemarks", + "syntaxKind": "block" + }, + { + "tagName": "@preapproved", + "syntaxKind": "modifier" + } + ], + "supportForTags": { + "@alpha": true, + "@beta": true, + "@defaultValue": true, + "@decorator": true, + "@deprecated": true, + "@eventProperty": true, + "@example": true, + "@experimental": true, + "@inheritDoc": true, + "@internal": true, + "@label": true, + "@link": true, + "@override": true, + "@packageDocumentation": true, + "@param": true, + "@privateRemarks": true, + "@public": true, + "@readonly": true, + "@remarks": true, + "@returns": true, + "@sealed": true, + "@see": true, + "@throws": true, + "@typeParam": true, + "@virtual": true, + "@betaDocumentation": true, + "@internalRemarks": true, + "@preapproved": true + }, + "reportUnsupportedHtmlElements": false + } + }, + "kind": "Package", + "canonicalReference": "test-package!", + "docComment": "/**\n * Test package\n */\n", + "name": "test-package", + "preserveMemberOrder": false, + "members": [ + { + "kind": "EntryPoint", + "canonicalReference": "test-package!", + "name": "", + "preserveMemberOrder": false, + "members": [ + { + "kind": "Namespace", + "canonicalReference": "package-b!TestNamespace:namespace", + "docComment": "/**\n * Test namespace\n *\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "export declare namespace TestNamespace " + } + ], + "fileUrlPath": "dist/index.d.ts", + "releaseTag": "Public", + "name": "TestNamespace", + "preserveMemberOrder": false, + "members": [ + { + "kind": "Variable", + "canonicalReference": "package-b!TestNamespace.bar:var", + "docComment": "/**\n * @beta\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "bar = " + }, + { + "kind": "Content", + "text": "\"bar\"" + } + ], + "initializerTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isReadonly": true, + "releaseTag": "Beta", + "name": "bar", + "variableTypeTokenRange": { + "startIndex": 0, + "endIndex": 0 + } + }, + { + "kind": "Variable", + "canonicalReference": "package-b!TestNamespace.baz:var", + "docComment": "/**\n * @alpha\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "baz = " + }, + { + "kind": "Content", + "text": "\"baz\"" + } + ], + "initializerTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isReadonly": true, + "releaseTag": "Alpha", + "name": "baz", + "variableTypeTokenRange": { + "startIndex": 0, + "endIndex": 0 + } + }, + { + "kind": "Variable", + "canonicalReference": "package-b!TestNamespace.foo:var", + "docComment": "/**\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "foo = " + }, + { + "kind": "Content", + "text": "\"foo\"" + } + ], + "initializerTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isReadonly": true, + "releaseTag": "Public", + "name": "foo", + "variableTypeTokenRange": { + "startIndex": 0, + "endIndex": 0 + } + } + ] + } + ] + } + ] +} diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html index 8214efa9d2c4..bf28aa5657f9 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/flat-config/simple-suite-test.html @@ -251,20 +251,6 @@

- - - testFunction(testParameter, testOptionalParameter) - - - ALPHA - - - TTypeParameter - - - Test function - - testFunctionReturningInlineType() @@ -2868,39 +2854,6 @@

-
-

- Interfaces -

- - - - - - - - - - - - - - - -
- Interface - - Alerts - - Description -
- TestInterface - - ALPHA - - Test interface -
-

Classes @@ -3081,11 +3034,6 @@

-
-

- Interface Details -

-

Class Details diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test.html index 5c98d358f222..c1c36121406d 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test.html @@ -249,20 +249,6 @@

- - - testFunction(testParameter, testOptionalParameter) - - - ALPHA - - - TTypeParameter - - - Test function - - testFunctionReturningInlineType() @@ -328,20 +314,6 @@

- - - testConst - - - BETA - - - readonly - - - Test Constant - - testConstWithEmptyDeprecatedBlock diff --git a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html index 9a99f41926cd..2a4521e179ba 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html +++ b/tools/api-markdown-documenter/src/test/snapshots/html/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.html @@ -62,39 +62,6 @@

-
-

- Interfaces -

- - - - - - - - - - - - - - - -
- Interface - - Alerts - - Description -
- TestInterface - - ALPHA - - Test interface -
-

Classes @@ -209,45 +176,6 @@

-
-

- Variables -

- - - - - - - - - - - - - - - - - -
- Variable - - Alerts - - Modifiers - - Description -
- TestConst - - BETA - - readonly - - Test Constant -
-

Namespaces diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md index 9027543fcd81..c998252a2202 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/flat-config/simple-suite-test.md @@ -72,7 +72,6 @@ const foo = bar; | Function | Alerts | Return Type | Description | | --- | --- | --- | --- | -| [testFunction(testParameter, testOptionalParameter)](docs/simple-suite-test#testfunction-function) | `ALPHA` | TTypeParameter | Test function | | [testFunctionReturningInlineType()](docs/simple-suite-test#testfunctionreturninginlinetype-function) | | { foo: number; bar: [TestEnum](docs/simple-suite-test#testenum-enum); } | Test function that returns an inline type | | [testFunctionReturningIntersectionType()](docs/simple-suite-test#testfunctionreturningintersectiontype-function) | `DEPRECATED` | [TestEmptyInterface](docs/simple-suite-test#testemptyinterface-interface) & [TestInterfaceWithTypeParameter](docs/simple-suite-test#testinterfacewithtypeparameter-interface)<number> | Test function that returns an inline type | | [testFunctionReturningUnionType()](docs/simple-suite-test#testfunctionreturninguniontype-function) | | string \| [TestInterface](docs/simple-suite-test#testinterface-interface) | Test function that returns an inline type | @@ -1005,12 +1004,6 @@ const foo = bar; const bar = foo ``` -### Interfaces - -| Interface | Alerts | Description | -| --- | --- | --- | -| [TestInterface](docs/simple-suite-test#testnamespace-testinterface-interface) | `ALPHA` | Test interface | - ### Classes | Class | Description | @@ -1047,8 +1040,6 @@ const bar = foo | --- | --- | | [TestSubNamespace](docs/simple-suite-test#testnamespace-testsubnamespace-namespace) | Test sub-namespace | -### Interface Details - ### Class Details #### TestClass {#testnamespace-testclass-class} diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test.md index edc82396a117..caf3fc1e73d1 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test.md @@ -70,7 +70,6 @@ const foo = bar; | Function | Alerts | Return Type | Description | | --- | --- | --- | --- | -| [testFunction(testParameter, testOptionalParameter)](docs/simple-suite-test/testfunction-function) | `ALPHA` | TTypeParameter | Test function | | [testFunctionReturningInlineType()](docs/simple-suite-test/testfunctionreturninginlinetype-function) | | { foo: number; bar: [TestEnum](docs/simple-suite-test/testenum-enum); } | Test function that returns an inline type | | [testFunctionReturningIntersectionType()](docs/simple-suite-test/testfunctionreturningintersectiontype-function) | `DEPRECATED` | [TestEmptyInterface](docs/simple-suite-test/testemptyinterface-interface) & [TestInterfaceWithTypeParameter](docs/simple-suite-test/testinterfacewithtypeparameter-interface)<number> | Test function that returns an inline type | | [testFunctionReturningUnionType()](docs/simple-suite-test/testfunctionreturninguniontype-function) | | string \| [TestInterface](docs/simple-suite-test/testinterface-interface) | Test function that returns an inline type | @@ -79,7 +78,6 @@ const foo = bar; | Variable | Alerts | Modifiers | Description | | --- | --- | --- | --- | -| [testConst](docs/simple-suite-test/testconst-variable) | `BETA` | `readonly` | Test Constant | | [testConstWithEmptyDeprecatedBlock](docs/simple-suite-test/testconstwithemptydeprecatedblock-variable) | `DEPRECATED` | `readonly` | I have a `@deprecated` tag with an empty comment block. | ### Namespaces diff --git a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md index a6baffdf5f1a..57265e93d89e 100644 --- a/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md +++ b/tools/api-markdown-documenter/src/test/snapshots/markdown/simple-suite-test/sparse-config/simple-suite-test/testnamespace-namespace.md @@ -26,12 +26,6 @@ const foo = bar; const bar = foo ``` -### Interfaces - -| Interface | Alerts | Description | -| --- | --- | --- | -| [TestInterface](docs/simple-suite-test/testnamespace-testinterface-interface) | `ALPHA` | Test interface | - ### Classes | Class | Description | @@ -56,12 +50,6 @@ const bar = foo | --- | --- | --- | | [testFunction(testParameter)](docs/simple-suite-test/testnamespace-testfunction-function) | number | Test function | -### Variables - -| Variable | Alerts | Modifiers | Description | -| --- | --- | --- | --- | -| [TestConst](docs/simple-suite-test/testnamespace-testconst-variable) | `BETA` | `readonly` | Test Constant | - ### Namespaces | Namespace | Description | From 44ef1a10258128111c18f9072b41f4e03d99a9f3 Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:12:46 -0600 Subject: [PATCH 17/20] docs(tree2): Typos and improvements (#18667) --- .../dds/tree2/src/class-tree/schemaFactory.ts | 2 +- experimental/dds/tree2/src/class-tree/tree.ts | 27 ++++++++++--------- .../dds/tree2/src/domains/leafDomain.ts | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/experimental/dds/tree2/src/class-tree/schemaFactory.ts b/experimental/dds/tree2/src/class-tree/schemaFactory.ts index 90c7fa8d78bc..a0d6f5e01e6e 100644 --- a/experimental/dds/tree2/src/class-tree/schemaFactory.ts +++ b/experimental/dds/tree2/src/class-tree/schemaFactory.ts @@ -158,7 +158,7 @@ export class SchemaFactory( config: TreeConfiguration, @@ -67,9 +68,9 @@ export interface ITree extends IChannel { export class TreeConfiguration { /** * @param schema - The schema which the application wants to view the tree with. - * @param initialTree - Default tree content to initialize the tree with iff the tree is uninitialized + * @param initialTree - A function that returns the default tree content to initialize the tree with iff the tree is uninitialized * (meaning it does not even have any schema set at all). - * If the `initialTree` returns any actual node instances, they should be recreated each time the `initialTree` runs. + * If `initialTree` returns any actual node instances, they should be recreated each time `initialTree` runs. * This is because if the config is used a second time any nodes that were not recreated could error since nodes cannot be inserted into the tree multiple times. */ public constructor( diff --git a/experimental/dds/tree2/src/domains/leafDomain.ts b/experimental/dds/tree2/src/domains/leafDomain.ts index 6456e664b7a7..4492a60b0593 100644 --- a/experimental/dds/tree2/src/domains/leafDomain.ts +++ b/experimental/dds/tree2/src/domains/leafDomain.ts @@ -73,7 +73,7 @@ export const leaf = { * * @remarks * There are good [reasons to avoid using null](https://www.npmjs.com/package/%40rushstack/eslint-plugin#rushstackno-new-null) in JavaScript, however sometimes it is desired. - * This {@link LeafNodeSchema} node provide the option to include nulls in trees when desired. + * This {@link LeafNodeSchema} node provides the option to include nulls in trees when desired. * Unless directly inter-operating with existing data using null, consider other approaches, like wrapping the value in an optional field, or using a more specifically named empty object node. */ null: nullSchema, From e9ff89c689b91ec4a6815dba2f3e2e94bbda9a3b Mon Sep 17 00:00:00 2001 From: yann-achard-MS <97201204+yann-achard-MS@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:40:34 -0800 Subject: [PATCH 18/20] feat(tree2): updated relevantRemovedRoots contract (#18606) --- .../tree2/src/feature-libraries/deltaUtils.ts | 8 +- .../modular-schema/fieldChangeHandler.ts | 8 +- .../modular-schema/genericFieldKind.ts | 2 +- .../optional-field/optionalField.ts | 17 +- .../sequence-field/relevantRemovedRoots.ts | 22 +-- .../modular-schema/genericFieldKind.spec.ts | 4 +- .../modularChangeFamily.spec.ts | 2 +- .../optional-field/optionalField.spec.ts | 181 ++++++++++-------- .../relevantRemovedRoots.spec.ts | 149 +++++++------- 9 files changed, 195 insertions(+), 198 deletions(-) diff --git a/experimental/dds/tree2/src/feature-libraries/deltaUtils.ts b/experimental/dds/tree2/src/feature-libraries/deltaUtils.ts index 40bd0d13205c..7e8454bef0ba 100644 --- a/experimental/dds/tree2/src/feature-libraries/deltaUtils.ts +++ b/experimental/dds/tree2/src/feature-libraries/deltaUtils.ts @@ -11,12 +11,16 @@ import { DeltaMark, DeltaRoot, FieldKey, + RevisionTag, makeDetachedNodeId, } from "../core"; import { Mutable } from "../util"; -export function nodeIdFromChangeAtom(changeAtom: ChangeAtomId): DeltaDetachedNodeId { - return makeDetachedNodeId(changeAtom.revision, changeAtom.localId); +export function nodeIdFromChangeAtom( + changeAtom: ChangeAtomId, + fallbackRevision?: RevisionTag, +): DeltaDetachedNodeId { + return makeDetachedNodeId(changeAtom.revision ?? fallbackRevision, changeAtom.localId); } /** diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts index 553bc5446a4a..88af1623eb98 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/fieldChangeHandler.ts @@ -40,7 +40,11 @@ export interface FieldChangeHandler< ): DeltaFieldChanges; /** * Returns the set of removed roots that should be in memory for the given change to be applied. - * A detached tree is relevant if it is being restored or being edited (or both). + * A removed root is relevant if any of the following is true: + * - It is being inserted + * - It is being restored + * - It is being edited + * - The ID it is associated with is being changed * * Implementations are allowed to be conservative by returning more removed roots than strictly necessary * (though they should, for the sake of performance, try to avoid doing so). @@ -51,7 +55,7 @@ export interface FieldChangeHandler< * @param relevantRemovedRootsFromChild - Delegate for collecting relevant removed roots from child changes. */ readonly relevantRemovedRoots: ( - change: TChangeset, + change: TaggedChange, relevantRemovedRootsFromChild: RelevantRemovedRootsFromChild, ) => Iterable; diff --git a/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts b/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts index b9acbc9f04a0..adc87b1f89a3 100644 --- a/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts +++ b/experimental/dds/tree2/src/feature-libraries/modular-schema/genericFieldKind.ts @@ -222,7 +222,7 @@ export function newGenericChangeset(): GenericChangeset { } function* relevantRemovedRoots( - change: GenericChangeset, + { change }: TaggedChange, relevantRemovedRootsFromChild: RelevantRemovedRootsFromChild, ): Iterable { for (const { nodeChange } of change) { diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts index efa32612b112..05b9a8826862 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts @@ -633,26 +633,23 @@ function areEqualRegisterIds(a: RegisterId, b: RegisterId): boolean { } function* relevantRemovedRoots( - change: OptionalChangeset, + { change, revision }: TaggedChange, relevantRemovedRootsFromChild: RelevantRemovedRootsFromChild, ): Iterable { - const alreadyYieldedOrNewlyBuilt = new RegisterMap(); - for (const { id } of change.build) { - alreadyYieldedOrNewlyBuilt.set(id, true); - } + const alreadyYielded = new RegisterMap(); for (const [src] of change.moves) { - if (src !== "self" && !alreadyYieldedOrNewlyBuilt.has(src)) { - alreadyYieldedOrNewlyBuilt.set(src, true); - yield nodeIdFromChangeAtom(src); + if (src !== "self" && !alreadyYielded.has(src)) { + alreadyYielded.set(src, true); + yield nodeIdFromChangeAtom(src, revision); } } for (const [id, childChange] of change.childChanges) { // Child changes make the tree they apply to relevant unless that tree existed in the starting context of // of this change. - if (id !== "self" && !alreadyYieldedOrNewlyBuilt.has(id)) { - alreadyYieldedOrNewlyBuilt.set(id, true); + if (id !== "self" && !alreadyYielded.has(id)) { + alreadyYielded.set(id, true); yield nodeIdFromChangeAtom(id); } yield* relevantRemovedRootsFromChild(childChange); diff --git a/experimental/dds/tree2/src/feature-libraries/sequence-field/relevantRemovedRoots.ts b/experimental/dds/tree2/src/feature-libraries/sequence-field/relevantRemovedRoots.ts index 6b1bf57a7f57..d02cda9b7860 100644 --- a/experimental/dds/tree2/src/feature-libraries/sequence-field/relevantRemovedRoots.ts +++ b/experimental/dds/tree2/src/feature-libraries/sequence-field/relevantRemovedRoots.ts @@ -4,32 +4,26 @@ */ import { assert } from "@fluidframework/core-utils"; -import { DeltaDetachedNodeId, offsetDetachId } from "../../core"; +import { DeltaDetachedNodeId, TaggedChange, offsetDetachId } from "../../core"; import { nodeIdFromChangeAtom } from "../deltaUtils"; import { Changeset, Mark } from "./types"; -import { - isInsert, - isNewAttach, - isReattachEffect, - isDetachOfRemovedNodes, - isAttachAndDetachEffect, -} from "./utils"; +import { isInsert, isDetachOfRemovedNodes, isAttachAndDetachEffect } from "./utils"; export type RelevantRemovedRootsFromTChild = ( child: TChild, ) => Iterable; export function* relevantRemovedRoots( - changeset: Changeset, + { change, revision }: TaggedChange>, relevantRemovedRootsFromChild: RelevantRemovedRootsFromTChild, ): Iterable { - for (const mark of changeset) { + for (const mark of change) { if (refersToRelevantRemovedRoots(mark)) { assert( mark.cellId !== undefined, 0x81d /* marks referring to removed trees must have an assigned cell ID */, ); - const nodeId = nodeIdFromChangeAtom(mark.cellId); + const nodeId = nodeIdFromChangeAtom(mark.cellId, revision); for (let i = 0; i < mark.count; i += 1) { yield offsetDetachId(nodeId, i); } @@ -43,14 +37,14 @@ export function* relevantRemovedRoots( function refersToRelevantRemovedRoots(mark: Mark): boolean { if (mark.cellId !== undefined) { const effect = isAttachAndDetachEffect(mark) ? mark.attach : mark; - if (isInsert(effect) && isReattachEffect(effect, mark.cellId)) { - // This tree is being restored. + if (isInsert(effect)) { + // This tree is being inserted or restored. return true; } else if (isDetachOfRemovedNodes(mark)) { // This removed tree is being restored as part of a detach. return true; } - if (!isNewAttach(mark) && mark.changes !== undefined) { + if (mark.changes !== undefined) { // This removed tree is being edited. // Note: there is a possibility that the child changes only affect a distant descendant // which may have been removed from this (removed) subtree. In such a case, this tree is not truly diff --git a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts index d6ab2ef6dd3f..58d5d39f6013 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/genericFieldKind.spec.ts @@ -452,7 +452,7 @@ describe("Generic FieldKind", () => { it("relevantRemovedRoots", () => { const actual = genericFieldKind.changeHandler.relevantRemovedRoots( - [ + makeAnonChange([ { index: 0, nodeChange: nodeChange0To1, @@ -461,7 +461,7 @@ describe("Generic FieldKind", () => { index: 2, nodeChange: nodeChange1To2, }, - ], + ]), (child) => child === nodeChange0To1 ? [{ minor: 42 }] diff --git a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts index 6afa4943bf63..7c3e81b63503 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/modular-schema/modularChangeFamily.spec.ts @@ -79,7 +79,7 @@ const singleNodeHandler: FieldChangeHandler = { local: [{ count: 1, fields: deltaFromChild(change) }], }), relevantRemovedRoots: (change, relevantRemovedRootsFromChild) => - relevantRemovedRootsFromChild(change), + relevantRemovedRootsFromChild(change.change), isEmpty: (change) => change.fieldChanges === undefined, }; diff --git a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts index c1b6149d79bb..d2879ad42af6 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts @@ -529,7 +529,7 @@ describe("optionalField", () => { optionalFieldEditor.buildChildChange(0, nodeChange1), mintRevisionTag(), ); - const relevantNestedTree = { major: "Child revision", minor: 4242 }; + const relevantNestedTree = { minor: 4242 }; const failingDelegate: RelevantRemovedRootsFromChild = (): never => assert.fail("Should not be called"); const noTreesDelegate: RelevantRemovedRootsFromChild = () => []; @@ -538,40 +538,22 @@ describe("optionalField", () => { return [relevantNestedTree]; }; describe("does not include", () => { - it("a tree being inserted", () => { - const actual = Array.from( - optionalChangeHandler.relevantRemovedRoots(fill.change, noTreesDelegate), - ); - assert.deepEqual(actual, []); - }); - it("a tree with child changes being inserted", () => { - const changes = [fill, hasChildChanges]; - const fillAndChange = optionalChangeRebaser.compose( - changes, - (): NodeChangeset => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - defaultRevisionMetadataFromChanges(changes), - ); - const actual = Array.from( - optionalChangeHandler.relevantRemovedRoots(fillAndChange, noTreesDelegate), - ); - assert.deepEqual(actual, []); - }); it("a tree being removed", () => { const actual = Array.from( - optionalChangeHandler.relevantRemovedRoots(clear.change, noTreesDelegate), + optionalChangeHandler.relevantRemovedRoots(clear, noTreesDelegate), ); assert.deepEqual(actual, []); }); it("a tree with child changes being removed", () => { const changes = [hasChildChanges, clear]; - const changeAndClear = optionalChangeRebaser.compose( - changes, - (): NodeChangeset => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - defaultRevisionMetadataFromChanges(changes), + const changeAndClear = makeAnonChange( + optionalChangeRebaser.compose( + changes, + (): NodeChangeset => nodeChange1, + fakeIdAllocator, + failCrossFieldManager, + defaultRevisionMetadataFromChanges(changes), + ), ); const actual = Array.from( optionalChangeHandler.relevantRemovedRoots(changeAndClear, noTreesDelegate), @@ -581,7 +563,7 @@ describe("optionalField", () => { it("a tree that remains untouched", () => { const actual = Array.from( optionalChangeHandler.relevantRemovedRoots( - { build: [], moves: [], childChanges: [] }, + makeAnonChange({ build: [], moves: [], childChanges: [] }), noTreesDelegate, ), ); @@ -589,22 +571,27 @@ describe("optionalField", () => { }); it("a tree that remains untouched aside from child changes", () => { const actual = Array.from( - optionalChangeHandler.relevantRemovedRoots( - hasChildChanges.change, - noTreesDelegate, - ), + optionalChangeHandler.relevantRemovedRoots(hasChildChanges, noTreesDelegate), ); assert.deepEqual(actual, []); }); }); describe("does include", () => { + it("a tree being inserted", () => { + const actual = Array.from( + optionalChangeHandler.relevantRemovedRoots(fill, noTreesDelegate), + ); + assert.deepEqual(actual, [makeDetachedNodeId(fill.revision, 2)]); + }); it("a tree being restored", () => { - const restore = optionalChangeRebaser.invert( - clear, - () => assert.fail("Should not need to invert children"), - fakeIdAllocator, - failCrossFieldManager, - defaultRevisionMetadataFromChanges([clear]), + const restore = makeAnonChange( + optionalChangeRebaser.invert( + clear, + () => assert.fail("Should not need to invert children"), + fakeIdAllocator, + failCrossFieldManager, + defaultRevisionMetadataFromChanges([clear]), + ), ); const actual = Array.from( optionalChangeHandler.relevantRemovedRoots(restore, failingDelegate), @@ -613,15 +600,17 @@ describe("optionalField", () => { assert.deepEqual(actual, expected); }); it("a tree that remains removed but has nested changes", () => { - const rebasedNestedChange = optionalChangeRebaser.rebase( - hasChildChanges.change, - clear, - () => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - rebaseRevisionMetadataFromInfo( - defaultRevInfosFromChanges([clear, hasChildChanges]), - [clear.revision], + const rebasedNestedChange = makeAnonChange( + optionalChangeRebaser.rebase( + hasChildChanges.change, + clear, + () => nodeChange1, + fakeIdAllocator, + failCrossFieldManager, + rebaseRevisionMetadataFromInfo( + defaultRevInfosFromChanges([clear, hasChildChanges]), + [clear.revision], + ), ), ); const actual = Array.from( @@ -633,35 +622,42 @@ describe("optionalField", () => { const expected = [makeDetachedNodeId(clear.revision, 1)]; assert.deepEqual(actual, expected); }); - it("relevant trees from nested changes under a tree being inserted", () => { + it("relevant roots from nested changes under a tree being inserted", () => { const changes = [fill, hasChildChanges]; - const fillAndChange = optionalChangeRebaser.compose( - changes, - (): NodeChangeset => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - defaultRevisionMetadataFromChanges(changes), + const fillAndChange = makeAnonChange( + optionalChangeRebaser.compose( + changes, + (): NodeChangeset => nodeChange1, + fakeIdAllocator, + failCrossFieldManager, + defaultRevisionMetadataFromChanges(changes), + ), ); const actual = Array.from( optionalChangeHandler.relevantRemovedRoots(fillAndChange, oneTreeDelegate), ); - assert.deepEqual(actual, [relevantNestedTree]); + assert.deepEqual(actual, [ + makeDetachedNodeId(fill.revision, 2), + relevantNestedTree, + ]); }); - it("relevant trees from nested changes under a tree being removed", () => { + it("relevant roots from nested changes under a tree being removed", () => { const changes = [hasChildChanges, clear]; - const changeAndClear = optionalChangeRebaser.compose( - changes, - (): NodeChangeset => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - defaultRevisionMetadataFromChanges(changes), + const changeAndClear = makeAnonChange( + optionalChangeRebaser.compose( + changes, + (): NodeChangeset => nodeChange1, + fakeIdAllocator, + failCrossFieldManager, + defaultRevisionMetadataFromChanges(changes), + ), ); const actual = Array.from( optionalChangeHandler.relevantRemovedRoots(changeAndClear, oneTreeDelegate), ); assert.deepEqual(actual, [relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being restored", () => { + it("relevant roots from nested changes under a tree being restored", () => { const restore = tagChange( optionalChangeRebaser.invert( clear, @@ -673,12 +669,14 @@ describe("optionalField", () => { mintRevisionTag(), ); const changes = [restore, hasChildChanges]; - const restoreAndChange = optionalChangeRebaser.compose( - changes, - (): NodeChangeset => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - defaultRevisionMetadataFromChanges(changes), + const restoreAndChange = makeAnonChange( + optionalChangeRebaser.compose( + changes, + (): NodeChangeset => nodeChange1, + fakeIdAllocator, + failCrossFieldManager, + defaultRevisionMetadataFromChanges(changes), + ), ); const actual = Array.from( optionalChangeHandler.relevantRemovedRoots(restoreAndChange, oneTreeDelegate), @@ -686,16 +684,18 @@ describe("optionalField", () => { const expected = [makeDetachedNodeId(clear.revision, 1), relevantNestedTree]; assert.deepEqual(actual, expected); }); - it("relevant trees from nested changes under a tree that remains removed", () => { - const rebasedNestedChange = optionalChangeRebaser.rebase( - hasChildChanges.change, - clear, - () => nodeChange1, - fakeIdAllocator, - failCrossFieldManager, - rebaseRevisionMetadataFromInfo( - defaultRevInfosFromChanges([clear, hasChildChanges]), - [clear.revision], + it("relevant roots from nested changes under a tree that remains removed", () => { + const rebasedNestedChange = makeAnonChange( + optionalChangeRebaser.rebase( + hasChildChanges.change, + clear, + () => nodeChange1, + fakeIdAllocator, + failCrossFieldManager, + rebaseRevisionMetadataFromInfo( + defaultRevInfosFromChanges([clear, hasChildChanges]), + [clear.revision], + ), ), ); const actual = Array.from( @@ -707,15 +707,26 @@ describe("optionalField", () => { const expected = [makeDetachedNodeId(clear.revision, 1), relevantNestedTree]; assert.deepEqual(actual, expected); }); - it("relevant trees from nested changes under a tree that remains in-doc ", () => { + it("relevant roots from nested changes under a tree that remains in-doc", () => { const actual = Array.from( - optionalChangeHandler.relevantRemovedRoots( - hasChildChanges.change, - oneTreeDelegate, - ), + optionalChangeHandler.relevantRemovedRoots(hasChildChanges, oneTreeDelegate), ); assert.deepEqual(actual, [relevantNestedTree]); }); }); + it("uses passed down revision", () => { + const restore = tagChange( + { + moves: [[{ localId: brand(42) }, "self", "nodeTargeting"]], + build: [], + childChanges: [], + }, + tag, + ); + const actual = Array.from( + optionalChangeHandler.relevantRemovedRoots(restore, failingDelegate), + ); + assert.deepEqual(actual, [{ major: tag, minor: 42 }]); + }); }); }); diff --git a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/relevantRemovedRoots.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/relevantRemovedRoots.spec.ts index 422969ed618d..d8079e26a65f 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/sequence-field/relevantRemovedRoots.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/sequence-field/relevantRemovedRoots.spec.ts @@ -4,16 +4,23 @@ */ import { strict as assert } from "assert"; -import { ChangeAtomId, DeltaDetachedNodeId, mintRevisionTag } from "../../../core"; +import { + ChangeAtomId, + DeltaDetachedNodeId, + makeAnonChange, + mintRevisionTag, + tagChange, +} from "../../../core"; import { SequenceField as SF } from "../../../feature-libraries"; import { brand } from "../../../util"; import { TestChange } from "../../testChange"; import { TestChangeset, MarkMaker as Mark } from "./testEdits"; -const atomId: ChangeAtomId = { revision: mintRevisionTag(), localId: brand(0) }; -const deltaId: DeltaDetachedNodeId = { major: atomId.revision, minor: atomId.localId }; +const tag = mintRevisionTag(); +const atomId: ChangeAtomId = { localId: brand(0) }; +const deltaId: DeltaDetachedNodeId = { minor: atomId.localId }; const childChange = TestChange.mint([0], 1); -const relevantNestedTree = { major: "Child revision", minor: 4242 }; +const relevantNestedTree = { minor: 4242 }; const oneTreeDelegate = (child: TestChange) => { assert.deepEqual(child, childChange); return [relevantNestedTree]; @@ -27,49 +34,37 @@ describe("SequenceField - relevantRemovedRoots", () => { describe("does not include", () => { it("a tree that remains in-doc", () => { const input: TestChangeset = [{ count: 1 }]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); it("a tree with child changes that remains in-doc", () => { const input: TestChangeset = [Mark.modify(childChange)]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); it("a tree that remains removed", () => { const input: TestChangeset = [{ count: 1, cellId: atomId }]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); it("a tree being removed", () => { const input: TestChangeset = [Mark.delete(1, atomId)]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); it("a tree with child changes being removed", () => { const input: TestChangeset = [Mark.delete(1, atomId, { changes: childChange })]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); - const array = Array.from(actual); - assert.deepEqual(array, []); - }); - it("a tree being inserted", () => { - const input: TestChangeset = [Mark.insert(1, atomId)]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); - const array = Array.from(actual); - assert.deepEqual(array, []); - }); - it("a tree with child changes being inserted", () => { - const input: TestChangeset = [Mark.insert(1, atomId, { changes: childChange })]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); it("a tree being moved", () => { const input: TestChangeset = [Mark.moveOut(1, atomId), Mark.moveIn(1, atomId)]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); @@ -78,152 +73,144 @@ describe("SequenceField - relevantRemovedRoots", () => { Mark.moveOut(1, atomId, { changes: childChange }), Mark.moveIn(1, atomId), ]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); - it("a tree being transiently inserted", () => { - const input: TestChangeset = [ - Mark.attachAndDetach(Mark.insert(1, atomId), Mark.delete(1, atomId)), - ]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + it("a live tree being pinned", () => { + const input: TestChangeset = [Mark.pin(1, brand(0))]; + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, []); }); - it("a tree with child changes being transiently inserted", () => { - const input: TestChangeset = [ - Mark.attachAndDetach(Mark.insert(1, atomId), Mark.delete(1, atomId), { - changes: childChange, - }), - ]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + }); + describe("does include", () => { + it("a tree being inserted", () => { + const input: TestChangeset = [Mark.insert(1, atomId)]; + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); - assert.deepEqual(array, []); + assert.deepEqual(array, [deltaId]); }); - it("a tree being transiently inserted and moved out", () => { + it("a tree being transiently inserted", () => { const input: TestChangeset = [ - Mark.attachAndDetach(Mark.insert(1, atomId), Mark.moveOut(1, atomId)), + Mark.attachAndDetach(Mark.insert(1, atomId), Mark.delete(1, atomId)), ]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); - assert.deepEqual(array, []); + assert.deepEqual(array, [deltaId]); }); - it("a tree with child changes being transiently inserted and moved out", () => { + it("a tree being transiently inserted and moved out", () => { const input: TestChangeset = [ - Mark.attachAndDetach(Mark.insert(1, atomId), Mark.moveOut(1, atomId), { - changes: childChange, - }), + Mark.attachAndDetach(Mark.insert(1, atomId), Mark.moveOut(1, atomId)), ]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); - const array = Array.from(actual); - assert.deepEqual(array, []); - }); - it("a live tree being pinned", () => { - const input: TestChangeset = [Mark.pin(1, brand(0))]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); - assert.deepEqual(array, []); + assert.deepEqual(array, [deltaId]); }); - }); - describe("does include", () => { - it("relevant trees from nested changes under a tree that remains in-doc", () => { + it("relevant roots from nested changes under a tree that remains in-doc", () => { const input: TestChangeset = [Mark.modify(childChange)]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [relevantNestedTree]); }); - it("relevant trees from nested changes under a tree that remains removed", () => { + it("relevant roots from nested changes under a tree that remains removed", () => { const input: TestChangeset = [Mark.modify(childChange, atomId)]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId, relevantNestedTree]); }); it("a removed tree with nested changes", () => { const input: TestChangeset = [Mark.modify(childChange, atomId)]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId]); }); it("a tree being restored by revive", () => { const input: TestChangeset = [Mark.revive(1, atomId)]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId]); }); it("a tree being restored by pin", () => { const input: TestChangeset = [Mark.pin(1, brand(0), { cellId: atomId })]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId]); }); it("a tree being transiently restored", () => { const input: TestChangeset = [Mark.delete(1, brand(0), { cellId: atomId })]; - const actual = SF.relevantRemovedRoots(input, noTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), noTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId]); }); - it("relevant trees from nested changes under a tree being restored by revive", () => { + it("relevant roots from nested changes under a tree being restored by revive", () => { const input: TestChangeset = [Mark.revive(1, atomId, { changes: childChange })]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId, relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being restored by pin", () => { + it("relevant roots from nested changes under a tree being restored by pin", () => { const input: TestChangeset = [ Mark.pin(1, brand(0), { cellId: atomId, changes: childChange }), ]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId, relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being removed", () => { + it("relevant roots from nested changes under a tree being removed", () => { const input: TestChangeset = [Mark.delete(1, atomId, { changes: childChange })]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being inserted", () => { + it("relevant roots from nested changes under a tree being inserted", () => { const input: TestChangeset = [Mark.insert(1, atomId, { changes: childChange })]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); - assert.deepEqual(array, [relevantNestedTree]); + assert.deepEqual(array, [deltaId, relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being moved", () => { + it("relevant roots from nested changes under a tree being moved", () => { const input: TestChangeset = [ Mark.moveOut(1, atomId, { changes: childChange }), Mark.moveIn(1, atomId), ]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being transiently inserted", () => { + it("relevant roots from nested changes under a tree being transiently inserted", () => { const input: TestChangeset = [ Mark.attachAndDetach(Mark.insert(1, atomId), Mark.delete(1, atomId), { changes: childChange, }), ]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); - assert.deepEqual(array, [relevantNestedTree]); + assert.deepEqual(array, [deltaId, relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being transiently restored", () => { + it("relevant roots from nested changes under a tree being transiently restored", () => { const input: TestChangeset = [ Mark.delete(1, brand(0), { cellId: atomId, changes: childChange }), ]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); assert.deepEqual(array, [deltaId, relevantNestedTree]); }); - it("relevant trees from nested changes under a tree being transiently inserted and moved out", () => { + it("relevant roots from nested changes under a tree being transiently inserted and moved out", () => { const input: TestChangeset = [ Mark.attachAndDetach(Mark.insert(1, atomId), Mark.moveOut(1, atomId), { changes: childChange, }), ]; - const actual = SF.relevantRemovedRoots(input, oneTreeDelegate); + const actual = SF.relevantRemovedRoots(makeAnonChange(input), oneTreeDelegate); const array = Array.from(actual); - assert.deepEqual(array, [relevantNestedTree]); + assert.deepEqual(array, [deltaId, relevantNestedTree]); }); }); + it("uses passed down revision", () => { + const input: TestChangeset = [Mark.modify(childChange, { localId: brand(42) })]; + const actual = SF.relevantRemovedRoots(tagChange(input, tag), noTreeDelegate); + const array = Array.from(actual); + assert.deepEqual(array, [{ major: tag, minor: 42 }]); + }); }); From 1e69222fa23b5c211e7794060f78d62f1410082c Mon Sep 17 00:00:00 2001 From: justus-camp <102552460+justus-camp@users.noreply.github.com> Date: Thu, 7 Dec 2023 16:26:23 -0800 Subject: [PATCH 19/20] WIP: Optional field root builds (#18701) ## Description Migrating the OptionalField FieldKind to use root builds. ## Reviewer Guidance The review process is outlined on [this wiki page](https://github.com/microsoft/FluidFramework/wiki/PR-Guidelines#guidelines). --------- Co-authored-by: Yann Achard --- .../default-schema/defaultChangeFamily.ts | 57 +++- .../default-schema/defaultFieldKinds.ts | 18 +- .../optional-field/optionalField.ts | 47 +-- .../optionalFieldChangeFormat.ts | 3 +- .../optionalFieldChangeTypes.ts | 7 +- .../optional-field/optionalFieldCodecs.ts | 22 +- .../sequence-field/sequenceFieldToDelta.ts | 5 - .../defaultFieldKinds.spec.ts | 30 +- .../optionalChangeRebaser.spec.ts | 12 +- .../optional-field/optionalField.spec.ts | 43 +-- .../optionalFieldChangeCodecs.spec.ts | 7 +- .../optional-field/optionalFieldUtils.ts | 2 - .../snapshots/files/has-handle-final.json | 50 ++- .../files/optional-field-scenarios-final.json | 316 ++++++++++++------ 14 files changed, 315 insertions(+), 304 deletions(-) diff --git a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts index b6ddc984a98f..ddfea329f5e1 100644 --- a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts +++ b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultChangeFamily.ts @@ -4,6 +4,7 @@ */ import { assert } from "@fluidframework/core-utils"; +import { OptionalChangeset } from "../optional-field"; import { ICodecFamily, ICodecOptions } from "../../codec"; import { ChangeFamily, @@ -17,6 +18,7 @@ import { TaggedChange, DeltaRoot, StoredSchemaCollection, + ChangesetLocalId, } from "../../core"; import { brand, isReadonlyArray } from "../../util"; import { @@ -27,6 +29,7 @@ import { FieldEditDescription, FullSchemaPolicy, intoDelta as intoModularDelta, + EditDescription, } from "../modular-schema"; import { fieldKinds, optional, sequence, required as valueFieldKind } from "./defaultFieldKinds"; @@ -153,13 +156,23 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild public valueField(field: FieldUpPath): ValueFieldEditBuilder { return { set: (newContent: ITreeCursor): void => { + const fillId = this.modularBuilder.generateId(); + + const build = this.modularBuilder.buildTrees(fillId, [newContent]); const change: FieldChangeset = brand( - valueFieldKind.changeHandler.editor.set(newContent, { - fill: this.modularBuilder.generateId(), + valueFieldKind.changeHandler.editor.set({ + fill: fillId, detach: this.modularBuilder.generateId(), }), ); - this.modularBuilder.submitChange(field, valueFieldKind.identifier, change); + + const edit: FieldEditDescription = { + type: "field", + field, + fieldKind: valueFieldKind.identifier, + change, + }; + this.modularBuilder.submitChanges([build, edit]); }, }; } @@ -167,16 +180,36 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild public optionalField(field: FieldUpPath): OptionalFieldEditBuilder { return { set: (newContent: ITreeCursor | undefined, wasEmpty: boolean): void => { - const id = this.modularBuilder.generateId(); - const optionalChange = - newContent === undefined - ? optional.changeHandler.editor.clear(wasEmpty, id) - : optional.changeHandler.editor.set(newContent, wasEmpty, { - fill: this.modularBuilder.generateId(), - detach: this.modularBuilder.generateId(), - }); + const detachId = this.modularBuilder.generateId(); + let fillId: ChangesetLocalId | undefined; + const edits: EditDescription[] = []; + let optionalChange: OptionalChangeset; + if (newContent !== undefined) { + fillId = this.modularBuilder.generateId(); + const build = this.modularBuilder.buildTrees(fillId, [newContent]); + edits.push(build); + + optionalChange = optional.changeHandler.editor.set(wasEmpty, { + fill: fillId, + detach: detachId, + }); + } else { + optionalChange = optional.changeHandler.editor.clear(wasEmpty, detachId); + } + const change: FieldChangeset = brand(optionalChange); - this.modularBuilder.submitChange(field, optional.identifier, change); + const edit: FieldEditDescription = { + type: "field", + field, + fieldKind: optional.identifier, + change, + }; + edits.push(edit); + + this.modularBuilder.submitChanges( + edits, + newContent === undefined ? detachId : fillId, + ); }, }; } diff --git a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts index cb8999428a0d..c769ab65acb7 100644 --- a/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts +++ b/experimental/dds/tree2/src/feature-libraries/default-schema/defaultFieldKinds.ts @@ -5,7 +5,6 @@ import { FieldKindIdentifier, - ITreeCursor, forbiddenFieldKindIdentifier, ChangesetLocalId, DeltaDetachedNodeId, @@ -53,13 +52,7 @@ export interface ValueFieldEditor extends FieldEditor { * @param changeId - the ID associated with the replacement of the current content. * @param buildId - the ID associated with the creation of the `newContent`. */ - set( - newContent: ITreeCursor, - ids: { - fill: ChangesetLocalId; - detach: ChangesetLocalId; - }, - ): OptionalChangeset; + set(ids: { fill: ChangesetLocalId; detach: ChangesetLocalId }): OptionalChangeset; } const optionalIdentifier = "Optional"; @@ -79,13 +72,8 @@ export const optional = new FieldKindWithEditor( export const valueFieldEditor: ValueFieldEditor = { ...optionalFieldEditor, - set: ( - newContent: ITreeCursor, - ids: { - fill: ChangesetLocalId; - detach: ChangesetLocalId; - }, - ): OptionalChangeset => optionalFieldEditor.set(newContent, false, ids), + set: (ids: { fill: ChangesetLocalId; detach: ChangesetLocalId }): OptionalChangeset => + optionalFieldEditor.set(false, ids), }; export const valueChangeHandler: FieldChangeHandler = { diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts index 05b9a8826862..b662e99b89d3 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalField.ts @@ -5,23 +5,19 @@ import { assert } from "@fluidframework/core-utils"; import { - ITreeCursor, TaggedChange, tagChange, ChangesetLocalId, RevisionTag, areEqualChangeAtomIds, - JsonableTree, RevisionMetadataSource, DeltaFieldChanges, DeltaDetachedNodeRename, DeltaMark, - DeltaDetachedNodeBuild, DeltaDetachedNodeId, DeltaDetachedNodeChanges, } from "../../core"; import { fail, Mutable, IdAllocator, SizedNestedMap } from "../../util"; -import { cursorForJsonableTreeNode, jsonableTreeFromCursor } from "../treeTextCursor"; import { ToDelta, FieldChangeRebaser, @@ -185,8 +181,6 @@ export const optionalChangeRebaser: FieldChangeRebaser = { let latestReservedDetachId: RegisterId | undefined; let inputContext: "empty" | "filled" | undefined; - // Using a map here avoids potential duplicate builds from sandwich rebases. - const builds = new RegisterMap(); const childChangesByOriginalId = new RegisterMap[]>(); // TODO: It might be possible to compose moves in place rather than repeatedly copy. // Additionally, working out a 'register allocation' strategy which enables frequent cancellation of noop moves @@ -202,16 +196,6 @@ export const optionalChangeRebaser: FieldChangeRebaser = { return { revision: intention, localId: id.localId }; }; - for (const { id, set } of change.build) { - builds.set( - { - revision: id.revision ?? revision, - localId: id.localId, - }, - set, - ); - } - const nextSrcToDst = new RegisterMap< [dst: RegisterId, target: "nodeTargeting" | "cellTargeting"] >(); @@ -275,13 +259,7 @@ export const optionalChangeRebaser: FieldChangeRebaser = { composedMoves.push([src, dst, target]); } - const composedBuilds: OptionalChangeset["build"] = []; - for (const [id, set] of builds.entries()) { - assert(id !== "self", "Detached trees should not be built directly to self register"); - composedBuilds.push({ id, set }); - } const composed: OptionalChangeset = { - build: composedBuilds, moves: composedMoves, childChanges: Array.from(childChangesByOriginalId.entries(), ([id, childChanges]) => [ id, @@ -334,7 +312,6 @@ export const optionalChangeRebaser: FieldChangeRebaser = { } } const inverted: OptionalChangeset = { - build: [], moves: invertedMoves, childChanges: childChanges.map(([id, childChange]) => [ withIntention(invertIdMap.get(id) ?? id), @@ -365,7 +342,7 @@ export const optionalChangeRebaser: FieldChangeRebaser = { return { revision: intention, localId: id.localId }; }; - const { moves, childChanges, build } = change; + const { moves, childChanges } = change; const { change: overChange } = overTagged; const rebasedMoves: typeof moves = []; @@ -442,7 +419,6 @@ export const optionalChangeRebaser: FieldChangeRebaser = { } const rebased: OptionalChangeset = { - build, moves: rebasedMoves, childChanges: rebasedChildChanges, }; @@ -455,7 +431,6 @@ export const optionalChangeRebaser: FieldChangeRebaser = { prune: (change: OptionalChangeset, pruneChild: NodeChangePruner): OptionalChangeset => { const childChanges: OptionalChangeset["childChanges"] = []; const prunedChange: OptionalChangeset = { - build: change.build, moves: change.moves, childChanges, }; @@ -483,7 +458,6 @@ export interface OptionalFieldEditor extends FieldEditor { * @param buildId - the ID associated with the creation of the `newContent`. */ set( - newContent: ITreeCursor, wasEmpty: boolean, ids: { fill: ChangesetLocalId; @@ -501,7 +475,6 @@ export interface OptionalFieldEditor extends FieldEditor { export const optionalFieldEditor: OptionalFieldEditor = { set: ( - newContent: ITreeCursor, wasEmpty: boolean, ids: { fill: ChangesetLocalId; @@ -510,7 +483,6 @@ export const optionalFieldEditor: OptionalFieldEditor = { }, ): OptionalChangeset => { const result: OptionalChangeset = { - build: [{ id: { localId: ids.fill }, set: jsonableTreeFromCursor(newContent) }], moves: [[{ localId: ids.fill }, "self", "nodeTargeting"]], childChanges: [], }; @@ -524,9 +496,8 @@ export const optionalFieldEditor: OptionalFieldEditor = { clear: (wasEmpty: boolean, detachId: ChangesetLocalId): OptionalChangeset => wasEmpty - ? { build: [], moves: [], childChanges: [], reservedDetachId: { localId: detachId } } + ? { moves: [], childChanges: [], reservedDetachId: { localId: detachId } } : { - build: [], moves: [["self", { localId: detachId }, "cellTargeting"]], childChanges: [], }, @@ -534,7 +505,6 @@ export const optionalFieldEditor: OptionalFieldEditor = { buildChildChange: (index: number, childChange: NodeChangeset): OptionalChangeset => { assert(index === 0, 0x404 /* Optional fields only support a single child node */); return { - build: [], moves: [], childChanges: [["self", childChange]], }; @@ -547,17 +517,6 @@ export function optionalFieldIntoDelta( ): DeltaFieldChanges { const delta: Mutable = {}; - if (change.build.length > 0) { - const builds: DeltaDetachedNodeBuild[] = []; - for (const build of change.build) { - builds.push({ - id: { major: build.id.revision ?? revision, minor: build.id.localId }, - trees: [cursorForJsonableTreeNode(build.set)], - }); - } - delta.build = builds; - } - let markIsANoop = true; const mark: Mutable = { count: 1 }; @@ -621,7 +580,7 @@ export const optionalChangeHandler: FieldChangeHandler - change.childChanges.length === 0 && change.moves.length === 0 && change.build.length === 0, + change.childChanges.length === 0 && change.moves.length === 0, }; function areEqualRegisterIds(a: RegisterId, b: RegisterId): boolean { diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeFormat.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeFormat.ts index f91589c750ed..236b9280b287 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeFormat.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeFormat.ts @@ -4,7 +4,6 @@ */ import { Static, ObjectOptions, TSchema, Type } from "@sinclair/typebox"; -import { EncodedJsonableTree } from "../../core"; import { EncodedChangeAtomId } from "../modular-schema"; const noAdditionalProps: ObjectOptions = { additionalProperties: false }; @@ -14,7 +13,7 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; export const EncodedRegisterId = Type.Union([EncodedChangeAtomId, Type.Literal(0)]); export type EncodedRegisterId = Static; -export const EncodedBuild = Type.Tuple([EncodedChangeAtomId, EncodedJsonableTree]); +export const EncodedBuild = Type.Tuple([EncodedChangeAtomId]); export type EncodedBuild = Static; export const EncodedOptionalChangeset = (tNodeChange: Schema) => diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeTypes.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeTypes.ts index a94435ea6f6c..4e692c10f357 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeTypes.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldChangeTypes.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. */ -import { ChangeAtomId, JsonableTree } from "../../core"; +import { ChangeAtomId } from "../../core"; import { NodeChangeset } from "../modular-schema"; /** @@ -24,11 +24,6 @@ export type RegisterId = ChangeAtomId | "self"; * The active register holds the current value of the field, and other registers hold detached roots. */ export interface OptionalChangeset { - /** - * Detached trees to build. - */ - build: { set: JsonableTree; id: ChangeAtomId }[]; - /** * Each entry signifies the intent to move a node from `src` to `dst`. * diff --git a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts index e1df17a9a7f9..1167879685f6 100644 --- a/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts +++ b/experimental/dds/tree2/src/feature-libraries/optional-field/optionalFieldCodecs.ts @@ -4,17 +4,12 @@ */ import { TAnySchema, Type } from "@sinclair/typebox"; -import { decodeChangeAtomId, encodeChangeAtomId } from "../utils"; import { ICodecFamily, IJsonCodec, makeCodecFamily, unitCodec } from "../../codec"; import { EncodedRevisionTag, RevisionTag } from "../../core"; import { Mutable } from "../../util"; import type { NodeChangeset } from "../modular-schema"; import type { OptionalChangeset, RegisterId } from "./optionalFieldChangeTypes"; -import { - EncodedOptionalChangeset, - EncodedRegisterId, - EncodedBuild, -} from "./optionalFieldChangeFormat"; +import { EncodedOptionalChangeset, EncodedRegisterId } from "./optionalFieldChangeFormat"; export const noChangeCodecFamily: ICodecFamily<0> = makeCodecFamily([[0, unitCodec]]); @@ -65,14 +60,6 @@ function makeOptionalFieldCodec( return { encode: (change: OptionalChangeset) => { const encoded: EncodedOptionalChangeset = {}; - if (change.build.length > 0) { - const builds: EncodedBuild[] = []; - for (const build of change.build) { - builds.push([encodeChangeAtomId(revisionTagCodec, build.id), build.set]); - } - encoded.b = builds; - } - if (change.moves.length > 0) { encoded.m = []; for (const [src, dst, type] of change.moves) { @@ -109,13 +96,6 @@ function makeOptionalFieldCodec( ] as const, ) ?? []; const decoded: OptionalChangeset = { - build: - encoded.b?.map(([id, set]) => { - return { - id: decodeChangeAtomId(revisionTagCodec, id), - set, - }; - }) ?? [], moves, childChanges: encoded.c?.map(([id, encodedChange]) => [ diff --git a/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldToDelta.ts b/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldToDelta.ts index d1648c4f936c..b9551be0d2dc 100644 --- a/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldToDelta.ts +++ b/experimental/dds/tree2/src/feature-libraries/sequence-field/sequenceFieldToDelta.ts @@ -6,7 +6,6 @@ import { assert, unreachableCase } from "@fluidframework/core-utils"; import { fail, Mutable } from "../../util"; import { - DeltaDetachedNodeBuild, DeltaDetachedNodeChanges, DeltaDetachedNodeRename, DeltaFieldChanges, @@ -37,7 +36,6 @@ export function sequenceFieldToDelta( ): DeltaFieldChanges { const local: DeltaMark[] = []; const global: DeltaDetachedNodeChanges[] = []; - const build: DeltaDetachedNodeBuild[] = []; const rename: DeltaDetachedNodeRename[] = []; for (const mark of change) { @@ -195,9 +193,6 @@ export function sequenceFieldToDelta( if (global.length > 0) { delta.global = global; } - if (build.length > 0) { - delta.build = build; - } if (rename.length > 0) { delta.rename = rename; } diff --git a/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultFieldKinds.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultFieldKinds.spec.ts index 725a001e26e6..c1aff1744393 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultFieldKinds.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/default-field-kinds/defaultFieldKinds.spec.ts @@ -17,7 +17,7 @@ import { brand, fakeIdAllocator } from "../../../util"; import { defaultRevisionMetadataFromChanges } from "../../utils"; // eslint-disable-next-line import/no-internal-modules import { OptionalChangeset } from "../../../feature-libraries/optional-field"; -import { changesetForChild, testTree, testTreeCursor } from "../fieldKindTestUtils"; +import { changesetForChild } from "../fieldKindTestUtils"; // eslint-disable-next-line import/no-internal-modules import { assertEqual } from "../optional-field/optionalFieldUtils"; // eslint-disable-next-line import/no-internal-modules @@ -50,7 +50,6 @@ describe("defaultFieldKinds", () => { describe("valueFieldEditor.set", () => { it("valueFieldEditor.set", () => { const expected: OptionalChangeset = { - build: [{ id: { localId: brand(41) }, set: testTree("tree1") }], moves: [ [{ localId: brand(41) }, "self", "nodeTargeting"], ["self", { localId: brand(1) }, "cellTargeting"], @@ -58,7 +57,7 @@ describe("defaultFieldKinds", () => { childChanges: [], }; assert.deepEqual( - valueFieldEditor.set(testTreeCursor("tree1"), { + valueFieldEditor.set({ detach: brand(1), fill: brand(41), }), @@ -75,32 +74,28 @@ describe("defaultFieldKinds", () => { valueChangeHandler; const childChange1: OptionalChangeset = { - build: [], moves: [], childChanges: [["self", nodeChange1]], }; const childChange2: OptionalChangeset = { - build: [], moves: [], childChanges: [["self", nodeChange2]], }; const childChange3: OptionalChangeset = { - build: [], moves: [], childChanges: [["self", arbitraryChildChange]], }; const change1 = tagChange( - fieldHandler.editor.set(testTreeCursor("tree1"), { detach: brand(1), fill: brand(41) }), + fieldHandler.editor.set({ detach: brand(1), fill: brand(41) }), mintRevisionTag(), ); const change2 = tagChange( - fieldHandler.editor.set(testTreeCursor("tree2"), { detach: brand(2), fill: brand(42) }), + fieldHandler.editor.set({ detach: brand(2), fill: brand(42) }), mintRevisionTag(), ); const change1WithChildChange: OptionalChangeset = { - build: [{ id: { localId: brand(41) }, set: testTree("tree1") }], moves: [ [{ localId: brand(41) }, "self", "nodeTargeting"], ["self", { localId: brand(1) }, "cellTargeting"], @@ -112,10 +107,6 @@ describe("defaultFieldKinds", () => { * Represents the outcome of composing change1 and change2. */ const change1And2: TaggedChange = makeAnonChange({ - build: [ - { id: { localId: brand(41), revision: change1.revision }, set: testTree("tree1") }, - { id: { localId: brand(42), revision: change2.revision }, set: testTree("tree2") }, - ], moves: [ [ { localId: brand(41), revision: change1.revision }, @@ -149,12 +140,6 @@ describe("defaultFieldKinds", () => { it("a field change and a child change", () => { const taggedChildChange1 = tagChange(childChange1, mintRevisionTag()); const expected: OptionalChangeset = { - build: [ - { - id: { localId: brand(41), revision: change1.revision }, - set: testTree("tree1"), - }, - ], moves: [ [ { localId: brand(41), revision: change1.revision }, @@ -190,12 +175,6 @@ describe("defaultFieldKinds", () => { defaultRevisionMetadataFromChanges([change1]), ); const expected2: OptionalChangeset = { - build: [ - { - id: { localId: brand(41), revision: change1.revision }, - set: testTree("tree1"), - }, - ], moves: [ [ { localId: brand(41), revision: change1.revision }, @@ -247,7 +226,6 @@ describe("defaultFieldKinds", () => { assertEqual( makeAnonChange(inverted), makeAnonChange({ - build: [], moves: [ [ { localId: brand(1), revision: taggedChange.revision }, diff --git a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalChangeRebaser.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalChangeRebaser.spec.ts index f40e174785c1..d4dee682c33e 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalChangeRebaser.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalChangeRebaser.spec.ts @@ -4,11 +4,7 @@ */ import { strict as assert } from "assert"; -import { - CrossFieldManager, - NodeChangeset, - cursorForJsonableTreeNode, -} from "../../../feature-libraries"; +import { CrossFieldManager, NodeChangeset } from "../../../feature-libraries"; import { ChangesetLocalId, DeltaFieldChanges, @@ -75,7 +71,7 @@ const OptionalChange = { detach: ChangesetLocalId; }, ) { - return optionalFieldEditor.set(cursorForJsonableTreeNode({ type, value }), wasEmpty, ids); + return optionalFieldEditor.set(wasEmpty, ids); }, clear(wasEmpty: boolean, id: ChangesetLocalId) { @@ -107,10 +103,6 @@ function getMaxId(...changes: OptionalChangeset[]): ChangesetLocalId | undefined }; for (const change of changes) { - for (const build of change.build ?? []) { - ingest(build.id.localId); - } - for (const [src, dst] of change.moves) { if (src !== "self") { ingest(src.localId); diff --git a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts index d2879ad42af6..6f61f0c24f03 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalField.spec.ts @@ -34,7 +34,7 @@ import { defaultRevInfosFromChanges, defaultRevisionMetadataFromChanges, } from "../../utils"; -import { changesetForChild, fooKey, testTree, testTreeCursor } from "../fieldKindTestUtils"; +import { changesetForChild, fooKey, testTreeCursor } from "../fieldKindTestUtils"; // eslint-disable-next-line import/no-internal-modules import { rebaseRevisionMetadataFromInfo } from "../../../feature-libraries/modular-schema/modularChangeFamily"; import { assertEqual } from "./optionalFieldUtils"; @@ -96,7 +96,6 @@ const deltaFromChild2 = ({ change, revision }: TaggedChange): Del const tag = mintRevisionTag(); const change1: TaggedChange = tagChange( { - build: [{ id: { localId: brand(41) }, set: testTree("tree1") }], moves: [[{ localId: brand(41) }, "self", "nodeTargeting"]], childChanges: [[{ localId: brand(41) }, nodeChange1]], reservedDetachId: { localId: brand(1) }, @@ -105,7 +104,7 @@ const change1: TaggedChange = tagChange( ); const change2: TaggedChange = tagChange( - optionalFieldEditor.set(testTreeCursor("tree2"), false, { fill: brand(42), detach: brand(2) }), + optionalFieldEditor.set(false, { fill: brand(42), detach: brand(2) }), mintRevisionTag(), ); @@ -125,7 +124,7 @@ const revertChange2: TaggedChange = tagChange( * Represents what change2 would have been had it been concurrent with change1. */ const change2PreChange1: TaggedChange = tagChange( - optionalFieldEditor.set(testTreeCursor("tree2"), true, { fill: brand(42), detach: brand(2) }), + optionalFieldEditor.set(true, { fill: brand(42), detach: brand(2) }), change2.revision, ); @@ -139,12 +138,11 @@ describe("optionalField", () => { // TODO: more editor tests describe("editor", () => { it("can be created", () => { - const actual: OptionalChangeset = optionalFieldEditor.set(testTreeCursor("x"), true, { + const actual: OptionalChangeset = optionalFieldEditor.set(true, { fill: brand(42), detach: brand(43), }); const expected: OptionalChangeset = { - build: [{ id: { localId: brand(42) }, set: testTree("x") }], moves: [[{ localId: brand(42) }, "self", "nodeTargeting"]], childChanges: [], reservedDetachId: { localId: brand(43) }, @@ -168,16 +166,6 @@ describe("optionalField", () => { ); const change1And2: OptionalChangeset = { - build: [ - { - id: { localId: brand(41), revision: change1.revision }, - set: testTree("tree1"), - }, - { - id: { localId: brand(42), revision: change2.revision }, - set: testTree("tree2"), - }, - ], moves: [ [ { localId: brand(41), revision: change1.revision }, @@ -195,12 +183,6 @@ describe("optionalField", () => { it("can compose child changes", () => { const expected: OptionalChangeset = { - build: [ - { - id: { localId: brand(41), revision: change1.revision }, - set: testTree("tree1"), - }, - ], moves: [ [{ localId: brand(41), revision: change1.revision }, "self", "nodeTargeting"], ], @@ -235,7 +217,6 @@ describe("optionalField", () => { }; const expected: OptionalChangeset = { - build: [], moves: [ ["self", { localId: brand(41), revision: change1.revision }, "cellTargeting"], ], @@ -277,12 +258,10 @@ describe("optionalField", () => { it("can rebase child change", () => { const baseChange: OptionalChangeset = { - build: [], moves: [], childChanges: [["self", nodeChange1]], }; const changeToRebase: OptionalChangeset = { - build: [], moves: [], childChanges: [["self", nodeChange2]], }; @@ -297,7 +276,6 @@ describe("optionalField", () => { }; const expected: OptionalChangeset = { - build: [], moves: [], childChanges: [["self", arbitraryChildChange]], }; @@ -368,7 +346,6 @@ describe("optionalField", () => { it("can rebase child change (field change ↷ field change)", () => { const baseChange: OptionalChangeset = { - build: [], moves: [["self", { localId: brand(0) }, "cellTargeting"]], childChanges: [["self", nodeChange1]], }; @@ -377,9 +354,6 @@ describe("optionalField", () => { // Note: this sort of change (has field changes as well as nested child changes) // can only be created for production codepaths using transactions. const changeToRebase: OptionalChangeset = { - build: [ - { id: { localId: brand(41) }, set: { type: brand("value"), value: "X" } }, - ], moves: [ [{ localId: brand(41) }, "self", "nodeTargeting"], ["self", { localId: brand(1) }, "cellTargeting"], @@ -397,9 +371,6 @@ describe("optionalField", () => { }; const expected: OptionalChangeset = { - build: [ - { id: { localId: brand(41) }, set: { type: brand("value"), value: "X" } }, - ], moves: [[{ localId: brand(41) }, "self", "nodeTargeting"]], childChanges: [ [ @@ -430,7 +401,6 @@ describe("optionalField", () => { const outerNodeId = makeDetachedNodeId(tag, 41); const innerNodeId = makeDetachedNodeId(tag, 1); const expected: DeltaFieldChanges = { - build: [{ id: outerNodeId, trees: [testTreeCursor("tree1")] }], global: [ { id: outerNodeId, @@ -521,7 +491,7 @@ describe("optionalField", () => { describe("relevantRemovedRoots", () => { const fill = tagChange( - optionalFieldEditor.set(testTreeCursor(""), true, { detach: brand(1), fill: brand(2) }), + optionalFieldEditor.set(true, { detach: brand(1), fill: brand(2) }), mintRevisionTag(), ); const clear = tagChange(optionalFieldEditor.clear(false, brand(1)), mintRevisionTag()); @@ -563,7 +533,7 @@ describe("optionalField", () => { it("a tree that remains untouched", () => { const actual = Array.from( optionalChangeHandler.relevantRemovedRoots( - makeAnonChange({ build: [], moves: [], childChanges: [] }), + makeAnonChange({ moves: [], childChanges: [] }), noTreesDelegate, ), ); @@ -718,7 +688,6 @@ describe("optionalField", () => { const restore = tagChange( { moves: [[{ localId: brand(42) }, "self", "nodeTargeting"]], - build: [], childChanges: [], }, tag, diff --git a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts index 040d17d72385..5a706fee8f37 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldChangeCodecs.spec.ts @@ -14,7 +14,7 @@ import { // eslint-disable-next-line import/no-internal-modules } from "../../../feature-libraries/optional-field"; import { IJsonCodec } from "../../../codec"; -import { changesetForChild, testTree, testTreeCursor } from "../fieldKindTestUtils"; +import { changesetForChild } from "../fieldKindTestUtils"; import { RevisionTagCodec } from "../../../shared-tree-core"; const nodeChange1 = changesetForChild("nodeChange1"); @@ -33,13 +33,12 @@ const childCodec1: IJsonCodec = { }; const change1: OptionalChangeset = { - build: [{ id: { localId: brand(41) }, set: testTree("tree1") }], moves: [[{ localId: brand(41) }, "self", "nodeTargeting"]], childChanges: [], reservedDetachId: { localId: brand(1) }, }; -const change2: OptionalChangeset = optionalFieldEditor.set(testTreeCursor("tree2"), false, { +const change2: OptionalChangeset = optionalFieldEditor.set(false, { fill: brand(42), detach: brand(2), }); @@ -50,13 +49,11 @@ const change2Inverted: OptionalChangeset = { ["self", { localId: brand(42) }, "cellTargeting"], ], childChanges: [], - build: [], }; const changeWithChildChange = optionalFieldEditor.buildChildChange(0, nodeChange1); const change1WithChildChange: OptionalChangeset = { - build: [{ id: { localId: brand(41) }, set: testTree("tree1") }], moves: [ [{ localId: brand(41) }, "self", "nodeTargeting"], ["self", { localId: brand(1) }, "cellTargeting"], diff --git a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldUtils.ts b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldUtils.ts index 7b9a66bcfef3..1dae3a355011 100644 --- a/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldUtils.ts +++ b/experimental/dds/tree2/src/test/feature-libraries/optional-field/optionalFieldUtils.ts @@ -30,8 +30,6 @@ export function assertEqual( const bCopy = { ...b, change: { ...b.change, moves: [...b.change.moves] } }; aCopy.change.moves.sort(([c], [d]) => compareRegisterIds(c, d)); bCopy.change.moves.sort(([c], [d]) => compareRegisterIds(c, d)); - aCopy.change.build.sort((c, d) => compareRegisterIds(c.id, d.id)); - bCopy.change.build.sort((c, d) => compareRegisterIds(c.id, d.id)); assert.equal( aCopy.change.reservedDetachId !== undefined, diff --git a/experimental/dds/tree2/src/test/snapshots/files/has-handle-final.json b/experimental/dds/tree2/src/test/snapshots/files/has-handle-final.json index cd762b8c5c18..a770cbd4747d 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/has-handle-final.json +++ b/experimental/dds/tree2/src/test/snapshots/files/has-handle-final.json @@ -17,6 +17,7 @@ "trunk": [ { "change": { + "maxId": 0, "changes": [ { "fieldKey": "rootFieldKey", @@ -35,25 +36,12 @@ }, { "change": { + "maxId": 1, "changes": [ { "fieldKey": "rootFieldKey", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 1 - }, - { - "type": "com.fluidframework.leaf.handle", - "value": { - "type": "__fluid_handle__", - "url": "/test/test" - } - } - ] - ], "m": [ [ { @@ -64,10 +52,42 @@ ] ], "d": { - "localId": 2 + "localId": 0 } } } + ], + "builds": [ + [ + 1, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.handle", + true, + { + "type": "__fluid_handle__", + "url": "/test/test" + }, + [] + ] + ] + } + ] ] }, "revision": "beefbeef-beef-4000-8000-000000000004", diff --git a/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json b/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json index 6b4564f2d14f..5867f4b49f9c 100644 --- a/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json +++ b/experimental/dds/tree2/src/test/snapshots/files/optional-field-scenarios-final.json @@ -17,6 +17,7 @@ "trunk": [ { "change": { + "maxId": 1, "changes": [ { "fieldKey": "rootFieldKey", @@ -30,17 +31,6 @@ "fieldKey": "root 1 child", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 1 - }, - { - "type": "com.fluidframework.leaf.number", - "value": 40 - } - ] - ], "m": [ [ { @@ -51,7 +41,7 @@ ] ], "d": { - "localId": 2 + "localId": 0 } } } @@ -60,6 +50,35 @@ } ] } + ], + "builds": [ + [ + 1, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.number", + true, + 40, + [] + ] + ] + } + ] ] }, "revision": "beefbeef-beef-4000-8000-000000000006", @@ -68,33 +87,16 @@ }, { "change": { + "maxId": 3, "changes": [ { "fieldKey": "rootFieldKey", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 4 - }, - { - "type": "optional-field.TestNode", - "fields": { - "root 2 child": [ - { - "type": "com.fluidframework.leaf.number", - "value": 41 - } - ] - } - } - ] - ], "m": [ [ { - "localId": 4 + "localId": 3 }, 0, true @@ -102,13 +104,49 @@ [ 0, { - "localId": 5 + "localId": 2 }, false ] ] } } + ], + "builds": [ + [ + 3, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "optional-field.TestNode", + false, + [ + "root 2 child", + [ + "com.fluidframework.leaf.number", + true, + 41, + [] + ] + ] + ] + ] + } + ] ] }, "revision": "beefbeef-beef-4000-8000-000000000007", @@ -117,25 +155,16 @@ }, { "change": { + "maxId": 5, "changes": [ { "fieldKey": "rootFieldKey", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 4 - }, - { - "type": "optional-field.TestNode" - } - ] - ], "m": [ [ { - "localId": 4 + "localId": 3 }, 0, true @@ -143,7 +172,7 @@ [ 0, { - "localId": 5 + "localId": 2 }, false ] @@ -151,7 +180,7 @@ "c": [ [ { - "localId": 5, + "localId": 2, "revision": "beefbeef-beef-4000-8000-000000000007" }, { @@ -160,17 +189,6 @@ "fieldKey": "root 1 child", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 1 - }, - { - "type": "com.fluidframework.leaf.number", - "value": 42 - } - ] - ], "m": [ [ { @@ -182,7 +200,7 @@ [ 0, { - "localId": 2 + "localId": 0 }, false ] @@ -194,7 +212,7 @@ ], [ { - "localId": 4 + "localId": 3 }, { "fieldChanges": [ @@ -202,28 +220,17 @@ "fieldKey": "root 3 child", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 7 - }, - { - "type": "com.fluidframework.leaf.number", - "value": 43 - } - ] - ], "m": [ [ { - "localId": 7 + "localId": 5 }, 0, true ] ], "d": { - "localId": 8 + "localId": 4 } } } @@ -233,6 +240,88 @@ ] } } + ], + "builds": [ + [ + 1, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.number", + true, + 42, + [] + ] + ] + } + ], + [ + 3, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "optional-field.TestNode", + false, + [] + ] + ] + } + ], + [ + 5, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.number", + true, + 43, + [] + ] + ] + } + ] ] }, "revision": "beefbeef-beef-4000-8000-00000000000b", @@ -241,6 +330,7 @@ }, { "change": { + "maxId": 5, "changes": [ { "fieldKey": "rootFieldKey", @@ -255,17 +345,6 @@ "fieldKey": "root 3 child", "fieldKind": "Optional", "change": { - "b": [ - [ - { - "localId": 1 - }, - { - "type": "com.fluidframework.leaf.number", - "value": 44 - } - ] - ], "m": [ [ { @@ -277,7 +356,7 @@ [ 0, { - "localId": 2 + "localId": 0 }, false ] @@ -290,6 +369,35 @@ ] } } + ], + "builds": [ + [ + 1, + { + "version": 1, + "identifiers": [], + "shapes": [ + { + "c": { + "extraFields": 1, + "fields": [] + } + }, + { + "a": 0 + } + ], + "data": [ + 1, + [ + "com.fluidframework.leaf.number", + true, + 44, + [] + ] + ] + } + ] ] }, "revision": "beefbeef-beef-4000-8000-00000000000c", @@ -448,7 +556,7 @@ } ], [ - "repair-7", + "repair-8", { "version": 1, "identifiers": [], @@ -466,15 +574,16 @@ "data": [ 1, [ - "optional-field.TestNode", - false, + "com.fluidframework.leaf.number", + true, + 42, [] ] ] } ], [ - "repair-8", + "repair-9", { "version": 1, "identifiers": [], @@ -492,16 +601,15 @@ "data": [ 1, [ - "com.fluidframework.leaf.number", - true, - 43, + "optional-field.TestNode", + false, [] ] ] } ], [ - "repair-9", + "repair-10", { "version": 1, "identifiers": [], @@ -521,7 +629,7 @@ [ "com.fluidframework.leaf.number", true, - 44, + 43, [] ] ] @@ -548,7 +656,7 @@ [ "com.fluidframework.leaf.number", true, - 42, + 44, [] ] ] @@ -669,42 +777,42 @@ "data": [ [ "beefbeef-beef-4000-8000-00000000000b", - 4, - 7 + 1, + 8 ], [ "beefbeef-beef-4000-8000-00000000000b", - 7, - 8 + 3, + 9 ], [ "beefbeef-beef-4000-8000-00000000000b", - 1, - 11 + 5, + 10 ], [ "beefbeef-beef-4000-8000-00000000000b", - 2, + 0, 12 ], [ "beefbeef-beef-4000-8000-00000000000b", - 5, + 2, 15 ], [ "beefbeef-beef-4000-8000-00000000000c", - 2, + 0, 6 ], [ "beefbeef-beef-4000-8000-00000000000c", 1, - 9 + 11 ], [ "beefbeef-beef-4000-8000-000000000007", - 5, + 2, 14 ] ], From b4d4d03a642e5bdb8f9827c80a6c150e480137a3 Mon Sep 17 00:00:00 2001 From: Joshua Smithrud <54606601+Josmithr@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:57:07 -0800 Subject: [PATCH 20/20] test(tree2): Add a (currently skipped) test verifying support of unhydrated `initialTree` input (#18707) Disabled test illustrates a currently unsupported input case that we will want to address in the future. --- .../tree2/src/test/shared-tree/sharedTree.spec.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/experimental/dds/tree2/src/test/shared-tree/sharedTree.spec.ts b/experimental/dds/tree2/src/test/shared-tree/sharedTree.spec.ts index eab5b52f29b4..ddfa5a82249b 100644 --- a/experimental/dds/tree2/src/test/shared-tree/sharedTree.spec.ts +++ b/experimental/dds/tree2/src/test/shared-tree/sharedTree.spec.ts @@ -69,6 +69,7 @@ import { typeboxValidator } from "../../external-utilities"; import { EditManager } from "../../shared-tree-core"; import { leaf, SchemaBuilder } from "../../domains"; import { noopValidator } from "../../codec"; +import { SchemaFactory, TreeConfiguration } from "../../class-tree"; const schemaCodec = makeSchemaCodec({ jsonValidator: typeboxValidator }); @@ -158,6 +159,17 @@ describe("SharedTree", () => { // Initial tree should not be applied assert.equal(schematized.editableTree.content, undefined); }); + + // TODO: ensure unhydrated initialTree input is correctly hydrated. + it.skip("unhydrated tree input", () => { + const tree = factory.create(new MockFluidDataStoreRuntime(), "the tree") as SharedTree; + const sb = new SchemaFactory("test-factory"); + class Foo extends sb.object("Foo", {}) {} + + const unhydratedInitialTree = new Foo({}); + const view = tree.schematize(new TreeConfiguration(Foo, () => unhydratedInitialTree)); + assert(view.root === unhydratedInitialTree); + }); }); describe("requireSchema", () => {