fix: another attempt to migrate empty document fields

Signed-off-by: Alexander Onnikov <[email protected]>
hcengineering · Oct 21, 2024 · c5d26fd · c5d26fd
1 parent 7bd0db4
commit c5d26fd
Show file tree

Hide file tree

Showing 10 changed files with 65 additions and 56 deletions.
diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts
@@ -1270,7 +1270,7 @@ async function updateYDoc (
   doc: RelatedDocument
 ): Promise<void> {
   try {
-    const ydoc = await loadCollaborativeDoc(storage, workspaceId, _id, ctx)
+    const ydoc = await loadCollaborativeDoc(ctx, storage, workspaceId, _id)
     if (ydoc === undefined) {
       ctx.error('document content not found', { document: contentDoc._id })
       return
@@ -1284,7 +1284,7 @@ async function updateYDoc (
     })
 
     if (updatedYDoc !== undefined) {
-      await saveCollaborativeDoc(storage, workspaceId, _id, updatedYDoc, ctx)
+      await saveCollaborativeDoc(ctx, storage, workspaceId, _id, updatedYDoc)
     }
   } catch {
     // do nothing, the collaborative doc does not sem to exist yet

diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts
@@ -199,7 +199,7 @@ async function processMigrateMarkupFor (
           if (blob === undefined) {
             try {
               const ydoc = markupToYDoc(value, attribute.name)
-              await saveCollaborativeDoc(storageAdapter, workspaceId, collaborativeDoc, ydoc, ctx)
+              await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, collaborativeDoc, ydoc)
             } catch (err) {
               console.error('failed to process document', doc._class, doc._id, err)
             }
@@ -298,7 +298,7 @@ export async function restoreLostMarkup (
             console.log(doc._class, doc._id, attr.name, markup)
             if (command === 'restore') {
               const ydoc = markupToYDoc(markup, attr.name)
-              await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx)
+              await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc)
             }
             restored = true
             break
@@ -329,7 +329,7 @@ export async function restoreLostMarkup (
                 console.log(doc._class, doc._id, attr.name, markup)
                 if (command === 'restore') {
                   const ydoc = markupToYDoc(markup, attr.name)
-                  await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx)
+                  await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc)
                 }
               }
             }

diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts
@@ -292,7 +292,7 @@ async function migrateDocSections (client: MigrationClient): Promise<void> {
 
     // Migrate sections headers + content
     try {
-      const ydoc = await loadCollaborativeDoc(storage, client.workspaceId, document.content, ctx)
+      const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content)
       if (ydoc === undefined) {
         ctx.error('collaborative document content not found', { document: document.title })
         continue
@@ -334,7 +334,7 @@ async function migrateDocSections (client: MigrationClient): Promise<void> {
         }
       })
 
-      await saveCollaborativeDoc(storage, client.workspaceId, document.content, ydoc, ctx)
+      await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.content, ydoc)
     } catch (err) {
       ctx.error('error collaborative document content migration', { error: err, document: document.title })
     }

diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts
@@ -229,7 +229,7 @@ async function processMigrateContentFor (
             if (blob === undefined) {
               try {
                 const ydoc = markupToYDoc(value, attribute.name)
-                await saveCollaborativeDoc(storageAdapter, client.workspaceId, collaborativeDoc, ydoc, ctx)
+                await saveCollaborativeDoc(ctx, storageAdapter, client.workspaceId, collaborativeDoc, ydoc)
               } catch (err) {
                 console.error('failed to process document', doc._class, doc._id, err)
               }

diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts
@@ -100,36 +100,6 @@ async function migrateTeamspacesMixins (client: MigrationClient): Promise<void>
   )
 }
 
-async function migrateContentField (client: MigrationClient): Promise<void> {
-  const ctx = new MeasureMetricsContext('migrate_content_field', {})
-  const storage = client.storageAdapter
-
-  const documents = await client.find<Document>(DOMAIN_DOCUMENT, {
-    _class: document.class.Document,
-    content: { $exists: true }
-  })
-
-  for (const document of documents) {
-    try {
-      const ydoc = await loadCollaborativeDoc(storage, client.workspaceId, document.content, ctx)
-      if (ydoc === undefined) {
-        ctx.error('document content not found', { document: document.title })
-        continue
-      }
-
-      if (!ydoc.share.has('') || ydoc.share.has('content')) {
-        continue
-      }
-
-      yDocCopyXmlField(ydoc, '', 'content')
-
-      await saveCollaborativeDoc(storage, client.workspaceId, document.content, ydoc, ctx)
-    } catch (err) {
-      ctx.error('error document content migration', { error: err, document: document.title })
-    }
-  }
-}
-
 async function migrateRank (client: MigrationClient): Promise<void> {
   const documents = await client.find<Document>(
     DOMAIN_DOCUMENT,
@@ -228,7 +198,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise<void> {
 
     if (document.description.includes('%description:')) {
       try {
-        const ydoc = await loadCollaborativeDoc(storage, client.workspaceId, document.description, ctx)
+        const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.description)
         if (ydoc === undefined) {
           continue
         }
@@ -239,7 +209,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise<void> {
 
         yDocCopyXmlField(ydoc, 'description', 'content')
 
-        await saveCollaborativeDoc(storage, client.workspaceId, document.description, ydoc, ctx)
+        await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.description, ydoc)
       } catch (err) {
         ctx.error('error document content migration', { error: err, document: document.title })
       }
@@ -264,6 +234,42 @@ async function renameFieldsRevert (client: MigrationClient): Promise<void> {
   }
 }
 
+async function restoreContentField (client: MigrationClient): Promise<void> {
+  const ctx = new MeasureMetricsContext('restoreContentField', {})
+  const storage = client.storageAdapter
+
+  const documents = await client.find<Document>(DOMAIN_DOCUMENT, {
+    _class: document.class.Document,
+    content: { $exists: true }
+  })
+
+  for (const document of documents) {
+    try {
+      const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content)
+      if (ydoc === undefined) {
+        ctx.error('document content not found', { document: document.title })
+        continue
+      }
+
+      // ignore if content is already present
+      if (ydoc.share.has('content') || ydoc.share.has('description')) {
+        continue
+      }
+
+      if (ydoc.share.has('')) {
+        yDocCopyXmlField(ydoc, '', 'content')
+        if (ydoc.share.has('content')) {
+          await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.content, ydoc)
+        } else {
+          ctx.error('document content still not found', { document: document.title })
+        }
+      }
+    } catch (err) {
+      ctx.error('error document content migration', { error: err, document: document.title })
+    }
+  }
+}
+
 export const documentOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     await tryMigrate(client, documentId, [
@@ -279,10 +285,6 @@ export const documentOperation: MigrateOperation = {
         state: 'migrate-teamspaces-mixins',
         func: migrateTeamspacesMixins
       },
-      {
-        state: 'migrateContentField',
-        func: migrateContentField
-      },
       {
         state: 'migrateRank',
         func: migrateRank
@@ -300,6 +302,10 @@ export const documentOperation: MigrateOperation = {
       {
         state: 'renameFieldsRevert',
         func: renameFieldsRevert
+      },
+      {
+        state: 'restoreContentField',
+        func: restoreContentField
       }
     ])
   },

diff --git a/server-plugins/activity-resources/src/references.ts b/server-plugins/activity-resources/src/references.ts
@@ -415,7 +415,7 @@ async function getCreateReferencesTxes (
     } else if (attr.type._class === core.class.TypeCollaborativeDoc) {
       const collaborativeDoc = (createdDoc as any)[attr.name] as CollaborativeDoc
       try {
-        const ydoc = await loadCollaborativeDoc(storage, control.workspace, collaborativeDoc, control.ctx)
+        const ydoc = await loadCollaborativeDoc(ctx, storage, control.workspace, collaborativeDoc)
         if (ydoc !== undefined) {
           const attrReferences = getReferencesData(
             srcDocId,
@@ -467,7 +467,7 @@ async function getUpdateReferencesTxes (
       hasReferenceAttrs = true
       try {
         const collaborativeDoc = (updatedDoc as any)[attr.name] as CollaborativeDoc
-        const ydoc = await loadCollaborativeDoc(storage, control.workspace, collaborativeDoc, control.ctx)
+        const ydoc = await loadCollaborativeDoc(ctx, storage, control.workspace, collaborativeDoc)
         if (ydoc !== undefined) {
           const attrReferences = getReferencesData(
             srcDocId,

diff --git a/server/collaboration/src/utils/__tests__/ydoc.test.ts b/server/collaboration/src/utils/__tests__/ydoc.test.ts
@@ -43,10 +43,12 @@ describe('ydoc', () => {
 
       const source = ydoc.getXmlFragment('source')
       source.insertAfter(null, [new YXmlElement('p'), new YXmlText('foo'), new YXmlElement('p')])
+      expect(ydoc.share.has('target')).toBeFalsy()
 
       yDocCopyXmlField(ydoc, 'source', 'target')
       const target = ydoc.getXmlFragment('target')
 
+      expect(ydoc.share.has('target')).toBeTruthy()
       expect(target.toJSON()).toEqual(source.toJSON())
     })
 
@@ -61,6 +63,7 @@ describe('ydoc', () => {
       expect(target.toJSON()).not.toEqual(source.toJSON())
 
       yDocCopyXmlField(ydoc, 'source', 'target')
+      expect(ydoc.share.has('target')).toBeTruthy()
       expect(target.toJSON()).toEqual(source.toJSON())
     })
   })

diff --git a/server/collaboration/src/utils/collaborative-doc.ts b/server/collaboration/src/utils/collaborative-doc.ts
@@ -78,10 +78,10 @@ async function loadCollaborativeDocVersion (
 
 /** @public */
 export async function loadCollaborativeDoc (
+  ctx: MeasureContext,
   storageAdapter: StorageAdapter,
   workspace: WorkspaceId,
-  collaborativeDoc: CollaborativeDoc,
-  ctx: MeasureContext
+  collaborativeDoc: CollaborativeDoc
 ): Promise<YDoc | undefined> {
   const sources = collaborativeDocUnchain(collaborativeDoc)
 
@@ -101,24 +101,24 @@ export async function loadCollaborativeDoc (
 
 /** @public */
 export async function saveCollaborativeDoc (
+  ctx: MeasureContext,
   storageAdapter: StorageAdapter,
   workspace: WorkspaceId,
   collaborativeDoc: CollaborativeDoc,
-  ydoc: YDoc,
-  ctx: MeasureContext
+  ydoc: YDoc
 ): Promise<void> {
   const { documentId, versionId } = collaborativeDocParse(collaborativeDoc)
-  await saveCollaborativeDocVersion(storageAdapter, workspace, documentId, versionId, ydoc, ctx)
+  await saveCollaborativeDocVersion(ctx, storageAdapter, workspace, documentId, versionId, ydoc)
 }
 
 /** @public */
 export async function saveCollaborativeDocVersion (
+  ctx: MeasureContext,
   storageAdapter: StorageAdapter,
   workspace: WorkspaceId,
   documentId: string,
   versionId: CollaborativeDocVersion,
-  ydoc: YDoc,
-  ctx: MeasureContext
+  ydoc: YDoc
 ): Promise<void> {
   await ctx.with('saveCollaborativeDoc', {}, async (ctx) => {
     if (versionId === 'HEAD') {

diff --git a/server/collaborator/src/storage/platform.ts b/server/collaborator/src/storage/platform.ts
@@ -124,7 +124,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
 
     return await ctx.with('load-document', {}, async (ctx) => {
       return await withRetry(ctx, 5, async () => {
-        return await loadCollaborativeDoc(this.storage, context.workspaceId, collaborativeDoc, ctx)
+        return await loadCollaborativeDoc(ctx, this.storage, context.workspaceId, collaborativeDoc)
       })
     })
   }
@@ -139,7 +139,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
 
     await ctx.with('save-document', {}, async (ctx) => {
       await withRetry(ctx, 5, async () => {
-        await saveCollaborativeDoc(this.storage, context.workspaceId, collaborativeDoc, document, ctx)
+        await saveCollaborativeDoc(ctx, this.storage, context.workspaceId, collaborativeDoc, document)
       })
     })
   }

diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts
@@ -272,7 +272,7 @@ export class WorkspaceInitializer {
     const json = parseMessageMarkdown(data ?? '', this.imageUrl)
     const yDoc = jsonToYDocNoSchema(json, field)
 
-    await saveCollaborativeDoc(this.storageAdapter, this.wsUrl, collabId, yDoc, this.ctx)
+    await saveCollaborativeDoc(this.ctx, this.storageAdapter, this.wsUrl, collabId, yDoc)
     return collabId
   }