Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <[email protected]>

dev/tool/package.json

@@ -52,8 +52,7 @@
     "@types/request": "~2.48.8",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
-    "@types/jest": "^29.5.5",
-    "@types/pg": "^8.11.6"
+    "@types/jest": "^29.5.5"
   },
   "dependencies": {
     "@elastic/elasticsearch": "^7.14.0",
@@ -157,7 +156,7 @@
     "libphonenumber-js": "^1.9.46",
     "mime-types": "~2.1.34",
     "mongodb": "6.9.0-dev.20241016.sha.3d5bd513",
-    "pg": "8.12.0",
+    "postgres": "^3.4.4",
     "ws": "^8.18.0"
   }
 }

dev/tool/src/db.ts

@@ -26,11 +26,12 @@ import {
   retryTxn,
   translateDomain
 } from '@hcengineering/postgres'
+import { type DBDoc } from '@hcengineering/postgres/types/utils'
 import { getTransactorEndpoint } from '@hcengineering/server-client'
 import { generateToken } from '@hcengineering/server-token'
 import { connect } from '@hcengineering/server-tool'
 import { type MongoClient } from 'mongodb'
-import { type Pool } from 'pg'
+import type postgres from 'postgres'
 
 export async function moveFromMongoToPG (
   accountDb: AccountDB,
@@ -64,7 +65,7 @@ export async function moveFromMongoToPG (
 async function moveWorkspace (
   accountDb: AccountDB,
   mongo: MongoClient,
-  pgClient: Pool,
+  pgClient: postgres.Sql,
   ws: Workspace,
   region: string
 ): Promise<void> {
@@ -84,17 +85,16 @@ async function moveWorkspace (
     for (const collection of collections) {
       const cursor = collection.find()
       const domain = translateDomain(collection.collectionName)
-      const current = await pgClient.query(`SELECT _id FROM ${domain} WHERE "workspaceId" = $1`, [ws.workspace])
-      const currentIds = new Set(current.rows.map((r) => r._id))
+      const current = await pgClient`SELECT _id FROM ${pgClient(domain)} WHERE "workspaceId" = ${ws.workspace}`
+      const currentIds = new Set(current.map((r) => r._id))
       console.log('move domain', domain)
       const docs: Doc[] = []
       const fields = getDocFieldsByDomains(domain)
       const filedsWithData = [...fields, 'data']
-      const insertFields: string[] = []
+      const insertFields: string[] = ['workspaceId']
       for (const field of filedsWithData) {
-        insertFields.push(`"${field}"`)
+        insertFields.push(field)
       }
-      const insertStr = insertFields.join(', ')
       while (true) {
         while (docs.length < 50000) {
           const doc = (await cursor.next()) as Doc | null
@@ -105,35 +105,15 @@ async function moveWorkspace (
         if (docs.length === 0) break
         while (docs.length > 0) {
           const part = docs.splice(0, 500)
-          const values: any[] = []
-          const vars: string[] = []
-          let index = 1
+          const values: DBDoc[] = []
           for (let i = 0; i < part.length; i++) {
             const doc = part[i]
-            const variables: string[] = []
             const d = convertDoc(domain, doc, ws.workspace)
-            values.push(d.workspaceId)
-            variables.push(`$${index++}`)
-            for (const field of fields) {
-              values.push(d[field])
-              variables.push(`$${index++}`)
-            }
-            values.push(d.data)
-            variables.push(`$${index++}`)
-            vars.push(`(${variables.join(', ')})`)
-          }
-          const vals = vars.join(',')
-          try {
-            await retryTxn(pgClient, async (client) => {
-              await client.query(
-                `INSERT INTO ${translateDomain(domain)} ("workspaceId", ${insertStr}) VALUES ${vals}`,
-                values
-              )
-            })
-          } catch (err) {
-            console.log('error when move doc to', domain, err)
-            continue
+            values.push(d)
           }
+          await retryTxn(pgClient, async (client) => {
+            await client`INSERT INTO ${client(translateDomain(domain))} ${client(values, insertFields)}`
+          })
         }
       }
     }

models/controlled-documents/src/migration.ts

@@ -294,7 +294,7 @@ async function migrateDocSections (client: MigrationClient): Promise<void> {
     try {
       const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content)
       if (ydoc === undefined) {
-        ctx.error('collaborative document content not found', { document: document.title })
+        // no content, ignore
         continue
       }
 

packages/core/src/__tests__/memdb.test.ts

@@ -29,7 +29,7 @@ import {
   SearchResult
 } from '../storage'
 import { Tx } from '../tx'
-import { genMinModel, test, TestMixin } from './minmodel'
+import { createDoc, deleteDoc, genMinModel, test, TestMixin, updateDoc } from './minmodel'
 
 const txes = genMinModel()
 
@@ -59,17 +59,17 @@ class ClientModel extends ModelDb implements Client {
   async close (): Promise<void> {}
 }
 
-async function createModel (): Promise<{ model: ClientModel, hierarchy: Hierarchy, txDb: TxDb }> {
+async function createModel (modelTxes: Tx[] = txes): Promise<{ model: ClientModel, hierarchy: Hierarchy, txDb: TxDb }> {
   const hierarchy = new Hierarchy()
-  for (const tx of txes) {
+  for (const tx of modelTxes) {
     hierarchy.tx(tx)
   }
   const model = new ClientModel(hierarchy)
-  for (const tx of txes) {
+  for (const tx of modelTxes) {
     await model.tx(tx)
   }
   const txDb = new TxDb(hierarchy)
-  for (const tx of txes) await txDb.tx(tx)
+  for (const tx of modelTxes) await txDb.tx(tx)
   return { model, hierarchy, txDb }
 }
 
@@ -78,7 +78,7 @@ describe('memdb', () => {
     const { txDb } = await createModel()
 
     const result = await txDb.findAll(core.class.Tx, {})
-    expect(result.length).toBe(txes.filter((tx) => tx._class === core.class.TxCreateDoc).length)
+    expect(result.length).toBe(txes.length)
   })
 
   it('should create space', async () => {
@@ -396,4 +396,41 @@ describe('memdb', () => {
       expect(e).toEqual(new Error('createDoc cannot be used for objects inherited from AttachedDoc'))
     }
   })
+
+  it('has correct accounts', async () => {
+    const modTxes = [...txes]
+
+    modTxes.push(
+      createDoc(core.class.Account, {
+        email: 'system_admin',
+        role: AccountRole.Owner
+      })
+    )
+
+    const system1Account = createDoc(core.class.Account, {
+      email: 'system1',
+      role: AccountRole.Maintainer
+    })
+    modTxes.push(system1Account)
+
+    const user1Account = createDoc(core.class.Account, {
+      email: 'user1',
+      role: AccountRole.User
+    })
+    modTxes.push(user1Account)
+
+    modTxes.push(updateDoc(core.class.Account, core.space.Model, system1Account.objectId, { email: 'user1' }))
+
+    modTxes.push(deleteDoc(core.class.Account, core.space.Model, user1Account.objectId))
+
+    const { model } = await createModel(modTxes)
+
+    expect(model.getAccountByEmail('system_admin')).not.toBeUndefined()
+    expect(model.getAccountByEmail('system_admin')?.role).toBe(AccountRole.Owner)
+
+    expect(model.getAccountByEmail('system1')).toBeUndefined()
+
+    expect(model.getAccountByEmail('user1')).not.toBeUndefined()
+    expect(model.getAccountByEmail('user1')?.role).toBe(AccountRole.Maintainer)
+  })
 })

packages/core/src/__tests__/minmodel.ts

@@ -15,10 +15,10 @@
 
 import type { IntlString, Plugin } from '@hcengineering/platform'
 import { plugin } from '@hcengineering/platform'
-import type { Arr, Class, Data, Doc, Interface, Mixin, Obj, Ref } from '../classes'
+import type { Arr, Class, Data, Doc, Interface, Mixin, Obj, Ref, Space } from '../classes'
 import { AttachedDoc, ClassifierKind, DOMAIN_MODEL } from '../classes'
 import core from '../component'
-import type { TxCUD, TxCreateDoc } from '../tx'
+import type { DocumentUpdate, TxCUD, TxCreateDoc, TxRemoveDoc, TxUpdateDoc } from '../tx'
 import { DOMAIN_TX, TxFactory } from '../tx'
 
 const txFactory = new TxFactory(core.account.System)
@@ -31,10 +31,23 @@ function createInterface (_interface: Ref<Interface<Doc>>, attributes: Data<Inte
   return txFactory.createTxCreateDoc(core.class.Interface, core.space.Model, attributes, _interface)
 }
 
-export function createDoc<T extends Doc> (_class: Ref<Class<T>>, attributes: Data<T>): TxCreateDoc<Doc> {
+export function createDoc<T extends Doc> (_class: Ref<Class<T>>, attributes: Data<T>): TxCreateDoc<T> {
   return txFactory.createTxCreateDoc(_class, core.space.Model, attributes)
 }
 
+export function updateDoc<T extends Doc> (
+  _class: Ref<Class<T>>,
+  space: Ref<Space>,
+  objectId: Ref<T>,
+  operations: DocumentUpdate<T>
+): TxUpdateDoc<Doc> {
+  return txFactory.createTxUpdateDoc(_class, space, objectId, operations)
+}
+
+export function deleteDoc<T extends Doc> (_class: Ref<Class<T>>, space: Ref<Space>, objectId: Ref<T>): TxRemoveDoc<Doc> {
+  return txFactory.createTxRemoveDoc(_class, space, objectId)
+}
+
 export interface TestMixin extends Doc {
   arr: Arr<string>
 }

packages/core/src/client.ts

@@ -441,6 +441,8 @@ async function buildModel (
     )
   })
 
+  userTx.sort(compareTxes)
+
   let txes = systemTx.concat(userTx)
   if (modelFilter !== undefined) {
     txes = await modelFilter(txes)
@@ -473,3 +475,8 @@ function getLastTxTime (txes: Tx[]): number {
   }
   return lastTxTime
 }
+
+function compareTxes (a: Tx, b: Tx): number {
+  const result = a._id.localeCompare(b._id)
+  return result !== 0 ? result : a.modifiedOn - b.modifiedOn
+}