Fix cockroach (#6965)

Signed-off-by: Denis Bykhov <[email protected]>
hcengineering · Oct 23, 2024 · 5eeb828 · 5eeb828
1 parent 498b390
commit 5eeb828
Show file tree

Hide file tree

Showing 88 changed files with 445 additions and 312 deletions.
diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts
@@ -18,7 +18,14 @@ import {
   type MeasureMetricsContext
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
-import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres'
+import {
+  convertDoc,
+  createTable,
+  getDBClient,
+  getDocFieldsByDomains,
+  retryTxn,
+  translateDomain
+} from '@hcengineering/postgres'
 import { getTransactorEndpoint } from '@hcengineering/server-client'
 import { generateToken } from '@hcengineering/server-token'
 import { connect } from '@hcengineering/server-tool'
@@ -54,10 +61,6 @@ export async function moveFromMongoToPG (
   client.close()
 }
 
-function escapeBackticks (str: string): string {
-  return str.replaceAll("'", "''")
-}
-
 async function moveWorkspace (
   accountDb: AccountDB,
   mongo: MongoClient,
@@ -85,6 +88,13 @@ async function moveWorkspace (
       const currentIds = new Set(current.rows.map((r) => r._id))
       console.log('move domain', domain)
       const docs: Doc[] = []
+      const fields = getDocFieldsByDomains(domain)
+      const filedsWithData = [...fields, 'data']
+      const insertFields: string[] = []
+      for (const field of filedsWithData) {
+        insertFields.push(`"${field}"`)
+      }
+      const insertStr = insertFields.join(', ')
       while (true) {
         while (docs.length < 50000) {
           const doc = (await cursor.next()) as Doc | null
@@ -95,18 +105,29 @@ async function moveWorkspace (
         if (docs.length === 0) break
         while (docs.length > 0) {
           const part = docs.splice(0, 500)
-          const vals = part
-            .map((doc) => {
-              const d = convertDoc(doc, ws.workspace)
-              return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
-                d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
-              }, '${escapeBackticks(JSON.stringify(d.data))}')`
-            })
-            .join(', ')
+          const values: any[] = []
+          const vars: string[] = []
+          let index = 1
+          for (let i = 0; i < part.length; i++) {
+            const doc = part[i]
+            const variables: string[] = []
+            const d = convertDoc(domain, doc, ws.workspace)
+            values.push(d.workspaceId)
+            variables.push(`$${index++}`)
+            for (const field of fields) {
+              values.push(d[field])
+              variables.push(`$${index++}`)
+            }
+            values.push(d.data)
+            variables.push(`$${index++}`)
+            vars.push(`(${variables.join(', ')})`)
+          }
+          const vals = vars.join(',')
           try {
             await retryTxn(pgClient, async (client) => {
               await client.query(
-                `INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}`
+                `INSERT INTO ${translateDomain(domain)} ("workspaceId", ${insertStr}) VALUES ${vals}`,
+                values
               )
             })
           } catch (err) {

diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts
@@ -371,7 +371,7 @@ export function devTool (
           lastProcessingTime: Date.now() + 1000 * 60
         })
 
-        await createWorkspace(measureCtx, version, brandingObj, wsInfo, txes, migrateOperations)
+        await createWorkspace(measureCtx, version, brandingObj, wsInfo, txes, migrateOperations, undefined, true)
 
         await updateWorkspace(db, wsInfo, {
           mode: 'active',
@@ -1717,7 +1717,7 @@ export function devTool (
           lastProcessingTime: Date.now() + 1000 * 60
         })
 
-        await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations)
+        await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations, undefined, true)
 
         await updateWorkspace(db, wsInfo, {
           mode: 'active',

diff --git a/packages/core/src/__tests__/memdb.test.ts b/packages/core/src/__tests__/memdb.test.ts
@@ -133,18 +133,6 @@ describe('memdb', () => {
     })
     const objClass = (await model.findAll(core.class.Class, { _id: core.class.Obj }))[0] as any
     expect(objClass['test:mixin:TestMixin'].arr).toEqual(expect.arrayContaining(['hello']))
-
-    await ops.updateDoc(test.mixin.TestMixin, core.space.Model, core.class.Obj as unknown as Ref<TestMixin>, {
-      $pushMixin: {
-        $mixin: test.mixin.TestMixin,
-        values: {
-          arr: 'there'
-        }
-      }
-    })
-
-    const objClass2 = (await model.findAll(core.class.Class, { _id: core.class.Obj }))[0] as any
-    expect(objClass2['test:mixin:TestMixin'].arr).toEqual(expect.arrayContaining(['hello', 'there']))
   })
 
   it('should allow delete', async () => {

diff --git a/packages/core/src/operator.ts b/packages/core/src/operator.ts
@@ -116,37 +116,6 @@ function $update (document: Doc, keyval: Record<string, PropertyType>): void {
   }
 }
 
-function $move (document: Doc, keyval: Record<string, PropertyType>): void {
-  const doc = document as any
-  for (const key in keyval) {
-    if (doc[key] === undefined) {
-      doc[key] = []
-    }
-    const arr = doc[key] as Array<any>
-    const desc = keyval[key]
-    doc[key] = (arr ?? []).filter((val) => val !== desc.$value)
-    doc[key].splice(desc.$position, 0, desc.$value)
-  }
-}
-
-function $pushMixin (document: Doc, options: any): void {
-  const doc = document as any
-  const mixinId = options.$mixin
-  if (mixinId === undefined) {
-    throw new Error('$mixin must be specified for $push_mixin operation')
-  }
-  const mixin = doc[mixinId]
-  const keyval = options.values
-  for (const key in keyval) {
-    const arr = mixin[key]
-    if (arr == null) {
-      mixin[key] = [keyval[key]]
-    } else {
-      arr.push(keyval[key])
-    }
-  }
-}
-
 function $inc (document: Doc, keyval: Record<string, number>): void {
   const doc = document as unknown as Record<string, number | undefined>
   for (const key in keyval) {
@@ -180,8 +149,6 @@ const operators: Record<string, _OperatorFunc> = {
   $push,
   $pull,
   $update,
-  $move,
-  $pushMixin,
   $inc,
   $unset,
   $rename

diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts
@@ -247,7 +247,6 @@ export type OmitNever<T extends object> = Omit<T, KeysByType<T, never>>
 export interface PushOptions<T extends object> {
   $push?: Partial<OmitNever<ArrayAsElementPosition<Required<T>>>>
   $pull?: Partial<OmitNever<ArrayAsElement<Required<T>>>>
-  $move?: Partial<OmitNever<ArrayMoveDescriptor<Required<T>>>>
 }
 
 /**
@@ -269,16 +268,6 @@ export interface SetEmbeddedOptions<T extends object> {
   $update?: Partial<OmitNever<ArrayAsElementUpdate<Required<T>>>>
 }
 
-/**
- * @public
- */
-export interface PushMixinOptions<D extends Doc> {
-  $pushMixin?: {
-    $mixin: Ref<Mixin<D>>
-    values: Partial<OmitNever<ArrayAsElement<D>>>
-  }
-}
-
 /**
  * @public
  */
@@ -299,7 +288,6 @@ export interface SpaceUpdate {
 export type DocumentUpdate<T extends Doc> = Partial<Data<T>> &
 PushOptions<T> &
 SetEmbeddedOptions<T> &
-PushMixinOptions<T> &
 IncOptions<T> &
 SpaceUpdate
 

diff --git a/packages/model/src/migration.ts b/packages/model/src/migration.ts
@@ -29,7 +29,7 @@ import { ModelLogger } from './utils'
  * @public
  */
 export type MigrateUpdate<T extends Doc> = Partial<T> &
-Omit<PushOptions<T>, '$move'> &
+PushOptions<T> &
 IncOptions<T> &
 UnsetOptions &
 Record<string, any>

diff --git a/server-plugins/contact-resources/src/index.ts b/server-plugins/contact-resources/src/index.ts
@@ -117,7 +117,7 @@ async function createPersonSpace (
   person: Ref<Person>,
   control: TriggerControl
 ): Promise<TxCUD<PersonSpace>[]> {
-  const personSpace = (await control.findAll(control.ctx, contact.class.PersonSpace, { person }, { limit: 1 })).shift()
+  const personSpace = (await control.findAll(control.ctx, contact.class.PersonSpace, { person }, { limit: 1 }))[0]
   if (personSpace !== undefined) {
     const toAdd = account.filter((it) => !personSpace.members.includes(it))
     if (toAdd.length === 0) return []

diff --git a/server-plugins/notification-resources/src/index.ts b/server-plugins/notification-resources/src/index.ts
@@ -1717,7 +1717,7 @@ async function updateCollaborators (
 
   if (hierarchy.classHierarchyMixin(objectClass, activity.mixin.ActivityDoc) === undefined) return res
 
-  const contexts = await control.findAll(control.ctx, notification.class.DocNotifyContext, { attachedTo: objectId })
+  const contexts = await control.findAll(control.ctx, notification.class.DocNotifyContext, { objectId })
   const addedInfo = await getUsersInfo(ctx, toAdd as Ref<PersonAccount>[], control)
 
   for (const addedUser of addedInfo.values()) {

diff --git a/server/backup/src/storage.ts b/server/backup/src/storage.ts
@@ -118,6 +118,7 @@ class AdapterStorage implements BackupStorage {
  */
 export async function createFileBackupStorage (fileName: string): Promise<BackupStorage> {
   if (!existsSync(fileName)) {
+    console.log(__dirname)
     await mkdir(fileName, { recursive: true })
   }
   return new FileStorage(fileName)

diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts
@@ -1407,23 +1407,6 @@ class MongoAdapter extends MongoAdapterBase {
       modifiedOn: tx.modifiedOn
     }
     if (isOperator(tx.attributes)) {
-      const operator = Object.keys(tx.attributes)[0]
-      if (operator === '$move') {
-        const keyval = (tx.attributes as any).$move
-        const arr = tx.mixin + '.' + Object.keys(keyval)[0]
-        const desc = keyval[arr]
-        const ops: any = [
-          { updateOne: { filter, update: { $pull: { [arr]: desc.$value } } } },
-          {
-            updateOne: {
-              filter,
-              update: { $set: modifyOp, $push: { [arr]: { $each: [desc.$value], $position: desc.$position } } }
-            }
-          }
-        ]
-        bulk.bulkOperations.push(...ops)
-        return
-      }
       const update = { ...this.translateMixinAttrs(tx.mixin, tx.attributes), $set: { ...modifyOp } }
 
       bulk.bulkOperations.push({
@@ -1475,46 +1458,7 @@ class MongoAdapter extends MongoAdapterBase {
   protected txUpdateDoc (bulk: OperationBulk, tx: TxUpdateDoc<Doc>): void {
     if (isOperator(tx.operations)) {
       const operator = Object.keys(tx.operations)[0]
-      if (operator === '$move') {
-        const keyval = (tx.operations as any).$move
-        const arr = Object.keys(keyval)[0]
-        const desc = keyval[arr]
-
-        const ops: any = [
-          {
-            updateOne: {
-              filter: { _id: tx.objectId },
-              update: {
-                $set: {
-                  '%hash%': null
-                },
-                $pull: {
-                  [arr]: desc.$value
-                }
-              }
-            }
-          },
-          {
-            updateOne: {
-              filter: { _id: tx.objectId },
-              update: {
-                $set: {
-                  modifiedBy: tx.modifiedBy,
-                  modifiedOn: tx.modifiedOn,
-                  '%hash%': null
-                },
-                $push: {
-                  [arr]: {
-                    $each: [desc.$value],
-                    $position: desc.$position
-                  }
-                }
-              }
-            }
-          }
-        ]
-        bulk.bulkOperations.push(...ops)
-      } else if (operator === '$update') {
+      if (operator === '$update') {
         const keyval = (tx.operations as any).$update
         const arr = Object.keys(keyval)[0]
         const desc = keyval[arr] as QueryUpdate<any>

diff --git a/server/postgres/src/index.ts b/server/postgres/src/index.ts
@@ -14,4 +14,4 @@
 //
 
 export * from './storage'
-export { getDBClient, convertDoc, createTable, retryTxn, translateDomain } from './utils'
+export { getDBClient, convertDoc, createTable, retryTxn, translateDomain, getDocFieldsByDomains } from './utils'
diff --git a/server/postgres/src/schemas.ts b/server/postgres/src/schemas.ts
@@ -0,0 +1,36 @@
+import { DOMAIN_SPACE } from '@hcengineering/core'
+
+type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
+
+type Schema = Record<string, [DataType, boolean]>
+
+export const defaultSchema: Schema = {
+  _id: ['text', true],
+  _class: ['text', true],
+  space: ['text', true],
+  modifiedBy: ['text', true],
+  createdBy: ['text', false],
+  modifiedOn: ['bigint', true],
+  createdOn: ['bigint', false],
+  attachedTo: ['text', false]
+}
+
+export const spaceSchema: Schema = {
+  _id: ['text', true],
+  _class: ['text', true],
+  space: ['text', true],
+  modifiedBy: ['text', true],
+  createdBy: ['text', false],
+  modifiedOn: ['bigint', true],
+  createdOn: ['bigint', false],
+  private: ['bool', true],
+  members: ['text[]', true]
+}
+
+export const domainSchemas: Record<string, Schema> = {
+  [DOMAIN_SPACE]: spaceSchema
+}
+
+export function getSchema (domain: string): Schema {
+  return domainSchemas[domain] ?? defaultSchema
+}