From 453be085fc6858c78e532603d68b76a6abf576e9 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 21 Oct 2024 12:58:12 +0700 Subject: [PATCH] Fix backup check tool (#6997) Signed-off-by: Andrey Sobolev --- common/config/rush/pnpm-lock.yaml | 30 ++++++++++++------------- server/backup-service/package.json | 4 ++-- server/backup/package.json | 4 ++-- server/backup/src/backup.ts | 35 ++++++++++++++++-------------- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index f07ee37bdb..35d45d4d99 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1332,8 +1332,8 @@ dependencies: specifier: ~0.32.0 version: 0.32.0 '@types/tar-stream': - specifier: ^2.2.2 - version: 2.2.3 + specifier: ^3.1.3 + version: 3.1.3 '@types/toposort': specifier: ^2.0.3 version: 2.0.7 @@ -1617,7 +1617,7 @@ dependencies: specifier: ^9.7.1 version: 9.13.0 itty-router: - specifier: ^5.0.17 + specifier: ^5.0.18 version: 5.0.18 jest: specifier: ^29.7.0 @@ -1839,8 +1839,8 @@ dependencies: specifier: ^3.0.0 version: 3.0.3 tar-stream: - specifier: ^2.2.0 - version: 2.2.0 + specifier: ^3.1.7 + version: 3.1.7 telegraf: specifier: ^4.16.3 version: 4.16.3 @@ -9833,8 +9833,8 @@ packages: resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==} dev: false - /@types/tar-stream@2.2.3: - resolution: {integrity: sha512-if3mugZfjVkXOMZdFjIHySxY13r6GXPpyOlsDmLffvyI7tLz9wXE8BFjNivXsvUeyJ1KNlOpfLnag+ISmxgxPw==} + /@types/tar-stream@3.1.3: + resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==} dependencies: '@types/node': 20.11.19 dev: false @@ -26087,14 +26087,14 @@ packages: dev: false file:projects/backup-service.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-yVHrvyvHvtRFu894lGDl0o19qrJv3scuxI80Dg+rQDzjb4nWXAPGAj477N6TR7vrZSmYAmsR5qEaHkXxc35ipg==, tarball: file:projects/backup-service.tgz} + resolution: {integrity: sha512-q/p/O9tTXiKT+qik2nsPL+zmBUwyvvfwBIH3RN4Z7vItg7gV/mI38jal1ppQU/Ddnwvfifn/h7tFc/sSTpjG2A==, tarball: file:projects/backup-service.tgz} id: file:projects/backup-service.tgz name: '@rush-temp/backup-service' version: 0.0.0 dependencies: '@types/jest': 29.5.12 '@types/node': 20.11.19 - '@types/tar-stream': 2.2.3 + '@types/tar-stream': 3.1.3 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) eslint: 8.56.0 @@ -26104,7 +26104,7 @@ packages: eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) prettier: 3.2.5 - tar-stream: 2.2.0 + tar-stream: 3.1.7 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 transitivePeerDependencies: @@ -26623,7 +26623,7 @@ packages: dev: false file:projects/cloud-branding.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-LuePDPK46tYQhJ9inFPje2LSd/RqlwYSW0k0UDsNWKNyJn4WwWGngaNblMp22YGBw8wMDgJpbaIjS/BPvZ7RQg==, tarball: file:projects/cloud-branding.tgz} + resolution: {integrity: sha512-f1NgjqZw48X0+O4cblNzjU8SeW4RYnvaMvYNFyN+eKbqTDHI1N97+q5f2OoDgRonvaZa+CXMETNFXMSkniWwyA==, tarball: file:projects/cloud-branding.tgz} id: file:projects/cloud-branding.tgz name: '@rush-temp/cloud-branding' version: 0.0.0 @@ -26658,7 +26658,7 @@ packages: dev: false file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-AA2lTsmPKPeYA1MTwIscZFRO40m9Ctc59Er2x8VRLNBBt4mQ01b1CCay4VFVPWYxAzh+Ru9RoUIB7lS+m8sj9Q==, tarball: file:projects/cloud-datalake.tgz} + resolution: {integrity: sha512-KdKIEaVTjeWtCEUYfkvyrIYxglGcfM8iICOnTxRzPiVmOIP3kswu7PaUKeH/xeFZQcvRBvhtNI7XT0dBr5mODA==, tarball: file:projects/cloud-datalake.tgz} id: file:projects/cloud-datalake.tgz name: '@rush-temp/cloud-datalake' version: 0.0.0 @@ -32916,14 +32916,14 @@ packages: dev: false file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-RCYMutijYbZbV1jzpWwwzQL9bZS8uzTOYX0eZgxmh3aYyU1Ub4v4TixQpdiiH72ujj0V2hPn9bM2hpYaxzbD1w==, tarball: file:projects/server-backup.tgz} + resolution: {integrity: sha512-oDyT5+30r1kpzb+Cx/HTsY1f4fRjjtSy7n13JjzYPmkn37nVcHYb1X1RImHxhoiPehYWdftTsE5v5l2on0mOPg==, tarball: file:projects/server-backup.tgz} id: file:projects/server-backup.tgz name: '@rush-temp/server-backup' version: 0.0.0 dependencies: '@types/jest': 29.5.12 '@types/node': 20.11.19 - '@types/tar-stream': 2.2.3 + '@types/tar-stream': 3.1.3 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) eslint: 8.56.0 @@ -32934,7 +32934,7 @@ packages: jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) prettier: 3.2.5 prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) - tar-stream: 2.2.0 + tar-stream: 3.1.7 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 transitivePeerDependencies: diff --git a/server/backup-service/package.json b/server/backup-service/package.json index d048040661..554529dda6 100644 --- a/server/backup-service/package.json +++ b/server/backup-service/package.json @@ -33,7 +33,7 @@ "eslint-config-standard-with-typescript": "^40.0.0", "prettier": "^3.1.0", "typescript": "^5.3.3", - "@types/tar-stream": "^2.2.2", + "@types/tar-stream": "^3.1.3", "@types/node": "~20.11.16", "jest": "^29.7.0", "ts-jest": "^29.1.1", @@ -46,7 +46,7 @@ "@hcengineering/client-resources": "^0.6.27", "@hcengineering/client": "^0.6.18", "@hcengineering/model": "^0.6.11", - "tar-stream": "^2.2.0", + "tar-stream": "^3.1.7", "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-storage": "^0.6.0", diff --git a/server/backup/package.json b/server/backup/package.json index 808422a622..e5339813e6 100644 --- a/server/backup/package.json +++ b/server/backup/package.json @@ -33,7 +33,7 @@ "eslint-config-standard-with-typescript": "^40.0.0", "prettier": "^3.1.0", "typescript": "^5.3.3", - "@types/tar-stream": "^2.2.2", + "@types/tar-stream": "^3.1.3", "@types/node": "~20.11.16", "jest": "^29.7.0", "ts-jest": "^29.1.1", @@ -47,7 +47,7 @@ "@hcengineering/client": "^0.6.18", "@hcengineering/model": "^0.6.11", "@hcengineering/analytics": "^0.6.0", - "tar-stream": "^2.2.0", + "tar-stream": "^3.1.7", "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-client": "^0.6.0", "@hcengineering/server-token": "^0.6.11", diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 1aef026c39..c1e4ac133e 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -45,7 +45,7 @@ import { type StorageAdapter } from '@hcengineering/server-core' import { fullTextPushStagePrefix } from '@hcengineering/server-indexer' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' -import { createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs' +import { createReadStream, createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs' import { rm } from 'node:fs/promises' import { basename, dirname } from 'node:path' import { PassThrough } from 'node:stream' @@ -178,7 +178,7 @@ async function loadDigest ( result.delete(k as Ref) } } catch (err: any) { - ctx.error('digest is broken, will do full backup for', { domain }) + ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot }) } } // Stop if stop date is matched and provided @@ -236,14 +236,10 @@ async function verifyDigest ( blobs.set(bname, { doc, buffer: undefined }) } else { blobs.delete(bname) - const blob = doc as Blob - - if (blob.size === bf.length) { - validDocs.add(name as Ref) - } + validDocs.add(bname as Ref) } } else { - validDocs.add(name as Ref) + validDocs.add(bname as Ref) } next() }) @@ -265,10 +261,7 @@ async function verifyDigest ( sz = bf.length } - // If blob size matches doc size, remove from requiredDocs - if (sz === bf.length) { - validDocs.add(name as Ref) - } + validDocs.add(name as Ref) } next() }) @@ -364,7 +357,7 @@ async function verifyDigest ( } } catch (err: any) { digestToRemove.add(snapshot) - ctx.error('digest is broken, will do full backup for', { domain }) + ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot }) modified = true } } @@ -1490,6 +1483,7 @@ export async function backupSize (storage: BackupStorage): Promise { */ export async function backupDownload (storage: BackupStorage, storeIn: string): Promise { const infoFile = 'backup.json.gz' + const sizeFile = 'backup.size.gz' if (!(await storage.exists(infoFile))) { throw new Error(`${infoFile} should present to restore`) @@ -1499,6 +1493,12 @@ export async function backupDownload (storage: BackupStorage, storeIn: string): const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString()) console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + let sizeInfo: Record = {} + if (await storage.exists(sizeFile)) { + sizeInfo = JSON.parse(gunzipSync(await storage.loadFile(sizeFile)).toString()) + } + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise => { if (file != null) { const target = join(storeIn, file) @@ -1506,8 +1506,11 @@ export async function backupDownload (storage: BackupStorage, storeIn: string): if (!existsSync(dir)) { mkdirSync(dir, { recursive: true }) } - if (!existsSync(target) || force) { - const fileSize = await storage.stat(file) + + const serverSize: number | undefined = sizeInfo[file] + + if (!existsSync(target) || force || (serverSize !== undefined && serverSize !== statSync(target).size)) { + const fileSize = serverSize ?? (await storage.stat(file)) console.log('downloading', file, fileSize) const readStream = await storage.load(file) const outp = createWriteStream(target) @@ -1781,7 +1784,7 @@ export async function restore ( if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) { totalSend += docs.length - ctx.info('upload', { + ctx.info('upload-' + c, { docs: docs.length, totalSend, from: docsToAdd.size + totalSend,