From c6fea036d72022bb54cc94734ad36b282b8b14a0 Mon Sep 17 00:00:00 2001 From: Alain Date: Mon, 18 Oct 2021 17:19:51 +0800 Subject: [PATCH] =?UTF-8?q?[KODO-13155]=20=E6=B7=BB=E5=8A=A0=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=AB=AF=E6=96=87=E4=BB=B6=E7=AD=BE=E5=90=8D=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=20(#532)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加服务端文件校验 * 补充 config test * 优化 README * 添加一些注释、说明、以及部分实现调整 * 添加 0 size 的文件测试 * 提升兼容性、优化 README * 更新注释文档 * 更新版本号 * update --- README.md | 3 +- package-lock.json | 4 +- package.json | 2 +- src/api/index.ts | 7 +++- src/upload/base.ts | 2 + src/upload/direct.ts | 7 ++++ src/upload/resume.ts | 1 + src/utils/config.test.ts | 6 +++ src/utils/config.ts | 1 + src/utils/crc32.test.ts | 25 +++++++++++ src/utils/crc32.ts | 90 ++++++++++++++++++++++++++++++++++++++++ test/demo1/main.js | 3 ++ 12 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 src/utils/crc32.test.ts create mode 100644 src/utils/crc32.ts diff --git a/README.md b/README.md index 17fdc9cb..2d1ac12b 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,8 @@ qiniu.compressImage(file, options).then(data => { * config.region: 选择上传域名区域；当为 `null` 或 `undefined` 时，自动分析上传域名区域，当指定了 `uphost` 时，此设置项无效。 * config.retryCount: 上传自动重试次数（整体重试次数，而不是某个分片的重试次数）；默认 3 次（即上传失败后最多重试两次）。 * config.concurrentRequestLimit: 分片上传的并发请求量，`number`，默认为3；因为浏览器本身也会限制最大并发量，所以最大并发量与浏览器有关。 - * config.checkByMD5: 是否开启 MD5 校验，为布尔值；在断点续传时，开启 MD5 校验会将已上传的分片与当前分片进行 MD5 值比对，若不一致，则重传该分片，避免使用错误的分片。读取分片内容并计算 MD5 需要花费一定的时间，因此会稍微增加断点续传时的耗时，默认为 false，不开启。 + * config.checkByServer: 是否开启服务端文件签名校验，为布尔值；开启后在文件上传时会计算本地的文件签名，服务端会根据本地的签名与接收到的数据的签名进行比对，如果不相同、则说明文件可能存在问题，此时会返回错误（`code`: 406），默认为 `false`，不开启。 + * config.checkByMD5: 是否开启 `MD5` 校验，为布尔值；在断点续传时，开启 `MD5` 校验会将已上传的分片与当前分片进行 `MD5` 值比对，若不一致，则重传该分片，避免使用错误的分片。读取分片内容并计算 `MD5` 需要花费一定的时间，因此会稍微增加断点续传时的耗时，默认为 `false`，不开启。 * config.forceDirect: 是否上传全部采用直传方式，为布尔值；为 `true` 时则上传方式全部为直传 form 方式，禁用断点续传，默认 `false`。 * config.chunkSize: `number`，分片上传时每片的大小，必须为正整数，单位为 `MB`，且最大不能超过 1024，默认值 4。因为 chunk 数最大 10000，所以如果文件以你所设的 `chunkSize` 进行分片并且 chunk 数超过 10000，我们会把你所设的 `chunkSize` 扩大二倍，如果仍不符合则继续扩大，直到符合条件。 * config.debugLogLevel: `INFO` | `WARN` | `ERROR` | `OFF`，允许程序在控制台输出日志，默认为 `OFF`，不输出任何日志，本功能仅仅用于本地调试，不建议在线上环境开启。 diff --git a/package-lock.json b/package-lock.json index 9cceb950..ca89e875 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "qiniu-js", - "version": "3.3.3", + "version": "3.4.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "qiniu-js", - "version": "3.3.3", + "version": "3.4.0", "license": "MIT", "dependencies": { "@babel/runtime-corejs2": "^7.10.2", diff --git a/package.json b/package.json index 73c9e1f0..ddc9ca89 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "qiniu-js", "jsName": "qiniu", - "version": "3.3.3", + "version": "3.4.0", "private": false, "description": "Javascript SDK for Qiniu Resource (Cloud) Storage AP", "main": "lib/index.js", diff --git a/src/api/index.ts b/src/api/index.ts index 4b27fd53..c13ea6ca 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -76,14 +76,17 @@ export function uploadChunk( key: string | null | undefined, index: number, uploadInfo: UploadInfo, - options: Partial + options: Partial ): utils.Response { const bucket = utils.getPutPolicy(token).bucketName const url = getBaseUrl(bucket, key, uploadInfo) + `/${index}` + const headers = utils.getHeadersForChunkUpload(token) + if (options.md5) headers['Content-MD5'] = options.md5 + return utils.request(url, { ...options, method: 'PUT', - headers: utils.getHeadersForChunkUpload(token) + headers }) } diff --git a/src/upload/base.ts b/src/upload/base.ts index cc453007..527a8296 100644 --- a/src/upload/base.ts +++ b/src/upload/base.ts @@ -26,6 +26,8 @@ export interface Extra { export interface InternalConfig { /** 是否开启 cdn 加速 */ useCdnDomain: boolean + /** 是否开启服务端校验 */ + checkByServer: boolean /** 是否对分片进行 md5校验 */ checkByMD5: boolean /** 强制直传 */ diff --git a/src/upload/direct.ts b/src/upload/direct.ts index feecd7d8..ddd5a549 100644 --- a/src/upload/direct.ts +++ b/src/upload/direct.ts @@ -1,3 +1,5 @@ +import { CRC32 } from '../utils/crc32' + import { direct } from '../api' import Base from './base' @@ -15,6 +17,11 @@ export default class Direct extends Base { } formData.append('fname', this.putExtra.fname) + if (this.config.checkByServer) { + const crcSign = await CRC32.file(this.file) + formData.append('crc32', crcSign.toString()) + } + if (this.putExtra.customVars) { this.logger.info('init customVars.') const { customVars } = this.putExtra diff --git a/src/upload/resume.ts b/src/upload/resume.ts index 4cd776d6..ae8f271e 100644 --- a/src/upload/resume.ts +++ b/src/upload/resume.ts @@ -153,6 +153,7 @@ export default class Resume extends Base { const requestOptions = { body: chunk, + md5: this.config.checkByServer ? md5 : undefined, onProgress, onCreate: (xhr: XMLHttpRequest) => this.addXhr(xhr) } diff --git a/src/utils/config.test.ts b/src/utils/config.test.ts index 2eeacfca..4c39f76e 100644 --- a/src/utils/config.test.ts +++ b/src/utils/config.test.ts @@ -9,6 +9,7 @@ describe('test config ', () => { uphost: [], retryCount: 3, checkByMD5: false, + checkByServer: false, forceDirect: false, useCdnDomain: true, concurrentRequestLimit: 3, @@ -23,6 +24,7 @@ describe('test config ', () => { uphost: [], retryCount: 3, checkByMD5: false, + checkByServer: false, forceDirect: false, useCdnDomain: true, concurrentRequestLimit: 3, @@ -38,6 +40,7 @@ describe('test config ', () => { uphost: regionUphostMap[region.z0].cdnUphost, retryCount: 3, checkByMD5: false, + checkByServer: false, forceDirect: false, useCdnDomain: true, concurrentRequestLimit: 3, @@ -52,6 +55,7 @@ describe('test config ', () => { uphost: ['test'], retryCount: 3, checkByMD5: false, + checkByServer: false, forceDirect: false, useCdnDomain: true, concurrentRequestLimit: 3, @@ -67,6 +71,7 @@ describe('test config ', () => { uphost: ['test'], retryCount: 3, checkByMD5: false, + checkByServer: false, forceDirect: false, useCdnDomain: true, concurrentRequestLimit: 3, @@ -82,6 +87,7 @@ describe('test config ', () => { uphost: regionUphostMap[region.z0].srcUphost, retryCount: 3, checkByMD5: false, + checkByServer: false, forceDirect: false, useCdnDomain: false, concurrentRequestLimit: 3, diff --git a/src/utils/config.ts b/src/utils/config.ts index eea07282..0a7d6989 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -12,6 +12,7 @@ export function normalizeUploadConfig(config?: Partial, logger?: Logger) checkByMD5: false, forceDirect: false, useCdnDomain: true, + checkByServer: false, concurrentRequestLimit: 3, chunkSize: DEFAULT_CHUNK_SIZE, diff --git a/src/utils/crc32.test.ts b/src/utils/crc32.test.ts new file mode 100644 index 00000000..034a0360 --- /dev/null +++ b/src/utils/crc32.test.ts @@ -0,0 +1,25 @@ +import { CRC32 } from './crc32' +import { MB } from './helper' + +function mockFile(size = 4, name = 'mock.jpg', type = 'image/jpg'): File { + if (size >= 1024) throw new Error('the size is set too large.') + + const blob = new Blob(['1'.repeat(size * MB)], { type }) + return new File([blob], name) +} + +describe('test crc32', async () => { + test('file', async () => { + const crc32One = new CRC32() + await expect(crc32One.file(mockFile(0))).resolves.toEqual(0) + + const crc32Two = new CRC32() + await expect(crc32Two.file(mockFile(0.5))).resolves.toEqual(1610895105) + + const crc32Three = new CRC32() + await expect(crc32Three.file(mockFile(1))).resolves.toEqual(3172987001) + + const crc32Four = new CRC32() + await expect(crc32Four.file(mockFile(2))).resolves.toEqual(847982614) + }) +}) diff --git a/src/utils/crc32.ts b/src/utils/crc32.ts new file mode 100644 index 00000000..85f2082f --- /dev/null +++ b/src/utils/crc32.ts @@ -0,0 +1,90 @@ +/* eslint-disable no-bitwise */ + +import { MB } from './helper' + +/** + * 以下 class 实现参考 + * https://github.com/Stuk/jszip/blob/d4702a70834bd953d4c2d0bc155fad795076631a/lib/crc32.js + * 该实现主要针对大文件优化、对计算的值进行了 `>>> 0` 运算（为与服务端保持一致） + */ +export class CRC32 { + private crc = -1 + private table = this.makeTable() + + private makeTable() { + const table = new Array() + for (let i = 0; i < 256; i++) { + let t = i + for (let j = 0; j < 8; j++) { + if (t & 1) { + // IEEE 标准 + t = (t >>> 1) ^ 0xEDB88320 + } else { + t >>>= 1 + } + } + table[i] = t + } + + return table + } + + private append(data: Uint8Array) { + let crc = this.crc + for (let offset = 0; offset < data.byteLength; offset++) { + crc = (crc >>> 8) ^ this.table[(crc ^ data[offset]) & 0xFF] + } + this.crc = crc + } + + private compute() { + return (this.crc ^ -1) >>> 0 + } + + private async readAsUint8Array(file: File | Blob): Promise { + if (typeof file.arrayBuffer === 'function') { + return new Uint8Array(await file.arrayBuffer()) + } + + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => { + if (reader.result == null) { + reject() + return + } + + if (typeof reader.result === 'string') { + reject() + return + } + + resolve(new Uint8Array(reader.result)) + } + reader.readAsArrayBuffer(file) + }) + } + + async file(file: File): Promise { + if (file.size <= MB) { + this.append(await this.readAsUint8Array(file)) + return this.compute() + } + + const count = Math.ceil(file.size / MB) + for (let index = 0; index < count; index++) { + const start = index * MB + const end = index === (count - 1) ? file.size : start + MB + // eslint-disable-next-line no-await-in-loop + const chuck = await this.readAsUint8Array(file.slice(start, end)) + this.append(new Uint8Array(chuck)) + } + + return this.compute() + } + + static file(file: File): Promise { + const crc = new CRC32() + return crc.file(file) + } +} diff --git a/test/demo1/main.js b/test/demo1/main.js index f4ea02c2..e9b1df48 100644 --- a/test/demo1/main.js +++ b/test/demo1/main.js @@ -2,6 +2,9 @@ var token = res.uptoken; var domain = res.domain; var config = { + checkByServer: true, + checkByMD5: true, + forceDirect: false, useCdnDomain: true, disableStatisticsReport: false, retryCount: 6,