From ea73ef4504d6e6f5b581e1c86740ffd0c2bc31c5 Mon Sep 17 00:00:00 2001 From: Kurt Thiemann Date: Tue, 5 Sep 2023 15:08:50 +0200 Subject: [PATCH] added data processors for nodejs deflate/inflate transform streams --- index.js | 16 ++-- .../DefaultDeflateDataProcessor.js | 4 +- .../DefaultInflateDataProcessor.js | 4 +- .../{ => Fflate}/FflateDataProcessor.js | 19 +--- .../FflateDeflateDataProcessor.js | 0 .../FflateInflateDataProcessor.js | 0 .../NativeDeflateDataProcessor.js | 0 .../NativeInflateDataProcessor.js | 0 .../{ => Native}/NativeStreamDataProcessor.js | 4 +- .../Node/NodeDeflateDataProcessor.js | 12 +++ .../Node/NodeInflateDataProcessor.js | 13 +++ .../Node/NodeStreamDataProcessor.js | 88 +++++++++++++++++++ src/Util/BufferUtils.js | 19 ++++ test/archive.test.js | 47 +++++++++- 14 files changed, 196 insertions(+), 30 deletions(-) rename src/DataProcessor/{ => Fflate}/FflateDataProcessor.js (70%) rename src/DataProcessor/{ => Fflate}/FflateDeflateDataProcessor.js (100%) rename src/DataProcessor/{ => Fflate}/FflateInflateDataProcessor.js (100%) rename src/DataProcessor/{ => Native}/NativeDeflateDataProcessor.js (100%) rename src/DataProcessor/{ => Native}/NativeInflateDataProcessor.js (100%) rename src/DataProcessor/{ => Native}/NativeStreamDataProcessor.js (95%) create mode 100644 src/DataProcessor/Node/NodeDeflateDataProcessor.js create mode 100644 src/DataProcessor/Node/NodeInflateDataProcessor.js create mode 100644 src/DataProcessor/Node/NodeStreamDataProcessor.js create mode 100644 src/Util/BufferUtils.js diff --git a/index.js b/index.js index 94e670b..98404bb 100644 --- a/index.js +++ b/index.js @@ -40,19 +40,23 @@ export { default as ArrayBufferReader } from "./src/Reader/ArrayBufferReader.js" export { default as CP437 } from "./src/Util/CP437.js"; export { default as CRC32 } from "./src/Util/CRC32.js"; export { default as MsDosTime } from "./src/Util/MsDosTime.js"; +export { default as BufferUtils } from "./src/Util/BufferUtils.js"; export { default as DataProcessor } from "./src/DataProcessor/DataProcessor.js"; export { default as AbstractDataProcessor } from "./src/DataProcessor/AbstractDataProcessor.js"; -export { default as FflateInflateDataProcessor } from "./src/DataProcessor/FflateInflateDataProcessor.js"; +export { default as FflateInflateDataProcessor } from "./src/DataProcessor/Fflate/FflateInflateDataProcessor.js"; export { default as PassThroughDataProcessor } from "./src/DataProcessor/PassThroughDataProcessor.js"; -export { default as FflateDeflateDataProcessor } from "./src/DataProcessor/FflateDeflateDataProcessor.js"; -export { default as FflateDataProcessor } from "./src/DataProcessor/FflateDataProcessor.js"; -export { default as NativeDeflateDataProcessor } from "./src/DataProcessor/NativeDeflateDataProcessor.js"; -export { default as NativeInflateDataProcessor } from "./src/DataProcessor/NativeInflateDataProcessor.js"; -export { default as NativeStreamDataProcessor } from "./src/DataProcessor/NativeStreamDataProcessor.js"; +export { default as FflateDeflateDataProcessor } from "./src/DataProcessor/Fflate/FflateDeflateDataProcessor.js"; +export { default as FflateDataProcessor } from "./src/DataProcessor/Fflate/FflateDataProcessor.js"; +export { default as NativeDeflateDataProcessor } from "./src/DataProcessor/Native/NativeDeflateDataProcessor.js"; +export { default as NativeInflateDataProcessor } from "./src/DataProcessor/Native/NativeInflateDataProcessor.js"; +export { default as NativeStreamDataProcessor } from "./src/DataProcessor/Native/NativeStreamDataProcessor.js"; export { default as FallbackDataProcessor } from "./src/DataProcessor/FallbackDataProcessor.js"; export { default as DefaultInflateDataProcessor } from "./src/DataProcessor/DefaultInflateDataProcessor.js"; export { default as DefaultDeflateDataProcessor } from "./src/DataProcessor/DefaultDeflateDataProcessor.js"; +export { default as NodeStreamDataProcessor } from "./src/DataProcessor/Node/NodeStreamDataProcessor.js"; +export { default as NodeDeflateDataProcessor } from "./src/DataProcessor/Node/NodeDeflateDataProcessor.js"; +export { default as NodeInflateDataProcessor } from "./src/DataProcessor/Node/NodeInflateDataProcessor.js"; export { default as Options } from "./src/Options/Options.js"; export { default as EntrySourceOptions } from "./src/Options/EntrySourceOptions.js"; diff --git a/src/DataProcessor/DefaultDeflateDataProcessor.js b/src/DataProcessor/DefaultDeflateDataProcessor.js index a126f96..c8edddf 100644 --- a/src/DataProcessor/DefaultDeflateDataProcessor.js +++ b/src/DataProcessor/DefaultDeflateDataProcessor.js @@ -1,6 +1,6 @@ import FallbackDataProcessor from './FallbackDataProcessor.js'; -import NativeDeflateDataProcessor from './NativeDeflateDataProcessor.js'; -import FflateDeflateDataProcessor from './FflateDeflateDataProcessor.js'; +import NativeDeflateDataProcessor from './Native/NativeDeflateDataProcessor.js'; +import FflateDeflateDataProcessor from './Fflate/FflateDeflateDataProcessor.js'; export default class DefaultDeflateDataProcessor extends FallbackDataProcessor { /** diff --git a/src/DataProcessor/DefaultInflateDataProcessor.js b/src/DataProcessor/DefaultInflateDataProcessor.js index f66dcee..21410f8 100644 --- a/src/DataProcessor/DefaultInflateDataProcessor.js +++ b/src/DataProcessor/DefaultInflateDataProcessor.js @@ -1,6 +1,6 @@ import FallbackDataProcessor from './FallbackDataProcessor.js'; -import NativeInflateDataProcessor from './NativeInflateDataProcessor.js'; -import FflateInflateDataProcessor from './FflateInflateDataProcessor.js'; +import NativeInflateDataProcessor from './Native/NativeInflateDataProcessor.js'; +import FflateInflateDataProcessor from './Fflate/FflateInflateDataProcessor.js'; export default class DefaultInflateDataProcessor extends FallbackDataProcessor { /** diff --git a/src/DataProcessor/FflateDataProcessor.js b/src/DataProcessor/Fflate/FflateDataProcessor.js similarity index 70% rename from src/DataProcessor/FflateDataProcessor.js rename to src/DataProcessor/Fflate/FflateDataProcessor.js index 7a3bc83..289d27d 100644 --- a/src/DataProcessor/FflateDataProcessor.js +++ b/src/DataProcessor/Fflate/FflateDataProcessor.js @@ -1,4 +1,5 @@ -import AbstractDataProcessor from './AbstractDataProcessor.js'; +import AbstractDataProcessor from '../AbstractDataProcessor.js'; +import BufferUtils from '../../Util/BufferUtils.js'; /** * @abstract @@ -35,21 +36,7 @@ export default class FflateDataProcessor extends AbstractDataProcessor { let chunks = this.chunks; this.chunks = []; - if (!chunks.length) { - return new Uint8Array(0); - } - if (chunks.length === 1) { - return chunks[0]; - } - - let length = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); - let res = new Uint8Array(length); - let offset = 0; - for (let chunk of chunks) { - res.set(chunk, offset); - offset += chunk.byteLength; - } - return res; + return BufferUtils.concatBuffers(chunks); } /** diff --git a/src/DataProcessor/FflateDeflateDataProcessor.js b/src/DataProcessor/Fflate/FflateDeflateDataProcessor.js similarity index 100% rename from src/DataProcessor/FflateDeflateDataProcessor.js rename to src/DataProcessor/Fflate/FflateDeflateDataProcessor.js diff --git a/src/DataProcessor/FflateInflateDataProcessor.js b/src/DataProcessor/Fflate/FflateInflateDataProcessor.js similarity index 100% rename from src/DataProcessor/FflateInflateDataProcessor.js rename to src/DataProcessor/Fflate/FflateInflateDataProcessor.js diff --git a/src/DataProcessor/NativeDeflateDataProcessor.js b/src/DataProcessor/Native/NativeDeflateDataProcessor.js similarity index 100% rename from src/DataProcessor/NativeDeflateDataProcessor.js rename to src/DataProcessor/Native/NativeDeflateDataProcessor.js diff --git a/src/DataProcessor/NativeInflateDataProcessor.js b/src/DataProcessor/Native/NativeInflateDataProcessor.js similarity index 100% rename from src/DataProcessor/NativeInflateDataProcessor.js rename to src/DataProcessor/Native/NativeInflateDataProcessor.js diff --git a/src/DataProcessor/NativeStreamDataProcessor.js b/src/DataProcessor/Native/NativeStreamDataProcessor.js similarity index 95% rename from src/DataProcessor/NativeStreamDataProcessor.js rename to src/DataProcessor/Native/NativeStreamDataProcessor.js index 83b96f0..25faa28 100644 --- a/src/DataProcessor/NativeStreamDataProcessor.js +++ b/src/DataProcessor/Native/NativeStreamDataProcessor.js @@ -1,5 +1,5 @@ -import Constants from '../Constants.js'; -import AbstractDataProcessor from './AbstractDataProcessor.js'; +import Constants from '../../Constants.js'; +import AbstractDataProcessor from '../AbstractDataProcessor.js'; export default class NativeStreamDataProcessor extends AbstractDataProcessor { /** @type {?boolean} */ static supported = null; diff --git a/src/DataProcessor/Node/NodeDeflateDataProcessor.js b/src/DataProcessor/Node/NodeDeflateDataProcessor.js new file mode 100644 index 0000000..9e06e05 --- /dev/null +++ b/src/DataProcessor/Node/NodeDeflateDataProcessor.js @@ -0,0 +1,12 @@ +import NodeStreamDataProcessor from './NodeStreamDataProcessor.js'; +import {createDeflateRaw} from 'zlib'; + +export default class NodeDeflateDataProcessor extends NodeStreamDataProcessor { + /** + * @inheritDoc + */ + initTransform() { + this.transform = createDeflateRaw(); + this.transform.on('data', this.onData.bind(this)); + } +} diff --git a/src/DataProcessor/Node/NodeInflateDataProcessor.js b/src/DataProcessor/Node/NodeInflateDataProcessor.js new file mode 100644 index 0000000..d815a0a --- /dev/null +++ b/src/DataProcessor/Node/NodeInflateDataProcessor.js @@ -0,0 +1,13 @@ +import NodeStreamDataProcessor from './NodeStreamDataProcessor.js'; +import {createInflateRaw} from 'zlib'; + + +export default class NodeInflateDataProcessor extends NodeStreamDataProcessor { + /** + * @inheritDoc + */ + initTransform() { + this.transform = createInflateRaw(); + this.transform.on('data', this.onData.bind(this)); + } +} diff --git a/src/DataProcessor/Node/NodeStreamDataProcessor.js b/src/DataProcessor/Node/NodeStreamDataProcessor.js new file mode 100644 index 0000000..ca86916 --- /dev/null +++ b/src/DataProcessor/Node/NodeStreamDataProcessor.js @@ -0,0 +1,88 @@ +import AbstractDataProcessor from '../AbstractDataProcessor.js'; +import BufferUtils from '../../Util/BufferUtils.js'; + +export default class NodeStreamDataProcessor extends AbstractDataProcessor { + /** @type {Uint8Array[]} */ chunks = []; + /** @type {module:stream.internal.Transform} */ transform = null; + + /** + * @inheritDoc + */ + constructor(reader, createPreCrc = false, createPostCrc = false) { + super(reader, createPreCrc, createPostCrc); + this.initTransform(); + } + + /** + * @abstract + */ + initTransform() { + } + + /** + * @inheritDoc + */ + async generate(length) { + if(this.chunkReader.isEof()) { + return null; + } + + await this.writeAsync(this.transform, await this.chunkReader.getChunk(length)); + if (this.chunkReader.isEof()) { + await this.endAsync(this.transform); + } + return this.concatChunks(); + } + + writeAsync(stream, data) { + return new Promise((resolve, reject) => { + stream.write(data, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } + + endAsync(stream) { + return new Promise((resolve, reject) => { + stream.on('end', resolve); + stream.end((err) => { + if (err) { + reject(err); + } + }); + }); + } + + /** + * @protected + * @returns {Uint8Array} + */ + concatChunks() { + let chunks = this.chunks; + this.chunks = []; + + return BufferUtils.concatBuffers(chunks); + } + + /** + * @protected + * @param {Uint8Array} chunk + * @param {boolean} final + */ + onData(chunk, final) { + this.chunks.push(chunk); + } + + /** + * @inheritDoc + */ + async reset() { + await super.reset(); + this.chunks = []; + this.initTransform(); + } +} diff --git a/src/Util/BufferUtils.js b/src/Util/BufferUtils.js new file mode 100644 index 0000000..489b056 --- /dev/null +++ b/src/Util/BufferUtils.js @@ -0,0 +1,19 @@ +export default class BufferUtils { + static concatBuffers(chunks) { + if (!chunks.length) { + return new Uint8Array(0); + } + if (chunks.length === 1) { + return chunks[0]; + } + + let length = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + let res = new Uint8Array(length); + let offset = 0; + for (let chunk of chunks) { + res.set(chunk, offset); + offset += chunk.byteLength; + } + return res; + } +} diff --git a/test/archive.test.js b/test/archive.test.js index bf7a347..46c741e 100644 --- a/test/archive.test.js +++ b/test/archive.test.js @@ -2,8 +2,8 @@ import { ArchiveEntry, ArchiveMerger, ArrayBufferReader, - Constants, - DataReaderEntrySource, ExtendedTimestamp, + Constants, CRC32, + DataReaderEntrySource, ExtendedTimestamp, NodeDeflateDataProcessor, NodeInflateDataProcessor, ReadArchive, UnicodeExtraField, WriteArchive @@ -12,6 +12,7 @@ import {FakeDataReader, openFileReader, writeArchive, writeArchiveToBuffer} from import BigInt from '../src/Util/BigInt.js'; const encoder = new TextEncoder(); +const decoder = new TextDecoder(); const timestamp = 1655914036000; const timestampSeconds = Math.floor(timestamp/1000); const date = new Date(timestamp); @@ -46,6 +47,48 @@ test('Write large file archive', async () => { expect(entry.getCrc()).toBe(126491095); }); +test('Write and read archive using NodeJS data processors', async () => { + let fileName = 'file.txt'; + let i = 0; + let payloadString = 'Hello World!'; + let payload = encoder.encode(payloadString); + + let writeDataProcessors = new Map([ + [Constants.COMPRESSION_METHOD_DEFLATE, NodeDeflateDataProcessor] + ]); + + let readDataProcessors = new Map([ + [Constants.COMPRESSION_METHOD_DEFLATE, NodeInflateDataProcessor] + ]); + + let archive = new WriteArchive(() => { + if(i > 0) { + return null; + } + i++; + return new DataReaderEntrySource(new ArrayBufferReader(payload.buffer, payload.byteOffset, payload.byteLength), { + fileName: fileName, + dataProcessors: writeDataProcessors + }); + }); + let data = await writeArchiveToBuffer(archive); + + let readArchive = new ReadArchive(new ArrayBufferReader(data.buffer, data.byteOffset, data.byteLength), { + entryOptions: { + dataProcessors: readDataProcessors + } + }); + await readArchive.init(); + let entries = await readArchive.getAllEntries(); + + expect(entries.length).toBe(1); + + let entry = entries.pop(); + expect(entry.getFileNameString()).toBe(fileName); + expect(decoder.decode(await entry.getData())).toBe(payloadString); + expect(entry.getCrc()).toBe(CRC32.hash(payload)); +}); + test('Write many entries', async () => { let count = 70000; let i = 0;