diff --git a/javascript/package.json b/javascript/package.json index 23e35dac7c..1b4e79ee3e 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -1,6 +1,6 @@ { "scripts": { - "test": "npm run build && enableHps=true jest", + "test": "npm run build && enableHps=true jest && ECMA_ONLY=true jest", "build": "npm run build -w packages/fury -w packages/hps", "lint": "eslint .", "lint-fix": "eslint . --fix" diff --git a/javascript/packages/fury/index.ts b/javascript/packages/fury/index.ts index 3d91cd953d..0f5f1fda13 100644 --- a/javascript/packages/fury/index.ts +++ b/javascript/packages/fury/index.ts @@ -238,7 +238,7 @@ export type ToRecordType = T extends { : T extends { type: InternalSerializerType.BINARY; } - ? Buffer + ? Uint8Array : T extends { type: InternalSerializerType.ANY; } @@ -264,7 +264,7 @@ export default class { serialize: (data: ToRecordType) => { return this.fury.serialize(data, serializer); }, - deserialize: (bytes: Buffer) => { + deserialize: (bytes: Uint8Array) => { return this.fury.deserialize(bytes, serializer) as ToRecordType; }, }; @@ -274,7 +274,7 @@ export default class { return this.fury.serialize(v, serialize); } - deserialize(bytes: Buffer) { + deserialize(bytes: Uint8Array) { return this.fury.deserialize(bytes); } } diff --git a/javascript/packages/fury/lib/classResolver.ts b/javascript/packages/fury/lib/classResolver.ts index 52c9e16381..0463f7fe3e 100644 --- a/javascript/packages/fury/lib/classResolver.ts +++ b/javascript/packages/fury/lib/classResolver.ts @@ -24,6 +24,7 @@ import boolSerializer from "./internalSerializer/bool"; import { uInt16Serializer, int16Serializer, int32Serializer, uInt32Serializer, uInt64Serializer, floatSerializer, doubleSerializer, uInt8Serializer, int64Serializer, int8Serializer } from "./internalSerializer/number"; import { InternalSerializerType, Serializer, Fury, BinaryReader, BinaryWriter } from "./type"; import anySerializer from './internalSerializer/any'; +import { PlatformBuffer } from "./platformBuffer"; const USESTRINGVALUE = 0; const USESTRINGID = 1 @@ -91,7 +92,7 @@ export default class SerializerResolver { return this.customSerializer[tag]; } - writeTag(binaryWriter: BinaryWriter, tag: string, tagHash: number, tagBuffer: Buffer, bufferLen: number) { + writeTag(binaryWriter: BinaryWriter, tag: string, tagHash: number, tagBuffer: PlatformBuffer, bufferLen: number) { const index = this.writeStringIndex.indexOf(tag); if (index > -1) { binaryWriter.uint8(USESTRINGID) diff --git a/javascript/packages/fury/lib/codeGen.ts b/javascript/packages/fury/lib/codeGen.ts index dd4a4b3f07..1ef27d9e4c 100644 --- a/javascript/packages/fury/lib/codeGen.ts +++ b/javascript/packages/fury/lib/codeGen.ts @@ -20,6 +20,7 @@ import mapSerializer from './internalSerializer/map'; import setSerializer from './internalSerializer/set'; import { arraySerializer } from './internalSerializer/array'; import { x64hash128 } from './murmurHash3'; +import { fromString } from './platformBuffer'; export interface TypeDescription { type: InternalSerializerType, label?: string, @@ -171,7 +172,7 @@ export const genSerializer = (fury: Fury, description: TypeDescription) => { } fury.classResolver.registerSerializerByTag(tag, fury.classResolver.getSerializerById(InternalSerializerType.ANY)); - const tagByteLen = Buffer.from(tag).byteLength; + const tagByteLen = fromString(tag).byteLength; const expectHash = computeStructHash(description); const read = ` // relation tag: ${Cast(description).options?.tag} @@ -197,9 +198,9 @@ export const genSerializer = (fury: Fury, description: TypeDescription) => { return function (fury, scope) { const { referenceResolver, binaryWriter, classResolver, binaryReader } = fury; const { writeNullOrRef, pushReadObject } = referenceResolver; - const { RefFlags, InternalSerializerType, arraySerializer, mapSerializer, setSerializer, x64hash128 } = scope; + const { RefFlags, InternalSerializerType, arraySerializer, mapSerializer, setSerializer, x64hash128, fromString } = scope; ${declarations.join('')} - const tagBuffer = Buffer.from("${validTag}"); + const tagBuffer = fromString("${validTag}"); let tagHash = x64hash128(tagBuffer, 47).getBigUint64(0); if (tagHash == 0) { tagHash = 1; @@ -237,5 +238,6 @@ return function (fury, scope) { mapSerializer, setSerializer, x64hash128, + fromString, })); } \ No newline at end of file diff --git a/javascript/packages/fury/lib/fury.ts b/javascript/packages/fury/lib/fury.ts index 8213b735ec..18f1b3fd73 100644 --- a/javascript/packages/fury/lib/fury.ts +++ b/javascript/packages/fury/lib/fury.ts @@ -39,7 +39,7 @@ export default (config: Config) => { classResolver.init(fury); - function deserialize(bytes: Buffer, serializer?: Serializer): T | null { + function deserialize(bytes: Uint8Array, serializer?: Serializer): T | null { referenceResolver.reset(); classResolver.reset(); binaryReader.reset(bytes); diff --git a/javascript/packages/fury/lib/platformBuffer.ts b/javascript/packages/fury/lib/platformBuffer.ts new file mode 100644 index 0000000000..557a325df7 --- /dev/null +++ b/javascript/packages/fury/lib/platformBuffer.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2023 The Fury Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isNodeEnv } from "./util"; + +export interface PlatformBuffer extends Uint8Array { + latin1Slice(start: number, end: number): string, + utf8Slice(start: number, end: number): string, + latin1Write(v: string, offset: number): void, + utf8Write(v: string, offset: number): void, + copy(target: Uint8Array, targetStart?: number, sourceStart?: number, sourceEnd?: number): Uint8Array +} + + +class BrowserBuffer extends Uint8Array { + static alloc(size: number) { + return new BrowserBuffer(new Uint8Array(size)); + } + + latin1Slice(start: number, end: number) { + if (end - start < 1) { + return ""; + } + let str = ""; + for (let i = start; i < end;) { + str += String.fromCharCode(this[i++]); + } + return str; + } + + utf8Slice(start: number, end: number) { + if (end - start < 1) { + return ""; + } + let str = ""; + for (let i = start; i < end;) { + const t = this[i++]; + if (t <= 0x7F) { + str += String.fromCharCode(t); + } else if (t >= 0xC0 && t < 0xE0) { + str += String.fromCharCode((t & 0x1F) << 6 | this[i++] & 0x3F); + } else if (t >= 0xE0 && t < 0xF0) { + str += String.fromCharCode((t & 0xF) << 12 | (this[i++] & 0x3F) << 6 | this[i++] & 0x3F); + } else if (t >= 0xF0) { + const t2 = ((t & 7) << 18 | (this[i++] & 0x3F) << 12 | (this[i++] & 0x3F) << 6 | this[i++] & 0x3F) - 0x10000; + str += String.fromCharCode(0xD800 + (t2 >> 10)); + str += String.fromCharCode(0xDC00 + (t2 & 0x3FF)); + } + } + + return str; + } + + copy(target: Uint8Array, targetStart?: number, sourceStart?: number, sourceEnd?: number) { + target.set(this.subarray(sourceStart, sourceEnd), targetStart); + } +} + +export function fromString(str: string) { + if (isNodeEnv) { + return Buffer.from(str); + } else { + return new BrowserBuffer(new TextEncoder().encode(str)) + } +} + +export function fromUint8Array(ab: Buffer | Uint8Array): PlatformBuffer { + if (isNodeEnv) { + if (!Buffer.isBuffer(ab)) { + return (Buffer.from(ab) as unknown as PlatformBuffer) + } else { + return ab as unknown as PlatformBuffer; + } + } else { + const result = new BrowserBuffer(ab); + return result as unknown as PlatformBuffer; + } +} + + + +export function alloc(size: number): PlatformBuffer { + if (isNodeEnv) { + return Buffer.allocUnsafe(size) as unknown as PlatformBuffer; + } else { + const result = BrowserBuffer.alloc(size); + return result as PlatformBuffer; + } +} + + +export function strByteLength(str: string): number { + if (isNodeEnv) { + return Buffer.byteLength(str); + } else { + let len = 0; + let c = 0; + for (let i = 0; i < str.length; ++i) { + c = str.charCodeAt(i); + if (c < 128) + len += 1; + else if (c < 2048) + len += 2; + else if ((c & 0xFC00) === 0xD800 && (str.charCodeAt(i + 1) & 0xFC00) === 0xDC00) { + ++i; + len += 4; + } else + len += 3; + } + return len; + } +} diff --git a/javascript/packages/fury/lib/reader.ts b/javascript/packages/fury/lib/reader.ts index 8fc3f262da..a5c4a47af8 100644 --- a/javascript/packages/fury/lib/reader.ts +++ b/javascript/packages/fury/lib/reader.ts @@ -15,146 +15,148 @@ */ import { Config } from "./type"; +import { isNodeEnv } from "./util"; +import { PlatformBuffer, alloc, fromUint8Array } from './platformBuffer'; export const BinaryReader = (config: Config) => { - let cursor = 0; - let dataView!: DataView; - let buffer!: Buffer; - let bigString: string; - function reset(bf: Buffer) { - buffer = bf; - dataView = new DataView(buffer.buffer, buffer.byteOffset); - if (config.useSliceString) { - bigString = (buffer as any).latin1Slice(0, buffer.byteLength); - } - cursor = 0; - } - - function uint8() { - return dataView.getUint8(cursor++); - } - - function int8() { - return dataView.getInt8(cursor++); - } - - function uint16() { - const result = dataView.getUint16(cursor, true); - cursor += 2; - return result; - } - - function int16() { - const result = dataView.getInt16(cursor, true); - cursor += 2; - return result; - } - - function skip(len: number) { - cursor += len; - } - - function int32() { - const result = dataView.getInt32(cursor, true); - cursor += 4; - return result; - } - - function uint32() { - const result = dataView.getUint32(cursor, true); - cursor += 4; - return result; - } - - function int64() { - const result = dataView.getBigInt64(cursor, true); - cursor += 8; - return Number(result); - } - - function uint64() { - const result = dataView.getBigUint64(cursor, true); - cursor += 8; - return Number(result); - } - - function float() { - const result = dataView.getFloat32(cursor, true); - cursor += 4; - return result; - } - - function double() { - const result = dataView.getFloat64(cursor, true); - cursor += 8; - return result; - } - - function stringUtf8(len: number) { - const result = (buffer as any).utf8Slice(cursor, cursor + len); - cursor += len; - return result; - } - - function stringLatin1Fast(len: number) { - const result = bigString.substring(cursor, cursor + len); - cursor += len; - return result; - } + const sliceStringEnable = isNodeEnv && config.useSliceString; + let cursor = 0; + let dataView!: DataView; + let buffer!: PlatformBuffer; + let bigString: string; + function reset(ab: Uint8Array) { + buffer = fromUint8Array(ab); + dataView = new DataView(buffer.buffer, buffer.byteOffset); + if (sliceStringEnable) { + bigString = buffer.latin1Slice(0, buffer.byteLength); + } + cursor = 0; + } - function stringLatin1Slow(len: number) { - const result = (buffer as any).latin1Slice(cursor, cursor + len); - cursor += len; - return result; - } - - function binary(len: number) { - const result = Buffer.allocUnsafe(len); - buffer.copy(result, 0, cursor, cursor + len); - return result; - } - - function varInt32() { - let byte_ = int8(); - let result = byte_ & 0x7f; + function uint8() { + return dataView.getUint8(cursor++); + } + + function int8() { + return dataView.getInt8(cursor++); + } + + function uint16() { + const result = dataView.getUint16(cursor, true); + cursor += 2; + return result; + } + + function int16() { + const result = dataView.getInt16(cursor, true); + cursor += 2; + return result; + } + + function skip(len: number) { + cursor += len; + } + + function int32() { + const result = dataView.getInt32(cursor, true); + cursor += 4; + return result; + } + + function uint32() { + const result = dataView.getUint32(cursor, true); + cursor += 4; + return result; + } + + function int64() { + const result = dataView.getBigInt64(cursor, true); + cursor += 8; + return Number(result); + } + + function uint64() { + const result = dataView.getBigUint64(cursor, true); + cursor += 8; + return Number(result); + } + + function float() { + const result = dataView.getFloat32(cursor, true); + cursor += 4; + return result; + } + + function double() { + const result = dataView.getFloat64(cursor, true); + cursor += 8; + return result; + } + + function stringUtf8(len: number) { + const result = buffer.utf8Slice(cursor, cursor + len); + cursor += len; + return result; + } + + function stringLatin1Fast(len: number) { + const result = bigString.substring(cursor, cursor + len); + cursor += len; + return result; + } + + function stringLatin1Slow(len: number) { + const result = buffer.latin1Slice(cursor, cursor + len); + cursor += len; + return result; + } + + function binary(len: number) { + const result = alloc(len); + buffer.copy(result, 0, cursor, cursor + len); + return result; + } + + function varInt32() { + let byte_ = int8(); + let result = byte_ & 0x7f; + if ((byte_ & 0x80) != 0) { + byte_ = int8(); + result |= (byte_ & 0x7f) << 7; if ((byte_ & 0x80) != 0) { byte_ = int8(); - result |= (byte_ & 0x7f) << 7; + result |= (byte_ & 0x7f) << 14; if ((byte_ & 0x80) != 0) { byte_ = int8(); - result |= (byte_ & 0x7f) << 14; + result |= (byte_ & 0x7f) << 21; if ((byte_ & 0x80) != 0) { byte_ = int8(); - result |= (byte_ & 0x7f) << 21; - if ((byte_ & 0x80) != 0) { - byte_ = int8(); - result |= (byte_ & 0x7f) << 28; - } + result |= (byte_ & 0x7f) << 28; } } } - return result; } - - return { - getCursor: () => cursor, - setCursor: (v: number) => (cursor = v), - varInt32, - int8, - buffer: binary, - uint8, - reset, - stringUtf8, - stringLatin1: config.useSliceString ? stringLatin1Fast : stringLatin1Slow, - double, - float, - uint16, - int16, - uint64, - skip, - int64, - uint32, - int32, - }; + return result; + } + + return { + getCursor: () => cursor, + setCursor: (v: number) => (cursor = v), + varInt32, + int8, + buffer: binary, + uint8, + reset, + stringUtf8, + stringLatin1: sliceStringEnable ? stringLatin1Fast : stringLatin1Slow, + double, + float, + uint16, + int16, + uint64, + skip, + int64, + uint32, + int32, }; - \ No newline at end of file +}; diff --git a/javascript/packages/fury/lib/util.ts b/javascript/packages/fury/lib/util.ts index ee3edca2c5..f9dc7ee720 100644 --- a/javascript/packages/fury/lib/util.ts +++ b/javascript/packages/fury/lib/util.ts @@ -44,4 +44,4 @@ export const safePropName = (prop: string) => { return prop; } - +export const isNodeEnv = typeof process === "object" && process.env.ECMA_ONLY === 'true' && typeof require === "function"; \ No newline at end of file diff --git a/javascript/packages/fury/lib/writer.ts b/javascript/packages/fury/lib/writer.ts index 205f2c27a8..b04f47b246 100644 --- a/javascript/packages/fury/lib/writer.ts +++ b/javascript/packages/fury/lib/writer.ts @@ -15,19 +15,23 @@ */ import { Config, LATIN1, UTF8 } from "./type"; +import { PlatformBuffer, alloc, strByteLength } from './platformBuffer'; const MAX_POOL_SIZE = 1024 * 1024 * 3; // 3MB + + + export const BinaryWriter = (config: Config) => { let cursor = 0; let byteLength: number; - let arrayBuffer: Buffer; + let arrayBuffer: PlatformBuffer; let dataView: DataView; let reserved = 0; function initPoll() { byteLength = 1024 * 100; - arrayBuffer = Buffer.allocUnsafe(byteLength); + arrayBuffer = alloc(byteLength); dataView = new DataView(arrayBuffer.buffer); } @@ -36,7 +40,9 @@ export const BinaryWriter = (config: Config) => { function reserve(len: number) { reserved += len; if (byteLength - cursor <= reserved) { - arrayBuffer = Buffer.concat([arrayBuffer, Buffer.allocUnsafe(byteLength + len)]); + const newAb = alloc(byteLength + len); + arrayBuffer.copy(newAb, 0); + arrayBuffer = newAb; byteLength = arrayBuffer.byteLength; dataView = new DataView(arrayBuffer.buffer); } @@ -121,13 +127,13 @@ export const BinaryWriter = (config: Config) => { cursor += 8; } - function utf8StringOfInt16(bf: Buffer, bufferLen: number) { + function utf8StringOfInt16(bf: PlatformBuffer, bufferLen: number) { int16(bufferLen); bf.copy(arrayBuffer, cursor); cursor += bufferLen; } - function fastWriteStringUtf8(string: string, buffer: Buffer, offset: number) { + function fastWriteStringUtf8(string: string, buffer: Uint8Array, offset: number) { let c1: number; let c2: number; for (let i = 0; i < string.length; ++i) { @@ -165,7 +171,7 @@ export const BinaryWriter = (config: Config) => { const { isLatin1: detectIsLatin1, stringCopy } = config!.hps!; return function (v: string) { const isLatin1 = detectIsLatin1(v); - const len = isLatin1 ? v.length : Buffer.byteLength(v); + const len = isLatin1 ? v.length : strByteLength(v); if (config.useLatin1) { dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8); } @@ -177,7 +183,7 @@ export const BinaryWriter = (config: Config) => { if (len < 40) { fastWriteStringUtf8(v, arrayBuffer, cursor); } else { - (arrayBuffer as any).utf8Write(v, cursor); + arrayBuffer.utf8Write(v, cursor); } } cursor += len; @@ -185,7 +191,7 @@ export const BinaryWriter = (config: Config) => { } function stringOfVarInt32Slow(v: string) { - const len = Buffer.byteLength(v); + const len = strByteLength(v); const isLatin1 = len === v.length; if (config.useLatin1) { dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8); @@ -198,13 +204,13 @@ export const BinaryWriter = (config: Config) => { arrayBuffer[cursor + index] = v.charCodeAt(index); } } else { - (arrayBuffer as any).latin1Write(v, cursor); + arrayBuffer.latin1Write(v, cursor); } } else { if (len < 40) { fastWriteStringUtf8(v, arrayBuffer, cursor); } else { - (arrayBuffer as any).utf8Write(v, cursor); + arrayBuffer.utf8Write(v, cursor); } } cursor += len; @@ -256,7 +262,7 @@ export const BinaryWriter = (config: Config) => { } function dump() { - const result = Buffer.allocUnsafe(cursor); + const result = alloc(cursor); arrayBuffer.copy(result, 0, 0, cursor); tryFreePool(); return result; diff --git a/javascript/test/binary.test.ts b/javascript/test/binary.test.ts index 9f1695d444..f8a9e9506a 100644 --- a/javascript/test/binary.test.ts +++ b/javascript/test/binary.test.ts @@ -38,7 +38,10 @@ describe('binary', () => { const result = fury.deserialize( input ); - expect(result).toEqual({ a: Buffer.from([1, 2, 3]) }) + expect(result instanceof Uint8Array) + expect(result.a[0] === 1); + expect(result.a[1] === 2); + expect(result.a[2] === 3); }); }); diff --git a/javascript/test/protocol.test.ts b/javascript/test/protocol.test.ts index 50e88479f9..a5506250fa 100644 --- a/javascript/test/protocol.test.ts +++ b/javascript/test/protocol.test.ts @@ -62,7 +62,7 @@ describe('protocol', () => { } }); - const obj = deserialize(Buffer.from([ + const obj = deserialize(new Uint8Array([ 134, 2, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 159, 160, 124, 69, 240, 2, 120, 21, 0, 101, 120, 97, 109, 112, 108, 101, 46, 67, 111, 109, 112, 108, 101, 120, 79, 98, 106, 101, 99, 116, 71, 168, 32, 21, 0, 13, 0, 3, 115, 116, 114, 0, 30, 0, 2, 255, 7, 0, 1, 0, 0, 0,