Skip to content

Commit

Permalink
[JavaScript] compress numbers (#1290)
Browse files Browse the repository at this point in the history 
1. Compress the numbers so that they correspond to standard protocol
2. Remove the arrow dependency in cargo, it is unnecessary at this stage
  • Loading branch information
@theweipeng
theweipeng authored Jan 2, 2024
1 parent 1134cce commit 53b166b
Show file tree
Hide file tree
Showing 11 changed files with 83 additions and 53 deletions.
2 changes: 1 addition & 1 deletion javascript/packages/fury/lib/internalSerializer/any.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export default (fury: Fury) => {
case RefFlags.RefValueFlag:
return detectSerializer(cursor).read();
case RefFlags.RefFlag:
return referenceResolver.getReadObjectByRefId(binaryReader.varInt32());
return referenceResolver.getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/array.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ export const buildArray = (fury: Fury, item: Serializer, type: InternalSerialize
const { binaryReader, binaryWriter, referenceResolver } = fury;

const { pushReadObject } = referenceResolver;
const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { varUInt32: readVarUInt32 } = binaryReader;
const { write, read } = item;
const innerHeadSize = (item.config().reserve);
return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Array(len);
pushReadObject(result);
for (let i = 0; i < result.length; i++) {
Expand All @@ -39,7 +39,7 @@ export const buildArray = (fury: Fury, item: Serializer, type: InternalSerialize
return result;
}),
write: referenceResolver.withNullableOrRefWriter(type, (v: any[]) => {
writeVarInt32(v.length);
writeVarUInt32(v.length);

reserves(innerHeadSize * v.length);

Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/map.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ import { InternalSerializerType, Fury, Serializer } from "../type";

export default (fury: Fury, keySerializer: Serializer, valueSerializer: Serializer) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: readVarUInt32 } = binaryReader;

const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { pushReadObject } = referenceResolver;
const innerHeadSize = keySerializer.config().reserve + valueSerializer.config().reserve;
return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Map();
pushReadObject(result);
for (let index = 0; index < len; index++) {
Expand All @@ -40,7 +40,7 @@ export default (fury: Fury, keySerializer: Serializer, valueSerializer: Serializ
}),
write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.MAP, (v: Map<any, any>) => {
const len = v.size;
writeVarInt32(len);
writeVarUInt32(len);
reserves(innerHeadSize * v.size);
for (const [key, value] of v.entries()) {
keySerializer.write(key);
Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/set.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ import { InternalSerializerType } from "../type";

export default (fury: Fury, nestedSerializer: Serializer) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;
const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { varUInt32: readVarUInt32 } = binaryReader;
const { pushReadObject } = referenceResolver;
const innerHeadSize = nestedSerializer.config().reserve;
return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Set();
pushReadObject(result);
for (let index = 0; index < len; index++) {
Expand All @@ -38,7 +38,7 @@ export default (fury: Fury, nestedSerializer: Serializer) => {
}),
write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.FURY_SET, (v: Set<any>) => {
const len = v.size;
writeVarInt32(len);
writeVarUInt32(len);
reserves(innerHeadSize * v.size);
for (const value of v.values()) {
nestedSerializer.write(value);
Expand Down
10 changes: 5 additions & 5 deletions javascript/packages/fury/lib/internalSerializer/string.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,23 @@ import { InternalSerializerType, RefFlags } from "../type";

export default (fury: Fury) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;
const { stringOfVarInt32: writeStringOfVarInt32, int8 } = binaryWriter;
const { stringOfVarInt32: readStringOfVarInt32 } = binaryReader;
const { stringOfVarUInt32: writeStringOfVarUInt32, int8 } = binaryWriter;
const { stringOfVarUInt32: readStringOfVarUInt32 } = binaryReader;

return {
...referenceResolver.deref(() => {
return readStringOfVarInt32();
return readStringOfVarUInt32();
}),
write: referenceResolver.withNotNullableWriter(InternalSerializerType.STRING, "", (v: string) => {
writeStringOfVarInt32(v);
writeStringOfVarUInt32(v);
}),
writeWithoutType: (v: string) => {
if (v === null) {
binaryWriter.int8(RefFlags.NullFlag);
return;
}
int8(RefFlags.NotNullValueFlag);
writeStringOfVarInt32(v);
writeStringOfVarUInt32(v);
},
config: () => {
return {
Expand Down
8 changes: 4 additions & 4 deletions javascript/packages/fury/lib/internalSerializer/tuple.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => {
const { binaryReader, binaryWriter, referenceResolver } = fury;

const { pushReadObject } = referenceResolver;
const { varInt32: writeVarInt32, reserve: reserves } = binaryWriter;
const { varInt32: readVarInt32 } = binaryReader;
const { varUInt32: writeVarUInt32, reserve: reserves } = binaryWriter;
const { varUInt32: readVarUInt32 } = binaryReader;

return {
...referenceResolver.deref(() => {
const len = readVarInt32();
const len = readVarUInt32();
const result = new Array(len);
pushReadObject(result);
for (let i = 0; i < len; i++) {
Expand All @@ -39,7 +39,7 @@ export const tupleSerializer = (fury: Fury, serializers: Serializer[]) => {
return result;
}),
write: referenceResolver.withNullableOrRefWriter(InternalSerializerType.TUPLE, (v: any[]) => {
writeVarInt32(serializers.length);
writeVarUInt32(serializers.length);

for (let i = 0; i < serializers.length; i++) {
const item = serializers[i];
Expand Down
17 changes: 13 additions & 4 deletions javascript/packages/fury/lib/reader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,9 @@ export const BinaryReader = (config: Config) => {
return result;
}

function stringOfVarInt32() {
function stringOfVarUInt32() {
const isLatin1 = uint8() === LATIN1;
const len = varInt32();
const len = varUInt32();
return isLatin1 ? stringLatin1(len) : stringUtf8(len);
}

Expand Down Expand Up @@ -140,7 +140,11 @@ export const BinaryReader = (config: Config) => {
return result;
}

function varInt32() {
function zigZag(v: number) {
return (v >> 1) ^ -(v & 1);
}

function varUInt32() {
let byte_ = int8();
let result = byte_ & 0x7f;
if ((byte_ & 0x80) != 0) {
Expand All @@ -162,10 +166,15 @@ export const BinaryReader = (config: Config) => {
return result;
}

function varInt32() {
return zigZag(varUInt32());
}

return {
getCursor: () => cursor,
setCursor: (v: number) => (cursor = v),
varInt32,
varUInt32,
int8,
buffer: binary,
bufferRef,
Expand All @@ -174,7 +183,7 @@ export const BinaryReader = (config: Config) => {
stringUtf8At,
stringUtf8,
stringLatin1,
stringOfVarInt32,
stringOfVarUInt32,
double,
float,
uint16,
Expand Down
6 changes: 3 additions & 3 deletions javascript/packages/fury/lib/referenceResolver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ export const ReferenceResolver = (
const existsId = existsWriteObject(v);
if (typeof existsId === "number") {
binaryWriter.int8(RefFlags.RefFlag);
binaryWriter.varInt32(existsId);
binaryWriter.varUInt32(existsId);
} else {
int24(head);
pushWriteObject(v);
Expand Down Expand Up @@ -137,7 +137,7 @@ export const ReferenceResolver = (
skipType();
return fn();
case RefFlags.RefFlag:
return getReadObjectByRefId(binaryReader.varInt32());
return getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
Expand All @@ -150,7 +150,7 @@ export const ReferenceResolver = (
case RefFlags.RefValueFlag:
return fn();
case RefFlags.RefFlag:
return getReadObjectByRefId(binaryReader.varInt32());
return getReadObjectByRefId(binaryReader.varUInt32());
case RefFlags.NullFlag:
return null;
case RefFlags.NotNullValueFlag:
Expand Down
23 changes: 16 additions & 7 deletions javascript/packages/fury/lib/writer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,13 @@ export const BinaryWriter = (config: Config) => {
}
}

function stringOfVarInt32Fast() {
function stringOfVarUInt32Fast() {
const { isLatin1: detectIsLatin1, stringCopy } = config!.hps!;
return function (v: string) {
const isLatin1 = detectIsLatin1(v);
const len = isLatin1 ? v.length : strByteLength(v);
dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8);
varInt32(len);
varUInt32(len);
reserve(len);
if (isLatin1) {
stringCopy(v, arrayBuffer, cursor);
Expand All @@ -191,11 +191,11 @@ export const BinaryWriter = (config: Config) => {
};
}

function stringOfVarInt32Slow(v: string) {
function stringOfVarUInt32Slow(v: string) {
const len = strByteLength(v);
const isLatin1 = len === v.length;
dataView.setUint8(cursor++, isLatin1 ? LATIN1 : UTF8);
varInt32(len);
varUInt32(len);
reserve(len);
if (isLatin1) {
if (len < 40) {
Expand All @@ -215,7 +215,15 @@ export const BinaryWriter = (config: Config) => {
cursor += len;
}

function zigZag(v: number) {
return (v << 1) ^ (v >> 31);
}

function varInt32(val: number) {
return varUInt32(zigZag(val));
}

function varUInt32(val: number) {
val = val >>> 0;
while (val > 127) {
arrayBuffer[cursor++] = val & 127 | 128;
Expand Down Expand Up @@ -278,9 +286,10 @@ export const BinaryWriter = (config: Config) => {
uint8,
int16,
varInt32,
stringOfVarInt32: config?.hps
? stringOfVarInt32Fast()
: stringOfVarInt32Slow,
varUInt32,
stringOfVarUInt32: config?.hps
? stringOfVarUInt32Fast()
: stringOfVarUInt32Slow,
bufferWithoutMemCheck,
uint64,
buffer,
Expand Down
Loading

0 comments on commit 53b166b

Please sign in to comment.