From 4b217cfa214105bda177fb813c37f7a6a9e73b01 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 11 Jul 2023 00:12:08 +0300 Subject: [PATCH 1/3] Sped up diffing by several orders of magnitude. Also started on a different more fancy approach to diffing involving boolean operations on object maps --- packages/viewer-sandbox/src/Sandbox.ts | 2 + packages/viewer/package.json | 3 +- packages/viewer/src/modules/Differ.ts | 230 ++++++++++++++++++++++++- yarn.lock | 8 + 4 files changed, 234 insertions(+), 9 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index a23f1d93e4..0be3a6f442 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -1016,6 +1016,8 @@ export default class Sandbox { // bug // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212', // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9', + // 'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc', + // 'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d', VisualDiffMode.COLORED, localStorage.getItem('AuthTokenLatest') as string diff --git a/packages/viewer/package.json b/packages/viewer/package.json index de27b07177..1b9a5e39e9 100644 --- a/packages/viewer/package.json +++ b/packages/viewer/package.json @@ -61,7 +61,8 @@ "three": "^0.140.0", "three-mesh-bvh": "0.5.17", "tree-model": "1.0.7", - "troika-three-text": "0.47.2" + "troika-three-text": "0.47.2", + "underscore": "1.13.6" }, "devDependencies": { "@babel/core": "^7.18.2", diff --git a/packages/viewer/src/modules/Differ.ts b/packages/viewer/src/modules/Differ.ts index 208a6c99b9..d07433d834 100644 --- a/packages/viewer/src/modules/Differ.ts +++ b/packages/viewer/src/modules/Differ.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { Color, FrontSide } from 'three' import { SpeckleTypeAllRenderables } from './converter/GeometryConverter' import SpeckleStandardMaterial from './materials/SpeckleStandardMaterial' @@ -7,6 +8,7 @@ import { GeometryType } from './batching/Batch' import SpeckleLineMaterial from './materials/SpeckleLineMaterial' import Logger from 'js-logger' import { NodeRenderView } from './tree/NodeRenderView' +import _, { omit } from 'underscore' // eslint-disable-next-line @typescript-eslint/no-explicit-any type SpeckleObject = Record @@ -184,7 +186,187 @@ export class Differ { this.removedMaterialPoint.toneMapped = false } + intersection(o1, o2) { + const [k1, k2] = [Object.keys(o1), Object.keys(o2)] + const [first, next] = k1.length > k2.length ? [k2, o1] : [k1, o2] + return first.filter((k) => k in next) + } + public diff(urlA: string, urlB: string): Promise { + const all = performance.now() + // const modifiedNew: Array = [] + // const modifiedOld: Array = [] + + const diffResult: DiffResult = { + unchanged: [], + added: [], + removed: [], + modified: [] + } + + const renderTreeA = this.tree.getRenderTree(urlA) + const renderTreeB = this.tree.getRenderTree(urlB) + // const rootA = this.tree.findId(urlA) + // const rootB = this.tree.findId(urlB) + let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables) + let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables) + + const idMapA = {} + const appIdMapA = {} + rvsA = rvsA.map((value) => { + const atomicRv = renderTreeA.getAtomicParent(value) + + const applicationId = atomicRv.model.raw.applicationId + ? atomicRv.model.raw.applicationId + : this.tree + .getAncestors(atomicRv) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + if (applicationId) { + appIdMapA[applicationId] = 1 + } + idMapA[atomicRv.model.raw.id] = { + node: atomicRv, + applicationId + } + return atomicRv + }) + + const idMapB = {} + const appIdMapB = {} + rvsB = rvsB.map((value) => { + const atomicRv = renderTreeB.getAtomicParent(value) + + const applicationId = atomicRv.model.raw.applicationId + ? atomicRv.model.raw.applicationId + : this.tree + .getAncestors(atomicRv) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + if (applicationId) { + appIdMapB[applicationId] = 1 + } + idMapB[atomicRv.model.raw.id] = { + node: atomicRv, + applicationId + } + return atomicRv + }) + + rvsA = [...Array.from(new Set(rvsA))] + rvsB = [...Array.from(new Set(rvsB))] + + const start = performance.now() + + const unchanged = this.intersection(idMapA, idMapB) + const addedModified = _.omit(_.omit(idMapB, unchanged), Object.keys(idMapA)) + const removedModified = _.omit(_.omit(idMapA, unchanged), Object.keys(idMapB)) + const modifiedA = _.omit(addedModified, function (value, key, object) { + return value.applicationId && appIdMapA[value.applicationId] !== undefined + }) + const modifiedB = _.omit(removedModified, function (value, key, object) { + return value.applicationId && appIdMapB[value.applicationId] !== undefined + }) + // const added = _.omit(addedModified, Object.keys(modifiedA)) + // const removed = _.omit(removedModified, Object.keys(modifiedB)) + + const modifiedOld = Object.values(modifiedA).map( + (value: { node: TreeNode }) => value.node + ) + const modifiedNew = Object.values(modifiedB).map( + (value: { node: TreeNode }) => value.node + ) + diffResult.unchanged.push( + ...Object.values(unchanged).map((value) => idMapA[value].node) + ) + diffResult.removed.push( + ...Object.values(removedModified).map((value: { node: TreeNode }) => value.node) + ) + diffResult.added.push( + ...Object.values(addedModified).map((value: { node: TreeNode }) => value.node) + ) + + console.warn('Boolean ops -> ', performance.now() - start) + console.warn(unchanged) + console.warn(modifiedOld) + console.warn(modifiedNew) + + // start = performance.now() + // for (let k = 0; k < rvsB.length; k++) { + // // const res = rootA.first((node: TreeNode) => { + // // return rvsB[k].model.raw.id === node.model.raw.id + // // }) + // const res = idMapA[rvsB[k].model.raw.id] + + // if (res) { + // diffResult.unchanged.push(res) + // } else { + // const applicationId = rvsB[k].model.raw.applicationId + // ? rvsB[k].model.raw.applicationId + // : this.tree + // .getAncestors(rvsB[k]) + // .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + // if (!applicationId) { + // Logger.error( + // `No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!` + // ) + // diffResult.added.push(rvsB[k]) + // continue + // } + // // const res2 = rootA.first((node: TreeNode) => { + // // return applicationId === node.model.raw.applicationId + // // }) + // const res2 = appIdMapA[applicationId] + // if (res2) { + // modifiedNew.push(rvsB[k]) + // } else { + // diffResult.added.push(rvsB[k]) + // } + // } + // } + // console.log('First pass -> ', performance.now() - start) + // start = performance.now() + // for (let k = 0; k < rvsA.length; k++) { + // // const res = rootB.first((node: TreeNode) => { + // // return rvsA[k].model.raw.id === node.model.raw.id + // // }) + // const res = idMapB[rvsA[k].model.raw.id] + // if (!res) { + // const applicationId = rvsA[k].model.raw.applicationId + // ? rvsA[k].model.raw.applicationId + // : this.tree + // .getAncestors(rvsA[k]) + // .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + // if (!applicationId) { + // Logger.error( + // `No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!` + // ) + // diffResult.removed.push(rvsA[k]) + // continue + // } + // // const res2 = rootB.first((node: TreeNode) => { + // // return applicationId === node.model.raw.applicationId + // // }) + // const res2 = appIdMapB[applicationId] + // if (!res2) { + // diffResult.removed.push(rvsA[k]) + // } else { + // modifiedOld.push(rvsA[k]) + // } + // } else { + // diffResult.unchanged.push(res) + // } + // } + // console.log('Second pass -> ', performance.now() - start) + // modifiedNew.forEach((value, index) => { + // value + // diffResult.modified.push([modifiedOld[index], modifiedNew[index]]) + // }) + console.warn('Total time -> ', performance.now() - all) + console.warn(diffResult) + return Promise.resolve(diffResult) + } + + public diffOld(urlA: string, urlB: string): Promise { + const all = performance.now() const modifiedNew: Array = [] const modifiedOld: Array = [] @@ -202,21 +384,47 @@ export class Differ { let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables) let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables) + const idMapA = {} + const appIdMapA = {} rvsA = rvsA.map((value) => { - return renderTreeA.getAtomicParent(value) + const atomicRv = renderTreeA.getAtomicParent(value) + idMapA[atomicRv.model.raw.id] = atomicRv + const applicationId = atomicRv.model.raw.applicationId + ? atomicRv.model.raw.applicationId + : this.tree + .getAncestors(atomicRv) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + if (applicationId) { + appIdMapA[applicationId] = 1 + } + return atomicRv }) + const idMapB = {} + const appIdMapB = {} rvsB = rvsB.map((value) => { - return renderTreeB.getAtomicParent(value) + const atomicRv = renderTreeB.getAtomicParent(value) + idMapB[atomicRv.model.raw.id] = atomicRv + const applicationId = atomicRv.model.raw.applicationId + ? atomicRv.model.raw.applicationId + : this.tree + .getAncestors(atomicRv) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + if (applicationId) { + appIdMapB[applicationId] = 1 + } + return atomicRv }) rvsA = [...Array.from(new Set(rvsA))] rvsB = [...Array.from(new Set(rvsB))] - + let start = performance.now() for (let k = 0; k < rvsB.length; k++) { const res = rootA.first((node: TreeNode) => { return rvsB[k].model.raw.id === node.model.raw.id }) + // const res = idMapA[rvsB[k].model.raw.id] + if (res) { diffResult.unchanged.push(res) } else { @@ -224,7 +432,7 @@ export class Differ { ? rvsB[k].model.raw.applicationId : this.tree .getAncestors(rvsB[k]) - .find((value) => value.model.raw.applicationId) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId if (!applicationId) { Logger.error( `No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!` @@ -235,6 +443,7 @@ export class Differ { const res2 = rootA.first((node: TreeNode) => { return applicationId === node.model.raw.applicationId }) + // const res2 = appIdMapA[applicationId] if (res2) { modifiedNew.push(rvsB[k]) } else { @@ -242,17 +451,19 @@ export class Differ { } } } - + console.warn('First pass -> ', performance.now() - start) + start = performance.now() for (let k = 0; k < rvsA.length; k++) { const res = rootB.first((node: TreeNode) => { return rvsA[k].model.raw.id === node.model.raw.id }) + // const res = idMapB[rvsA[k].model.raw.id] if (!res) { const applicationId = rvsA[k].model.raw.applicationId ? rvsA[k].model.raw.applicationId : this.tree .getAncestors(rvsA[k]) - .find((value) => value.model.raw.applicationId) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId if (!applicationId) { Logger.error( `No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!` @@ -263,6 +474,7 @@ export class Differ { const res2 = rootB.first((node: TreeNode) => { return applicationId === node.model.raw.applicationId }) + // const res2 = appIdMapB[applicationId] if (!res2) { diffResult.removed.push(rvsA[k]) } else { @@ -272,12 +484,12 @@ export class Differ { diffResult.unchanged.push(res) } } - + console.warn('Second pass -> ', performance.now() - start) modifiedOld.forEach((value, index) => { value diffResult.modified.push([modifiedOld[index], modifiedNew[index]]) }) - + console.warn('Total time -> ', performance.now() - all) console.warn(diffResult) return Promise.resolve(diffResult) } @@ -322,6 +534,7 @@ export class Differ { [id: string]: SpeckleStandardMaterial | SpecklePointMaterial | SpeckleLineMaterial } ) { + const start = performance.now() switch (mode) { case VisualDiffMode.COLORED: this._materialGroups = this.getColoredMaterialGroups( @@ -337,6 +550,7 @@ export class Differ { default: Logger.error(`Unsupported visual diff mode ${mode}`) } + console.warn('Material groups -> ', performance.now() - start) return this._materialGroups } diff --git a/yarn.lock b/yarn.lock index 7d661ff6cb..0b09716057 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11536,6 +11536,7 @@ __metadata: tree-model: 1.0.7 troika-three-text: 0.47.2 typescript: ^4.5.4 + underscore: 1.13.6 languageName: unknown linkType: soft @@ -41698,6 +41699,13 @@ __metadata: languageName: node linkType: hard +"underscore@npm:1.13.6": + version: 1.13.6 + resolution: "underscore@npm:1.13.6" + checksum: d5cedd14a9d0d91dd38c1ce6169e4455bb931f0aaf354108e47bd46d3f2da7464d49b2171a5cf786d61963204a42d01ea1332a903b7342ad428deaafaf70ec36 + languageName: node + linkType: hard + "undici@npm:^5.1.0, undici@npm:^5.12.0, undici@npm:^5.19.1, undici@npm:^5.22.0, undici@npm:^5.8.0": version: 5.22.1 resolution: "undici@npm:5.22.1" From 6122f017a09ba50d1f8fa150842b804ec890b7be Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 11 Jul 2023 16:34:12 +0300 Subject: [PATCH 2/3] Finished with boolean version of diffing. Improved the speed of both by 50% on top of the previous speed improvements --- packages/viewer-sandbox/src/Sandbox.ts | 13 +- packages/viewer-sandbox/src/main.ts | 4 +- packages/viewer/src/modules/Differ.ts | 284 ++++++++----------------- 3 files changed, 98 insertions(+), 203 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 0be3a6f442..cc0cda241d 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -982,16 +982,16 @@ export default class Sandbox { diffResult = await this.viewer.diff( //building // 'https://latest.speckle.dev/streams/aea12cab71/objects/bcf37136dea9fe9397cdfd84012f616a', - // 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867' + // 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867', // cubes // 'https://latest.speckle.dev/streams/aea12cab71/objects/d2510c59c203b73473f8bbfe637e0552', // 'https://latest.speckle.dev/streams/aea12cab71/objects/1c327da824fdb04629eb48675101d7b7', // sketchup // 'https://latest.speckle.dev/streams/aea12cab71/objects/06bed1819e6c61d9df7196d424ab1eec', - // 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef' + // 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef', //latest - 'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472', - 'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8', + // 'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472', + // 'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8', //lines // 'https://latest.speckle.dev/streams/92b620fb17/objects/3b42d6ef51d3110b4e33b9f8cdc9f357', // 'https://latest.speckle.dev/streams/92b620fb17/objects/774384d431fb34d447d4696abbc4b816', @@ -1016,8 +1016,9 @@ export default class Sandbox { // bug // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212', // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9', - // 'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc', - // 'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d', + // large + 'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc', + 'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d', VisualDiffMode.COLORED, localStorage.getItem('AuthTokenLatest') as string diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 393210a3b4..b2596fd845 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -110,7 +110,7 @@ const getStream = () => { // prettier-ignore // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.dev/streams/c1faab5c62/commits/6c6e43e5f3' // 'https://latest.speckle.dev/streams/58b5648c4d/commits/60371ecb2d' // 'Super' heavy revit shit @@ -278,7 +278,7 @@ const getStream = () => { // 'https://latest.speckle.dev/streams/b68abcbf2e/commits/4e94ecad62' // Big ass mafa' // 'https://speckle.xyz/streams/88307505eb/objects/a232d760059046b81ff97e6c4530c985' - // 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d' + 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d' // 'Blocks with elements // 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c' // 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add' diff --git a/packages/viewer/src/modules/Differ.ts b/packages/viewer/src/modules/Differ.ts index d07433d834..b64cb17775 100644 --- a/packages/viewer/src/modules/Differ.ts +++ b/packages/viewer/src/modules/Differ.ts @@ -186,17 +186,41 @@ export class Differ { this.removedMaterialPoint.toneMapped = false } - intersection(o1, o2) { + private intersection(o1, o2) { const [k1, k2] = [Object.keys(o1), Object.keys(o2)] const [first, next] = k1.length > k2.length ? [k2, o1] : [k1, o2] return first.filter((k) => k in next) } + private buildIdMaps( + rvs: Array, + idMap: { [id: string]: { node: TreeNode; applicationId: string } }, + appIdMap: { [id: string]: number } + ) { + for (let k = 0; k < rvs.length; k++) { + const atomicRv = rvs[k] + const applicationId = atomicRv.model.raw.applicationId + ? atomicRv.model.raw.applicationId + : this.tree + .getAncestors(atomicRv) + .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + + idMap[atomicRv.model.raw.id] = { + node: atomicRv, + applicationId + } + if (applicationId) { + appIdMap[applicationId] = 1 + } + } + } + public diff(urlA: string, urlB: string): Promise { - const all = performance.now() - // const modifiedNew: Array = [] - // const modifiedOld: Array = [] + return this.diffIterative(urlA, urlB) + } + private diffBoolean(urlA: string, urlB: string): Promise { + const start = performance.now() const diffResult: DiffResult = { unchanged: [], added: [], @@ -206,167 +230,79 @@ export class Differ { const renderTreeA = this.tree.getRenderTree(urlA) const renderTreeB = this.tree.getRenderTree(urlB) - // const rootA = this.tree.findId(urlA) - // const rootB = this.tree.findId(urlB) let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables) let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables) - const idMapA = {} - const appIdMapA = {} rvsA = rvsA.map((value) => { - const atomicRv = renderTreeA.getAtomicParent(value) - - const applicationId = atomicRv.model.raw.applicationId - ? atomicRv.model.raw.applicationId - : this.tree - .getAncestors(atomicRv) - .find((value) => value.model.raw.applicationId)?.model.raw.applicationId - if (applicationId) { - appIdMapA[applicationId] = 1 - } - idMapA[atomicRv.model.raw.id] = { - node: atomicRv, - applicationId - } - return atomicRv + return renderTreeA.getAtomicParent(value) }) - const idMapB = {} - const appIdMapB = {} rvsB = rvsB.map((value) => { - const atomicRv = renderTreeB.getAtomicParent(value) - - const applicationId = atomicRv.model.raw.applicationId - ? atomicRv.model.raw.applicationId - : this.tree - .getAncestors(atomicRv) - .find((value) => value.model.raw.applicationId)?.model.raw.applicationId - if (applicationId) { - appIdMapB[applicationId] = 1 - } - idMapB[atomicRv.model.raw.id] = { - node: atomicRv, - applicationId - } - return atomicRv + return renderTreeB.getAtomicParent(value) }) rvsA = [...Array.from(new Set(rvsA))] rvsB = [...Array.from(new Set(rvsB))] - const start = performance.now() + const idMapA = {} + const appIdMapA = {} + this.buildIdMaps(rvsA, idMapA, appIdMapA) - const unchanged = this.intersection(idMapA, idMapB) - const addedModified = _.omit(_.omit(idMapB, unchanged), Object.keys(idMapA)) - const removedModified = _.omit(_.omit(idMapA, unchanged), Object.keys(idMapB)) - const modifiedA = _.omit(addedModified, function (value, key, object) { + const idMapB = {} + const appIdMapB = {} + this.buildIdMaps(rvsB, idMapB, appIdMapB) + + /** Get the ids which are common between the two maps. This will be objects + * which have not changed + */ + const unchanged: Array = this.intersection(idMapA, idMapB) + /** We remove the unchanged objects from B and end up with changed + added */ + const addedModified = _.omit(idMapB, unchanged) + /** We remove the unchanged objects from A and end up with changed + removed */ + const removedModified = _.omit(idMapA, unchanged) + /** We remove the changed objects from B. An object from B is changed if + * it's application ID exists in A + */ + const added = _.omit(addedModified, function (value, key, object) { return value.applicationId && appIdMapA[value.applicationId] !== undefined }) - const modifiedB = _.omit(removedModified, function (value, key, object) { + /** We remove the changed objects from A. An object from A is changed if + * it's application ID exists in B + */ + const removed = _.omit(removedModified, function (value, key, object) { return value.applicationId && appIdMapB[value.applicationId] !== undefined }) - // const added = _.omit(addedModified, Object.keys(modifiedA)) - // const removed = _.omit(removedModified, Object.keys(modifiedB)) + /** We remove the removed objects from A, leaving us only changed objects */ + const modifiedRemoved = _.omit(removedModified, Object.keys(removed)) + /** We remove the removed objects from B, leaving us only changed objects */ + const modifiedAdded = _.omit(addedModified, Object.keys(added)) - const modifiedOld = Object.values(modifiedA).map( + /** We fill the arrays from here on out */ + const modifiedOld = Object.values(modifiedRemoved).map( (value: { node: TreeNode }) => value.node ) - const modifiedNew = Object.values(modifiedB).map( + const modifiedNew = Object.values(modifiedAdded).map( (value: { node: TreeNode }) => value.node ) - diffResult.unchanged.push( - ...Object.values(unchanged).map((value) => idMapA[value].node) - ) + diffResult.unchanged.push(...unchanged.map((value) => idMapA[value].node)) + diffResult.unchanged.push(...unchanged.map((value) => idMapB[value].node)) diffResult.removed.push( - ...Object.values(removedModified).map((value: { node: TreeNode }) => value.node) + ...Object.values(removed).map((value: { node: TreeNode }) => value.node) ) diffResult.added.push( - ...Object.values(addedModified).map((value: { node: TreeNode }) => value.node) + ...Object.values(added).map((value: { node: TreeNode }) => value.node) ) - console.warn('Boolean ops -> ', performance.now() - start) - console.warn(unchanged) - console.warn(modifiedOld) - console.warn(modifiedNew) - - // start = performance.now() - // for (let k = 0; k < rvsB.length; k++) { - // // const res = rootA.first((node: TreeNode) => { - // // return rvsB[k].model.raw.id === node.model.raw.id - // // }) - // const res = idMapA[rvsB[k].model.raw.id] - - // if (res) { - // diffResult.unchanged.push(res) - // } else { - // const applicationId = rvsB[k].model.raw.applicationId - // ? rvsB[k].model.raw.applicationId - // : this.tree - // .getAncestors(rvsB[k]) - // .find((value) => value.model.raw.applicationId)?.model.raw.applicationId - // if (!applicationId) { - // Logger.error( - // `No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!` - // ) - // diffResult.added.push(rvsB[k]) - // continue - // } - // // const res2 = rootA.first((node: TreeNode) => { - // // return applicationId === node.model.raw.applicationId - // // }) - // const res2 = appIdMapA[applicationId] - // if (res2) { - // modifiedNew.push(rvsB[k]) - // } else { - // diffResult.added.push(rvsB[k]) - // } - // } - // } - // console.log('First pass -> ', performance.now() - start) - // start = performance.now() - // for (let k = 0; k < rvsA.length; k++) { - // // const res = rootB.first((node: TreeNode) => { - // // return rvsA[k].model.raw.id === node.model.raw.id - // // }) - // const res = idMapB[rvsA[k].model.raw.id] - // if (!res) { - // const applicationId = rvsA[k].model.raw.applicationId - // ? rvsA[k].model.raw.applicationId - // : this.tree - // .getAncestors(rvsA[k]) - // .find((value) => value.model.raw.applicationId)?.model.raw.applicationId - // if (!applicationId) { - // Logger.error( - // `No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!` - // ) - // diffResult.removed.push(rvsA[k]) - // continue - // } - // // const res2 = rootB.first((node: TreeNode) => { - // // return applicationId === node.model.raw.applicationId - // // }) - // const res2 = appIdMapB[applicationId] - // if (!res2) { - // diffResult.removed.push(rvsA[k]) - // } else { - // modifiedOld.push(rvsA[k]) - // } - // } else { - // diffResult.unchanged.push(res) - // } - // } - // console.log('Second pass -> ', performance.now() - start) - // modifiedNew.forEach((value, index) => { - // value - // diffResult.modified.push([modifiedOld[index], modifiedNew[index]]) - // }) - console.warn('Total time -> ', performance.now() - all) - console.warn(diffResult) + modifiedOld.forEach((value, index) => { + value + diffResult.modified.push([modifiedOld[index], modifiedNew[index]]) + }) + console.warn('Boolean Time -> ', performance.now() - start) return Promise.resolve(diffResult) } - public diffOld(urlA: string, urlB: string): Promise { - const all = performance.now() + private diffIterative(urlA: string, urlB: string): Promise { + const start = performance.now() const modifiedNew: Array = [] const modifiedOld: Array = [] @@ -379,60 +315,35 @@ export class Differ { const renderTreeA = this.tree.getRenderTree(urlA) const renderTreeB = this.tree.getRenderTree(urlB) - const rootA = this.tree.findId(urlA) - const rootB = this.tree.findId(urlB) let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables) let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables) - const idMapA = {} - const appIdMapA = {} rvsA = rvsA.map((value) => { - const atomicRv = renderTreeA.getAtomicParent(value) - idMapA[atomicRv.model.raw.id] = atomicRv - const applicationId = atomicRv.model.raw.applicationId - ? atomicRv.model.raw.applicationId - : this.tree - .getAncestors(atomicRv) - .find((value) => value.model.raw.applicationId)?.model.raw.applicationId - if (applicationId) { - appIdMapA[applicationId] = 1 - } - return atomicRv + return renderTreeA.getAtomicParent(value) }) - const idMapB = {} - const appIdMapB = {} rvsB = rvsB.map((value) => { - const atomicRv = renderTreeB.getAtomicParent(value) - idMapB[atomicRv.model.raw.id] = atomicRv - const applicationId = atomicRv.model.raw.applicationId - ? atomicRv.model.raw.applicationId - : this.tree - .getAncestors(atomicRv) - .find((value) => value.model.raw.applicationId)?.model.raw.applicationId - if (applicationId) { - appIdMapB[applicationId] = 1 - } - return atomicRv + return renderTreeB.getAtomicParent(value) }) rvsA = [...Array.from(new Set(rvsA))] rvsB = [...Array.from(new Set(rvsB))] - let start = performance.now() + + const idMapA = {} + const appIdMapA = {} + this.buildIdMaps(rvsA, idMapA, appIdMapA) + + const idMapB = {} + const appIdMapB = {} + this.buildIdMaps(rvsB, idMapB, appIdMapB) + for (let k = 0; k < rvsB.length; k++) { - const res = rootA.first((node: TreeNode) => { - return rvsB[k].model.raw.id === node.model.raw.id - }) - // const res = idMapA[rvsB[k].model.raw.id] + const res = idMapA[rvsB[k].model.raw.id]?.node if (res) { diffResult.unchanged.push(res) } else { - const applicationId = rvsB[k].model.raw.applicationId - ? rvsB[k].model.raw.applicationId - : this.tree - .getAncestors(rvsB[k]) - .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + const applicationId = idMapB[rvsB[k].model.raw.id].applicationId if (!applicationId) { Logger.error( `No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!` @@ -440,10 +351,7 @@ export class Differ { diffResult.added.push(rvsB[k]) continue } - const res2 = rootA.first((node: TreeNode) => { - return applicationId === node.model.raw.applicationId - }) - // const res2 = appIdMapA[applicationId] + const res2 = appIdMapA[applicationId] if (res2) { modifiedNew.push(rvsB[k]) } else { @@ -451,19 +359,10 @@ export class Differ { } } } - console.warn('First pass -> ', performance.now() - start) - start = performance.now() for (let k = 0; k < rvsA.length; k++) { - const res = rootB.first((node: TreeNode) => { - return rvsA[k].model.raw.id === node.model.raw.id - }) - // const res = idMapB[rvsA[k].model.raw.id] + const res = idMapB[rvsA[k].model.raw.id]?.node if (!res) { - const applicationId = rvsA[k].model.raw.applicationId - ? rvsA[k].model.raw.applicationId - : this.tree - .getAncestors(rvsA[k]) - .find((value) => value.model.raw.applicationId)?.model.raw.applicationId + const applicationId = idMapA[rvsA[k].model.raw.id].applicationId if (!applicationId) { Logger.error( `No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!` @@ -471,10 +370,7 @@ export class Differ { diffResult.removed.push(rvsA[k]) continue } - const res2 = rootB.first((node: TreeNode) => { - return applicationId === node.model.raw.applicationId - }) - // const res2 = appIdMapB[applicationId] + const res2 = appIdMapB[applicationId] if (!res2) { diffResult.removed.push(rvsA[k]) } else { @@ -484,13 +380,11 @@ export class Differ { diffResult.unchanged.push(res) } } - console.warn('Second pass -> ', performance.now() - start) modifiedOld.forEach((value, index) => { value diffResult.modified.push([modifiedOld[index], modifiedNew[index]]) }) - console.warn('Total time -> ', performance.now() - all) - console.warn(diffResult) + console.warn('Interative Time -> ', performance.now() - start) return Promise.resolve(diffResult) } From 27b62fbbb37f0aeafaa4f288b8a33d61c404aa7b Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 21 Jul 2023 12:25:14 +0300 Subject: [PATCH 3/3] #1690 Completely transparent objects are ignored during picking via a toggle-able flag in renderer --- packages/viewer-sandbox/src/Sandbox.ts | 3 +++ packages/viewer-sandbox/src/main.ts | 1 + .../viewer/src/modules/SpeckleRenderer.ts | 20 ++++++++++++-- packages/viewer/src/modules/batching/Batch.ts | 1 + .../viewer/src/modules/batching/Batcher.ts | 13 +++++++-- .../viewer/src/modules/batching/LineBatch.ts | 6 +++++ .../viewer/src/modules/batching/PointBatch.ts | 27 +++++++++++++++++++ .../viewer/src/modules/batching/TextBatch.ts | 4 +-- 8 files changed, 68 insertions(+), 7 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index cc0cda241d..e34b982fe3 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -1019,6 +1019,9 @@ export default class Sandbox { // large 'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc', 'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d', + // cubes + // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212', + // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9', VisualDiffMode.COLORED, localStorage.getItem('AuthTokenLatest') as string diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index b2596fd845..778973f081 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -283,6 +283,7 @@ const getStream = () => { // 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c' // 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add' // 'https://latest.speckle.dev/streams/e258b0e8db/commits/c19577c7d6?c=%5B15.88776,-8.2182,12.17095,18.64059,1.48552,0.6025,0,1%5D' + // 'https://speckle.xyz/streams/46caea9b53/commits/71938adcd1' ) } diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 3b259f8743..5666122476 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -77,6 +77,7 @@ export enum ObjectLayers { export default class SpeckleRenderer { private readonly SHOW_HELPERS = false + private readonly IGNORE_ZERO_OPACITY_OBJECTS = true public SHOW_BVH = false private container: HTMLElement private _renderer: WebGLRenderer @@ -890,13 +891,28 @@ export default class SpeckleRenderer { const rvs = [] const points = [] for (let k = 0; k < results.length; k++) { - let rv = results[k].batchObject?.renderView - if (!rv) { + const batchObject = results[k].batchObject + let rv = null + if (batchObject) { + rv = batchObject.renderView + const material = (results[k].object as SpeckleMesh).getBatchObjectMaterial( + results[k].batchObject + ) + if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue + } else { rv = this.batcher.getRenderView( results[k].object.uuid, results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index ) + if (rv) { + const material = this.batcher.getRenderViewMaterial( + results[k].object.uuid, + results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index + ) + if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue + } } + if (rv) { rvs.push(rv) points.push(results[k].point) diff --git a/packages/viewer/src/modules/batching/Batch.ts b/packages/viewer/src/modules/batching/Batch.ts index 16cd863a14..9ea3063016 100644 --- a/packages/viewer/src/modules/batching/Batch.ts +++ b/packages/viewer/src/modules/batching/Batch.ts @@ -31,6 +31,7 @@ export interface Batch { resetDrawRanges() buildBatch() getRenderView(index: number): NodeRenderView + getMaterialAtIndex(index: number): Material onUpdate(deltaTime: number) onRender(renderer: WebGLRenderer) purge() diff --git a/packages/viewer/src/modules/batching/Batcher.ts b/packages/viewer/src/modules/batching/Batcher.ts index dd10dae515..b360a2ffb0 100644 --- a/packages/viewer/src/modules/batching/Batcher.ts +++ b/packages/viewer/src/modules/batching/Batcher.ts @@ -460,6 +460,15 @@ export default class Batcher { return this.batches[batchId].getRenderView(index) } + public getRenderViewMaterial(batchId: string, index: number) { + if (!this.batches[batchId]) { + Logger.error('Invalid batch id!') + return null + } + + return this.batches[batchId].getMaterialAtIndex(index) + } + public resetBatchesDrawRanges() { for (const k in this.batches) { this.batches[k].resetDrawRanges() @@ -576,7 +585,7 @@ export default class Batcher { if (k !== rv.batchId) { this.batches[k].setDrawRanges({ offset: 0, - count: Infinity, + count: this.batches[k].getCount(), material: this.materials.getFilterMaterial( this.batches[k].renderViews[0], FilterMaterialType.GHOST @@ -591,7 +600,7 @@ export default class Batcher { if (k !== batchId) { this.batches[k].setDrawRanges({ offset: 0, - count: Infinity, + count: this.batches[k].getCount(), material: this.materials.getFilterMaterial( this.batches[k].renderViews[0], FilterMaterialType.GHOST diff --git a/packages/viewer/src/modules/batching/LineBatch.ts b/packages/viewer/src/modules/batching/LineBatch.ts index 5e64ea5c2e..23085fa79f 100644 --- a/packages/viewer/src/modules/batching/LineBatch.ts +++ b/packages/viewer/src/modules/batching/LineBatch.ts @@ -5,6 +5,7 @@ import { InstancedInterleavedBuffer, InterleavedBufferAttribute, Line, + Material, Object3D, Vector4, WebGLRenderer @@ -253,6 +254,11 @@ export default class LineBatch implements Batch { } } + public getMaterialAtIndex(index: number): Material { + index + return this.batchMaterial + } + private makeLineGeometry(position: Float64Array) { this.geometry = this.makeLineGeometryTriangle(new Float32Array(position)) Geometry.updateRTEGeometry(this.geometry, position) diff --git a/packages/viewer/src/modules/batching/PointBatch.ts b/packages/viewer/src/modules/batching/PointBatch.ts index 879ab2ac50..c05143600a 100644 --- a/packages/viewer/src/modules/batching/PointBatch.ts +++ b/packages/viewer/src/modules/batching/PointBatch.ts @@ -17,6 +17,7 @@ import { } from './Batch' import { GeometryConverter } from '../converter/GeometryConverter' import { ObjectLayers } from '../SpeckleRenderer' +import Logger from 'js-logger' export default class PointBatch implements Batch { public id: string @@ -353,6 +354,32 @@ export default class PointBatch implements Batch { } } + public getMaterialAtIndex(index: number): Material { + for (let k = 0; k < this.renderViews.length; k++) { + if ( + index >= this.renderViews[k].batchStart && + index < this.renderViews[k].batchEnd + ) { + const rv = this.renderViews[k] + const group = this.geometry.groups.find((value) => { + return ( + rv.batchStart >= value.start && + rv.batchStart + rv.batchCount <= value.count + value.start + ) + }) + if (!Array.isArray(this.mesh.material)) { + return this.mesh.material + } else { + if (!group) { + Logger.warn(`Malformed material index!`) + return null + } + return this.mesh.material[group.materialIndex] + } + } + } + } + private makePointGeometry( position: Float64Array, color: Float32Array diff --git a/packages/viewer/src/modules/batching/TextBatch.ts b/packages/viewer/src/modules/batching/TextBatch.ts index 63c127c207..a14721b11f 100644 --- a/packages/viewer/src/modules/batching/TextBatch.ts +++ b/packages/viewer/src/modules/batching/TextBatch.ts @@ -131,9 +131,7 @@ export default class TextBatch implements Batch { } public getMaterialAtIndex(index: number): Material { - index - console.warn('Deprecated! Do not call this anymore') - return null + return this.batchMaterial } public purge() {