Alex/#1678 faster diff

packages/viewer-sandbox/src/Sandbox.ts

@@ -982,16 +982,16 @@ export default class Sandbox {
       diffResult = await this.viewer.diff(
         //building
         // 'https://latest.speckle.dev/streams/aea12cab71/objects/bcf37136dea9fe9397cdfd84012f616a',
-        // 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867'
+        // 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867',
         // cubes
         // 'https://latest.speckle.dev/streams/aea12cab71/objects/d2510c59c203b73473f8bbfe637e0552',
         // 'https://latest.speckle.dev/streams/aea12cab71/objects/1c327da824fdb04629eb48675101d7b7',
         // sketchup
         // 'https://latest.speckle.dev/streams/aea12cab71/objects/06bed1819e6c61d9df7196d424ab1eec',
-        // 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef'
+        // 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef',
         //latest
-        'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472',
-        'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8',
+        // 'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472',
+        // 'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8',
         //lines
         // 'https://latest.speckle.dev/streams/92b620fb17/objects/3b42d6ef51d3110b4e33b9f8cdc9f357',
         // 'https://latest.speckle.dev/streams/92b620fb17/objects/774384d431fb34d447d4696abbc4b816',
@@ -1016,6 +1016,12 @@ export default class Sandbox {
         // bug
         // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212',
         // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9',
+        // large
+        'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc',
+        'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d',
+        // cubes
+        // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212',
+        // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9',
 
         VisualDiffMode.COLORED,
         localStorage.getItem('AuthTokenLatest') as string

packages/viewer-sandbox/src/main.ts

@@ -110,7 +110,7 @@ const getStream = () => {
     // prettier-ignore
     // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
     // Revit sample house (good for bim-like stuff with many display meshes)
-    'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
+    // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.dev/streams/c1faab5c62/commits/6c6e43e5f3'
     // 'https://latest.speckle.dev/streams/58b5648c4d/commits/60371ecb2d'
     // 'Super' heavy revit shit
@@ -278,11 +278,12 @@ const getStream = () => {
     // 'https://latest.speckle.dev/streams/b68abcbf2e/commits/4e94ecad62'
     // Big ass mafa'
     // 'https://speckle.xyz/streams/88307505eb/objects/a232d760059046b81ff97e6c4530c985'
-    // 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
+    'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
     // 'Blocks with elements
     // 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c'
     // 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add'
     // 'https://latest.speckle.dev/streams/e258b0e8db/commits/c19577c7d6?c=%5B15.88776,-8.2182,12.17095,18.64059,1.48552,0.6025,0,1%5D'
+    // 'https://speckle.xyz/streams/46caea9b53/commits/71938adcd1'
   )
 }
 

packages/viewer/package.json

@@ -61,7 +61,8 @@
     "three": "^0.140.0",
     "three-mesh-bvh": "0.5.17",
     "tree-model": "1.0.7",
-    "troika-three-text": "0.47.2"
+    "troika-three-text": "0.47.2",
+    "underscore": "1.13.6"
   },
   "devDependencies": {
     "@babel/core": "^7.18.2",

packages/viewer/src/modules/Differ.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
 import { Color, FrontSide } from 'three'
 import { SpeckleTypeAllRenderables } from './converter/GeometryConverter'
 import SpeckleStandardMaterial from './materials/SpeckleStandardMaterial'
@@ -7,6 +8,7 @@ import { GeometryType } from './batching/Batch'
 import SpeckleLineMaterial from './materials/SpeckleLineMaterial'
 import Logger from 'js-logger'
 import { NodeRenderView } from './tree/NodeRenderView'
+import _, { omit } from 'underscore'
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type SpeckleObject = Record<string, any>
@@ -184,7 +186,123 @@ export class Differ {
     this.removedMaterialPoint.toneMapped = false
   }
 
+  private intersection(o1, o2) {
+    const [k1, k2] = [Object.keys(o1), Object.keys(o2)]
+    const [first, next] = k1.length > k2.length ? [k2, o1] : [k1, o2]
+    return first.filter((k) => k in next)
+  }
+
+  private buildIdMaps(
+    rvs: Array<TreeNode>,
+    idMap: { [id: string]: { node: TreeNode; applicationId: string } },
+    appIdMap: { [id: string]: number }
+  ) {
+    for (let k = 0; k < rvs.length; k++) {
+      const atomicRv = rvs[k]
+      const applicationId = atomicRv.model.raw.applicationId
+        ? atomicRv.model.raw.applicationId
+        : this.tree
+            .getAncestors(atomicRv)
+            .find((value) => value.model.raw.applicationId)?.model.raw.applicationId
+
+      idMap[atomicRv.model.raw.id] = {
+        node: atomicRv,
+        applicationId
+      }
+      if (applicationId) {
+        appIdMap[applicationId] = 1
+      }
+    }
+  }
+
   public diff(urlA: string, urlB: string): Promise<DiffResult> {
+    return this.diffIterative(urlA, urlB)
+  }
+
+  private diffBoolean(urlA: string, urlB: string): Promise<DiffResult> {
+    const start = performance.now()
+    const diffResult: DiffResult = {
+      unchanged: [],
+      added: [],
+      removed: [],
+      modified: []
+    }
+
+    const renderTreeA = this.tree.getRenderTree(urlA)
+    const renderTreeB = this.tree.getRenderTree(urlB)
+    let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables)
+    let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables)
+
+    rvsA = rvsA.map((value) => {
+      return renderTreeA.getAtomicParent(value)
+    })
+
+    rvsB = rvsB.map((value) => {
+      return renderTreeB.getAtomicParent(value)
+    })
+
+    rvsA = [...Array.from(new Set(rvsA))]
+    rvsB = [...Array.from(new Set(rvsB))]
+
+    const idMapA = {}
+    const appIdMapA = {}
+    this.buildIdMaps(rvsA, idMapA, appIdMapA)
+
+    const idMapB = {}
+    const appIdMapB = {}
+    this.buildIdMaps(rvsB, idMapB, appIdMapB)
+
+    /** Get the ids which are common between the two maps. This will be objects
+     *  which have not changed
+     */
+    const unchanged: Array<string> = this.intersection(idMapA, idMapB)
+    /** We remove the unchanged objects from B and end up with changed + added */
+    const addedModified = _.omit(idMapB, unchanged)
+    /** We remove the unchanged objects from A and end up with changed + removed */
+    const removedModified = _.omit(idMapA, unchanged)
+    /** We remove the changed objects from B. An object from B is changed if
+     *  it's application ID exists in A
+     */
+    const added = _.omit(addedModified, function (value, key, object) {
+      return value.applicationId && appIdMapA[value.applicationId] !== undefined
+    })
+    /** We remove the changed objects from A. An object from A is changed if
+     *  it's application ID exists in B
+     */
+    const removed = _.omit(removedModified, function (value, key, object) {
+      return value.applicationId && appIdMapB[value.applicationId] !== undefined
+    })
+    /** We remove the removed objects from A, leaving us only changed objects */
+    const modifiedRemoved = _.omit(removedModified, Object.keys(removed))
+    /** We remove the removed objects from B, leaving us only changed objects */
+    const modifiedAdded = _.omit(addedModified, Object.keys(added))
+
+    /** We fill the arrays from here on out */
+    const modifiedOld = Object.values(modifiedRemoved).map(
+      (value: { node: TreeNode }) => value.node
+    )
+    const modifiedNew = Object.values(modifiedAdded).map(
+      (value: { node: TreeNode }) => value.node
+    )
+    diffResult.unchanged.push(...unchanged.map((value) => idMapA[value].node))
+    diffResult.unchanged.push(...unchanged.map((value) => idMapB[value].node))
+    diffResult.removed.push(
+      ...Object.values(removed).map((value: { node: TreeNode }) => value.node)
+    )
+    diffResult.added.push(
+      ...Object.values(added).map((value: { node: TreeNode }) => value.node)
+    )
+
+    modifiedOld.forEach((value, index) => {
+      value
+      diffResult.modified.push([modifiedOld[index], modifiedNew[index]])
+    })
+    console.warn('Boolean Time -> ', performance.now() - start)
+    return Promise.resolve(diffResult)
+  }
+
+  private diffIterative(urlA: string, urlB: string): Promise<DiffResult> {
+    const start = performance.now()
     const modifiedNew: Array<SpeckleObject> = []
     const modifiedOld: Array<SpeckleObject> = []
 
@@ -197,8 +315,6 @@ export class Differ {
 
     const renderTreeA = this.tree.getRenderTree(urlA)
     const renderTreeB = this.tree.getRenderTree(urlB)
-    const rootA = this.tree.findId(urlA)
-    const rootB = this.tree.findId(urlB)
     let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables)
     let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables)
 
@@ -213,56 +329,48 @@ export class Differ {
     rvsA = [...Array.from(new Set(rvsA))]
     rvsB = [...Array.from(new Set(rvsB))]
 
+    const idMapA = {}
+    const appIdMapA = {}
+    this.buildIdMaps(rvsA, idMapA, appIdMapA)
+
+    const idMapB = {}
+    const appIdMapB = {}
+    this.buildIdMaps(rvsB, idMapB, appIdMapB)
+
     for (let k = 0; k < rvsB.length; k++) {
-      const res = rootA.first((node: TreeNode) => {
-        return rvsB[k].model.raw.id === node.model.raw.id
-      })
+      const res = idMapA[rvsB[k].model.raw.id]?.node
+
       if (res) {
         diffResult.unchanged.push(res)
       } else {
-        const applicationId = rvsB[k].model.raw.applicationId
-          ? rvsB[k].model.raw.applicationId
-          : this.tree
-              .getAncestors(rvsB[k])
-              .find((value) => value.model.raw.applicationId)
+        const applicationId = idMapB[rvsB[k].model.raw.id].applicationId
         if (!applicationId) {
           Logger.error(
             `No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!`
           )
           diffResult.added.push(rvsB[k])
           continue
         }
-        const res2 = rootA.first((node: TreeNode) => {
-          return applicationId === node.model.raw.applicationId
-        })
+        const res2 = appIdMapA[applicationId]
         if (res2) {
           modifiedNew.push(rvsB[k])
         } else {
           diffResult.added.push(rvsB[k])
         }
       }
     }
-
     for (let k = 0; k < rvsA.length; k++) {
-      const res = rootB.first((node: TreeNode) => {
-        return rvsA[k].model.raw.id === node.model.raw.id
-      })
+      const res = idMapB[rvsA[k].model.raw.id]?.node
       if (!res) {
-        const applicationId = rvsA[k].model.raw.applicationId
-          ? rvsA[k].model.raw.applicationId
-          : this.tree
-              .getAncestors(rvsA[k])
-              .find((value) => value.model.raw.applicationId)
+        const applicationId = idMapA[rvsA[k].model.raw.id].applicationId
         if (!applicationId) {
           Logger.error(
             `No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!`
           )
           diffResult.removed.push(rvsA[k])
           continue
         }
-        const res2 = rootB.first((node: TreeNode) => {
-          return applicationId === node.model.raw.applicationId
-        })
+        const res2 = appIdMapB[applicationId]
         if (!res2) {
           diffResult.removed.push(rvsA[k])
         } else {
@@ -272,13 +380,11 @@ export class Differ {
         diffResult.unchanged.push(res)
       }
     }
-
     modifiedOld.forEach((value, index) => {
       value
       diffResult.modified.push([modifiedOld[index], modifiedNew[index]])
     })
-
-    console.warn(diffResult)
+    console.warn('Interative Time -> ', performance.now() - start)
     return Promise.resolve(diffResult)
   }
 
@@ -322,6 +428,7 @@ export class Differ {
       [id: string]: SpeckleStandardMaterial | SpecklePointMaterial | SpeckleLineMaterial
     }
   ) {
+    const start = performance.now()
     switch (mode) {
       case VisualDiffMode.COLORED:
         this._materialGroups = this.getColoredMaterialGroups(
@@ -337,6 +444,7 @@ export class Differ {
       default:
         Logger.error(`Unsupported visual diff mode ${mode}`)
     }
+    console.warn('Material groups -> ', performance.now() - start)
     return this._materialGroups
   }
 

packages/viewer/src/modules/SpeckleRenderer.ts

@@ -77,6 +77,7 @@ export enum ObjectLayers {
 
 export default class SpeckleRenderer {
   private readonly SHOW_HELPERS = false
+  private readonly IGNORE_ZERO_OPACITY_OBJECTS = true
   public SHOW_BVH = false
   private container: HTMLElement
   private _renderer: WebGLRenderer
@@ -890,13 +891,28 @@ export default class SpeckleRenderer {
     const rvs = []
     const points = []
     for (let k = 0; k < results.length; k++) {
-      let rv = results[k].batchObject?.renderView
-      if (!rv) {
+      const batchObject = results[k].batchObject
+      let rv = null
+      if (batchObject) {
+        rv = batchObject.renderView
+        const material = (results[k].object as SpeckleMesh).getBatchObjectMaterial(
+          results[k].batchObject
+        )
+        if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue
+      } else {
         rv = this.batcher.getRenderView(
           results[k].object.uuid,
           results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index
         )
+        if (rv) {
+          const material = this.batcher.getRenderViewMaterial(
+            results[k].object.uuid,
+            results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index
+          )
+          if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue
+        }
       }
+
       if (rv) {
         rvs.push(rv)
         points.push(results[k].point)

packages/viewer/src/modules/batching/Batch.ts

@@ -31,6 +31,7 @@ export interface Batch {
   resetDrawRanges()
   buildBatch()
   getRenderView(index: number): NodeRenderView
+  getMaterialAtIndex(index: number): Material
   onUpdate(deltaTime: number)
   onRender(renderer: WebGLRenderer)
   purge()

packages/viewer/src/modules/batching/Batcher.ts

@@ -460,6 +460,15 @@ export default class Batcher {
     return this.batches[batchId].getRenderView(index)
   }
 
+  public getRenderViewMaterial(batchId: string, index: number) {
+    if (!this.batches[batchId]) {
+      Logger.error('Invalid batch id!')
+      return null
+    }
+
+    return this.batches[batchId].getMaterialAtIndex(index)
+  }
+
   public resetBatchesDrawRanges() {
     for (const k in this.batches) {
       this.batches[k].resetDrawRanges()
@@ -576,7 +585,7 @@ export default class Batcher {
       if (k !== rv.batchId) {
         this.batches[k].setDrawRanges({
           offset: 0,
-          count: Infinity,
+          count: this.batches[k].getCount(),
           material: this.materials.getFilterMaterial(
             this.batches[k].renderViews[0],
             FilterMaterialType.GHOST
@@ -591,7 +600,7 @@ export default class Batcher {
       if (k !== batchId) {
         this.batches[k].setDrawRanges({
           offset: 0,
-          count: Infinity,
+          count: this.batches[k].getCount(),
           material: this.materials.getFilterMaterial(
             this.batches[k].renderViews[0],
             FilterMaterialType.GHOST