View Modes

packages/frontend-2/components/viewer/gendo/Panel.vue

@@ -76,7 +76,14 @@ const timeOutWait = ref(false)
 
 const enqueMagic = async () => {
   isLoading.value = true
-  const screenshot = await viewerInstance.getExtension(PassReader).read()
+  const [depthData, width, height] = await viewerInstance
+    .getExtension(PassReader)
+    .read('DEPTH')
+  const screenshot = PassReader.toBase64(
+    PassReader.decodeDepth(depthData),
+    width,
+    height
+  )
   void lodgeRequest(screenshot)
 
   timeOutWait.value = true

packages/frontend-2/lib/viewer/composables/setup.ts

@@ -66,6 +66,7 @@ import {
 import { useSynchronizedCookie } from '~~/lib/common/composables/reactiveCookie'
 import { buildManualPromise } from '@speckle/ui-components'
 import { PassReader } from '../extensions/PassReader'
+import { ViewModesKeys } from '../extensions/ViewModesKeys'
 
 export type LoadedModel = NonNullable<
   Get<ViewerLoadedResourcesQuery, 'project.models.items[0]'>
@@ -337,6 +338,7 @@ function createViewerDataBuilder(params: { viewerDebug: boolean }) {
       verbose: !!(import.meta.client && params.viewerDebug)
     })
     viewer.createExtension(PassReader)
+    viewer.createExtension(ViewModesKeys)
     const initPromise = viewer.init()
 
     return {

packages/frontend-2/lib/viewer/extensions/PassReader.ts

@@ -1,26 +1,31 @@
-import { type SpecklePass } from '@speckle/viewer'
+import type { SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
-import type SpeckleRenderer from '@speckle/viewer/dist/modules/SpeckleRenderer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
 
 export class PassReader extends Extension {
   private outputBuffer: Uint8ClampedArray = new Uint8ClampedArray()
-  private renderTarget: WebGLRenderTarget | undefined = undefined
+  private renderTarget: WebGLRenderTarget | null = null
   private needsRead: boolean = false
-  private readbackExecutor: ((arg: string) => void) | null = null
+  private readbackExecutor:
+    | ((arg: [Uint8ClampedArray, number, number]) => void)
+    | null = null
 
-  public async read(): Promise<string> {
-    return new Promise<string>((resolve, reject) => {
+  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+    return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
 
-      const dephPass: SpecklePass = renderer.pipeline.composer
-        .passes[0] as unknown as SpecklePass
-      // o_0
+      const depthPass = renderer.pipeline.getPass(passName)[0]
+
+      if (!depthPass) {
+        reject(`Pipeline does not have a ${passName} pass`)
+        return
+      }
+
+      this.renderTarget = depthPass.outputTarget
 
-      this.renderTarget = dephPass.outputRenderTarget
       if (!this.renderTarget) {
-        reject('Issue with depth pass render target')
+        reject('Pass does not have a render target assigned')
         return
       }
 
@@ -34,7 +39,7 @@ export class PassReader extends Extension {
   }
 
   public onRender(): void {
-    if (!this.needsRead || this.renderTarget === undefined) return
+    if (!this.needsRead || !this.renderTarget) return
 
     this.viewer
       .getRenderer()
@@ -46,6 +51,17 @@ export class PassReader extends Extension {
         this.renderTarget.height,
         this.outputBuffer
       )
+
+    if (this.readbackExecutor)
+      this.readbackExecutor([
+        this.outputBuffer,
+        this.renderTarget.width,
+        this.renderTarget.height
+      ])
+    this.needsRead = false
+  }
+
+  public static decodeDepth(buffer: Uint8ClampedArray): Uint8ClampedArray {
     const UnpackDownscale = 255 / 256
     const PackFactors = new Vector3(256 * 256 * 256, 256 * 256, 256)
     const UnpackFactors = new Vector4(
@@ -56,31 +72,39 @@ export class PassReader extends Extension {
     )
 
     const v4 = new Vector4()
-    for (let k = 0; k < this.outputBuffer.length; k += 4) {
+    for (let k = 0; k < buffer.length; k += 4) {
       v4.set(
-        this.outputBuffer[k] / 255,
-        this.outputBuffer[k + 1] / 255,
-        this.outputBuffer[k + 2] / 255,
-        this.outputBuffer[k + 3] / 255
+        buffer[k] / 255,
+        buffer[k + 1] / 255,
+        buffer[k + 2] / 255,
+        buffer[k + 3] / 255
       )
       const res = v4.dot(UnpackFactors)
-      this.outputBuffer[k] = res * 255
-      this.outputBuffer[k + 1] = res * 255
-      this.outputBuffer[k + 2] = res * 255
-      this.outputBuffer[k + 3] = 255
+      buffer[k] = res * 255
+      buffer[k + 1] = res * 255
+      buffer[k + 2] = res * 255
+      buffer[k + 3] = 255
     }
 
+    return buffer
+  }
+
+  public static toBase64(
+    buffer: Uint8ClampedArray,
+    width: number,
+    height: number
+  ): string {
     const canvas = document.createElement('canvas')
     const ctx = canvas.getContext('2d')
-    if (!ctx) return
-    canvas.width = this.renderTarget.width
-    canvas.height = this.renderTarget.height
+    if (!ctx) return ''
+    canvas.width = width
+    canvas.height = height
 
     // create imageData object
-    const idata = ctx.createImageData(this.renderTarget.width, this.renderTarget.height)
+    const idata = ctx.createImageData(width, height)
 
     // set our buffer as source
-    idata.data.set(this.outputBuffer)
+    idata.data.set(buffer)
 
     // update canvas with new data
     ctx.putImageData(idata, 0, 0)
@@ -89,10 +113,9 @@ export class PassReader extends Extension {
      */
     ctx.globalCompositeOperation = 'copy'
     ctx.scale(1, -1)
-    ctx.drawImage(canvas, 0, 0, this.renderTarget.width, -this.renderTarget.height)
+    ctx.drawImage(canvas, 0, 0, width, -height)
     ctx.restore()
 
-    if (this.readbackExecutor) this.readbackExecutor(canvas.toDataURL())
-    this.needsRead = false
+    return canvas.toDataURL()
   }
 }

packages/frontend-2/lib/viewer/extensions/ViewModesKeys.ts

@@ -0,0 +1,48 @@
+import {
+  Extension,
+  InputEvent,
+  ViewMode,
+  ViewModes,
+  type IViewer
+} from '@speckle/viewer'
+
+export class ViewModesKeys extends Extension {
+  public get inject() {
+    return [ViewModes]
+  }
+
+  constructor(viewer: IViewer, protected viewModes: ViewModes) {
+    super(viewer)
+    const renderer = viewer.getRenderer()
+
+    renderer.input.on(InputEvent.KeyUp, (arg: KeyboardEvent) => {
+      // Dont trigger on inputs, textareas or contenteditable elements
+      // We should handle this more gracefully but it works for now
+      if (
+        arg.target &&
+        ((arg.target as HTMLElement).tagName.toLowerCase() === 'input' ||
+          (arg.target as HTMLElement).tagName.toLowerCase() === 'textarea' ||
+          (arg.target as HTMLElement).getAttribute('contenteditable') === 'true')
+      )
+        return
+
+      switch (arg.key) {
+        case '1':
+          viewModes.setViewMode(ViewMode.DEFAULT)
+          break
+        case '2':
+          viewModes.setViewMode(ViewMode.DEFAULT_EDGES)
+          break
+        case '3':
+          viewModes.setViewMode(ViewMode.SHADED)
+          break
+        case '4':
+          viewModes.setViewMode(ViewMode.PEN)
+          break
+        case '5':
+          viewModes.setViewMode(ViewMode.ARCTIC)
+          break
+      }
+    })
+  }
+}

packages/viewer-sandbox/src/Extensions/PassReader.ts

@@ -0,0 +1,117 @@
+import type { SpeckleRenderer } from '@speckle/viewer'
+import { Extension } from '@speckle/viewer'
+import type { WebGLRenderTarget } from 'three'
+import { Vector3, Vector4 } from 'three'
+
+export class PassReader extends Extension {
+  private outputBuffer: Uint8ClampedArray = new Uint8ClampedArray()
+  private renderTarget: WebGLRenderTarget | null = null
+  private needsRead: boolean = false
+  private readbackExecutor:
+    | ((arg: [Uint8ClampedArray, number, number]) => void)
+    | null = null
+
+  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+    return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
+      const renderer: SpeckleRenderer = this.viewer.getRenderer()
+
+      const depthPass = renderer.pipeline.getPass(passName)[0]
+
+      if (!depthPass) {
+        reject(`Pipeline does not have a ${passName} pass`)
+        return
+      }
+
+      this.renderTarget = depthPass.outputTarget
+
+      if (!this.renderTarget) {
+        reject('Pass does not have a render target assigned')
+        return
+      }
+
+      const bufferSize =
+        Math.floor(this.renderTarget.width) * Math.floor(this.renderTarget.height) * 4
+      if (this.outputBuffer.length !== bufferSize)
+        this.outputBuffer = new Uint8ClampedArray(bufferSize)
+      this.needsRead = true
+      this.readbackExecutor = resolve
+    })
+  }
+
+  public onRender(): void {
+    if (!this.needsRead || !this.renderTarget) return
+
+    this.viewer
+      .getRenderer()
+      .renderer.readRenderTargetPixels(
+        this.renderTarget,
+        0,
+        0,
+        this.renderTarget.width,
+        this.renderTarget.height,
+        this.outputBuffer
+      )
+
+    if (this.readbackExecutor)
+      this.readbackExecutor([
+        this.outputBuffer,
+        this.renderTarget.width,
+        this.renderTarget.height
+      ])
+    this.needsRead = false
+  }
+
+  public static decodeDepth(buffer: Uint8ClampedArray): Uint8ClampedArray {
+    const UnpackDownscale = 255 / 256
+    const PackFactors = new Vector3(256 * 256 * 256, 256 * 256, 256)
+    const UnpackFactors = new Vector4(
+      UnpackDownscale / PackFactors.x,
+      UnpackDownscale / PackFactors.y,
+      UnpackDownscale / PackFactors.z,
+      1
+    )
+
+    const v4 = new Vector4()
+    for (let k = 0; k < buffer.length; k += 4) {
+      v4.set(
+        buffer[k] / 255,
+        buffer[k + 1] / 255,
+        buffer[k + 2] / 255,
+        buffer[k + 3] / 255
+      )
+      const res = v4.dot(UnpackFactors)
+      buffer[k] = res * 255
+      buffer[k + 1] = res * 255
+      buffer[k + 2] = res * 255
+      buffer[k + 3] = 255
+    }
+
+    return buffer
+  }
+
+  public static toBase64(buffer: Uint8ClampedArray, width: number, height: number) {
+    const canvas = document.createElement('canvas')
+    const ctx = canvas.getContext('2d')
+    if (!ctx) return
+    canvas.width = width
+    canvas.height = height
+
+    // create imageData object
+    const idata = ctx.createImageData(width, height)
+
+    // set our buffer as source
+    idata.data.set(buffer)
+
+    // update canvas with new data
+    ctx.putImageData(idata, 0, 0)
+    ctx.save()
+    /** Flipping the image by drawing it on itself
+     */
+    ctx.globalCompositeOperation = 'copy'
+    ctx.scale(1, -1)
+    ctx.drawImage(canvas, 0, 0, width, -height)
+    ctx.restore()
+
+    return canvas.toDataURL()
+  }
+}

packages/viewer-sandbox/src/Extensions/ViewModesKeys.ts

@@ -0,0 +1,37 @@
+import {
+  Extension,
+  InputEvent,
+  ViewMode,
+  ViewModes,
+  type IViewer
+} from '@speckle/viewer'
+
+export class ViewModesKeys extends Extension {
+  public get inject() {
+    return [ViewModes]
+  }
+
+  constructor(viewer: IViewer, protected viewModes: ViewModes) {
+    super(viewer)
+    const renderer = viewer.getRenderer()
+    renderer.input.on(InputEvent.KeyUp, (arg: KeyboardEvent) => {
+      switch (arg.key) {
+        case '1':
+          viewModes.setViewMode(ViewMode.DEFAULT)
+          break
+        case '2':
+          viewModes.setViewMode(ViewMode.DEFAULT_EDGES)
+          break
+        case '3':
+          viewModes.setViewMode(ViewMode.SHADED)
+          break
+        case '4':
+          viewModes.setViewMode(ViewMode.PEN)
+          break
+        case '5':
+          viewModes.setViewMode(ViewMode.ARCTIC)
+          break
+      }
+    })
+  }
+}