From a9939d6ac39a41416f857db8c225c087cca98e0c Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 6 Feb 2024 21:32:06 +0100 Subject: [PATCH] Volume rendering compositing changes and wireframe rendering to vis chunks in volume rendering mode (#503) * feat: VR front-to-back compositing and chunk vis * chore: add comment about opacity correction source * refactor: clearer naming for compositing * chore: fix formatting * feat: add gain parameter to VR and python binds The gain float can be used to scale the VR intensity. Also bind that gain to Python controls and bind VR bool to Python. * refactor: pull OIT emit function out as standalone * feat: perform OIT along rays during VR * test: ensure that color and revealage are independent of sampling rate Refactored volume rendering shader to accomodate * feat: add depth culling to VR rays during marching * chore: format file * fix: break composite loop if VR ray goes behind opaque * fix: remove console.log during testing * feat: volume rendering chunk vis in wireframe mode * refactor: rename gain to volumeRenderingGain Also update relevant Python bindings * feat: change gain scale from linear to exponential * refactor: rename to depthBufferTexture (remove ID) * format: run python formatting * fix: default volume rendering slider gain of 0 (e^0 passed to shader) --- python/neuroglancer/__init__.py | 2 + python/neuroglancer/viewer_state.py | 18 +++ src/image_user_layer.ts | 18 +++ src/perspective_view/panel.ts | 16 ++- src/perspective_view/render_layer.ts | 5 + .../volume_render_layer.spec.ts | 73 ++++++++++++ src/volume_rendering/volume_render_layer.ts | 109 +++++++++++++++--- 7 files changed, 221 insertions(+), 20 deletions(-) create mode 100644 src/volume_rendering/volume_render_layer.spec.ts diff --git a/python/neuroglancer/__init__.py b/python/neuroglancer/__init__.py index 996270585..5275b5c87 100644 --- a/python/neuroglancer/__init__.py +++ b/python/neuroglancer/__init__.py @@ -48,6 +48,8 @@ PlaceEllipsoidTool, # noqa: F401 BlendTool, # noqa: F401 OpacityTool, # noqa: F401 + VolumeRenderingTool, # noqa: F401 + VolumeRenderingGainTool, # noqa: F401 VolumeRenderingDepthSamplesTool, # noqa: F401 CrossSectionRenderScaleTool, # noqa: F401 SelectedAlphaTool, # noqa: F401 diff --git a/python/neuroglancer/viewer_state.py b/python/neuroglancer/viewer_state.py index eea97d763..7431a300f 100644 --- a/python/neuroglancer/viewer_state.py +++ b/python/neuroglancer/viewer_state.py @@ -164,6 +164,18 @@ class OpacityTool(Tool): TOOL_TYPE = "opacity" +@export_tool +class VolumeRenderingTool(Tool): + __slots__ = () + TOOL_TYPE = "volumeRendering" + + +@export_tool +class VolumeRenderingGainTool(Tool): + __slots__ = () + TOOL_TYPE = "volumeRenderingGain" + + @export_tool class VolumeRenderingDepthSamplesTool(Tool): __slots__ = () @@ -543,6 +555,12 @@ def __init__(self, *args, **kwargs): ) opacity = wrapped_property("opacity", optional(float, 0.5)) blend = wrapped_property("blend", optional(str)) + volume_rendering = volumeRendering = wrapped_property( + "volumeRendering", optional(bool, False) + ) + volume_rendering_gain = volumeRenderingGain = wrapped_property( + "volumeRenderingGain", optional(float, 1) + ) volume_rendering_depth_samples = volumeRenderingDepthSamples = wrapped_property( "volumeRenderingDepthSamples", optional(float, 64) ) diff --git a/src/image_user_layer.ts b/src/image_user_layer.ts index 8e1544a0d..8e4237a4e 100644 --- a/src/image_user_layer.ts +++ b/src/image_user_layer.ts @@ -98,6 +98,7 @@ import { ShaderControls, } from "#/widget/shader_controls"; import { Tab } from "#/widget/tab_view"; +import { trackableFiniteFloat } from "#/trackable_finite_float"; const OPACITY_JSON_KEY = "opacity"; const BLEND_JSON_KEY = "blend"; @@ -106,6 +107,7 @@ const SHADER_CONTROLS_JSON_KEY = "shaderControls"; const CROSS_SECTION_RENDER_SCALE_JSON_KEY = "crossSectionRenderScale"; const CHANNEL_DIMENSIONS_JSON_KEY = "channelDimensions"; const VOLUME_RENDERING_JSON_KEY = "volumeRendering"; +const VOLUME_RENDERING_GAIN_JSON_KEY = "volumeRenderingGain"; const VOLUME_RENDERING_DEPTH_SAMPLES_JSON_KEY = "volumeRenderingDepthSamples"; export interface ImageLayerSelectionState extends UserLayerSelectionState { @@ -125,6 +127,7 @@ export class ImageUserLayer extends Base { dataType = new WatchableValue(undefined); sliceViewRenderScaleHistogram = new RenderScaleHistogram(); sliceViewRenderScaleTarget = trackableRenderScaleTarget(1); + volumeRenderingGain = trackableFiniteFloat(0); volumeRenderingChunkResolutionHistogram = new RenderScaleHistogram( volumeRenderingDepthSamplesOriginLogScale, ); @@ -199,6 +202,7 @@ export class ImageUserLayer extends Base { isLocalDimension; this.blendMode.changed.add(this.specificationChanged.dispatch); this.opacity.changed.add(this.specificationChanged.dispatch); + this.volumeRenderingGain.changed.add(this.specificationChanged.dispatch); this.fragmentMain.changed.add(this.specificationChanged.dispatch); this.shaderControlState.changed.add(this.specificationChanged.dispatch); this.sliceViewRenderScaleTarget.changed.add( @@ -252,6 +256,7 @@ export class ImageUserLayer extends Base { ); const volumeRenderLayer = context.registerDisposer( new VolumeRenderingRenderLayer({ + gain: this.volumeRenderingGain, multiscaleSource: volume, shaderControlState: this.shaderControlState, shaderError: this.shaderError, @@ -299,6 +304,9 @@ export class ImageUserLayer extends Base { specification[CHANNEL_DIMENSIONS_JSON_KEY], ); this.volumeRendering.restoreState(specification[VOLUME_RENDERING_JSON_KEY]); + this.volumeRenderingGain.restoreState( + specification[VOLUME_RENDERING_GAIN_JSON_KEY], + ); this.volumeRenderingDepthSamplesTarget.restoreState( specification[VOLUME_RENDERING_DEPTH_SAMPLES_JSON_KEY], ); @@ -313,6 +321,7 @@ export class ImageUserLayer extends Base { this.sliceViewRenderScaleTarget.toJSON(); x[CHANNEL_DIMENSIONS_JSON_KEY] = this.channelCoordinateSpace.toJSON(); x[VOLUME_RENDERING_JSON_KEY] = this.volumeRendering.toJSON(); + x[VOLUME_RENDERING_GAIN_JSON_KEY] = this.volumeRenderingGain.toJSON(); x[VOLUME_RENDERING_DEPTH_SAMPLES_JSON_KEY] = this.volumeRenderingDepthSamplesTarget.toJSON(); return x; @@ -451,6 +460,15 @@ const LAYER_CONTROLS: LayerControlDefinition[] = [ toolJson: VOLUME_RENDERING_JSON_KEY, ...checkboxLayerControl((layer) => layer.volumeRendering), }, + { + label: "Gain (3D)", + toolJson: VOLUME_RENDERING_GAIN_JSON_KEY, + isValid: (layer) => layer.volumeRendering, + ...rangeLayerControl((layer) => ({ + value: layer.volumeRenderingGain, + options: { min: -10.0, max: 10.0, step: 0.1 }, + })), + }, { label: "Resolution (3D)", toolJson: VOLUME_RENDERING_DEPTH_SAMPLES_JSON_KEY, diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 811a2292c..0157f53e8 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -109,22 +109,26 @@ void emit(vec4 color, highp uint pickId) { * http://casual-effects.blogspot.com/2015/03/implemented-weighted-blended-order.html */ export const glsl_computeOITWeight = ` -float computeOITWeight(float alpha) { +float computeOITWeight(float alpha, float depth) { float a = min(1.0, alpha) * 8.0 + 0.01; - float b = -gl_FragCoord.z * 0.95 + 1.0; + float b = -depth * 0.95 + 1.0; return a * a * a * b * b * b; } `; // Color must be premultiplied by alpha. +// Can use emitAccumAndRevealage() to emit a pre-weighted OIT result. export const glsl_perspectivePanelEmitOIT = [ glsl_computeOITWeight, ` +void emitAccumAndRevealage(vec4 accum, float revealage, highp uint pickId) { + v4f_fragData0 = vec4(accum.rgb, revealage); + v4f_fragData1 = vec4(accum.a, 0.0, 0.0, 0.0); +} void emit(vec4 color, highp uint pickId) { - float weight = computeOITWeight(color.a); + float weight = computeOITWeight(color.a, gl_FragCoord.z); vec4 accum = color * weight; - v4f_fragData0 = vec4(accum.rgb, color.a); - v4f_fragData1 = vec4(accum.a, 0.0, 0.0, 0.0); + emitAccumAndRevealage(accum, color.a, pickId); } `, ]; @@ -846,6 +850,8 @@ export class PerspectivePanel extends RenderedDataPanel { renderContext.emitPickID = false; for (const [renderLayer, attachment] of visibleLayers) { if (renderLayer.isTransparent) { + renderContext.depthBufferTexture = + this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture; renderLayer.draw(renderContext, attachment); } } diff --git a/src/perspective_view/render_layer.ts b/src/perspective_view/render_layer.ts index 16af1b097..9b6a3b67a 100644 --- a/src/perspective_view/render_layer.ts +++ b/src/perspective_view/render_layer.ts @@ -50,6 +50,11 @@ export interface PerspectiveViewRenderContext * Specifies whether there was a previous pick ID pass. */ alreadyEmittedPickID: boolean; + + /** + * Specifies the ID of the depth frame buffer texture to query during rendering. + */ + depthBufferTexture?: WebGLTexture | null; } export class PerspectiveViewRenderLayer< diff --git a/src/volume_rendering/volume_render_layer.spec.ts b/src/volume_rendering/volume_render_layer.spec.ts new file mode 100644 index 000000000..6b87e965b --- /dev/null +++ b/src/volume_rendering/volume_render_layer.spec.ts @@ -0,0 +1,73 @@ +import { fragmentShaderTest } from "#/webgl/shader_testing"; +import { glsl_computeOITWeight } from "#/perspective_view/panel"; +import { glsl_emitRGBAVolumeRendering } from "#/volume_rendering/volume_render_layer"; +import { vec3 } from "gl-matrix"; + +describe("volume rendering compositing", () => { + const steps = [16, 22, 32, 37, 64, 100, 128, 256, 512, 551, 1024, 2048]; + const revealages = new Float32Array(steps.length); + it("combines uniform colors the same regardless of sampling rate", () => { + fragmentShaderTest( + { + inputSteps: "float", + }, + { + outputValue1: "float", + outputValue2: "float", + outputValue3: "float", + outputValue4: "float", + revealage: "float", + }, + (tester) => { + const { builder } = tester; + builder.addFragmentCode(glsl_computeOITWeight); + builder.addFragmentCode(` +vec4 color = vec4(0.1, 0.3, 0.5, 0.1); +float idealSamplingRate = 512.0; +float uGain = 0.01; +float uBrightnessFactor; +vec4 outputColor; +float depthAtRayPosition; +`); + builder.addFragmentCode(glsl_emitRGBAVolumeRendering); + builder.setFragmentMain(` +outputColor = vec4(0.0); +revealage = 1.0; +uBrightnessFactor = idealSamplingRate / inputSteps; +for (int i = 0; i < int(inputSteps); ++i) { + depthAtRayPosition = mix(0.0, 1.0, float(i) / (inputSteps - 1.0)); + emitRGBA(color); +} +outputValue1 = outputColor.r; +outputValue2 = outputColor.g; +outputValue3 = outputColor.b; +outputValue4 = outputColor.a; +`); + for (let i = 0; i < steps.length; ++i) { + const inputSteps = steps[i]; + tester.execute({ inputSteps }); + const values = tester.values; + const { + revealage, + outputValue1, + outputValue2, + outputValue3, + outputValue4, + } = values; + const color = vec3.fromValues( + outputValue1 / outputValue4, + outputValue2 / outputValue4, + outputValue3 / outputValue4, + ); + expect(color[0]).toBeCloseTo(0.1, 5); + expect(color[1]).toBeCloseTo(0.3, 5); + expect(color[2]).toBeCloseTo(0.5, 5); + revealages[i] = revealage; + } + for (let i = 1; i < revealages.length; ++i) { + expect(revealages[i]).toBeCloseTo(revealages[i - 1], 2); + } + }, + ); + }); +}); diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 14536d220..42d817614 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -83,6 +83,17 @@ export const VOLUME_RENDERING_DEPTH_SAMPLES_DEFAULT_VALUE = 64; const VOLUME_RENDERING_DEPTH_SAMPLES_LOG_SCALE_ORIGIN = 1; const VOLUME_RENDERING_RESOLUTION_INDICATOR_BAR_HEIGHT = 10; +const depthSamplerTextureUnit = Symbol("depthSamplerTextureUnit"); + +export const glsl_emitRGBAVolumeRendering = ` +void emitRGBA(vec4 rgba) { + float correctedAlpha = clamp(rgba.a * uBrightnessFactor * uGain, 0.0, 1.0); + float weightedAlpha = correctedAlpha * computeOITWeight(correctedAlpha, depthAtRayPosition); + outputColor += vec4(rgba.rgb * weightedAlpha, weightedAlpha); + revealage *= 1.0 - correctedAlpha; +} +`; + type TransformedVolumeSource = FrontendTransformedSource< SliceViewRenderLayer, VolumeChunkSource @@ -93,6 +104,7 @@ interface VolumeRenderingAttachmentState { } export interface VolumeRenderingRenderLayerOptions { + gain: WatchableValueInterface; multiscaleSource: MultiscaleVolumeChunkSource; transform: WatchableValueInterface; shaderError: WatchableShaderError; @@ -128,6 +140,7 @@ function clampAndRoundResolutionTargetValue(value: number) { } export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { + gain: WatchableValueInterface; multiscaleSource: MultiscaleVolumeChunkSource; transform: WatchableValueInterface; channelCoordinateSpace: WatchableValueInterface; @@ -139,7 +152,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { private vertexIdHelper: VertexIdHelper; private shaderGetter: ParameterizedContextDependentShaderGetter< - { emitter: ShaderModule; chunkFormat: ChunkFormat }, + { emitter: ShaderModule; chunkFormat: ChunkFormat; wireFrame: boolean }, ShaderControlsBuilderState, number >; @@ -158,6 +171,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { constructor(options: VolumeRenderingRenderLayerOptions) { super(); + this.gain = options.gain; this.multiscaleSource = options.multiscaleSource; this.transform = options.transform; this.channelCoordinateSpace = options.channelCoordinateSpace; @@ -180,13 +194,13 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { { memoizeKey: "VolumeRenderingRenderLayer", parameters: options.shaderControlState.builderState, - getContextKey: ({ emitter, chunkFormat }) => - `${getObjectId(emitter)}:${chunkFormat.shaderKey}`, + getContextKey: ({ emitter, chunkFormat, wireFrame }) => + `${getObjectId(emitter)}:${chunkFormat.shaderKey}:${wireFrame}`, shaderError: options.shaderError, extraParameters: numChannelDimensions, defineShader: ( builder, - { emitter, chunkFormat }, + { emitter, chunkFormat, wireFrame }, shaderBuilderState, numChannelDimensions, ) => { @@ -214,12 +228,19 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { // Chunk size in voxels. builder.addUniform("highp vec3", "uChunkDataSize"); + builder.addUniform("highp float", "uChunkNumber"); builder.addUniform("highp vec3", "uLowerClipBound"); builder.addUniform("highp vec3", "uUpperClipBound"); builder.addUniform("highp float", "uBrightnessFactor"); + builder.addUniform("highp float", "uGain"); builder.addVarying("highp vec4", "vNormalizedPosition"); + builder.addTextureSampler( + "sampler2D", + "uDepthSampler", + depthSamplerTextureUnit, + ); builder.addVertexCode(glsl_getBoxFaceVertexPosition); builder.setVertexMain(` @@ -230,7 +251,9 @@ gl_Position.z = 0.0; `); builder.addFragmentCode(` vec3 curChunkPosition; +float depthAtRayPosition; vec4 outputColor; +float revealage; void userMain(); `); defineChunkDataShaderAccess( @@ -239,22 +262,37 @@ void userMain(); numChannelDimensions, "curChunkPosition", ); - builder.addFragmentCode(` -void emitRGBA(vec4 rgba) { - float alpha = rgba.a * uBrightnessFactor; - outputColor += vec4(rgba.rgb * alpha, alpha); -} + builder.addFragmentCode([ + glsl_emitRGBAVolumeRendering, + ` void emitRGB(vec3 rgb) { emitRGBA(vec4(rgb, 1.0)); } void emitGrayscale(float value) { - emitRGB(vec3(value, value, value)); + emitRGBA(vec4(value, value, value, value)); } void emitTransparent() { emitRGBA(vec4(0.0, 0.0, 0.0, 0.0)); } +float computeDepthFromClipSpace(vec4 clipSpacePosition) { + float NDCDepthCoord = clipSpacePosition.z / clipSpacePosition.w; + return (NDCDepthCoord + 1.0) * 0.5; +} +vec2 computeUVFromClipSpace(vec4 clipSpacePosition) { + vec2 NDCPosition = clipSpacePosition.xy / clipSpacePosition.w; + return (NDCPosition + 1.0) * 0.5; +} +`, + ]); + if (wireFrame) { + builder.setFragmentMainFunction(` +void main() { + outputColor = vec4(uChunkNumber, uChunkNumber, uChunkNumber, 1.0); + emit(outputColor, 0u); +} `); - builder.setFragmentMainFunction(` + } else { + builder.setFragmentMainFunction(` void main() { vec2 normalizedPosition = vNormalizedPosition.xy / vNormalizedPosition.w; vec4 nearPointH = uInvModelViewProjectionMatrix * vec4(normalizedPosition, -1.0, 1.0); @@ -291,14 +329,24 @@ void main() { int startStep = int(floor((intersectStart - uNearLimitFraction) / stepSize)); int endStep = min(uMaxSteps, int(floor((intersectEnd - uNearLimitFraction) / stepSize)) + 1); outputColor = vec4(0, 0, 0, 0); + revealage = 1.0; for (int step = startStep; step < endStep; ++step) { vec3 position = mix(nearPoint, farPoint, uNearLimitFraction + float(step) * stepSize); + vec4 clipSpacePosition = uModelViewProjectionMatrix * vec4(position, 1.0); + depthAtRayPosition = computeDepthFromClipSpace(clipSpacePosition); + vec2 uv = computeUVFromClipSpace(clipSpacePosition); + float depthInBuffer = texture(uDepthSampler, uv).r; + bool rayPositionBehindOpaqueObject = (1.0 - depthAtRayPosition) < depthInBuffer; + if (rayPositionBehindOpaqueObject) { + break; + } curChunkPosition = position - uTranslation; userMain(); } - emit(outputColor, 0u); + emitAccumAndRevealage(outputColor, 1.0 - revealage, 0u); } `); + } builder.addFragmentCode(glsl_COLORMAPS); addControlsToBuilder(shaderBuilderState, builder); builder.addFragmentCode( @@ -314,6 +362,7 @@ void main() { this.registerDisposer( this.depthSamplesTarget.changed.add(this.redrawNeeded.dispatch), ); + this.registerDisposer(this.gain.changed.add(this.redrawNeeded.dispatch)); this.registerDisposer( this.shaderControlState.changed.add(this.redrawNeeded.dispatch), ); @@ -434,6 +483,9 @@ void main() { if (prevChunkFormat !== null) { prevChunkFormat!.endDrawing(gl, shader); } + const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit); + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + depthTextureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); if (presentCount !== 0 || notPresentCount !== 0) { let index = curHistogramInformation.spatialScales.size - 1; const alreadyStoredSamples = new Set([ @@ -475,6 +527,7 @@ void main() { let presentCount = 0; let notPresentCount = 0; let chunkDataSize: Uint32Array | undefined; + let chunkNumber = 1; const chunkRank = this.multiscaleSource.rank; const chunkPosition = vec3.create(); @@ -517,6 +570,7 @@ void main() { shaderResult = this.shaderGetter({ emitter: renderContext.emitter, chunkFormat: chunkFormat!, + wireFrame: renderContext.wireFrame, }); shader = shaderResult.shader; if (shader !== null) { @@ -528,6 +582,25 @@ void main() { this.shaderControlState, shaderResult.parameters.parseResult.controls, ); + if ( + renderContext.depthBufferTexture !== undefined && + renderContext.depthBufferTexture !== null + ) { + const depthTextureUnit = shader.textureUnit( + depthSamplerTextureUnit, + ); + gl.activeTexture( + WebGL2RenderingContext.TEXTURE0 + depthTextureUnit, + ); + gl.bindTexture( + WebGL2RenderingContext.TEXTURE_2D, + renderContext.depthBufferTexture, + ); + } else { + throw new Error( + "Depth buffer texture ID for volume rendering is undefined or null", + ); + } chunkFormat.beginDrawing(gl, shader); chunkFormat.beginSource(gl, shader); } @@ -564,14 +637,15 @@ void main() { transformedSource.lowerClipDisplayBound, transformedSource.upperClipDisplayBound, ); - const step = - (adjustedFar - adjustedNear) / (this.depthSamplesTarget.value - 1); - const brightnessFactor = step / (far - near); + const optimalSampleRate = optimalSamples; + const actualSampleRate = this.depthSamplesTarget.value; + const brightnessFactor = optimalSampleRate / actualSampleRate; gl.uniform1f(shader.uniform("uBrightnessFactor"), brightnessFactor); const nearLimitFraction = (adjustedNear - near) / (far - near); const farLimitFraction = (adjustedFar - near) / (far - near); gl.uniform1f(shader.uniform("uNearLimitFraction"), nearLimitFraction); gl.uniform1f(shader.uniform("uFarLimitFraction"), farLimitFraction); + gl.uniform1f(shader.uniform("uGain"), Math.exp(this.gain.value)); gl.uniform1i( shader.uniform("uMaxSteps"), this.depthSamplesTarget.value, @@ -597,6 +671,11 @@ void main() { fixedPositionWithinChunk, chunkTransform: { channelToChunkDimensionIndices }, } = transformedSource; + if (renderContext.wireFrame) { + const normChunkNumber = chunkNumber / chunks.size; + gl.uniform1f(shader.uniform("uChunkNumber"), normChunkNumber); + ++chunkNumber; + } if (newChunkDataSize !== chunkDataSize) { chunkDataSize = newChunkDataSize;