Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WebXR: Implement antialiased multiview using OCULUS_multiview #15

Merged
merged 4 commits into from
Oct 20, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/webxr_xr_ballshooter.html
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@

//

renderer = new THREE.WebGLRenderer( { antialias: true } );
renderer = new THREE.WebGLRenderer( { antialias: true, multiviewStereo: true } );
renderer.setPixelRatio( window.devicePixelRatio );
renderer.setSize( window.innerWidth, window.innerHeight );
renderer.setAnimationLoop( render );
Expand Down
35 changes: 35 additions & 0 deletions src/renderers/WebGLMultiviewRenderTarget.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/**
* @author fernandojsg / http://fernandojsg.com
* @author Takahiro https://github.com/takahirox
*/

import { WebGLRenderTarget } from './WebGLRenderTarget.js';

class WebGLMultiviewRenderTarget extends WebGLRenderTarget {

constructor( width, height, numViews, options = {} ) {

super( width, height, options );

this.depthBuffer = false;
this.stencilBuffer = false;

this.numViews = numViews;

}

copy( source ) {

super.copy( source );

this.numViews = source.numViews;

return this;

}

}

WebGLMultiviewRenderTarget.prototype.isWebGLMultiviewRenderTarget = true;

export { WebGLMultiviewRenderTarget };
77 changes: 57 additions & 20 deletions src/renderers/WebGLRenderer.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import { WebGLGeometries } from './webgl/WebGLGeometries.js';
import { WebGLIndexedBufferRenderer } from './webgl/WebGLIndexedBufferRenderer.js';
import { WebGLInfo } from './webgl/WebGLInfo.js';
import { WebGLMorphtargets } from './webgl/WebGLMorphtargets.js';
import { WebGLMultiview } from './webgl/WebGLMultiview.js';
import { WebGLObjects } from './webgl/WebGLObjects.js';
import { WebGLPrograms } from './webgl/WebGLPrograms.js';
import { WebGLProperties } from './webgl/WebGLProperties.js';
Expand Down Expand Up @@ -79,6 +80,7 @@ class WebGLRenderer {
preserveDrawingBuffer = false,
powerPreference = 'default',
failIfMajorPerformanceCaveat = false,
multiviewStereo = false,
} = parameters;

this.isWebGLRenderer = true;
Expand Down Expand Up @@ -313,6 +315,7 @@ class WebGLRenderer {
let extensions, capabilities, state, info;
let properties, textures, cubemaps, cubeuvmaps, attributes, geometries, objects;
let programCache, materials, renderLists, renderStates, clipping, shadowMap;
let multiview;

let background, morphtargets, bufferRenderer, indexedBufferRenderer;

Expand Down Expand Up @@ -346,6 +349,7 @@ class WebGLRenderer {
renderLists = new WebGLRenderLists();
renderStates = new WebGLRenderStates( extensions, capabilities );
background = new WebGLBackground( _this, cubemaps, cubeuvmaps, state, objects, _alpha, premultipliedAlpha );
multiview = new WebGLMultiview( _this, extensions, _gl );
shadowMap = new WebGLShadowMap( _this, objects, capabilities );
uniformsGroups = new WebGLUniformsGroups( _gl, info, capabilities, state );

Expand All @@ -368,7 +372,7 @@ class WebGLRenderer {

// xr

const xr = ( typeof navigator !== 'undefined' && 'xr' in navigator ) ? new WebXRManager( _this, _gl ) : new WebVRManager( _this );
const xr = ( typeof navigator !== 'undefined' && 'xr' in navigator ) ? new WebXRManager( _this, _gl, extensions, multiviewStereo ) : new WebVRManager( _this );

this.xr = xr;

Expand Down Expand Up @@ -1115,13 +1119,23 @@ class WebGLRenderer {

if ( camera.isArrayCamera ) {

const cameras = camera.cameras;
if ( xr.enabled && xr.isMultiview ) {

for ( let i = 0, l = cameras.length; i < l; i ++ ) {
textures.setDeferTextureUploads( true );

const camera2 = cameras[ i ];
renderScene( currentRenderList, scene, camera, camera.cameras[ 0 ].viewport );

renderScene( currentRenderList, scene, camera2, camera2.viewport );
} else {

const cameras = camera.cameras;

for ( let i = 0, l = cameras.length; i < l; i ++ ) {

const camera2 = cameras[ i ];

renderScene( currentRenderList, scene, camera2, camera2.viewport );

}

}

Expand Down Expand Up @@ -1149,6 +1163,8 @@ class WebGLRenderer {

if ( scene.isScene === true ) scene.onAfterRender( _this, scene, camera );

textures.runDeferredUploads();

if ( xr.enabled && xr.submitFrame ) {

xr.submitFrame();
Expand Down Expand Up @@ -1616,6 +1632,7 @@ class WebGLRenderer {
materialProperties.vertexAlphas = parameters.vertexAlphas;
materialProperties.vertexTangents = parameters.vertexTangents;
materialProperties.toneMapping = parameters.toneMapping;
materialProperties.numMultiviewViews = parameters.numMultiviewViews;

}

Expand Down Expand Up @@ -1647,6 +1664,8 @@ class WebGLRenderer {

}

const numMultiviewViews = _currentRenderTarget && _currentRenderTarget.isWebGLMultiviewRenderTarget ? _currentRenderTarget.numViews : 0;

const morphAttribute = geometry.morphAttributes.position || geometry.morphAttributes.normal || geometry.morphAttributes.color;
const morphTargetsCount = ( morphAttribute !== undefined ) ? morphAttribute.length : 0;

Expand Down Expand Up @@ -1750,6 +1769,10 @@ class WebGLRenderer {

needsProgramChange = true;

} else if ( materialProperties.numMultiviewViews !== numMultiviewViews ) {

needsProgramChange = true;

}

} else {
Expand Down Expand Up @@ -1796,8 +1819,17 @@ class WebGLRenderer {

// common camera uniforms

p_uniforms.setValue( _gl, 'projectionMatrix', camera.projectionMatrix );
p_uniforms.setValue( _gl, 'viewMatrix', camera.matrixWorldInverse );
if ( program.numMultiviewViews > 0 ) {

multiview.updateCameraProjectionMatricesUniform( camera, p_uniforms );
multiview.updateCameraViewMatricesUniform( camera, p_uniforms );

} else {

p_uniforms.setValue( _gl, 'projectionMatrix', camera.projectionMatrix );
p_uniforms.setValue( _gl, 'viewMatrix', camera.matrixWorldInverse );

}

const uCamPos = p_uniforms.map.cameraPosition;

Expand Down Expand Up @@ -1945,8 +1977,17 @@ class WebGLRenderer {

// common matrices

p_uniforms.setValue( _gl, 'modelViewMatrix', object.modelViewMatrix );
p_uniforms.setValue( _gl, 'normalMatrix', object.normalMatrix );
if ( program.numMultiviewViews > 0 ) {

multiview.updateObjectMatricesUniforms( object, camera, p_uniforms );

} else {

p_uniforms.setValue( _gl, 'modelViewMatrix', object.modelViewMatrix );
p_uniforms.setValue( _gl, 'normalMatrix', object.normalMatrix );

}

p_uniforms.setValue( _gl, 'modelMatrix', object.matrixWorld );

// UBOs
Expand Down Expand Up @@ -2056,20 +2097,16 @@ class WebGLRenderer {
const renderTargetProperties = properties.get( renderTarget );
renderTargetProperties.__hasExternalTextures = true;

if ( renderTargetProperties.__hasExternalTextures ) {
renderTargetProperties.__autoAllocateDepthBuffer = depthTexture === undefined;

renderTargetProperties.__autoAllocateDepthBuffer = depthTexture === undefined;
if ( ! renderTargetProperties.__autoAllocateDepthBuffer && ! _currentRenderTarget.isWebGLMultiviewRenderTarget ) {

if ( ! renderTargetProperties.__autoAllocateDepthBuffer ) {
// The multisample_render_to_texture extension doesn't work properly if there
// are midframe flushes and an external depth buffer. Disable use of the extension.
if ( extensions.has( 'WEBGL_multisampled_render_to_texture' ) === true ) {

// The multisample_render_to_texture extension doesn't work properly if there
// are midframe flushes and an external depth buffer. Disable use of the extension.
if ( extensions.has( 'WEBGL_multisampled_render_to_texture' ) === true ) {

console.warn( 'THREE.WebGLRenderer: Render-to-texture extension was disabled because an external texture was provided' );
renderTargetProperties.__useRenderToTexture = false;

}
console.warn( 'THREE.WebGLRenderer: Render-to-texture extension was disabled because an external texture was provided' );
renderTargetProperties.__useRenderToTexture = false;

}

Expand Down
2 changes: 1 addition & 1 deletion src/renderers/webgl/WebGLBackground.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ function WebGLBackground( renderer, cubemaps, cubeuvmaps, state, objects, alpha,
if ( boxMesh === undefined ) {

boxMesh = new Mesh(
new BoxGeometry( 1, 1, 1 ),
new BoxGeometry( 10000, 10000, 10000 ),
Copy link
Member

@dmarcos dmarcos Oct 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this needed related to multiview?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to @snagy: the WebGLBackground code has some special casing and is not multiview-aware, the original dimension of 1x1x1 becomes really obvious as a box around your head with multiview, and making the box really big effectively negates this effect.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. concern is that this might affect non-vr (2D) applications perhaps causing the background to be clipped? 10,000 is 10km

new ShaderMaterial( {
name: 'BackgroundCubeMaterial',
uniforms: cloneUniforms( ShaderLib.backgroundCube.uniforms ),
Expand Down
100 changes: 100 additions & 0 deletions src/renderers/webgl/WebGLMultiview.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/**
* @author fernandojsg / http://fernandojsg.com
* @author Takahiro https://github.com/takahirox
*/
import { Matrix3 } from '../../math/Matrix3.js';
import { Matrix4 } from '../../math/Matrix4.js';

class WebGLMultiview {

constructor( renderer, extensions, gl ) {

this.renderer = renderer;

this.DEFAULT_NUMVIEWS = 2;
this.maxNumViews = 0;
this.gl = gl;

this.extensions = extensions;

this.available = this.extensions.has( 'OCULUS_multiview' );

if ( this.available ) {

const extension = this.extensions.get( 'OCULUS_multiview' );

this.maxNumViews = this.gl.getParameter( extension.MAX_VIEWS_OVR );

this.mat4 = [];
this.mat3 = [];
this.cameraArray = [];

for ( var i = 0; i < this.maxNumViews; i ++ ) {

this.mat4[ i ] = new Matrix4();
this.mat3[ i ] = new Matrix3();

}

}

}

//
getCameraArray( camera ) {

if ( camera.isArrayCamera ) return camera.cameras;

this.cameraArray[ 0 ] = camera;

return this.cameraArray;

}

updateCameraProjectionMatricesUniform( camera, uniforms ) {

var cameras = this.getCameraArray( camera );

for ( var i = 0; i < cameras.length; i ++ ) {

this.mat4[ i ].copy( cameras[ i ].projectionMatrix );

}

uniforms.setValue( this.gl, 'projectionMatrices', this.mat4 );

}

updateCameraViewMatricesUniform( camera, uniforms ) {

var cameras = this.getCameraArray( camera );

for ( var i = 0; i < cameras.length; i ++ ) {

this.mat4[ i ].copy( cameras[ i ].matrixWorldInverse );

}

uniforms.setValue( this.gl, 'viewMatrices', this.mat4 );

}

updateObjectMatricesUniforms( object, camera, uniforms ) {

var cameras = this.getCameraArray( camera );

for ( var i = 0; i < cameras.length; i ++ ) {

this.mat4[ i ].multiplyMatrices( cameras[ i ].matrixWorldInverse, object.matrixWorld );
this.mat3[ i ].getNormalMatrix( this.mat4[ i ] );

}

uniforms.setValue( this.gl, 'modelViewMatrices', this.mat4 );
uniforms.setValue( this.gl, 'normalMatrices', this.mat3 );

}

}

export { WebGLMultiview };
50 changes: 50 additions & 0 deletions src/renderers/webgl/WebGLProgram.js
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,8 @@ function WebGLProgram( renderer, cacheKey, parameters, bindingStates ) {
let prefixVertex, prefixFragment;
let versionString = parameters.glslVersion ? '#version ' + parameters.glslVersion + '\n' : '';

const numMultiviewViews = parameters.numMultiviewViews;

if ( parameters.isRawShaderMaterial ) {

prefixVertex = [
Expand Down Expand Up @@ -861,6 +863,53 @@ function WebGLProgram( renderer, cacheKey, parameters, bindingStates ) {
'#define textureCubeGradEXT textureGrad'
].join( '\n' ) + '\n' + prefixFragment;

// Multiview

if ( numMultiviewViews > 0 ) {

// TODO: fix light transforms here?

prefixVertex = [
'#extension GL_OVR_multiview : require',
'layout(num_views = ' + numMultiviewViews + ') in;',
'#define VIEW_ID gl_ViewID_OVR'
].join( '\n' ) + '\n' + prefixVertex;

prefixVertex = prefixVertex.replace(
[
'uniform mat4 modelViewMatrix;',
'uniform mat4 projectionMatrix;',
'uniform mat4 viewMatrix;',
'uniform mat3 normalMatrix;'
].join( '\n' ),
[
'uniform mat4 modelViewMatrices[' + numMultiviewViews + '];',
'uniform mat4 projectionMatrices[' + numMultiviewViews + '];',
'uniform mat4 viewMatrices[' + numMultiviewViews + '];',
'uniform mat3 normalMatrices[' + numMultiviewViews + '];',

'#define modelViewMatrix modelViewMatrices[VIEW_ID]',
'#define projectionMatrix projectionMatrices[VIEW_ID]',
'#define viewMatrix viewMatrices[VIEW_ID]',
'#define normalMatrix normalMatrices[VIEW_ID]'
].join( '\n' )
);

prefixFragment = [
'#extension GL_OVR_multiview : require',
'#define VIEW_ID gl_ViewID_OVR'
].join( '\n' ) + '\n' + prefixFragment;

prefixFragment = prefixFragment.replace(
'uniform mat4 viewMatrix;',
[
'uniform mat4 viewMatrices[' + numMultiviewViews + '];',
'#define viewMatrix viewMatrices[VIEW_ID]'
].join( '\n' )
);

}

}

const vertexGlsl = versionString + prefixVertex + vertexShader;
Expand Down Expand Up @@ -1025,6 +1074,7 @@ function WebGLProgram( renderer, cacheKey, parameters, bindingStates ) {
this.program = program;
this.vertexShader = glVertexShader;
this.fragmentShader = glFragmentShader;
this.numMultiviewViews = numMultiviewViews;

return this;

Expand Down
Loading