Why is my occlusion culling failed with Three.js? - three.js

Question

First I knew that Three.js does not have official support for occlusion culling. However, I thought it's possible to do occlusion culling in an offscreen canvas and copy the result back to my Three.js WebGLCanvas.
Basically, I want to transform this demo
to a Three.JS demo. I use Three.js to create everything, and in a synced offscreen canvas, I test occlusion culling against each bounding box. If any bounding box is occluded, I turn off the visibility of that sphere in the main canvas. Those are what I did in this snippet. but I don't know why it failed to occlude any sphere.
I think a possible issue might be coming from calculating the ModelViewProjection matrix of the bounding box, but I don't see anything wrong. Could somebody please help?
var camera, scene, renderer, light;
var spheres = [];
var NUM_SPHERES, occludedSpheres = 0;
var gl;
var boundingBoxPositions;
var boundingBoxProgram, boundingBoxArray, boundingBoxModelMatrixLocation, viewProjMatrixLocation;
var viewMatrix, projMatrix;
var firstRender = true;
var sphereCountElement = document.getElementById("num-spheres");
var occludedSpheresElement = document.getElementById("num-invisible-spheres");
// depth sort variables
var sortPositionA = new THREE.Vector3();
var sortPositionB = new THREE.Vector3();
var sortModelView = new THREE.Matrix4();
init();
animate();
function init() {
scene = new THREE.Scene();
scene.add( new THREE.AmbientLight( 0x222222 ) );
light = new THREE.DirectionalLight( 0xffffff, 1 );
scene.add( light );
camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 1, 1000);
renderer = new THREE.WebGLRenderer( { antialias: true } );
renderer.setSize(window.innerWidth, window.innerHeight);
document.body.appendChild(renderer.domElement);
// set up offscreen canvas
var offscreenCanvas = new OffscreenCanvas(window.innerWidth, window.innerHeight);
gl = offscreenCanvas.getContext('webgl2');
if ( !gl ) {
console.error("WebGL 2 not available");
document.body.innerHTML = "This example requires WebGL 2 which is unavailable on this system."
}
// define spheres
var GRID_DIM = 6;
var GRID_OFFSET = GRID_DIM / 2 - 0.5;
NUM_SPHERES = GRID_DIM * GRID_DIM;
sphereCountElement.innerHTML = NUM_SPHERES;
var geometry = new THREE.SphereGeometry(20, 64, 64);
var material = new THREE.MeshPhongMaterial( {
color: 0xff0000,
specular: 0x050505,
shininess: 50,
emissive: 0x000000
} );
geometry.computeBoundingBox();
for ( var i = 0; i < NUM_SPHERES; i ++ ) {
var x = Math.floor(i / GRID_DIM) - GRID_OFFSET;
var z = i % GRID_DIM - GRID_OFFSET;
var mesh = new THREE.Mesh( geometry, material );
spheres.push(mesh);
scene.add(mesh);
mesh.position.set(x * 35, 0, z * 35);
mesh.userData.query = gl.createQuery();
mesh.userData.queryInProgress = false;
mesh.userData.occluded = false;
}
//////////////////////////
// WebGL code
//////////////////////////
// boundingbox shader
var boundingBoxVSource = document.getElementById("vertex-boundingBox").text.trim();
var boundingBoxFSource = document.getElementById("fragment-boundingBox").text.trim();
var boundingBoxVertexShader = gl.createShader(gl.VERTEX_SHADER);
gl.shaderSource(boundingBoxVertexShader, boundingBoxVSource);
gl.compileShader(boundingBoxVertexShader);
if (!gl.getShaderParameter(boundingBoxVertexShader, gl.COMPILE_STATUS)) {
console.error(gl.getShaderInfoLog(boundingBoxVertexShader));
}
var boundingBoxFragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
gl.shaderSource(boundingBoxFragmentShader, boundingBoxFSource);
gl.compileShader(boundingBoxFragmentShader);
if (!gl.getShaderParameter(boundingBoxFragmentShader, gl.COMPILE_STATUS)) {
console.error(gl.getShaderInfoLog(boundingBoxFragmentShader));
}
boundingBoxProgram = gl.createProgram();
gl.attachShader(boundingBoxProgram, boundingBoxVertexShader);
gl.attachShader(boundingBoxProgram, boundingBoxFragmentShader);
gl.linkProgram(boundingBoxProgram);
if (!gl.getProgramParameter(boundingBoxProgram, gl.LINK_STATUS)) {
console.error(gl.getProgramInfoLog(boundingBoxProgram));
}
// uniform location
boundingBoxModelMatrixLocation = gl.getUniformLocation(boundingBoxProgram, "uModel");
viewProjMatrixLocation = gl.getUniformLocation(boundingBoxProgram, "uViewProj");
// vertex location
boundingBoxPositions = computeBoundingBoxPositions(geometry.boundingBox);
boundingBoxArray = gl.createVertexArray();
gl.bindVertexArray(boundingBoxArray);
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, boundingBoxPositions, gl.STATIC_DRAW);
gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 0, 0);
gl.enableVertexAttribArray(0);
gl.bindVertexArray(null);
window.addEventListener( 'resize', onWindowResize, false );
}
function onWindowResize() {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize( window.innerWidth, window.innerHeight );
}
function animate() {
requestAnimationFrame(animate);
render();
}
function depthSort(a, b) {
sortPositionA.copy(a.position);
sortPositionB.copy(b.position);
sortModelView.copy(viewMatrix).multiply(a.matrix);
sortPositionA.applyMatrix4(sortModelView);
sortModelView.copy(viewMatrix).multiply(b.matrix);
sortPositionB.applyMatrix4(sortModelView);
return sortPositionB[2] - sortPositionA[2];
}
function render() {
var timer = Date.now() * 0.0001;
camera.position.x = Math.cos( timer ) * 250;
camera.position.z = Math.sin( timer ) * 250;
camera.lookAt( scene.position );
light.position.copy( camera.position );
occludedSpheres = 0;
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
gl.clearColor(0, 0, 0, 1);
gl.enable(gl.DEPTH_TEST);
gl.colorMask(true, true, true, true);
gl.depthMask(true);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
if (!firstRender) {
viewMatrix = camera.matrixWorldInverse.clone();
projMatrix = camera.projectionMatrix.clone();
var viewProjMatrix = projMatrix.multiply(viewMatrix);
spheres.sort(depthSort);
// for occlusion test
gl.colorMask(false, false, false, false);
gl.depthMask(false);
gl.useProgram(boundingBoxProgram);
gl.bindVertexArray(boundingBoxArray);
for (var i = 0; i < NUM_SPHERES; i ++) {
spheres[i].visible = true;
spheres[i].rotation.y += 0.003;
var sphereData = spheres[i].userData;
gl.uniformMatrix4fv(boundingBoxModelMatrixLocation, false, spheres[i].matrix.elements);
gl.uniformMatrix4fv(viewProjMatrixLocation, false, viewProjMatrix.elements);
// check query results here (will be from previous frame)
if (sphereData.queryInProgress && gl.getQueryParameter(sphereData.query, gl.QUERY_RESULT_AVAILABLE)) {
sphereData.occluded = !gl.getQueryParameter(sphereData.query, gl.QUERY_RESULT);
if (sphereData.occluded) occludedSpheres ++;
sphereData.queryInProgress = false;
}
// Query is initiated here by drawing the bounding box of the sphere
if (!sphereData.queryInProgress) {
gl.beginQuery(gl.ANY_SAMPLES_PASSED_CONSERVATIVE, sphereData.query);
gl.drawArrays(gl.TRIANGLES, 0, boundingBoxPositions.length / 3);
gl.endQuery(gl.ANY_SAMPLES_PASSED_CONSERVATIVE);
sphereData.queryInProgress = true;
}
if (sphereData.occluded) {
spheres[i].visible = false;
}
}
occludedSpheresElement.innerHTML = occludedSpheres;
}
firstRender = false;
renderer.render(scene, camera);
}
function computeBoundingBoxPositions(box) {
var dimension = box.max.sub(box.min);
var width = dimension.x;
var height = dimension.y;
var depth = dimension.z;
var x = box.min.x;
var y = box.min.y;
var z = box.min.z;
var fbl = {x: x, y: y, z: z + depth};
var fbr = {x: x + width, y: y, z: z + depth};
var ftl = {x: x, y: y + height, z: z + depth};
var ftr = {x: x + width, y: y + height, z: z + depth};
var bbl = {x: x, y: y, z: z };
var bbr = {x: x + width, y: y, z: z };
var btl = {x: x, y: y + height, z: z };
var btr = {x: x + width, y: y + height, z: z };
var positions = new Float32Array([
//front
fbl.x, fbl.y, fbl.z,
fbr.x, fbr.y, fbr.z,
ftl.x, ftl.y, ftl.z,
ftl.x, ftl.y, ftl.z,
fbr.x, fbr.y, fbr.z,
ftr.x, ftr.y, ftr.z,
//right
fbr.x, fbr.y, fbr.z,
bbr.x, bbr.y, bbr.z,
ftr.x, ftr.y, ftr.z,
ftr.x, ftr.y, ftr.z,
bbr.x, bbr.y, bbr.z,
btr.x, btr.y, btr.z,
//back
fbr.x, bbr.y, bbr.z,
bbl.x, bbl.y, bbl.z,
btr.x, btr.y, btr.z,
btr.x, btr.y, btr.z,
bbl.x, bbl.y, bbl.z,
btl.x, btl.y, btl.z,
//left
bbl.x, bbl.y, bbl.z,
fbl.x, fbl.y, fbl.z,
btl.x, btl.y, btl.z,
btl.x, btl.y, btl.z,
fbl.x, fbl.y, fbl.z,
ftl.x, ftl.y, ftl.z,
//top
ftl.x, ftl.y, ftl.z,
ftr.x, ftr.y, ftr.z,
btl.x, btl.y, btl.z,
btl.x, btl.y, btl.z,
ftr.x, ftr.y, ftr.z,
btr.x, btr.y, btr.z,
//bottom
bbl.x, bbl.y, bbl.z,
bbr.x, bbr.y, bbr.z,
fbl.x, fbl.y, fbl.z,
fbl.x, fbl.y, fbl.z,
bbr.x, bbr.y, bbr.z,
fbr.x, fbr.y, fbr.z,
]);
return positions;
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r125/three.js"></script>
<div id="occlusion-controls">
Spheres: <span id="num-spheres"></span><br> Culled spheres: <span id="num-invisible-spheres"></span><br>
</div>
<script type="x-shader/vs" id="vertex-boundingBox">#version 300 es
layout(std140, column_major) uniform;
layout(location=0) in vec4 position;
uniform mat4 uModel;
uniform mat4 uViewProj;
void main() {
gl_Position = uViewProj * uModel * position;
}
</script>
<script type="x-shader/vf" id="fragment-boundingBox">#version 300 es
precision highp float;
layout(std140, column_major) uniform;
out vec4 fragColor;
void main() {
fragColor = vec4(1.0, 1.0, 1.0, 1.0);
}
</script>

Answer 1

At a minimum, these are the issues I found.
you need to write to the depth buffer (otherwise how would anything occlude?)
so remove gl.depthMask(false)
you need to gl.flush the OffscreenCanvas because being offscreen, one is not automatically added for you. I found this out by using a normal canvas and adding it to the page. I also turned on drawing by commenting out gl.colorMask(false, false, false, false) just to double check that your boxes are drawn correctly. I noticed that when I got something kind of working it behaved differently when I switched back to the offscreen canvas. I found the same different behavior if I didn't add the normal canvas to the page. Adding in the gl.flush fixed the different behavior.
depthSort was not working
I checked this by changing the shader to use a color and I passed in i / NUM_SPHERES as the color which made it clear they were not being sorted. The issue was this
return sortPositionB[2] - sortPositionA[2];
needs to be
return sortPositionB.z - sortPositionA.z;
var camera, scene, renderer, light;
var spheres = [];
var NUM_SPHERES, occludedSpheres = 0;
var gl;
var boundingBoxPositions;
var boundingBoxProgram, boundingBoxArray, boundingBoxModelMatrixLocation, viewProjMatrixLocation;
var viewMatrix, projMatrix;
var firstRender = true;
var sphereCountElement = document.getElementById("num-spheres");
var occludedSpheresElement = document.getElementById("num-invisible-spheres");
// depth sort variables
var sortPositionA = new THREE.Vector3();
var sortPositionB = new THREE.Vector3();
var sortModelView = new THREE.Matrix4();
init();
animate();
function init() {
scene = new THREE.Scene();
scene.add( new THREE.AmbientLight( 0x222222 ) );
light = new THREE.DirectionalLight( 0xffffff, 1 );
scene.add( light );
camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 1, 1000);
renderer = new THREE.WebGLRenderer( { antialias: true } );
renderer.setSize(window.innerWidth, window.innerHeight);
document.body.appendChild(renderer.domElement);
// set up offscreen canvas
var offscreenCanvas = new OffscreenCanvas(window.innerWidth, window.innerHeight);
//var offscreenCanvas = document.createElement('canvas');
//offscreenCanvas.width = window.innerWidth;
//offscreenCanvas.height = window.innerHeight;
//document.body.appendChild(offscreenCanvas);
gl = offscreenCanvas.getContext('webgl2');
if ( !gl ) {
console.error("WebGL 2 not available");
document.body.innerHTML = "This example requires WebGL 2 which is unavailable on this system."
}
// define spheres
var GRID_DIM = 6;
var GRID_OFFSET = GRID_DIM / 2 - 0.5;
NUM_SPHERES = GRID_DIM * GRID_DIM;
sphereCountElement.innerHTML = NUM_SPHERES;
var geometry = new THREE.SphereGeometry(20, 64, 64);
var material = new THREE.MeshPhongMaterial( {
color: 0xff0000,
specular: 0x050505,
shininess: 50,
emissive: 0x000000
} );
geometry.computeBoundingBox();
for ( var i = 0; i < NUM_SPHERES; i ++ ) {
var x = Math.floor(i / GRID_DIM) - GRID_OFFSET;
var z = i % GRID_DIM - GRID_OFFSET;
var mesh = new THREE.Mesh( geometry, material );
spheres.push(mesh);
scene.add(mesh);
mesh.position.set(x * 35, 0, z * 35);
mesh.userData.query = gl.createQuery();
mesh.userData.queryInProgress = false;
mesh.userData.occluded = false;
}
//////////////////////////
// WebGL code
//////////////////////////
// boundingbox shader
var boundingBoxVSource = document.getElementById("vertex-boundingBox").text.trim();
var boundingBoxFSource = document.getElementById("fragment-boundingBox").text.trim();
var boundingBoxVertexShader = gl.createShader(gl.VERTEX_SHADER);
gl.shaderSource(boundingBoxVertexShader, boundingBoxVSource);
gl.compileShader(boundingBoxVertexShader);
if (!gl.getShaderParameter(boundingBoxVertexShader, gl.COMPILE_STATUS)) {
console.error(gl.getShaderInfoLog(boundingBoxVertexShader));
}
var boundingBoxFragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
gl.shaderSource(boundingBoxFragmentShader, boundingBoxFSource);
gl.compileShader(boundingBoxFragmentShader);
if (!gl.getShaderParameter(boundingBoxFragmentShader, gl.COMPILE_STATUS)) {
console.error(gl.getShaderInfoLog(boundingBoxFragmentShader));
}
boundingBoxProgram = gl.createProgram();
gl.attachShader(boundingBoxProgram, boundingBoxVertexShader);
gl.attachShader(boundingBoxProgram, boundingBoxFragmentShader);
gl.linkProgram(boundingBoxProgram);
if (!gl.getProgramParameter(boundingBoxProgram, gl.LINK_STATUS)) {
console.error(gl.getProgramInfoLog(boundingBoxProgram));
}
// uniform location
boundingBoxModelMatrixLocation = gl.getUniformLocation(boundingBoxProgram, "uModel");
viewProjMatrixLocation = gl.getUniformLocation(boundingBoxProgram, "uViewProj");
// vertex location
boundingBoxPositions = computeBoundingBoxPositions(geometry.boundingBox);
boundingBoxArray = gl.createVertexArray();
gl.bindVertexArray(boundingBoxArray);
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, boundingBoxPositions, gl.STATIC_DRAW);
gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 0, 0);
gl.enableVertexAttribArray(0);
gl.bindVertexArray(null);
window.addEventListener( 'resize', onWindowResize, false );
}
function onWindowResize() {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize( window.innerWidth, window.innerHeight );
}
function animate() {
requestAnimationFrame(animate);
render();
}
function depthSort(a, b) {
sortPositionA.copy(a.position);
sortPositionB.copy(b.position);
sortModelView.copy(viewMatrix).multiply(a.matrix);
sortPositionA.applyMatrix4(sortModelView);
sortModelView.copy(viewMatrix).multiply(b.matrix);
sortPositionB.applyMatrix4(sortModelView);
return sortPositionB.z - sortPositionA.z;
}
function render() {
var timer = Date.now() * 0.0001;
camera.position.x = Math.cos( timer ) * 250;
camera.position.z = Math.sin( timer ) * 250;
camera.lookAt( scene.position );
light.position.copy( camera.position );
occludedSpheres = 0;
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
gl.clearColor(0, 0, 0, 1);
gl.enable(gl.DEPTH_TEST);
gl.colorMask(true, true, true, true);
gl.depthMask(true);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
if (!firstRender) {
viewMatrix = camera.matrixWorldInverse.clone();
projMatrix = camera.projectionMatrix.clone();
var viewProjMatrix = projMatrix.multiply(viewMatrix);
spheres.sort(depthSort);
// for occlusion test
gl.colorMask(false, false, false, false);
//gl.depthMask(false);
gl.useProgram(boundingBoxProgram);
gl.bindVertexArray(boundingBoxArray);
for (var i = 0; i < NUM_SPHERES; i ++) {
spheres[i].visible = true;
spheres[i].rotation.y += 0.003;
var sphereData = spheres[i].userData;
gl.uniformMatrix4fv(boundingBoxModelMatrixLocation, false, spheres[i].matrix.elements);
gl.uniformMatrix4fv(viewProjMatrixLocation, false, viewProjMatrix.elements);
gl.uniform4f(gl.getUniformLocation(boundingBoxProgram, 'color'),
i / NUM_SPHERES, 0, 0, 1);
// check query results here (will be from previous frame)
if (sphereData.queryInProgress && gl.getQueryParameter(sphereData.query, gl.QUERY_RESULT_AVAILABLE)) {
sphereData.occluded = !gl.getQueryParameter(sphereData.query, gl.QUERY_RESULT);
if (sphereData.occluded) occludedSpheres ++;
sphereData.queryInProgress = false;
}
// Query is initiated here by drawing the bounding box of the sphere
if (!sphereData.queryInProgress) {
gl.beginQuery(gl.ANY_SAMPLES_PASSED_CONSERVATIVE, sphereData.query);
gl.drawArrays(gl.TRIANGLES, 0, boundingBoxPositions.length / 3);
gl.endQuery(gl.ANY_SAMPLES_PASSED_CONSERVATIVE);
sphereData.queryInProgress = true;
}
if (sphereData.occluded) {
spheres[i].visible = false;
}
}
gl.flush();
occludedSpheresElement.innerHTML = occludedSpheres;
}
firstRender = false;
renderer.render(scene, camera);
}
function computeBoundingBoxPositions(box) {
var dimension = box.max.sub(box.min);
var width = dimension.x;
var height = dimension.y;
var depth = dimension.z;
var x = box.min.x;
var y = box.min.y;
var z = box.min.z;
var fbl = {x: x, y: y, z: z + depth};
var fbr = {x: x + width, y: y, z: z + depth};
var ftl = {x: x, y: y + height, z: z + depth};
var ftr = {x: x + width, y: y + height, z: z + depth};
var bbl = {x: x, y: y, z: z };
var bbr = {x: x + width, y: y, z: z };
var btl = {x: x, y: y + height, z: z };
var btr = {x: x + width, y: y + height, z: z };
var positions = new Float32Array([
//front
fbl.x, fbl.y, fbl.z,
fbr.x, fbr.y, fbr.z,
ftl.x, ftl.y, ftl.z,
ftl.x, ftl.y, ftl.z,
fbr.x, fbr.y, fbr.z,
ftr.x, ftr.y, ftr.z,
//right
fbr.x, fbr.y, fbr.z,
bbr.x, bbr.y, bbr.z,
ftr.x, ftr.y, ftr.z,
ftr.x, ftr.y, ftr.z,
bbr.x, bbr.y, bbr.z,
btr.x, btr.y, btr.z,
//back
fbr.x, bbr.y, bbr.z,
bbl.x, bbl.y, bbl.z,
btr.x, btr.y, btr.z,
btr.x, btr.y, btr.z,
bbl.x, bbl.y, bbl.z,
btl.x, btl.y, btl.z,
//left
bbl.x, bbl.y, bbl.z,
fbl.x, fbl.y, fbl.z,
btl.x, btl.y, btl.z,
btl.x, btl.y, btl.z,
fbl.x, fbl.y, fbl.z,
ftl.x, ftl.y, ftl.z,
//top
ftl.x, ftl.y, ftl.z,
ftr.x, ftr.y, ftr.z,
btl.x, btl.y, btl.z,
btl.x, btl.y, btl.z,
ftr.x, ftr.y, ftr.z,
btr.x, btr.y, btr.z,
//bottom
bbl.x, bbl.y, bbl.z,
bbr.x, bbr.y, bbr.z,
fbl.x, fbl.y, fbl.z,
fbl.x, fbl.y, fbl.z,
bbr.x, bbr.y, bbr.z,
fbr.x, fbr.y, fbr.z,
]);
return positions;
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r125/three.js"></script>
<div id="occlusion-controls">
Spheres: <span id="num-spheres"></span><br> Culled spheres: <span id="num-invisible-spheres"></span><br>
</div>
<script type="x-shader/vs" id="vertex-boundingBox">#version 300 es
layout(std140, column_major) uniform;
layout(location=0) in vec4 position;
uniform mat4 uModel;
uniform mat4 uViewProj;
void main() {
gl_Position = uViewProj * uModel * position;
}
</script>
<script type="x-shader/vf" id="fragment-boundingBox">#version 300 es
precision highp float;
layout(std140, column_major) uniform;
out vec4 fragColor;
void main() {
fragColor = vec4(1.0, 1.0, 1.0, 1.0);
}
</script>