How does raycasting in three.js work with an offscreen canvas? - three.js

Question

I can't seem to get raycasting to work in an offscreen canvas.
A click event sends data to the worker like this:
var r = document.getElementById('webGL').getBoundingClientRect()
offscreen.postMessage({
action: 'set_scene',
mesh: mesh.toJSON(),
camera: camera.toJSON(),
canvasSize: {
width: document.getElementById('webGL').clientWidth,
height: document.getElementById('webGL').clientHeight
},
coordinates: { x: e.originalEvent.clientX - r.x, y: e.originalEvent.clientY - r.y },
time: (new Date())
});

while the worker looks like this:
self.importScripts( './three.min.js' );
var loader = new THREE.ObjectLoader();
var scene = new THREE.Scene();
self.onmessage = function(e) {
// var workerResult = 'Result: ' + (e.data[0] * e.data[1]);
var canvas = new OffscreenCanvas(e.data.canvasSize.width, e.data.canvasSize.height);
var renderer = new THREE.WebGLRenderer( { antialias: true, canvas: canvas, preserveDrawingBuffer: true } );
Promise.all([
(new Promise(function(resolve, reject) {
loader.parse(
e.data.mesh,
function ( obj ) {
resolve( obj );
})
})),
(new Promise(function(resolve, reject) {
loader.parse(
e.data.camera,
function ( obj ) {
resolve( obj );
})
}))
]).then(obj => {
var mesh, camera
[mesh, camera] = obj;
scene.add( mesh );
renderer.render( scene, camera );
var raycaster = new THREE.Raycaster();
p = { x: e.data.coordinates.x, y: e.data.coordinates.y };
var m = {};
m.x = (p.x) / e.data.canvasSize.width * 2 - 1;
m.y = 1 - (p.y) / e.data.canvasSize.height * 2;
raycaster.setFromCamera( m, camera );
var intersects = raycaster.intersectObjects( [ mesh ], true );
return intersects;
}).then(r => {
self.postMessage(r);
}).catch(e => {
console.log(e);
})
}
Same code onscreen works ok, and the values resulting from the transforms check out ok.
Is it possible to do such a thing at all, or what am I getting wrong?

Answer 1

I don't see the issue in your code but there is absolutely nothing special about picking offscreen. Here's a working example to prove it. It doesn't have any three or camera or mesh in the main page, only in the worker.
All the main page does is start the worker, transfer control of the canvas to the worker, then send resize and mouse events to the worker. That's it. Otherwise the code in the worker is 99% the same as the code would be in the main page. The only major difference is resizeCanvasToDisplaySize gets the display size from state.width and state.height. The non-offscreen version code comes from here
You need to post more code.
function main() {
const canvas = document.querySelector("#c");
if (!canvas.transferControlToOffscreen) {
alert('no offscreen canvas support');
return;
}
const offscreen = canvas.transferControlToOffscreen();
const workerScript = document.querySelector('#foo').text;
const blob = new Blob([workerScript], {type: 'application/javascript'});
const url = URL.createObjectURL(blob);
const worker = new Worker(url);
worker.postMessage({type: 'init', canvas: offscreen}, [offscreen]);
function sendSize() {
worker.postMessage({
type: 'size',
width: canvas.clientWidth,
height: canvas.clientHeight,
});
}
sendSize();
window.addEventListener('resize', sendSize);
function getNormaizedMousePosition(element, e) {
const rect = element.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
return {
x: x / canvas.clientWidth * 2 - 1,
y: y / canvas.clientHeight * -2 + 1,
}
}
canvas.addEventListener('mousemove', (e) => {
const pos = getNormaizedMousePosition(canvas, e);
worker.postMessage({
type: 'mousemove',
x: pos.x,
y: pos.y,
});
});
}
main();
body {
margin: 0;
}
#c {
width: 100vw;
height: 100vh;
display: block;
}
<canvas id="c"></canvas>
<script type="foo" id="foo">
'use strict';
importScripts('https://threejsfundamentals.org/threejs/resources/threejs/r103/three.min.js');
/* global THREE */
const state = {
width: 300,
height: 150,
mouse: {
x: -2,
y: -2,
},
};
function init(data) {
const {canvas} = data;
const renderer = new THREE.WebGLRenderer({
canvas
});
state.width = canvas.width;
state.height = canvas.height;
const fov = 75;
const aspect = 2; // the canvas default
const near = 0.1;
const far = 100;
const camera = new THREE.PerspectiveCamera(fov, aspect, near, far);
camera.position.z = 4;
const scene = new THREE.Scene();
{
const color = 0xFFFFFF;
const intensity = 1;
const light = new THREE.DirectionalLight(color, intensity);
light.position.set(-1, 2, 4);
scene.add(light);
}
const boxWidth = 1;
const boxHeight = 1;
const boxDepth = 1;
const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);
function makeInstance(geometry, color, x) {
const material = new THREE.MeshPhongMaterial({
color
});
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);
cube.position.x = x;
return cube;
}
const cubes = [
makeInstance(geometry, 0x44aa88, 0),
makeInstance(geometry, 0x8844aa, -2),
makeInstance(geometry, 0xaa8844, 2),
];
class PickHelper {
constructor() {
this.raycaster = new THREE.Raycaster();
this.pickedObject = null;
this.pickedObjectSavedColor = 0;
}
pick(normalizedPosition, scene, camera, time) {
// restore the color if there is a picked object
if (this.pickedObject) {
this.pickedObject.material.emissive.setHex(this.pickedObjectSavedColor);
this.pickedObject = undefined;
}
// cast a ray through the frustum
this.raycaster.setFromCamera(normalizedPosition, camera);
// get the list of objects the ray intersected
const intersectedObjects = this.raycaster.intersectObjects(scene.children);
if (intersectedObjects.length) {
// pick the first object. It's the closest one
this.pickedObject = intersectedObjects[0].object;
// save its color
this.pickedObjectSavedColor = this.pickedObject.material.emissive.getHex();
// set its emissive color to flashing red/yellow
this.pickedObject.material.emissive.setHex((time * 8) % 2 > 1 ? 0xFFFF00 : 0xFF0000);
}
}
}
const pickHelper = new PickHelper();
function resizeRendererToDisplaySize(renderer) {
const canvas = renderer.domElement;
const width = state.width;
const height = state.height;
const needResize = canvas.width !== width || canvas.height !== height;
if (needResize) {
renderer.setSize(width, height, false);
}
return needResize;
}
function render(time) {
time *= 0.001;
if (resizeRendererToDisplaySize(renderer)) {
const canvas = renderer.domElement;
camera.aspect = canvas.width / canvas.height;
camera.updateProjectionMatrix();
}
cubes.forEach((cube, ndx) => {
const speed = 1 + ndx * .1;
const rot = time * speed;
cube.rotation.x = rot;
cube.rotation.y = rot;
});
pickHelper.pick(state.mouse, scene, camera, time);
renderer.render(scene, camera);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
}
function size(data) {
state.width = data.width;
state.height = data.height;
}
function mousemove(data) {
state.mouse.x = data.x;
state.mouse.y = data.y;
}
const handlers = {
init,
size,
mousemove,
};
self.onmessage = function(e) {
const fn = handlers[e.data.type];
if (!fn) {
throw new Error('no handler for type: ' + e.data.type);
}
fn(e.data);
};
</script>
For your particular case though you shouldn't need a camera. Send the ray. You don't need the canvas size either. Picking in three.js is CPU based
function main() {
const canvas = document.querySelector("#c");
const renderer = new THREE.WebGLRenderer({canvas});
const fov = 60;
const aspect = 2; // the canvas default
const near = 0.1;
const far = 200;
const camera = new THREE.PerspectiveCamera(fov, aspect, near, far);
camera.position.z = 4;
const scene = new THREE.Scene();
scene.background = new THREE.Color('white');
{
const color = 0xFFFFFF;
const intensity = 1;
const light = new THREE.DirectionalLight(color, intensity);
light.position.set(-1, 2, 4);
scene.add(light);
}
const boxWidth = 1;
const boxHeight = 1;
const boxDepth = 1;
const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);
function makeInstance(geometry, color, x, name) {
const material = new THREE.MeshPhongMaterial({
color
});
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);
cube.name = name;
cube.position.x = x;
cube.rotation.x = x;
cube.rotation.z = x + .7;
return cube;
}
makeInstance(geometry, 0x44aa88, 0, 'cyan cube');
makeInstance(geometry, 0x8844aa, -2, 'purple cube');
makeInstance(geometry, 0xaa8844, 2, 'brown cube');
function resizeRendererToDisplaySize(renderer) {
const canvas = renderer.domElement;
const width = canvas.clientWidth;
const height = canvas.clientHeight;
const needResize = canvas.width !== width || canvas.height !== height;
if (needResize) {
renderer.setSize(width, height, false);
}
return needResize;
}
function render() {
if (resizeRendererToDisplaySize(renderer)) {
const canvas = renderer.domElement;
camera.aspect = canvas.width / canvas.height;
camera.updateProjectionMatrix();
}
renderer.render(scene, camera);
}
render();
window.addEventListener('resize', render);
const workerScript = document.querySelector('#foo').text;
const blob = new Blob([workerScript], {type: 'application/javascript'});
const url = URL.createObjectURL(blob);
const worker = new Worker(url);
const msgElem = document.querySelector('#msg');
worker.onmessage = (e) => {
msgElem.textContent = e.data;
};
worker.postMessage({type: 'init', scene: scene.toJSON()});
function getNormaizedMousePosition(element, e) {
const rect = element.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
return {
x: x / canvas.clientWidth * 2 - 1,
y: y / canvas.clientHeight * -2 + 1,
}
}
const raycaster = new THREE.Raycaster();
canvas.addEventListener('mousemove', (e) => {
const pos = getNormaizedMousePosition(canvas, e);
raycaster.setFromCamera(pos, camera);
worker.postMessage({
type: 'intersect',
origin: raycaster.ray.origin.toArray(),
direction: raycaster.ray.direction.toArray(),
});
});
}
main();
body {
margin: 0;
}
#c {
width: 100vw;
height: 100vh;
display: block;
}
#msg {
position: absolute;
left: 1em;
top: 1em;
}
<canvas id="c"></canvas>
<div id="msg"></div>
<script src="https://threejsfundamentals.org/threejs/resources/threejs/r103/three.min.js"></script>
<script type="foo" id="foo">
'use strict';
importScripts('https://threejsfundamentals.org/threejs/resources/threejs/r103/three.min.js');
/* global THREE */
function loadObject(json) {
return new Promise((resolve) => {
const loader = new THREE.ObjectLoader();
loader.parse(json, resolve);
});
}
const renderer = new THREE.WebGLRenderer({
canvas: new OffscreenCanvas(1, 1),
});
// settings not important
const camera = new THREE.PerspectiveCamera(1, 1, 0.1, 100);
const raycaster = new THREE.Raycaster();
let scene;
let lastIntersectedObject;
async function init(data) {
scene = await loadObject(data.scene);
// we only need to render once to init the scene
renderer.render(scene, camera);
}
function intersect(data) {
raycaster.set(
new THREE.Vector3(...data.origin),
new THREE.Vector3(...data.direction));
const intersections = raycaster.intersectObjects(scene.children);
const intersectedObject = intersections.length
? intersections[0].object
: null;
if (intersectedObject !== lastIntersectedObject) {
lastIntersectedObject = intersectedObject;
log('intersection:', lastIntersectedObject ? lastIntersectedObject.name : 'none');
}
}
const handlers = {
init,
intersect,
};
self.onmessage = function(e) {
const fn = handlers[e.data.type];
if (!fn) {
throw new Error('no handler for type: ' + e.data.type);
}
fn(e.data);
};
function log(...args) {
postMessage([...args].join(' '));
}
</script>