gpu picking - invisible pixels around sprites - three.js

Question

I'm rendering a picking scene that contains sprites. As my cursor gets close to the sprite, it registers as a color and gets "picked". This invisible border gets larger closer you zoom into the sprites.
Open up your console to see the IDs printed in real time. Move your cursor closer and further away to large and small sprites. You'll see that sprites get selected on an invisible border. This behavior does not happen with regular geometry, just with sprites.
It's weird because I'm rendering out what renderer.readRenderTargetPixels actually sees.
How can I get rid of the invisible borders for more accurate picking?
var renderer, scene, camera, controls;
var particles, uniforms;
var PARTICLE_SIZE = 50;
var raycaster, intersects;
var mouse, INTERSECTED;
var pickingTexture;
var numOfVertices;
init();
animate();
function init() {
container = document.getElementById('container');
scene = new THREE.Scene();
camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 1, 10000);
camera.position.z = 150;
//
var geometry1 = new THREE.BoxGeometry(200, 200, 200, 4, 4, 4);
var vertices = geometry1.vertices;
numOfVertices = vertices.length;
var positions = new Float32Array(vertices.length * 3);
var colors = new Float32Array(vertices.length * 3);
var sizes = new Float32Array(vertices.length);
var vertex;
var color = new THREE.Color();
for (var i = 0, l = vertices.length; i < l; i++) {
vertex = vertices[i];
vertex.toArray(positions, i * 3);
color.setHex(i + 1);
color.toArray(colors, i * 3);
sizes[i] = PARTICLE_SIZE * 0.5;
}
var geometry = new THREE.BufferGeometry();
geometry.addAttribute('position', new THREE.BufferAttribute(positions, 3));
geometry.addAttribute('customColor', new THREE.BufferAttribute(colors, 3));
geometry.addAttribute('size', new THREE.BufferAttribute(sizes, 1));
//
var material = new THREE.ShaderMaterial({
uniforms: {
// texture: {type: "t", value: THREE.ImageUtils.loadTexture("../textures/circle.png")}
texture: {type: "t", value: THREE.ImageUtils.loadTexture("../textures/disc.png")}
},
vertexShader: document.getElementById('vertexshader').textContent,
fragmentShader: document.getElementById('fragmentshader').textContent,
depthTest: false,
transparent: false
// alphaTest: 0.9
});
//
particles = new THREE.Points(geometry, material);
scene.add(particles);
//
renderer = new THREE.WebGLRenderer({
antialias: true,
alpha: true
});
renderer.setPixelRatio(window.devicePixelRatio);
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setClearColor(0xffffff);
container.appendChild(renderer.domElement);
//
raycaster = new THREE.Raycaster();
mouse = new THREE.Vector2();
//
//
window.addEventListener('resize', onWindowResize, false);
document.addEventListener('mousemove', onDocumentMouseMove, false);
// defaults are on the right (except minFilter)
var options = {
format: THREE.RGBAFormat, // THREE.RGBAFormat
type: THREE.UnsignedByteType, // THREE.UnsignedByteType
anisotropy: 1, // 1
magFilter: THREE.LinearFilter, // THREE.LinearFilter
minFilter: THREE.LinearFilter, // THREE.LinearFilter
depthBuffer: true, // true
stencilBuffer: true // true
};
pickingTexture = new THREE.WebGLRenderTarget(window.innerWidth, window.innerHeight, options);
pickingTexture.texture.generateMipmaps = false;
controls = new THREE.OrbitControls(camera, container);
controls.damping = 0.2;
controls.enableDamping = false;
}
function onDocumentMouseMove(e) {
// event.preventDefault();
mouse.x = e.clientX;
mouse.y = e.clientY;
}
function onWindowResize() {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
}
function animate() {
requestAnimationFrame(animate);
controls.update();
render();
}
function render() {
pick();
renderer.render(scene, camera);
}
function pick() {
renderer.render(scene, camera, pickingTexture);
//create buffer for reading single pixel
var pixelBuffer = new Uint8Array(4);
//read the pixel under the mouse from the texture
renderer.readRenderTargetPixels(pickingTexture, mouse.x, pickingTexture.height - mouse.y, 1, 1, pixelBuffer);
//interpret the pixel as an ID
var id = ( pixelBuffer[0] << 16 ) | ( pixelBuffer[1] << 8 ) | ( pixelBuffer[2] );
if (id <= numOfVertices) console.log(id);
}
body {
color: #ffffff;
background-color: #000000;
margin: 0px;
overflow: hidden;
}
<script src="http://threejs.org/build/three.min.js"></script>
<script src="http://threejs.org/examples/js/controls/OrbitControls.js"></script>
<script type="x-shader/x-fragment" id="fragmentshader">
uniform sampler2D texture;
varying vec3 vColor;
void main() {
// solid squares of color
gl_FragColor = vec4( vColor, 1.0 );
}
</script>
<script type="x-shader/x-vertex" id="vertexshader">
attribute float size;
attribute vec3 customColor;
varying vec3 vColor;
void main() {
vColor = customColor;
vec4 mvPosition = modelViewMatrix * vec4( position, 1.0 );
gl_PointSize = size * ( 300.0 / length( mvPosition.xyz ) );
gl_Position = projectionMatrix * mvPosition;
}
</script>
<div id="container"></div>

Answer 1

The problem is you're on a device that has a devicePixelRatio != 1.0 and three.js lying about the size.
Because you called renderer.setPixelRatio now magic happens behind the scenes. Your canvas is not the size you requested it's some other size based on some formula hidden in the three.js code.
So, what happens. Your canvas is one size but your render target is a different size. Your shader uses gl_PointSize to draw its points. That size is in device pixels. Because your render target is a different size the size of the points are different in your render target than they are on screen.
Remove the call to render.setPixelRatio and it will start working.
IMO the correct way to fix this is to use devicePixelRatio yourself because that way everything that is happening is 100% visible to you. No magic happening behind the scenes.
So,
Get rid of the container and use a canvas directly
<canvas id="c"></canvas>
set the canvas to use 100vw for width, 100vh for height and made the body margin: 0;
canvas { width: 100vw; height: 100vh; display: block; }
body { margin: 0; }
This will make your canvas stretch automatically to fill the window.
Use the size the browser stretched the canvas to choose the size its drawingBuffer should be and multiply by devicePixelRatio. That assumes you actually want to support device pixel ratio. No need to do this twice so following D.R.Y. so just do it in onWindowResize.
canvas = document.getElementById("c");
renderer = new THREE.WebGLRenderer({
antialias: true,
alpha: true,
canvas: canvas,
});
pickingTexture = new THREE.WebGLRenderTarget(1, 1, options);
onWindowResize();
...
function onWindowResize() {
var width = canvas.clientWidth * window.devicePixelRatio;
var height = canvas.clientHeight * window.devicePixelRatio;
camera.aspect = width / height;
camera.updateProjectionMatrix();
renderer.setSize(width, height, false); // YOU MUST PASS FALSE HERE otherwise three.js will muck with the CSS
pickingTexture.setSize(width, height);
}
Convert the mouse coordinates into device coordinates
renderer.readRenderTargetPixels(
pickingTexture,
mouse.x * window.devicePixelRatio,
pickingTexture.height - mouse.y * window.devicePixelRatio,
1, 1, pixelBuffer);
Here's that solution
var renderer, scene, camera, controls;
var particles, uniforms;
var PARTICLE_SIZE = 50;
var raycaster, intersects;
var mouse, INTERSECTED;
var pickingTexture;
var numOfVertices;
var info = document.querySelector('#info');
init();
animate();
function init() {
canvas = document.getElementById('c');
scene = new THREE.Scene();
camera = new THREE.PerspectiveCamera(45, 1, 1, 10000);
camera.position.z = 150;
//
var geometry1 = new THREE.BoxGeometry(200, 200, 200, 4, 4, 4);
var vertices = geometry1.vertices;
numOfVertices = vertices.length;
var positions = new Float32Array(vertices.length * 3);
var colors = new Float32Array(vertices.length * 3);
var sizes = new Float32Array(vertices.length);
var vertex;
var color = new THREE.Color();
for (var i = 0, l = vertices.length; i < l; i++) {
vertex = vertices[i];
vertex.toArray(positions, i * 3);
color.setHex(i + 1);
color.toArray(colors, i * 3);
sizes[i] = PARTICLE_SIZE * 0.5;
}
var geometry = new THREE.BufferGeometry();
geometry.setAttribute('position', new THREE.BufferAttribute(positions, 3));
geometry.setAttribute('customColor', new THREE.BufferAttribute(colors, 3));
geometry.setAttribute('size', new THREE.BufferAttribute(sizes, 1));
//
var loader = new THREE.TextureLoader();
var material = new THREE.ShaderMaterial({
uniforms: {
// texture: {type: "t", value: THREE.ImageUtils.loadTexture("../textures/circle.png")}
texture: {value: loader.load("https://i.imgur.com/iXT97XR.png")}
},
vertexShader: document.getElementById('vertexshader').textContent,
fragmentShader: document.getElementById('fragmentshader').textContent,
depthTest: false,
transparent: false
// alphaTest: 0.9
});
//
particles = new THREE.Points(geometry, material);
scene.add(particles);
//
renderer = new THREE.WebGLRenderer({
antialias: true,
alpha: true,
canvas: canvas,
});
renderer.setClearColor(0xffffff);
//
raycaster = new THREE.Raycaster();
mouse = new THREE.Vector2();
//
//
window.addEventListener('resize', onWindowResize, false);
document.addEventListener('mousemove', onDocumentMouseMove, false);
// defaults are on the right (except minFilter)
var options = {
format: THREE.RGBAFormat, // THREE.RGBAFormat
type: THREE.UnsignedByteType, // THREE.UnsignedByteType
anisotropy: 1, // 1
magFilter: THREE.LinearFilter, // THREE.LinearFilter
minFilter: THREE.LinearFilter, // THREE.LinearFilter
depthBuffer: true, // true
stencilBuffer: true // true
};
pickingTexture = new THREE.WebGLRenderTarget(1, 1, options);
pickingTexture.texture.generateMipmaps = false;
controls = new THREE.OrbitControls(camera, canvas);
controls.damping = 0.2;
controls.enableDamping = false;
onWindowResize();
}
function onDocumentMouseMove(e) {
// event.preventDefault();
mouse.x = e.clientX;
mouse.y = e.clientY;
}
function onWindowResize() {
var width = canvas.clientWidth * window.devicePixelRatio;
var height = canvas.clientHeight * window.devicePixelRatio;
camera.aspect = width / height;
camera.updateProjectionMatrix();
renderer.setSize(width, height, false); // YOU MUST PASS FALSE HERE!
pickingTexture.setSize(width, height);
}
function animate() {
requestAnimationFrame(animate);
controls.update();
render();
}
function render() {
pick();
renderer.render(scene, camera);
}
function pick() {
renderer.setRenderTarget(pickingTexture);
renderer.setClearColor(0);
renderer.render(scene, camera);
renderer.setClearColor(0xFFFFFF);
renderer.setRenderTarget(null)
//create buffer for reading single pixel
var pixelBuffer = new Uint8Array(4);
//read the pixel under the mouse from the texture
renderer.readRenderTargetPixels(pickingTexture, mouse.x * window.devicePixelRatio, pickingTexture.height - mouse.y * window.devicePixelRatio, 1, 1, pixelBuffer);
//interpret the pixel as an ID
var id = ( pixelBuffer[0] << 16 ) | ( pixelBuffer[1] << 8 ) | ( pixelBuffer[2] );
//if (id > 0) console.log(id);
info.textContent = id;
}
body {
color: #ffffff;
background-color: #000000;
margin: 0;
}
canvas { width: 100vw; height: 100vh; display: block; }
#info { position: absolute; left: 0; top: 0; color: red; background: black; padding: 0.5em; font-family: monospace; }
<script src="https://threejs.org/build/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>
<script type="x-shader/x-fragment" id="fragmentshader">
uniform sampler2D texture;
varying vec3 vColor;
void main() {
// solid squares of color
gl_FragColor = vec4( vColor, 1.0 );
}
</script>
<script type="x-shader/x-vertex" id="vertexshader">
attribute float size;
attribute vec3 customColor;
varying vec3 vColor;
void main() {
vColor = customColor;
vec4 mvPosition = modelViewMatrix * vec4( position, 1.0 );
gl_PointSize = size * ( 300.0 / length( mvPosition.xyz ) );
gl_Position = projectionMatrix * mvPosition;
}
</script>
<canvas id="c"></canvas>
<div id="info"></div>
Note a few other things.
I'd guess you really want to clear the picking texture to zero instead of white. That way 0 = nothing there, anything else = something there.
renderer.setClearColor(0);
renderer.render(scene, camera, pickingTexture);
renderer.setClearColor(0xFFFFFF);
No idea what the id <= numOfVertices means
So given that it's clearing to zero now the code is just
if (id) console.log(id);
I don't set the renderer size, the pickingTexture size nor the camera aspect at init time.
Why repeat myself. onWindowResize already sets it
You need to resize the pickingTexture render target when the canvas is resizes so it matches in size.
I removed most references to window.innerWidth and window.innerHeight
I would have removed all of them but I didn't want to change even more code for this example. Using window.innerWidth ties the code to the window. If you ever want to use the code in something that's not the fullsize of the window, for example lets say you make an editor. You'll have to change the code.
It's not any harder to write the code in a way that works in more situations so why make more work for yourself later.
Other solutions I didn't chose
You could call render.setPixelRatio and then set the pickingTexture render target's size with window.devicePixelRatio
I didn't pick this solution because you have to guess what three.js is doing behind the scenes. Your guess might be correct today but wrong tomorrow. It seems better if you tell three.js make something width by height it should just make it width by height and not make it something else. Similarly you'd have to guess when three.js is going to apply pixelRatio and when it's not. As you noticed above it doesn't apply it to the size of the render target and it can't because it doesn't know what your purpose is. Are you making a render target for picking? For a fullscreen effect? For capture? for a non-fullscreen effect? Since it can't know it can't apply the pixelRatio for you. This happens all over the three.js code. Some places it applies pixelRatio, other places it doesn't. You're left guessing. If you never set pixelRatio that problem disappears.
You could pass in devicePixelRatio into your shader
<script type="x-shader/x-vertex" id="vertexshader">
attribute float size;
attribute vec3 customColor;
varying vec3 vColor;
uniform float devicePixelRatio; // added
void main() {
vColor = customColor;
vec4 mvPosition = modelViewMatrix * vec4( position, 1.0 );
gl_PointSize = size * ( 300.0 / length( mvPosition.xyz ) ) * devicePixelRatio;
gl_Position = projectionMatrix * mvPosition;
}
</script>
and of course you'd need to set devicePixelRatio in your uniforms.
I might pick this solution. The minor problem is if the pickingTexture is not the same resolution as the canvas's backbuffer you can get off by 1 errors. In this case if the canvas was 2x the pickingTexture then 3 of every 4 pixels in the canvas don't exist in the pickingTexture. Depending on your application that might be ok though. You can't pick 1/2 pixels, at least not with the mouse.
Another other reason I would probably not pick this solution is it just leaves the issue to pop up other places. lineWidth is one, gl_FragCoord is another. So are the viewport and scissor settings. It seems better to make the render target size match that canvas so that everything is the same rather than make more and more workarounds and have to remember where to use one size vs another. Tomorrow I start using the PointsMaterial. It also has issues with devicePixelRatio. By not calling renderer.setPixelRatio those problems go away.