Related
I am trying to get the depth values of each pixel in the canvas element. Is there a way to find these depth values using WebGL and Three.js?
What I majorly want is that for eg. in the image below, the red background should have 0 as the depth value whereas the 3D model should have the depth values based on the distance from the camera.
Using the X,Y coordinates of the canvas, is there a method to access the depth values?
[Edit 1]: Adding more information
I pick three random points as shown below, then I ask the user to input the depth values for each of these points. Once the input is received from the user, I will compute the difference between the depth values in three.js and the values inputted from the user.
Basically, I would require a 2D array of the canvas size where each pixel corresponds to an array value. This 2D array must contain the value 0 if the pixel is a red background, or contain the depth value if the pixel contains the 3D model.
Two ways come to mind.
One you can just use RayCaster
body {
margin: 0;
}
#c {
width: 100vw;
height: 100vh;
display: block;
}
.info {
position: absolute;
left: 1em;
top: 1em;
padding: 1em;
background: rgba(0, 0, 0, 0.7);
color: white;
font-size: xx-small;
}
.info::after{
content: '';
position: absolute;
border: 10px solid transparent;
border-top: 10px solid rgba(0, 0, 0, 0.7);
top: 0;
left: -10px;
}
<canvas id="c"></canvas>
<script type="module">
// Three.js - Picking - RayCaster
// from https://threejsfundamentals.org/threejs/threejs-picking-raycaster.html
import * as THREE from 'https://threejsfundamentals.org/threejs/resources/threejs/r110/build/three.module.js';
function main() {
const canvas = document.querySelector('#c');
const renderer = new THREE.WebGLRenderer({canvas});
const fov = 60;
const aspect = 2; // the canvas default
const near = 0.1;
const far = 200;
const camera = new THREE.PerspectiveCamera(fov, aspect, near, far);
camera.position.z = 30;
const points = [
[170, 20],
[400, 50],
[225, 120],
].map((point) => {
const infoElem = document.createElement('pre');
document.body.appendChild(infoElem);
infoElem.className = "info";
infoElem.style.left = `${point[0] + 10}px`;
infoElem.style.top = `${point[1]}px`;
return {
point,
infoElem,
};
});
const scene = new THREE.Scene();
scene.background = new THREE.Color('white');
// put the camera on a pole (parent it to an object)
// so we can spin the pole to move the camera around the scene
const cameraPole = new THREE.Object3D();
scene.add(cameraPole);
cameraPole.add(camera);
{
const color = 0xFFFFFF;
const intensity = 1;
const light = new THREE.DirectionalLight(color, intensity);
light.position.set(-1, 2, 4);
camera.add(light);
}
const boxWidth = 1;
const boxHeight = 1;
const boxDepth = 1;
const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);
function rand(min, max) {
if (max === undefined) {
max = min;
min = 0;
}
return min + (max - min) * Math.random();
}
function randomColor() {
return `hsl(${rand(360) | 0}, ${rand(50, 100) | 0}%, 50%)`;
}
const numObjects = 100;
for (let i = 0; i < numObjects; ++i) {
const material = new THREE.MeshPhongMaterial({
color: randomColor(),
});
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);
cube.position.set(rand(-20, 20), rand(-20, 20), rand(-20, 20));
cube.rotation.set(rand(Math.PI), rand(Math.PI), 0);
cube.scale.set(rand(3, 6), rand(3, 6), rand(3, 6));
}
function resizeRendererToDisplaySize(renderer) {
const canvas = renderer.domElement;
const width = canvas.clientWidth;
const height = canvas.clientHeight;
const needResize = canvas.width !== width || canvas.height !== height;
if (needResize) {
renderer.setSize(width, height, false);
}
return needResize;
}
const raycaster = new THREE.Raycaster();
function render(time) {
time *= 0.001; // convert to seconds;
if (resizeRendererToDisplaySize(renderer)) {
const canvas = renderer.domElement;
camera.aspect = canvas.clientWidth / canvas.clientHeight;
camera.updateProjectionMatrix();
}
cameraPole.rotation.y = time * .1;
for (const {point, infoElem} of points) {
const pickPosition = {
x: (point[0] / canvas.clientWidth ) * 2 - 1,
y: (point[1] / canvas.clientHeight) * -2 + 1, // note we flip Y
};
raycaster.setFromCamera(pickPosition, camera);
const intersectedObjects = raycaster.intersectObjects(scene.children);
if (intersectedObjects.length) {
// pick the first object. It's the closest one
const intersection = intersectedObjects[0];
infoElem.textContent = `position : ${point[0]}, ${point[1]}
distance : ${intersection.distance.toFixed(2)}
z depth : ${((intersection.distance - near) / (far - near)).toFixed(3)}
local pos: ${intersection.point.x.toFixed(2)}, ${intersection.point.y.toFixed(2)}, ${intersection.point.z.toFixed(2)}
local uv : ${intersection.uv.x.toFixed(2)}, ${intersection.uv.y.toFixed(2)}`;
} else {
infoElem.textContent = `position : ${point[0]}, ${point[1]}`;
}
}
renderer.render(scene, camera);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
}
main();
</script>
The other way is to do as you mentioned and read the depth buffer. Unfortunately there is no direct way to read the depth buffer.
To read the depth values you need 2 render targets. You'd render to the first target. That gives you both a color texture with the rendered image and a depth texture with the depth values. You can't read a depth texture directly but you can draw it to another color texture and then read the color texture. Finally you can draw the first color texture to the cavnas.
body {
margin: 0;
}
#c {
width: 100vw;
height: 100vh;
display: block;
}
.info {
position: absolute;
left: 1em;
top: 1em;
padding: 1em;
background: rgba(0, 0, 0, 0.7);
color: white;
font-size: xx-small;
}
.info::after{
content: '';
position: absolute;
border: 10px solid transparent;
border-top: 10px solid rgba(0, 0, 0, 0.7);
top: 0;
left: -10px;
}
<canvas id="c"></canvas>
<script type="module">
import * as THREE from 'https://threejsfundamentals.org/threejs/resources/threejs/r110/build/three.module.js';
function main() {
const canvas = document.querySelector('#c');
const renderer = new THREE.WebGLRenderer({canvas});
const points = [
[170, 20],
[400, 50],
[225, 120],
].map((point) => {
const infoElem = document.createElement('pre');
document.body.appendChild(infoElem);
infoElem.className = "info";
infoElem.style.left = `${point[0] + 10}px`;
infoElem.style.top = `${point[1]}px`;
return {
point,
infoElem,
};
});
const renderTarget = new THREE.WebGLRenderTarget(1, 1);
renderTarget.depthTexture = new THREE.DepthTexture();
const depthRenderTarget = new THREE.WebGLRenderTarget(1, 1, {
depthBuffer: false,
stenciBuffer: false,
});
const rtFov = 60;
const rtAspect = 1;
const rtNear = 0.1;
const rtFar = 200;
const rtCamera = new THREE.PerspectiveCamera(rtFov, rtAspect, rtNear, rtFar);
rtCamera.position.z = 30;
const rtScene = new THREE.Scene();
rtScene.background = new THREE.Color('white');
// put the camera on a pole (parent it to an object)
// so we can spin the pole to move the camera around the scene
const cameraPole = new THREE.Object3D();
rtScene.add(cameraPole);
cameraPole.add(rtCamera);
{
const color = 0xFFFFFF;
const intensity = 1;
const light = new THREE.DirectionalLight(color, intensity);
light.position.set(-1, 2, 4);
rtCamera.add(light);
}
const boxWidth = 1;
const boxHeight = 1;
const boxDepth = 1;
const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);
function rand(min, max) {
if (max === undefined) {
max = min;
min = 0;
}
return min + (max - min) * Math.random();
}
function randomColor() {
return `hsl(${rand(360) | 0}, ${rand(50, 100) | 0}%, 50%)`;
}
const numObjects = 100;
for (let i = 0; i < numObjects; ++i) {
const material = new THREE.MeshPhongMaterial({
color: randomColor(),
});
const cube = new THREE.Mesh(geometry, material);
rtScene.add(cube);
cube.position.set(rand(-20, 20), rand(-20, 20), rand(-20, 20));
cube.rotation.set(rand(Math.PI), rand(Math.PI), 0);
cube.scale.set(rand(3, 6), rand(3, 6), rand(3, 6));
}
const camera = new THREE.OrthographicCamera(-1, 1, 1, -1, -1, 1);
const scene = new THREE.Scene();
camera.position.z = 1;
const sceneMaterial = new THREE.MeshBasicMaterial({
map: renderTarget.texture,
});
const planeGeo = new THREE.PlaneBufferGeometry(2, 2);
const plane = new THREE.Mesh(planeGeo, sceneMaterial);
scene.add(plane);
const depthScene = new THREE.Scene();
const depthMaterial = new THREE.MeshBasicMaterial({
map: renderTarget.depthTexture,
});
const depthPlane = new THREE.Mesh(planeGeo, depthMaterial);
depthScene.add(depthPlane);
function resizeRendererToDisplaySize(renderer) {
const canvas = renderer.domElement;
const width = canvas.clientWidth;
const height = canvas.clientHeight;
const needResize = canvas.width !== width || canvas.height !== height;
if (needResize) {
renderer.setSize(width, height, false);
}
return needResize;
}
let depthValues = new Uint8Array(0);
function render(time) {
time *= 0.001;
if (resizeRendererToDisplaySize(renderer)) {
const canvas = renderer.domElement;
renderTarget.setSize(canvas.width, canvas.height);
depthRenderTarget.setSize(canvas.width, canvas.height);
rtCamera.aspect = canvas.clientWidth / canvas.clientHeight;
rtCamera.updateProjectionMatrix();
}
cameraPole.rotation.y = time * .1;
// draw render target scene to render target
renderer.setRenderTarget(renderTarget);
renderer.render(rtScene, rtCamera);
renderer.setRenderTarget(null);
// render the depth texture to another render target
renderer.setRenderTarget(depthRenderTarget);
renderer.render(depthScene, camera);
renderer.setRenderTarget(null);
{
const {width, height} = depthRenderTarget;
const spaceNeeded = width * height * 4;
if (depthValues.length !== spaceNeeded) {
depthValues = new Uint8Array(spaceNeeded);
}
renderer.readRenderTargetPixels(
depthRenderTarget,
0,
0,
depthRenderTarget.width,
depthRenderTarget.height,
depthValues);
for (const {point, infoElem} of points) {
const offset = ((height - point[1] - 1) * width + point[0]) * 4;
infoElem.textContent = `position : ${point[0]}, ${point[1]}
z depth : ${(depthValues[offset] / 255).toFixed(3)}`;
}
}
// render the color texture to the canvas
renderer.render(scene, camera);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
}
main();
</script>
The problem is you can only read UNSIGNED_BYTE values from the texture so your depth values only go from 0 to 255 which is not really enough resolution to do much.
To solve that issue you have to encode the depth values across channels when drawing the depth texture to the 2nd render target which means you need to make your own shader. three.js has some shader snippets for packing the values so hacking a shader using ideas from this article we can get better depth values.
body {
margin: 0;
}
#c {
width: 100vw;
height: 100vh;
display: block;
}
.info {
position: absolute;
left: 1em;
top: 1em;
padding: 1em;
background: rgba(0, 0, 0, 0.7);
color: white;
font-size: xx-small;
}
.info::after{
content: '';
position: absolute;
border: 10px solid transparent;
border-top: 10px solid rgba(0, 0, 0, 0.7);
top: 0;
left: -10px;
}
<canvas id="c"></canvas>
<script type="module">
import * as THREE from 'https://threejsfundamentals.org/threejs/resources/threejs/r110/build/three.module.js';
function main() {
const canvas = document.querySelector('#c');
const renderer = new THREE.WebGLRenderer({canvas});
const points = [
[170, 20],
[400, 50],
[225, 120],
].map((point) => {
const infoElem = document.createElement('pre');
document.body.appendChild(infoElem);
infoElem.className = "info";
infoElem.style.left = `${point[0] + 10}px`;
infoElem.style.top = `${point[1]}px`;
return {
point,
infoElem,
};
});
const renderTarget = new THREE.WebGLRenderTarget(1, 1);
renderTarget.depthTexture = new THREE.DepthTexture();
const depthRenderTarget = new THREE.WebGLRenderTarget(1, 1, {
depthBuffer: false,
stenciBuffer: false,
});
const rtFov = 60;
const rtAspect = 1;
const rtNear = 0.1;
const rtFar = 200;
const rtCamera = new THREE.PerspectiveCamera(rtFov, rtAspect, rtNear, rtFar);
rtCamera.position.z = 30;
const rtScene = new THREE.Scene();
rtScene.background = new THREE.Color('white');
// put the camera on a pole (parent it to an object)
// so we can spin the pole to move the camera around the scene
const cameraPole = new THREE.Object3D();
rtScene.add(cameraPole);
cameraPole.add(rtCamera);
{
const color = 0xFFFFFF;
const intensity = 1;
const light = new THREE.DirectionalLight(color, intensity);
light.position.set(-1, 2, 4);
rtCamera.add(light);
}
const boxWidth = 1;
const boxHeight = 1;
const boxDepth = 1;
const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);
function rand(min, max) {
if (max === undefined) {
max = min;
min = 0;
}
return min + (max - min) * Math.random();
}
function randomColor() {
return `hsl(${rand(360) | 0}, ${rand(50, 100) | 0}%, 50%)`;
}
const numObjects = 100;
for (let i = 0; i < numObjects; ++i) {
const material = new THREE.MeshPhongMaterial({
color: randomColor(),
});
const cube = new THREE.Mesh(geometry, material);
rtScene.add(cube);
cube.position.set(rand(-20, 20), rand(-20, 20), rand(-20, 20));
cube.rotation.set(rand(Math.PI), rand(Math.PI), 0);
cube.scale.set(rand(3, 6), rand(3, 6), rand(3, 6));
}
const camera = new THREE.OrthographicCamera(-1, 1, 1, -1, -1, 1);
const scene = new THREE.Scene();
camera.position.z = 1;
const sceneMaterial = new THREE.MeshBasicMaterial({
map: renderTarget.texture,
});
const planeGeo = new THREE.PlaneBufferGeometry(2, 2);
const plane = new THREE.Mesh(planeGeo, sceneMaterial);
scene.add(plane);
const depthScene = new THREE.Scene();
const depthMaterial = new THREE.MeshBasicMaterial({
map: renderTarget.depthTexture,
});
depthMaterial.onBeforeCompile = function(shader) {
// the <packing> GLSL chunk from three.js has the packDeathToRGBA function.
// then at the end of the shader the default MaterialBasicShader has
// already read from the material's `map` texture (the depthTexture)
// which has depth in 'r' and assigned it to gl_FragColor
shader.fragmentShader = shader.fragmentShader.replace(
'#include <common>',
'#include <common>\n#include <packing>',
).replace(
'#include <fog_fragment>',
'gl_FragColor = packDepthToRGBA( gl_FragColor.r );',
);
};
const depthPlane = new THREE.Mesh(planeGeo, depthMaterial);
depthScene.add(depthPlane);
function resizeRendererToDisplaySize(renderer) {
const canvas = renderer.domElement;
const width = canvas.clientWidth;
const height = canvas.clientHeight;
const needResize = canvas.width !== width || canvas.height !== height;
if (needResize) {
renderer.setSize(width, height, false);
}
return needResize;
}
let depthValues = new Uint8Array(0);
function render(time) {
time *= 0.001;
if (resizeRendererToDisplaySize(renderer)) {
const canvas = renderer.domElement;
renderTarget.setSize(canvas.width, canvas.height);
depthRenderTarget.setSize(canvas.width, canvas.height);
rtCamera.aspect = canvas.clientWidth / canvas.clientHeight;
rtCamera.updateProjectionMatrix();
}
cameraPole.rotation.y = time * .1;
// draw render target scene to render target
renderer.setRenderTarget(renderTarget);
renderer.render(rtScene, rtCamera);
renderer.setRenderTarget(null);
// render the depth texture to another render target
renderer.setRenderTarget(depthRenderTarget);
renderer.render(depthScene, camera);
renderer.setRenderTarget(null);
{
const {width, height} = depthRenderTarget;
const spaceNeeded = width * height * 4;
if (depthValues.length !== spaceNeeded) {
depthValues = new Uint8Array(spaceNeeded);
}
renderer.readRenderTargetPixels(
depthRenderTarget,
0,
0,
depthRenderTarget.width,
depthRenderTarget.height,
depthValues);
for (const {point, infoElem} of points) {
const offset = ((height - point[1] - 1) * width + point[0]) * 4;
const depth = depthValues[offset ] * ((255 / 256) / (256 * 256 * 256)) +
depthValues[offset + 1] * ((255 / 256) / (256 * 256)) +
depthValues[offset + 2] * ((255 / 256) / 256);
infoElem.textContent = `position : ${point[0]}, ${point[1]}
z depth : ${depth.toFixed(3)}`;
}
}
// render the color texture to the canvas
renderer.render(scene, camera);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
}
main();
</script>
Note depthTexture uses a webgl extension which is an optional feature not found on all devices
To work around that would require drawing the scene twice. Once with your normal materials and then again to a color render target using the MeshDepthMaterial.
I have created a flutter widget that moves the circle with accelerometer. It is very laggy as I have to use setState to change the position of the circle when phone is moved. Is there an alternative to creating this?
I have used AnimatedBuilder here but not sure how that can change the position of circle when device is moved smoothly.
class _AnimationWidgetState extends State<AnimationWidget>
with TickerProviderStateMixin {
AnimationController _animeController;
Animation _anime;
double x = 0.0, y = 0.0;
#override
void initState() {
super.initState();
_animeController =
AnimationController(vsync: this, duration: const Duration(seconds: 2));
_anime = Tween(begin: 0.5, end: 0.5).animate(
CurvedAnimation(parent: _animeController, curve: Curves.ease));
accelerometerEvents.listen((AccelerometerEvent event) {
var a = ((event.x * 100).round() / 100).clamp(-1.0, 1.0) * -1;
var b = ((event.y * 100).round() / 100).clamp(-1.0, 1.0);
if ((x - a).abs() > 0.02 || (y - b).abs() > 0.02) {
setState(() {
x = a; y = b;
});
}
});
}
#override
void dispose() {
_animeController.dispose();
super.dispose();
}
#override
Widget build(BuildContext context) {
_animeController.forward();
final double width = MediaQuery.of(context).size.width;
final double height = MediaQuery.of(context).size.height;
return AnimatedBuilder(
animation: _animeController,
builder: (context, child) {
return Scaffold(
body: Transform(
transform: Matrix4.translationValues(
_anime.value * width * x, _anime.value * height * y, 0.0),
child: Center(
child: CircleAvatar(
radius: 15.0,
backgroundColor: Colors.green,
),
),
),
);
},
);
}
}
Animation is not at all smooth. This is because I have to use setState but the movement of circle is working as desired.
The whole purpose of using AnimationController is to listen to its events - that's what AnimatedBuilder does and rebuilds its subtree accordingly.
I will post here my overall recommendations on what to change in the code.
Remove setState - that's what makes your entire layout rebuild all over again, i.e. lags.
Also trigger _animeController listeners, i.e. AnimatedBuilder in your case - to rebuild itself.
accelerometerEvents.listen((AccelerometerEvent event) {
var a = ((event.x * 100).round() / 100).clamp(-1.0, 1.0) * -1;
var b = ((event.y * 100).round() / 100).clamp(-1.0, 1.0);
if ((x - a).abs() > 0.02 || (y - b).abs() > 0.02) {
x = a; y = b;
_animeController.value = _animeController.value; // Trigger controller's listeners
}
});
Start animation in initState, instead of build. This is the second thing that produces lags in your case. .forward triggers rebuild of your widgets, which leads to an infinite loop.
#override
void initState() {
super.initState();
_animeController.forward();
}
Use child property of your AnimatedBuilder to save up resources on rebuilding the avatar block every time. Also I'm not sure what Scaffold is here for - let's remove it if it's not needed.
AnimatedBuilder(
animation: _animeController,
builder: (context, child) => Transform(
transform: Matrix4.translationValues(_anime.value * width * x, _anime.value * height * y, 0.0),
child: child,
),
child: Center(
child: CircleAvatar(
radius: 15.0,
backgroundColor: Colors.green,
),
),
);
Follow Official Animations Tutorial to master Flutter animations.
Let me know if this helps.
I have a texture of size 800x600. How do I scale it on a webgl <canvas> at another size and keep the original aspect ratio? Assuming that the drawing buffer and the canvas have the same dimensions.
Given the WebGL only cares about clipsapce coordinates you can just draw a 2 unit quad (-1 to +1) and scale it by the aspect of the canvas vs the aspect of the image.
In other words
const canvasAspect = canvas.clientWidth / canvas.clientHeight;
const imageAspect = image.width / image.height;
let scaleY = 1;
let scaleX = imageAspect / canvasAspect;
Note that you need to decide how you want to fit the image. scaleY= 1 means the image will always fit vertically and horizontally will just be whatever it comes out to.
If you want it to fit horizontally then you need to make scaleX = 1
let scaleX = 1;
let scaleY = canvasAspect / imageAspect;
If you want it to contain then
let scaleY = 1;
let scaleX = imageAspect / canvasAspect;
if (scaleX > 1) {
scaleY = 1 / scaleX;
scaleX = 1;
}
If you want it to cover then
let scaleY = 1;
let scaleX = imageAspect / canvasAspect;
if (scaleX < 1) {
scaleY = 1 / scaleX;
scaleX = 1;
}
let scaleMode = 'fitV';
const gl = document.querySelector("canvas").getContext('webgl');
const vs = `
attribute vec4 position;
uniform mat4 u_matrix;
varying vec2 v_texcoord;
void main() {
gl_Position = u_matrix * position;
v_texcoord = position.xy * .5 + .5; // because we know we're using a -1 + 1 quad
}
`;
const fs = `
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_tex;
void main() {
gl_FragColor = texture2D(u_tex, v_texcoord);
}
`;
let image = { width: 1, height: 1 }; // dummy until loaded
const tex = twgl.createTexture(gl, {
src: 'https://i.imgur.com/TSiyiJv.jpg',
crossOrigin: 'anonymous',
}, (err, tex, img) => {
// called after image as loaded
image = img;
render();
});
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
const bufferInfo = twgl.createBufferInfoFromArrays(gl, {
position: {
numComponents: 2,
data: [
-1, -1, // tri 1
1, -1,
-1, 1,
-1, 1, // tri 2
1, -1,
1, 1,
],
}
});
function render() {
// this line is not needed if you don't
// care that the canvas drawing buffer size
// matches the canvas display size
twgl.resizeCanvasToDisplaySize(gl.canvas);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.useProgram(programInfo.program);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
const canvasAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const imageAspect = image.width / image.height;
let scaleX;
let scaleY;
switch (scaleMode) {
case 'fitV':
scaleY = 1;
scaleX = imageAspect / canvasAspect;
break;
case 'fitH':
scaleX = 1;
scaleY = canvasAspect / imageAspect;
break;
case 'contain':
scaleY = 1;
scaleX = imageAspect / canvasAspect;
if (scaleX > 1) {
scaleY = 1 / scaleX;
scaleX = 1;
}
break;
case 'cover':
scaleY = 1;
scaleX = imageAspect / canvasAspect;
if (scaleX < 1) {
scaleY = 1 / scaleX;
scaleX = 1;
}
break;
}
twgl.setUniforms(programInfo, {
u_matrix: [
scaleX, 0, 0, 0,
0, -scaleY, 0, 0,
0, 0, 1, 0,
0, 0, 0, 1,
],
});
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
render();
window.addEventListener('resize', render);
document.querySelectorAll('button').forEach((elem) => {
elem.addEventListener('click', setScaleMode);
});
function setScaleMode(e) {
scaleMode = e.target.id;
render();
}
html, body {
margin: 0;
height: 100%;
}
canvas {
width: 100%;
height: 100%;
display: block;
}
.ui {
position: absolute;
left: 0;
top: 0;
}
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
<div class="ui">
<button id="fitV">fit vertical</button>
<button id="fitH">fit horizontal</button>
<button id="contain">contain</button>
<button id="cover">cover</button>
</div>
The code above uses a 4x4 matrix to apply the scale
gl_Position = u_matrix * position;
It could just as easily pass in the scale directly
uniform vec2 scale;
...
gl_Position = vec4(scale * position.xy, 0, 1);
I'm trying to work out how to warp all coordinates in a Three.js scene around a specific pivot point / axis. The best way to describe it is as if I was to place a tube somewhere in the scene and everything else in the scene would curve around that axis and keep the same distance from that axis.
If it helps, this diagram is what I'm trying to achieve. The top part is as if you were looking at the scene from the side and the bottom part is as if you were looking at it from a perspective. The red dot / line is where the pivot point is.
To further complicate matters, I'd like to stop the curve / warp from wrapping back on itself, so the curve stops when it's horizontal or vertical like the top-right example in the diagram.
Any insight into how to achieve this using GLSL shaders, ideally in Three.js but I'll try to translate if they can be described clearly otherwise?
I'm also open to alternative approaches to this as I'm unsure how best to describe what I'm after. Basically I want an inverted "curved world" effect where the scene is bending up and away from you.
First I'd do it in 2D just like your top diagram.
I have no idea if this is the correct way to do this or even a good way but, doing it in 2D seemed easier than 3D and besides the effect you want is actually a 2D. X is not changing at all, only Y, and Z so solving it in 2D seems like it would lead to solution.
Basically we choose a radius for a circle. At that radius for every unit of X past the circle's center we want to wrap one horizontal unit to one unit around the circle. Given the radius we know the distance around the circle is 2 * PI * radius so we can easily compute how far to rotate around our circle to get one unit. It's just 1 / circumference * Math.PI * 2 We do that for some specified distance past the circle's center
const m4 = twgl.m4;
const v3 = twgl.v3;
const ctx = document.querySelector('canvas').getContext('2d');
const gui = new dat.GUI();
resizeToDisplaySize(ctx.canvas);
const g = {
rotationPoint: {x: 100, y: ctx.canvas.height / 2 - 50},
radius: 50,
range: 60,
};
gui.add(g.rotationPoint, 'x', 0, ctx.canvas.width).onChange(render);
gui.add(g.rotationPoint, 'y', 0, ctx.canvas.height).onChange(render);
gui.add(g, 'radius', 1, 100).onChange(render);
gui.add(g, 'range', 0, 300).onChange(render);
render();
window.addEventListener('resize', render);
function render() {
resizeToDisplaySize(ctx.canvas);
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
const start = g.rotationPoint.x;
const curveAmount = g.range / g.radius;
const y = ctx.canvas.height / 2;
drawDot(ctx, g.rotationPoint.x, g.rotationPoint.y, 'red');
ctx.beginPath();
ctx.arc(g.rotationPoint.x, g.rotationPoint.y, g.radius, 0, Math.PI * 2, false);
ctx.strokeStyle = 'red';
ctx.stroke();
ctx.fillStyle = 'black';
const invRange = g.range > 0 ? 1 / g.range : 0; // so we don't divide by 0
for (let x = 0; x < ctx.canvas.width; x += 5) {
for (let yy = 0; yy <= 30; yy += 10) {
const sign = Math.sign(g.rotationPoint.y - y);
const amountToApplyCurve = clamp((x - start) * invRange, 0, 1);
let mat = m4.identity();
mat = m4.translate(mat, [g.rotationPoint.x, g.rotationPoint.y, 0]);
mat = m4.rotateZ(mat, curveAmount * amountToApplyCurve * sign);
mat = m4.translate(mat, [-g.rotationPoint.x, -g.rotationPoint.y, 0]);
const origP = [x, y + yy, 0];
origP[0] += -g.range * amountToApplyCurve;
const newP = m4.transformPoint(mat, origP);
drawDot(ctx, newP[0], newP[1], 'black');
}
}
}
function drawDot(ctx, x, y, color) {
ctx.fillStyle = color;
ctx.fillRect(x - 1, y - 1, 3, 3);
}
function clamp(v, min, max) {
return Math.min(max, Math.max(v, min));
}
function resizeToDisplaySize(canvas) {
const width = canvas.clientWidth;
const height = canvas.clientHeight;
if (canvas.width !== width || canvas.height !== height) {
canvas.width = width;
canvas.height = height;
}
}
body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }
<canvas></canvas>
<!-- using twgl just for its math library -->
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/dat-gui/0.7.2/dat.gui.min.js"></script>
Notice the only place that matches perfectly is when the radius touches a line of points. Inside the radius things will get pinched, outside they'll get stretched.
Putting that in a shader in the Z direction for actual use
const renderer = new THREE.WebGLRenderer({
canvas: document.querySelector('canvas'),
});
const gui = new dat.GUI();
const scene = new THREE.Scene();
const fov = 75;
const aspect = 2; // the canvas default
const zNear = 1;
const zFar = 1000;
const camera = new THREE.PerspectiveCamera(fov, aspect, zNear, zFar);
function lookSide() {
camera.position.set(-170, 35, 210);
camera.lookAt(0, 25, 210);
}
function lookIn() {
camera.position.set(0, 35, -50);
camera.lookAt(0, 25, 0);
}
{
scene.add(new THREE.HemisphereLight(0xaaaaaa, 0x444444, .5));
const light = new THREE.DirectionalLight(0xffffff, 1);
light.position.set(-1, 20, 4 - 15);
scene.add(light);
}
const point = function() {
const material = new THREE.MeshPhongMaterial({
color: 'red',
emissive: 'hsl(0,50%,25%)',
wireframe: true,
});
const radiusTop = 1;
const radiusBottom = 1;
const height = 0.001;
const radialSegments = 32;
const geo = new THREE.CylinderBufferGeometry(
radiusTop, radiusBottom, height, radialSegments);
const sphere = new THREE.Mesh(geo, material);
sphere.rotation.z = Math.PI * .5;
const mesh = new THREE.Object3D();
mesh.add(sphere);
scene.add(mesh);
mesh.position.y = 88;
mesh.position.z = 200;
return {
point: mesh,
rep: sphere,
};
}();
const vs = `
// -------------------------------------- [ VS ] ---
#define PI radians(180.0)
uniform mat4 center;
uniform mat4 invCenter;
uniform float range;
uniform float radius;
varying vec3 vNormal;
mat4 rotZ(float angleInRadians) {
float s = sin(angleInRadians);
float c = cos(angleInRadians);
return mat4(
c,-s, 0, 0,
s, c, 0, 0,
0, 0, 1, 0,
0, 0, 0, 1);
}
mat4 rotX(float angleInRadians) {
float s = sin(angleInRadians);
float c = cos(angleInRadians);
return mat4(
1, 0, 0, 0,
0, c, s, 0,
0, -s, c, 0,
0, 0, 0, 1);
}
void main() {
float curveAmount = range / radius;
float invRange = range > 0.0 ? 1.0 / range : 0.0;
vec4 mvPosition = modelViewMatrix * vec4(position, 1.0);
vec4 point = invCenter * mvPosition;
float amountToApplyCurve = clamp(point.z * invRange, 0.0, 1.0);
float s = sign(point.y);
mat4 mat = rotX(curveAmount * amountToApplyCurve * s);
point = center * mat * (point + vec4(0, 0, -range * amountToApplyCurve, 0));
vNormal = mat3(mat) * normalMatrix * normal;
gl_Position = projectionMatrix * point;
}
`;
const fs = `
// -------------------------------------- [ FS ] ---
varying vec3 vNormal;
uniform vec3 color;
void main() {
vec3 light = vec3( 0.5, 2.2, 1.0 );
light = normalize( light );
float dProd = dot( vNormal, light ) * 0.5 + 0.5;
gl_FragColor = vec4( vec3( dProd ) * vec3( color ), 1.0 );
}
`;
const centerUniforms = {
radius: { value: 0 },
range: { value: 0 },
center: { value: new THREE.Matrix4() },
invCenter: { value: new THREE.Matrix4() },
};
function addUniforms(uniforms) {
return Object.assign(uniforms, centerUniforms);
}
{
const uniforms = addUniforms({
color: { value: new THREE.Color('hsl(100,50%,50%)') },
});
const material = new THREE.ShaderMaterial( {
uniforms: uniforms,
vertexShader: vs,
fragmentShader: fs,
});
const planeGeo = new THREE.PlaneBufferGeometry(1000, 1000, 100, 100);
const mesh = new THREE.Mesh(planeGeo, material);
mesh.rotation.x = Math.PI * -.5;
scene.add(mesh);
}
{
const uniforms = addUniforms({
color: { value: new THREE.Color('hsl(180,50%,50%)' ) },
});
const material = new THREE.ShaderMaterial( {
uniforms: uniforms,
vertexShader: vs,
fragmentShader: fs,
});
const boxGeo = new THREE.BoxBufferGeometry(10, 10, 10, 20, 20, 20);
for (let x = -41; x <= 41; x += 2) {
for (let z = 0; z <= 40; z += 2) {
const base = new THREE.Object3D();
const mesh = new THREE.Mesh(boxGeo, material);
mesh.position.set(0, 5, 0);
base.position.set(x * 10, 0, z * 10);
base.scale.y = 1 + Math.random() * 2;
base.add(mesh);
scene.add(base);
}
}
}
const g = {
radius: 59,
range: 60,
side: true,
};
class DegRadHelper {
constructor(obj, prop) {
this.obj = obj;
this.prop = prop;
}
get v() {
return THREE.Math.radToDeg(this.obj[this.prop]);
}
set v(v) {
this.obj[this.prop] = THREE.Math.degToRad(v);
}
}
gui.add(point.point.position, 'z', -300, 300).onChange(render);
gui.add(point.point.position, 'y', -150, 300).onChange(render);
gui.add(g, 'radius', 1, 100).onChange(render);
gui.add(g, 'range', 0, 300).onChange(render);
gui.add(g, 'side').onChange(render);
gui.add(new DegRadHelper(point.point.rotation, 'x'), 'v', -180, 180).name('rotX').onChange(render);
gui.add(new DegRadHelper(point.point.rotation, 'y'), 'v', -180, 180).name('rotY').onChange(render);
gui.add(new DegRadHelper(point.point.rotation, 'z'), 'v', -180, 180).name('rotZ').onChange(render);
render();
window.addEventListener('resize', render);
function render() {
if (resizeToDisplaySize(renderer)) {
const canvas = renderer.domElement;
camera.aspect = canvas.clientWidth / canvas.clientHeight;
camera.updateProjectionMatrix();
}
if (g.side) {
lookSide();
} else {
lookIn();
}
camera.updateMatrixWorld();
point.rep.scale.set(g.radius, g.radius, g.radius);
point.point.updateMatrixWorld();
centerUniforms.center.value.multiplyMatrices(
camera.matrixWorldInverse, point.point.matrixWorld);
centerUniforms.invCenter.value.getInverse(centerUniforms.center.value);
centerUniforms.range.value = g.range;
centerUniforms.radius.value = g.radius;
renderer.render(scene, camera);
}
function resizeToDisplaySize(renderer) {
const canvas = renderer.domElement;
const width = canvas.clientWidth;
const height = canvas.clientHeight;
const needUpdate = canvas.width !== width || canvas.height !== height;
if (needUpdate) {
renderer.setSize(width, height, false);
}
return needUpdate;
}
body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }
<canvas></canvas>
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/95/three.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/dat-gui/0.7.2/dat.gui.min.js"></script>
Honestly I have a feeling there's an easier way I'm missing but for the moment it seems to kind of be working.
I don't really know linear algebra, I'm trying to learn it because of this issue now.
I'm pretty sure it's kind of simple question for those who knows linear algebra.
I'll try to explain and show some pictures, videos and code to describe it better.
First, my goal is pretty simple at first glance.
Is to create this sort of animation
Now I managed to get almost there
Widget getWidget(Type type)
{
Matrix4 _pmat(num pv) {
return new Matrix4(
1.0, 0.0, 0.0, 0.0, //
0.0, 1.0, 0.0, 0.0, //
0.0, 0.0, 1.0, pv * 0.001, //
0.0, 0.0, 0.0, 1.0,
);
}
double delta = _panDown ? _delta : _animationOffset.value;
Matrix4 perspective = _pmat(0.35);
if (type == Type.NEXT)
{
delta -= MAX_DELTA;
}
double xOffset = ((delta / 70) / 2.5) * (deviceWidth / 2);
double zOffset = (delta / 70 / (math.pi - 0.55)).abs() * 180.0;
double rotateY = (math.pi - (delta / 3) * math.pi / 180) - math.pi;
if (type == Type.NEXT)
{
xOffset = ((delta / 70) / 2.5) * (deviceWidth);
zOffset *= 2;
}
perspective.translate(xOffset, 0.0, zOffset);
double angleOffset = delta * -0.5;
return new Transform(
alignment: FractionalOffset.center,
transform: perspective.scaled(1.0, 1.0, 1.0)
..rotateX(0.0)
..rotateY(rotateY)
..rotateZ(0.0),
child: new Container(color: Colors.red),
);
}
As you can see I'm currently moving each "page" with "magic numbers" and I call it magic numbers because I'm pretty dam noob at linear algebra and what I've done here is with the little understanding I have + trial and error. Clearly not the right way.
I have the delta which is basically just an offset taken from the user's GestureDetector to just play with it.
Now, with all the samples of this cube transition I saw let's say for example in CSS, they just rotated/translated each individual side. And then moved/rotated the "camera" or the parent I guess ?
I tried to do that method as well:
Like this:
Widget getWidget(Type type)
{
AlignmentGeometry alignmentGeometry;
Matrix4 matrix4;
switch (type)
{
case Type.PREVIOUS:
alignmentGeometry = FractionalOffset.center;
matrix4 = Matrix4.identity()
..setEntry(3, 2, 0.001) // perspective
..rotateY(-90.0 * math64.degrees2Radians)
..translate(deviceWidth / 2, 0.0, deviceWidth / 2);
break;
case Type.CURRENT:
alignmentGeometry = FractionalOffset.center;
matrix4 = Matrix4.identity()
..setEntry(3, 2, 0.001);
break;
case Type.NEXT:
alignmentGeometry = FractionalOffset.center;
matrix4 = Matrix4.identity()
..setEntry(3, 2, 0.001) // perspective
..rotateY(90.0 * math64.degrees2Radians)
..translate(-deviceWidth / 2, 0.0, deviceWidth / 2);
break;
}
return new Transform(
alignment: alignmentGeometry,
transform: matrix4,
child: new Container(
color: type == Type.NEXT ? Colors.green : type == Type.PREVIOUS ? Colors.red : Colors.black.withOpacity(0.5),
),
);
}
And then let's simply rotate the parent right ?
Widget build(BuildContext context)
{
List<Widget> widgets = [];
widgets.add(getWidget(Type.PREVIOUS));
widgets.add(getWidget(Type.CURRENT));
widgets.add(getWidget(Type.NEXT));
return new Scaffold(
key: _scaffoldKey,
body: new Container(
color: Colors.black,
child: new GestureDetector(
child: new Container(
color: Colors.white,
child: new Transform(
alignment: FractionalOffset.center,
transform: Matrix4.identity()
..setEntry(3, 2, 0.0001)
..rotateY(45.0 * math64.degrees2Radians),
child: new Stack(
children: widgets,
),
)
),
)
)
);
}
No:
Having it 90 degrees will make it all disappear.
Now, I do understand that I need sort of "pivot rotation point" or change the "origin" or "anchor point" not sure even how to call it.
So to picture it, let's say for example we also have a bottom side. so I want the center of that bottom side (let's say for example in a cube). So it will look like this:
Now if I'll rotate that "origin point" in the Y axis I should get the right animation right ?