Most of the people have opposite problem, but I want to ask is there any simple way to get texture data back to the data URI.
I am using render to texture (RTT) to render the scene in background (with different camera). I want to be able to send this data to the server or just display in img element for test purpose.
Since you asked specifically for the render to texture case, there's a function for that in another answer: https://stackoverflow.com/a/18804083/3640489
Suppose you have a THREE.WebGLRenderTarget defined like so:
var renderTarget = new THREE.WebGLRenderTarget(window.innerWidth, window.innerHeight, {
minFilter: THREE.NearestFilter,
magFilter: THREE.NearestFilter,
format: THREE.RGBAFormat
});
Then you get the WebGLRenderingContext from the renderer and call the function:
var gl = renderer.getContext();
var webglTexture = renderTarget.__webglTexture;
var img = createImageFromTexture(gl, webglTexture, width, height);
Thanks guys for the help. I have figured out one solution. Probably it could be done simpler, but this solution works and looks quite fast.
RTT
this.renderer.render(this.scene, camera, this.rttTexture, true);
Get scene data.
this.pixelsIn = new Uint8Array(4 * 1024 * 1024);
var gl = this.renderer.context;
var framebuffer = this.rttTexture.__webglFramebuffer;
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
gl.viewport(0, 0, 1024, 1024);
gl.readPixels(0, 0, 1024, 1024, gl.RGBA, gl.UNSIGNED_BYTE, this.pixelsIn);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
Create 2d context.
this.printScreenCanvas = document.createElement("canvas");
this.printScreenCanvas.width = 1024;
this.printScreenCanvas.height = 1024;
this.printScreenContext = this.printScreenCanvas.getContext('2d');
Create storage for 2d context.
this.pixelsOut = this.printScreenContext.createImageData(1024, 1024);
Must convert data + flip Y.
var row, col, k = 4*1024;
for(var i=0; i<this.pixelsIn.length; i++) {
row = Math.floor(i/k);
col = i % k;
this.pixelsOut.data[(1023-row)*k+col] = this.pixelsIn[i];
}
Store data in 2d context.
this.printScreenContext.putImageData(this.pixelsOut,0,0);
Get data URI data.
return this.printScreenCanvas.toDataURL();
Would this do?
var gl = renderer.getContext();
var pixels = new Uint8Array(width * height * 4);
gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
your pixels should end up in the pixels array
its a little late but there is a really simple way.
The WebGLRenderer finally draws to a canvas object accessed by the domElement property.
HTML Canvas object has a standard implementation to grab the buffer and encode it into a Data URL.
If you want to display this screenshot into say an img HTML element then here is the example code. Assuming that the img HTML element has an id of "screenshotImageBox"
var imageDataURL = threeJSRenderer.domElement.toDataURL( "image/png" );
document.getElementById("screenshotImageBox").src = imageDataURL;
Related
I upload a large texture to my fragment shader as a map in three.js.
This texture is a 2D canvas on which I draw random shapes.
While drawing, I just want to update the part of the texture that has changed. Updating the whole texture (up to 3000x3000 pixel) is just too slow.
However, I can not get to perform a partial update. texSubImage2D doesn't have any effect in my scene.
No errors in the console - I expect that I am missing a step but can not figure it out yet.
//
// create drawing canvas 2D and related textures
//
this._canvas = document.createElement('canvas');
this._canvas.width = maxSliceDimensionsX;
this._canvas.height = maxSliceDimensionsY;
this._canvasContext = this._canvas.getContext('2d')!;
this._canvasContext.fillStyle = 'rgba(0, 0, 0, 0)';
this._canvasContext.fillRect(0, 0, window.innerWidth, window.innerHeight);
this._texture = new Texture(this._canvas);
this._texture.magFilter = NearestFilter;
this._texture.minFilter = NearestFilter;
// const data = new Uint8Array(maxSliceDimensionsX * maxSliceDimensionsY * 4);
// this._texture2 = new THREE.DataTexture(data, maxSliceDimensionsX, maxSliceDimensionsY, THREE.RGBAFormat);
this._brushMaterial = new MeshBasicMaterial({
map: this._texture,
side: DoubleSide,
transparent: true,
});
...
// in the render loop
renderLoop() {
...
// grab random values as a test
const imageData = this._canvasContext.getImageData(100, 100, 200, 200);
const uint8Array = new Uint8Array(imageData.data.buffer);
// activate texture?
renderer.setTexture2D(this._texture, 0 );
// update subtexture
const context = renderer.getContext();
context.texSubImage2D( context.TEXTURE_2D, 0, 0, 0, 200, 200, context.RGBA, context.UNSIGNED_BYTE, uint8Array);
// updating the whole texture works as expected but is slow
// this._texture.needsUpdate = true;
// Render new scene
renderer.render(scene, this._camera);
}
Thanks,
Nicolas
Right after creating the texture, we must notify three that the texture needs to be uploaded to the GPU.
We just have to set it 1 time at creation time and not in the render loop.
this._texture = new Texture(this._canvas);
this._texture.magFilter = NearestFilter;
this._texture.minFilter = NearestFilter;
// MUST BE ADDED
this._texture.needsUpdate = true;
I retrieve a rect from openSeadragonSelection:
viewer:
this.viewer = OpenSeadragon(this.config);
this.selection = this.viewer.selection({
showConfirmDenyButtons: true,
styleConfirmDenyButtons: true,
returnPixelCoordinates: true,
onSelection: rect => console.log(rect)
});
this.selection.enable();
rect by onSelection:
t.SelectionRect {x: 3502, y: 2265, width: 1122, height: 887, rotation:0, degrees: 0, …}
I have no idea how to get the canvas by rect from my viewer instance.
this.viewer.open(new OpenSeadragon.ImageTileSource(this.getTile(this.src)));
A self implemented imageViewer returned the canvas of the selected area. So I could get the blob and post it to the server:
onSave(canvas){
let source = canvas.toDataURL();
this.setState({source:source, crop: false, angle: 0});
save(this.dataURItoBlob(source), source.match(new RegExp("\/(.*);"))1]);
}
dataURItoBlob(dataURI) {
// convert base64/URLEncoded data component to raw binary data held in a string
var byteString;
if (dataURI.split(',')[0].indexOf('base64') >= 0)
byteString = atob(dataURI.split(',')[1]);
else
byteString = unescape(dataURI.split(',')[1]);
// separate out the mime component
var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
// write the bytes of the string to a typed array
var ia = new Uint8Array(byteString.length);
for (var i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
return new Blob([ia], {type:mimeString});
}
How can I get the image of the viewer by rect. Rotation should be considered as well.
#iangilman:
Thank's alot for your advice. I created another canvas which I crop and after that put it back into the viewer. I was not sure if something similar was supported by your library yet:
const viewportRect = self.viewer.viewport.imageToViewportRectangle(rect);
const webRect = self.viewer.viewport.viewportToViewerElementRectangle(viewportRect);
const { x, y, width, height } = webRect || {};
const { canvas } = self.viewer.drawer;
let source = canvas.toDataURL();
const img = new Image();
img.onload = function () {
let croppedCanvas = document.createElement('canvas');
let ctx = croppedCanvas.getContext('2d');
croppedCanvas.width = width;
croppedCanvas.height = height;
ctx.drawImage(img, x, y, width, height, 0, 0, width, height);
let croppedSrc = croppedCanvas.toDataURL();
//update viewer with cropped image
self.tile = self.getTile(croppedSrc);
self.ImageTileSource = new OpenSeadragon.ImageTileSource(self.tile);
self.viewer.open(self.ImageTileSource);
}
img.src = source;
Rotation hasn't been considered yet.
I imagine you'll need to convert the rectangle into the proper coordinates, then create a second canvas and copy the appropriate bit out of the OSD canvas into the second one.
Looks like maybe the selection rectangle is in image coordinates? The OSD canvas will be in web coordinates, or maybe double that on an HDPI display. OSD has a number of conversion functions, for instance:
var viewportRect = viewer.viewport.imageToViewportRectangle(imageRect);
var webRect = viewer.viewport.viewportToViewerElementRectangle(viewportRect);
You can find out the pixel density via OpenSeadragon.pixelDensityRatio.
Once you have the appropriate rectangle it should be easy to copy out of the one canvas into another. I'm not sure how you incorporate rotation, but it might be as simple as adding a rotation call to one of the canvas contexts.
Sorry this is kind of vague, but I hope it helps!
I have some code at https://jsfiddle.net/72mnd2yt/1/ that doesn't display the sprite I'm trying to draw. I tried to follow the code over at https://stemkoski.github.io/Three.js/Sprite-Text-Labels.html and read it line by line, but I'm not sure where I went wrong. would someone mind taking a look at it?
Here is some relevant code:
// picture
var getPicture = function(message){
var canvas = document.createElement('canvas');
canvas.width = "100%";
canvas.height = "100%";
var context = canvas.getContext('2d');
context.font = "10px";
context.fillText(message, 0, 10);
var picture = canvas.toDataURL();
// return picture;
return canvas;
};
// let there be light and so forth...
var getScene = function(){
var scene = new THREE.Scene();
var camera = new THREE.PerspectiveCamera(
70,
$('body').width(),
$('body').height(),
1,
1000
);
var light = new THREE.PointLight(0xeeeeee);
scene.add(camera);
scene.add(light);
var renderer = new THREE.WebGLRenderer();
renderer.setClearColor(0xefefef);
renderer.setSize($('body').width(), $('body').height());
camera.position.z = -10;
camera.lookAt(new THREE.Vector3(0, 0, 0));
return [scene, renderer, camera];
};
// now for the meat
var getLabel = function(message){
var texture = new THREE.Texture(getPicture(message));
var spriteMaterial = new THREE.SpriteMaterial(
{map: texture }
);
var sprite = new THREE.Sprite(spriteMaterial);
//sprite.scale.set(100, 50, 1.0);
return sprite
};
var setup = function(){
var scene;
var renderer;
var camera;
[scene, renderer, camera] = getScene();
$('body').append(renderer.domElement);
var label = getLabel("Hello, World!");
scene.add(label);
var animate = function(){
requestAnimationFrame(animate);
renderer.render(scene, camera);
};
animate();
};
setup();
A few points:
1) canvas.width = "100%" should be canvas.width = "100" (canvas sizes are assumed to be px). Same with canvas.height.
2) $('body').height() is 0, so the renderer canvas is not visible (you can check this out in dev tools, it's in the element tree but squashed to 0px high). I know nothing about jQuery, so not sure why this is, but I would recommend using window.innerHeight and window.innerWidth instead anyways. So renderer.setSize(window.innerWidth, window.innerHeight). You'll also want to make this change in the camera initialization.
3) Speaking of the camera initialization, you are passing in width and height as separate arguments, when there should only be an aspect ratio argument. See the docs. So
var camera = new THREE.PerspectiveCamera(70, window.innerWidth, window.innerHeight, 1, 1000)
becomes
var camera = new THREE.PerspectiveCamera(70, window.innerWidth/window.innerHeight, 1, 1000)
4) Because textures are assumed to be static, you need to add something to this part:
var texture = new THREE.Texture(getPicture(message))
texture.needsUpdate = true // this signals that the texture has changed
That's a one-time flag, so you need to set it every time the canvas changes if you want a dynamic texture. You don't have to make a new THREE.Texture each time, just add texture.needsUpdate in the render loop (or in an event that only fires when you want the texture to change, if you're going for efficiency). See the docs, under .needsUpdate.
At this point, it should work. Here are some further things to consider:
5) Instead of using Texture you could use CanvasTexture, which sets .needsUpdate for you. The fiddle you posted is using Three.js r71, which doesn't have it, but newer versions do. That would look like this:
var texture = new THREE.CanvasTexture(getPicture(message));
// no needsUpdate necessary
6) It looks like you were on this path already based on the commented out return picture, but you can use either canvas element, or a data url generated from the canvas for a texture. If getPicture now returns a data url, try this:
var texture = new THREE.TextureLoader().load(getPicture(message))
You can also indirectly use a data url with Texture:
var img = document.createElement('img')
var img = new Image()
img.src = getPicture(message)
var texture = new THREE.Texture(img);
texture.needsUpdate = true
If not, just stick with Texture or CanvasTexture, both will take a canvas element. Texture can indirectly take a url by passing in an image element whose src is set to the data url.
Fiddle with the outlined fixes:
https://jsfiddle.net/jbjw/x0uL1kbh/
Firstly, I'd like to make it clear that I am an absolute beginner when it comes to canvas & three.js
I've taken an existing project and am trying to make some minor modifications, namely, drawing an image on to a texture instead of just writing text. I would like the texture to have a set background color and will be attempting to draw a transparent png image on top of it.
The method that I am attempting to modify returns a texture for use in another function which is currently doing a lot more stuff that I probably won't be able to explain here.
I've tried a few approaches and I can't seem to get the behaviour that I want.
Approach #1
In this approach, I am getting the background color but I am not getting the image displayed
var ts = calc_texture_size(size + size * 2 * margin) * 2;
canvas.width = canvas.height = ts;
context.fillStyle = back_color;
context.fillRect(0, 0, canvas.width, canvas.height);
var img = new Image();
function drawIcon() {
context.drawImage(img,0,0);
}
img.onload = drawIcon();
img.src = "img/test.png";
var texture = new THREE.Texture(canvas);
texture.needsUpdate = true;
return texture;
Approach #2
After doing some research, it seemed I was supposed to use a TextureLoader but I couldn't figure out how to draw the color/shape on the canvas. In this case, the image appears on its own without any background color.
var imgTex = new new THREE.TextureLoader().load( "img/test.png",function(t){
texture.map = imgTex;
});
texture.needsUpdate = true;
return texture;
How can I successfully draw an image on top of a coloured background and return it as a texture for use by other functions?
Your problem is much related to JavaScript event handling, like a question I answered yesterday: How to make a Json model auto-rotates in the scene?. You can never have an "onload" and then expect that to return the variable for you - the variable is just not going to be defined until the callback is executed. Instead you need to defer the work to the callback, or elsewhere keep polling the variable until it's defined.
More specific to your question though:
With approach #1, you're sending the backfilled texture to the GPU, then rendering additional content to it, but not updating the texture that was sent to the GPU after the image was loaded. Moving the texture.needsUpdate = true; to your drawIcon() would probably solve it here. Untested example:
var ts = calc_texture_size(size + size * 2 * margin) * 2;
var img = new Image();
var texture = new THREE.Texture(canvas);
canvas.width = canvas.height = ts;
context.fillStyle = back_color;
context.fillRect(0, 0, canvas.width, canvas.height);
img.onload = function() {
context.drawImage(img,0,0);
texture.needsUpdate = true;
};
img.src = "img/test.png";
return texture;
NB: You should always define your var variables at the start of a scope in JavaScript to make things clearer ... drawIcon() has access to that texture variable, though it doesn't look like it from your example code because it appears to be defined after it, but that's not how JavaScript works: var variables are always global to the scope they're defined in.
With approach #2, you need a little extra work to paint on the texture, turning the texture back in to a canvas and sending the resulting canvas to the GPU. The ImageLoader and Texture documentation combined hint at how you need to paint to the image you get from the TextureLoader.
Combining the two approaches, it should be something like this (also utterly untested):
var ts = calc_texture_size(size + size * 2 * margin) * 2;
canvas.width = canvas.height = ts;
context.fillStyle = back_color;
context.fillRect(0, 0, canvas.width, canvas.height);
var textureLoader = new THREE.TextureLoader().load(
"img/test.png",
function( texture ) {
var image = texture.image;
context.drawImage( texture.image, 0, 0 0 );
var texture = new THREE.Texture(canvas);
texture.needsUpdate = true;
// Now do something with texture, like assigning it
// to your material
}
);
NB: Don't forget about using data URL's, if the image you need to paint with is small then this can be included in your code as a "data:" URL and this avoids the need to go to the server and wait for it to load in an event handler. Example at MDN.
I need to get the color of an image texture on a mesh at a given xyz point (mouse click + ray cast). How can I achieve it in THREE.js?
I know I could use gl.readPixels from plain webgl, but it's not a valid option for me.
Thanks!
So I ended using a separate canvas, in which I load the image texture, translate the three.js coordinates into canvas one and read the pixel. Something like this:
// point is a THREE.Vector3
var getColor = function(point, callback) {
var img = new Image();
img.src = 'assets/img/myImage.png';
img.onload = function() {
// get the xy coords from the point
var xyCoords = convertVector3ToXY(point);
// create a canvas to manipulate the image
var canvas = document.createElement('canvas');
canvas.width = img.width;
canvas.height = img.height;
canvas.getContext('2d').drawImage(img, 0, 0, img.width, img.height);
// get the pixel data and callback
var pixelData = canvas.getContext('2d').getImageData(x, y, 1, 1).data;
callback(pixelData);
}
};
Thanks