I am working on an in-browser image processing tool, and have run into a bizarre performance issue in Firefox.
If I modify a canvas, then immediately get the canvas' image data, data is retrieved at the expected fast speed.
If I modify a canvas, await anything, then get the image data, this takes orders of magnitude longer.
I have reduced the issue to a minimal example, provided below. This example consists of two tests. Test 1 writes to the canvas, then immediately simulates some canvas work by calculating the average luminosity of the canvas' pixels on a column-by-column basis. Test 2 writes to the canvas, goes to sleep for 0ms, then simulates the same work.
As an entirely unscientific sample, running this test on my desktop (Win10, i7-5820K, 16GB RAM, GeForce GT 1030, Firefox 90.0.2) produces these values for different canvas sizes:
Canvas size Test 1 (ms) Test 2 (ms) ΔFactor
512x 512 8.0 170.0 21x
1024x1024 31.0 1116.0 36x
2048x2048 109.0 8720.0 80x
4096x4096 631.0 76447.0 121x
8192x8192 3415.0 n/a*) n/a *) Script terminated by timeout
During the runtime, Firefox did not consume an extraordinary amount of memory. However, the tab's thread was consistently and fully loading its CPU core throughout.
Chrome does not exhibit this behavior, completing the 8192x8192 tests in 5428ms and 4536ms respectively.
Can anyone explain this?
canvas.html:
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<script defer src="canvas.js"></script>
</head>
<body>
<div style="display: flex; flex-direction: row;">
<button id="test1">Run Test 1</button>
<div id="test1-result"></div>
</div>
<div style="display: flex; flex-direction: row;">
<button id="test2">Run Test 2</button>
<div id="test2-result"></div>
</div>
<canvas id="canvas"></canvas>
</body>
</html>
canvas.js:
window.sleep = ((ms) => new Promise((r) => setTimeout(r, ms)));
const SIZE = 2048;
const canvas = document.getElementById('canvas');
let doSetup = (() =>
{
canvas.width = SIZE;
canvas.height = SIZE;
const ctx = canvas.getContext('2d');
ctx.fillStyle = '#fff';
ctx.fillRect(0,0,SIZE,SIZE);
});
let doTest = (() =>
{
const ctx = canvas.getContext('2d');
let sum = 0;
for (let x=0; x<SIZE; ++x)
{
const data = ctx.getImageData(x,0,1,SIZE);
for (let i=0; i<data.data.length; i+=4)
sum += (data.data[i+0]*.299 + data.data[i+1]*.587 + data.data[i+2]*.114);
}
return sum/SIZE/SIZE;
});
document.getElementById('test1').addEventListener('click', async () =>
{
doSetup();
/* await sleep(0); */
const start = performance.now();
const result = doTest();
const elapsed = (performance.now() - start);
document.getElementById('test1-result').innerText = ('Test1 done, took '+elapsed+'ms, avg lum='+result);
});
document.getElementById('test2').addEventListener('click', async () =>
{
doSetup();
await sleep(0);
const start = performance.now();
const result = doTest();
const elapsed = (performance.now() - start);
document.getElementById('test2-result').innerText = ('Test2 done, took '+elapsed+'ms, avg lum='+result);
});
Related
I would like to use MediaStream.captureStream() method, but it is either rendered useless due to specification and bugs or I am using it totally wrong.
I know that captureStream gets maximal framerate as the parameter, not constant and it does not even guarantee that, but it is possible to change MediaStream currentTime (currently in Chrome, in Firefox it has no effect but in return there is requestFrame, not available at Chrome), but the idea of manual frame requests or setting the placement of the frame in the MediaStream should override this effect. It doesn't.
In Firefox it smoothly renders the video, frame by frame, but the video result is as long as wall clock time used for processing.
In Chrome there are some dubious black frames or reordered ones (currently I do not care about it until the FPS matches), and the manual setting of currentTime gives nothing, the same result as in FF.
I use modified code from MediaStream Capture Canvas and Audio Simultaneously answer.
const FPS = 30;
var cStream, vid, recorder, chunks = [], go = true,
Q = 61, rec = document.getElementById('rec'),
canvas = document.getElementById('canvas'),
ctx = canvas.getContext('2d');
ctx.strokeStyle = 'rgb(255, 0, 0)';
function clickHandler() {
this.textContent = 'stop recording';
//it has no effect no matter if it is empty or set to 30
cStream = canvas.captureStream(FPS);
recorder = new MediaRecorder(cStream);
recorder.ondataavailable = saveChunks;
recorder.onstop = exportStream;
this.onclick = stopRecording;
recorder.start();
draw();
}
function exportStream(e) {
if (chunks.length) {
var blob = new Blob(chunks)
var vidURL = URL.createObjectURL(blob);
var vid2 = document.createElement('video');
vid2.controls = true;
vid2.src = vidURL;
vid2.onend = function() {
URL.revokeObjectURL(vidURL);
}
document.body.insertBefore(vid2, vid);
} else {
document.body.insertBefore(document.createTextNode('no data saved'), canvas);
}
}
function saveChunks(e) {
e.data.size && chunks.push(e.data);
}
function stopRecording() {
go = false;
this.parentNode.removeChild(this);
recorder.stop();
}
var loadVideo = function() {
vid = document.createElement('video');
document.body.insertBefore(vid, canvas);
vid.oncanplay = function() {
rec.onclick = clickHandler;
rec.disabled = false;
canvas.width = vid.videoWidth;
canvas.height = vid.videoHeight;
vid.oncanplay = null;
ctx.drawImage(vid, 0, 0);
}
vid.onseeked = function() {
ctx.drawImage(vid, 0, 0);
/*
Here I want to include additional drawing per each frame,
for sure taking more than 180ms
*/
if(cStream && cStream.requestFrame) cStream.requestFrame();
draw();
}
vid.crossOrigin = 'anonymous';
vid.src = 'https://dl.dropboxusercontent.com/s/bch2j17v6ny4ako/movie720p.mp4';
vid.currentTime = 0;
}
function draw() {
if(go && cStream) {
++Q;
cStream.currentTime = Q / FPS;
vid.currentTime = Q / FPS;
}
};
loadVideo();
<button id="rec" disabled>record</button><br>
<canvas id="canvas" width="500" height="500"></canvas>
Is there a way to make it operational?
The goal is to load video, process every frame (which is time consuming in my case) and return the processed one.
Footnote: I do not want to use ffmpeg.js, external server or other technologies. I can process it by classic ffmpeg without using JavaScript at all, but this is not the point of this question, it is more about MediaStream usability / maturity. The context is Firefox/Chrome here, but it may be node.js or nw.js as well. If this is possible at all or awaiting bug fixes, the next question would be feeding audio to it, but I think it would be good as separate question.
I am using Angular 4 with TypeScript for a web application. I am allowing users to upload a thumbnail profile photo from their device as either a png, jpeg, or jpg, and I want to convert that photo to a jpg on the frontend. I am looking for some way to do this, as the file type is readonly.
The reason that I am doing this is so that when users load the profile page, they don't have to download large images so the page is quick. I think that converting to a jpg might be the best bet because when tested with a sample image, a png of an image was 35.4KB while a converted jpg of the same image was 6.7KB. If there is a better solution, I would love to hear it!
Thank you in advance!
I just wrote some code like your requirements, here is what I did:
load a local image by FileReader and add a listener to its onload event
in the onload listener, create an "Image" object by new Image(), and set "src" attribute by the loaded image "src" (Data URL format)
create a "Canvas" element, and draw the image on this canvas
use Canvas.toDataURL() to fetch the converted image data (in base64)
post the image data to server
After you draw an image to canvas, call Canvas.toDataURL() will get the canvas content in Data URL string, please note that it's the canvas data not original image data, for example, if image size is 100 x 100 and canvas size is 50 x 50, you'll get an image in 50 x 50 pixel with this function, so if you want a full size image, you need to resize the canvas to the certain size.
this function has two parameters:
canvas.toDataURL(type, encoderOptions);
type - A DOMString indicating the image format. The default format type is image/png. in your code, set to "image/jpeg" in your case
encoderOptions - A Number between 0 and 1 indicating image quality if the requested type is image/jpeg or image/webp.
Here is my "preview and upload" function write in TypesScript for reference:
preview(input: HTMLInputElement) {
if (input.files.length) {
let reader = new FileReader();
reader.onload = (e) => {
if (!this.img) {
this.img = new Image();
}
this.img.src = (e.target as any).result;
this.img.onload = () => {
// omit code of setting 'dx', 'dy', 'width', 'height'
let ctx = <CanvasRenderingContext2D>this.canvas.nativeElement.getContext('2d');
ctx.drawImage(this.img, dx, dy, width, height);
let dataUrl = (<HTMLCanvasElement>this.canvas.nativeElement).toDataURL('image/jpeg', 0.7);
this.uploadService.upload(dataUrl).then((resp: any) => {
if (resp.key) {
this.asset.image = resp.key;
}
});
};
}
reader.readAsDataURL(input.files[0]);
}
}
I omitted some variables: dx, dx, width, height in the above code, I use these variables to adjust image position (for clipping purpose).
This is JavaScript example:
function _elm(id) {
return document.getElementById(id);
}
_elm('fileInput').onchange= function(event) {
if (event.target.files.length) {
var fileReader = new FileReader();
fileReader.onload = function(event) {
var img = new Image();
img.src = event.target.result;
_elm('sizeRaw').innerText = '+ data-url size ' + event.target.result.length;
img.onload = function() {
var canvas = _elm('canvas');
var context = canvas.getContext('2d');
context.drawImage(img, 0, 0, 200, 200);
var dataUrl = canvas.toDataURL('image/jpeg', 0.5);
_elm('output').innerText = dataUrl;
_elm('sizeNew').innerText = '+ data-url size ' + dataUrl.length;
}
};
fileReader.readAsDataURL(event.target.files[0]);
}
};
#canvas {
border: 1px solid blue;
}
#output {
word-break: break-all;
}
<h3>Input file <span id="sizeRaw"></span>: </h3>
<input id="fileInput" type="file" accept="image/*">
<h3>Canvas</h3>
<div>
<canvas id="canvas" width="200" height="200"></canvas>
</div>
<h3>Output <span id="sizeNew"></span>: </h3>
<div id="output">
</div>
i new start a video in a specific position when the web is loading, i get the position using a eloquent var coming from the controler.
I also use jquery for other functions in the same view.
I have checked the variables arrive correctly so I do not know where is the error.
The strange thing is this method only works if I show a alert popup before in firefox. (not work in chrome or other browser).
This is my code:
<video id="video" style="display:none; width:100%; height:100%;" autoplay>
<source src="/files/convert/videos/{{$moviesNow->url}}" type="video/mp4" />
Su navegador no soporta el tag video.
</video>
<script>
var vid = document.getElementById("video");
var time = {{$difTime}};
var isPlaying = {{$playNow}};
var moviesArr = [];
};
var j = jQuery.noConflict();
j(document).ready(function() {
if(time >= 0 && isPlaying == 1){
//vid.currentTime = time;
}
});
function setCurTime() {
vid.currentTime = time;
};
</script>
Also i trying use:
document.getElementById('video').addEventListener('loadedmetadata', function() {
this.currentTime = time;
}, false);
But the problem is not solved.
The loadedmetadata event is helpful, but it doesn't ensure that the section of video you are trying to access is seekable yet. You could try a simpler solution using Media Fragments:
Just add #t=120 to the end of the video url to have it auto-forward to that spot. Like this:
video {
height: 180px;
width: 320px;
}
<video src="http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4#t=120" controls autoplay></video>
Aloha Stockoverflow.
In advance, thank you!
I am trying to modify my randomly changing background on my webpage, to add a FADE effect, so the change from 1 background to another is not so sudden and sharp.
I have tried to search through the web endlessly for a solution to my issue, but it all points towards adding a jQuery plugin which I would preferably avoid if it is possible.
My working code is as follows and needs to have added some kind of fadein / fadeout effect.
<script type="text/javascript">
var num;
var temp=0;
var speed=5000; /* this is set for 5 seconds, edit value to suit requirements */
var preloads=[];
/* add any number of images here */
preload(
'images/bg1.jpg',
'images/bg2.jpg',
'images/bg3.jpg',
'images/bg4.jpg',
'images/bg5.jpg'
);
function preload(){
for(var c=0;c<arguments.length;c++) {
preloads[preloads.length]=new Image();
preloads[preloads.length-1].src=arguments[c];
}
}
function rotateImages() {
num=Math.floor(Math.random()*preloads.length);
if(num==temp){
rotateImages();
}
else {
document.body.style.backgroundImage='url('+preloads[num].src+')';
temp=num;
setTimeout(function(){rotateImages()},speed);
}
}
if(window.addEventListener){
window.addEventListener('load',rotateImages,false);
}
else {
if(window.attachEvent){
window.attachEvent('onload',rotateImages);
}
}
</script>
Thank you very much for taking the time to look at it. :)
How to do it without plugins:
Use 2 layers for the background image, position them on top of each other.
Init the page with the first image on the bottom layer, make the top layer invisible (using CSS opacity property, make sure to Google this, different browsers use different approaches).
When fading:
Set the new image for the top layer.
Use a short, looping (frameduration < 40ms) setTimeout to increment the opacity of your top layer to 1. Use increments of 1/(speed/frameduration).
When comletely faded in, set the bottom layer to use the new (now visible) image, and set the top layer to opacity 0.
Like this:
<html>
<head>
<script type="text/javascript">
var num;
var current=0;
var speed=5000; /* this is set for 5 seconds, edit value to suit requirements */
var fps = 25;
var fadeDuration = 1000;
var opacityIncrement = 1/(fadeDuration/(1000/fps));
var preloads=[];
var topLayerOpacity = 0;
var topLayer = document.createElement("div");
var bottomLayer = document.createElement("div");
setOpacity(topLayer, 0);
/* add any number of images here */
preload(
'images/bg1.jpg',
'images/bg2.jpg',
'images/bg3.jpg',
'images/bg4.jpg'
);
function loadComplete(){
//add layers to background div
document.getElementById('backgroundContainer').appendChild(bottomLayer);
document.getElementById('backgroundContainer').appendChild(topLayer);
rotateImages();
}
function preload(){
//preload images
for(var c=0;c<arguments.length;c++) {
preloads[preloads.length]=new Image();
preloads[preloads.length-1].src=arguments[c];
}
}
// selecte new random image from preloads and start fade-in
function rotateImages() {
num=Math.floor(Math.random()*preloads.length);
//don't select current image
if(num==current){
rotateImages();
}
else {
topLayer.style.backgroundImage = 'url('+preloads[num].src+')';
current=num;
//start fade-in
fadeIn();
setTimeout(function(){rotateImages()},speed);
}
}
// fade in topLayer
function fadeIn(){
if (topLayerOpacity < 1){
topLayerOpacity += opacityIncrement;
setOpacity(topLayer, topLayerOpacity);// opacityIncrement);
setTimeout(fadeIn, 1000/fps);
}else{
fadeInComplete();
}
}
//return opacity for element
function getOpacity(el){
alert (el.style.opacity);
return el.style.opacity;
}
//sets opacity on element
function setOpacity(el, val){
el.style.opacity = val;
el.style.filter = 'alpha(opacity=' + val*100 + ')';
}
//called when fadeIn completed
function fadeInComplete(){
bottomLayer.style.backgroundImage = topLayer.style.backgroundImage;
topLayerOpacity = 0;
setOpacity(topLayer, topLayerOpacity);
}
if(window.addEventListener){
window.addEventListener('load',loadComplete,false);
}
else {
if(window.attachEvent){
window.attachEvent('onload',loadComplete);
}
}
</script>
<style type="text/css">
#backgroundContainer{
width:100%;
height:100%;
position:absolute;
/*background-color:green;*/
}
#backgroundContainer div{
width:100%;
height:100%;
position:absolute;
top:0;
}
.page {
width:100%;
text-align:center;
position:absolute;
}
.contents{
width:400px;
margin:0 auto;
background-color:lightblue;
}
</style>
</head>
<body>
<!-- holds background layers -->
<div id="backgroundContainer"></div>
<!-- substitutes for 'body' on this webpage -->
<div class="page">
<!-- contents for your webpage, through css centered within page-div -->
<div class="contents">
<p>Contents</p>
</div>
</div>
</body>
</html>
OR
Use jQuery/mootools/script.aculo.us/...
Best of luck!
The leak is pretty easy to create. Place the HTML below alongside a list of large images named "TestImage0.jpg", "TestImage1.jpg",..."TestImage9.jpg". The page will leak memory (I used sIEve for testing) on every click of the page. If the resize css is removed, the page will not leak. Can anyone confirm that this is an IE8 problem, or that my experiment is flawed?
Test Code
<html>
<head>
<title>Memory Leak Testing</title>
<script type="text/javascript">
var count = 0;
window.onload =
function() {
AppendImage();
window.document.body.onclick = ReplaceImage;
}
var ReplaceImage = function() {
window.document.body.removeChild(document.getElementById('MemTestObject' + count));
count++;
if (count > 9) {
alert('No more images to load.');
} else {
AppendImage();
}
}
var AppendImage = function() {
var imageObject = document.createElement('img');
imageObject.id = 'MemTestObject' + count;
imageObject.className = 'MemTestObject';
imageObject.src = 'TestImage' + count + '.jpg';
window.document.body.appendChild(imageObject);
}
</script>
<style type="text/css">
.MemTestObject {
width: 140px;
height: 178px;
}
</style>
</head>
<body>
<h1>Memory Leak Testing</h1>
</body>
</html>
The question is "Can someone verify that this is an IE8 memory leak?" To which the answer can only be, yes some one could verify this.
If you really think it is a valid memory leak in IE, first make sure that it is just IE. Then carry on working out specifics. Once you can describe exactly how to recreate this leak, you can consider reporting to MS, but they probably will not bother fixing it any time soon. But by all means spread the news so that others can avoid the trap