Capture frames from a canvas at 60 fps - html5-canvas

Hey everyone so I have a canvas that I write a rather complex animation to. Let's say I want to take screenshots of the canvas at 60 frames a second. The canvas doesn't have to play in real-time I just need it to capture 60 frames a second so I can send the screenshots to FFmpeg and make a video. I know I can use canvas.toDataURL but how do I capture the frames smoothly?

Use this code to pause the video and lottie animations if you are using lottie-web for after effects content in the browser. Than take screenshots and use Whammy to compile a webm file which you can than run through ffmpeg to get your desired output.
generateVideo(){
const vid = new Whammy.fromImageArray(this.captures, 30);
vid.name = "project_id_238.webm";
vid.lastModifiedDate = new Date();
this.file = URL.createObjectURL(vid);
},
async pauseAll(){
this.pauseVideo();
if(this.animations.length){
this.pauseLotties()
}
this.captures.push(this.canvas.toDataURL('image/webp'));
if(!this.ended){
setTimeout(()=>{
this.pauseAll();
}, 500);
}
},
async pauseVideo(){
console.log("curretTime",this.video.currentTime);
console.log("duration", this.video.duration);
this.video.pause();
const oneFrame = 1/30;
this.video.currentTime += oneFrame;
},
async pauseLotties(){
lottie.freeze();
for(let i =0; i<this.animations.length; i++){
let step =0;
let animation = this.animations[i].lottie;
if(animation.currentFrame<=animation.totalFrames){
step = animation.currentFrame + animation.totalFrames/30;
}
lottie.goToAndStop(step, true, animation.name);
}
}

Related

Reset a webp animation so that it will play from frame 1, restart animated webp

This issue has been discussed for gif files.
Here is what DID NOT WORK for webp files. Script starts with,
var animatedWebpImg = document.getElementById('idOfTheWebpInsideAnImgElement');
First failed attempt was,
animatedWebpImg.style.display = "none";
setTimeout(function() { animatedWebpImg.style.display = "block"; },100);
Second, I have tried removeChild() and appendChild() with exactly the same logic but that didn't work either.
Finally I got close however without success,
var srcOfTheImg = animatedWebpImg.src;
animatedWebpImg.src = ""; // empty and then back to original
animatedWebpImg.src = srcOfTheImg;
Here is what worked,
var srcOfTheImg = animatedWebpImg.src;
animatedWebpImg.src = "data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==";
setTimeout(function () { animatedWebpImg.src = srcOfTheImg; },100);
That means, instead of emptying the src to nothing, I had to fill it with a transparent pixel - an invisible something.

Embed logo in antmedia live stream via canvas

Am following the blog at https://antmedia.io/how-to-merge-live-stream-and-canvas-in-webrtc-easily/ that explains how to embed a logo in antmedia live stream. However, I couldn't quite figure out to initialise a localStream with javascript SDK as illustrated in the blog. Specifically, where is the implementation of initWebRTCAdaptor():
//initialize the webRTCAdaptor with the localStream created.
//initWebRTCAdaptor method is implemented below
initWebRTCAdaptor(localStream);
A complete working sample would be very helpful.
It seems that blog post is not fully up to date.
Let me share what to do to have this feature.
Just add a localStream parameter to the WebRTCAdaptor constructor.
Secondly, use the below code in place of initWebRTCAdaptor
For the full code, please take a look at this gist.
https://gist.github.com/mekya/d7d21f78e7ecb2c34d89bd6ec5bf5799
Make sure that you use your own image in image.src.(Use local images)
var canvas = document.getElementById('canvas');
var vid = document.getElementById('localVideo');
var image=new Image();
image.src="images/play.png";
function draw() {
if (canvas.getContext) {
var ctx = canvas.getContext('2d');
ctx.drawImage(vid, 0, 0, 200, 150);
ctx.drawImage(image,50, 10, 100, 30);
}
}
setInterval(function() { draw(); }, 50);
//capture stream from canvas
var localStream = canvas.captureStream(25);
navigator.mediaDevices.getUserMedia({video: true, audio:true}).then(function (stream) {
var video = document.querySelector('video#localVideo');
video.srcObject = stream;
video.onloadedmetadata = function(e) {
video.play();
};
//initialize the webRTCAdaptor with the localStream created.
//initWebRTCAdaptor method is implemented below
localStream.addTrack(stream.getAudioTracks()[0]);
initWebRTCAdaptor(false, autoRepublishEnabled);
});

Is it possible to feed MediaStream frames at lower framerate?

I would like to use MediaStream.captureStream() method, but it is either rendered useless due to specification and bugs or I am using it totally wrong.
I know that captureStream gets maximal framerate as the parameter, not constant and it does not even guarantee that, but it is possible to change MediaStream currentTime (currently in Chrome, in Firefox it has no effect but in return there is requestFrame, not available at Chrome), but the idea of manual frame requests or setting the placement of the frame in the MediaStream should override this effect. It doesn't.
In Firefox it smoothly renders the video, frame by frame, but the video result is as long as wall clock time used for processing.
In Chrome there are some dubious black frames or reordered ones (currently I do not care about it until the FPS matches), and the manual setting of currentTime gives nothing, the same result as in FF.
I use modified code from MediaStream Capture Canvas and Audio Simultaneously answer.
const FPS = 30;
var cStream, vid, recorder, chunks = [], go = true,
Q = 61, rec = document.getElementById('rec'),
canvas = document.getElementById('canvas'),
ctx = canvas.getContext('2d');
ctx.strokeStyle = 'rgb(255, 0, 0)';
function clickHandler() {
this.textContent = 'stop recording';
//it has no effect no matter if it is empty or set to 30
cStream = canvas.captureStream(FPS);
recorder = new MediaRecorder(cStream);
recorder.ondataavailable = saveChunks;
recorder.onstop = exportStream;
this.onclick = stopRecording;
recorder.start();
draw();
}
function exportStream(e) {
if (chunks.length) {
var blob = new Blob(chunks)
var vidURL = URL.createObjectURL(blob);
var vid2 = document.createElement('video');
vid2.controls = true;
vid2.src = vidURL;
vid2.onend = function() {
URL.revokeObjectURL(vidURL);
}
document.body.insertBefore(vid2, vid);
} else {
document.body.insertBefore(document.createTextNode('no data saved'), canvas);
}
}
function saveChunks(e) {
e.data.size && chunks.push(e.data);
}
function stopRecording() {
go = false;
this.parentNode.removeChild(this);
recorder.stop();
}
var loadVideo = function() {
vid = document.createElement('video');
document.body.insertBefore(vid, canvas);
vid.oncanplay = function() {
rec.onclick = clickHandler;
rec.disabled = false;
canvas.width = vid.videoWidth;
canvas.height = vid.videoHeight;
vid.oncanplay = null;
ctx.drawImage(vid, 0, 0);
}
vid.onseeked = function() {
ctx.drawImage(vid, 0, 0);
/*
Here I want to include additional drawing per each frame,
for sure taking more than 180ms
*/
if(cStream && cStream.requestFrame) cStream.requestFrame();
draw();
}
vid.crossOrigin = 'anonymous';
vid.src = 'https://dl.dropboxusercontent.com/s/bch2j17v6ny4ako/movie720p.mp4';
vid.currentTime = 0;
}
function draw() {
if(go && cStream) {
++Q;
cStream.currentTime = Q / FPS;
vid.currentTime = Q / FPS;
}
};
loadVideo();
<button id="rec" disabled>record</button><br>
<canvas id="canvas" width="500" height="500"></canvas>
Is there a way to make it operational?
The goal is to load video, process every frame (which is time consuming in my case) and return the processed one.
Footnote: I do not want to use ffmpeg.js, external server or other technologies. I can process it by classic ffmpeg without using JavaScript at all, but this is not the point of this question, it is more about MediaStream usability / maturity. The context is Firefox/Chrome here, but it may be node.js or nw.js as well. If this is possible at all or awaiting bug fixes, the next question would be feeding audio to it, but I think it would be good as separate question.

phantomjs screenshots and ffmpeg frame missing

I have problem making video from website screenshots taken from phantomjs.
the phantomjs did not make screenshots for all frames within the same second and even not all seconds there , there is huge missing frames .
the result is high speed video playing with many jumps in video effects .
test.js :
var page = require('webpage').create(),
address = 'http://raphaeljs.com/polar-clock.html',
duration = 5, // duration of the video, in seconds
framerate = 24, // number of frames per second. 24 is a good value.
counter = 0,
width = 1024,
height = 786;
frame = 10001;
page.viewportSize = { width: width, height: height };
page.open(address, function(status) {
if (status !== 'success') {
console.log('Unable to load the address!');
phantom.exit(1);
} else {
window.setTimeout(function () {
page.clipRect = { top: 0, left: 0, width: width, height: height };
window.setInterval(function () {
counter++;
page.render('newtest/image'+(frame++)+'.png', { format: 'png' });
if (counter > duration * framerate) {
phantom.exit();
}
}, 1/framerate);
}, 200);
}
});
this will create 120 image , this is correct count , but when you see the images one by one you will see many duplicate the same contents and many missing frames
ffmpeg :
fmpeg -start_number 10001 -i newtest/image%05d.png -c:v libx264 -r 24 -pix_fmt yuv420p out.mp4
I know this script and ffmpeg command not perfect , because I did hundred of changes without lucky, and I lost the correct setting understanding .
an anyone guide me to fix this ?.
thank you all

Initialize youtube api to target ajax loaded iframe

it's my first time to work with Youtube API and after I got most things done, I'm having one last issue with it which i can't seem to solve, so I would appreciate if you could take a few minutes to help.
Here it goes in simple words:
I have a video slider (carousel). Under the main video, are thumbnails of other related videos (this is the site: kushtube.com)
when I click on a related video thumbnail, the main player content is loaded via AJAX
What I wanted to ultimately achieve is that when the current video ends,the next vicdo starts playing automatically.
Now after I did some work, I managed to make it work like that, BUT
it only work on page load...
If i click on one of the related video thumbnails, the content of the player loads via AJAX and I cant seem to be able to re-initialize Youtube API to target the new ajax-loaded player.
This is my current code:
<script>
// global variable for the player
var player;
function ytInit(){
// create the global player from the specific iframe (#video)
player = new YT.Player('test', {
events: {
// call this function when player is ready to use
'onReady': onPlayerReady,
'onStateChange': onPlayerStatusChange
}
});
}
// this function gets called when API is ready to use
function onYouTubeIframeAPIReady() {
ytInit();
}
jQuery(document).ready( function($){
jQuery('ul.carousel-list li').click( function(){
ytInit();
jQuery('ul.carousel-list li.active').removeClass('active');
});
});
function onPlayerStatusChange(event) {
/* video status
----------------
-1 (unstarted)
0 (ended)
1 (playing)
2 (paused)
3 (buffering)
5 (video cued)
*/
console.log( event );
if( event.data == 0 ){
var thisLI = jQuery('ul.carousel-list li.active'); //get active slider item
var thisLINext = jQuery('ul.carousel-list li.active').next(); //get next slider item
var nextVideoCode = thisLINext.attr('video_code'); //get next video code
var nextVideoTitle = thisLINext.attr('video_title'); //get next video code
var nextVideoUrl = thisLINext.attr('video_url'); //get next video code
thisLINext.addClass('active'); //activate next slider item
thisLI.removeClass('active'); //deactivate current cslider item
//load next video
player.loadVideoById(nextVideoCode);
//update video title in header
jQuery('.entry-header h1.entry-title a').attr('href', decodeURIComponent(nextVideoUrl)).text(decodeURIComponent(nextVideoTitle));
}
}
function onPlayerReady(event) {
// bind events
var playButton = document.getElementById("play-button");
playButton.addEventListener("click", function() {
player.playVideo();
});
var pauseButton = document.getElementById("pause-button");
pauseButton.addEventListener("click", function() {
player.pauseVideo();
});
}
// Inject YouTube API script
var tag = document.createElement('script');
tag.src = "//www.youtube.com/player_api";
var firstScriptTag = document.getElementsByTagName('script')[0];
firstScriptTag.parentNode.insertBefore(tag, firstScriptTag);
</script>
Do you have any helpful suggestions? I would really appreciate.
Calling ytInit() when you click the <li> creates a new player object. I think that's not quite what you want.
Have you tried:
jQuery('ul.carousel-list li').click( function(){
// ytInit();
// just load the video from the "video_code" attr
player.loadVideoById(this.getAttribute('video_code'));
jQuery('ul.carousel-list li.active').removeClass('active');
});
If that doesn't autoplay, then you may have to wait for the YT.PlayerState.CUED (i.e. 5) event in your player status change event listener to start the player.

Resources