I'm using a simple p5.js programme to take a webcam stream and show a still image from this on the screen every few seconds.
let capture;
function setup() {
createCanvas(320, 240);
capture = createCapture(VIDEO);
capture.size(320, 240);
capture.hide();
}
function draw() {
if(frameCount % 120 == 0)
{image(capture, 0, 0, 320, 240);}
}
What I want to do then is to take each still image and convert it to a format that I can send using HTTP POST. The online manual is not helpful, and extensive internet searches haven't thrown up anything that works. Any good ideas, please?
Use .toDataURL() to convert the contents of a canvas element to string.
https://editor.p5js.org/EthanHermsey/sketches/bTvntCc35
let cnv;
let capture;
function setup() {
cnv = createCanvas(320, 240);
capture = createCapture(VIDEO);
capture.size(320, 240);
capture.hide();
}
function draw() {
//show live webcam feed
image(capture, 0, 0, 320, 240);
if (frameCount % 120 == 0){
captureFrame();
}
}
function captureFrame() {
// convert the graphics canvas to string, you can send that string in a POST request.
let imageBase64String = cnv.elt.toDataURL();
// for example purpose, after you retreive the string from the server with a GET request,
// convert it back to an image.
// let showImg = createImg(imageBase64String, "");
// showImg.hide();
// You can use
// image( showImg, 0, 0, 320, 240)
// to draw in on canvas.
}
Related
I'm trying to adapt Apple's AVCamFilter sample to MacOS. The filtering appears to work, but rendering the processed image through Metal gives me a framerate of several seconds per frame. I've tried different approaches, but have been stuck for a long time.
This is the project AVCamFilterMacOS - Can anyone with better knowledge of AVFoundation with Metal tell me what's wrong? I've been reading the documentation and practicing getting the unprocessed image to display, as well as rendering other things like models to the metal view but I can't seem to get the processed CMSampleBuffer to render at a reasonable framerate.
Even if I skip the renderer and send the videoPixelBuffer to the metal view directly, the view's performance is pretty jittery.
Here is some of the relevant rendering code I'm using in the controller:
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
processVideo(sampleBuffer: sampleBuffer)
}
func processVideo(sampleBuffer: CMSampleBuffer) {
if !renderingEnabled {
return
}
guard let videoPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return
}
if !self.videoFilter.isPrepared {
/*
outputRetainedBufferCountHint is the number of pixel buffers the renderer retains. This value informs the renderer
how to size its buffer pool and how many pixel buffers to preallocate. Allow 3 frames of latency to cover the dispatch_async call.
*/
self.videoFilter.prepare(with: formatDescription, outputRetainedBufferCountHint: 3)
}
// Send the pixel buffer through the filter
guard let filteredBuffer = self.videoFilter.render(pixelBuffer: videoPixelBuffer) else {
print("Unable to filter video buffer")
return
}
self.previewView.pixelBuffer = filteredBuffer
}
And from the renderer:
func render(pixelBuffer: CVPixelBuffer) -> CVPixelBuffer? {
if !isPrepared {
assertionFailure("Invalid state: Not prepared.")
return nil
}
var newPixelBuffer: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, outputPixelBufferPool!, &newPixelBuffer)
guard let outputPixelBuffer = newPixelBuffer else {
print("Allocation failure: Could not get pixel buffer from pool. (\(self.description))")
return nil
}
guard let inputTexture = makeTextureFromCVPixelBuffer(pixelBuffer: pixelBuffer, textureFormat: .bgra8Unorm),
let outputTexture = makeTextureFromCVPixelBuffer(pixelBuffer: outputPixelBuffer, textureFormat: .bgra8Unorm) else {
return nil
}
// Set up command queue, buffer, and encoder.
guard let commandQueue = commandQueue,
let commandBuffer = commandQueue.makeCommandBuffer(),
let commandEncoder = commandBuffer.makeComputeCommandEncoder() else {
print("Failed to create a Metal command queue.")
CVMetalTextureCacheFlush(textureCache!, 0)
return nil
}
commandEncoder.label = "Rosy Metal"
commandEncoder.setComputePipelineState(computePipelineState!)
commandEncoder.setTexture(inputTexture, index: 0)
commandEncoder.setTexture(outputTexture, index: 1)
// Set up the thread groups.
let width = computePipelineState!.threadExecutionWidth
let height = computePipelineState!.maxTotalThreadsPerThreadgroup / width
let threadsPerThreadgroup = MTLSizeMake(width, height, 1)
let threadgroupsPerGrid = MTLSize(width: (inputTexture.width + width - 1) / width,
height: (inputTexture.height + height - 1) / height,
depth: 1)
commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup)
commandEncoder.endEncoding()
commandBuffer.commit()
return outputPixelBuffer
}
func makeTextureFromCVPixelBuffer(pixelBuffer: CVPixelBuffer, textureFormat: MTLPixelFormat) -> MTLTexture? {
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
// Create a Metal texture from the image buffer.
var cvTextureOut: CVMetalTexture?
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, textureFormat, width, height, 0, &cvTextureOut)
guard let cvTexture = cvTextureOut, let texture = CVMetalTextureGetTexture(cvTexture) else {
CVMetalTextureCacheFlush(textureCache, 0)
return nil
}
return texture
}
And finally the metal view:
override func draw(_ rect: CGRect) {
var pixelBuffer: CVPixelBuffer?
var mirroring = false
var rotation: Rotation = .rotate0Degrees
syncQueue.sync {
pixelBuffer = internalPixelBuffer
mirroring = internalMirroring
rotation = internalRotation
}
guard let drawable = currentDrawable,
let currentRenderPassDescriptor = currentRenderPassDescriptor,
let previewPixelBuffer = pixelBuffer else {
return
}
// Create a Metal texture from the image buffer.
let width = CVPixelBufferGetWidth(previewPixelBuffer)
let height = CVPixelBufferGetHeight(previewPixelBuffer)
if textureCache == nil {
createTextureCache()
}
var cvTextureOut: CVMetalTexture?
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
textureCache!,
previewPixelBuffer,
nil,
.bgra8Unorm,
width,
height,
0,
&cvTextureOut)
guard let cvTexture = cvTextureOut, let texture = CVMetalTextureGetTexture(cvTexture) else {
print("Failed to create preview texture")
CVMetalTextureCacheFlush(textureCache!, 0)
return
}
if texture.width != textureWidth ||
texture.height != textureHeight ||
self.bounds != internalBounds ||
mirroring != textureMirroring ||
rotation != textureRotation {
setupTransform(width: texture.width, height: texture.height, mirroring: mirroring, rotation: rotation)
}
// Set up command buffer and encoder
guard let commandQueue = commandQueue else {
print("Failed to create Metal command queue")
CVMetalTextureCacheFlush(textureCache!, 0)
return
}
guard let commandBuffer = commandQueue.makeCommandBuffer() else {
print("Failed to create Metal command buffer")
CVMetalTextureCacheFlush(textureCache!, 0)
return
}
guard let commandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor) else {
print("Failed to create Metal command encoder")
CVMetalTextureCacheFlush(textureCache!, 0)
return
}
commandEncoder.label = "Preview display"
commandEncoder.setRenderPipelineState(renderPipelineState!)
commandEncoder.setVertexBuffer(vertexCoordBuffer, offset: 0, index: 0)
commandEncoder.setVertexBuffer(textCoordBuffer, offset: 0, index: 1)
commandEncoder.setFragmentTexture(texture, index: 0)
commandEncoder.setFragmentSamplerState(sampler, index: 0)
commandEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
commandEncoder.endEncoding()
// Draw to the screen.
commandBuffer.present(drawable)
commandBuffer.commit()
}
All of this code is in the linked project
Capture device delegates don't own the sample buffers they receive in their callbacks, so it's incumbent on the receiver to make sure they're retained for as long as their contents are needed. This project doesn't currently ensure that.
Rather, by calling CMSampleBufferGetImageBuffer and wrapping the resulting pixel buffer in a texture, the view controller is allowing the sample buffer to be released, meaning that future operations on its corresponding pixel buffer are undefined.
One way to ensure the sample buffer lives long enough to be processed is to add a private member to the camera view controller class that retains the most-recently received sample buffer:
private var sampleBuffer: CMSampleBuffer!
and then set this member in the captureOutput(...) method before calling processVideo. You don't even have to reference it further; the fact that it's retained should prevent the stuttery and unpredictable behavior you're seeing.
This solution may not be perfect, since it retains the sample buffer for longer than strictly necessary in the event of a capture session interruption or other pause. You can devise your own scheme for managing object lifetimes; the important thing is to ensure that the root sample buffer object sticks around until you're done with any textures that refer to its contents.
I would like to use MediaStream.captureStream() method, but it is either rendered useless due to specification and bugs or I am using it totally wrong.
I know that captureStream gets maximal framerate as the parameter, not constant and it does not even guarantee that, but it is possible to change MediaStream currentTime (currently in Chrome, in Firefox it has no effect but in return there is requestFrame, not available at Chrome), but the idea of manual frame requests or setting the placement of the frame in the MediaStream should override this effect. It doesn't.
In Firefox it smoothly renders the video, frame by frame, but the video result is as long as wall clock time used for processing.
In Chrome there are some dubious black frames or reordered ones (currently I do not care about it until the FPS matches), and the manual setting of currentTime gives nothing, the same result as in FF.
I use modified code from MediaStream Capture Canvas and Audio Simultaneously answer.
const FPS = 30;
var cStream, vid, recorder, chunks = [], go = true,
Q = 61, rec = document.getElementById('rec'),
canvas = document.getElementById('canvas'),
ctx = canvas.getContext('2d');
ctx.strokeStyle = 'rgb(255, 0, 0)';
function clickHandler() {
this.textContent = 'stop recording';
//it has no effect no matter if it is empty or set to 30
cStream = canvas.captureStream(FPS);
recorder = new MediaRecorder(cStream);
recorder.ondataavailable = saveChunks;
recorder.onstop = exportStream;
this.onclick = stopRecording;
recorder.start();
draw();
}
function exportStream(e) {
if (chunks.length) {
var blob = new Blob(chunks)
var vidURL = URL.createObjectURL(blob);
var vid2 = document.createElement('video');
vid2.controls = true;
vid2.src = vidURL;
vid2.onend = function() {
URL.revokeObjectURL(vidURL);
}
document.body.insertBefore(vid2, vid);
} else {
document.body.insertBefore(document.createTextNode('no data saved'), canvas);
}
}
function saveChunks(e) {
e.data.size && chunks.push(e.data);
}
function stopRecording() {
go = false;
this.parentNode.removeChild(this);
recorder.stop();
}
var loadVideo = function() {
vid = document.createElement('video');
document.body.insertBefore(vid, canvas);
vid.oncanplay = function() {
rec.onclick = clickHandler;
rec.disabled = false;
canvas.width = vid.videoWidth;
canvas.height = vid.videoHeight;
vid.oncanplay = null;
ctx.drawImage(vid, 0, 0);
}
vid.onseeked = function() {
ctx.drawImage(vid, 0, 0);
/*
Here I want to include additional drawing per each frame,
for sure taking more than 180ms
*/
if(cStream && cStream.requestFrame) cStream.requestFrame();
draw();
}
vid.crossOrigin = 'anonymous';
vid.src = 'https://dl.dropboxusercontent.com/s/bch2j17v6ny4ako/movie720p.mp4';
vid.currentTime = 0;
}
function draw() {
if(go && cStream) {
++Q;
cStream.currentTime = Q / FPS;
vid.currentTime = Q / FPS;
}
};
loadVideo();
<button id="rec" disabled>record</button><br>
<canvas id="canvas" width="500" height="500"></canvas>
Is there a way to make it operational?
The goal is to load video, process every frame (which is time consuming in my case) and return the processed one.
Footnote: I do not want to use ffmpeg.js, external server or other technologies. I can process it by classic ffmpeg without using JavaScript at all, but this is not the point of this question, it is more about MediaStream usability / maturity. The context is Firefox/Chrome here, but it may be node.js or nw.js as well. If this is possible at all or awaiting bug fixes, the next question would be feeding audio to it, but I think it would be good as separate question.
I just made my image generator work with PNG files. For now, it's divided into 3 categories (backgrounds, objects & texts). These are now all combined, and with every mouse click it randomises these PNGs.
I made three toggles, where you could to choose to show either the background and the objects on top, all of them, or all separate. Whenever I run the sketch, it shows the "grey" background, but when I use the toggles, it doesn't show anything, or shows a flickering image, where the mouse-click can't be used to go to the next image. I can't seem to find the problem. Hopefully, you can help. :)
import controlP5.*;
boolean showBackground = false;
boolean showObjects = false;
boolean showGrids = false;
ControlP5 cp5;
PImage[] myImageArray = new PImage[8];
PImage[] myImageArray2 = new PImage[15];
PImage[] myImageArray3 = new PImage[15];
void setup() {
size(1436, 847);
background(211, 211, 211);
for (int i=0; i<myImageArray.length; i++) {
myImageArray[i] = loadImage ( "o" + i + ".png");
myImageArray2[i] = loadImage ( "g" + i + ".png");
myImageArray3[i] = loadImage( "b" + i + ".jpg");
cp5 = new ControlP5(this);
// create a toggle and change the default look to a (on/off) switch look
cp5.addToggle("showBackground")
.setPosition(40, 250)
.setSize(50, 20)
.setValue(true)
.setMode(ControlP5.SWITCH);
cp5.addToggle("showObjects")
.setPosition(40, 400)
.setSize(50, 20)
.setValue(true)
.setMode(ControlP5.SWITCH);
cp5.addToggle("showGrid")
.setPosition(40, 600)
.setSize(50, 20)
.setValue(true)
.setMode(ControlP5.SWITCH);
}
display();
}
void display() {
image(myImageArray3[(int)random(myImageArray.length)], 0, 0, 1436, 847); // b
image(myImageArray2[(int)random(myImageArray.length)], 0, 0, 1436, 847); // g
image(myImageArray[(int)random(myImageArray.length)], 0, 0, 1436, 847); // o
}
void mousePressed() {
display();
}
void draw() {
pushMatrix();
if (showBackground==false) {
image(myImageArray3[(int)random(myImageArray.length)], 0, 0, 1436, 847); // b
} else {
background(211, 211, 211);
}
if (showGrids==false) {
image(myImageArray2[(int)random(myImageArray.length)], 0, 0, 1436, 847); // g
} else {
background(211, 211, 211);
}
if (showObjects==false) {
image(myImageArray[(int)random(myImageArray.length)], 0, 0, 1436, 847); // o
} else {
background(211, 211, 211);
}
popMatrix();
}
Here are a couple of things where the logic your wrote in your code might not match what you had in mind:
When you call display() on mouse it renders those 3 images once (also it will be different images within those since it's using a randomised index). Similarly in draw(), when an does get picked to be rendered, frames will be flickering fast as a random index is generated multiple times per second(each frame). You may want to randomise indices in a different event (e.g. mouse or key press) and store this value in a variable you can re-use.
the conditions you use in draw(): you probably meant to check if the values are true(toggled enabled/turned on in controlP5) ? (e.g. e.g. if (showBackground==true) and initialise the toggles with false, instead of true?)
a big one: in draw() , after each condition(showBackground,showGrids,showObjects), if it's false, you're clearing the the whole frame (so a previous image would be erased)
you have 3 arrays, but you use the size of the first(myImageArray.length) only, which means, even though you may have more images for myImageArray2 and myImageArray3, you're not loading, nor displaying them.
The third grid is labeled "showGrid" when it should be "showGrids": if you aren't consistent with the toggle labels and variable names, toggles won't update the variable names.
you should use more descriptive names for the arrays: it will make it easier to scan/follow your code on the long run.
there's no need to add toggles multiple times in the for loop where you load images: once will do.
Here's what I mean:
import controlP5.*;
boolean showBackground = false;
boolean showObjects = false;
boolean showGrids = false;
ControlP5 cp5;
PImage[] objects = new PImage[8];
PImage[] grids = new PImage[15];
PImage[] backgrounds = new PImage[15];
int currentImage = 0;
void setup() {
size(1436, 847);
//load objects
for (int i=0; i<objects.length; i++) {
objects[i] = loadImage ( "o" + i + ".png");
}
//load grids
for(int i = 0 ; i < grids.length; i++){
grids[i] = loadImage ( "g" + i + ".png");
}
//load backgrounds
for(int i = 0 ; i < grids.length; i++){
backgrounds[i] = loadImage( "b" + i + ".jpg");
}
//setup UI
cp5 = new ControlP5(this);
// create a toggle and change the default look to a (on/off) switch look
cp5.addToggle("showBackground")
.setPosition(40, 250)
.setSize(50, 20)
.setValue(false)
.setMode(ControlP5.SWITCH);
cp5.addToggle("showObjects")
.setPosition(40, 400)
.setSize(50, 20)
.setValue(false)
.setMode(ControlP5.SWITCH);
cp5.addToggle("showGrids")
.setPosition(40, 600)
.setSize(50, 20)
.setValue(false)
.setMode(ControlP5.SWITCH);
}
void mousePressed() {
//go to next image index
currentImage = currentImage + 1;
//check if the incremented index is still valid, otherwise, reset it to 0 (so it doesn't go out of bounds)
if (currentImage >= objects.length) {
currentImage = 0;
}
}
void draw() {
//clear current frame
background(211);//for gray scale value you can just use one value: the brightness level :)
if (showBackground==true) {
image(backgrounds[currentImage], 0, 0, 1436, 847); // b
}
if (showGrids==true) {
image(grids[currentImage], 0, 0, 1436, 847); // g
}
if (showObjects==true) {
image(objects[currentImage], 0, 0, 1436, 847); // o
}
}
Note that currently the same index is used for all 3 arrays.
You may want to add a separate index variable for each array (e.g. currentObjectIndex, currentBackgroundIndex, currentGridIndex) that you can increment independently of each other.
I recommend having a bit more patience and double checking your code first.
Visualise what each line of code will do, then check if it actually does what you expect it to do. Either you will learn something new or improve your logic.
Also, if mentally joggling 3 arrays is tricky (and it can be), go one step back: try your logic with one array only until you get the hang of it, then move on.
A step backwards is sometimes a step forward when you're going in the wrong direction.
As creative as you'd like to be with Processing, bare in mind, the interface to plugging your ideas to it is still a series of one instruction at a time, each precisely tuned to do exactly what you want it to do. There's room for fun, but unfortunately you need to get past the boring parts first
I have a small bit of code to get the camera:
void setup() {
if (cam.available() == false) {
cam.start();
}
}
void draw() {
if (cam.available() == true) {
cam.read();
}
image(cam, w/2, h/2, w, 480.0/640.0*w); // resized according to size()
}
If I use cam.get(), the image is not resized, it keeps the camera resolution.
Is there any solution to get the "resized" camera image?
I tried
big = copy(cam, int(w/2), int(h/2), int(w), int(480/640*w), 0, 0, int(w), int(h));
but it doesn't seem to work (same for cam.copy(...).
Thank you in advance!
Assuming you're using the video library and cam is a Capture, then I would expect these methods to work. Capture extends PImage, so you should be able to copy and resize it.
So the first thing I'd check is what values you're passing into these functions. The println() function is your best friend.
Or try passing hardcoded values so you know what to expect:
image(cam, 0, 0, 100, 100);
If that really doesn't work, then as a worst case scenario you could use the Capture.get() function to get the pixels yourself, then do the resizing manually. I really don't think you'll have to do that though.
I want to stream an image to webpage via websocket. the data is in RGBA. how do I change the blog into image data?
this is my current code, it doesn't work and it will be slow. is there a direct way of assigning event.data to canvas' image data?
void onMessage(MessageEvent event)
{
print("received!");
var imgData = canvas.getImageData(0, 0, 100, 100);
var j = 0;
for (var i=0;i<imgData.data.length;i+=4)
{
imgData.data[i+0]=event.data[j];
imgData.data[i+1]=event.data[j+1];
imgData.data[i+2]=event.data[j+2];
imgData.data[i+3]=255;
j+=3;
}
canvas.putImageData(imgData,0,0);
}
On Firefox you can use the toBlob method. Put the image data on a temporany canvas and call toBlob method. Proof of concept example:
var canvas = document.createElement('canvas');
canvas.width = imageData.width;
canvas.height = imageData.width;
me._dstCanvas.getContext('2d').putImageData(a.dstImgData, 0, 0);
me._dstCanvas.toBlob(function(blob) {
blob// this is yout file
}, 'image/png', 1);
For more have a look at Moz Dev:
https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/toBlob