Webserver network timing
Why AJAX is so laggy?
The most take a time: Initial connection.
How to speed up refreshing status of 72 rgb led?
I try to reach 100Hz.
Why AJAX is so laggy?
The most take a time: Initial connection.
How to speed up refreshing status of 72 rgb led?
I try to reach 100Hz.
Why AJAX is so laggy?
The most take a time: Initial connection.
How to speed up refreshing status of 72 rgb led?
I try to reach 100Hz.
#include <WiFi.h>
#include <WiFiClient.h>
#include <WebServer.h>
const char* ssid = "ssid";
const char* password = "password";
WebServer server(80);
bool refresh = true;
void handleRoot() {
String html = "<html><head><style>.led-box {width: 10px; height: 10px; border: 1px solid black; float: left; margin-right: 1px;}</style>";
html += "<script>";
html += "function updateLEDs() {";
html += " var xhr = new XMLHttpRequest();";
html += " xhr.open('GET', '/updateLEDs', true);";
html += " xhr.onreadystatechange = function() {";
html += " if (xhr.readyState == XMLHttpRequest.DONE) {";
html += " var colors = xhr.responseText.split(',');";
for (int i = 1; i <= 72; i++) {
html += " document.getElementById('led" + String(i) + "').style.backgroundColor = '#' + colors[" + String(i - 1) + "];";
}
html += " }";
html += " };";
html += " xhr.send();";
html += "}";
html += "setInterval(updateLEDs, 1000);";
html += "</script></head><body>";
for (int i = 1; i <= 72; i++) {
html += "<div id='led" + String(i) + "' class='led-box'></div>";
}
html += "</body></html>";
server.send(200, "text/html", html);
}
void handleUpdateLEDs() {
String colorCodes = "";
for (int i = 1; i <= 72; i++) {
if (i > 1) colorCodes += ",";
colorCodes += String(random(0xFFFFFF), HEX);
}
server.send(200, "text/plain", colorCodes);
}
void setup() {
Serial.begin(115200);
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED) {
delay(1000);
Serial.println("Connecting to WiFi...");
}
Serial.println("Connected to WiFi");
server.on("/", handleRoot);
server.on("/updateLEDs", handleUpdateLEDs);
server.begin();
Serial.println("HTTP server started");
}
void loop() {
server.handleClient();
}
The ESP32 is a fairly sluggish little microcontroller to begin with, and you give it lots of unnecessary string processing to do on each HTTP request.
The worst bottleneck is your handleRoot() function - it dynamically assembles the exact same HTML document for every request. You do several hundreds of String concatenations every time someone GETs the root document. That's going to be slow. Pre-assemble the string (e.g. using raw string literals in setup()), only return the value in handleRoot().
String rootHtml;
void handleRoot() {
server.send(200, "text/html", rootHtml);
}
void setup() {
rootHtml = R"MYRAW(
<html><head><style>.led-box {width: 10px; height: 10px; border: 1px solid black; float: left; margin-right: 1px;}</style>
<script>
function updateLEDs() {
var xhr = new XMLHttpRequest();
xhr.open('GET', '/updateLEDs', true);
xhr.onreadystatechange = function() {
if (xhr.readyState == XMLHttpRequest.DONE) {
var colors = xhr.responseText.split(',');
)MYRAW";
for (int i = 1; i <= 72; i++) {
rootHtml += " document.getElementById('led" + String(i) + "').style.backgroundColor = '#' + colors[" + String(i - 1) + "];";
}
// ... etc
}
The handleUpdateLEDs() function also does at least 144 string concatenations, which is probably not the fastest algorithm for assembling a large string. I'd experiment with snprintf() and friends.
Note that the Arduino WebServer class is not built for speed, it's built for simplicity. Experiment with ESP IDF HTTP server to get better performance.
Related
I am trying to serve a webpage that has a color wheel. When the user changes the color on the wheel I want to send the new rgb value to the esp but the code on the esp never enters
if(type == WStype_TEXT)
Backend (esp8266 code):
#include <ESP8266WiFi.h>
#include <WiFiClient.h>
#include <ESP8266WebServer.h>
#include <WebSocketsServer.h>
#include "index.h"
const char* WiFissid = "ssid";
const char* WiFiPassword = "password";
const int WebServerPort = port;
const int WebSocketsPort = port;
ESP8266WebServer WebServer(WebServerPort);
WebSocketsServer WebSockets = WebSocketsServer(WebSocketsPort);
const int RedPin = 13;
const int GreenPin = 12;
const int BluePin = 14;
const int Resolution = 256;
int RedValue = 0;
int GreenValue = 0;
int BlueValue = 255;
/* Serve Web Page */
void ServeMainPage()
{
WebServer.send(200, "text/html", MainPage);
}
/* Web Sockets Get Input */
void WebSocketsGetData(uint8_t num, WStype_t type, uint8_t * payload, size_t length)
{
if(type == WStype_CONNECTED)
{
IPAddress ip = WebSockets.remoteIP(num);
Serial.println(String(ip[0]) + String(ip[1]) + String(ip[2]) + String(ip[3]));
String _payload = String((char *) &payload[0]);
Serial.println(_payload);
}
else if(type == WStype_TEXT)
{
String _payload = String((char *) &payload[0]);
Serial.println(_payload);
}
else if(type == WStype_DISCONNECTED)
{
Serial.println("Client disconnected");
}
}
/* Setup */
void setup()
{
/* Map Pins and Set Default Color */
analogWriteRange(Resolution);
analogWrite(RedPin, RedValue);
analogWrite(GreenPin, GreenValue);
analogWrite(BluePin, BlueValue);
/* Start Serial */
Serial.begin(9600);
/* Connect to WiFi */
WiFi.mode(WIFI_STA);
WiFi.begin(WiFissid, WiFiPassword);
while (WiFi.status() != WL_CONNECTED)
{
delay(500);
}
Serial.println(WiFi.localIP());
/* Start and Config Web Server */
WebServer.on("/", ServeMainPage);
WebServer.begin();
/* Start and Config Web Sockets Server */
WebSockets.onEvent(WebSocketsGetData);
WebSockets.begin();
}
void loop()
{
WebServer.handleClient();
WebSockets.loop();
}
index.h:
const char MainPage[] PROGMEM = R"=====(
<!DOCTYPE html>
<html>
<head>
<meta name = "viewport" content = "width=device-width, initial-scale=1">
<meta charset="UTF-8">
<link rel = "stylesheet" href = "https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css">
<script src = "https://cdnjs.cloudflare.com/ajax/libs/jscolor/2.0.4/jscolor.min.js"></script>
<script src = "https://cdn.jsdelivr.net/npm/#jaames/iro"></script>
<title>Leduri Rares</title>
</head>
<body>
<div class = "container-fluid">
<div class = "row">
<div class = "wheel" id = "ColorWheel"></div>
</div>
</div>
</body>
</html>
<style>
body
{
background-color: #000000;
overflow-x: hidden;
}
h1
{
color: white;
}
.container-fluid
{
margin-top: 5%;
}
.row
{
text-align: center;
}
#ColorWheel
{
margin: 0 auto;
}
#media only screen and (max-width: 800px)
{
.container-fluid
{
margin-top: 10%;
}
#ColorWheel
{
margin: 5% auto;
}
}
</style>
<script>
let Socket = new WebSocket("wss://myIP:WSPort");
Socket.onclose = function(event)
{
if (event.wasClean)
{
alert("Connection with the server was closed smoothly");
} else
{
alert("Connection with the server was closed unexpectedly");
}
};
Socket.onerror = function(error)
{
alert("Connection Error: " + String(error.message));
};
var colorWheel = new iro.ColorPicker("#ColorWheel",
{
color: "rgb(255, 255, 255)",
borderWidth: 2,
borderColor: '#ffffff',
width: 275,
});
colorWheel.on('color:change', function(color, changes)
{
var Color = colorWheel.color.rgb;
let ColorsJSON = {
Red: Color.r,
Green: Color.g,
Blue: Color.b
};
Socket.send(JSON.stringify(ColorsJSON));
});
</script>
)=====";
I an new to programming and I have no idee why this is not working, any help is appreciated!
Your ESP8266 skecth is OK. Just tested with my own client.
Double check your index.h or replace it by a simple one for testing purpose.
I try to make an HTTPS GET request and make an HTTPS POST request with the data recieved... I am using an arduino UNO with the library GSM_GPRS_GPS_Shield_GSMSHIELD and the GSM GPRS shield SIM900. Here is my current code below:
//include libs
#include "SIM900.h"
#include "inetGSM.h"
#include <SoftwareSerial.h>
InetGSM inet;
//data holder
int par1 = -1;
int par2 = -1;
void setup() {
Serial.begin(9600);
Serial.println("BEGIN");
boolean started = false;
SIM900power();
//initialize the SIM900
if (gsm.begin(2400)){
Serial.println("READY");
started=true;
}else Serial.println("IDLE");
//connect it to the network
if(started){
if (inet.attachGPRS("free", "", ""))
Serial.println("ATTACHED");
else Serial.println("ERROR");
delay(1000);
gsm.SimpleWriteln("AT+CIFSR");
delay(3000);
gsm.WhileSimpleRead();
//GET request
char * json = "";
while(strlen(json) < 4){
delay(2000);
char msg[200] = "";
Serial.println(inet.httpGET("my.site.com", 80, "/somethingToGet?param=1", msg, 200));
//interpret Json
char * msg_tmp = msg;
json = strstr (msg_tmp, "[{");
}
if(json != ""){
const byte posPar1 = (int)(strstr(json, "par1") - json) + 7;
const byte posPar2 = (int)(strstr(json, "par2") - json) + 7;
if(json[posPar1] != 'u')
par1 = extractNum(json, posPar1);
if(json[posPar2] != 'u')
par2 = extractNum(json, posPar2);
}
if(json == "" || par1 == -1 || par2 == -1){
SIM900power();
Serial.println("A JSON ERROR OCCURED");
while(1){}}
}
};
void loop() {
aPostRequest();
while(1){}
};
void SIM900power()
{
digitalWrite(9, HIGH);
delay(1000);
digitalWrite(9, LOW);
delay(8000);
}
//extract the data from the Json string
int extractPar(char * json, byte pos){
int num = 0;
while (json[pos] != '"'){
num = json[pos]-'0' + num * 10;
pos++;
}
return num;
}
//POST request
void aPostRequest(){
if( par1 != -1 && par2 != -1){
boolean dataFound = true;
while(dataFound){
delay(2000);
char params[100];
snprintf(params, 100, "par1=%d&par2=%d", par1,par2);
char msg[200] = "";
dataFound = (inet.httpPOST("my.site.com ", 80, "/something", params , msg, 200) == 0);
}
}
}
I have two web sites, an HTTP one for my tests and the other one in HTTPS. As you can imagine, it's working on my HTTP one.
I don't know how to solve this problem but I think I need to do some tricky things with certificates in the library... can somebody help?
PS: if you want to test the code, you need to uncomment the HTTPPOST() function in the file inetGSM.h of the library. You can edit the functions httpGET() and HTTPPOST() in the file inetGSM.cpp.
UPDATE
There is the library code for the GET function below (httpPOST() works the same) :
int InetGSM::httpGET(const char* server, int port, const char* path, char* result, int resultlength)
{
boolean connected=false;
int n_of_at=0;
int length_write;
char end_c[2];
end_c[0]=0x1a;
end_c[1]='\0';
/*
Status = ATTACHED.
if(gsm.getStatus()!=GSM::ATTACHED)
return 0;
*/
while(n_of_at<3) {
if(!connectTCP(server, port)) {
#ifdef DEBUG_ON
Serial.println("DB:NOT CONN");
#endif
n_of_at++;
} else {
connected=true;
n_of_at=3;
}
}
if(!connected) return 0;
gsm.SimpleWrite("GET ");
gsm.SimpleWrite(path);
gsm.SimpleWrite(" HTTP/1.0\r\nHost: ");
gsm.SimpleWrite(server);
gsm.SimpleWrite("\r\n");
gsm.SimpleWrite("User-Agent: Arduino");
gsm.SimpleWrite("\r\n\r\n");
gsm.SimpleWrite(end_c);
switch(gsm.WaitResp(10000, 10, "SEND OK")) {
case RX_TMOUT_ERR:
return 0;
break;
case RX_FINISHED_STR_NOT_RECV:
return 0;
break;
}
delay(50);
#ifdef DEBUG_ON
Serial.println("DB:SENT");
#endif
int res = gsm.read(result, resultlength);
//gsm.disconnectTCP();
//int res=1;
return res;
}
I have already tried to change the HTTP/1.0 for HTTPS/1.0, but nothing appends.
UPDATE 2
I redirected my request through my HTTP server because I still have not found an answer, if someone could answer for those who could be blocked!
I was trying to make a HTTPS request to a Lambda function that i coded in AWS. The function had to send a json body from a WEMOS D1 mini via a POST to AWS. TBH, I don't know, if this will solve your Issue on your Controller, but it might be worth trying :)
#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>
#ifndef STASSID
#define STASSID "<yourWiFiSSID>"
#define STAPSK "<yourWifiPW>"
#endif
const char* ssid = STASSID;
const char* password = STAPSK;
const char* host = "<the host link here (has to look like **google.com** important, dont add the route here) >";
const int httpsPort = 443;
const String data = "<Json Object here e.g -> **{\"temperature\": 20.5, \"humidity\": 60}**>";
// Use web browser to view and copy
// SHA1 fingerprint of the certificate
const char fingerprint[] PROGMEM = "5F F1 60 31 09 04 3E F2 90 D2 B0 8A 50 38 04 E8 37 9F BC 76";
void setup() {
Serial.begin(115200);
Serial.println();
Serial.print("connecting to ");
Serial.println(ssid);
WiFi.mode(WIFI_STA);
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
// Use WiFiClientSecure class to create TLS connection
WiFiClientSecure client;
Serial.print("connecting to ");
Serial.println(host);
Serial.printf("Using fingerprint '%s'\n", fingerprint);
client.setFingerprint(fingerprint);
client.setInsecure();
if (!client.connect(host, httpsPort)) {
Serial.println("connection failed");
return;
}
String url = "<your route here e.g **/photos/test**> ";
Serial.print("requesting URL: ");
Serial.println(url);
client.print(String("POST ") + url + " HTTP/1.1\r\n" +
"Host: " + host + "\r\n" +
"Connection: close\r\n"+
"Content-Length: " + data.length() + "\r\n" +
"Content-Type: application/json;charset=UTF-8\r\n\r\n"+
data +"\r\n");
Serial.println("request sent");
//READ INCOMING HTML
uint8_t * _buffer = new uint8_t[128];
String line = "";
if (client.connected()) {
int actualLength = client.read(_buffer, 128);
// If it fails for whatever reason
if(actualLength <= 0)
{
return;
}
// Concatenate the buffer content to the final response string
// I used an arduino String for convenience
// but you can use strcat or whatever you see fit
//TRANSFORM EVERY CHAR FROM INCOMING HTML
line += String((char*)_buffer).substring(0, actualLength);
if (line == "\r") {
Serial.println("headers received");
}
}
Serial.print("Line: ");
Serial.println(line);
if (line.startsWith("{\"state\":\"success\"")) {
Serial.println("esp8266/Arduino CI successfull!");
} else {
Serial.println("esp8266/Arduino CI has failed");
}
Serial.println("reply was:");
Serial.println("==========");
Serial.println(line);
Serial.println("==========");
Serial.println("closing connection");
}
void loop() {
}
I really hope this might have helped someone.
I am trying to replicate a project for Kinect for this music video, but the code is seriously outdated.
After weeks searching, I have not found anything about this.
I would be greatly thankful to anyone who points out to me what is deprecated in the following code:
(I'm using Processing 3)
import org.openkinect.*;
import org.openkinect.processing.*;
import java.io.*;
// Kinect Library object
Kinect kinect;
float a = 0;
// Size of kinect image
int w = 640;
int h = 480;
// writing state indicator
boolean write = false;
// treshold filter initial value
int fltValue = 950;
// "recording" object. each vector element holds a coordinate map vector
Vector <Object> recording = new Vector<Object>();
// We'll use a lookup table so that we don't have to repeat the math over and over
float[] depthLookUp = new float[2048];
void setup() {
size(800,600,P3D);
kinect = new Kinect(this);
kinect.start();
kinect.enableDepth(true);
// We don't need the grayscale image in this example
// so this makes it more efficient
kinect.processDepthImage(false);
// Lookup table for all possible depth values (0 - 2047)
for (int i = 0; i < depthLookUp.length; i++) {
depthLookUp[i] = rawDepthToMeters(i);
}
}
void draw() {
background(0);
fill(255);
textMode(SCREEN);
text("Kinect FR: " + (int)kinect.getDepthFPS() + "\nProcessing FR: " + (int)frameRate,10,16);
// Get the raw depth as array of integers
int[] depth = kinect.getRawDepth();
// We're just going to calculate and draw every 4th pixel (equivalent of 160x120)
int skip = 4;
// Translate and rotate
translate(width/2,height/2,-50);
rotateY(a);
//noStroke();
//lights();
int index = 0;
PVector[] frame = new PVector[19200];
for(int x=0; x<w; x+=skip) {
for(int y=0; y<h; y+=skip) {
int offset = x+y*w;
// Convert kinect data to world xyz coordinate
int rawDepth = depth[offset];
boolean flt = true;
PVector v = depthToWorld(x,y,rawDepth);
if (flt && rawDepth > fltValue)
{
v = depthToWorld(x,y,2047);
}
frame[index] = v;
index++;
stroke(map(rawDepth,0,2048,0,256));
pushMatrix();
// Scale up by 200
float factor = 400;
translate(v.x*factor,v.y*factor,factor-v.z*factor);
//sphere(1);
point(0,0);
//line (0,0,1,1);
popMatrix();
}
}
if (write == true) {
recording.add(frame);
}
// Rotate
//a += 0.015f;
}
// These functions come from:http://graphics.stanford.edu/~mdfisher/Kinect.html
float rawDepthToMeters(int depthValue) {
if (depthValue < 2047) {
return (float)(1.0 / ((double)(depthValue) * -0.0030711016 + 3.3309495161));
}
return 0.0f;
}
PVector depthToWorld(int x, int y, int depthValue) {
final double fx_d = 1.0 / 5.9421434211923247e+02;
final double fy_d = 1.0 / 5.9104053696870778e+02;
final double cx_d = 3.3930780975300314e+02;
final double cy_d = 2.4273913761751615e+02;
PVector result = new PVector();
double depth = depthLookUp[depthValue];//rawDepthToMeters(depthValue);
result.x = (float)((x - cx_d) * depth * fx_d);
result.y = (float)((y - cy_d) * depth * fy_d);
result.z = (float)(depth);
return result;
}
void stop() {
kinect.quit();
super.stop();
}
int currentFile = 0;
void saveFile() {
}
void keyPressed() { // Press a key to save the data
if (key == '1')
{
fltValue += 50;
println("fltValue: " + fltValue);
}
else if (key == '2')
{
fltValue -= 50;
println("fltValue: " + fltValue);
}
else if (key=='4'){
if (write == true) {
write = false;
println( "recorded " + recording.size() + " frames.");
// saveFile();
// save
Enumeration e = recording.elements();
println("Stopped Recording " + currentFile);
int i = 0;
while (e.hasMoreElements()) {
// Create one directory
boolean success = (new File("out"+currentFile)).mkdir();
PrintWriter output = createWriter("out"+currentFile+"/frame" + i++ +".txt");
PVector [] frame = (PVector []) e.nextElement();
for (int j = 0; j < frame.length; j++) {
output.println(j + ", " + frame[j].x + ", " + frame[j].y + ", " + frame[j].z );
}
output.flush(); // Write the remaining data
output.close();
}
currentFile++;
}
}
else if (key == '3') {
println("Started Recording "+currentFile);
recording.clear();
write = true;
}
}
If the code works, then I wouldn't worry too much about it. Deprecated can just mean that a newer version is available, not that the older version stopped working.
However, if the code does not work, then updating to a newer library is probably a good idea anyway. Check out the library section of the Processing homepage, which lists several Kinect libraries.
In fact, one of those libraries is the updated version of the old library you're using: Open Kinect for Processing.
Edit: It looks like both of the errors you mentioned are due to missing import statements. You need to import both Vector and Enumeration to use them:
import java.util.Vector;
import java.util.Enumeration;
I'm trying to save an image after certain time, the problem is that the image size is bigger than the display so when I use the save or saveFrame function it only saves the image that I can see in the display. There is any other way to save the whole image?
This is my code:
PImage picture, pictureFilter, img;
int total, cont, current;
ArrayList<ArrayList<Position>> columns;
String[] fontList;
public class Position {
public int x;
public int y;
}
void setup() {
fontList = PFont.list();
picture = loadImage("DSC05920b.JPG");
pictureFilter = loadImage("filtrePort2.jpg");
frame.setResizable(true);
size(picture.width, picture.height);
columns = new ArrayList<ArrayList<Position>>();
for(int i = 0; i < picture.width; i++) {
ArrayList<Position> row = new ArrayList<Position>();
for(int j = 0; j < picture.height; j++){
Position p = new Position();
p.x = i;
p.y = j;
row.add(p);
}
columns.add(row);
}
total = picture.width * picture.height;
cont = total;
current = 0;
img = createImage(picture.width, picture.height, RGB);
}
float randomLetter() {
float value = 23;
boolean found = false;
while(!found) {
value = random(48, 122);
if(value >48 && value <58) found = true;
if(value >65 && value <91) found = true;
if(value >97 && value <123) found = true;
}
return value;
}
void draw() {
int x = int(random(0, columns.size()));
ArrayList<Position> rows = columns.get(x);
int y = int(random(0, rows.size()));
Position p = rows.get(y);
color c = pictureFilter.get(p.x, p.y);
int r = (c >> 16) & 0xFF; // Faster way of getting red(argb)
if(r < 240) {
PFont f = createFont(fontList[int(random(0,fontList.length))],random(5, 24),true);
textFont(f);
fill(picture.get(p.x,p.y));
char letter = (char) int(randomLetter());
text(letter, p.x, p.y);
}
if(rows.size() == 1) {
if(columns.size() == 1) {
saveFrame("lol.jpg");
columns.remove(x);
} else {
columns.remove(x);
}
} else {
println(rows.size());
rows.remove(y);
}
--cont;
float percent = float(total-cont)/float(total)*100;
if(int(percent) != current) {
current = int(percent);
save("image_" + current + ".jpg");
}
println("DONE: " + (total-cont) + "/" + total + " Progress: " + percent + "%");
}
The code do a lot of stuff but the part that its not working well is at the final when I check if the percentage have been increased in order to save the image
You can write this into a PGraphics context - aka a graphics buffer.
The buffer can be as big as you need it to be, and you can choose whether to draw it on the screen or not..
// Create the buffer at the size you need, and choose the renderer
PGraphics pg = createGraphics(myImage.width, myImage.height, P2D);
// Wrap all your drawing functions in the pg context - e.g.
PFont f = createFont(fontList[int(random(0,fontList.length))],random(5, 24),true);
textFont(f);
pg.beginDraw();
pg.fill(picture.get(p.x,p.y));
char letter = (char) int(randomLetter());
pg.text(letter, p.x, p.y);
pg.endDraw();
// Draw your PG to the screen and resize the representation of it to the screen bounds
image(pg, 0, 0, width, height); // <-- this wont actually clip/resize the image
// Save it
pg.save("image_" + current + ".jpg");
The PImage class contains a save() function that exports to file. The API should be your first stop for questions like this.
I've created a processing sketch which saves each frame of point cloud data from the kinect to a text file, where each line of the file is a point (or vertex) that the kinect has registered. I plan to pull the data into a 3d program to visualize the animation in 3d space and apply various effects. The problem is, when I do this, the first frame seems proper, and the rest of the frames seem to be spitting out what looks like the first image, plus a bunch of random noise. This is my code, in its entirety. It requires simple openni to work properly. You can see the comments
import SimpleOpenNI.*;
//import processing.opengl.*;
SimpleOpenNI context;
float zoomF =0.5f;
float rotX = radians(180); // by default rotate the hole scene 180deg around the x-axis,
float rotY = radians(0); // the data from openni comes upside down
int maxZ = 2000;
Vector <Object> recording = new Vector<Object>();
boolean isRecording = false;
boolean canDraw = true;
boolean mouseMode = false;
int currentFile = 0;
int depthWidth = 640; //MH - assuming this is static?
int depthHeight = 480;
int steps = 5;
int arrayLength = (depthWidth/steps) * (depthHeight/steps); //total lines in each output file
void setup()
{
size(1024,768,P3D); // strange, get drawing error in the cameraFrustum if i use P3D, in opengl there is no problem
//size(1024,768,OPENGL);
context = new SimpleOpenNI(this);
context.setMirror(true);
depthWidth = context.depthWidth();
depthHeight = context.depthHeight();
// enable depthMap generation
if(context.enableDepth() == false)
{
println("Can't open the depthMap, maybe the camera is not connected!");
exit();
return;
}
stroke(255,255,255);
smooth();
perspective(radians(45),
float(width)/float(height),
10.0f,150000.0f);
}
void draw()
{
//println(isRecording);
// update the cam
context.update();
background(0,0,0);
// set the scene pos
translate(width/2, height/2, 0);
rotateX(rotX);
rotateY(rotY);
scale(zoomF);
// draw the 3d point depth map
int[] depthMap = context.depthMap();
int index = 0;
PVector realWorldPoint;
PVector[] frame = new PVector[arrayLength];
translate(0,0,-1000); // set the rotation center of the scene 1000 infront of the camera
stroke(200);
for(int y=0;y < context.depthHeight();y+=steps)
{
for(int x=0;x < context.depthWidth();x+=steps)
{
int offset = x + y * context.depthWidth();
realWorldPoint = context.depthMapRealWorld()[offset];
if (isRecording == true){
if (realWorldPoint.z < maxZ){
frame[index] = realWorldPoint;
} else {
frame[index] = new PVector(-0.0,-0.0,0.0);
}
index++;
} else {
if (realWorldPoint.z < maxZ){
if (canDraw == true){
point(realWorldPoint.x,realWorldPoint.y,realWorldPoint.z);
}
}
}
}
}
if (isRecording == true){
recording.add(frame);
}
if (mouseMode == true){
float rotVal = map (mouseX,0,1024,-1,1); //comment these out to disable mouse orientation
float rotValX = map (mouseY,0,768,2,4);
rotY = rotVal;
rotX = rotValX;
}
}
// -----------------------------------------------------------------
// Keyboard event
void keyPressed()
{
switch(key)
{
case ' ':
context.setMirror(!context.mirror());
break;
case 'm':
mouseMode = !mouseMode;
break;
case 'r':
isRecording = !isRecording;
break;
case 's':
if (isRecording == true){
isRecording = false;
canDraw = false;
println("Stopped Recording");
Enumeration e = recording.elements();
int i = 0;
while (e.hasMoreElements()) {
// Create one directory
boolean success = (new File("out"+currentFile)).mkdir();
PrintWriter output = createWriter("out"+currentFile+"/frame" + i++ +".txt");
PVector [] frame = (PVector []) e.nextElement();
for (int j = 0; j < frame.length; j++) {
output.println(j + ", " + frame[j].x + ", " + frame[j].y + ", " + frame[j].z );
}
output.flush(); // Write the remaining data
output.close();
//exit();
}
canDraw = true;
println("done recording");
}
currentFile++;
break;
}
switch(keyCode)
{
case LEFT:
if(keyEvent.isShiftDown())
maxZ -= 100;
else
rotY += 0.1f;
break;
case RIGHT:
if(keyEvent.isShiftDown())
maxZ += 100;
else
rotY -= 0.1f;
break;
case UP:
if(keyEvent.isShiftDown())
zoomF += 0.01f;
else
rotX += 0.1f;
break;
case DOWN:
if(keyEvent.isShiftDown())
{
zoomF -= 0.01f;
if(zoomF < 0.01)
zoomF = 0.01;
}
else
rotX -= 0.1f;
break;
}
}
I imagine the loop is where the problems begin occurring: for(int y=0;y < context.depthHeight();y+=steps)
{ , etc. although it could just be a problem with the python script I wrote for the 3d program. Anyway, this is a cool sketch, and I think would be super useful for anyone wanting to do 3d effects to point cloud data (or build models, etc), but I'm stuck at the moment. Thanks for your help!
Unfortunately I can't explain a lot right now, but I've sone something similar a few months back saving to PLY and CSV:
import processing.opengl.*;
import SimpleOpenNI.*;
SimpleOpenNI context;
float zoomF =0.5f;
float rotX = radians(180);
float rotY = radians(0);
boolean recording = false;
ArrayList<PVector> pts = new ArrayList<PVector>();//points for one frame
float minZ = 100,maxZ = 150;
void setup()
{
size(1024,768,OPENGL);
context = new SimpleOpenNI(this);
context.setMirror(false);
context.enableDepth();
context.enableScene();
stroke(255);
smooth();
perspective(95,float(width)/float(height), 10,150000);
}
void draw()
{
context.update();
background(0);
translate(width/2, height/2, 0);
rotateX(rotX);
rotateY(rotY);
scale(zoomF);
int[] depthMap = context.depthMap();
int[] sceneMap = context.sceneMap();
int steps = 10;
int index;
PVector realWorldPoint;
pts.clear();//reset points
translate(0,0,-1000);
//*
//stroke(100);
for(int y=0;y < context.depthHeight();y+=steps)
{
for(int x=0;x < context.depthWidth();x+=steps)
{
index = x + y * context.depthWidth();
if(depthMap[index] > 0)
{
realWorldPoint = context.depthMapRealWorld()[index];
if(realWorldPoint.z > minZ && realWorldPoint.z < maxZ){//if within range
stroke(0,255,0);
point(realWorldPoint.x,realWorldPoint.y,realWorldPoint.z);
pts.add(realWorldPoint.get());//store each point
}
}
}
}
if(recording){
savePLY(pts);//save to disk as PLY
saveCSV(pts);//save to disk as CSV
}
//*/
}
// -----------------------------------------------------------------
// Keyboard events
void keyPressed()
{
if(key == 'q') minZ += 10;
if(key == 'w') minZ -= 10;
if(key == 'a') maxZ += 10;
if(key == 's') maxZ -= 10;
switch(key)
{
case ' ':
context.setMirror(!context.mirror());
break;
case 'r':
recording = !recording;
break;
}
switch(keyCode)
{
case LEFT:
rotY += 0.1f;
break;
case RIGHT:
// zoom out
rotY -= 0.1f;
break;
case UP:
if(keyEvent.isShiftDown())
zoomF += 0.01f;
else
rotX += 0.1f;
break;
case DOWN:
if(keyEvent.isShiftDown())
{
zoomF -= 0.01f;
if(zoomF < 0.01)
zoomF = 0.01;
}
else
rotX -= 0.1f;
break;
}
}
void savePLY(ArrayList<PVector> pts){
String ply = "ply\n";
ply += "format ascii 1.0\n";
ply += "element vertex " + pts.size() + "\n";
ply += "property float x\n";
ply += "property float y\n";
ply += "property float z\n";
ply += "end_header\n";
for(PVector p : pts)ply += p.x + " " + p.y + " " + p.z + "\n";
saveStrings("frame_"+frameCount+".ply",ply.split("\n"));
}
void saveCSV(ArrayList<PVector> pts){
String csv = "x,y,z\n";
for(PVector p : pts) csv += p.x + "," + p.y + "," + p.z + "\n";
saveStrings("frame_"+frameCount+".csv",csv.split("\n"));
}
I'm using an if statement to save only the points within a certain Z threshold, but feel free to alter/use as you see fit.
The post processing idea reminds of the Moullinex video for Catalina. Check it out, it's well documented and includes source code as well.
Update
The posted code saves 1 file per frame. Even though the playback speed would be low, the sketch should still save a file for each frame. The code be simplified a bit:
import processing.opengl.*;
import SimpleOpenNI.*;
SimpleOpenNI context;
float zoomF =0.5f;
float rotX = radians(180);
float rotY = radians(0);
boolean recording = false;
String csv;
void setup()
{
size(1024,768,OPENGL);
context = new SimpleOpenNI(this);
context.setMirror(false);
context.enableDepth();
stroke(255);
smooth();
perspective(95,float(width)/float(height), 10,150000);
}
void draw()
{
csv = "x,y,z\n";//reset csv for this frame
context.update();
background(0);
translate(width/2, height/2, 0);
rotateX(rotX);
rotateY(rotY);
scale(zoomF);
int[] depthMap = context.depthMap();
int[] sceneMap = context.sceneMap();
int steps = 10;
int index;
PVector realWorldPoint;
translate(0,0,-1000);
//*
beginShape(POINTS);
for(int y=0;y < context.depthHeight();y+=steps)
{
for(int x=0;x < context.depthWidth();x+=steps)
{
index = x + y * context.depthWidth();
if(depthMap[index] > 0)
{
realWorldPoint = context.depthMapRealWorld()[index];
vertex(realWorldPoint.x,realWorldPoint.y,realWorldPoint.z);
if(recording) csv += realWorldPoint.x + "," + realWorldPoint.y + "," + realWorldPoint.z + "\n";
}
}
}
endShape();
if(recording) saveStrings("frame_"+frameCount+".csv",csv.split("\n"));
frame.setTitle((int)frameRate + " fps");
//*/
}
// -----------------------------------------------------------------
// Keyboard events
void keyPressed()
{
switch(key)
{
case ' ':
context.setMirror(!context.mirror());
break;
case 'r':
recording = !recording;
break;
}
switch(keyCode)
{
case LEFT:
rotY += 0.1f;
break;
case RIGHT:
// zoom out
rotY -= 0.1f;
break;
case UP:
if(keyEvent.isShiftDown())
zoomF += 0.01f;
else
rotX += 0.1f;
break;
case DOWN:
if(keyEvent.isShiftDown())
{
zoomF -= 0.01f;
if(zoomF < 0.01)
zoomF = 0.01;
}
else
rotX -= 0.1f;
break;
}
}
The preview can be separated from the recording with different loops and you could have a low res preview, but save more data, still, it would be slow.
I've got another suggestion: Record to the .oni format instead. If you've installed OpenNI, you could make use of a couple of samples like NiViewer and NiBackRecorder. SimpleOpenNI also exposes this functionality, have a look at the RecorderPlay sample.
I suggest trying something like this:
Record your scene to an .oni file. It should be fast/responsive
When you're happy with you're .oni recording, process each frame (convert depth to x,y,z points/ filter as needed/ save to the desired format/etc.)
Here's another sketch to illustrate the idea:
import SimpleOpenNI.*;
SimpleOpenNI context;
boolean recordFlag = true;
int frames = 0;
void setup(){
context = new SimpleOpenNI(this);
if(! recordFlag){
if(! context.openFileRecording("test.oni") ){
println("can't find recording !!!!");
exit();
}
context.enableDepth();
}else{
// recording
context.enableDepth();
// setup the recording
context.enableRecorder(SimpleOpenNI.RECORD_MEDIUM_FILE,"test.oni");
// select the recording channels
context.addNodeToRecording(SimpleOpenNI.NODE_DEPTH,SimpleOpenNI.CODEC_16Z_EMB_TABLES);
}
// set window size
if((context.nodes() & SimpleOpenNI.NODE_DEPTH) != 0)
size(context.depthWidth() , context.depthHeight());
else
exit();
}
void draw()
{
background(0);
context.update();
if((context.nodes() & SimpleOpenNI.NODE_DEPTH) != 0) image(context.depthImage(),0,0);
if(recordFlag) frames++;
}
void keyPressed(){
if(key == ' '){
if(recordFlag){
saveStrings(dataPath("frames.txt"),split(frames+" ",' '));
exit();
}else saveONIToPLY();
}
}
void saveONIToPLY(){
frames = int(loadStrings(dataPath("frames.txt"))[0]);
println("recording " + frames + " frames");
int w = context.depthWidth();
int h = context.depthHeight();
noLoop();
for(int i = 0 ; i < frames; i++){
PrintWriter output = createWriter(dataPath("frame_"+i+".ply"));
output.println("ply");
output.println("format ascii 1.0");
output.println("element vertex " + (w*h));
output.println("property float x");
output.println("property float y");
output.println("property float z");
output.println("end_header\n");
context.update();
int[] depthMap = context.depthMap();
int index;
PVector realWorldPoint;
for(int y=0;y < h;y++){
for(int x=0;x < w;x++){
index = x + y * w;
realWorldPoint = context.depthMapRealWorld()[index];
output.println(realWorldPoint.x + " " + realWorldPoint.y + " " + realWorldPoint.z);
}
}
output.flush();
output.close();
println("saved " + (i+1) + " of " + frames);
}
loop();
println("recorded " + frames + " frames");
}
When the recordFlag is set to true, data will be saved to an .oni file.
I haven't found anything in the docs to read how many frames there are in an .oni file so as a quick workaround I've added the frame counter. If you hit space, the recording will stop, but will also save the number of frames in a txt file then exit the app. This will be useful later.
When the recordFlag is set to false, if there is a recording already, it will playback.
If you hit space in this 'mode', drawing will stop, the frame number will be load from the .txt file and for each frame:
The context will be updated (moving to the next frame)
Each pixel in the depth map will be converted to a point
ALL the points will be written to a .ply file (you can process with meshlab)
After all frames were saved, the sketch will resume drawing. Since there's no 3D drawing and the sketch is fairly simple, performance should be better, but bare in mind that large .oni file will require a lot of RAM. Feel free to modify the sketch to your needs (e.g. filter out the information you don't want saved, etc.).
Also note that the above, although should save to PLY each separate frame, it saves the same. It seems the context doesn't update() when noLoop() has been called. Here's a modified hacky version that uses a 3s. delay (hopefully the .ply fille will be written to disk by then).
import SimpleOpenNI.*;
SimpleOpenNI context;
boolean recordFlag = false;
boolean saving = false;
int frames = 0;
int savedFrames = 0;
void setup(){
context = new SimpleOpenNI(this);
if(! recordFlag){
if(! context.openFileRecording("test.oni") ){
println("can't find recording !!!!");
exit();
}
context.enableDepth();
}else{
// recording
context.enableDepth();
// setup the recording
context.enableRecorder(SimpleOpenNI.RECORD_MEDIUM_FILE,"test.oni");
// select the recording channels
context.addNodeToRecording(SimpleOpenNI.NODE_DEPTH,SimpleOpenNI.CODEC_16Z_EMB_TABLES);
}
// set window size
if((context.nodes() & SimpleOpenNI.NODE_DEPTH) != 0)
size(context.depthWidth() , context.depthHeight());
else
exit();
}
void draw()
{
background(0);
context.update();
if((context.nodes() & SimpleOpenNI.NODE_DEPTH) != 0) image(context.depthImage(),0,0);
if(recordFlag) frames++;
if(saving && savedFrames < frames){
delay(3000);//hack
int i = savedFrames;
int w = context.depthWidth();
int h = context.depthHeight();
PrintWriter output = createWriter(dataPath("frame_"+i+".ply"));
output.println("ply");
output.println("format ascii 1.0");
output.println("element vertex " + (w*h));
output.println("property float x");
output.println("property float y");
output.println("property float z");
output.println("end_header\n");
rect(random(width),random(height),100,100);
int[] depthMap = context.depthMap();
int index;
PVector realWorldPoint;
for(int y=0;y < h;y++){
for(int x=0;x < w;x++){
index = x + y * w;
realWorldPoint = context.depthMapRealWorld()[index];
output.println(realWorldPoint.x + " " + realWorldPoint.y + " " + realWorldPoint.z);
}
}
output.flush();
output.close();
println("saved " + (i+1) + " of " + frames);
savedFrames++;
}
}
void keyPressed(){
if(key == ' '){
if(recordFlag){
saveStrings(dataPath("frames.txt"),split(frames+" ",' '));
exit();
}else saveONIToPLY();
}
}
void saveONIToPLY(){
frames = int(loadStrings(dataPath("frames.txt"))[0]);
saving = true;
println("recording " + frames + " frames");
}
I'm not sure frames and files sync and the depth data is saved at medium quality, but I hope my answer provides some ideas.