I have an running and working system which is an ESP32 with an LAN8720 to communicate over the internet.
Now just plane HTTP requests (via the WiFiClientSecure client) work like a charm. But I also need (over https) to update the device.
Now I currently have this code:
#include <Arduino.h>
#include <Update.h>
#include <HTTPUpdate.h>
WiFiClientSecure otaClient;
Serial.println("Preparing to update");
// Do OTA update
otaClient.setInsecure(); //skip verification of SSL cert
if (!otaClient.connect(DIGITAL_HQ_BASE_URL, 443)) {
Serial.println("Could not connect.");
}
otaClient.print("GET "); // watch the space!
otaClient.print(DIGITAL_HQ_BINARY_ENDPOINT); // API endpoint
otaClient.println(" HTTP/1.1"); // watch the space!
otaClient.print("Host: ");
otaClient.println(DIGITAL_HQ_BASE_URL);
otaClient.println("Connection: keep-alive"); // Don't close, since we need to perform OTA
otaClient.print("User-Agent: ");
otaClient.println(DIGITAL_HQ_USER_AGENT);
otaClient.println("Cache-Control: no-cache");
otaClient.println();
unsigned long timeout = millis();
while (otaClient.available() == 0) {
if (millis() - timeout > 5000) {
Serial.println("Client Timeout !");
otaClient.stop();
return;
}
}
while (otaClient.available()) {
// read line till /n
String line = otaClient.readStringUntil('\n');
// remove space, to check if the line is end of headers
line.trim();
// if the the line is empty,
// this is end of headers
// break the while and feed the
// remaining `client` to the
// Update.writeStream();
if (!line.length()) {
//headers ended
break; // and get the OTA started
}
// Check if the HTTP Response is 200
// else break and Exit Update
if (line.startsWith("HTTP/1.1")) {
if (line.indexOf("200") < 0) {
Serial.println("Got a non 200 status code from server. Exiting OTA Update.");
break;
}
}
// extract headers here
// Start with content length
if (line.startsWith("Content-Length: ")) {
contentLength = atol((getHeaderValue(line, "Content-Length: ")).c_str());
Serial.println("Got " + String(contentLength) + " bytes from server");
}
// Next, the content type
if (line.startsWith("Content-Type: ")) {
String contentType = getHeaderValue(line, "Content-Type: ");
Serial.println("Got " + contentType + " payload.");
if (contentType == "application/octet-stream") {
isValidContentType = true;
}
}
}
Serial.println("contentLength : " + String(contentLength) + ", isValidContentType : " + String(isValidContentType));
if (contentLength && isValidContentType) {
// Check if there is enough to OTA Update
bool canBegin = Update.begin(contentLength);
// If yes, begin
if (canBegin) {
Serial.println("Begin OTA. This may take 2 - 5 mins to complete. Things might be quite for a while.. Patience!");
// No activity would appear on the Serial monitor
// So be patient. This may take 2 - 5mins to complete
size_t written = Update.writeStream(otaClient);
if (written == contentLength) {
Serial.println("Written : " + String(written) + " successfully");
} else {
Serial.println("Written only : " + ConvertFormatBytes(written) + "/" + ConvertFormatBytes(contentLength));
// retry??
// execOTA();
}
if (Update.end()) {
Serial.println("OTA done!");
if (Update.isFinished()) {
Serial.println("Update successfully completed. Rebooting.");
ESP.restart();
} else {
Serial.println("Update not finished? Something went wrong!");
}
} else {
Serial.println("Error Occurred. Error #: " + String(Update.getError()));
}
} else {
// not enough space to begin OTA
// Understand the partitions and
// space availability
Serial.println("Not enough space to begin OTA");
otaClient.stop();
}
} else {
Serial.println("There was no content in the response");
otaClient.stop();
}
This runs without errors, but is frozen on the Preparing to update console message. Anyone who has an idea what I'm doing wrong here?
The file NEEDS to come from an https domain.
Related
I am trying to download my hex file of size 1500KB via UDS with CAPL test module,
p2 timer = 50ms
p2* timer = 5000ms
Here is snippet of my code for data transfer :
void TS_transferData()
{
byte transferData_serviceid = 0x36;
byte blockSequenceCounter = 0x1;
byte buffer[4093];
byte binarydata[4095];
long i,ret1,ret2,ret3,temp,timeout = 0,Counter = 0;
char filename[30] = "xxx.bin";
dword readaccess_handle;
diagrequest ECU_QUALIFIER.* request;
long valueleft;
readaccess_handle = OpenFileRead(filename,1);
if (readaccess_handle != 0 )
{
while( (valueleft = fileGetBinaryBlock(buffer,elcount(buffer),readaccess_handle))==4093 )
{
binarydata[0] = transferData_serviceid;
binarydata[1] = blockSequenceCounter;
for(i=0;i<elcount(buffer);i++)
{
binarydata[i+2] = buffer[i];
}
diagResize(request, elCount(binarydata));
DiagSetPrimitiveData(request,binarydata,elcount(binarydata));
DiagSendRequest(request);
write("length of binarydata %d ",elcount(binarydata));
// Wait until the request has been completely sent
ret1 = TestWaitForDiagRequestSent(request, 20000);
if(ret1 == 1) // Request sent
{
ret2=TestWaitForDiagResponse(request,50);
if(ret2==1) // Response received
{
ret3=DiagGetLastResponseCode(request); // Get the code of the response
if(ret3==-1) // Is it a positive response?
{
;
}
else
{
testStepFail(0, "4.0","Binary Datatransfer on server Failed");
break;
}
}
else if(ret2 == timeout)
{
testStepFail(0, "4.0","Binary Datatransfer on server Failed");
write("timeout occured while TestWaitForDiagResponse with block %d ",blockSequenceCounter);
}
}
else if(ret1 == timeout)
{
testStepFail(0, "4.0","Binary Datatransfer on server Failed");
write("timeout occured while TestWaitForDiagRequestSent %d ",blockSequenceCounter);
}
if(blockSequenceCounter == 255)
blockSequenceCounter = 0;
else
++blockSequenceCounter;
}
}
//handle the rest of the bytes to be transmitted
fileClose (readaccess_handle);
}
The software downloading is happening but it is taking a long.... time for download.
For TestWaitForDiagRequestSent() function any value for timeout less than 20000 is giving me timeout error.
Is there any other way I can reduce the software transfer time or where am I going wrong with calculation?
Is there any example I can refer to see How to transmit such a long data using CAPL ?
Sorry, I am a beginner to CAPL and UDS protocol.
I'm trying to post a (large) json file that's stored in the SPIFFS (LittleFS- of an arduino.
I first of all, fetch the JSON from a local IP address, which can change or is dynamic. Then, that json file gets stored in the internal flash of my ESP32 and sent over to our servers.
The goal is to have an ESP32 act as a bridge for local addresses.
For example:
Locally, there is a server running, that server is never accessible from outside of a network, but it is from within. We place an ESP32 in that network, to be able to read out data from the internal device and pass them through to our servers, for further processing.
The current code I have is the following;
HTTPClient http;
if (LittleFS.exists("/downloadedFile.json")) {
LittleFS.remove("/downloadedFile.json");
}
File f = LittleFS.open("/downloadedFile.json", "w", true);
if (f) {
String url;
// Build the URL...
if (username != "" && password != "") {
url = "http://" + username + ":" + password + "#" + ip + ":" + String(port) + "/data/file.json";
} else {
url = "http://" + ip + ":" + String(port) + "/data/file.json";
}
http.begin(url);
int httpCode = http.GET();
if (httpCode > 0) {
if (httpCode == HTTP_CODE_OK) {
http.writeToStream(&f);
}
} else {
// failed.
}
f.close();
} else {
// Unable to open file
}
http.end();
if (LittleFS.exists("/downloadedFile.json")) {
File f = LittleFS.open("/downloadedFile.json", "r");
if (f) {
WiFiClienSecure client;
client.setInsecure(); // test
if (!client.connect("my.somain.com", 443)) {
// failed to connect. (never gets in here)
return;
}
client.println("POST /path/to/my/endpoint HTTP/1.1");
client.println("Host: my.somain.com");
client.println("Content-Type: application/json");
client.println("Connection: keep-alive");
client.println("Content-Length: " + String(f.size()));
client.println();
// Build the body
int len = 0;
int total = 0;
uint8_t buf[1025] = {};
size_t _len = 256;
do {
len = f.read(buf, _len);
client.write(buf, len);
total += len;
} while (len > 0);
client.println(); // end line
client.stop();
f.close();
}
}
Downloading the file always works, except for sometimes random crashing the whole system. But that's the least of my concerns.
Locally I got it working to download and re-upload the file. Struggle is when I try to achieve the same result on the server (which uses https)
Only difference is this;
Server
Local
WiFiClienSecure client;
WiFiClient client;
client.setInsecure();
client.connect("my.somain.com", 443)
client.connect("IP", 80)
The main problem is that it doesn't work on the server and that it works really unstable. the file size (when I do f.size()) is 310831.
Any help and thoughts on getting this stable is appriciated!
I'm trying to make Multiple GET calls using the same client. I've tried multiple things. New Variables, New instances. Closing Clients. The second Stock Call doesnt go through ever. It fails at "while (client.connected())" on the second Call. It Doesnt get to returning headers.
Any Ideas?
Code look like this
void loop()
{
WiFiClientSecure client;
//Stock
String payload;
// Use WiFiClientSecure class to create TLS connection
Serial.print("connecting to ");
Serial.println(host);
Serial.printf("Using fingerprint '%s'\n", fingerprint);
client.setFingerprint(fingerprint);
if (!client.connect(host, httpsPort))
{
Serial.println("connection failed");
client.stop();
//return;
}
//AAPL
String url = "/api/v1/quote?symbol=AAPL&token=XXXX";
Serial.print("requesting URL: ");
Serial.println(url);
client.print(String("GET ") + url + " HTTP/1.0\r\n" +
"Host: " + host + "\r\n" +
"User-Agent: BuildFailureDetectorESP8266\r\n" +
"Connection: close\r\n\r\n");
Serial.println("request sent");
while (client.connected())
{
String line = client.readStringUntil('\n');
// String line = client.readStringUntil('\r');
Serial.println(line);
if (line == "\r")
{
Serial.println("headers received");
payload = client.readStringUntil('\r');
//break;
}
}
Serial.println("payload");
Serial.println(payload);
// String line = client.readString();
//JsonObject& obj = doc.parseObject(payload);
StaticJsonDocument<200> doc;
DeserializationError error = deserializeJson(doc, payload);
// Test if parsing succeeds.
if (error) {
Serial.print(F("deserializeJson() failed: "));
Serial.println(error.f_str());
// return;
client.stop();
}
//{"c":23.35,"d":-0.73,"dp":-3.0316,"h":24.6986,"l":23.25,"o":24.54,"pc":24.08,"t":1627588802}
double currentprice = doc["c"];
double pctchg = doc["d"];
// Print values.
Serial.println(currentprice);
Serial.println(pctchg);
Serial.println("closing connection");
client.stop();
delay(20000);
//SPY
String spyurl = "/api/v1/quote?symbol=SPY&token=XXXX";
Serial.print("requesting URL: ");
Serial.println(spyurl);
client.print(String("GET ") + spyurl + " HTTP/1.0\r\n" +
"Host: " + host + "\r\n" +
"User-Agent: BuildFailureDetectorESP8266\r\n" +
"Connection: close\r\n\r\n");
Serial.println("request sent");
String spypayload;
while (client.connected())
{
String spyline = client.readStringUntil('\n');
// String line = client.readStringUntil('\r');
Serial.println(spyline);
if (spyline == "\r")
{
Serial.println("headers received");
spypayload = client.readStringUntil('\r');
//break;
}
}
Serial.println("spypayload");
Serial.println(spypayload);
// String line = client.readString();
//JsonObject& obj = doc.parseObject(payload);
StaticJsonDocument<200> spydoc;
DeserializationError spyerror = deserializeJson(spydoc, spypayload);
// Test if parsing succeeds.
if (spyerror) {
Serial.print(F("deserializeJson() failed: "));
Serial.println(spyerror.f_str());
// return;
client.stop();
}
//{"c":23.35,"d":-0.73,"dp":-3.0316,"h":24.6986,"l":23.25,"o":24.54,"pc":24.08,"t":1627588802}
double SPYcurrentprice = spydoc["c"];
double SPYpctchg = spydoc["d"];
// Print values.
Serial.println(SPYcurrentprice);
Serial.println(SPYpctchg);
Serial.println("closing connection");
client.stop();
I'm an idiot.
Needed to keep the connection open on the first call
"Connection: Keep-Alive\r\n\r\n");
I'm working on a project using a google script to publish images from an ESP32-CAM to google drive.
I keep getting the error "no response", no matter how long I set the wait peroid to. I tried using http and https(wificlient and wificlientsecure) but it doesn't work no matter what.
What could be the issue?
WiFiClientSecure clienthttps;
Serial.println("Connect to " + String(myDomain));
if (clienthttps.connect(myDomain, 443)) {
Serial.println("Connection successful");
camera_fb_t * fb = NULL;
fb = esp_camera_fb_get();
if (!fb) {
Serial.println("Camera capture failed");
delay(1000);
ESP.restart();
return;
}
char *input = (char *)fb->buf;
char output[base64_enc_len(3)];
String imageFile = "";
for (int i = 0; i < fb->len; i++) {
base64_encode(output, (input++), 3);
if (i % 3 == 0) imageFile += urlencode(String(output));
}
String Data = myFilename + mimeType + myImage;
esp_camera_fb_return(fb);
Serial.println("Send a captured image to Google Drive.");
Serial.println("POST " + myScript + " HTTP/1.1");
Serial.println("Host: " + String(myDomain));
Serial.println("Content-Length: " + String(Data.length() + imageFile.length()));
Serial.println("Content-Type: application/x-www-form-urlencoded");
Serial.println(Data);
Serial.println();
clienthttps.println("POST " + myScript + " HTTP/1.1");
clienthttps.println("Host: " + String(myDomain));
clienthttps.println("Content-Length: " + String(Data.length() + imageFile.length()));
clienthttps.println("Content-Type: application/x-www-form-urlencoded");
clienthttps.println();
clienthttps.print(Data);
int Index;
for (Index = 0; Index < imageFile.length(); Index = Index + 1000) {
clienthttps.print(imageFile.substring(Index, Index + 1000));
}
Serial.println("Waiting for response.");
long int StartTime = millis();
while (!clienthttps.available()) {
Serial.print(".");
delay(100);
if ((StartTime + waitingTime) < millis()) {
Serial.println();
Serial.println("No response.");
//If you have no response, maybe need a greater value of waitingTime
break;
}
}
Serial.println();
while (clienthttps.available()) {
Serial.print(char(clienthttps.read()));
}
} else {
Serial.println("Connected to " + String(myDomain) + " failed.");
}
clienthttps.stop();
Note that in the latest "github.com/espressif/arduino-esp32" (as at 17Jan23) the command
"WiFiClientSecure.println();" (your "clienthttps.println()") seems to corrupt the SSL-buffer somehow, and although the command returns after a bit, the SSL-connection fails at that point (this didn't happen in earlier versions of the software).
The solution which worked for me was to ensure I at very least sent "WiFiClientSecure.println(" ");" (ie. " \r\n") as this seems to work.
(This solution appears also often necessary if sending data over a GPRS connection because in that instance a ".println()" seems to be ingored!)
Im using a Teensy 3.2 microcontroller paired with an ESP8266. Right now im just trying to serve a simple HTML web page that is updated with ajax. I can connect to the ESP and serve a page but im having trouble updating the page with XML data. The problem is somewhere in the loop function. Im not sure how to get the ESP to properly send XML data, or maybe im missing a critical function. Help greatly appreciated!
#define LED1 11
#define LED2 12
#define BUFFER_SIZE 4096
#define SSID "xxxx" // change this to match your WiFi SSID
#define PASS "xxxx" // change this to match your WiFi password
#define PORT "8080" // Port 8080 is default webserver port
char buffer[BUFFER_SIZE];
int n = 0;
String webSite, javaScript, XML, header, content;
void buildWebsite() {
header = "HTTP/1.1 200 OK\r\n";
header += "Content-Type: text/html\r\n";
header += "Connection: close\r\n";
//header += "Refresh: 5\r\n";
buildJavascript();
content = "<!DOCTYPE HTML>\n";
content += javaScript;
content += "<BODY onload='process()'>\n";
content += "<BR>This is the ESP website.<BR>\n";
content += "Runtime = <A id='runtime'></A>\n";
content += "</BODY>\n";
content += "</HTML>\n";
header += "Content-Length:";
header += (int)(content.length());
header += "\r\n\r\n";
webSite = header + content;
}
void buildJavascript() {
javaScript = "<SCRIPT>\n";
javaScript += "var xmlHttp = createXmlHttpObject();\n";
javaScript += "function createXmlHttpObject() {\n";
javaScript += " if(window.XMLHttpRequest) {\n";
javaScript += " xmlHttp = new XMLHttpRequest();\n";
javaScript += " } else {\n";
javaScript += " xmlHttp = new ActiveXObject('Microsoft.XMLHTTP');\n";
javaScript += " }\n";
javaScript += " return xmlHttp;\n";
javaScript += "}\n";
javaScript += "function process(){\n";
javaScript += " if(xmlHttp.readyState == 0 || xmlHttp.readyState == 4){\n";
javaScript += " xmlHttp.open('GET','xml',true);\n";
javaScript += " xmlHttp.onreadystatechange = handleServerResponse();\n"; // no brackets?????
javaScript += " xmlHttp.send();\n";
javaScript += " }\n";
javaScript += " setTimeout('process()',1000);\n";
javaScript += "}\n";
javaScript += "function handleServerResponse(){\n";
javaScript += " if(xmlHttp.readyState == 4 && xmlHttp.status == 200){\n";
javaScript += " xmlResponse = xmlHttp.responseXML;\n";
javaScript += " xmldoc = xmlResponse.getElementsByTagName('response');\n";
javaScript += " message = xmldoc[0].firstChild.nodeValue;\n";
javaScript += " document.getElementById('runtime').innerHTML = message;\n";
javaScript += " }\n";
javaScript += "}\n";
javaScript += "</SCRIPT>\n";
}
void buildXML() {
XML = "<?xml version='1.0' encoding='UTF-8'?>\n";
XML += "<response>\n";
XML += millis2time();
XML += "</response>\n";
}
String millis2time() {
String Time = "";
unsigned long ss;
byte mm, hh;
ss = millis() / 1000;
hh = ss / 3600;
mm = (ss - hh * 3600) / 60;
ss = (ss - hh * 3600) - mm * 60;
if (hh < 10)Time += "0";
Time += (String)hh + ":";
if (mm < 10)Time += "0";
Time += (String)mm + ":";
if (ss < 10)Time += "0";
Time += (String)ss;
return Time;
}
/*******************************************************************
* PROGRAM SETUP
********************************************************************/
void setup() {
delay(1000);
Serial1.begin(115200); // Teensy to ESP8266
Serial.begin(115200); // Teensy to USB Serial
Serial.println("Begin program.");
pinMode(LED1, OUTPUT);
pinMode(LED2, OUTPUT);
// Initialize ESP8266.
setupWiFi();
}
/*******************************************************************
* DEVICE FUNCTIONS
********************************************************************/
// Read line responses from ESP8266.
bool read_till_eol() {
static int i = 0;
if (Serial1.available()) {
buffer[i++] = Serial1.read();
if (i == BUFFER_SIZE) i = 0;
if (i > 1 && buffer[i - 2] == 13 && buffer[i - 1] == 10) {
buffer[i] = 0;
i = 0;
Serial.print(buffer);
return true;
}
}
return false;
}
// Listen for ESP8266 response. By default we are looking for OK\r\n
char OK[] = "OK\r\n";
byte wait_for_esp_response(int timeout, char* term = OK) {
unsigned long t = millis();
bool found = false;
int i = 0;
int len = strlen(term); // compute length of (string)
// wait for at most timeout milliseconds, or if OK\r\n is found
while (millis() < t + timeout) {
if (Serial1.available()) {
digitalWrite(LED2, HIGH);
buffer[i++] = Serial1.read();
if (i >= len) {
if (strncmp(buffer + i - len, term, len) == 0) {
found = true;
break;
}
}
digitalWrite(LED2, LOW);
}
}
buffer[i] = 0;
Serial.print(buffer);
return found;
}
/*******************************************************************
* LOOP
********************************************************************/
void loop() {
int ch_id, packet_len;
char *pb;
// Look for received IDP (unsolicited data packet) from browser refresh.
if (read_till_eol()) {
if (strncmp(buffer, "+IPD,", 5) == 0) // If strings match...
{
// Request: (+IPD, connection channel, data length)
sscanf(buffer + 5, "%d,%d", &ch_id, &packet_len);
if (packet_len > 0) {
// Read serial until packet_len character received
// start from :
pb = buffer + 5;
while (*pb != ':') pb++;
pb++;
if (strncmp(pb, "GET / HTTP", 10) == 0)
{
// Send HTML data.
wait_for_esp_response(2000);
Serial.println("Serving HTML ->");
buildWebsite();
serve(webSite, ch_id);
}
else if (strncmp(pb, "GET /xml", 8) == 0)
{
// Send XML data.
wait_for_esp_response(2000);
Serial.println("Serving XML ->");
buildXML();
serve(XML, ch_id);
Serial.println(millis2time());
}
}
}
}
}
/*******************************************************************
* SEND DATA
********************************************************************/
// Send the data to the ESP8266.
void serve(String data, int ch_id)
{
Serial1.print("AT+CIPSEND=");
Serial1.print(ch_id);
Serial1.print(",");
Serial1.println(data.length());
if (wait_for_esp_response(1000)) {
Serial1.print(data);
}
else {
Serial1.print("AT+CIPCLOSE=");
Serial1.println(ch_id);
}
}
/*******************************************************************
* SETUP WIFI
********************************************************************/
void setupWiFi() {
// Turn on echo.
Serial1.println("ATE1");
wait_for_esp_response(1000);
// Set mode 3 (client + AP).
Serial1.println("AT+CWMODE=3");
wait_for_esp_response(1000);
// Reset WiFi module.
Serial1.print("AT+RST\r\n");
wait_for_esp_response(1500);
// Join AP.
Serial1.print("AT+CWJAP=\"");
Serial1.print(SSID);
Serial1.print("\",\"");
Serial1.print(PASS);
Serial1.println("\"");
wait_for_esp_response(5000);
// Start server.
Serial1.println("AT+CIPMUX=1");
wait_for_esp_response(1000);
// Create TCP Server.
Serial1.print("AT+CIPSERVER=1,");
Serial1.println(PORT);
wait_for_esp_response(1000);
// Set the automatic socket client disconnection timeout from 1 to 28800 seconds.
Serial1.println("AT+CIPSTO=6000");
wait_for_esp_response(1000);
Serial1.println("AT+GMR");
wait_for_esp_response(1000);
Serial1.println("AT+CWJAP?");
wait_for_esp_response(1000);
Serial1.println("AT+CIPSTA?");
wait_for_esp_response(1000);
Serial1.println("AT+CWMODE?");
wait_for_esp_response(1000);
Serial1.println("AT+CIFSR");
wait_for_esp_response(5000);
Serial1.println("AT+CWLAP");
wait_for_esp_response(5000);
Serial1.println("AT+CIPSTATUS");
wait_for_esp_response(5000);
Serial.println("------------------------------------");
}
One of the problems is here:
xmlHttp.onreadystatechange = handleServerResponse();
You are not binding your handler to the readystatechange event. You are just setting what handleServerResponse() returns to xmlHttp.onreadystatechange.
It should be:
xmlHttp.onreadystatechange = handleServerResponse;
The second problem is that you aren't sending your XML as a proper HTTP response.
You should send it with HTTP headers like you are sending your HTML. And your XML response should set the Content-Type header as text/xml.
There are other possible problems/improvements like:
You are putting your <script> tag directly into the <html> tag. It should be under the <head> tag.
You are using Refresh: 5 header, but you are already refreshing your page with AJAX.
You don't need so send XML, if you just want a single value. You could just send what millis2time() returns and use document.getElementById('runtime').innerHTML = xmlHttp.responseText; in your JavaScript.
Your HTML code is static and doesn't change between requests. It doesn't make sense to build the whole request on every HTTP request.
You could just put the whole request string into a const variable and then serve that.