update I have updated the code(second code segment) so that the null character is inserted into the array which solves the problem of writing past the end of the array. in order to stop the program from crashing i can only put the null character at the end of the array. this means that i need to increase the size of the array to accommodate for it. infuriatingly as soon as i do anything with the size variable, ie. size = size+1 the program takes a great deal longer to execute. i have no idea whats going on here so any help would be much appreciated. i have simplified the code further and the actual text file i am using on the server is included if anyone wanted to try it out. many thanks.
original post i have written a sketch which uses the HTTPClient library to download 3 files from a server and write them to 3 SPIFFS files on an ESP32 using arduino IDE. It works fine but I am trying to improve the speed. The following code is the code which works:
#include <Arduino.h>
#include <WiFi.h>
#include <WiFiMulti.h>
#include <HTTPClient.h>
#define Serial Serial
WiFiMulti wifiMulti;
#include <SPIFFS.h>
#include "SPIFFS.h"
#define FORMAT_SPIFFS_IF_FAILED true
String url = "http://server.com";
String fileName[] = {"/file1", "/file2", "/file3"};
String fileSuf = ".dat";
String imageSuf = ".gif";
int fileIndex = 0;
int numFiles = sizeof(fileName) / sizeof(fileName[0]);
String filePath = url + fileName[fileIndex] + fileSuf;
String imagePath = fileName[fileIndex] + imageSuf;
void setup() {
Serial.begin(115200);
if (!SPIFFS.begin(FORMAT_SPIFFS_IF_FAILED)) {
Serial.println("SPIFFS Mount Failed");
return;
}
SPIFFS.format();
///connect to the internet
wifiMulti.addAP("xxx", "xxx"); //work network
// download files
while(fileIndex < numFiles){
getFiles(); //retreive gif image
fileIndex++;
}
fileIndex = 0;
}
void loop() {
}
long timer;
int speed;
void getFiles() {
// wait for WiFi connection
if ((wifiMulti.run() == WL_CONNECTED)) {
//generate paths
filePath = url + fileName[fileIndex] + fileSuf;
imagePath = fileName[fileIndex] + imageSuf;
Serial.println(filePath);
Serial.println(imagePath);
HTTPClient http;
Serial.print("[HTTP] begin...\n");
// configure server and url
http.begin(filePath);
timer = millis(); //set timer
Serial.print("[HTTP] GET...\n");
// start connection and send HTTP header
int httpCode = http.GET();
if (httpCode > 0) {
// HTTP header has been send and Server response header has been handled
Serial.printf("[HTTP] GET... code: %d\n", httpCode);
// file found at server
if (httpCode == HTTP_CODE_OK) {
// get lenght of document (is -1 when Server sends no Content-Length header)
int len = http.getSize();
// create buffer for read
byte buff[2048] = { 0 };
// get tcp stream
WiFiClient * stream = http.getStreamPtr();
// read all data from server
while (http.connected() && (len > 0 || len == -1)) {
// get available data size
size_t size = stream->available();
if (size) { ///
// read up to 128 byte
int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size)); //calculate size of buffer(is it full?)
appendFile(SPIFFS, imagePath, buff, c); //addi buffer to file in flash
if (len > 0) {
len -= c;
}
}
}
Serial.println();
Serial.print("[HTTP] connection closed or file end.\n");
}
} else {
Serial.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
}
speed = millis() - timer;
Serial.println(speed); //print time taken in millis;
http.end();
}
}
void appendFile(fs::FS &fs, String _path, byte message[], int arr_size) {
File file = fs.open(_path, FILE_APPEND);
file.write(message,arr_size);
}
So thats all well and good but i experimented with how to speed it up.The following code finishes the task of downloading and writing to flash in a third of the time but because i pass the array to the print method without telling it where the buffer ends it prints 'junk' bytes on the end making the file corrupted.
My question is, why is my working method so much slower than the other method. is there any way i can get the same speed as method 2 but without printing the unwanted bytes each time? it seems like i should be able to specify this without sacrificing so much speed.
as an example of the speed file1 took 9102 ms to complete in the first approach and 3303 ms in the second. a big difference! any advice would be much appreciated, thanks! Heres the second approach' code: (The only changes are changing the buffer array from byte to char and using file.print(message) in the appendFile function instead of file.write(message,array_size);
#include <Arduino.h>
#include <WiFi.h>
#include <WiFiMulti.h>
#include <HTTPClient.h>
#define Serial Serial
WiFiMulti wifiMulti;
#include <SPIFFS.h>
#include "SPIFFS.h"
#define FORMAT_SPIFFS_IF_FAILED true
String path = "/test.txt";
String filePath = "http://gonadgranny.atwebpages.com/text.txt";
void setup() {
Serial.begin(115200);
if (!SPIFFS.begin(FORMAT_SPIFFS_IF_FAILED)) {
Serial.println("SPIFFS Mount Failed");
return;
}
SPIFFS.format();
///connect to the internet
wifiMulti.addAP("xxx", "xxx"); //work network
getFiles(); //retreive gif image
}
void loop() {
readFile(SPIFFS, path);
delay(5000);
}
long timer;
int speed;
void getFiles() {
// wait for WiFi connection
if ((wifiMulti.run() == WL_CONNECTED)) {
//generate paths
Serial.println(filePath);
Serial.println(path);
HTTPClient http;
Serial.print("[HTTP] begin...\n");
// configure server and url
http.begin(filePath);
timer = millis(); //set timer
Serial.print("[HTTP] GET...\n");
// start connection and send HTTP header
int httpCode = http.GET();
if (httpCode > 0) {
// HTTP header has been send and Server response header has been handled
Serial.printf("[HTTP] GET... code: %d\n", httpCode);
// file found at server
if (httpCode == HTTP_CODE_OK) {
// get lenght of document (is -1 when Server sends no Content-Length header)
int len = http.getSize();
// create buffer for read
char buff[2048] = { 0 };
// get tcp stream
WiFiClient * stream = http.getStreamPtr();
// read all data from server
while (http.connected() && (len > 0 || len == -1)) {
// get available data size
size_t size = (stream->available()) ; //make one bigger
if (size) { ///
// read up to 128 byte
int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size)); //calculate size of buffer(is it full?)
appendFile(SPIFFS, path, buff, c); //addi buffer to file in flash
if (len > 0) {
len -= c;
}
}
}
Serial.println();
Serial.print("[HTTP] connection closed or file end.\n");
}
} else {
Serial.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
}
speed = millis() - timer;
Serial.println(speed); //print time taken in millis;
http.end();
}
}
void appendFile(fs::FS &fs, String _path, char message[], int arr_size) {
message[(arr_size-1)] = '\0'; //put in end of array
File file = fs.open(_path, FILE_APPEND);
file.print(message);
}
void readFile(fs::FS &fs, String path) {
Serial.printf("Reading file: %s\r\n", path);
File file = SPIFFS.open(path);
if (!file || file.isDirectory()) {
// Serial.println("- failed to open file for reading");
return;
}
// Serial.println("- read from file:");
while (file.available()) {
Serial.write(file.read());
}
}
