Files
WLED_MM_Infinity/wled00/file.cpp
Frank 6247ee066d fix presets.json corruption, part 1: avoid re-assigning open file pointer
* this prevents "open after open", that could lead to file corruption when "f" was still holding some changes in its buffer (written out with f.close())
2025-12-29 20:08:34 +01:00

564 lines
20 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#include "wled.h"
/*
* Utility for SPIFFS filesystem
*/
#ifdef ARDUINO_ARCH_ESP32 //FS info bare IDF function until FS wrapper is available for ESP32
#if WLED_FS != LITTLEFS && ESP_IDF_VERSION_MAJOR < 4
#include "esp_spiffs.h"
#endif
//#define yield() {delay(0);} // WLEDMM yield() is completely unnecessary on esp32, but delay(0) can reduce task contention
#endif
//WLEDMM seems that 256 is indeed the optimal buffer length
#define FS_BUFSIZE 256
/*
* Structural requirements for files managed by writeObjectToFile() and readObjectFromFile() utilities:
* 1. File must be a string representation of a valid JSON object
* 2. File must have '{' as first character
* 3. There must not be any additional characters between a root-level key and its value object (e.g. space, tab, newline)
* 4. There must not be any characters between a root object-separating ',' and the next object key string
* 5. There may be any number of spaces, tabs, and/or newlines before such object-separating ','
* 6. There must not be more than 5 consecutive spaces at any point except for those permitted in condition 5
* 7. If it is desired to delete the first usable object (e.g. preset file), a dummy object '"0":{}' is inserted at the beginning.
* It shall be disregarded by receiving software.
* The reason for it is that deleting the first preset would require special code to handle commas between it and the 2nd preset
*/
// There are no consecutive spaces longer than this in the file, so if more space is required, findSpace() can return false immediately
// Actual space may be lower
constexpr size_t MAX_SPACE = UINT16_MAX * 2U; // smallest supported config has 128Kb flash size
static volatile size_t knownLargestSpace = MAX_SPACE;
static File f; // don't export to other cpp files
//wrapper to find out how long closing takes
void closeFile() {
#ifdef ARDUINO_ARCH_ESP32
// WLEDMM: file.close() triggers flash writing. While flash is writing, the NPB RMT driver cannot fill its buffer which may create glitches.
// WLEDMM more precisely (thanks to a web research done by AI):
// the RMT peripheral itself doesnt stall, but the refill path often does. In Arduino-ESP32/WLED
// typical builds, close() that commits flash writes frequently causes enough blocking that the LED pipeline under-runs, resulting in visible glitches.
// So the assumption is practically correct for this project context.
// --> with neopixelBus 2.7.5, the practical ISR stall budget is about 0.080.12 ms — far less than LittleFS flash commit times.
// typical flash write "commit" times are between 0.5ms and 10ms, but they can be a few 100ms in worst case
// --> file reads rarely cause refill stalls compared to writes, but large/fragmented reads can still exceed the ~0.080.12 ms budget.
// esp32 recommendations: use f.setBufferSize() (5121024 for reads is reasonable); use delay(0) after file reads, to reduce task contention
if (!f) {doCloseFile = false; return;} // WLEDMM only do all this hick-hack when f is an open file
unsigned long t_wait = millis();
bool oldLock = suspendStripService;
if (strip.isUpdating()) suspendStripService = true; // WLEDMM schedule short pause to prevent LEDs glitching during flash write
while(strip.isUpdating() && (millis() - t_wait < 72)) delay(1); // WLEDMM try to catch a moment when strip is idle
while(strip.isUpdating() && (millis() - t_wait < 96)) delay(0); // try harder
//if (strip.isUpdating()) USER_PRINTLN("closeFile: strip still updating.");
delay(2); // might help
#else
bool oldLock = suspendStripService; // fix build f***u* on 8266
#endif
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINT(F("Close -> "));
uint32_t s = millis();
#endif
if ((suspendStripService == false) && (oldLock == true)) oldLock = false; // update in case of parallel lock release by another task
f.close();
#ifdef ARDUINO_ARCH_ESP32
delay(1); // might help
#endif
suspendStripService = oldLock; // restore previous lock
DEBUGFS_PRINTF("took %d ms\n", millis() - s);
doCloseFile = false;
}
//find() that reads and buffers data from file stream in 256-byte blocks.
//Significantly faster, f.find(key) can take SECONDS for multi-kB files
static bool bufferedFind(const char *target, bool fromStart = true) {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINT("Find ");
DEBUGFS_PRINTLN(target);
uint32_t s = millis();
#endif
if (!f || !f.size()) return false; // fast return when current file closed, or file size is zero
size_t targetLen = strlen(target);
size_t index = 0;
byte buf[FS_BUFSIZE];
if (fromStart) f.seek(0);
while (f.position() < f.size() -1) {
size_t bufsize = f.read(buf, FS_BUFSIZE); // better to use size_t instead if uint16_t
size_t count = 0;
while (count < bufsize) {
if(buf[count] != target[index])
index = 0; // reset index if any char does not match
if(buf[count] == target[index]) {
if(++index >= targetLen) { // return true if all chars in the target match
f.seek((f.position() - bufsize) + count +1);
DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
return true;
}
}
count++;
}
}
DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
return false;
}
//find empty spots in file stream in 256-byte blocks.
static bool bufferedFindSpace(size_t targetLen, bool fromStart = true) {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTF("Find %d spaces\n", targetLen);
uint32_t s = millis();
#endif
if (knownLargestSpace < targetLen) {
DEBUGFS_PRINT(F("No match, KLS "));
DEBUGFS_PRINTLN(knownLargestSpace);
return false;
}
if (!f || !f.size()) return false;
size_t index = 0; // better to use size_t instead if uint16_t
byte buf[FS_BUFSIZE];
if (fromStart) f.seek(0);
while (f.position() < f.size() -1) {
size_t bufsize = f.read(buf, FS_BUFSIZE);
size_t count = 0;
while (count < bufsize) {
if(buf[count] == ' ') {
if(++index >= targetLen) { // return true if space long enough
if (fromStart) {
f.seek((f.position() - bufsize) + count +1 - targetLen);
knownLargestSpace = MAX_SPACE; //there may be larger spaces after, so we don't know
}
DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
return true;
}
} else {
if (!fromStart) return false;
if (index) {
if (knownLargestSpace < index || (knownLargestSpace == MAX_SPACE)) knownLargestSpace = index;
index = 0; // reset index if not space
}
}
count++;
}
}
DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
return false;
}
//find the closing bracket corresponding to the opening bracket at the file pos when calling this function
static bool bufferedFindObjectEnd() {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTLN(F("Find obj end"));
uint32_t s = millis();
#endif
if (!f || !f.size()) return false;
uint16_t objDepth = 0; //num of '{' minus num of '}'. return once 0
//size_t start = f.position();
byte buf[FS_BUFSIZE];
while (f.position() < f.size() -1) {
size_t bufsize = f.read(buf, FS_BUFSIZE); // better to use size_t instead of uint16_t
size_t count = 0;
while (count < bufsize) {
if (buf[count] == '{') objDepth++;
if (buf[count] == '}') objDepth--;
if (objDepth == 0) {
f.seek((f.position() - bufsize) + count +1);
DEBUGFS_PRINTF("} at pos %d, took %d ms", f.position(), millis() - s);
return true;
}
count++;
}
}
DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
return false;
}
//fills n bytes from current file pos with ' ' characters
static void writeSpace(size_t l)
{
byte buf[FS_BUFSIZE];
memset(buf, ' ', FS_BUFSIZE);
while (l > 0) {
size_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l;
f.write(buf, block);
l -= block;
}
if (knownLargestSpace < l) knownLargestSpace = l;
}
bool appendObjectToFile(const char* key, JsonDocument* content, uint32_t s, uint32_t contentLen = 0)
{
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTLN(F("Append"));
uint32_t s1 = millis();
#endif
uint32_t pos = 0;
if (!f) return false;
if (f.size() < 3) {
char init[12];
strcpy_P(init, PSTR("{\"0\":{}}"));
f.print(init);
}
if (content->isNull()) {
doCloseFile = true;
return true; //nothing to append
}
//if there is enough empty space in file, insert there instead of appending
if (!contentLen) contentLen = measureJson(*content);
DEBUGFS_PRINTF("CLen %d\n", contentLen);
if (bufferedFindSpace(contentLen + strlen(key) + 1)) {
if (f.position() > 2) f.write(','); //add comma if not first object
f.print(key);
serializeJson(*content, f);
DEBUGFS_PRINTF("Inserted, took %d ms (total %d)", millis() - s1, millis() - s);
doCloseFile = true;
return true;
}
//not enough space, append at end
//permitted space for presets exceeded
updateFSInfo();
if (f.size() + 9000 > (fsBytesTotal - fsBytesUsed)) { //make sure there is enough space to at least copy the file once
errorFlag = ERR_FS_QUOTA;
doCloseFile = true;
return false;
}
//check if last character in file is '}' (typical)
uint32_t eof = f.size() -1;
f.seek(eof, SeekSet);
if (f.read() == '}') pos = eof;
if (pos == 0) //not found
{
DEBUGFS_PRINTLN("not }");
f.seek(0);
while (bufferedFind("}",false)) //find last closing bracket in JSON if not last char
{
pos = f.position();
}
if (pos > 0) pos--;
}
DEBUGFS_PRINT("pos "); DEBUGFS_PRINTLN(pos);
if (pos > 2)
{
f.seek(pos, SeekSet);
f.write(',');
} else { //file content is not valid JSON object
f.seek(0, SeekSet);
f.print('{'); //start JSON
}
f.print(key);
//Append object
serializeJson(*content, f);
f.write('}');
doCloseFile = true;
DEBUGFS_PRINTF("Appended, took %d ms (total %d)", millis() - s1, millis() - s);
return true;
}
bool writeObjectToFileUsingId(const char* file, uint16_t id, JsonDocument* content)
{
char objKey[12];
sprintf(objKey, "\"%d\":", id);
return writeObjectToFile(file, objKey, content);
}
bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
{
uint32_t s = 0; //timing
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTF("Write to %s with key %s >>>\n", file, (key==nullptr)?"nullptr":key);
serializeJson(*content, Serial); DEBUGFS_PRINTLN();
s = millis();
#endif
if (doCloseFile) {
if (f) { DEBUG_PRINTLN("writeObjectToFile("+String(file)+"): file f is already open, closing to prevent file corruption."); }
closeFile(); // WLEDMM: Ensure previous file is closed
}
size_t pos = 0;
f = WLED_FS.open(file, "r+");
if (!f && !WLED_FS.exists(file)) { f = WLED_FS.open(file, "w+");
if(f) { DEBUG_PRINTF(PSTR("FILE '%s' open to write, size =%d\n"), file, (int)f.size());} // WLEDMM additional debug message
}
if (!f) {
DEBUGFS_PRINTLN(F("Failed to open!"));
return false;
}
#if ESP_IDF_VERSION_MAJOR >= 4
f.setBufferSize(FS_BUFSIZE); // reduced internal buffer leads to shorter blocking delay, and might prevent LED glitches
#endif
if (!bufferedFind(key)) //key does not exist in file
{
return appendObjectToFile(key, content, s);
}
//an object with this key already exists, replace or delete it
pos = f.position();
//measure out end of old object
bufferedFindObjectEnd();
size_t pos2 = f.position();
uint32_t oldLen = pos2 - pos;
DEBUGFS_PRINTF("Old obj len %d\n", oldLen);
//Three cases:
//1. The new content is null, overwrite old obj with spaces
//2. The new content is smaller than the old, overwrite and fill diff with spaces
//3. The new content is larger than the old, but smaller than old + trailing spaces, overwrite with new
//4. The new content is larger than old + trailing spaces, delete old and append
size_t contentLen = 0;
if (!content->isNull()) contentLen = measureJson(*content);
if (contentLen && contentLen <= oldLen) { //replace and fill diff with spaces
DEBUGFS_PRINTLN(F("replace"));
f.seek(pos);
serializeJson(*content, f);
writeSpace(pos2 - f.position());
} else if (contentLen && bufferedFindSpace(contentLen - oldLen, false)) { //enough leading spaces to replace
DEBUGFS_PRINTLN(F("replace (trailing)"));
f.seek(pos);
serializeJson(*content, f);
} else {
DEBUGFS_PRINTLN(F("delete"));
pos -= strlen(key);
if (pos > 3) pos--; //also delete leading comma if not first object
f.seek(pos);
writeSpace(pos2 - pos);
if (contentLen) return appendObjectToFile(key, content, s, contentLen);
}
doCloseFile = true;
DEBUGFS_PRINTF("Replaced/deleted, took %d ms\n", millis() - s);
return true;
}
bool readObjectFromFileUsingId(const char* file, uint16_t id, JsonDocument* dest)
{
char objKey[12];
sprintf(objKey, "\"%d\":", id);
return readObjectFromFile(file, objKey, dest);
}
//if the key is a nullptr, deserialize entire object
//WLEDMM: if key is not a nullptr, nothing seems to be done with it!!! (except check for existing), still whole json is loaded
bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
{
if (doCloseFile) closeFile();
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTF("Read from %s with key %s >>>\n", file, (key==nullptr)?"nullptr":key);
uint32_t s = millis();
#endif
f = WLED_FS.open(file, "r");
if (!f) return false;
else { DEBUG_PRINTF(PSTR("FILE '%s' open to read, size %d bytes\n"), file, (int)f.size());} // WLEDMM additional debug message
#if ESP_IDF_VERSION_MAJOR >= 4
f.setBufferSize(FS_BUFSIZE*2); // reduced internal buffer leads to shorter blocking delay, and might prevent LED glitches
#endif
if (key != nullptr && !bufferedFind(key)) //key does not exist in file
{
f.close();
dest->clear();
DEBUGFS_PRINTLN(F("Obj not found."));
return false;
}
deserializeJson(*dest, f);
f.close();
DEBUGFS_PRINTF("Read, took %d ms\n", millis() - s);
return true;
}
void updateFSInfo() {
#ifdef ARDUINO_ARCH_ESP32
#if WLED_FS == LITTLEFS || ESP_IDF_VERSION_MAJOR >= 4
fsBytesTotal = WLED_FS.totalBytes();
fsBytesUsed = WLED_FS.usedBytes();
#else
esp_spiffs_info(nullptr, &fsBytesTotal, &fsBytesUsed);
#endif
#else
FSInfo fsi;
WLED_FS.info(fsi);
fsBytesUsed = fsi.usedBytes;
fsBytesTotal = fsi.totalBytes;
#endif
}
//Un-comment any file types you need
static String getContentType(AsyncWebServerRequest* request, String filename){
if(request->hasArg("download")) return "application/octet-stream";
else if(filename.endsWith(".htm")) return "text/html";
else if(filename.endsWith(".html")) return "text/html";
else if(filename.endsWith(".css")) return "text/css";
else if(filename.endsWith(".js")) return "application/javascript";
else if(filename.endsWith(".json")) return "application/json";
else if(filename.endsWith(".png")) return "image/png";
else if(filename.endsWith(".gif")) return "image/gif";
else if(filename.endsWith(".jpg")) return "image/jpeg";
else if(filename.endsWith(".ico")) return "image/x-icon";
// else if(filename.endsWith(".xml")) return "text/xml";
// else if(filename.endsWith(".pdf")) return "application/x-pdf";
// else if(filename.endsWith(".zip")) return "application/x-zip";
// else if(filename.endsWith(".gz")) return "application/x-gzip";
return "text/plain";
}
#if defined(BOARD_HAS_PSRAM) && (defined(WLED_USE_PSRAM) || defined(WLED_USE_PSRAM_JSON))
// caching presets in PSRAM may prevent occasional flashes seen when HomeAssistant polls WLED
// original idea by @akaricchi (https://github.com/Akaricchi)
// returns a pointer to the PSRAM buffer, updates size parameter
static const uint8_t *getPresetCache(size_t &size) {
static unsigned long presetsCachedTime = 0;
static uint8_t *presetsCached = nullptr;
static size_t presetsCachedSize = 0;
static byte presetsCachedValidate = 0;
if (!psramFound()) {
size = 0;
return nullptr;
}
//if (presetsModifiedTime != presetsCachedTime) DEBUG_PRINTLN(F("getPresetCache(): presetsModifiedTime changed."));
//if (presetsCachedValidate != cacheInvalidate) DEBUG_PRINTLN(F("getPresetCache(): cacheInvalidate changed."));
if ((presetsModifiedTime != presetsCachedTime) || (presetsCachedValidate != cacheInvalidate)) {
if (presetsCached) {
free(presetsCached);
presetsCached = nullptr;
}
}
if (!presetsCached) {
File file = WLED_FS.open("/presets.json", "r");
#if ESP_IDF_VERSION_MAJOR >= 4
if (file) file.setBufferSize(FS_BUFSIZE*2); // reduced internal buffer leads to shorter blocking delay, and might prevent LED glitches
#endif
if (file) {
presetsCachedTime = presetsModifiedTime;
presetsCachedValidate = cacheInvalidate;
presetsCachedSize = 0;
presetsCached = (uint8_t*)ps_malloc(file.size() + 1);
if (presetsCached) {
presetsCachedSize = file.size();
file.read(presetsCached, presetsCachedSize);
presetsCached[presetsCachedSize] = 0;
file.close();
//USER_PRINTLN(F("getPresetCache(): /presets.json cached in PSRAM."));
}
}
} else {
//USER_PRINTLN(F("getPresetCache(): /presets.json served from PSRAM."));
}
size = presetsCachedSize;
return presetsCached;
}
#endif
// WLEDMM
static volatile bool haveLedmapFile = true;
static volatile bool haveIndexFile = true;
static volatile bool haveSkinFile = true;
static volatile bool haveICOFile = true;
static volatile bool haveCpalFile = true;
void invalidateFileNameCache() { // reset "file not found" cache
haveLedmapFile = true;
haveIndexFile = true;
haveSkinFile = true;
haveICOFile = true;
haveCpalFile = true;
#if defined(BOARD_HAS_PSRAM) && (defined(WLED_USE_PSRAM) || defined(WLED_USE_PSRAM_JSON))
// WLEDMM hack to clear presets.json cache
size_t dummy;
unsigned long realpresetsTime = presetsModifiedTime;
presetsModifiedTime = toki.second(); // pretend we have changes
(void) getPresetCache(dummy); // clear presets.json cache
presetsModifiedTime = realpresetsTime; // restore correct value
#endif
//USER_PRINTLN("WS FileRead cache cleared");
}
bool handleFileRead(AsyncWebServerRequest* request, String path){
DEBUG_PRINTLN("WS FileRead: " + path);
if(path.endsWith("/")) path += "index.htm";
if(path.indexOf("sec") > -1) return false;
// WLEDMM shortcuts
if ((haveLedmapFile == false) && path.equals("/ledmap.json")) return false;
if ((haveIndexFile == false) && path.equals("/index.htm")) return false;
if ((haveSkinFile == false) && path.equals("/skin.css")) return false;
if ((haveICOFile == false) && path.equals("/favicon.ico")) return false;
if ((haveCpalFile == false) && path.equals("/cpal.htm")) return false;
// WLEDMM toDO: add file caching (PSRAM) for /presets.json an /cfg.json
String contentType = getContentType(request, path);
/*String pathWithGz = path + ".gz";
if(WLED_FS.exists(pathWithGz)){
request->send(WLED_FS, pathWithGz, contentType);
return true;
}*/
#if defined(BOARD_HAS_PSRAM) && (defined(WLED_USE_PSRAM) || defined(WLED_USE_PSRAM_JSON))
if (path.endsWith("/presets.json")) {
size_t psize;
const uint8_t *presets = getPresetCache(psize);
if (presets) {
AsyncWebServerResponse *response = request->beginResponse_P(200, contentType, presets, psize);
request->send(response);
return true;
}
}
#endif
if(WLED_FS.exists(path) || WLED_FS.exists(path + ".gz")) {
request->send(WLED_FS, path, String(), request->hasArg(F("download")));
return true;
}
//USER_PRINTLN("WS FileRead failed: " + path + " (" + contentType + ")");
// WLEDMM cache "file not found" results (reduces LED flickering during UI activities)
if (path.equals("/ledmap.json")) haveLedmapFile = false;
if (path.equals("/index.htm")) haveIndexFile = false;
if (path.equals("/skin.css")) haveSkinFile = false;
if (path.equals("/favicon.ico")) haveICOFile = false;
if (path.equals("/cpal.htm")) haveCpalFile = false;
return false;
}