WLED_MM_Infinity/wled00/file.cpp

#include "wled.h"

/*
 * Utility for SPIFFS filesystem
 */

#ifdef ARDUINO_ARCH_ESP32 //FS info bare IDF function until FS wrapper is available for ESP32
#if WLED_FS != LITTLEFS && ESP_IDF_VERSION_MAJOR < 4
  #include "esp_spiffs.h"
#endif
//#define yield() {delay(0);}  // WLEDMM yield() is completely unnecessary on esp32, but delay(0) can reduce task contention
#endif

//WLEDMM seems that 256 is indeed the optimal buffer length
#define FS_BUFSIZE 256

/*
 * Structural requirements for files managed by writeObjectToFile() and readObjectFromFile() utilities:
 * 1. File must be a string representation of a valid JSON object
 * 2. File must have '{' as first character
 * 3. There must not be any additional characters between a root-level key and its value object (e.g. space, tab, newline)
 * 4. There must not be any characters between a root object-separating ',' and the next object key string
 * 5. There may be any number of spaces, tabs, and/or newlines before such object-separating ','
 * 6. There must not be more than 5 consecutive spaces at any point except for those permitted in condition 5
 * 7. If it is desired to delete the first usable object (e.g. preset file), a dummy object '"0":{}' is inserted at the beginning.
 *    It shall be disregarded by receiving software.
 *    The reason for it is that deleting the first preset would require special code to handle commas between it and the 2nd preset
 */

// There are no consecutive spaces longer than this in the file, so if more space is required, findSpace() can return false immediately
// Actual space may be lower
constexpr size_t MAX_SPACE = UINT16_MAX * 2U;           // smallest supported config has 128Kb flash size
static volatile size_t knownLargestSpace = MAX_SPACE;

static File f; // don't export to other cpp files

//wrapper to find out how long closing takes
void closeFile() {
  #ifdef ARDUINO_ARCH_ESP32
  // WLEDMM: file.close() triggers flash writing. While flash is writing, the NPB RMT driver cannot fill its buffer which may create glitches.
  // WLEDMM more precisely (thanks to a web research done by AI):
  //        the RMT peripheral itself doesn’t stall, but the refill path often does. In Arduino-ESP32/WLED 
  //        typical builds, close() that commits flash writes frequently causes enough blocking that the LED pipeline under-runs, resulting in visible glitches. 
  //        So the assumption is practically correct for this project context.
  //    --> with neopixelBus 2.7.5, the practical ISR stall budget is about 0.08–0.12 ms — far less than LittleFS flash commit times.
  //        typical flash write "commit" times are between 0.5ms and 10ms, but they can be a few 100ms in worst case
  //    --> file reads rarely cause refill stalls compared to writes, but large/fragmented reads can still exceed the ~0.08–0.12 ms budget.
  //        esp32 recommendations: use f.setBufferSize() (512–1024 for reads is reasonable); use delay(0) after file reads, to reduce task contention

  if (!f) {doCloseFile = false; return;} // WLEDMM only do all this hick-hack when f is an open file

  bool oldLock = suspendStripService;
  #if defined(WLEDMM_FILEWAIT) // only wait if we don't have the flicker-free RMTHI driver
  unsigned long t_wait = millis();
  if (strip.isUpdating()) suspendStripService = true;             // WLEDMM schedule short pause to prevent LEDs glitching during flash write
  while(strip.isUpdating() && (millis() - t_wait < 72)) delay(1); // WLEDMM try to catch a moment when strip is idle
  while(strip.isUpdating() && (millis() - t_wait < 96)) delay(0); //        try harder
  //if (strip.isUpdating()) USER_PRINTLN("closeFile: strip still updating.");
  delay(2); // might help
  #endif
  #else
    bool oldLock = suspendStripService; // fix build f***u* on 8266
  #endif
  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINT(F("Close -> "));
    uint32_t s = millis();
  #endif
  if ((suspendStripService == false) && (oldLock == true)) oldLock = false; // update in case of parallel lock release by another task
  f.close();
  #ifdef ARDUINO_ARCH_ESP32
  delay(1); // might help
  #endif
  suspendStripService = oldLock; // restore previous lock
  DEBUGFS_PRINTF("took %d ms\n", millis() - s);
  doCloseFile = false;
}

//find() that reads and buffers data from file stream in 256-byte blocks.
//Significantly faster, f.find(key) can take SECONDS for multi-kB files
static bool bufferedFind(const char *target, bool fromStart = true) {
  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINT("Find ");
    DEBUGFS_PRINTLN(target);
    uint32_t s = millis();
  #endif

  if (!f || !f.size()) return false; // fast return when current file closed, or file size is zero
  size_t targetLen = strlen(target);

  size_t index = 0;
  byte buf[FS_BUFSIZE];
  if (fromStart) f.seek(0);

  while (f.position() < f.size() -1) {
    size_t bufsize = f.read(buf, FS_BUFSIZE); // better to use size_t instead if uint16_t
    size_t count = 0;
    while (count < bufsize) {
      if(buf[count] != target[index])
      index = 0; // reset index if any char does not match

      if(buf[count] == target[index]) {
        if(++index >= targetLen) { // return true if all chars in the target match
          f.seek((f.position() - bufsize) + count +1);
          DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
          return true;
        }
      }
      count++;
    }
  }
  DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
  return false;
}

//find empty spots in file stream in 256-byte blocks.
static bool bufferedFindSpace(size_t targetLen, bool fromStart = true) {

  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINTF("Find %d spaces\n", targetLen);
    uint32_t s = millis();
  #endif

  if (knownLargestSpace < targetLen) {
    DEBUGFS_PRINT(F("No match, KLS "));
    DEBUGFS_PRINTLN(knownLargestSpace);
    return false;
  }

  if (!f || !f.size()) return false;

  size_t index = 0; // better to use size_t instead if uint16_t
  byte buf[FS_BUFSIZE];
  if (fromStart) f.seek(0);

  while (f.position() < f.size() -1) {
    size_t bufsize = f.read(buf, FS_BUFSIZE);
    size_t count = 0;

    while (count < bufsize) {
      if(buf[count] == ' ') {
        if(++index >= targetLen) { // return true if space long enough
          if (fromStart) {
            f.seek((f.position() - bufsize) + count +1 - targetLen);
            knownLargestSpace = MAX_SPACE; //there may be larger spaces after, so we don't know
          }
          DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
          return true;
        }
      } else {
        if (!fromStart) return false;
        if (index) {
          if (knownLargestSpace < index || (knownLargestSpace == MAX_SPACE)) knownLargestSpace = index;
          index = 0; // reset index if not space
        }
      }

      count++;
    }
  }
  DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
  return false;
}

//find the closing bracket corresponding to the opening bracket at the file pos when calling this function
static bool bufferedFindObjectEnd() {
  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINTLN(F("Find obj end"));
    uint32_t s = millis();
  #endif

  if (!f || !f.size()) return false;

  uint16_t objDepth = 0; //num of '{' minus num of '}'. return once 0
  //size_t start = f.position();
  byte buf[FS_BUFSIZE];
  while (f.position() < f.size() -1) {
    size_t bufsize = f.read(buf, FS_BUFSIZE); // better to use size_t instead of uint16_t
    size_t count = 0;

    while (count < bufsize) {
      if (buf[count] == '{') objDepth++;
      if (buf[count] == '}') objDepth--;
      if (objDepth == 0) {
        f.seek((f.position() - bufsize) + count +1);
        DEBUGFS_PRINTF("} at pos %d, took %d ms", f.position(), millis() - s);
        return true;
      }
      count++;
    }
  }
  DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
  return false;
}

//fills n bytes from current file pos with ' ' characters
static void writeSpace(size_t l)
{
  byte buf[FS_BUFSIZE];
  memset(buf, ' ', FS_BUFSIZE);
  while (l > 0) {
    size_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l;
    f.write(buf, block);
    l -= block;
  }

  if (knownLargestSpace < l) knownLargestSpace = l;
}

bool appendObjectToFile(const char* key, JsonDocument* content, uint32_t s, uint32_t contentLen = 0)
{
  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINTLN(F("Append"));
    uint32_t s1 = millis();
  #endif
  uint32_t pos = 0;
  if (!f) return false;

  if (f.size() < 3) {
    char init[12];
    strcpy_P(init, PSTR("{\"0\":{}}"));
    f.print(init);
  }

  if (content->isNull()) {
    doCloseFile = true;
    return true; //nothing  to append
  }

  //if there is enough empty space in file, insert there instead of appending
  if (!contentLen) contentLen = measureJson(*content);
  DEBUGFS_PRINTF("CLen %d\n", contentLen);
  if (bufferedFindSpace(contentLen + strlen(key) + 1)) {
    if (f.position() > 2) f.write(','); //add comma if not first object
    f.print(key);
    serializeJson(*content, f);
    DEBUGFS_PRINTF("Inserted, took %d ms (total %d)", millis() - s1, millis() - s);
    doCloseFile = true;
    return true;
  }

  //not enough space, append at end

  //permitted space for presets exceeded
  updateFSInfo();

  if (f.size() + 9000 > (fsBytesTotal - fsBytesUsed)) { //make sure there is enough space to at least copy the file once
    errorFlag = ERR_FS_QUOTA;
    doCloseFile = true;
    return false;
  }

  //check if last character in file is '}' (typical)
  uint32_t eof = f.size() -1;
  f.seek(eof, SeekSet);
  if (f.read() == '}') pos = eof;

  if (pos == 0) //not found
  {
    DEBUGFS_PRINTLN("not }");
    f.seek(0);
    while (bufferedFind("}",false)) //find last closing bracket in JSON if not last char
    {
      pos = f.position();
    }
    if (pos > 0) pos--;
  }
  DEBUGFS_PRINT("pos "); DEBUGFS_PRINTLN(pos);
  if (pos > 2)
  {
    f.seek(pos, SeekSet);
    f.write(',');
  } else { //file content is not valid JSON object
    f.seek(0, SeekSet);
    f.print('{'); //start JSON
  }

  f.print(key);

  //Append object
  serializeJson(*content, f);
  f.write('}');

  doCloseFile = true;
  DEBUGFS_PRINTF("Appended, took %d ms (total %d)", millis() - s1, millis() - s);
  return true;
}

bool writeObjectToFileUsingId(const char* file, uint16_t id, JsonDocument* content)
{
  char objKey[12];
  sprintf(objKey, "\"%d\":", id);
  return writeObjectToFile(file, objKey, content);
}

bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
{
  uint32_t s = 0; //timing
  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINTF("Write to %s with key %s >>>\n", file, (key==nullptr)?"nullptr":key);
    serializeJson(*content, Serial); DEBUGFS_PRINTLN();
    s = millis();
  #endif

  if (doCloseFile) {
    if (f) { DEBUG_PRINTLN("writeObjectToFile("+String(file)+"): file f is already open, closing to prevent file corruption."); }
    closeFile();  // WLEDMM: Ensure previous file is closed
  }

  size_t pos = 0;
  f = WLED_FS.open(file, "r+");
  if (!f && !WLED_FS.exists(file)) { f = WLED_FS.open(file, "w+");
    if(f) { DEBUG_PRINTF(PSTR("FILE '%s' open to write, size =%d\n"), file, (int)f.size());} // WLEDMM additional debug message
  }
  if (!f) {
    DEBUGFS_PRINTLN(F("Failed to open!"));
    return false;
  }
  #if ESP_IDF_VERSION_MAJOR >= 4
  f.setBufferSize(FS_BUFSIZE);        // reduced internal buffer leads to shorter blocking delay, and might prevent LED glitches
  #endif

  if (!bufferedFind(key)) //key does not exist in file
  {
    return appendObjectToFile(key, content, s);
  }

  //an object with this key already exists, replace or delete it
  pos = f.position();
  //measure out end of old object
  bufferedFindObjectEnd();
  size_t pos2 = f.position();

  uint32_t oldLen = pos2 - pos;
  DEBUGFS_PRINTF("Old obj len %d\n", oldLen);

  //Three cases:
  //1. The new content is null, overwrite old obj with spaces
  //2. The new content is smaller than the old, overwrite and fill diff with spaces
  //3. The new content is larger than the old, but smaller than old + trailing spaces, overwrite with new
  //4. The new content is larger than old + trailing spaces, delete old and append

  size_t contentLen = 0;
  if (!content->isNull()) contentLen = measureJson(*content);

  if (contentLen && contentLen <= oldLen) { //replace and fill diff with spaces
    DEBUGFS_PRINTLN(F("replace"));
    f.seek(pos);
    serializeJson(*content, f);
    writeSpace(pos2 - f.position());
  } else if (contentLen && bufferedFindSpace(contentLen - oldLen, false)) { //enough leading spaces to replace
    DEBUGFS_PRINTLN(F("replace (trailing)"));
    f.seek(pos);
    serializeJson(*content, f);
  } else {
    DEBUGFS_PRINTLN(F("delete"));
    pos -= strlen(key);
    if (pos > 3) pos--; //also delete leading comma if not first object
    f.seek(pos);
    writeSpace(pos2 - pos);
    if (contentLen) return appendObjectToFile(key, content, s, contentLen);
  }

  doCloseFile = true;
  DEBUGFS_PRINTF("Replaced/deleted, took %d ms\n", millis() - s);
  return true;
}

bool readObjectFromFileUsingId(const char* file, uint16_t id, JsonDocument* dest)
{
  char objKey[12];
  sprintf(objKey, "\"%d\":", id);
  return readObjectFromFile(file, objKey, dest);
}

//if the key is a nullptr, deserialize entire object
//WLEDMM: if key is not a nullptr, nothing seems to be done with it!!! (except check for existing), still whole json is loaded
bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
{
  if (doCloseFile) closeFile();
  #ifdef WLED_DEBUG_FS
    DEBUGFS_PRINTF("Read from %s with key %s >>>\n", file, (key==nullptr)?"nullptr":key);
    uint32_t s = millis();
  #endif
  f = WLED_FS.open(file, "r");
  if (!f) return false;
  else { DEBUG_PRINTF(PSTR("FILE '%s' open to read, size %d bytes\n"), file, (int)f.size());} // WLEDMM additional debug message

  #if ESP_IDF_VERSION_MAJOR >= 4
  f.setBufferSize(FS_BUFSIZE*2);        // reduced internal buffer leads to shorter blocking delay, and might prevent LED glitches
  #endif

  if (key != nullptr && !bufferedFind(key)) //key does not exist in file
  {
    f.close();
    dest->clear();
    DEBUGFS_PRINTLN(F("Obj not found."));
    return false;
  }

  deserializeJson(*dest, f);

  f.close();
  DEBUGFS_PRINTF("Read, took %d ms\n", millis() - s);
  return true;
}

void updateFSInfo() {
  #ifdef ARDUINO_ARCH_ESP32
    #if WLED_FS == LITTLEFS || ESP_IDF_VERSION_MAJOR >= 4
    fsBytesTotal = WLED_FS.totalBytes();
    fsBytesUsed = WLED_FS.usedBytes();
    #else
    esp_spiffs_info(nullptr, &fsBytesTotal, &fsBytesUsed);
    #endif
  #else
    FSInfo fsi;
    WLED_FS.info(fsi);
    fsBytesUsed  = fsi.usedBytes;
    fsBytesTotal = fsi.totalBytes;
  #endif
}


//Un-comment any file types you need
static String getContentType(AsyncWebServerRequest* request, String filename){
  if(request->hasArg("download")) return "application/octet-stream";
  else if(filename.endsWith(".htm")) return "text/html";
  else if(filename.endsWith(".html")) return "text/html";
  else if(filename.endsWith(".css")) return "text/css";
  else if(filename.endsWith(".js")) return "application/javascript";
  else if(filename.endsWith(".json")) return "application/json";
  else if(filename.endsWith(".png")) return "image/png";
  else if(filename.endsWith(".gif")) return "image/gif";
  else if(filename.endsWith(".jpg")) return "image/jpeg";
  else if(filename.endsWith(".ico")) return "image/x-icon";
//  else if(filename.endsWith(".xml")) return "text/xml";
//  else if(filename.endsWith(".pdf")) return "application/x-pdf";
//  else if(filename.endsWith(".zip")) return "application/x-zip";
//  else if(filename.endsWith(".gz")) return "application/x-gzip";
  return "text/plain";
}

#if defined(BOARD_HAS_PSRAM) && (defined(WLED_USE_PSRAM) || defined(WLED_USE_PSRAM_JSON))
// caching presets in PSRAM may prevent occasional flashes seen when HomeAssistant polls WLED
// original idea by @akaricchi (https://github.com/Akaricchi)
// returns a pointer to the PSRAM buffer, updates size parameter
static const uint8_t *getPresetCache(size_t &size) {
  static unsigned long presetsCachedTime = 0;
  static uint8_t *presetsCached = nullptr;
  static size_t presetsCachedSize = 0;
  static byte presetsCachedValidate = 0;

  if (!psramFound()) {
    size = 0;
    return nullptr;
  }

  //if (presetsModifiedTime != presetsCachedTime) DEBUG_PRINTLN(F("getPresetCache(): presetsModifiedTime changed."));
  //if (presetsCachedValidate != cacheInvalidate) DEBUG_PRINTLN(F("getPresetCache(): cacheInvalidate changed."));

  if ((presetsModifiedTime != presetsCachedTime) || (presetsCachedValidate != cacheInvalidate)) {
    if (presetsCached) {
      p_free(presetsCached);
      presetsCached = nullptr;
    }
  }

  if (!presetsCached) {
    File file = WLED_FS.open("/presets.json", "r");

  #if ESP_IDF_VERSION_MAJOR >= 4
    if (file) file.setBufferSize(FS_BUFSIZE*2);        // reduced internal buffer leads to shorter blocking delay, and might prevent LED glitches
  #endif
    if (file) {
      presetsCachedTime = presetsModifiedTime;
      presetsCachedValidate = cacheInvalidate;
      presetsCachedSize = 0;
      presetsCached = (uint8_t*)p_malloc(file.size() + 1);
      if (presetsCached) {
        presetsCachedSize = file.size();
        file.read(presetsCached, presetsCachedSize);
        presetsCached[presetsCachedSize] = 0;
        file.close();
        //USER_PRINTLN(F("getPresetCache(): /presets.json cached in PSRAM."));
      }
    }
  } else {
    //USER_PRINTLN(F("getPresetCache(): /presets.json served from PSRAM."));
  }

  size = presetsCachedSize;
  return presetsCached;
}
#endif

// WLEDMM
static volatile bool haveLedmapFile = true;
static volatile bool haveIndexFile = true;
static volatile bool haveSkinFile = true;
static volatile bool haveICOFile = true;
static volatile bool haveCpalFile = true;
void invalidateFileNameCache() { // reset "file not found" cache
  haveLedmapFile = true;
  haveIndexFile = true;
  haveSkinFile = true;
  haveICOFile = true;
  haveCpalFile = true;

  #if (defined(BOARD_HAS_PSRAM) || ESP_IDF_VERSION_MAJOR > 3) && (defined(WLED_USE_PSRAM) || defined(WLED_USE_PSRAM_JSON))
  // WLEDMM hack to clear presets.json cache
  if (psramFound()) {
    size_t dummy;
    unsigned long realpresetsTime = presetsModifiedTime;
    presetsModifiedTime = toki.second();   // pretend we have changes
    (void) getPresetCache(dummy);          // clear presets.json cache
    presetsModifiedTime = realpresetsTime; // restore correct value
  }
  #endif
  //USER_PRINTLN("WS FileRead cache cleared");
}

bool handleFileRead(AsyncWebServerRequest* request, String path){
  DEBUG_PRINTLN("WS FileRead: " + path);
  if(path.endsWith("/")) path += "index.htm";
  if(path.indexOf("sec") > -1) return false;

  // WLEDMM shortcuts
  if ((haveLedmapFile == false) && path.equals("/ledmap.json")) return false;
  if ((haveIndexFile == false)  && path.equals("/index.htm")) return false;
  if ((haveSkinFile == false)   && path.equals("/skin.css")) return false;
  if ((haveICOFile == false)    && path.equals("/favicon.ico")) return false;
  if ((haveCpalFile == false)   && path.equals("/cpal.htm")) return false;
  // WLEDMM toDO: add file caching (PSRAM) for /presets.json an /cfg.json

  String contentType = getContentType(request, path);
  /*String pathWithGz = path + ".gz";
  if(WLED_FS.exists(pathWithGz)){
    request->send(WLED_FS, pathWithGz, contentType);
    return true;
  }*/

  #if defined(BOARD_HAS_PSRAM) && (defined(WLED_USE_PSRAM) || defined(WLED_USE_PSRAM_JSON))
  if (path.endsWith("/presets.json")) {
    size_t psize;
    const uint8_t *presets = getPresetCache(psize);
    if (presets) {
      AsyncWebServerResponse *response = request->beginResponse_P(200, contentType, presets, psize);
      request->send(response);
      return true;
    }
  }
  #endif

  // wait for strip to finish updating, accessing FS during sendout causes glitches
  #if defined(ARDUINO_ARCH_ESP32) && defined(WLEDMM_FILEWAIT)  // only wait if we don't have the flicker-free RMTHI driver
  unsigned wait_start = millis();
  while (strip.isUpdating() && (millis() - wait_start < 40)) delay(1); // wait max 40ms
  #endif

  if(WLED_FS.exists(path) || WLED_FS.exists(path + ".gz")) {
      request->send(WLED_FS, path, String(), request->hasArg(F("download")));
    return true;
  }
  //USER_PRINTLN("WS FileRead failed: "  + path + " (" + contentType + ")");

  // WLEDMM cache "file not found" results (reduces LED flickering during UI activities)
  if (path.equals("/ledmap.json")) haveLedmapFile = false;
  if (path.equals("/index.htm"))   haveIndexFile = false;
  if (path.equals("/skin.css"))    haveSkinFile = false;
  if (path.equals("/favicon.ico")) haveICOFile = false;
  if (path.equals("/cpal.htm"))    haveCpalFile = false;
  return false;
}