Jan/WLED_MM_Infinity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfix for large JSON files (>64Kb)

Browse Source 
WLED was using uint16_t internally to index files, which leads to random behaviour when the real filesize is above 64Kb.

Better to use size_t which is the "right" type that can hold the largest possible size.
This commit is contained in:
Frank
2023-05-05 18:07:00 +02:00
parent 3df6f286a6
commit 61bbea0e5e
1 changed files with 30 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
wled00/file.cpp
View File

@@ -10,7 +10,11 @@
#endif #endif
#endif #endif
#if defined(WLEDMM_FASTPATH) // WLEMM use bigger buffer when reading, to reduce number of file operations
#define FS_BUFSIZE 512
#else
#define FS_BUFSIZE 256 #define FS_BUFSIZE 256
#endif
/* /*
* Structural requirements for files managed by writeObjectToFile() and readObjectFromFile() utilities: * Structural requirements for files managed by writeObjectToFile() and readObjectFromFile() utilities:
@@ -27,9 +31,10 @@
// There are no consecutive spaces longer than this in the file, so if more space is required, findSpace() can return false immediately // There are no consecutive spaces longer than this in the file, so if more space is required, findSpace() can return false immediately
// Actual space may be lower // Actual space may be lower
uint16_t knownLargestSpace = UINT16_MAX; constexpr size_t MAX_SPACE = UINT16_MAX * 2U; // WLEDMM smallest supported config has 128Kb flash size
static volatile size_t knownLargestSpace = MAX_SPACE;
File f; static File f; // WLEDMM
//wrapper to find out how long closing takes //wrapper to find out how long closing takes
void closeFile() { void closeFile() {
@@ -44,7 +49,7 @@ void closeFile() {
//find() that reads and buffers data from file stream in 256-byte blocks. //find() that reads and buffers data from file stream in 256-byte blocks.
//Significantly faster, f.find(key) can take SECONDS for multi-kB files //Significantly faster, f.find(key) can take SECONDS for multi-kB files
bool bufferedFind(const char *target, bool fromStart = true) { static bool bufferedFind(const char *target, bool fromStart = true) {
#ifdef WLED_DEBUG_FS #ifdef WLED_DEBUG_FS
DEBUGFS_PRINT("Find "); DEBUGFS_PRINT("Find ");
DEBUGFS_PRINTLN(target); DEBUGFS_PRINTLN(target);
@@ -59,8 +64,8 @@ bool bufferedFind(const char *target, bool fromStart = true) {
if (fromStart) f.seek(0); if (fromStart) f.seek(0);
while (f.position() < f.size() -1) { while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE); size_t bufsize = f.read(buf, FS_BUFSIZE); // WLEDMM size_t instead if uint16_t
uint16_t count = 0; size_t count = 0; // WLEDMM
while (count < bufsize) { while (count < bufsize) {
if(buf[count] != target[index]) if(buf[count] != target[index])
index = 0; // reset index if any char does not match index = 0; // reset index if any char does not match
@@ -80,7 +85,7 @@ bool bufferedFind(const char *target, bool fromStart = true) {
} }
//find empty spots in file stream in 256-byte blocks. //find empty spots in file stream in 256-byte blocks.
bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) { static bool bufferedFindSpace(size_t targetLen, bool fromStart = true) {
#ifdef WLED_DEBUG_FS #ifdef WLED_DEBUG_FS
DEBUGFS_PRINTF("Find %d spaces\n", targetLen); DEBUGFS_PRINTF("Find %d spaces\n", targetLen);
@@ -95,20 +100,20 @@ bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
if (!f || !f.size()) return false; if (!f || !f.size()) return false;
uint16_t index = 0; size_t index = 0; // WLEDMM size_t instead if uint16_t
byte buf[FS_BUFSIZE]; byte buf[FS_BUFSIZE];
if (fromStart) f.seek(0); if (fromStart) f.seek(0);
while (f.position() < f.size() -1) { while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE); size_t bufsize = f.read(buf, FS_BUFSIZE); // WLEDMM
uint16_t count = 0; size_t count = 0; // WLEDMM
while (count < bufsize) { while (count < bufsize) {
if(buf[count] == ' ') { if(buf[count] == ' ') {
if(++index >= targetLen) { // return true if space long enough if(++index >= targetLen) { // return true if space long enough
if (fromStart) { if (fromStart) {
f.seek((f.position() - bufsize) + count +1 - targetLen); f.seek((f.position() - bufsize) + count +1 - targetLen);
knownLargestSpace = UINT16_MAX; //there may be larger spaces after, so we don't know knownLargestSpace = MAX_SPACE; //there may be larger spaces after, so we don't know // WLEDMM smallest supported config has 128Kb flash size
} }
DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s); DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
return true; return true;
@@ -116,7 +121,7 @@ bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
} else { } else {
if (!fromStart) return false; if (!fromStart) return false;
if (index) { if (index) {
if (knownLargestSpace < index || knownLargestSpace == UINT16_MAX) knownLargestSpace = index; if (knownLargestSpace < index || (knownLargestSpace == MAX_SPACE)) knownLargestSpace = index; // WLEDMM
index = 0; // reset index if not space index = 0; // reset index if not space
} }
} }
@@ -129,7 +134,7 @@ bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
} }
//find the closing bracket corresponding to the opening bracket at the file pos when calling this function //find the closing bracket corresponding to the opening bracket at the file pos when calling this function
bool bufferedFindObjectEnd() { static bool bufferedFindObjectEnd() {
#ifdef WLED_DEBUG_FS #ifdef WLED_DEBUG_FS
DEBUGFS_PRINTLN(F("Find obj end")); DEBUGFS_PRINTLN(F("Find obj end"));
uint32_t s = millis(); uint32_t s = millis();
@@ -142,8 +147,8 @@ bool bufferedFindObjectEnd() {
byte buf[FS_BUFSIZE]; byte buf[FS_BUFSIZE];
while (f.position() < f.size() -1) { while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE); size_t bufsize = f.read(buf, FS_BUFSIZE); // WLEDMM size_t instead of uint16_t
uint16_t count = 0; size_t count = 0; // WLEDMM
while (count < bufsize) { while (count < bufsize) {
if (buf[count] == '{') objDepth++; if (buf[count] == '{') objDepth++;
@@ -161,13 +166,13 @@ bool bufferedFindObjectEnd() {
} }
//fills n bytes from current file pos with ' ' characters //fills n bytes from current file pos with ' ' characters
void writeSpace(uint16_t l) static void writeSpace(size_t l)
{ {
byte buf[FS_BUFSIZE]; byte buf[FS_BUFSIZE];
memset(buf, ' ', FS_BUFSIZE); memset(buf, ' ', FS_BUFSIZE);
while (l > 0) { while (l > 0) {
uint16_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l; size_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l; // WLEDMM size_t instead of uint16_t
f.write(buf, block); f.write(buf, block);
l -= block; l -= block;
} }
@@ -270,9 +275,11 @@ bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
s = millis(); s = millis();
#endif #endif
uint32_t pos = 0; size_t pos = 0;
f = WLED_FS.open(file, "r+"); f = WLED_FS.open(file, "r+");
if (!f && !WLED_FS.exists(file)) f = WLED_FS.open(file, "w+"); if (!f && !WLED_FS.exists(file)) { f = WLED_FS.open(file, "w+");
if(f) { DEBUG_PRINTF(PSTR("FILE '%s' open to write, size =%d\n"), file, (int)f.size());} // WLEDMM additional debug message
}
if (!f) { if (!f) {
DEBUGFS_PRINTLN(F("Failed to open!")); DEBUGFS_PRINTLN(F("Failed to open!"));
return false; return false;
@@ -287,7 +294,7 @@ bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
pos = f.position(); pos = f.position();
//measure out end of old object //measure out end of old object
bufferedFindObjectEnd(); bufferedFindObjectEnd();
uint32_t pos2 = f.position(); size_t pos2 = f.position(); // WLEDMM
uint32_t oldLen = pos2 - pos; uint32_t oldLen = pos2 - pos;
DEBUGFS_PRINTF("Old obj len %d\n", oldLen); DEBUGFS_PRINTF("Old obj len %d\n", oldLen);
@@ -298,7 +305,7 @@ bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
//3. The new content is larger than the old, but smaller than old + trailing spaces, overwrite with new //3. The new content is larger than the old, but smaller than old + trailing spaces, overwrite with new
//4. The new content is larger than old + trailing spaces, delete old and append //4. The new content is larger than old + trailing spaces, delete old and append
uint32_t contentLen = 0; size_t contentLen = 0; // WLEDMM
if (!content->isNull()) contentLen = measureJson(*content); if (!content->isNull()) contentLen = measureJson(*content);
if (contentLen && contentLen <= oldLen) { //replace and fill diff with spaces if (contentLen && contentLen <= oldLen) { //replace and fill diff with spaces
@@ -341,6 +348,7 @@ bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
#endif #endif
f = WLED_FS.open(file, "r"); f = WLED_FS.open(file, "r");
if (!f) return false; if (!f) return false;
else { DEBUG_PRINTF(PSTR("FILE '%s' open to read, size %d bytes\n"), file, (int)f.size());} // WLEDMM additional debug message
if (key != nullptr && !bufferedFind(key)) //key does not exist in file if (key != nullptr && !bufferedFind(key)) //key does not exist in file
{ {
@@ -375,7 +383,7 @@ void updateFSInfo() {
//Un-comment any file types you need //Un-comment any file types you need
String getContentType(AsyncWebServerRequest* request, String filename){ static String getContentType(AsyncWebServerRequest* request, String filename){
if(request->hasArg("download")) return "application/octet-stream"; if(request->hasArg("download")) return "application/octet-stream";
else if(filename.endsWith(".htm")) return "text/html"; else if(filename.endsWith(".htm")) return "text/html";
else if(filename.endsWith(".html")) return "text/html"; else if(filename.endsWith(".html")) return "text/html";
@@ -394,7 +402,7 @@ String getContentType(AsyncWebServerRequest* request, String filename){
} }
bool handleFileRead(AsyncWebServerRequest* request, String path){ bool handleFileRead(AsyncWebServerRequest* request, String path){
DEBUG_PRINTLN("FileRead: " + path); DEBUG_PRINTLN("WS FileRead: " + path);
if(path.endsWith("/")) path += "index.htm"; if(path.endsWith("/")) path += "index.htm";
if(path.indexOf("sec") > -1) return false; if(path.indexOf("sec") > -1) return false;
String contentType = getContentType(request, path); String contentType = getContentType(request, path);