Jan/WLED_MM_Infinity
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

bugfix for large JSON files (>64Kb)

Browse Source 
WLED was using uint16_t internally to index files, which leads to random behaviour when the real filesize is above 64Kb.

Better to use size_t which is the "right" type that can hold the largest possible size.
This commit is contained in:
Frank
2023-05-05 18:07:00 +02:00
parent 3df6f286a6
commit 61bbea0e5e
1 changed files with 30 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
wled00/file.cpp
View File

@@ -10,7 +10,11 @@
#endif
#endif
#if defined(WLEDMM_FASTPATH) // WLEMM use bigger buffer when reading, to reduce number of file operations
#define FS_BUFSIZE 512
#else
#define FS_BUFSIZE 256
#endif
/*
* Structural requirements for files managed by writeObjectToFile() and readObjectFromFile() utilities:
@@ -27,9 +31,10 @@
// There are no consecutive spaces longer than this in the file, so if more space is required, findSpace() can return false immediately
// Actual space may be lower
uint16_t knownLargestSpace = UINT16_MAX;
constexpr size_t MAX_SPACE = UINT16_MAX * 2U; // WLEDMM smallest supported config has 128Kb flash size
static volatile size_t knownLargestSpace = MAX_SPACE;
File f;
static File f; // WLEDMM
//wrapper to find out how long closing takes
void closeFile() {
@@ -44,7 +49,7 @@ void closeFile() {
//find() that reads and buffers data from file stream in 256-byte blocks.
//Significantly faster, f.find(key) can take SECONDS for multi-kB files
bool bufferedFind(const char *target, bool fromStart = true) {
static bool bufferedFind(const char *target, bool fromStart = true) {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINT("Find ");
DEBUGFS_PRINTLN(target);
@@ -59,8 +64,8 @@ bool bufferedFind(const char *target, bool fromStart = true) {
if (fromStart) f.seek(0);
while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE);
uint16_t count = 0;
size_t bufsize = f.read(buf, FS_BUFSIZE); // WLEDMM size_t instead if uint16_t
size_t count = 0; // WLEDMM
while (count < bufsize) {
if(buf[count] != target[index])
index = 0; // reset index if any char does not match
@@ -80,7 +85,7 @@ bool bufferedFind(const char *target, bool fromStart = true) {
}
//find empty spots in file stream in 256-byte blocks.
bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
static bool bufferedFindSpace(size_t targetLen, bool fromStart = true) {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTF("Find %d spaces\n", targetLen);
@@ -95,20 +100,20 @@ bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
if (!f || !f.size()) return false;
uint16_t index = 0;
size_t index = 0; // WLEDMM size_t instead if uint16_t
byte buf[FS_BUFSIZE];
if (fromStart) f.seek(0);
while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE);
uint16_t count = 0;
size_t bufsize = f.read(buf, FS_BUFSIZE); // WLEDMM
size_t count = 0; // WLEDMM
while (count < bufsize) {
if(buf[count] == ' ') {
if(++index >= targetLen) { // return true if space long enough
if (fromStart) {
f.seek((f.position() - bufsize) + count +1 - targetLen);
knownLargestSpace = UINT16_MAX; //there may be larger spaces after, so we don't know
knownLargestSpace = MAX_SPACE; //there may be larger spaces after, so we don't know // WLEDMM smallest supported config has 128Kb flash size
}
DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
return true;
@@ -116,7 +121,7 @@ bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
} else {
if (!fromStart) return false;
if (index) {
if (knownLargestSpace < index || knownLargestSpace == UINT16_MAX) knownLargestSpace = index;
if (knownLargestSpace < index || (knownLargestSpace == MAX_SPACE)) knownLargestSpace = index; // WLEDMM
index = 0; // reset index if not space
}
}
@@ -129,7 +134,7 @@ bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
}
//find the closing bracket corresponding to the opening bracket at the file pos when calling this function
bool bufferedFindObjectEnd() {
static bool bufferedFindObjectEnd() {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTLN(F("Find obj end"));
uint32_t s = millis();
@@ -142,8 +147,8 @@ bool bufferedFindObjectEnd() {
byte buf[FS_BUFSIZE];
while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE);
uint16_t count = 0;
size_t bufsize = f.read(buf, FS_BUFSIZE); // WLEDMM size_t instead of uint16_t
size_t count = 0; // WLEDMM
while (count < bufsize) {
if (buf[count] == '{') objDepth++;
@@ -161,13 +166,13 @@ bool bufferedFindObjectEnd() {
}
//fills n bytes from current file pos with ' ' characters
void writeSpace(uint16_t l)
static void writeSpace(size_t l)
{
byte buf[FS_BUFSIZE];
memset(buf, ' ', FS_BUFSIZE);
while (l > 0) {
uint16_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l;
size_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l; // WLEDMM size_t instead of uint16_t
f.write(buf, block);
l -= block;
}
@@ -270,9 +275,11 @@ bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
s = millis();
#endif
uint32_t pos = 0;
size_t pos = 0;
f = WLED_FS.open(file, "r+");
if (!f && !WLED_FS.exists(file)) f = WLED_FS.open(file, "w+");
if (!f && !WLED_FS.exists(file)) { f = WLED_FS.open(file, "w+");
if(f) { DEBUG_PRINTF(PSTR("FILE '%s' open to write, size =%d\n"), file, (int)f.size());} // WLEDMM additional debug message
}
if (!f) {
DEBUGFS_PRINTLN(F("Failed to open!"));
return false;
@@ -287,7 +294,7 @@ bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
pos = f.position();
//measure out end of old object
bufferedFindObjectEnd();
uint32_t pos2 = f.position();
size_t pos2 = f.position(); // WLEDMM
uint32_t oldLen = pos2 - pos;
DEBUGFS_PRINTF("Old obj len %d\n", oldLen);
@@ -298,7 +305,7 @@ bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
//3. The new content is larger than the old, but smaller than old + trailing spaces, overwrite with new
//4. The new content is larger than old + trailing spaces, delete old and append
uint32_t contentLen = 0;
size_t contentLen = 0; // WLEDMM
if (!content->isNull()) contentLen = measureJson(*content);
if (contentLen && contentLen <= oldLen) { //replace and fill diff with spaces
@@ -341,6 +348,7 @@ bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
#endif
f = WLED_FS.open(file, "r");
if (!f) return false;
else { DEBUG_PRINTF(PSTR("FILE '%s' open to read, size %d bytes\n"), file, (int)f.size());} // WLEDMM additional debug message
if (key != nullptr && !bufferedFind(key)) //key does not exist in file
{
@@ -375,7 +383,7 @@ void updateFSInfo() {
//Un-comment any file types you need
String getContentType(AsyncWebServerRequest* request, String filename){
static String getContentType(AsyncWebServerRequest* request, String filename){
if(request->hasArg("download")) return "application/octet-stream";
else if(filename.endsWith(".htm")) return "text/html";
else if(filename.endsWith(".html")) return "text/html";
@@ -394,7 +402,7 @@ String getContentType(AsyncWebServerRequest* request, String filename){
}
bool handleFileRead(AsyncWebServerRequest* request, String path){
DEBUG_PRINTLN("FileRead: " + path);
DEBUG_PRINTLN("WS FileRead: " + path);
if(path.endsWith("/")) path += "index.htm";
if(path.indexOf("sec") > -1) return false;
String contentType = getContentType(request, path);