Support for large ledmaps

optimize jMapC, enumerateLedmaps and deserializeMap

wled00/FX.h

@@ -618,7 +618,7 @@ typedef struct Segment {
     void blur2d(fract8 blur_amount) { blur(blur_amount); }
     void fill_solid(CRGB c) { fill(RGBW32(c.r,c.g,c.b,0)); }
     void nscale8(uint8_t scale);
-    bool jsonToPixels(char *name, uint8_t fileNr);
+    bool jsonToPixels(char *name, uint8_t fileNr); //WLEDMM for artifx
   #else
     uint16_t XY(uint16_t x, uint16_t y)                                    { return x; }
     void setPixelColorXY(int x, int y, uint32_t c)                         { setPixelColor(x, c); }

wled00/FX_2Dfcn.cpp

@@ -567,6 +567,7 @@ void Segment::drawArc(uint16_t x0, uint16_t y0, uint16_t radius, uint32_t color,
   }
 }
 
+//WLEDMM for artifx
 bool Segment::jsonToPixels(char * name, uint8_t fileNr) {
   char fileName[32];
   //WLEDMM: als support segment name ledmaps

wled00/FX_fcn.cpp

@@ -592,7 +592,7 @@ class JMapC {
       if (jVectorMap.size() > 0) {
         USER_PRINTLN("delete jVectorMap");
         for (size_t i=0; i<jVectorMap.size(); i++)
-          delete jVectorMap[i].array; 
+          delete[] jVectorMap[i].array; 
         jVectorMap.clear();
       }
     }
@@ -647,7 +647,7 @@ class JMapC {
 
         //https://arduinojson.org/v6/how-to/deserialize-a-very-large-document/
         jMapFile.find("[");
-        do {
+        do { //for each element in the array
           DeserializationError err = deserializeJson(docChunk, jMapFile);
           // serializeJson(docChunk, Serial); USER_PRINTLN();
           // USER_PRINTF("docChunk  %u / %u%% (%u %u %u) %u\n", (unsigned int)docChunk.memoryUsage(), 100 * docChunk.memoryUsage() / docChunk.capacity(), (unsigned int)docChunk.size(), docChunk.overflowed(), (unsigned int)docChunk.nesting(), jMapFile.size());
@@ -690,6 +690,8 @@ class JMapC {
 
         } while (jMapFile.findUntil(",", "]"));
 
+        jMapFile.close();
+
         maxWidth++; maxHeight++;
         scale = MIN(SEGMENT.virtualWidth() / maxWidth, SEGMENT.virtualHeight() / maxHeight);
 
@@ -1356,12 +1358,17 @@ void WS2812FX::enumerateLedmaps() {
 
       #ifndef ESP8266
       if (requestJSONBufferLock(21)) {
-        if (readObjectFromFile(fileName, nullptr, &doc)) {
+        //WLEDMM: upstream code loops over all ledmap files, read them all, every byte (!!!!) and only get the name of the file!!!
+        File f;
+        f = WLED_FS.open(fileName, "r");
+        if (f) {
+          f.find("\"n\":");
+          const char *name = f.readStringUntil('\n').c_str();
+          USER_PRINTF("enumerateLedmaps %s %s\n", fileName, name);
+
           size_t len = 0;
-          if (!doc["n"].isNull()) {
+          if (name != nullptr) {
-            // name field exists
+            len = strlen(name);
-            const char *name = doc["n"].as<const char*>();
-            if (name != nullptr) len = strlen(name);
             if (len > 0 && len < 33) {
               ledmapNames[i-1] = new char[len+1];
               if (ledmapNames[i-1]) strlcpy(ledmapNames[i-1], name, 33);
@@ -1375,6 +1382,7 @@ void WS2812FX::enumerateLedmaps() {
             if (ledmapNames[i-1]) strlcpy(ledmapNames[i-1], tmp, 33);
           }
         }
+        f.close();
         releaseJSONBufferLock();
       }
       #endif
@@ -2106,7 +2114,11 @@ bool WS2812FX::deserializeMap(uint8_t n) {
 
   if (!requestJSONBufferLock(7)) return false;
 
-  if (!readObjectFromFile(fileName, nullptr, &doc)) {
+  //WLEDMM: change upstream code: do not load complete ledmaps in json as this blows up memory, use file read instead
+  //read the file
+  File f;
+  f = WLED_FS.open(fileName, "r");
+  if (!f) {
     releaseJSONBufferLock();
     return false; //if file does not exist just exit
   }
@@ -2122,34 +2134,48 @@ bool WS2812FX::deserializeMap(uint8_t n) {
     loadedLedmap = 0;
   }
 
-  JsonArray map = doc[F("map")];
+  //WLEDMM: read width and height (mandatory in file!!)
-  if (!map.isNull() && map.size()) {  // not an empty map
+  f.find("\"width\":");
+  uint16_t maxWidth = f.readStringUntil('\n').toInt();
 
-    //WLEDMM: support ledmap file properties width and height
+  f.find("\"height\":");
-    if (doc[F("width")]>0 && doc[F("height")]>0) {
+  uint16_t maxHeight = f.readStringUntil('\n').toInt();
-      Segment::maxWidth = doc[F("width")];;
-      Segment::maxHeight = doc[F("height")];;
-      resetSegments(true); //WLEDMM not makeAutoSegments() as we only want to change bounds
-    }
 
-    customMappingSize  = map.size();
+  USER_PRINTF("deserializeMap %d x %d\n", maxWidth, maxHeight);
-    customMappingTable = new uint16_t[customMappingSize];
+  if (maxWidth * maxHeight <= 0) {
-
+    releaseJSONBufferLock();
-    for (uint16_t i=0; i<customMappingSize; i++) 
+    return false;
-      customMappingTable[i] = (uint16_t) (map[i]<0 ? 0xFFFFU : map[i]);
-
-    loadedLedmap = n;
-
-    #ifdef WLED_DEBUG
-      DEBUG_PRINTF("Custom ledmap: %d\n", loadedLedmap);
-      for (uint16_t i=0; i<customMappingSize; i++) {
-        if (!(i%Segment::maxWidth)) DEBUG_PRINTLN();
-        DEBUG_PRINTF("%4d,", customMappingTable[i]);
-      }
-      DEBUG_PRINTLN();
-    #endif
   }
 
+  //WLEDMM: support ledmap file properties width and height
+  Segment::maxWidth = maxWidth;
+  Segment::maxHeight = maxHeight;
+  resetSegments(true); //WLEDMM not makeAutoSegments() as we only want to change bounds
+
+  customMappingSize  = maxWidth * maxHeight;
+  customMappingTable = new uint16_t[customMappingSize];
+
+  //WLEDMM: find the map values
+  f.find("\"map\":[");
+  uint16_t i=0;
+  do { //for each element in the array
+    int mapi = f.readStringUntil(',').toInt();
+    // USER_PRINTF(", %d", mapi);
+    customMappingTable[i++] = (uint16_t) (mapi<0 ? 0xFFFFU : mapi);
+  } while (f.available());
+
+  loadedLedmap = n;
+  f.close();
+
+  #ifdef WLED_DEBUG
+    DEBUG_PRINTF("Custom ledmap: %d\n", loadedLedmap);
+    for (uint16_t i=0; i<customMappingSize; i++) {
+      if (!(i%Segment::maxWidth)) DEBUG_PRINTLN();
+      DEBUG_PRINTF("%4d,", customMappingTable[i]);
+    }
+    DEBUG_PRINTLN();
+  #endif
+
   releaseJSONBufferLock();
   return true;
 }

wled00/cfg.cpp

@@ -19,9 +19,9 @@ void getStringFromJson(char* dest, const char* src, size_t len) {
 bool deserializeConfig(JsonObject doc, bool fromFS) {
 
   //WLEDMM add USER_PRINT
-  String temp;
+  // String temp;
-  serializeJson(doc, temp);
+  // serializeJson(doc, temp);
-  USER_PRINTF("deserializeConfig %s\n", temp.c_str());
+  USER_PRINTF("deserializeConfig\n");
 
   bool needsSave = false;
   //int rev_major = doc["rev"][0]; // 1

wled00/data/index.js

@@ -1274,6 +1274,7 @@ function updateLen(s, draw=true) //WLEDMM conditonally draw segment view
 
 	gId(`seg${s}len`).innerHTML = out;
 
+	// console.log("drawSegmentView","updateLen");
 	if (draw && isM) drawSegmentView(); //WLEDMM draw new segmentview if something changes in a segment
 }
 
@@ -1490,7 +1491,7 @@ function drawSegmentView() {
 			}
 			post();
 		}, function(parms,error) { //error handling
-			console.log(error);
+			// console.log("drawledmap error fetching " + fileName +": ", error);
 			// downloadGHFile("LM", fileName, true, false); WLEDMM: remove as this has too much impact
 			post();
 		});
@@ -1669,6 +1670,8 @@ function makeWS() {
 		var s = json.state ? json.state : json;
 		displayRover(i, s);
 		readState(s);
+		// console.log("drawSegmentView","websocket", json);
+		// if (isM) drawSegmentView();
 	};
 	ws.onclose = (e)=>{
 		gId('connind').style.backgroundColor = "var(--c-r)";
@@ -1979,6 +1982,7 @@ function requestJson(command=null)
 
 		//WLEDMM init, gfx default on upon web page load
 		if (isM) {
+			// console.log("drawSegmentView","requestjson");
 			drawSegmentView();
 			toggleLiveview();
 		}
@@ -3065,7 +3069,6 @@ function fetchAndExecute(url, name, parms, callback, callError = null)
     return res.text();
   })
   .then(text => {
-	console.log("text", text);
     callback(parms, text);
   })
   .catch(function (error) {
@@ -3355,6 +3358,7 @@ function togglePcMode(fromB = false)
 	sCol('--bh', gId('bot').clientHeight + "px");
 	_C.style.width = (pcMode)?'100%':'400%';
 	//WLEDMM resize segmentview
+	// console.log("drawSegmentView","togglePCMode");
 	if (isM) drawSegmentView();
 }
 

wled00/file.cpp

@@ -339,6 +339,7 @@ bool readObjectFromFileUsingId(const char* file, uint16_t id, JsonDocument* dest
 }
 
 //if the key is a nullptr, deserialize entire object
+//WLEDMM: if key is not a nullptr, nothing seems to be done with it!!! (except check for existing), still whole json is loaded
 bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
 {
   if (doCloseFile) closeFile();

wled00/wled.h

@@ -8,7 +8,7 @@
  */
 
 // version code in format yymmddb (b = daily build)
-#define VERSION 2305130
+#define VERSION 2305160
 
 //uncomment this if you have a "my_config.h" file you'd like to use
 //#define WLED_USE_MY_CONFIG