upgrade HTML data build system - use same tools as in upstream WLED

align with tools/cdata.js from upstream WLED

--> prerequisite for pixelforge tool !

package.json

@@ -1,7 +1,7 @@
 {
   "name": "wled",
   "version": "14.7.0-mdev",
-  "description": "Tools for WLED project",
+  "description": "Tools for WLED-MM project",
   "main": "tools/cdata.js",
   "directories": {
     "lib": "lib",
@@ -9,6 +9,7 @@
   },
   "scripts": {
     "build": "node tools/cdata.js",
+    "test": "node --test",
     "dev": "nodemon -e js,html,htm,css,png,jpg,gif,ico,js -w tools/ -w wled00/data/ -x node tools/cdata.js"
   },
   "repository": {
@@ -22,10 +23,12 @@
   },
   "homepage": "https://github.com/MoonModules/WLED-MM#readme",
   "dependencies": {
-    "clean-css": "^4.2.3",
+    "clean-css": "^5.3.3",
-    "html-minifier-terser": "^5.1.1",
+    "html-minifier-terser": "^7.2.0",
-    "inliner": "^1.13.1",
+    "web-resource-inliner": "^7.0.0",
-    "nodemon": "^2.0.20",
+    "nodemon": "^3.1.9"
-    "zlib": "^1.0.5"
+  },
+  "engines": {
+    "node": ">=20.0.0"
   }
 }

pio-scripts/build_ui.py

@@ -0,0 +1,21 @@
+Import("env")
+import shutil
+
+node_ex = shutil.which("node")
+# Check if Node.js is installed and present in PATH if it failed, abort the build
+if node_ex is None:
+    print('\x1b[0;31;43m' + 'Node.js is not installed or missing from PATH html css js will not be processed check https://kno.wled.ge/advanced/compiling-wled/' + '\x1b[0m')
+    exitCode = env.Execute("null")
+    exit(exitCode)
+else:
+    # Install the necessary node packages for the pre-build asset bundling script
+    print('\x1b[6;33;42m' + 'Installing node packages' + '\x1b[0m')
+    env.Execute("npm ci")
+
+    # Call the bundling script
+    exitCode = env.Execute("npm run build")
+
+    # If it failed, abort the build
+    if (exitCode):
+      print('\x1b[0;31;43m' + 'npm run build fails check https://kno.wled.ge/advanced/compiling-wled/' + '\x1b[0m')
+      exit(exitCode)

pio-scripts/set_metadata.py

@@ -0,0 +1,116 @@
+Import('env')
+import subprocess
+import json
+import re
+
+def get_github_repo():
+    """Extract GitHub repository name from git remote URL.
+    
+    Uses the remote that the current branch tracks, falling back to 'origin'.
+    This handles cases where repositories have multiple remotes or where the
+    main remote is not named 'origin'.
+    
+    Returns:
+        str: Repository name in 'owner/repo' format for GitHub repos,
+             'unknown' for non-GitHub repos, missing git CLI, or any errors.
+    """
+    try:
+        remote_name = 'origin'  # Default fallback
+        
+        # Try to get the remote for the current branch
+        try:
+            # Get current branch name
+            branch_result = subprocess.run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], 
+                                         capture_output=True, text=True, check=True)
+            current_branch = branch_result.stdout.strip()
+            
+            # Get the remote for the current branch
+            remote_result = subprocess.run(['git', 'config', f'branch.{current_branch}.remote'], 
+                                         capture_output=True, text=True, check=True)
+            tracked_remote = remote_result.stdout.strip()
+            
+            # Use the tracked remote if we found one
+            if tracked_remote:
+                remote_name = tracked_remote
+        except subprocess.CalledProcessError:
+            # If branch config lookup fails, continue with 'origin' as fallback
+            pass
+        
+        # Get the remote URL for the determined remote
+        result = subprocess.run(['git', 'remote', 'get-url', remote_name], 
+                              capture_output=True, text=True, check=True)
+        remote_url = result.stdout.strip()
+        
+        # Check if it's a GitHub URL
+        if 'github.com' not in remote_url.lower():
+            return None
+        
+        # Parse GitHub URL patterns:
+        # https://github.com/owner/repo.git
+        # git@github.com:owner/repo.git
+        # https://github.com/owner/repo
+        
+        # Remove .git suffix if present
+        if remote_url.endswith('.git'):
+            remote_url = remote_url[:-4]
+        
+        # Handle HTTPS URLs
+        https_match = re.search(r'github\.com/([^/]+/[^/]+)', remote_url, re.IGNORECASE)
+        if https_match:
+            return https_match.group(1)
+        
+        # Handle SSH URLs
+        ssh_match = re.search(r'github\.com:([^/]+/[^/]+)', remote_url, re.IGNORECASE)
+        if ssh_match:
+            return ssh_match.group(1)
+        
+        return None
+        
+    except FileNotFoundError:
+        # Git CLI is not installed or not in PATH
+        return None
+    except subprocess.CalledProcessError:
+        # Git command failed (e.g., not a git repo, no remote, etc.)
+        return None
+    except Exception:
+        # Any other unexpected error
+        return None
+
+# WLED version is managed by package.json; this is picked up in several places
+# - It's integrated in to the UI code
+# - Here, for wled_metadata.cpp
+# - The output_bins script
+# We always take it from package.json to ensure consistency
+with open("package.json", "r") as package:
+    WLED_VERSION = json.load(package)["version"]
+
+def has_def(cppdefs, name):
+    """ Returns true if a given name is set in a CPPDEFINES collection """
+    for f in cppdefs:
+        if isinstance(f, tuple):
+            f = f[0]
+        if f == name:
+            return True
+    return False
+
+
+def add_wled_metadata_flags(env, node):    
+    cdefs = env["CPPDEFINES"].copy()
+
+    if not has_def(cdefs, "WLED_REPO"):
+        repo = get_github_repo()
+        if repo:
+            cdefs.append(("WLED_REPO", f"\\\"{repo}\\\""))
+
+    cdefs.append(("WLED_VERSION", WLED_VERSION))
+
+    # This transforms the node in to a Builder; it cannot be modified again
+    return env.Object(
+        node,
+        CPPDEFINES=cdefs
+    )
+   
+env.AddBuildMiddleware(
+    add_wled_metadata_flags,
+    "*/wled_metadata.cpp"
+)

pio-scripts/validate_modules.py

@@ -0,0 +1,80 @@
+import re
+from pathlib import Path   # For OS-agnostic path manipulation
+from typing import Iterable
+from click import secho
+from SCons.Script import Action, Exit
+from platformio.builder.tools.piolib import LibBuilderBase
+
+
+def is_wled_module(env, dep: LibBuilderBase) -> bool:
+  """Returns true if the specified library is a wled module
+  """
+  usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
+  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
+
+
+def read_lines(p: Path):
+    """ Read in the contents of a file for analysis """
+    with p.open("r", encoding="utf-8", errors="ignore") as f:
+        return f.readlines()
+
+
+def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
+    """ Identify which dirs contributed to the final build
+
+        Returns the (sub)set of dirs that are found in the output ELF
+    """
+    # Pattern to match symbols in object directories
+    # Join directories into alternation
+    usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
+    # Matches nonzero address, any size, and any path in a matching directory
+    object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
+
+    found = set()
+    for line in map_file:
+        matches = object_path_regex.findall(line)
+        for m in matches:
+            found.add(m)
+    return found
+
+
+def count_usermod_objects(map_file: list[str]) -> int:
+    """ Returns the number of usermod objects in the usermod list """
+    # Count the number of entries in the usermods table section
+    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
+
+
+def validate_map_file(source, target, env):
+    """ Validate that all modules appear in the output build """
+    build_dir = Path(env.subst("$BUILD_DIR"))
+    map_file_path = build_dir /  env.subst("${PROGNAME}.map")
+
+    if not map_file_path.exists():
+        secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
+        Exit(1)
+
+    # Identify the WLED module builders, set by load_usermods.py
+    module_lib_builders = env['WLED_MODULES']
+
+    # Extract the values we care about
+    modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
+    secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
+
+    # Now parse the map file
+    map_file_contents = read_lines(map_file_path)
+    usermod_object_count = count_usermod_objects(map_file_contents)
+    secho(f"INFO: {usermod_object_count} usermod object entries")
+
+    confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
+    missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
+    if missing_modules:
+        secho(
+            f"ERROR: No object files from {missing_modules} found in linked output!",
+            fg="red",
+            err=True)
+        Exit(1)
+    return None
+
+Import("env")
+env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

platformio.ini

@@ -220,7 +220,7 @@ ldscript_16m14m = eagle.flash.16m14m.ld
 [scripts_defaults]
 extra_scripts =
   pre:pio-scripts/set_version.py
-  pre:pio-scripts/build-html.py
+  pre:pio-scripts/build_ui.py
   pre:pio-scripts/conditional_usb_mode.py
   post:pio-scripts/output_bins.py
   post:pio-scripts/strip-floats.py

tools/cdata-test.js

@@ -0,0 +1,212 @@
+'use strict';
+
+const assert = require('node:assert');
+const { describe, it, before, after } = require('node:test');
+const fs = require('fs');
+const path = require('path');
+const child_process = require('child_process');
+const util = require('util');
+const execPromise = util.promisify(child_process.exec);
+
+process.env.NODE_ENV = 'test'; // Set the environment to testing
+const cdata = require('./cdata.js');
+
+describe('Function', () => {
+  const testFolderPath = path.join(__dirname, 'testFolder');
+  const oldFilePath = path.join(testFolderPath, 'oldFile.txt');
+  const newFilePath = path.join(testFolderPath, 'newFile.txt');
+
+  // Create a temporary file before the test
+  before(() => {
+    // Create test folder
+    if (!fs.existsSync(testFolderPath)) {
+      fs.mkdirSync(testFolderPath);
+    }
+
+    // Create an old file
+    fs.writeFileSync(oldFilePath, 'This is an old file.');
+    // Modify the 'mtime' to simulate an old file
+    const oldTime = new Date();
+    oldTime.setFullYear(oldTime.getFullYear() - 1);
+    fs.utimesSync(oldFilePath, oldTime, oldTime);
+
+    // Create a new file
+    fs.writeFileSync(newFilePath, 'This is a new file.');
+  });
+
+  // delete the temporary files after the test
+  after(() => {
+    fs.rmSync(testFolderPath, { recursive: true });
+  });
+
+  describe('isFileNewerThan', async () => {
+    it('should return true if the file is newer than the provided time', async () => {
+      const pastTime = Date.now() - 10000; // 10 seconds ago
+      assert.strictEqual(cdata.isFileNewerThan(newFilePath, pastTime), true);
+    });
+
+    it('should return false if the file is older than the provided time', async () => {
+      assert.strictEqual(cdata.isFileNewerThan(oldFilePath, Date.now()), false);
+    });
+
+    it('should throw an exception if the file does not exist', async () => {
+      assert.throws(() => {
+        cdata.isFileNewerThan('nonexistent.txt', Date.now());
+      });
+    });
+  });
+
+  describe('isAnyFileInFolderNewerThan', async () => {
+    it('should return true if a file in the folder is newer than the given time', async () => {
+      const time = fs.statSync(path.join(testFolderPath, 'oldFile.txt')).mtime;
+      assert.strictEqual(cdata.isAnyFileInFolderNewerThan(testFolderPath, time), true);
+    });
+
+    it('should return false if no files in the folder are newer than the given time', async () => {
+      assert.strictEqual(cdata.isAnyFileInFolderNewerThan(testFolderPath, new Date()), false);
+    });
+
+    it('should throw an exception if the folder does not exist', async () => {
+      assert.throws(() => {
+        cdata.isAnyFileInFolderNewerThan('nonexistent', new Date());
+      });
+    });
+  });
+});
+
+describe('Script', () => {
+  const folderPath = 'wled00';
+  const dataPath = path.join(folderPath, 'data');
+
+  before(() => {
+    process.env.NODE_ENV = 'production';
+    // Backup files
+    fs.cpSync("wled00/data", "wled00Backup", { recursive: true });
+    fs.cpSync("tools/cdata.js", "cdata.bak.js");
+    fs.cpSync("package.json", "package.bak.json");
+  });
+  after(() => {
+    // Restore backup
+    fs.rmSync("wled00/data", { recursive: true });
+    fs.renameSync("wled00Backup", "wled00/data");
+    fs.rmSync("tools/cdata.js");
+    fs.renameSync("cdata.bak.js", "tools/cdata.js");
+    fs.rmSync("package.json");
+    fs.renameSync("package.bak.json", "package.json");
+  });
+
+  // delete all html_*.h files
+  async function deleteBuiltFiles() {
+    const files = await fs.promises.readdir(folderPath);
+    await Promise.all(files.map(file => {
+      if (file.startsWith('html_') && path.extname(file) === '.h') {
+        return fs.promises.unlink(path.join(folderPath, file));
+      }
+    }));
+  }
+
+  // check if html_*.h files were created
+  async function checkIfBuiltFilesExist() {
+    const files = await fs.promises.readdir(folderPath);
+    const htmlFiles = files.filter(file => file.startsWith('html_') && path.extname(file) === '.h');
+    assert(htmlFiles.length > 0, 'html_*.h files were not created');
+  }
+
+  async function runAndCheckIfBuiltFilesExist() {
+    await execPromise('node tools/cdata.js');
+    await checkIfBuiltFilesExist();
+  }
+
+  async function checkIfFileWasNewlyCreated(file) {
+    const modifiedTime = fs.statSync(file).mtimeMs;
+    assert(Date.now() - modifiedTime < 500, file + ' was not modified');
+  }
+
+  async function testFileModification(sourceFilePath, resultFile) {
+    // run cdata.js to ensure html_*.h files are created
+    await execPromise('node tools/cdata.js');
+
+    // modify file
+    fs.appendFileSync(sourceFilePath, ' ');
+    // delay for 1 second to ensure the modified time is different
+    await new Promise(resolve => setTimeout(resolve, 1000));
+
+    // run script cdata.js again and wait for it to finish
+    await execPromise('node tools/cdata.js');
+
+    await checkIfFileWasNewlyCreated(path.join(folderPath, resultFile));
+  }
+
+  describe('should build if', () => {
+    it('html_*.h files are missing', async () => {
+      await deleteBuiltFiles();
+      await runAndCheckIfBuiltFilesExist();
+    });
+
+    it('only one html_*.h file is missing', async () => {
+      // run script cdata.js and wait for it to finish
+      await execPromise('node tools/cdata.js');
+
+      // delete a random html_*.h file
+      let files = await fs.promises.readdir(folderPath);
+      let htmlFiles = files.filter(file => file.startsWith('html_') && path.extname(file) === '.h');
+      const randomFile = htmlFiles[Math.floor(Math.random() * htmlFiles.length)];
+      await fs.promises.unlink(path.join(folderPath, randomFile));
+
+      await runAndCheckIfBuiltFilesExist();
+    });
+
+    it('script was executed with -f or --force', async () => {
+      await execPromise('node tools/cdata.js');
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      await execPromise('node tools/cdata.js --force');
+      await checkIfFileWasNewlyCreated(path.join(folderPath, 'html_ui.h'));
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      await execPromise('node tools/cdata.js -f');
+      await checkIfFileWasNewlyCreated(path.join(folderPath, 'html_ui.h'));
+    });
+
+    it('a file changes', async () => {
+      await testFileModification(path.join(dataPath, 'index.htm'), 'html_ui.h');
+    });
+
+    it('a inlined file changes', async () => {
+      await testFileModification(path.join(dataPath, 'index.js'), 'html_ui.h');
+    });
+
+    it('a settings file changes', async () => {
+      await testFileModification(path.join(dataPath, 'settings_leds.htm'), 'html_ui.h');
+    });
+
+    it('the favicon changes', async () => {
+      await testFileModification(path.join(dataPath, 'favicon.ico'), 'html_ui.h');
+    });
+
+    it('cdata.js changes', async () => {
+      await testFileModification('tools/cdata.js', 'html_ui.h');
+    });
+
+    it('package.json changes', async () => {
+      await testFileModification('package.json', 'html_ui.h');
+    });
+  });
+
+  describe('should not build if', () => {
+    it('the files are already built', async () => {
+      await deleteBuiltFiles();
+
+      // run script cdata.js and wait for it to finish
+      let startTime = Date.now();
+      await execPromise('node tools/cdata.js');
+      const firstRunTime = Date.now() - startTime;
+
+      // run script cdata.js and wait for it to finish
+      startTime = Date.now();
+      await execPromise('node tools/cdata.js');
+      const secondRunTime = Date.now() - startTime;
+
+      // check if second run was faster than the first (must be at least 2x faster)
+      assert(secondRunTime < firstRunTime / 2, 'html_*.h files were rebuilt');
+    });
+  });
+});

tools/cdata.js

@@ -2,7 +2,7 @@
  * Writes compressed C arrays of data files (web interface)
  * How to use it?
  *
- * 1) Install Node 11+ and npm
+ * 1) Install Node 20+ and npm
  * 2) npm install
  * 3) npm run build
  *
@@ -15,26 +15,70 @@
  * It uses NodeJS packages to inline, minify and GZIP files. See writeHtmlGzipped and writeChunks invocations at the bottom of the page.
  */
 
-const fs = require("fs");
+const fs = require("node:fs");
-const inliner = require("inliner");
+const path = require("path");
-const zlib = require("zlib");
+const inline = require("web-resource-inliner");
+const zlib = require("node:zlib");
 const CleanCSS = require("clean-css");
-const MinifyHTML = require("html-minifier-terser").minify;
+const minifyHtml = require("html-minifier-terser").minify;
 const packageJson = require("../package.json");
 
-/**
+// Export functions for testing
- *
+module.exports = { isFileNewerThan, isAnyFileInFolderNewerThan };
+
+//const output = ["wled00/html_ui.h", "wled00/html_pixart.h", "wled00/html_cpal.h", "wled00/html_edit.h", "wled00/html_pxmagic.h", "wled00/html_pixelforge.h", "wled00/html_settings.h", "wled00/html_other.h"]
+const output = ["wled00/html_ui.h", "wled00/html_pixart.h", "wled00/html_cpal.h", "wled00/html_edit.h", "wled00/html_pxmagic.h", "wled00/html_settings.h", "wled00/html_other.h"]
+
+// \x1b[34m is blue, \x1b[36m is cyan, \x1b[0m is reset
+const wledBanner = `
+\t\x1b[34m  ##  ##      ##        ######    ######
+\t\x1b[34m##      ##    ##      ##        ##  ##  ##
+\t\x1b[34m##  ##  ##  ##        ######        ##  ##
+\t\x1b[34m##  ##  ##  ##        ##            ##  ##
+\t\x1b[34m  ##  ##      ######    ######    ######
+\t\t\x1b[36m build script for web UI
+\x1b[0m`;
+
+// Generate build timestamp as UNIX timestamp (seconds since epoch)
+function generateBuildTime() {
+  return Math.floor(Date.now() / 1000);
+}
+
+const singleHeader = `/*
+ * Binary array for the Web UI.
+ * gzip is used for smaller size and improved speeds.
+ * 
+ * Please see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
+ * to find out how to easily modify the web UI source!
  */
-function hexdump(buffer,isHex=false) {
+
+// Automatically generated build time for cache busting (UNIX timestamp)
+#ifdef WEB_BUILD_TIME
+#undef WEB_BUILD_TIME
+#endif
+#define WEB_BUILD_TIME ${generateBuildTime()}
+
+`;
+
+const multiHeader = `/*
+ * More web UI HTML source arrays.
+ * This file is auto generated, please don't make any changes manually.
+ *
+ * Instead, see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
+ * to find out how to easily modify the web UI source!
+ */
+`;
+
+function hexdump(buffer, isHex = false) {
   let lines = [];
 
-  for (let i = 0; i < buffer.length; i +=(isHex?32:16)) {
+  for (let i = 0; i < buffer.length; i += (isHex ? 32 : 16)) {
     var block;
     let hexArray = [];
     if (isHex) {
       block = buffer.slice(i, i + 32)
-      for (let j = 0; j < block.length; j +=2 ) {
+      for (let j = 0; j < block.length; j += 2) {
-        hexArray.push("0x" + block.slice(j,j+2))
+        hexArray.push("0x" + block.slice(j, j + 2))
       }
     } else {
       block = buffer.slice(i, i + 16); // cut buffer into blocks of 16
@@ -51,219 +95,203 @@ function hexdump(buffer,isHex=false) {
   return lines.join(",\n");
 }
 
-function strReplace(str, search, replacement) {
-  return str.split(search).join(replacement);
-}
-
 function adoptVersionAndRepo(html) {
   let repoUrl = packageJson.repository ? packageJson.repository.url : undefined;
   if (repoUrl) {
     repoUrl = repoUrl.replace(/^git\+/, "");
     repoUrl = repoUrl.replace(/\.git$/, "");
-    // Replace we
+    // html = html.replaceAll("https://github.com/wled-dev/WLED", repoUrl); // WLEDMM replacing upstream break "credits"
-    html = strReplace(html, "https://github.com/atuline/WLED", repoUrl);
+    html = html.replaceAll("https://github.com/atuline/WLED", repoUrl);
-    html = strReplace(html, "https://github.com/Aircoookie/WLED", repoUrl);
+    html = html.replaceAll("https://github.com/Aircoookie/WLED", repoUrl);
-    // html = strReplace(html, "https://github.com/wled-dev/WLED", repoUrl); // replacing upstream break "credits"
-    // html = strReplace(html, "https://github.com/wled/WLED", repoUrl);
-    // html = strReplace(html, "https://github.com/MoonModules/WLED", repoUrl); //WLEDMM
-    // html = strReplace(html, "https://github.com/MoonModules/WLED-MM", repoUrl); //WLEDMM - not necessary to replace ourselves ;-)
   }
   let version = packageJson.version;
   if (version) {
-    html = strReplace(html, "##VERSION##", version);
+    html = html.replaceAll("##VERSION##", version);
   }
   return html;
 }
 
-function filter(str, type) {
+async function minify(str, type = "plain") {
-  str = adoptVersionAndRepo(str);
+  const options = {
-  if (type === undefined) {
+    collapseWhitespace: true,
+    conservativeCollapse: true, // preserve spaces in text
+    collapseBooleanAttributes: true,
+    collapseInlineTagWhitespace: true,
+    minifyCSS: true,
+    minifyJS: true,
+    removeAttributeQuotes: true,
+    removeComments: true,
+    sortAttributes: true,
+    sortClassName: true,
+  };
+
+  if (type == "plain") {
     return str;
   } else if (type == "css-minify") {
     return new CleanCSS({}).minify(str).styles;
   } else if (type == "js-minify") {
-    return MinifyHTML('<script>' + str + '</script>', {
+    let js = await minifyHtml('<script>' + str + '</script>', options);
-      collapseWhitespace: true,
+    return js.replace(/<[\/]*script>/g, '');
-      minifyJS: true, 
-      continueOnParseError: false,
-      removeComments: true,
-    }).replace(/<[\/]*script>/g,'');
   } else if (type == "html-minify") {
-    return MinifyHTML(str, {
+    return await minifyHtml(str, options);
-      collapseWhitespace: true,
-      maxLineLength: 80,
-      minifyCSS: true,
-      minifyJS: true, 
-      continueOnParseError: false,
-      removeComments: true,
-    });
-  } else if (type == "html-minify-ui") {
-    return MinifyHTML(str, {
-      collapseWhitespace: true,
-      conservativeCollapse: true,
-      maxLineLength: 80,
-      minifyCSS: true,
-      minifyJS: true, 
-      continueOnParseError: false,
-      removeComments: true,
-    });
-  } else {
-    console.warn("Unknown filter: " + type);
-    return str;
   }
+
+  throw new Error("Unknown filter: " + type);
 }
 
-// Generate build timestamp as UNIX timestamp (seconds since epoch)
+async function writeHtmlGzipped(sourceFile, resultFile, page, inlineCss = true) {
-function generateBuildTime() {
-  return Math.floor(Date.now() / 1000);
-}
-
-function writeHtmlGzipped(sourceFile, resultFile, page) {
   console.info("Reading " + sourceFile);
-  new inliner(sourceFile, function (error, html) {
+  inline.html({
-    console.info("Inlined " + html.length + " characters");
+    fileContent: fs.readFileSync(sourceFile, "utf8"),
-    html = filter(html, "html-minify-ui");
+    relativeTo: path.dirname(sourceFile),
-    console.info("Minified to " + html.length + " characters");
+    strict: inlineCss,     // when not inlining css, ignore errors (enables linking style.css from subfolder htm files)
+    stylesheets: inlineCss // when true (default), css is inlined
+  },
+    async function (error, html) {
+      if (error) throw error;
 
-    if (error) {
+      html = adoptVersionAndRepo(html);
-      console.warn(error);
+      const originalLength = html.length;
-      throw error;
+      html = await minify(html, "html-minify");
-    }
+      const result = zlib.gzipSync(html, { level: zlib.constants.Z_BEST_COMPRESSION });
-
+      console.info("Minified and compressed " + sourceFile + " from " + originalLength + " to " + result.length + " bytes");
-    html = adoptVersionAndRepo(html);
-    zlib.gzip(html, { level: zlib.constants.Z_BEST_COMPRESSION }, function (error, result) {
-      if (error) {
-        console.warn(error);
-        throw error;
-      }
-
-      console.info("Compressed " + result.length + " bytes");
       const array = hexdump(result);
-      const src = `/*
+      let src = singleHeader;
- * Binary array for the Web UI.
+      src += `const uint16_t PAGE_${page}_L = ${result.length};\n`;
- * gzip is used for smaller size and improved speeds.
+      src += `const uint8_t PAGE_${page}[] PROGMEM = {\n${array}\n};\n\n`;
- * 
- * Please see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
- * to find out how to easily modify the web UI source!
- */
-
-// Automatically generated build time for cache busting (UNIX timestamp)
-#ifdef WEB_BUILD_TIME // avoid duplicate defintions
-#undef WEB_BUILD_TIME
-#endif
-#define WEB_BUILD_TIME ${generateBuildTime()}
-
-// Autogenerated from ${sourceFile}, do not edit!!
-const uint16_t PAGE_${page}_L = ${result.length};
-const uint8_t PAGE_${page}[] PROGMEM = {
-${array}
-};
-`;
       console.info("Writing " + resultFile);
       fs.writeFileSync(resultFile, src);
     });
-  });
 }
 
-function specToChunk(srcDir, s) {
+async function specToChunk(srcDir, s) {
-  if (s.method == "plaintext") {
+  const buf = fs.readFileSync(srcDir + "/" + s.file);
-    const buf = fs.readFileSync(srcDir + "/" + s.file);
+  let chunk = `\n// Autogenerated from ${srcDir}/${s.file}, do not edit!!\n`
-    const str = buf.toString("utf-8");
-    const chunk = `
-// Autogenerated from ${srcDir}/${s.file}, do not edit!!
-const char ${s.name}[] PROGMEM = R"${s.prepend || ""}${filter(str, s.filter)}${
-      s.append || ""
-    }";
 
-`;
+  if (s.method == "plaintext" || s.method == "gzip") {
-    return s.mangle ? s.mangle(chunk) : chunk;
+    let str = buf.toString("utf-8");
-  } else if (s.method == "gzip") {
+    str = adoptVersionAndRepo(str);
-    const buf = fs.readFileSync(srcDir + "/" + s.file);
+    const originalLength = str.length;
-    var str = buf.toString('utf-8');
+    if (s.method == "gzip") {
-    if (s.mangle) str = s.mangle(str);
+      if (s.mangle) str = s.mangle(str);
-    const zip = zlib.gzipSync(filter(str, s.filter), { level: zlib.constants.Z_BEST_COMPRESSION });
+      const zip = zlib.gzipSync(await minify(str, s.filter), { level: zlib.constants.Z_BEST_COMPRESSION });
-    const result = hexdump(zip.toString('hex'), true);
+      console.info("Minified and compressed " + s.file + " from " + originalLength + " to " + zip.length + " bytes");
-    const chunk = `
+      const result = hexdump(zip);
-// Autogenerated from ${srcDir}/${s.file}, do not edit!!
+      chunk += `const uint16_t ${s.name}_length = ${zip.length};\n`;
-const uint16_t ${s.name}_length = ${zip.length};
+      chunk += `const uint8_t ${s.name}[] PROGMEM = {\n${result}\n};\n\n`;
-const uint8_t ${s.name}[] PROGMEM = {
+      return chunk;
-${result}
+    } else {
-};
+      const minified = await minify(str, s.filter);
-
+      console.info("Minified " + s.file + " from " + originalLength + " to " + minified.length + " bytes");
-`;
+      chunk += `const char ${s.name}[] PROGMEM = R"${s.prepend || ""}${minified}${s.append || ""}";\n\n`;
-    return chunk;
+      return s.mangle ? s.mangle(chunk) : chunk;
-  } else if (s.method == "binary") {
-    const buf = fs.readFileSync(srcDir + "/" + s.file);
-    const result = hexdump(buf);
-    const chunk = `
-// Autogenerated from ${srcDir}/${s.file}, do not edit!!
-const uint16_t ${s.name}_length = ${result.length};
-const uint8_t ${s.name}[] PROGMEM = {
-${result}
-};
-
-`;
-    return chunk;
-  } else {
-    console.warn("Unknown method: " + s.method);
-    return undefined;
-  }
-}
-
-function writeChunks(srcDir, specs, resultFile) {
-  let src = `/*
- * More web UI HTML source arrays.
- * This file is auto generated, please don't make any changes manually.
- * Instead, see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
- * to find out how to easily modify the web UI source!
- */ 
-`;
-  specs.forEach((s) => {
-    try {
-      console.info("Reading " + srcDir + "/" + s.file + " as " + s.name);
-      src += specToChunk(srcDir, s);
-    } catch (e) {
-      console.warn(
-        "Failed " + s.name + " from " + srcDir + "/" + s.file,
-        e.message.length > 60 ? e.message.substring(0, 60) : e.message
-      );
     }
-  });
+  } else if (s.method == "binary") {
+    const result = hexdump(buf);
+    chunk += `const uint16_t ${s.name}_length = ${buf.length};\n`;
+    chunk += `const uint8_t ${s.name}[] PROGMEM = {\n${result}\n};\n\n`;
+    return chunk;
+  }
+
+  throw new Error("Unknown method: " + s.method);
+}
+
+async function writeChunks(srcDir, specs, resultFile) {
+  let src = multiHeader;
+  for (const s of specs) {
+    console.info("Reading " + srcDir + "/" + s.file + " as " + s.name);
+    src += await specToChunk(srcDir, s);
+  }
   console.info("Writing " + src.length + " characters into " + resultFile);
   fs.writeFileSync(resultFile, src);
 }
 
-writeHtmlGzipped("wled00/data/index.htm", "wled00/html_ui.h", 'index');
+// Check if a file is newer than a given time
+function isFileNewerThan(filePath, time) {
+  const stats = fs.statSync(filePath);
+  return stats.mtimeMs > time;
+}
+
+// Check if any file in a folder (or its subfolders) is newer than a given time
+function isAnyFileInFolderNewerThan(folderPath, time) {
+  const files = fs.readdirSync(folderPath, { withFileTypes: true });
+  for (const file of files) {
+    const filePath = path.join(folderPath, file.name);
+    if (isFileNewerThan(filePath, time)) {
+      return true;
+    }
+    if (file.isDirectory() && isAnyFileInFolderNewerThan(filePath, time)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// Check if the web UI is already built
+function isAlreadyBuilt(webUIPath, packageJsonPath = "package.json") {
+  let lastBuildTime = Infinity;
+
+  for (const file of output) {
+    try {
+      lastBuildTime = Math.min(lastBuildTime, fs.statSync(file).mtimeMs);
+    } catch (e) {
+      if (e.code !== 'ENOENT') throw e;
+      console.info("File " + file + " does not exist. Rebuilding...");
+      return false;
+    }
+  }
+
+  return !isAnyFileInFolderNewerThan(webUIPath, lastBuildTime) && !isFileNewerThan(packageJsonPath, lastBuildTime) && !isFileNewerThan(__filename, lastBuildTime);
+}
+
+// Don't run this script if we're in a test environment
+if (process.env.NODE_ENV === 'test') {
+  return;
+}
+
+console.info(wledBanner);
+
+if (isAlreadyBuilt("wled00/data") && process.argv[2] !== '--force' && process.argv[2] !== '-f') {
+  console.info("Web UI is already built");
+  return;
+}
+
+writeHtmlGzipped("wled00/data/index.htm", "wled00/html_ui.h", 'index', false);
 writeHtmlGzipped("wled00/data/simple.htm", "wled00/html_simple.h", 'simple');
 writeHtmlGzipped("wled00/data/pixart/pixart.htm", "wled00/html_pixart.h", 'pixart');
+//writeHtmlGzipped("wled00/data/pxmagic/pxmagic.htm", "wled00/html_pxmagic.h", 'pxmagic');
+//writeHtmlGzipped("wled00/data/pixelforge/pixelforge.htm", "wled00/html_pixelforge.h", 'pixelforge', false); // do not inline css
 writeHtmlGzipped("wled00/data/cpal/cpal.htm", "wled00/html_cpal.h", 'cpal');
-writeHtmlGzipped("wled00/data/pxmagic/pxmagic.htm", "wled00/html_pxmagic.h", 'pxmagic');
+//writeHtmlGzipped("wled00/data/edit.htm", "wled00/html_edit.h", 'edit');
-/*
+
 writeChunks(
   "wled00/data",
   [
     {
-      file: "simple.css",
+      file: "edit.htm",
-      name: "PAGE_simpleCss",
+      name: "PAGE_edit",
       method: "gzip",
-      filter: "css-minify",
+      filter: "html-minify"
-    },
-    {
-      file: "simple.js",
-      name: "PAGE_simpleJs",
-      method: "gzip",
-      filter: "js-minify",
-    },
-    {
-      file: "simple.htm",
-      name: "PAGE_simple",
-      method: "gzip",
-      filter: "html-minify-ui",
     }
   ],
-  "wled00/html_simplex.h"
+  "wled00/html_edit.h"
+);
+
+/*
+writeChunks(
+  "wled00/data/cpal",
+  [
+    {
+      file: "cpal.htm",
+      name: "PAGE_cpal",
+      method: "gzip",
+      filter: "html-minify"
+    }
+  ],
+  "wled00/html_cpal.h"
 );
 */
+
 writeChunks(
   "wled00/data",
   [
@@ -274,8 +302,16 @@ writeChunks(
       filter: "css-minify",
       mangle: (str) =>
         str
-          .replace("%%","%")
+          .replace("%%", "%")
     },
+	/*
+    {
+      file: "common.js",
+      name: "JS_common",
+      method: "gzip",
+      filter: "js-minify",
+    },
+	*/
     {
       file: "settings.htm",
       name: "PAGE_settings",
@@ -430,6 +466,7 @@ const char PAGE_dmxmap[] PROGMEM = R"=====()=====";
       method: "gzip",
       filter: "html-minify",
     },
+    //WLEDMM
     {
       file: "404mini.htm",
       name: "PAGE_404_mini",
@@ -444,12 +481,14 @@ const char PAGE_dmxmap[] PROGMEM = R"=====()=====";
     {
       file: "iro.js",
       name: "iroJs",
-      method: "gzip"
+      method: "gzip",
+      filter: "js-minify",
     },
     {
       file: "rangetouch.js",
       name: "rangetouchJs",
-      method: "gzip"
+      method: "gzip",
+      filter: "js-minify"
     }
   ],
   "wled00/html_other.h"