Jan/WLED_MM_Infinity
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

upgrade HTML data build system - use same tools as in upstream WLED

Browse Source 
align with tools/cdata.js from upstream WLED

--> prerequisite for pixelforge tool !
This commit is contained in:
Frank
2025-12-18 23:40:23 +01:00
parent 5deaf92373
commit 114bdb416f
8 changed files with 1065 additions and 3701 deletions
Download Patch File Download Diff File Expand all files Collapse all files

401
tools/cdata.js
View File

@@ -2,7 +2,7 @@
* Writes compressed C arrays of data files (web interface)
* How to use it?
*
* 1) Install Node 11+ and npm
* 1) Install Node 20+ and npm
* 2) npm install
* 3) npm run build
*
@@ -15,26 +15,70 @@
* It uses NodeJS packages to inline, minify and GZIP files. See writeHtmlGzipped and writeChunks invocations at the bottom of the page.
*/
const fs = require("fs");
const inliner = require("inliner");
const zlib = require("zlib");
const fs = require("node:fs");
const path = require("path");
const inline = require("web-resource-inliner");
const zlib = require("node:zlib");
const CleanCSS = require("clean-css");
const MinifyHTML = require("html-minifier-terser").minify;
const minifyHtml = require("html-minifier-terser").minify;
const packageJson = require("../package.json");
/**
*
// Export functions for testing
module.exports = { isFileNewerThan, isAnyFileInFolderNewerThan };
//const output = ["wled00/html_ui.h", "wled00/html_pixart.h", "wled00/html_cpal.h", "wled00/html_edit.h", "wled00/html_pxmagic.h", "wled00/html_pixelforge.h", "wled00/html_settings.h", "wled00/html_other.h"]
const output = ["wled00/html_ui.h", "wled00/html_pixart.h", "wled00/html_cpal.h", "wled00/html_edit.h", "wled00/html_pxmagic.h", "wled00/html_settings.h", "wled00/html_other.h"]
// \x1b[34m is blue, \x1b[36m is cyan, \x1b[0m is reset
const wledBanner = `
\t\x1b[34m ## ## ## ###### ######
\t\x1b[34m## ## ## ## ## ## ##
\t\x1b[34m## ## ## ## ###### ## ##
\t\x1b[34m## ## ## ## ## ## ##
\t\x1b[34m ## ## ###### ###### ######
\t\t\x1b[36m build script for web UI
\x1b[0m`;
// Generate build timestamp as UNIX timestamp (seconds since epoch)
function generateBuildTime() {
return Math.floor(Date.now() / 1000);
}
const singleHeader = `/*
* Binary array for the Web UI.
* gzip is used for smaller size and improved speeds.
*
* Please see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
* to find out how to easily modify the web UI source!
*/
function hexdump(buffer,isHex=false) {
// Automatically generated build time for cache busting (UNIX timestamp)
#ifdef WEB_BUILD_TIME
#undef WEB_BUILD_TIME
#endif
#define WEB_BUILD_TIME ${generateBuildTime()}
`;
const multiHeader = `/*
* More web UI HTML source arrays.
* This file is auto generated, please don't make any changes manually.
*
* Instead, see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
* to find out how to easily modify the web UI source!
*/
`;
function hexdump(buffer, isHex = false) {
let lines = [];
for (let i = 0; i < buffer.length; i +=(isHex?32:16)) {
for (let i = 0; i < buffer.length; i += (isHex ? 32 : 16)) {
var block;
let hexArray = [];
if (isHex) {
block = buffer.slice(i, i + 32)
for (let j = 0; j < block.length; j +=2 ) {
hexArray.push("0x" + block.slice(j,j+2))
for (let j = 0; j < block.length; j += 2) {
hexArray.push("0x" + block.slice(j, j + 2))
}
} else {
block = buffer.slice(i, i + 16); // cut buffer into blocks of 16
@@ -51,219 +95,203 @@ function hexdump(buffer,isHex=false) {
return lines.join(",\n");
}
function strReplace(str, search, replacement) {
return str.split(search).join(replacement);
}
function adoptVersionAndRepo(html) {
let repoUrl = packageJson.repository ? packageJson.repository.url : undefined;
if (repoUrl) {
repoUrl = repoUrl.replace(/^git\+/, "");
repoUrl = repoUrl.replace(/\.git$/, "");
// Replace we
html = strReplace(html, "https://github.com/atuline/WLED", repoUrl);
html = strReplace(html, "https://github.com/Aircoookie/WLED", repoUrl);
// html = strReplace(html, "https://github.com/wled-dev/WLED", repoUrl); // replacing upstream break "credits"
// html = strReplace(html, "https://github.com/wled/WLED", repoUrl);
// html = strReplace(html, "https://github.com/MoonModules/WLED", repoUrl); //WLEDMM
// html = strReplace(html, "https://github.com/MoonModules/WLED-MM", repoUrl); //WLEDMM - not necessary to replace ourselves ;-)
// html = html.replaceAll("https://github.com/wled-dev/WLED", repoUrl); // WLEDMM replacing upstream break "credits"
html = html.replaceAll("https://github.com/atuline/WLED", repoUrl);
html = html.replaceAll("https://github.com/Aircoookie/WLED", repoUrl);
}
let version = packageJson.version;
if (version) {
html = strReplace(html, "##VERSION##", version);
html = html.replaceAll("##VERSION##", version);
}
return html;
}
function filter(str, type) {
str = adoptVersionAndRepo(str);
if (type === undefined) {
async function minify(str, type = "plain") {
const options = {
collapseWhitespace: true,
conservativeCollapse: true, // preserve spaces in text
collapseBooleanAttributes: true,
collapseInlineTagWhitespace: true,
minifyCSS: true,
minifyJS: true,
removeAttributeQuotes: true,
removeComments: true,
sortAttributes: true,
sortClassName: true,
};
if (type == "plain") {
return str;
} else if (type == "css-minify") {
return new CleanCSS({}).minify(str).styles;
} else if (type == "js-minify") {
return MinifyHTML('<script>' + str + '</script>', {
collapseWhitespace: true,
minifyJS: true,
continueOnParseError: false,
removeComments: true,
}).replace(/<[\/]*script>/g,'');
let js = await minifyHtml('<script>' + str + '</script>', options);
return js.replace(/<[\/]*script>/g, '');
} else if (type == "html-minify") {
return MinifyHTML(str, {
collapseWhitespace: true,
maxLineLength: 80,
minifyCSS: true,
minifyJS: true,
continueOnParseError: false,
removeComments: true,
});
} else if (type == "html-minify-ui") {
return MinifyHTML(str, {
collapseWhitespace: true,
conservativeCollapse: true,
maxLineLength: 80,
minifyCSS: true,
minifyJS: true,
continueOnParseError: false,
removeComments: true,
});
} else {
console.warn("Unknown filter: " + type);
return str;
return await minifyHtml(str, options);
}
throw new Error("Unknown filter: " + type);
}
// Generate build timestamp as UNIX timestamp (seconds since epoch)
function generateBuildTime() {
return Math.floor(Date.now() / 1000);
}
function writeHtmlGzipped(sourceFile, resultFile, page) {
async function writeHtmlGzipped(sourceFile, resultFile, page, inlineCss = true) {
console.info("Reading " + sourceFile);
new inliner(sourceFile, function (error, html) {
console.info("Inlined " + html.length + " characters");
html = filter(html, "html-minify-ui");
console.info("Minified to " + html.length + " characters");
inline.html({
fileContent: fs.readFileSync(sourceFile, "utf8"),
relativeTo: path.dirname(sourceFile),
strict: inlineCss, // when not inlining css, ignore errors (enables linking style.css from subfolder htm files)
stylesheets: inlineCss // when true (default), css is inlined
},
async function (error, html) {
if (error) throw error;
if (error) {
console.warn(error);
throw error;
}
html = adoptVersionAndRepo(html);
zlib.gzip(html, { level: zlib.constants.Z_BEST_COMPRESSION }, function (error, result) {
if (error) {
console.warn(error);
throw error;
}
console.info("Compressed " + result.length + " bytes");
html = adoptVersionAndRepo(html);
const originalLength = html.length;
html = await minify(html, "html-minify");
const result = zlib.gzipSync(html, { level: zlib.constants.Z_BEST_COMPRESSION });
console.info("Minified and compressed " + sourceFile + " from " + originalLength + " to " + result.length + " bytes");
const array = hexdump(result);
const src = `/*
* Binary array for the Web UI.
* gzip is used for smaller size and improved speeds.
*
* Please see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
* to find out how to easily modify the web UI source!
*/
// Automatically generated build time for cache busting (UNIX timestamp)
#ifdef WEB_BUILD_TIME // avoid duplicate defintions
#undef WEB_BUILD_TIME
#endif
#define WEB_BUILD_TIME ${generateBuildTime()}
// Autogenerated from ${sourceFile}, do not edit!!
const uint16_t PAGE_${page}_L = ${result.length};
const uint8_t PAGE_${page}[] PROGMEM = {
${array}
};
`;
let src = singleHeader;
src += `const uint16_t PAGE_${page}_L = ${result.length};\n`;
src += `const uint8_t PAGE_${page}[] PROGMEM = {\n${array}\n};\n\n`;
console.info("Writing " + resultFile);
fs.writeFileSync(resultFile, src);
});
});
}
function specToChunk(srcDir, s) {
if (s.method == "plaintext") {
const buf = fs.readFileSync(srcDir + "/" + s.file);
const str = buf.toString("utf-8");
const chunk = `
// Autogenerated from ${srcDir}/${s.file}, do not edit!!
const char ${s.name}[] PROGMEM = R"${s.prepend || ""}${filter(str, s.filter)}${
s.append || ""
}";
async function specToChunk(srcDir, s) {
const buf = fs.readFileSync(srcDir + "/" + s.file);
let chunk = `\n// Autogenerated from ${srcDir}/${s.file}, do not edit!!\n`
`;
return s.mangle ? s.mangle(chunk) : chunk;
} else if (s.method == "gzip") {
const buf = fs.readFileSync(srcDir + "/" + s.file);
var str = buf.toString('utf-8');
if (s.mangle) str = s.mangle(str);
const zip = zlib.gzipSync(filter(str, s.filter), { level: zlib.constants.Z_BEST_COMPRESSION });
const result = hexdump(zip.toString('hex'), true);
const chunk = `
// Autogenerated from ${srcDir}/${s.file}, do not edit!!
const uint16_t ${s.name}_length = ${zip.length};
const uint8_t ${s.name}[] PROGMEM = {
${result}
};
`;
return chunk;
} else if (s.method == "binary") {
const buf = fs.readFileSync(srcDir + "/" + s.file);
const result = hexdump(buf);
const chunk = `
// Autogenerated from ${srcDir}/${s.file}, do not edit!!
const uint16_t ${s.name}_length = ${result.length};
const uint8_t ${s.name}[] PROGMEM = {
${result}
};
`;
return chunk;
} else {
console.warn("Unknown method: " + s.method);
return undefined;
}
}
function writeChunks(srcDir, specs, resultFile) {
let src = `/*
* More web UI HTML source arrays.
* This file is auto generated, please don't make any changes manually.
* Instead, see https://mm.kno.wled.ge/advanced/custom-features/#changing-web-ui
* to find out how to easily modify the web UI source!
*/
`;
specs.forEach((s) => {
try {
console.info("Reading " + srcDir + "/" + s.file + " as " + s.name);
src += specToChunk(srcDir, s);
} catch (e) {
console.warn(
"Failed " + s.name + " from " + srcDir + "/" + s.file,
e.message.length > 60 ? e.message.substring(0, 60) : e.message
);
if (s.method == "plaintext" || s.method == "gzip") {
let str = buf.toString("utf-8");
str = adoptVersionAndRepo(str);
const originalLength = str.length;
if (s.method == "gzip") {
if (s.mangle) str = s.mangle(str);
const zip = zlib.gzipSync(await minify(str, s.filter), { level: zlib.constants.Z_BEST_COMPRESSION });
console.info("Minified and compressed " + s.file + " from " + originalLength + " to " + zip.length + " bytes");
const result = hexdump(zip);
chunk += `const uint16_t ${s.name}_length = ${zip.length};\n`;
chunk += `const uint8_t ${s.name}[] PROGMEM = {\n${result}\n};\n\n`;
return chunk;
} else {
const minified = await minify(str, s.filter);
console.info("Minified " + s.file + " from " + originalLength + " to " + minified.length + " bytes");
chunk += `const char ${s.name}[] PROGMEM = R"${s.prepend || ""}${minified}${s.append || ""}";\n\n`;
return s.mangle ? s.mangle(chunk) : chunk;
}
});
} else if (s.method == "binary") {
const result = hexdump(buf);
chunk += `const uint16_t ${s.name}_length = ${buf.length};\n`;
chunk += `const uint8_t ${s.name}[] PROGMEM = {\n${result}\n};\n\n`;
return chunk;
}
throw new Error("Unknown method: " + s.method);
}
async function writeChunks(srcDir, specs, resultFile) {
let src = multiHeader;
for (const s of specs) {
console.info("Reading " + srcDir + "/" + s.file + " as " + s.name);
src += await specToChunk(srcDir, s);
}
console.info("Writing " + src.length + " characters into " + resultFile);
fs.writeFileSync(resultFile, src);
}
writeHtmlGzipped("wled00/data/index.htm", "wled00/html_ui.h", 'index');
// Check if a file is newer than a given time
function isFileNewerThan(filePath, time) {
const stats = fs.statSync(filePath);
return stats.mtimeMs > time;
}
// Check if any file in a folder (or its subfolders) is newer than a given time
function isAnyFileInFolderNewerThan(folderPath, time) {
const files = fs.readdirSync(folderPath, { withFileTypes: true });
for (const file of files) {
const filePath = path.join(folderPath, file.name);
if (isFileNewerThan(filePath, time)) {
return true;
}
if (file.isDirectory() && isAnyFileInFolderNewerThan(filePath, time)) {
return true;
}
}
return false;
}
// Check if the web UI is already built
function isAlreadyBuilt(webUIPath, packageJsonPath = "package.json") {
let lastBuildTime = Infinity;
for (const file of output) {
try {
lastBuildTime = Math.min(lastBuildTime, fs.statSync(file).mtimeMs);
} catch (e) {
if (e.code !== 'ENOENT') throw e;
console.info("File " + file + " does not exist. Rebuilding...");
return false;
}
}
return !isAnyFileInFolderNewerThan(webUIPath, lastBuildTime) && !isFileNewerThan(packageJsonPath, lastBuildTime) && !isFileNewerThan(__filename, lastBuildTime);
}
// Don't run this script if we're in a test environment
if (process.env.NODE_ENV === 'test') {
return;
}
console.info(wledBanner);
if (isAlreadyBuilt("wled00/data") && process.argv[2] !== '--force' && process.argv[2] !== '-f') {
console.info("Web UI is already built");
return;
}
writeHtmlGzipped("wled00/data/index.htm", "wled00/html_ui.h", 'index', false);
writeHtmlGzipped("wled00/data/simple.htm", "wled00/html_simple.h", 'simple');
writeHtmlGzipped("wled00/data/pixart/pixart.htm", "wled00/html_pixart.h", 'pixart');
//writeHtmlGzipped("wled00/data/pxmagic/pxmagic.htm", "wled00/html_pxmagic.h", 'pxmagic');
//writeHtmlGzipped("wled00/data/pixelforge/pixelforge.htm", "wled00/html_pixelforge.h", 'pixelforge', false); // do not inline css
writeHtmlGzipped("wled00/data/cpal/cpal.htm", "wled00/html_cpal.h", 'cpal');
writeHtmlGzipped("wled00/data/pxmagic/pxmagic.htm", "wled00/html_pxmagic.h", 'pxmagic');
/*
//writeHtmlGzipped("wled00/data/edit.htm", "wled00/html_edit.h", 'edit');
writeChunks(
"wled00/data",
[
{
file: "simple.css",
name: "PAGE_simpleCss",
file: "edit.htm",
name: "PAGE_edit",
method: "gzip",
filter: "css-minify",
},
{
file: "simple.js",
name: "PAGE_simpleJs",
method: "gzip",
filter: "js-minify",
},
{
file: "simple.htm",
name: "PAGE_simple",
method: "gzip",
filter: "html-minify-ui",
filter: "html-minify"
}
],
"wled00/html_simplex.h"
"wled00/html_edit.h"
);
/*
writeChunks(
"wled00/data/cpal",
[
{
file: "cpal.htm",
name: "PAGE_cpal",
method: "gzip",
filter: "html-minify"
}
],
"wled00/html_cpal.h"
);
*/
writeChunks(
"wled00/data",
[
@@ -274,8 +302,16 @@ writeChunks(
filter: "css-minify",
mangle: (str) =>
str
.replace("%%","%")
.replace("%%", "%")
},
/*
{
file: "common.js",
name: "JS_common",
method: "gzip",
filter: "js-minify",
},
*/
{
file: "settings.htm",
name: "PAGE_settings",
@@ -430,6 +466,7 @@ const char PAGE_dmxmap[] PROGMEM = R"=====()=====";
method: "gzip",
filter: "html-minify",
},
//WLEDMM
{
file: "404mini.htm",
name: "PAGE_404_mini",
@@ -444,12 +481,14 @@ const char PAGE_dmxmap[] PROGMEM = R"=====()=====";
{
file: "iro.js",
name: "iroJs",
method: "gzip"
method: "gzip",
filter: "js-minify",
},
{
file: "rangetouch.js",
name: "rangetouchJs",
method: "gzip"
method: "gzip",
filter: "js-minify"
}
],
"wled00/html_other.h"