upgrade HTML data build system - use same tools as in upstream WLED

align with tools/cdata.js from upstream WLED

--> prerequisite for pixelforge tool !

pio-scripts/build_ui.py

@@ -0,0 +1,21 @@
+Import("env")
+import shutil
+
+node_ex = shutil.which("node")
+# Check if Node.js is installed and present in PATH if it failed, abort the build
+if node_ex is None:
+    print('\x1b[0;31;43m' + 'Node.js is not installed or missing from PATH html css js will not be processed check https://kno.wled.ge/advanced/compiling-wled/' + '\x1b[0m')
+    exitCode = env.Execute("null")
+    exit(exitCode)
+else:
+    # Install the necessary node packages for the pre-build asset bundling script
+    print('\x1b[6;33;42m' + 'Installing node packages' + '\x1b[0m')
+    env.Execute("npm ci")
+
+    # Call the bundling script
+    exitCode = env.Execute("npm run build")
+
+    # If it failed, abort the build
+    if (exitCode):
+      print('\x1b[0;31;43m' + 'npm run build fails check https://kno.wled.ge/advanced/compiling-wled/' + '\x1b[0m')
+      exit(exitCode)

pio-scripts/set_metadata.py

@@ -0,0 +1,116 @@
+Import('env')
+import subprocess
+import json
+import re
+
+def get_github_repo():
+    """Extract GitHub repository name from git remote URL.
+    
+    Uses the remote that the current branch tracks, falling back to 'origin'.
+    This handles cases where repositories have multiple remotes or where the
+    main remote is not named 'origin'.
+    
+    Returns:
+        str: Repository name in 'owner/repo' format for GitHub repos,
+             'unknown' for non-GitHub repos, missing git CLI, or any errors.
+    """
+    try:
+        remote_name = 'origin'  # Default fallback
+        
+        # Try to get the remote for the current branch
+        try:
+            # Get current branch name
+            branch_result = subprocess.run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], 
+                                         capture_output=True, text=True, check=True)
+            current_branch = branch_result.stdout.strip()
+            
+            # Get the remote for the current branch
+            remote_result = subprocess.run(['git', 'config', f'branch.{current_branch}.remote'], 
+                                         capture_output=True, text=True, check=True)
+            tracked_remote = remote_result.stdout.strip()
+            
+            # Use the tracked remote if we found one
+            if tracked_remote:
+                remote_name = tracked_remote
+        except subprocess.CalledProcessError:
+            # If branch config lookup fails, continue with 'origin' as fallback
+            pass
+        
+        # Get the remote URL for the determined remote
+        result = subprocess.run(['git', 'remote', 'get-url', remote_name], 
+                              capture_output=True, text=True, check=True)
+        remote_url = result.stdout.strip()
+        
+        # Check if it's a GitHub URL
+        if 'github.com' not in remote_url.lower():
+            return None
+        
+        # Parse GitHub URL patterns:
+        # https://github.com/owner/repo.git
+        # git@github.com:owner/repo.git
+        # https://github.com/owner/repo
+        
+        # Remove .git suffix if present
+        if remote_url.endswith('.git'):
+            remote_url = remote_url[:-4]
+        
+        # Handle HTTPS URLs
+        https_match = re.search(r'github\.com/([^/]+/[^/]+)', remote_url, re.IGNORECASE)
+        if https_match:
+            return https_match.group(1)
+        
+        # Handle SSH URLs
+        ssh_match = re.search(r'github\.com:([^/]+/[^/]+)', remote_url, re.IGNORECASE)
+        if ssh_match:
+            return ssh_match.group(1)
+        
+        return None
+        
+    except FileNotFoundError:
+        # Git CLI is not installed or not in PATH
+        return None
+    except subprocess.CalledProcessError:
+        # Git command failed (e.g., not a git repo, no remote, etc.)
+        return None
+    except Exception:
+        # Any other unexpected error
+        return None
+
+# WLED version is managed by package.json; this is picked up in several places
+# - It's integrated in to the UI code
+# - Here, for wled_metadata.cpp
+# - The output_bins script
+# We always take it from package.json to ensure consistency
+with open("package.json", "r") as package:
+    WLED_VERSION = json.load(package)["version"]
+
+def has_def(cppdefs, name):
+    """ Returns true if a given name is set in a CPPDEFINES collection """
+    for f in cppdefs:
+        if isinstance(f, tuple):
+            f = f[0]
+        if f == name:
+            return True
+    return False
+
+
+def add_wled_metadata_flags(env, node):    
+    cdefs = env["CPPDEFINES"].copy()
+
+    if not has_def(cdefs, "WLED_REPO"):
+        repo = get_github_repo()
+        if repo:
+            cdefs.append(("WLED_REPO", f"\\\"{repo}\\\""))
+
+    cdefs.append(("WLED_VERSION", WLED_VERSION))
+
+    # This transforms the node in to a Builder; it cannot be modified again
+    return env.Object(
+        node,
+        CPPDEFINES=cdefs
+    )
+   
+env.AddBuildMiddleware(
+    add_wled_metadata_flags,
+    "*/wled_metadata.cpp"
+)

pio-scripts/validate_modules.py

@@ -0,0 +1,80 @@
+import re
+from pathlib import Path   # For OS-agnostic path manipulation
+from typing import Iterable
+from click import secho
+from SCons.Script import Action, Exit
+from platformio.builder.tools.piolib import LibBuilderBase
+
+
+def is_wled_module(env, dep: LibBuilderBase) -> bool:
+  """Returns true if the specified library is a wled module
+  """
+  usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
+  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
+
+
+def read_lines(p: Path):
+    """ Read in the contents of a file for analysis """
+    with p.open("r", encoding="utf-8", errors="ignore") as f:
+        return f.readlines()
+
+
+def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
+    """ Identify which dirs contributed to the final build
+
+        Returns the (sub)set of dirs that are found in the output ELF
+    """
+    # Pattern to match symbols in object directories
+    # Join directories into alternation
+    usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
+    # Matches nonzero address, any size, and any path in a matching directory
+    object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
+
+    found = set()
+    for line in map_file:
+        matches = object_path_regex.findall(line)
+        for m in matches:
+            found.add(m)
+    return found
+
+
+def count_usermod_objects(map_file: list[str]) -> int:
+    """ Returns the number of usermod objects in the usermod list """
+    # Count the number of entries in the usermods table section
+    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
+
+
+def validate_map_file(source, target, env):
+    """ Validate that all modules appear in the output build """
+    build_dir = Path(env.subst("$BUILD_DIR"))
+    map_file_path = build_dir /  env.subst("${PROGNAME}.map")
+
+    if not map_file_path.exists():
+        secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
+        Exit(1)
+
+    # Identify the WLED module builders, set by load_usermods.py
+    module_lib_builders = env['WLED_MODULES']
+
+    # Extract the values we care about
+    modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
+    secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
+
+    # Now parse the map file
+    map_file_contents = read_lines(map_file_path)
+    usermod_object_count = count_usermod_objects(map_file_contents)
+    secho(f"INFO: {usermod_object_count} usermod object entries")
+
+    confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
+    missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
+    if missing_modules:
+        secho(
+            f"ERROR: No object files from {missing_modules} found in linked output!",
+            fg="red",
+            err=True)
+        Exit(1)
+    return None
+
+Import("env")
+env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))