FreeCADWasmDev/tools/generate_web_dependency_closure.py

#!/usr/bin/env python3
"""Generate a source-backed dependency closure for the first Web kernel slice.

The report deliberately keeps CMake conditions and unresolved variables visible.
It is an inventory of the current checkout, not a claim that the targets already
compile under Emscripten.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import re
import subprocess
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


SOURCE_EXTENSIONS = {
    ".c",
    ".cc",
    ".cpp",
    ".cxx",
    ".h",
    ".hh",
    ".hpp",
    ".hxx",
    ".inl",
    ".l",
    ".py",
    ".pyi",
    ".qrc",
    ".xsd",
    ".y",
}

TARGETS: dict[str, dict[str, Any]] = {
    "FreeCADBase": {
        "cmake": "src/Base/CMakeLists.txt",
        "source_root": "src/Base",
        "internal": [],
        "packages": [
            {"name": "Boost", "evidence": "${Boost_LIBRARIES}", "condition": "always"},
            {"name": "Xerces-C", "evidence": "${XercesC_LIBRARIES}", "condition": "always"},
            {"name": "zlib", "evidence": "${ZLIB_LIBRARIES}", "condition": "always"},
            {"name": "ICU", "evidence": "${ICU_LIBRARIES}", "condition": "always"},
            {"name": "Qt5::Core or Qt6::Core", "evidence": "${QtCore_LIBRARIES}", "condition": "always"},
            {"name": "libfastsignals", "evidence": "libfastsignals", "condition": "always; in-tree"},
            {"name": "fmt", "evidence": "fmt::fmt", "condition": "always"},
            {"name": "Python3", "evidence": "${Python3_LIBRARIES}", "condition": "BUILD_DYNAMIC_LINK_PYTHON"},
            {"name": "PyCXX", "evidence": "${PYCXX_SOURCES}; ${PYCXX_INCLUDE_DIR}", "condition": "always; source/include"},
            {"name": "Zipios++", "evidence": "${zipios_SRCS} or ${ZIPIOS_LIBRARY}", "condition": "FREECAD_USE_EXTERNAL_ZIPIOS selects link vs in-tree"},
            {"name": "POSIX libutil/libdl", "evidence": "-lutil -ldl", "condition": "CMAKE_SYSTEM_NAME == Linux"},
        ],
        "conditions": ["BUILD_GUI is not required by this target", "Python bindings are compiled in this target"],
    },
    "FreeCADApp": {
        "cmake": "src/App/CMakeLists.txt",
        "source_root": "src/App",
        "internal": ["FreeCADBase"],
        "packages": [
            {"name": "Boost", "evidence": "${Boost_LIBRARIES}", "condition": "always"},
            {"name": "fmt", "evidence": "fmt::fmt", "condition": "always"},
            {"name": "Qt5::Core or Qt6::Core", "evidence": "${QtCore_LIBRARIES}", "condition": "always"},
            {"name": "Qt5::Xml or Qt6::Xml", "evidence": "${QtXml_LIBRARIES}", "condition": "always"},
        ],
        "conditions": ["FreeCADBase must be built", "Python bindings are compiled in this target"],
    },
    "Materials": {
        "cmake": "src/Mod/Material/App/CMakeLists.txt",
        "source_root": "src/Mod/Material/App",
        "internal": ["FreeCADApp"],
        "packages": [
            {"name": "Qt5::Concurrent or Qt6::Concurrent", "evidence": "${QtConcurrent_LIBRARIES}", "condition": "always"},
            {"name": "yaml-cpp", "evidence": "${YAML_CPP_LIBRARIES} or yaml-cpp::yaml-cpp", "condition": "always"},
        ],
        "conditions": ["Built when BUILD_MATERIAL is enabled", "Material external manager is optional"],
    },
    "Part": {
        "cmake": "src/Mod/Part/App/CMakeLists.txt",
        "source_root": "src/Mod/Part/App",
        "internal": ["FreeCADApp", "Materials"],
        "packages": [
            {"name": "OpenCASCADE", "evidence": "${OCC_LIBRARIES}; ${OCC_DEBUG_LIBRARIES}", "condition": "always"},
            {"name": "FreeType", "evidence": "${FREETYPE_LIBRARIES}", "condition": "FREETYPE_FOUND"},
            {"name": "HarfBuzz", "evidence": "harfbuzz::harfbuzz", "condition": "FREETYPE_FOUND"},
        ],
        "conditions": ["BUILD_PART", "Materials target must be available"],
    },
    "PartDesign": {
        "cmake": "src/Mod/PartDesign/App/CMakeLists.txt",
        "source_root": "src/Mod/PartDesign/App",
        "internal": ["Part", "FreeCADApp"],
        "packages": [
            {"name": "OpenCASCADE", "evidence": "${OCC_LIBRARIES}; ${OCC_DEBUG_LIBRARIES}", "condition": "always"},
            {"name": "nlohmann-json", "evidence": "src/3rdParty/json include", "condition": "header-only; always"},
        ],
        "conditions": ["BUILD_PART_DESIGN", "Part target must be available"],
    },
    "Sketcher": {
        "cmake": "src/Mod/Sketcher/App/CMakeLists.txt",
        "source_root": "src/Mod/Sketcher/App",
        "internal": ["Part", "FreeCADApp"],
        "packages": [
            {"name": "Eigen3", "evidence": "${EIGEN3_INCLUDE_DIR}", "condition": "EIGEN3_FOUND"},
            {"name": "nlohmann-json", "evidence": "src/3rdParty/json include", "condition": "header-only; always"},
        ],
        "conditions": ["BUILD_SKETCHER", "EIGEN3_FOUND is required by src/Mod/CMakeLists.txt", "Part target must be available"],
    },
}

PACKAGE_ALIASES = {
    "Boost": "Boost",
    "Xerces-C": "Xerces-C",
    "zlib": "zlib",
    "ICU": "ICU",
    "Qt5::Core or Qt6::Core": "Qt",
    "Qt5::Xml or Qt6::Xml": "Qt",
    "Qt5::Concurrent or Qt6::Concurrent": "Qt",
    "OpenCASCADE": "Open CASCADE",
    "FreeType": "FreeType",
    "Eigen3": "Eigen",
    "Python3": "Python",
    "PyCXX": "PyCXX",
    "Zipios++": "Zipios++",
    "yaml-cpp": "yaml-cpp",
}

INCLUDE_FAMILIES = (
    ("OpenCASCADE", ("BRep", "ChFi", "Geom", "gp.", "GProp", "HLR", "IGES", "Interface_", "Message", "NCollection", "STEP", "Standard", "TCol", "TopAbs", "TopExp", "TopoDS", "Transfer", "UnitsMethods", "XSControl")),
    ("Eigen3", ("Eigen/",)),
    ("Boost", ("boost/",)),
    ("Xerces-C", ("xercesc/",)),
    ("Qt", ("Qt", "Q")),
    ("Python", ("Python.h", "python", "structmember.h")),
    ("PyCXX", ("CXX/", "CXX_")),
    ("yaml-cpp", ("yaml-cpp/",)),
    ("fmt", ("fmt/",)),
    ("nlohmann-json", ("nlohmann/",)),
    ("Zipios++", ("zipios",)),
    ("ICU", ("unicode/",)),
)


def run(*args: str) -> str | None:
    try:
        result = subprocess.run(args, check=False, capture_output=True, text=True)
    except OSError:
        return None
    if result.returncode != 0:
        return None
    return result.stdout.strip()


def read_text(path: Path) -> str:
    return path.read_text(encoding="utf-8", errors="replace")


def source_files(root: Path) -> list[Path]:
    return sorted(path for path in root.rglob("*") if path.is_file() and path.suffix in SOURCE_EXTENSIONS)


def classify_include(include: str, delimiter: str, source_file: Path, source_root: Path, source_tree: Path) -> str:
    if delimiter == '"':
        candidates = [source_file.parent / include, source_root / include, source_tree / include]
        if any(candidate.is_file() for candidate in candidates):
            return "FreeCAD source"
    for family, prefixes in INCLUDE_FAMILIES:
        if include.startswith(prefixes):
            return family
    if include == "json.hpp":
        return "nlohmann-json"
    if delimiter == "<":
        return "unresolved system header"
    return "unresolved quoted header"


def include_evidence(files: list[Path], source_root: Path, source_tree: Path) -> dict[str, Any]:
    counts: Counter[str] = Counter()
    files_by_family: dict[str, set[str]] = defaultdict(set)
    unresolved: Counter[str] = Counter()
    total = 0
    pattern = re.compile(r'^\s*#\s*include\s*([<"])([^>"]+)[>"]', re.MULTILINE)
    for path in files:
        text = read_text(path)
        for delimiter, include in pattern.findall(text):
            total += 1
            family = classify_include(include.strip(), delimiter, path, source_root, source_tree)
            counts[family] += 1
            rel = path.relative_to(source_tree).as_posix()
            files_by_family[family].add(rel)
            if family.startswith("unresolved"):
                unresolved[include.strip()] += 1
    return {
        "total_include_directives": total,
        "by_family": dict(sorted(counts.items())),
        "files_by_family": {key: sorted(value) for key, value in sorted(files_by_family.items())},
        "unresolved_headers": [
            {"header": name, "count": count} for name, count in unresolved.most_common()
        ],
    }


def cmake_evidence(path: Path) -> dict[str, Any]:
    text = read_text(path)
    variables = sorted(set(re.findall(r"\$\{([^}]+)\}", text)))
    commands = sorted(set(match.group(1) for match in re.finditer(r"\b(target_[a-z_]+|add_library|add_dependencies|target_link_directories)\s*\(", text)))
    return {
        "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
        "variable_references": variables,
        "commands": commands,
    }


def parse_lock(path: Path, archive_root: Path, hash_archives: bool) -> list[dict[str, Any]]:
    rows: list[dict[str, Any]] = []
    with path.open(encoding="utf-8", errors="replace", newline="") as handle:
        headers = handle.readline().rstrip("\n").split("\t")
        for line in handle:
            values = line.rstrip("\n").split("\t")
            if len(values) != len(headers):
                continue
            row = dict(zip(headers, values))
            archive = row.get("archive", "")
            archive_path = archive_root / archive if archive and archive != "-" else None
            item: dict[str, Any] = {
                "project": row.get("project", ""),
                "requested_version": row.get("requested_version", ""),
                "resolved_version": row.get("resolved_version", ""),
                "type": row.get("type", ""),
                "archive": archive,
                "directory": row.get("directory", ""),
                "note": row.get("note", ""),
                "archive_present": bool(archive_path and archive_path.is_file()),
                "archive_size": archive_path.stat().st_size if archive_path and archive_path.is_file() else None,
                "archive_sha256": None,
                "sha256_status": "not_requested" if archive_path else "not_applicable",
            }
            if archive_path and archive_path.is_file() and hash_archives:
                digest = hashlib.sha256()
                with archive_path.open("rb") as archive_handle:
                    for chunk in iter(lambda: archive_handle.read(1024 * 1024), b""):
                        digest.update(chunk)
                item["archive_sha256"] = digest.hexdigest()
                item["sha256_status"] = "computed"
            rows.append(item)
    return rows


def transitive_targets(name: str) -> list[str]:
    seen: set[str] = set()
    ordered: list[str] = []

    def visit(current: str) -> None:
        if current in seen:
            return
        seen.add(current)
        ordered.append(current)
        for dependency in TARGETS[current]["internal"]:
            visit(dependency)

    visit(name)
    return ordered


def transitive_packages(name: str) -> list[str]:
    return sorted(
        {
            requirement["name"]
            for target_name in transitive_targets(name)
            for requirement in TARGETS[target_name]["packages"]
        }
    )


def package_closure() -> list[dict[str, Any]]:
    packages: dict[str, dict[str, Any]] = {}
    for target_name, target in TARGETS.items():
        for requirement in target["packages"]:
            name = requirement["name"]
            entry = packages.setdefault(name, {"name": name, "used_by": [], "evidence": [], "conditions": []})
            entry["used_by"].append(target_name)
            entry["evidence"].append(requirement["evidence"])
            entry["conditions"].append(requirement["condition"])
    for entry in packages.values():
        entry["used_by"] = sorted(set(entry["used_by"]))
        entry["evidence"] = sorted(set(entry["evidence"]))
        entry["conditions"] = sorted(set(entry["conditions"]))
    return [packages[key] for key in sorted(packages)]


def relative(path: Path, root: Path) -> str:
    return path.relative_to(root).as_posix()


def build_report(source_root: Path, lock_path: Path, hash_archives: bool) -> dict[str, Any]:
    workspace_root = source_root.parent.parent
    source_tree = source_root / "src"
    archive_root = lock_path.parent / "sources" / "archives"
    lock_rows = parse_lock(lock_path, archive_root, hash_archives)
    lock_by_project = {row["project"]: row for row in lock_rows}
    target_rows: list[dict[str, Any]] = []
    all_source_files: set[str] = set()

    for name, target in TARGETS.items():
        cmake_path = source_root / target["cmake"]
        root = source_root / target["source_root"]
        files = source_files(root)
        all_source_files.update(relative(path, source_root) for path in files)
        extension_counts = Counter(path.suffix or "<none>" for path in files)
        target_rows.append(
            {
                "name": name,
                "cmake_file": relative(cmake_path, source_root),
                "cmake_file_exists": cmake_path.is_file(),
                "source_root": relative(root, source_root),
                "source_files": [relative(path, source_root) for path in files],
                "source_file_count": len(files),
                "source_file_count_by_extension": dict(sorted(extension_counts.items())),
                "cmake": cmake_evidence(cmake_path) if cmake_path.is_file() else None,
                "direct_internal_targets": sorted(target["internal"]),
                "direct_packages": target["packages"],
                "conditions": target["conditions"],
                "include_evidence": include_evidence(files, root, source_tree),
                "transitive_targets": transitive_targets(name),
                "transitive_external_packages": transitive_packages(name),
            }
        )

    relevant_projects = sorted(
        {
            PACKAGE_ALIASES[requirement["name"]]
            for target in TARGETS.values()
            for requirement in target["packages"]
            if requirement["name"] in PACKAGE_ALIASES
        }
    )
    locked = []
    for project in relevant_projects:
        if project in lock_by_project:
            locked.append(lock_by_project[project])

    git_head = run("git", "-C", str(source_root), "rev-parse", "HEAD")
    git_status = run("git", "-C", str(source_root), "status", "--short") or ""
    version_text = read_text(source_root / "CMakeLists.txt")
    version = ".".join(re.search(rf'set\(PACKAGE_VERSION_{part}\s+"([^"]+)"\)', version_text).group(1) for part in ("MAJOR", "MINOR", "PATCH"))
    emcc_version = run("emcc", "--version")
    cmake_version = run("cmake", "--version")
    report = {
        "schema": "freecad-web-dependency-closure/v1",
        "generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
        "scope": ["FreeCADBase", "FreeCADApp", "Part", "PartDesign", "Sketcher"],
        "source_baseline": {
            "source_root": str(source_root),
            "git_head": git_head,
            "git_status_porcelain": git_status.splitlines(),
            "package_version": version,
            "cmake_minimum_required": re.search(r"cmake_minimum_required\(VERSION ([^ )]+)", version_text).group(1),
            "cmake_version": cmake_version.splitlines()[0] if cmake_version else None,
            "emcc_version": emcc_version.splitlines()[0] if emcc_version else None,
        },
        "method": {
            "source_inventory": "all source/header/resource-like files below each target's App/Base source root",
            "include_scan": "preprocessor include directives in the inventory; unresolved/generated headers remain visible",
            "cmake_evidence": "target CMakeLists variable references and target commands, without evaluating conditions",
            "archive_sha256": "computed only with --hash-archives; null means no checksum was requested",
        },
        "targets": target_rows,
        "target_graph": {
            name: {
                "direct": sorted(target["internal"]),
                "transitive": transitive_targets(name),
                "transitive_external_packages": transitive_packages(name),
            }
            for name, target in TARGETS.items()
        },
        "root_closures": {
            name: {
                "targets": transitive_targets(name),
                "external_packages": transitive_packages(name),
            }
            for name in ("FreeCADBase", "FreeCADApp", "Part", "PartDesign", "Sketcher")
        },
        "external_packages": package_closure(),
        "source_lock": {
            "path": relative(lock_path, workspace_root),
            "relevant_rows": locked,
            "missing_from_lock": [project for project in relevant_projects if project not in lock_by_project],
        },
        "inventory_summary": {
            "target_count": len(TARGETS),
            "unique_inventory_files": len(all_source_files),
            "target_source_file_counts": {row["name"]: row["source_file_count"] for row in target_rows},
        },
        "known_limits": [
            "CMake conditionals, imported targets, generated sources, and platform-specific link resolution are not evaluated by this static scan.",
            "The source lock has no checksum column; archive_sha256 is null unless --hash-archives is used.",
            "A dependency in this report is not Web-compatible until an Emscripten configure/build probe succeeds.",
            "The report is tied to the recorded git checkout and must be regenerated after source or build-option changes.",
        ],
    }
    return report


def markdown(report: dict[str, Any]) -> str:
    baseline = report["source_baseline"]
    lines = [
        "# FreeCAD Web Dependency Closure",
        "",
        "This report is generated from the current FreeCAD checkout. It is an evidence inventory for the Web kernel P0 slice; it does not claim that the targets already compile under Emscripten or that Web behavior is feature-complete.",
        "",
        f"- Schema: `{report['schema']}`",
        f"- Generated: `{report['generated_at_utc']}`",
        f"- Source HEAD: `{baseline['git_head']}`",
        f"- FreeCAD package version: `{baseline['package_version']}`",
        f"- CMake: `{baseline['cmake_version'] or 'unavailable'}`; minimum `{baseline['cmake_minimum_required']}`",
        f"- Emscripten: `{baseline['emcc_version'] or 'unavailable'}`",
        "",
        "## Target Inventory",
        "",
        "| Target | CMake source root | Files | Direct internal targets |",
        "|---|---|---:|---|",
    ]
    for target in report["targets"]:
        lines.append(
            f"| `{target['name']}` | `{target['source_root']}` | {target['source_file_count']} | {', '.join(f'`{name}`' for name in target['direct_internal_targets']) or '-'} |"
        )
    lines.extend(["", "## Root Dependency Closure", ""])
    for name, closure in report["root_closures"].items():
        lines.append(f"### `{name}`")
        lines.append("")
        lines.append("- Targets: " + ", ".join(f"`{item}`" for item in closure["targets"]))
        lines.append("- External packages: " + ", ".join(f"`{item}`" for item in closure["external_packages"]))
        lines.append("")
    lines.extend(["", "`Materials` is included because `Part` links it, even though it is outside the five requested root targets.", "", "## External Package Closure", "", "| Package | Used by | Locked source | Conditions / caveat |", "|---|---|---|---|"])
    lock_rows = {row["project"]: row for row in report["source_lock"]["relevant_rows"]}
    for package in report["external_packages"]:
        lock_name = PACKAGE_ALIASES.get(package["name"])
        lock = lock_rows.get(lock_name) if lock_name else None
        locked = f"{lock['resolved_version']} (`{lock['archive']}`)" if lock else "not in sources.lock.tsv"
        conditions = "; ".join(package["conditions"])
        lines.append(f"| `{package['name']}` | {', '.join(f'`{name}`' for name in package['used_by'])} | {locked} | {conditions} |")
    lines.extend(["", "## Source Include Evidence", "", "The JSON report contains the full file lists and per-file include family evidence. The aggregate counts below show which external families are visible in the target source trees.", ""])
    for target in report["targets"]:
        families = target["include_evidence"]["by_family"]
        visible = ", ".join(f"`{name}`={count}" for name, count in families.items() if name != "FreeCAD source")
        lines.append(f"- `{target['name']}`: {visible or 'no classified external includes'}")
    lines.extend(["", "## Conditions and Limits", ""])
    for target in report["targets"]:
        lines.append(f"- `{target['name']}`: " + "; ".join(target["conditions"]) + ".")
    for limit in report["known_limits"]:
        lines.append(f"- {limit}")
    lines.extend(["", "## Reproduction", "", "```bash", "python3 FreeCAD/tools/generate_web_dependency_closure.py", "python3 FreeCAD/tools/generate_web_dependency_closure.py --hash-archives", "```", "", "The second command is the full checkpoint command. It reads the existing local archives and does not download anything. The first command is a faster inventory refresh that leaves archive checksums null.", ""])
    return "\n".join(lines)


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source-root", type=Path, help="FreeCAD source checkout")
    parser.add_argument("--output-dir", type=Path, help="directory for dependency-closure.json/.md")
    parser.add_argument("--hash-archives", action="store_true", help="compute SHA-256 for local source archives")
    args = parser.parse_args()

    script_root = Path(__file__).resolve().parents[1]
    source_root = (args.source_root or script_root / "freecad-source").resolve()
    output_dir = (args.output_dir or script_root / "docs").resolve()
    lock_path = script_root / "FreeCADBaseLib" / "sources.lock.tsv"
    if not source_root.is_dir():
        parser.error(f"source root does not exist: {source_root}")
    if not lock_path.is_file():
        parser.error(f"source lock does not exist: {lock_path}")
    output_dir.mkdir(parents=True, exist_ok=True)
    report = build_report(source_root, lock_path, args.hash_archives)
    json_path = output_dir / "dependency-closure.json"
    markdown_path = output_dir / "dependency-closure.md"
    json_path.write_text(json.dumps(report, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
    markdown_path.write_text(markdown(report), encoding="utf-8")
    print(f"wrote {json_path}")
    print(f"wrote {markdown_path}")
    print(f"targets={len(report['targets'])} unique_inventory_files={report['inventory_summary']['unique_inventory_files']}")
    return 0


if __name__ == "__main__":
    sys.exit(main())