SchOS/tools/python/stats.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later
"""The project's numbers, measured from the tree and written where they are shown.

    python3 tools/python/stats.py          rewrite README.md and the website
    python3 tools/python/stats.py --print  only print them

Both places carry the numbers between markers -- <!-- stats:begin --> and
<!-- stats:end --> -- and everything between them is regenerated. A number in
the README that somebody typed is a number that was true once; these are true
on the commit that ran this.

First-party code only: ports/ is other people's software, build/ is output,
and docs/Website/niveus is a vendored design system. Comment lines are counted
apart from code, because in this tree they are a large part of the work.
"""

import datetime
import os
import re
import subprocess
import sys

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

FIRST_PARTY = ["kernel", "include", "boot", "lib", "cmd", "tools", "ldls", "etc"]
LANGS = {
    ".c": "C", ".h": "C", ".S": "Assembly", ".s": "Assembly", ".asm": "Assembly",
    ".py": "Python", ".sh": "Shell", ".ps1": "PowerShell", ".pl": "Perl",
    ".ld": "Linker script", ".inc": "C",
}
GENERATED = re.compile(r"include/gfx/(logo|font)[^/]*\.h$|_data\.h$")


def git(*args):
    return subprocess.run(["git", "-C", ROOT, *args], capture_output=True,
                          text=True, check=True).stdout


def tracked():
    return [p for p in git("ls-files").splitlines()
            if p.split("/")[0] in FIRST_PARTY and not GENERATED.search(p)]


def count(path, lang):
    """(code, comment, blank) lines. Block comments are followed for C-family
    files; for the scripting languages a line starting with # is a comment."""
    code = comment = blank = 0
    in_block = False
    try:
        with open(os.path.join(ROOT, path), encoding="utf-8", errors="replace") as f:
            lines = f.read().splitlines()
    except OSError:
        return 0, 0, 0
    hash_comments = lang in ("Python", "Shell", "PowerShell", "Perl")
    for raw in lines:
        s = raw.strip()
        if not s:
            blank += 1
            continue
        if hash_comments:
            if s.startswith("#") and not s.startswith("#!"):
                comment += 1
            else:
                code += 1
            continue
        if in_block:
            comment += 1
            if "*/" in s:
                in_block = False
                if s.split("*/", 1)[1].strip():
                    comment -= 1
                    code += 1
            continue
        if s.startswith("//") or (lang == "Assembly" and s.startswith("#") and
                                  not re.match(r"#\s*(include|define|if|endif|else|elif|undef)", s)):
            comment += 1
        elif s.startswith("/*"):
            comment += 1
            if "*/" not in s[2:]:
                in_block = True
        else:
            code += 1
    return code, comment, blank


def measure():
    by_lang = {}
    kernel = [0, 0]
    files = 0
    for p in tracked():
        ext = os.path.splitext(p)[1]
        lang = LANGS.get(ext)
        if lang is None:
            continue
        c, m, _ = count(p, lang)
        files += 1
        t = by_lang.setdefault(lang, [0, 0])
        t[0] += c
        t[1] += m
        if p.startswith(("kernel/", "include/")):
            kernel[0] += c
            kernel[1] += m

    table = open(os.path.join(ROOT, "kernel/sys/table.inc")).read()
    calls = re.findall(r"^SYSCALL\(\s*\d+,\s*(\w+),", table, re.M)
    linux = [n for n in calls if n != "NO_LINUX"]

    drivers = set()
    for p in tracked():
        if p.startswith("kernel/drivers/") and p.endswith(".c"):
            parts = p.split("/")
            drivers.add("/".join(parts[2:4]) if len(parts) > 4 else parts[-1])
    drivers.add("i2c-hid")          # kernel/acpi/i2c_hid.c, found through ACPI

    programs = {p.split("/")[1] for p in tracked() if p.startswith("cmd/") and p.count("/") >= 2}

    world = [l for l in open(os.path.join(ROOT, "ports/alpine/world")).read().splitlines()
             if l.strip() and not l.startswith("#")]
    lock = [l for l in open(os.path.join(ROOT, "ports/alpine/world.lock")).read().splitlines()
            if l.strip() and not l.startswith("#")]

    commits = int(git("rev-list", "--count", "HEAD").strip())
    first = git("log", "--reverse", "--format=%ad", "--date=short").splitlines()[0]
    days = (datetime.date.today() - datetime.date.fromisoformat(first)).days

    code = sum(v[0] for v in by_lang.values())
    comments = sum(v[1] for v in by_lang.values())
    return {
        "code": code, "comments": comments, "files": files,
        "by_lang": sorted(by_lang.items(), key=lambda kv: -kv[1][0]),
        "kernel_code": kernel[0], "kernel_comments": kernel[1],
        "syscalls": len(calls), "linux_syscalls": len(linux),
        "drivers": len(drivers), "programs": len(programs),
        "world": len(world), "packages": len(lock),
        "commits": commits, "first": first, "days": days,
    }


def n(v):
    return f"{v:,}"


def readme_block(s):
    ratio = 100 * s["comments"] / max(1, s["code"] + s["comments"])
    rows = "\n".join(f"| {lang} | {n(c)} | {n(m)} |" for lang, (c, m) in s["by_lang"])
    return f"""<!-- stats:begin -->
| | |
|---|---|
| **First-party code** | {n(s['code'])} lines, plus {n(s['comments'])} lines of comments ({ratio:.0f}%) in {n(s['files'])} files |
| **Kernel** (`kernel/`, `include/`) | {n(s['kernel_code'])} lines of code, {n(s['kernel_comments'])} of comments |
| **Syscalls** | {s['syscalls']}, of which {s['linux_syscalls']} on Linux numbering |
| **Drivers** | {s['drivers']} in the kernel |
| **First-party programs** | {s['programs']} in `cmd/` |
| **Alpine packages in the root** | {s['packages']}, from {s['world']} asked for in `ports/alpine/world` |
| **History** | {n(s['commits'])} commits since {s['first']} ({s['days']} days) |

<details>
<summary>Lines by language</summary>

| Language | Code | Comments |
|---|--:|--:|
{rows}

</details>

<sub>Measured by <code>tools/python/stats.py</code>; first-party code only, so
<code>ports/</code> and generated tables are not counted.</sub>
<!-- stats:end -->"""


def site_block(s):
    items = [
        (f"lines of code in <code class=\"nv-code\">kernel/</code> and <code class=\"nv-code\">include/</code>", n(s["kernel_code"])),
        ("lines of comments alongside them", n(s["kernel_comments"])),
        ("first-party lines in all", n(s["code"])),
        ("syscalls, Linux numbering", str(s["linux_syscalls"])),
        ("drivers in the kernel", str(s["drivers"])),
        ("Alpine packages in the live root", str(s["packages"])),
        ("commits", n(s["commits"])),
        ("days since the first one", str(s["days"])),
        ("patches carried against fastfetch", "0"),
    ]
    out = ["<!-- stats:begin -->", "  <dl class=\"nv-stat-group\">"]
    for label, value in items:
        out += ["    <div class=\"nv-stat\">",
                f"      <dt class=\"nv-stat__label\">{label}</dt>",
                f"      <dd class=\"nv-stat__value\">{value}</dd>",
                "    </div>"]
    out += ["  </dl>", "  <!-- stats:end -->"]
    return "\n".join(out)


def rewrite(path, block):
    full = os.path.join(ROOT, path)
    text = open(full, encoding="utf-8").read()
    new, k = re.subn(r"<!-- stats:begin -->.*?<!-- stats:end -->", lambda _: block,
                     text, flags=re.S)
    if k != 1:
        sys.exit(f"stats: {path} needs exactly one stats:begin/stats:end pair")
    if new != text:
        open(full, "w", encoding="utf-8").write(new)
        print(f"stats: rewrote {path}")


def main():
    s = measure()
    if "--print" in sys.argv:
        for k, v in s.items():
            print(f"{k}: {v}")
        return
    rewrite("README.md", readme_block(s))
    rewrite("docs/Website/index.html", site_block(s))


if __name__ == "__main__":
    main()