a8caf27c14
Five weeks of forge evidence per week rather than a rolling window, so a finding survives under one key and "closed after three weeks" is computable. Rules run deterministically; agent.py hands the packet to whoever writes the prose and never calls a model itself. Model choice fails closed — no default, no substitute, and the exact named model must be served or the pass refuses to run.
475 lines
18 KiB
Python
475 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Weekly evidence for the project-state pass. Deterministic; no model runs here.
|
|
|
|
Reads the forge for one ISO week — commits, Actions runs, issues, pulls,
|
|
releases — and the local SouveraineOS clone for what the task index and the SAF
|
|
declared, then writes project-state/evidence/<week>.json. Every row carries a
|
|
stable id; a finding cites those ids and nothing else.
|
|
|
|
tools/project-state/collect.py --week last
|
|
tools/project-state/collect.py --week 2026-W33 --out -
|
|
|
|
Env: GITEA_URL, GITEA_TOKEN (falls back to ~/.git-credentials).
|
|
See docs/tasks/79-project-state.md.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import common as c
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
VERSION = 1
|
|
|
|
|
|
def commits(forge, repo, start, end, note):
|
|
"""Commits in the window, every branch, deduped by SHA.
|
|
|
|
A branch whose tip predates the window cannot hold work inside it, so it is
|
|
never walked — the same prefilter the ledger's collector uses.
|
|
"""
|
|
full = f"{c.OWNER}/{repo}"
|
|
edge = c.iso_z(start)
|
|
rows, seen = [], set()
|
|
try:
|
|
branches = forge.pages(f"repos/{full}/branches")
|
|
except urllib.error.HTTPError as e:
|
|
note(f"{repo}: branches unavailable ({e.code})")
|
|
return []
|
|
live = [b["name"] for b in branches
|
|
if ((b.get("commit") or {}).get("timestamp") or "") >= edge]
|
|
for branch in live:
|
|
path = (f"repos/{full}/commits?sha={branch}&since={edge}&until={c.iso_z(end)}"
|
|
f"&stat=false&verification=false&files=false")
|
|
try:
|
|
page = forge.pages(path)
|
|
except urllib.error.HTTPError as e:
|
|
note(f"{repo}@{branch}: commits unavailable ({e.code})")
|
|
continue
|
|
for row in page:
|
|
sha = row.get("sha", "")
|
|
if sha in seen or len((row.get("parents") or [])) > 1:
|
|
continue
|
|
seen.add(sha)
|
|
commit = row.get("commit") or {}
|
|
author = commit.get("author") or {}
|
|
subject, _, body = (commit.get("message") or "").partition("\n")
|
|
paths = c.commit_paths(repo, sha)
|
|
dimension, basis = c.classify(repo, subject.strip(), paths)
|
|
rows.append({
|
|
"id": f"c:{repo}:{sha[:8]}",
|
|
"repo": repo,
|
|
"sha": sha,
|
|
"branch": branch,
|
|
"at": author.get("date", ""),
|
|
"author": author.get("email", ""),
|
|
"subject": subject.strip(),
|
|
"detail": re.split(r"\n\s*\n", body.strip(), maxsplit=1)[0].strip(),
|
|
"dimension": dimension,
|
|
"basis": basis,
|
|
"paths": paths[:40],
|
|
"url": row.get("html_url", ""),
|
|
})
|
|
return rows
|
|
|
|
|
|
def runs(forge, repo, start, end, note):
|
|
"""Actions runs started inside the window.
|
|
|
|
This endpoint ignores limit and page and hands back the whole history, so it
|
|
is fetched once and sliced here — paging it would loop on the same rows.
|
|
"""
|
|
try:
|
|
payload = forge.get(f"repos/{c.OWNER}/{repo}/actions/runs")
|
|
except urllib.error.HTTPError as e:
|
|
if e.code != 404:
|
|
note(f"{repo}: actions unavailable ({e.code})")
|
|
return []
|
|
rows = []
|
|
for run in payload.get("workflow_runs", []) if isinstance(payload, dict) else payload:
|
|
started = c.parse_iso(run.get("started_at") or run.get("run_started_at"))
|
|
if not started or not (start <= started < end):
|
|
continue
|
|
rows.append({
|
|
"id": f"ci:{repo}:{run.get('id')}",
|
|
"repo": repo,
|
|
"run": run.get("id"),
|
|
"workflow": os.path.basename(run.get("path") or ""),
|
|
"branch": run.get("head_branch", ""),
|
|
"sha": run.get("head_sha", ""),
|
|
"commit": f"c:{repo}:{(run.get('head_sha') or '')[:8]}",
|
|
"conclusion": run.get("conclusion") or run.get("status") or "",
|
|
"started": run.get("started_at", ""),
|
|
"completed": run.get("completed_at", ""),
|
|
"title": run.get("display_title", ""),
|
|
})
|
|
return rows
|
|
|
|
|
|
def issues(forge, repo, start, end, note):
|
|
"""Every issue, open or closed — aging needs the ones that predate the week."""
|
|
try:
|
|
rows = forge.pages(f"repos/{c.OWNER}/{repo}/issues?state=all&type=issues")
|
|
except urllib.error.HTTPError as e:
|
|
if e.code != 404:
|
|
note(f"{repo}: issues unavailable ({e.code})")
|
|
return []
|
|
out = []
|
|
for i in rows:
|
|
created, closed = c.parse_iso(i.get("created_at")), c.parse_iso(i.get("closed_at"))
|
|
out.append({
|
|
"id": f"i:{repo}:{i.get('number')}",
|
|
"repo": repo,
|
|
"number": i.get("number"),
|
|
"title": i.get("title", ""),
|
|
"state": i.get("state", ""),
|
|
"labels": [l.get("name") for l in (i.get("labels") or [])],
|
|
"created": i.get("created_at", ""),
|
|
"closed": i.get("closed_at"),
|
|
"age_days": (end - created).days if created else None,
|
|
"opened_this_week": bool(created and start <= created < end),
|
|
"closed_this_week": bool(closed and start <= closed < end),
|
|
"url": i.get("html_url", ""),
|
|
})
|
|
return out
|
|
|
|
|
|
def pulls(forge, repo, start, end, note):
|
|
try:
|
|
rows = forge.pages(f"repos/{c.OWNER}/{repo}/pulls?state=all")
|
|
except urllib.error.HTTPError as e:
|
|
if e.code != 404:
|
|
note(f"{repo}: pulls unavailable ({e.code})")
|
|
return []
|
|
out = []
|
|
for p in rows:
|
|
created, merged = c.parse_iso(p.get("created_at")), c.parse_iso(p.get("merged_at"))
|
|
out.append({
|
|
"id": f"p:{repo}:{p.get('number')}",
|
|
"repo": repo,
|
|
"number": p.get("number"),
|
|
"title": p.get("title", ""),
|
|
"state": p.get("state", ""),
|
|
"created": p.get("created_at", ""),
|
|
"merged": p.get("merged_at"),
|
|
"age_days": (end - created).days if created else None,
|
|
"opened_this_week": bool(created and start <= created < end),
|
|
"merged_this_week": bool(merged and start <= merged < end),
|
|
"url": p.get("html_url", ""),
|
|
})
|
|
return out
|
|
|
|
|
|
def releases(forge, repo, start, end, note):
|
|
try:
|
|
rows = forge.pages(f"repos/{c.OWNER}/{repo}/releases")
|
|
except urllib.error.HTTPError as e:
|
|
if e.code != 404:
|
|
note(f"{repo}: releases unavailable ({e.code})")
|
|
return []
|
|
out = []
|
|
for r in rows:
|
|
published = c.parse_iso(r.get("published_at") or r.get("created_at"))
|
|
out.append({
|
|
"id": f"r:{repo}:{r.get('tag_name')}",
|
|
"repo": repo,
|
|
"tag": r.get("tag_name", ""),
|
|
"name": r.get("name", ""),
|
|
"published": r.get("published_at") or r.get("created_at", ""),
|
|
"assets": len(r.get("assets") or []),
|
|
"draft": bool(r.get("draft")),
|
|
"cut_this_week": bool(published and start <= published < end),
|
|
"age_days": (end - published).days if published else None,
|
|
"url": r.get("html_url", ""),
|
|
})
|
|
return out
|
|
|
|
|
|
# ---- declared state --------------------------------------------------------
|
|
|
|
TASK_ROW = re.compile(r"^\|\s*(\d+)\s*\|\s*\[([^\]]+)\]\(([^)]+)\)\s*\|\s*(.*?)\s*\|\s*$")
|
|
|
|
|
|
def parse_index(text):
|
|
"""The task index as {number: {section, title, left}} — the hand-written state."""
|
|
section, out = "", {}
|
|
for line in text.splitlines():
|
|
if line.startswith("## "):
|
|
section = line[3:].split("—")[0].strip()
|
|
continue
|
|
m = TASK_ROW.match(line)
|
|
if m:
|
|
# The index writes both `| 08 |` and `| 8 |` for task 8; unnormalised,
|
|
# one task carried two keys and closed itself while still standing.
|
|
out[str(int(m.group(1)))] = {
|
|
"section": section,
|
|
"title": m.group(2).strip(),
|
|
"file": m.group(3).strip(),
|
|
"left": m.group(4).strip(),
|
|
}
|
|
return out
|
|
|
|
|
|
def blob_at(path, sha, filename):
|
|
return c.git(path, "show", f"{sha}:{filename}")
|
|
|
|
|
|
def boundary(path, when, branch="main"):
|
|
"""Newest commit at or before an instant — the index as it stood then."""
|
|
out = c.git(path, "rev-list", "-1", f"--before={c.iso_z(when)}", branch)
|
|
return out.strip()
|
|
|
|
|
|
def declared(start, end, note, snapshot):
|
|
"""What the task index and the SAF said before and after the week."""
|
|
path = c.checkout("SouveraineOS")
|
|
if not path:
|
|
note("SouveraineOS is not checked out — declared state skipped")
|
|
return []
|
|
before, after = boundary(path, start), boundary(path, end)
|
|
if not before or not after:
|
|
note("no SouveraineOS commit on one side of the week — declared state skipped")
|
|
return []
|
|
index_file = "docs/tasks/README.md"
|
|
was = parse_index(blob_at(path, before, index_file))
|
|
now = parse_index(blob_at(path, after, index_file))
|
|
snapshot.update(now)
|
|
rows = []
|
|
for number in sorted(set(was) | set(now), key=int):
|
|
old, new = was.get(number), now.get(number)
|
|
if old and new and old == new:
|
|
continue
|
|
if not old:
|
|
change, detail = "listed", new["section"]
|
|
elif not new:
|
|
change, detail = "delisted", old["section"]
|
|
elif old["section"] != new["section"]:
|
|
change, detail = "moved", f"{old['section']} → {new['section']}"
|
|
else:
|
|
change, detail = "restated", "what's left rewritten"
|
|
rows.append({
|
|
"id": f"d:task-{number}",
|
|
"kind": "task-index",
|
|
"task": number,
|
|
"title": (new or old)["title"],
|
|
"file": (new or old)["file"],
|
|
"change": change,
|
|
"detail": detail,
|
|
"section": (new or old)["section"],
|
|
"was": old["left"] if old else None,
|
|
"now": new["left"] if new else None,
|
|
"dimension": c.classify_text(
|
|
f"{(new or old)['title']} {(new or old)['file']} {(new or old)['left']}")[0],
|
|
})
|
|
|
|
status = {"A": "added", "M": "revised", "D": "deleted", "R": "moved", "C": "copied"}
|
|
changed = c.git(path, "diff", "--name-status", before, after, "--", "saf/", "docs/")
|
|
for line in changed.splitlines():
|
|
fields = line.split("\t")
|
|
if len(fields) < 2:
|
|
continue
|
|
filename = fields[-1]
|
|
if filename.startswith("docs/tasks/"):
|
|
continue
|
|
log = c.git(path, "log", "--format=%h|%aI|%s", f"{before}..{after}", "--", filename)
|
|
touches = [entry.split("|", 2) for entry in log.splitlines() if entry]
|
|
rows.append({
|
|
"id": f"d:{filename}",
|
|
"kind": "document",
|
|
"file": filename,
|
|
"change": status.get(fields[0][0], fields[0]),
|
|
"detail": f"{len(touches)} commit(s)",
|
|
"commits": [t[0] for t in touches],
|
|
"subjects": [t[2] for t in touches],
|
|
"dimension": c.classify_text(f"{filename} {' '.join(t[2] for t in touches)}")[0],
|
|
})
|
|
if not rows:
|
|
note("declared state did not move this week")
|
|
return rows
|
|
|
|
|
|
# ---- rollup ----------------------------------------------------------------
|
|
|
|
def rollup(evidence, start, end):
|
|
by_dim = {}
|
|
for row in evidence["commits"]:
|
|
d = by_dim.setdefault(row["dimension"], {"commits": 0, "repos": [], "authors": []})
|
|
d["commits"] += 1
|
|
for key, value in (("repos", row["repo"]), ("authors", row["author"])):
|
|
if value and value not in d[key]:
|
|
d[key].append(value)
|
|
for row in evidence["declared"]:
|
|
d = by_dim.setdefault(row["dimension"], {"commits": 0, "repos": [], "authors": []})
|
|
d["declared"] = d.get("declared", 0) + 1
|
|
|
|
ci = {}
|
|
for row in evidence["ci"]:
|
|
r = ci.setdefault(row["repo"], {"runs": 0, "success": 0, "failure": 0, "other": 0})
|
|
r["runs"] += 1
|
|
key = row["conclusion"] if row["conclusion"] in ("success", "failure") else "other"
|
|
r[key] += 1
|
|
for repo, r in ci.items():
|
|
newest = max((row for row in evidence["ci"] if row["repo"] == repo),
|
|
key=lambda row: row["started"], default=None)
|
|
r["newest"] = {"id": newest["id"], "conclusion": newest["conclusion"]} if newest else None
|
|
|
|
open_issues = [i for i in evidence["issues"] if i["state"] == "open"]
|
|
open_pulls = [p for p in evidence["pulls"] if p["state"] == "open"]
|
|
basis = {}
|
|
for row in evidence["commits"]:
|
|
basis[row["basis"]] = basis.get(row["basis"], 0) + 1
|
|
return {
|
|
"by_dimension": by_dim,
|
|
"ci": ci,
|
|
"issues": {
|
|
"open": len(open_issues),
|
|
"opened_this_week": sum(i["opened_this_week"] for i in evidence["issues"]),
|
|
"closed_this_week": sum(i["closed_this_week"] for i in evidence["issues"]),
|
|
"ever_closed": sum(1 for i in evidence["issues"] if i["closed"]),
|
|
"oldest_open": max(open_issues, key=lambda i: i["age_days"] or 0)["id"] if open_issues else None,
|
|
"oldest_open_days": max((i["age_days"] or 0) for i in open_issues) if open_issues else 0,
|
|
},
|
|
"pulls": {
|
|
"open": len(open_pulls),
|
|
"merged_this_week": sum(p["merged_this_week"] for p in evidence["pulls"]),
|
|
"oldest_open_days": max((p["age_days"] or 0) for p in open_pulls) if open_pulls else 0,
|
|
},
|
|
"releases": {
|
|
"cut_this_week": [r["id"] for r in evidence["releases"] if r["cut_this_week"]],
|
|
"newest_age_days": min((r["age_days"] or 0) for r in evidence["releases"]) if evidence["releases"] else None,
|
|
},
|
|
"dimension_basis": basis,
|
|
}
|
|
|
|
|
|
def held():
|
|
"""Weeks that already have evidence on disk."""
|
|
path = os.path.join(ROOT, "project-state", "evidence")
|
|
return sorted(name[:-5] for name in os.listdir(path)) if os.path.isdir(path) else []
|
|
|
|
|
|
def gaps(since=None):
|
|
"""Weeks with no evidence, from the first week held (or --since) to now.
|
|
|
|
Nothing runs this on a schedule. The tool has to be able to say which weeks
|
|
are unfilled, or a missed Monday quietly becomes a hole in the record.
|
|
"""
|
|
have = held()
|
|
current, _, _ = c.parse_week("current")
|
|
first = since or (have[0] if have else current)
|
|
return [week for week in c.week_span(first, current) if week not in have]
|
|
|
|
|
|
def collect_week(week, args, forge=None):
|
|
label, monday, sunday = c.parse_week(week)
|
|
start, end = c.week_bounds_utc(monday, sunday)
|
|
repos = [r.strip() for r in args.repos.split(",")] if args.repos else c.REPOS
|
|
forge = forge or c.Forge()
|
|
|
|
notes = []
|
|
def note(message):
|
|
notes.append(message)
|
|
print(f"[project-state] {message}", file=sys.stderr)
|
|
|
|
evidence = {"commits": [], "ci": [], "issues": [], "pulls": [], "releases": []}
|
|
repo_summary = {}
|
|
before = forge.calls
|
|
for repo in repos:
|
|
rows = commits(forge, repo, start, end, note)
|
|
evidence["commits"] += rows
|
|
evidence["ci"] += runs(forge, repo, start, end, note)
|
|
evidence["issues"] += issues(forge, repo, start, end, note)
|
|
evidence["pulls"] += pulls(forge, repo, start, end, note)
|
|
evidence["releases"] += releases(forge, repo, start, end, note)
|
|
repo_summary[repo] = {"commits": len(rows), "checkout": bool(c.checkout(repo))}
|
|
index = {}
|
|
evidence["declared"] = declared(start, end, note, index)
|
|
evidence["index"] = index
|
|
|
|
for key in ("commits", "ci"):
|
|
evidence[key].sort(key=lambda row: row.get("at") or row.get("started") or "")
|
|
|
|
if args.redact:
|
|
for row in evidence["commits"]:
|
|
row["subject"] = c.redact(row["subject"])
|
|
row["detail"] = c.redact(row["detail"])
|
|
|
|
out = {
|
|
"week": label,
|
|
"range": {
|
|
"start": monday.isoformat(), "end": sunday.isoformat(), "tz": str(c.LOCAL),
|
|
"start_utc": c.iso_z(start), "end_utc": c.iso_z(end),
|
|
},
|
|
"collected_at": c.iso_z(dt.datetime.now(dt.timezone.utc)),
|
|
"collector": {
|
|
"tool": "tools/project-state/collect.py",
|
|
"version": VERSION,
|
|
"forge": f"gitea {forge.version()}",
|
|
"api_calls": forge.calls - before,
|
|
"redacted": bool(args.redact),
|
|
"repos": repo_summary,
|
|
"notes": notes,
|
|
},
|
|
**evidence,
|
|
}
|
|
out["rollup"] = rollup(out, start, end)
|
|
return out
|
|
|
|
|
|
def write(out, path):
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
open(path, "w").write(json.dumps(out, indent=2, ensure_ascii=False) + "\n")
|
|
print(f"[project-state] {out['week']}: {len(out['commits'])} commits, {len(out['ci'])} runs, "
|
|
f"{len(out['declared'])} declared changes, {out['collector']['api_calls']} api calls "
|
|
f"-> {path}", file=sys.stderr)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--week", default="last", help="2026-W33 | current | last (default)")
|
|
ap.add_argument("--repos", help="comma-separated override of the repo list")
|
|
ap.add_argument("--out", help="output path, '-' for stdout")
|
|
ap.add_argument("--redact", action="store_true",
|
|
help="mask hosts and home paths — for evidence leaving the forge")
|
|
ap.add_argument("--gaps", action="store_true", help="name the weeks holding no evidence")
|
|
ap.add_argument("--backfill", action="store_true", help="collect every week --gaps names")
|
|
ap.add_argument("--since", help="earliest week --gaps and --backfill consider")
|
|
args = ap.parse_args()
|
|
|
|
if args.gaps or args.backfill:
|
|
missing = gaps(args.since)
|
|
if not missing:
|
|
print(f"[project-state] no gaps; {len(held())} week(s) held", file=sys.stderr)
|
|
return 0
|
|
if args.gaps:
|
|
print("\n".join(missing))
|
|
return 0
|
|
forge = c.Forge()
|
|
if not forge.token:
|
|
print("project-state: no forge token (set GITEA_TOKEN)", file=sys.stderr)
|
|
return 2
|
|
for week in missing:
|
|
write(collect_week(week, args, forge),
|
|
os.path.join(ROOT, "project-state", "evidence", f"{week}.json"))
|
|
return 0
|
|
|
|
forge = c.Forge()
|
|
if not forge.token:
|
|
print("project-state: no forge token (set GITEA_TOKEN)", file=sys.stderr)
|
|
return 2
|
|
|
|
out = collect_week(args.week, args, forge)
|
|
if args.out == "-":
|
|
sys.stdout.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n")
|
|
return 0
|
|
write(out, args.out or os.path.join(ROOT, "project-state", "evidence", f"{out['week']}.json"))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|