project-state: weekly evidence, carried state, and an agent's door
Five weeks of forge evidence per week rather than a rolling window, so a finding survives under one key and "closed after three weeks" is computable. Rules run deterministically; agent.py hands the packet to whoever writes the prose and never calls a model itself. Model choice fails closed — no default, no substitute, and the exact named model must be served or the pass refuses to run.
This commit is contained in:
Fimeg
parent
1c565ce0e2
commit
a8caf27c14
26 changed files with 56303 additions and 0 deletions
353
tools/project-state/agent.py
Normal file
353
tools/project-state/agent.py
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
#!/usr/bin/env python3
|
||||
"""The agent's door into project state. No inference happens here, ever.
|
||||
|
||||
The deterministic layer establishes what is true; the agent reading this writes
|
||||
the prose. Nothing in this file selects, calls, or configures a model — the
|
||||
agent invoking it is the model, and it already has the identity, the context and
|
||||
the tools.
|
||||
|
||||
agent.py status what is collected, what is missing
|
||||
agent.py packet --week 2026-W33 everything needed to write the draft
|
||||
agent.py explain f-0069 one finding and every row it rests on
|
||||
agent.py draft --week 2026-W33 --file draft.md --author Annie
|
||||
agent.py verdict f-0069 promote --note "worth a task"
|
||||
|
||||
`packet` is the contract. It carries the week's state, what turned, what has
|
||||
stood longest, and an `investigate` list naming where the deterministic layer
|
||||
knows it is thin — go read those repositories rather than trusting the packet.
|
||||
|
||||
See docs/tasks/79-project-state.md.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import common as c
|
||||
import collect
|
||||
import synthesize as s
|
||||
|
||||
STORE = s.STORE
|
||||
CONTRACT = {
|
||||
"you_are": "the agent that writes Souveraine's weekly project-state draft",
|
||||
"the_split": (
|
||||
"Everything under `counted`, `movement`, `standing`, `carried` and `closed` was "
|
||||
"computed from the forge by deterministic code and is authoritative. Your draft is "
|
||||
"downstream of it and is disposable — it can be regenerated from this packet and "
|
||||
"the stored state at any time."
|
||||
),
|
||||
"produce": (
|
||||
"A markdown draft answering: what became true about Souveraine this week, what "
|
||||
"stopped being true, what advanced, what regressed, what remains unresolved, what "
|
||||
"decisions are waiting, and which milestones were crossed or are approaching."
|
||||
),
|
||||
"rules": [
|
||||
"Cite finding ids (f-0069) and evidence ids (c:, ci:, d:, i:, p:, r:) for every claim.",
|
||||
"Do not invent a shipment, release, decision, or outcome that is not in the evidence.",
|
||||
"'Nothing moved here this week' is a legal sentence.",
|
||||
"Where the packet's `investigate` list says the evidence is thin, go read the "
|
||||
"repository before asserting; say so inline if you did not.",
|
||||
"The report is internal. It publishes nowhere until a human promotes it.",
|
||||
],
|
||||
"when_the_packet_is_not_enough": (
|
||||
"Read the repositories directly. `evidence.file` holds every row including full "
|
||||
"commit bodies and changed paths; the working trees named in `repos` are checked "
|
||||
"out locally. `agent.py explain <id>` prints any finding with its rows in full."
|
||||
),
|
||||
"return_with": "agent.py draft --week <week> --file <path> --author <you> [--model <id>]",
|
||||
"no_inference_here": (
|
||||
"This tool never calls a model. Model choice belongs to whoever invokes it and is "
|
||||
"recorded, not decided, by --model."
|
||||
),
|
||||
}
|
||||
|
||||
EVIDENCE_KINDS = {
|
||||
"c:": "commit — repo, sha, subject, body, changed paths, dimension and the basis that chose it",
|
||||
"ci:": "Gitea Actions run — workflow, branch, head sha, conclusion; joins to a commit id",
|
||||
"d:": "declared change — a task-index row or a SAF/docs file that moved",
|
||||
"i:": "issue", "p:": "pull request", "r:": "release", "w:": "the week's rollup",
|
||||
}
|
||||
|
||||
|
||||
def load_state():
|
||||
return s.load(os.path.join(STORE, "state.json"), {"findings": []})
|
||||
|
||||
|
||||
def load_evidence(week):
|
||||
return s.load(os.path.join(STORE, "evidence", f"{week}.json"))
|
||||
|
||||
|
||||
def status():
|
||||
state = load_state()
|
||||
held = collect.held()
|
||||
return {
|
||||
"weeks_held": held,
|
||||
"latest_synthesized": state.get("latest_week"),
|
||||
"gaps": collect.gaps(),
|
||||
"pending": s.pending(state),
|
||||
"open_findings": sum(1 for f in state.get("findings", []) if f["status"] == "open"),
|
||||
"drafts": sorted(os.listdir(os.path.join(STORE, "drafts")))
|
||||
if os.path.isdir(os.path.join(STORE, "drafts")) else [],
|
||||
"next": next_action(state, held),
|
||||
}
|
||||
|
||||
|
||||
def next_action(state, held):
|
||||
if collect.gaps():
|
||||
return "collect.py --backfill — weeks are missing evidence"
|
||||
if s.pending(state):
|
||||
return "synthesize.py --catch-up — weeks are collected but not passed over"
|
||||
if not held:
|
||||
return "collect.py --backfill --since <week> — nothing collected yet"
|
||||
return f"agent.py packet --week {state.get('latest_week')} — state is current; write the draft"
|
||||
|
||||
|
||||
def thin_spots(ev, findings):
|
||||
"""Where the deterministic layer knows it did not see enough.
|
||||
|
||||
This is the half of the packet that asks for work: each entry says what is
|
||||
weak, and what reading it would take to settle.
|
||||
"""
|
||||
out = []
|
||||
unplaced = [r for r in ev["commits"] if r["dimension"] == c.UNASSIGNED]
|
||||
if unplaced:
|
||||
out.append({
|
||||
"why": f"{len(unplaced)} commit(s) matched no dimension keyword and were not guessed at",
|
||||
"how": "read these commits in their repositories and place them yourself",
|
||||
"rows": [{"id": r["id"], "repo": r["repo"], "subject": r["subject"],
|
||||
"paths": r["paths"][:6]} for r in unplaced],
|
||||
})
|
||||
blind = [name for name, meta in ev["collector"]["repos"].items()
|
||||
if not meta["checkout"] and meta["commits"]]
|
||||
if blind:
|
||||
out.append({
|
||||
"why": f"no local checkout for {', '.join(blind)}, so changed paths were unavailable "
|
||||
f"and those commits were placed by subject or repository alone",
|
||||
"how": "clone or fetch those repos if a dimension call there matters this week",
|
||||
"rows": [],
|
||||
})
|
||||
single = [f for f in findings if len(f["evidence"]) == 1
|
||||
and f["provenance"]["rule"] not in ("task-standing", "pull-aging", "issue-aging")]
|
||||
if single:
|
||||
out.append({
|
||||
"why": f"{len(single)} finding(s) rest on exactly one evidence row",
|
||||
"how": "agent.py explain <id>, then read the surrounding work before leaning on it",
|
||||
"rows": [{"id": f["id"], "title": f["title"], "evidence": f["evidence"]}
|
||||
for f in single[:12]],
|
||||
})
|
||||
declared_only = [dim for dim, r in ev["rollup"]["by_dimension"].items()
|
||||
if r.get("declared") and not r.get("commits")]
|
||||
if declared_only:
|
||||
out.append({
|
||||
"why": f"{', '.join(declared_only)} changed in the documentation with no commits behind it",
|
||||
"how": "a doc can lead the code or lag it — check which, before calling it advanced",
|
||||
"rows": [{"id": r["id"], "file": r.get("file"), "title": r.get("title"),
|
||||
"change": r["change"]}
|
||||
for r in ev["declared"] if r["dimension"] in declared_only][:12],
|
||||
})
|
||||
notes = ev["collector"].get("notes") or []
|
||||
if notes:
|
||||
out.append({"why": "the collector reported problems", "how": "treat these areas as incomplete",
|
||||
"rows": [{"note": n} for n in notes]})
|
||||
return out
|
||||
|
||||
|
||||
def packet(week, brief=False):
|
||||
label, monday, sunday = c.parse_week(week)
|
||||
ev = load_evidence(label)
|
||||
if not ev:
|
||||
return None, f"no evidence for {label} — collect.py --week {label}"
|
||||
state = s.load(os.path.join(STORE, "history", f"{label}.json")) or load_state()
|
||||
if label not in state.get("weeks", []):
|
||||
return None, f"{label} is collected but not synthesized — synthesize.py --catch-up"
|
||||
|
||||
findings = state["findings"]
|
||||
this_week = [f for f in findings if f["last_seen"] == label and f["status"] == "open"]
|
||||
quiet = [f for f in this_week if f["provenance"]["rule"] == "task-standing"
|
||||
and f["movement"] != "blocked"]
|
||||
moved = [f for f in this_week if f not in quiet]
|
||||
carried = [f for f in findings if f["status"] == "open" and f["last_seen"] != label]
|
||||
closed = [f for f in findings if f.get("resolved_in") == label]
|
||||
turned = [f for f in moved if f.get("was") and f["was"]["movement"] != f["movement"]]
|
||||
|
||||
def slim(f):
|
||||
row = {"id": f["id"], "key": f["key"], "dimension": f["dimension"],
|
||||
"movement": f["movement"], "title": f["title"], "statement": f["statement"],
|
||||
"evidence": f["evidence"], "rule": f["provenance"]["rule"],
|
||||
"first_seen": f["first_seen"], "weeks_open": f.get("weeks_open")}
|
||||
for extra in ("was", "weeks_in_section", "section_since", "task_state", "resolved_in"):
|
||||
if f.get(extra):
|
||||
row[extra] = f[extra]
|
||||
if (f.get("human") or {}).get("verdict"):
|
||||
row["human"] = f["human"]
|
||||
return row
|
||||
|
||||
commits = [{"id": r["id"], "repo": r["repo"], "at": r["at"], "dimension": r["dimension"],
|
||||
"basis": r["basis"], "subject": r["subject"],
|
||||
**({} if brief else {"detail": r["detail"], "paths": r["paths"][:12]})}
|
||||
for r in ev["commits"]]
|
||||
# Only a finding that carries can be the oldest one; a CI week or a
|
||||
# dimension's activity describes one week and lapses with it.
|
||||
oldest = max((f for f in findings if f["status"] == "open"
|
||||
and f["provenance"]["rule"] in s.CARRIES),
|
||||
key=lambda f: (f.get("weeks_open", 1), f["id"]), default=None)
|
||||
|
||||
return {
|
||||
"contract": CONTRACT,
|
||||
"week": label,
|
||||
"range": ev["range"],
|
||||
"completeness": {
|
||||
"weeks_held": collect.held(),
|
||||
"gaps": collect.gaps(),
|
||||
"pending": s.pending(load_state()),
|
||||
"collected_at": ev["collected_at"],
|
||||
"collector": {k: ev["collector"][k] for k in ("tool", "version", "forge", "api_calls",
|
||||
"redacted")},
|
||||
},
|
||||
"counted": ev["rollup"],
|
||||
"movement": [slim(f) for f in moved],
|
||||
"standing": [slim(f) for f in sorted(quiet, key=lambda f: -f.get("weeks_in_section", 1))],
|
||||
"carried": [slim(f) for f in sorted(carried, key=lambda f: -f.get("weeks_open", 1))],
|
||||
"closed": [slim(f) for f in closed],
|
||||
"continuity": {
|
||||
"weeks_available": state.get("weeks", []),
|
||||
"turned_this_week": [{"id": f["id"], "title": f["title"],
|
||||
"from": f["was"]["movement"], "to": f["movement"],
|
||||
"since": f["was"]["week"]} for f in turned],
|
||||
"oldest_open": slim(oldest) if oldest else None,
|
||||
},
|
||||
"investigate": thin_spots(ev, this_week),
|
||||
# --brief keeps every row the counts cannot replace and drops the rest:
|
||||
# green CI runs are already a tally, and the task index is `standing`.
|
||||
"evidence": {
|
||||
"file": f"project-state/evidence/{label}.json",
|
||||
"explain": "tools/project-state/agent.py explain <finding-id>",
|
||||
"kinds": EVIDENCE_KINDS,
|
||||
"commits": commits,
|
||||
"declared": ev["declared"],
|
||||
"ci": [r for r in ev["ci"] if r["conclusion"] != "success"] if brief else ev["ci"],
|
||||
"ci_note": "green runs omitted; counts are under `counted.ci`" if brief else None,
|
||||
"issues": ev["issues"],
|
||||
"pulls": ev["pulls"],
|
||||
"releases": ev["releases"],
|
||||
**({} if brief else {"task_index": ev.get("index", {})}),
|
||||
},
|
||||
"repos": {name: {"commits": meta["commits"],
|
||||
"checkout": os.path.join(c.PROJECTS, name) if meta["checkout"] else None}
|
||||
for name, meta in ev["collector"]["repos"].items() if meta["commits"]},
|
||||
}, None
|
||||
|
||||
|
||||
def record_draft(week, path, author, model, note):
|
||||
label, _, _ = c.parse_week(week)
|
||||
try:
|
||||
body = open(path).read()
|
||||
except OSError as e:
|
||||
note(f"cannot read {path}: {e}")
|
||||
return 2
|
||||
state = s.load(os.path.join(STORE, "history", f"{label}.json")) or load_state()
|
||||
folder = os.path.join(STORE, "drafts")
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
slug = "".join(ch if ch.isalnum() or ch in "-_" else "-" for ch in author).strip("-").lower()
|
||||
stem = os.path.join(folder, f"{label}--{slug}")
|
||||
open(stem + ".md", "w").write(body)
|
||||
# The draft names the state it was written against, so a later reader can
|
||||
# tell whether it still describes the findings that exist.
|
||||
open(stem + ".json", "w").write(json.dumps({
|
||||
"week": label,
|
||||
"author": author,
|
||||
"model": model,
|
||||
"recorded_at": c.iso_z(dt.datetime.now(dt.timezone.utc)),
|
||||
"state_updated": state.get("updated"),
|
||||
"findings_open": sum(1 for f in state.get("findings", []) if f["status"] == "open"),
|
||||
"evidence": f"project-state/evidence/{label}.json",
|
||||
"published": False,
|
||||
"source": os.path.abspath(path),
|
||||
}, indent=2, ensure_ascii=False) + "\n")
|
||||
note(f"{label}: draft by {author}"
|
||||
+ (f" ({model})" if model else "")
|
||||
+ f" -> project-state/drafts/{label}--{slug}.md; nothing published")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
sub = ap.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("status", help="what is collected, synthesized, and missing")
|
||||
|
||||
p = sub.add_parser("packet", help="everything needed to write the week's draft")
|
||||
p.add_argument("--week", default="last")
|
||||
p.add_argument("--brief", action="store_true", help="drop commit bodies and paths")
|
||||
p.add_argument("--out", help="write here instead of stdout")
|
||||
|
||||
e = sub.add_parser("explain", help="one finding and every evidence row it cites")
|
||||
e.add_argument("finding")
|
||||
e.add_argument("--json", action="store_true")
|
||||
|
||||
d = sub.add_parser("draft", help="record a draft written by an agent or a person")
|
||||
d.add_argument("--week", default="last")
|
||||
d.add_argument("--file", required=True)
|
||||
d.add_argument("--author", required=True)
|
||||
d.add_argument("--model", help="the model that wrote it, recorded not chosen")
|
||||
|
||||
v = sub.add_parser("verdict", help="a person's word on a finding")
|
||||
v.add_argument("finding")
|
||||
v.add_argument("verdict", choices=s.VERDICTS)
|
||||
v.add_argument("--note")
|
||||
v.add_argument("--by")
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
def note(message):
|
||||
print(f"[project-state] {message}", file=sys.stderr)
|
||||
|
||||
if args.command == "status":
|
||||
print(json.dumps(status(), indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
if args.command == "packet":
|
||||
data, problem = packet(args.week, args.brief)
|
||||
if problem:
|
||||
note(problem)
|
||||
return 2
|
||||
text = json.dumps(data, indent=2, ensure_ascii=False) + "\n"
|
||||
if args.out:
|
||||
open(args.out, "w").write(text)
|
||||
note(f"{data['week']}: packet -> {args.out} ({len(text):,} chars)")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
return 0
|
||||
|
||||
if args.command == "explain":
|
||||
if not args.json:
|
||||
return s.explain(load_state(), args.finding)
|
||||
state = load_state()
|
||||
f = next((x for x in state["findings"]
|
||||
if x["id"] == args.finding or x["key"] == args.finding), None)
|
||||
if not f:
|
||||
note(f"no finding {args.finding}")
|
||||
return 1
|
||||
week = f["provenance"].get("evidence_week", f["last_seen"])
|
||||
ev = load_evidence(week)
|
||||
index = {row["id"]: row for group in ("commits", "ci", "issues", "pulls",
|
||||
"releases", "declared")
|
||||
for row in ev.get(group, [])}
|
||||
index[f"w:{week}"] = ev.get("rollup", {})
|
||||
print(json.dumps({"finding": f, "week": week,
|
||||
"evidence": {eid: index.get(eid, "MISSING") for eid in f["evidence"]}},
|
||||
indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
if args.command == "draft":
|
||||
return record_draft(args.week, args.file, args.author, args.model, note)
|
||||
|
||||
if args.command == "verdict":
|
||||
return s.write_verdict(f"{args.finding}={args.verdict}", note, args.note, args.by)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
475
tools/project-state/collect.py
Normal file
475
tools/project-state/collect.py
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Weekly evidence for the project-state pass. Deterministic; no model runs here.
|
||||
|
||||
Reads the forge for one ISO week — commits, Actions runs, issues, pulls,
|
||||
releases — and the local SouveraineOS clone for what the task index and the SAF
|
||||
declared, then writes project-state/evidence/<week>.json. Every row carries a
|
||||
stable id; a finding cites those ids and nothing else.
|
||||
|
||||
tools/project-state/collect.py --week last
|
||||
tools/project-state/collect.py --week 2026-W33 --out -
|
||||
|
||||
Env: GITEA_URL, GITEA_TOKEN (falls back to ~/.git-credentials).
|
||||
See docs/tasks/79-project-state.md.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import common as c
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
VERSION = 1
|
||||
|
||||
|
||||
def commits(forge, repo, start, end, note):
|
||||
"""Commits in the window, every branch, deduped by SHA.
|
||||
|
||||
A branch whose tip predates the window cannot hold work inside it, so it is
|
||||
never walked — the same prefilter the ledger's collector uses.
|
||||
"""
|
||||
full = f"{c.OWNER}/{repo}"
|
||||
edge = c.iso_z(start)
|
||||
rows, seen = [], set()
|
||||
try:
|
||||
branches = forge.pages(f"repos/{full}/branches")
|
||||
except urllib.error.HTTPError as e:
|
||||
note(f"{repo}: branches unavailable ({e.code})")
|
||||
return []
|
||||
live = [b["name"] for b in branches
|
||||
if ((b.get("commit") or {}).get("timestamp") or "") >= edge]
|
||||
for branch in live:
|
||||
path = (f"repos/{full}/commits?sha={branch}&since={edge}&until={c.iso_z(end)}"
|
||||
f"&stat=false&verification=false&files=false")
|
||||
try:
|
||||
page = forge.pages(path)
|
||||
except urllib.error.HTTPError as e:
|
||||
note(f"{repo}@{branch}: commits unavailable ({e.code})")
|
||||
continue
|
||||
for row in page:
|
||||
sha = row.get("sha", "")
|
||||
if sha in seen or len((row.get("parents") or [])) > 1:
|
||||
continue
|
||||
seen.add(sha)
|
||||
commit = row.get("commit") or {}
|
||||
author = commit.get("author") or {}
|
||||
subject, _, body = (commit.get("message") or "").partition("\n")
|
||||
paths = c.commit_paths(repo, sha)
|
||||
dimension, basis = c.classify(repo, subject.strip(), paths)
|
||||
rows.append({
|
||||
"id": f"c:{repo}:{sha[:8]}",
|
||||
"repo": repo,
|
||||
"sha": sha,
|
||||
"branch": branch,
|
||||
"at": author.get("date", ""),
|
||||
"author": author.get("email", ""),
|
||||
"subject": subject.strip(),
|
||||
"detail": re.split(r"\n\s*\n", body.strip(), maxsplit=1)[0].strip(),
|
||||
"dimension": dimension,
|
||||
"basis": basis,
|
||||
"paths": paths[:40],
|
||||
"url": row.get("html_url", ""),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def runs(forge, repo, start, end, note):
|
||||
"""Actions runs started inside the window.
|
||||
|
||||
This endpoint ignores limit and page and hands back the whole history, so it
|
||||
is fetched once and sliced here — paging it would loop on the same rows.
|
||||
"""
|
||||
try:
|
||||
payload = forge.get(f"repos/{c.OWNER}/{repo}/actions/runs")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
note(f"{repo}: actions unavailable ({e.code})")
|
||||
return []
|
||||
rows = []
|
||||
for run in payload.get("workflow_runs", []) if isinstance(payload, dict) else payload:
|
||||
started = c.parse_iso(run.get("started_at") or run.get("run_started_at"))
|
||||
if not started or not (start <= started < end):
|
||||
continue
|
||||
rows.append({
|
||||
"id": f"ci:{repo}:{run.get('id')}",
|
||||
"repo": repo,
|
||||
"run": run.get("id"),
|
||||
"workflow": os.path.basename(run.get("path") or ""),
|
||||
"branch": run.get("head_branch", ""),
|
||||
"sha": run.get("head_sha", ""),
|
||||
"commit": f"c:{repo}:{(run.get('head_sha') or '')[:8]}",
|
||||
"conclusion": run.get("conclusion") or run.get("status") or "",
|
||||
"started": run.get("started_at", ""),
|
||||
"completed": run.get("completed_at", ""),
|
||||
"title": run.get("display_title", ""),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def issues(forge, repo, start, end, note):
|
||||
"""Every issue, open or closed — aging needs the ones that predate the week."""
|
||||
try:
|
||||
rows = forge.pages(f"repos/{c.OWNER}/{repo}/issues?state=all&type=issues")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
note(f"{repo}: issues unavailable ({e.code})")
|
||||
return []
|
||||
out = []
|
||||
for i in rows:
|
||||
created, closed = c.parse_iso(i.get("created_at")), c.parse_iso(i.get("closed_at"))
|
||||
out.append({
|
||||
"id": f"i:{repo}:{i.get('number')}",
|
||||
"repo": repo,
|
||||
"number": i.get("number"),
|
||||
"title": i.get("title", ""),
|
||||
"state": i.get("state", ""),
|
||||
"labels": [l.get("name") for l in (i.get("labels") or [])],
|
||||
"created": i.get("created_at", ""),
|
||||
"closed": i.get("closed_at"),
|
||||
"age_days": (end - created).days if created else None,
|
||||
"opened_this_week": bool(created and start <= created < end),
|
||||
"closed_this_week": bool(closed and start <= closed < end),
|
||||
"url": i.get("html_url", ""),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def pulls(forge, repo, start, end, note):
|
||||
try:
|
||||
rows = forge.pages(f"repos/{c.OWNER}/{repo}/pulls?state=all")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
note(f"{repo}: pulls unavailable ({e.code})")
|
||||
return []
|
||||
out = []
|
||||
for p in rows:
|
||||
created, merged = c.parse_iso(p.get("created_at")), c.parse_iso(p.get("merged_at"))
|
||||
out.append({
|
||||
"id": f"p:{repo}:{p.get('number')}",
|
||||
"repo": repo,
|
||||
"number": p.get("number"),
|
||||
"title": p.get("title", ""),
|
||||
"state": p.get("state", ""),
|
||||
"created": p.get("created_at", ""),
|
||||
"merged": p.get("merged_at"),
|
||||
"age_days": (end - created).days if created else None,
|
||||
"opened_this_week": bool(created and start <= created < end),
|
||||
"merged_this_week": bool(merged and start <= merged < end),
|
||||
"url": p.get("html_url", ""),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def releases(forge, repo, start, end, note):
|
||||
try:
|
||||
rows = forge.pages(f"repos/{c.OWNER}/{repo}/releases")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
note(f"{repo}: releases unavailable ({e.code})")
|
||||
return []
|
||||
out = []
|
||||
for r in rows:
|
||||
published = c.parse_iso(r.get("published_at") or r.get("created_at"))
|
||||
out.append({
|
||||
"id": f"r:{repo}:{r.get('tag_name')}",
|
||||
"repo": repo,
|
||||
"tag": r.get("tag_name", ""),
|
||||
"name": r.get("name", ""),
|
||||
"published": r.get("published_at") or r.get("created_at", ""),
|
||||
"assets": len(r.get("assets") or []),
|
||||
"draft": bool(r.get("draft")),
|
||||
"cut_this_week": bool(published and start <= published < end),
|
||||
"age_days": (end - published).days if published else None,
|
||||
"url": r.get("html_url", ""),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
# ---- declared state --------------------------------------------------------
|
||||
|
||||
TASK_ROW = re.compile(r"^\|\s*(\d+)\s*\|\s*\[([^\]]+)\]\(([^)]+)\)\s*\|\s*(.*?)\s*\|\s*$")
|
||||
|
||||
|
||||
def parse_index(text):
|
||||
"""The task index as {number: {section, title, left}} — the hand-written state."""
|
||||
section, out = "", {}
|
||||
for line in text.splitlines():
|
||||
if line.startswith("## "):
|
||||
section = line[3:].split("—")[0].strip()
|
||||
continue
|
||||
m = TASK_ROW.match(line)
|
||||
if m:
|
||||
# The index writes both `| 08 |` and `| 8 |` for task 8; unnormalised,
|
||||
# one task carried two keys and closed itself while still standing.
|
||||
out[str(int(m.group(1)))] = {
|
||||
"section": section,
|
||||
"title": m.group(2).strip(),
|
||||
"file": m.group(3).strip(),
|
||||
"left": m.group(4).strip(),
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def blob_at(path, sha, filename):
|
||||
return c.git(path, "show", f"{sha}:{filename}")
|
||||
|
||||
|
||||
def boundary(path, when, branch="main"):
|
||||
"""Newest commit at or before an instant — the index as it stood then."""
|
||||
out = c.git(path, "rev-list", "-1", f"--before={c.iso_z(when)}", branch)
|
||||
return out.strip()
|
||||
|
||||
|
||||
def declared(start, end, note, snapshot):
|
||||
"""What the task index and the SAF said before and after the week."""
|
||||
path = c.checkout("SouveraineOS")
|
||||
if not path:
|
||||
note("SouveraineOS is not checked out — declared state skipped")
|
||||
return []
|
||||
before, after = boundary(path, start), boundary(path, end)
|
||||
if not before or not after:
|
||||
note("no SouveraineOS commit on one side of the week — declared state skipped")
|
||||
return []
|
||||
index_file = "docs/tasks/README.md"
|
||||
was = parse_index(blob_at(path, before, index_file))
|
||||
now = parse_index(blob_at(path, after, index_file))
|
||||
snapshot.update(now)
|
||||
rows = []
|
||||
for number in sorted(set(was) | set(now), key=int):
|
||||
old, new = was.get(number), now.get(number)
|
||||
if old and new and old == new:
|
||||
continue
|
||||
if not old:
|
||||
change, detail = "listed", new["section"]
|
||||
elif not new:
|
||||
change, detail = "delisted", old["section"]
|
||||
elif old["section"] != new["section"]:
|
||||
change, detail = "moved", f"{old['section']} → {new['section']}"
|
||||
else:
|
||||
change, detail = "restated", "what's left rewritten"
|
||||
rows.append({
|
||||
"id": f"d:task-{number}",
|
||||
"kind": "task-index",
|
||||
"task": number,
|
||||
"title": (new or old)["title"],
|
||||
"file": (new or old)["file"],
|
||||
"change": change,
|
||||
"detail": detail,
|
||||
"section": (new or old)["section"],
|
||||
"was": old["left"] if old else None,
|
||||
"now": new["left"] if new else None,
|
||||
"dimension": c.classify_text(
|
||||
f"{(new or old)['title']} {(new or old)['file']} {(new or old)['left']}")[0],
|
||||
})
|
||||
|
||||
status = {"A": "added", "M": "revised", "D": "deleted", "R": "moved", "C": "copied"}
|
||||
changed = c.git(path, "diff", "--name-status", before, after, "--", "saf/", "docs/")
|
||||
for line in changed.splitlines():
|
||||
fields = line.split("\t")
|
||||
if len(fields) < 2:
|
||||
continue
|
||||
filename = fields[-1]
|
||||
if filename.startswith("docs/tasks/"):
|
||||
continue
|
||||
log = c.git(path, "log", "--format=%h|%aI|%s", f"{before}..{after}", "--", filename)
|
||||
touches = [entry.split("|", 2) for entry in log.splitlines() if entry]
|
||||
rows.append({
|
||||
"id": f"d:{filename}",
|
||||
"kind": "document",
|
||||
"file": filename,
|
||||
"change": status.get(fields[0][0], fields[0]),
|
||||
"detail": f"{len(touches)} commit(s)",
|
||||
"commits": [t[0] for t in touches],
|
||||
"subjects": [t[2] for t in touches],
|
||||
"dimension": c.classify_text(f"{filename} {' '.join(t[2] for t in touches)}")[0],
|
||||
})
|
||||
if not rows:
|
||||
note("declared state did not move this week")
|
||||
return rows
|
||||
|
||||
|
||||
# ---- rollup ----------------------------------------------------------------
|
||||
|
||||
def rollup(evidence, start, end):
|
||||
by_dim = {}
|
||||
for row in evidence["commits"]:
|
||||
d = by_dim.setdefault(row["dimension"], {"commits": 0, "repos": [], "authors": []})
|
||||
d["commits"] += 1
|
||||
for key, value in (("repos", row["repo"]), ("authors", row["author"])):
|
||||
if value and value not in d[key]:
|
||||
d[key].append(value)
|
||||
for row in evidence["declared"]:
|
||||
d = by_dim.setdefault(row["dimension"], {"commits": 0, "repos": [], "authors": []})
|
||||
d["declared"] = d.get("declared", 0) + 1
|
||||
|
||||
ci = {}
|
||||
for row in evidence["ci"]:
|
||||
r = ci.setdefault(row["repo"], {"runs": 0, "success": 0, "failure": 0, "other": 0})
|
||||
r["runs"] += 1
|
||||
key = row["conclusion"] if row["conclusion"] in ("success", "failure") else "other"
|
||||
r[key] += 1
|
||||
for repo, r in ci.items():
|
||||
newest = max((row for row in evidence["ci"] if row["repo"] == repo),
|
||||
key=lambda row: row["started"], default=None)
|
||||
r["newest"] = {"id": newest["id"], "conclusion": newest["conclusion"]} if newest else None
|
||||
|
||||
open_issues = [i for i in evidence["issues"] if i["state"] == "open"]
|
||||
open_pulls = [p for p in evidence["pulls"] if p["state"] == "open"]
|
||||
basis = {}
|
||||
for row in evidence["commits"]:
|
||||
basis[row["basis"]] = basis.get(row["basis"], 0) + 1
|
||||
return {
|
||||
"by_dimension": by_dim,
|
||||
"ci": ci,
|
||||
"issues": {
|
||||
"open": len(open_issues),
|
||||
"opened_this_week": sum(i["opened_this_week"] for i in evidence["issues"]),
|
||||
"closed_this_week": sum(i["closed_this_week"] for i in evidence["issues"]),
|
||||
"ever_closed": sum(1 for i in evidence["issues"] if i["closed"]),
|
||||
"oldest_open": max(open_issues, key=lambda i: i["age_days"] or 0)["id"] if open_issues else None,
|
||||
"oldest_open_days": max((i["age_days"] or 0) for i in open_issues) if open_issues else 0,
|
||||
},
|
||||
"pulls": {
|
||||
"open": len(open_pulls),
|
||||
"merged_this_week": sum(p["merged_this_week"] for p in evidence["pulls"]),
|
||||
"oldest_open_days": max((p["age_days"] or 0) for p in open_pulls) if open_pulls else 0,
|
||||
},
|
||||
"releases": {
|
||||
"cut_this_week": [r["id"] for r in evidence["releases"] if r["cut_this_week"]],
|
||||
"newest_age_days": min((r["age_days"] or 0) for r in evidence["releases"]) if evidence["releases"] else None,
|
||||
},
|
||||
"dimension_basis": basis,
|
||||
}
|
||||
|
||||
|
||||
def held():
|
||||
"""Weeks that already have evidence on disk."""
|
||||
path = os.path.join(ROOT, "project-state", "evidence")
|
||||
return sorted(name[:-5] for name in os.listdir(path)) if os.path.isdir(path) else []
|
||||
|
||||
|
||||
def gaps(since=None):
|
||||
"""Weeks with no evidence, from the first week held (or --since) to now.
|
||||
|
||||
Nothing runs this on a schedule. The tool has to be able to say which weeks
|
||||
are unfilled, or a missed Monday quietly becomes a hole in the record.
|
||||
"""
|
||||
have = held()
|
||||
current, _, _ = c.parse_week("current")
|
||||
first = since or (have[0] if have else current)
|
||||
return [week for week in c.week_span(first, current) if week not in have]
|
||||
|
||||
|
||||
def collect_week(week, args, forge=None):
|
||||
label, monday, sunday = c.parse_week(week)
|
||||
start, end = c.week_bounds_utc(monday, sunday)
|
||||
repos = [r.strip() for r in args.repos.split(",")] if args.repos else c.REPOS
|
||||
forge = forge or c.Forge()
|
||||
|
||||
notes = []
|
||||
def note(message):
|
||||
notes.append(message)
|
||||
print(f"[project-state] {message}", file=sys.stderr)
|
||||
|
||||
evidence = {"commits": [], "ci": [], "issues": [], "pulls": [], "releases": []}
|
||||
repo_summary = {}
|
||||
before = forge.calls
|
||||
for repo in repos:
|
||||
rows = commits(forge, repo, start, end, note)
|
||||
evidence["commits"] += rows
|
||||
evidence["ci"] += runs(forge, repo, start, end, note)
|
||||
evidence["issues"] += issues(forge, repo, start, end, note)
|
||||
evidence["pulls"] += pulls(forge, repo, start, end, note)
|
||||
evidence["releases"] += releases(forge, repo, start, end, note)
|
||||
repo_summary[repo] = {"commits": len(rows), "checkout": bool(c.checkout(repo))}
|
||||
index = {}
|
||||
evidence["declared"] = declared(start, end, note, index)
|
||||
evidence["index"] = index
|
||||
|
||||
for key in ("commits", "ci"):
|
||||
evidence[key].sort(key=lambda row: row.get("at") or row.get("started") or "")
|
||||
|
||||
if args.redact:
|
||||
for row in evidence["commits"]:
|
||||
row["subject"] = c.redact(row["subject"])
|
||||
row["detail"] = c.redact(row["detail"])
|
||||
|
||||
out = {
|
||||
"week": label,
|
||||
"range": {
|
||||
"start": monday.isoformat(), "end": sunday.isoformat(), "tz": str(c.LOCAL),
|
||||
"start_utc": c.iso_z(start), "end_utc": c.iso_z(end),
|
||||
},
|
||||
"collected_at": c.iso_z(dt.datetime.now(dt.timezone.utc)),
|
||||
"collector": {
|
||||
"tool": "tools/project-state/collect.py",
|
||||
"version": VERSION,
|
||||
"forge": f"gitea {forge.version()}",
|
||||
"api_calls": forge.calls - before,
|
||||
"redacted": bool(args.redact),
|
||||
"repos": repo_summary,
|
||||
"notes": notes,
|
||||
},
|
||||
**evidence,
|
||||
}
|
||||
out["rollup"] = rollup(out, start, end)
|
||||
return out
|
||||
|
||||
|
||||
def write(out, path):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
open(path, "w").write(json.dumps(out, indent=2, ensure_ascii=False) + "\n")
|
||||
print(f"[project-state] {out['week']}: {len(out['commits'])} commits, {len(out['ci'])} runs, "
|
||||
f"{len(out['declared'])} declared changes, {out['collector']['api_calls']} api calls "
|
||||
f"-> {path}", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--week", default="last", help="2026-W33 | current | last (default)")
|
||||
ap.add_argument("--repos", help="comma-separated override of the repo list")
|
||||
ap.add_argument("--out", help="output path, '-' for stdout")
|
||||
ap.add_argument("--redact", action="store_true",
|
||||
help="mask hosts and home paths — for evidence leaving the forge")
|
||||
ap.add_argument("--gaps", action="store_true", help="name the weeks holding no evidence")
|
||||
ap.add_argument("--backfill", action="store_true", help="collect every week --gaps names")
|
||||
ap.add_argument("--since", help="earliest week --gaps and --backfill consider")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.gaps or args.backfill:
|
||||
missing = gaps(args.since)
|
||||
if not missing:
|
||||
print(f"[project-state] no gaps; {len(held())} week(s) held", file=sys.stderr)
|
||||
return 0
|
||||
if args.gaps:
|
||||
print("\n".join(missing))
|
||||
return 0
|
||||
forge = c.Forge()
|
||||
if not forge.token:
|
||||
print("project-state: no forge token (set GITEA_TOKEN)", file=sys.stderr)
|
||||
return 2
|
||||
for week in missing:
|
||||
write(collect_week(week, args, forge),
|
||||
os.path.join(ROOT, "project-state", "evidence", f"{week}.json"))
|
||||
return 0
|
||||
|
||||
forge = c.Forge()
|
||||
if not forge.token:
|
||||
print("project-state: no forge token (set GITEA_TOKEN)", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
out = collect_week(args.week, args, forge)
|
||||
if args.out == "-":
|
||||
sys.stdout.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n")
|
||||
return 0
|
||||
write(out, args.out or os.path.join(ROOT, "project-state", "evidence", f"{out['week']}.json"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
276
tools/project-state/common.py
Normal file
276
tools/project-state/common.py
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
"""Shared pieces for the project-state pass: forge client, weeks, dimensions.
|
||||
|
||||
See docs/tasks/79-project-state.md for what this is for and why it is shaped
|
||||
this way.
|
||||
"""
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
LOCAL = ZoneInfo("America/Toronto")
|
||||
OWNER = "Fimeg"
|
||||
PROJECTS = os.path.expanduser("~/Projects")
|
||||
|
||||
# Listed, not discovered. A repo joining Souveraine is a deliberate addition
|
||||
# here, and one that goes quiet should be visible as a quiet row rather than
|
||||
# silently stop being read. Same reasoning as tools/ci-status.sh.
|
||||
REPOS = [
|
||||
"souveraine", "SouveraineOS", "Pixel3Arch", "souveraine-viewtop",
|
||||
"souveraine-updater", "souveraine-lens", "souveraine-speech",
|
||||
"souveraine-player", "souveraine-usb", "culver", "linux-blueline",
|
||||
"hexagonrpc", "smithay", "pocketboot", "libcmatrix", "upower",
|
||||
]
|
||||
|
||||
# The SAF sections, plus the two the pipeline owns and SAF has no page for.
|
||||
DIMENSIONS = [
|
||||
"memory", "runtime", "reflection", "identity", "surface",
|
||||
"sensorium", "release", "reliability", "federation", "adoption",
|
||||
]
|
||||
|
||||
UNASSIGNED = "unassigned"
|
||||
|
||||
# Longer keywords score higher, so "device state" beats a bare "state" and a
|
||||
# path full of a subsystem's name beats one incidental word in a subject.
|
||||
KEYWORDS = {
|
||||
"memory": ["memory", "memfs", "compaction", "archivist", "recall", "vault", "memoir"],
|
||||
"runtime": ["server", "handler", "runtime", "substrate", "provider", "tool call",
|
||||
"stream", "sse", "prompt", "context", "turn", "inference", "token"],
|
||||
"reflection": ["reflection", "subconscious", "cadence", "n+1", "aster", "introspect"],
|
||||
"identity": ["identity", "authority", "sessiond", "principal", "secrets", "machined",
|
||||
"trust", "login", "credential", "permission", "polkit", "device state",
|
||||
"device_state", "admission"],
|
||||
"surface": ["shell", "panel", "tui", "surface", "qml", "dock", "dial", "osk",
|
||||
"keyboard", "island", "overview", "compositor", "viewtop", "window",
|
||||
"gesture", "lockscreen", "card", "render"],
|
||||
"sensorium": ["sensor", "grip", "battery", "charge", "audio", "microphone", "camera",
|
||||
"haptic", "bluetooth", "airpods", "librepods", "modem", "bearer",
|
||||
"brightness", "fingerprint", "touch", "squeeze"],
|
||||
"release": ["release", "package", "pkgbuild", "pacman", "repo-add", "distribution",
|
||||
"artifact", "signing", "manifest"],
|
||||
"reliability": ["crash", "stall", "deadlock", "leak", "regression", "revert",
|
||||
"hotfix", "clippy", "flake", "panic", "timeout", "retry", "workflow"],
|
||||
"federation": ["federation", "enrol", "enroll", "peer", "mirror", "distributed",
|
||||
"multi-device", "household", "sync"],
|
||||
"adoption": ["license", "licence", "install", "provision", "onboard",
|
||||
"quickstart", "getting started", "contributor"],
|
||||
}
|
||||
|
||||
# Two path shapes that no keyword scan should have to guess at.
|
||||
PATH_RULES = [
|
||||
("reliability", r"(^|/)\.gitea/"),
|
||||
("release", r"(^|/)(packaging|pkgs?)/|PKGBUILD"),
|
||||
]
|
||||
|
||||
# Only for a commit that neither its paths nor its subject place. A repo whose
|
||||
# whole purpose is one dimension answers for it.
|
||||
REPO_DEFAULT = {
|
||||
"souveraine": "runtime",
|
||||
"Pixel3Arch": "sensorium",
|
||||
"linux-blueline": "sensorium",
|
||||
"souveraine-viewtop": "surface",
|
||||
"souveraine-lens": "adoption",
|
||||
"souveraine-updater": "release",
|
||||
"souveraine-player": "surface",
|
||||
"souveraine-speech": "sensorium",
|
||||
"souveraine-usb": "federation",
|
||||
"culver": "surface",
|
||||
"hexagonrpc": "sensorium",
|
||||
"smithay": "surface",
|
||||
"pocketboot": "release",
|
||||
"libcmatrix": "surface",
|
||||
"upower": "sensorium",
|
||||
}
|
||||
|
||||
# Commit subjects are written for the man who owns the machines. Evidence stays
|
||||
# raw because it never leaves the forge; this is here for whatever gets promoted
|
||||
# to a public surface later. Same patterns as the ledger's collector.
|
||||
REDACTIONS = [
|
||||
(re.compile(r'\b(?:10|127)(?:\.\d{1,3}){2,3}(?::\d+)?\b'), "[host]"),
|
||||
(re.compile(r'\b192\.168(?:\.\d{1,3}){1,2}(?::\d+)?\b'), "[host]"),
|
||||
(re.compile(r'\b172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){1,2}(?::\d+)?\b'), "[host]"),
|
||||
(re.compile(r'\b(?:WIUF[-\w]*|wiufph|archdev)\b', re.I), "[host]"),
|
||||
(re.compile(r'/home/casey|/root\b'), "~"),
|
||||
]
|
||||
|
||||
|
||||
def redact(text):
|
||||
for pattern, mask in REDACTIONS:
|
||||
text = pattern.sub(mask, text)
|
||||
return text
|
||||
|
||||
|
||||
def score(text):
|
||||
"""Dimension scores for any text; ties go to the earlier dimension."""
|
||||
low = (text or "").lower()
|
||||
hits = {d: sum(len(k) for k in words if k in low) for d, words in KEYWORDS.items()}
|
||||
hits = {d: s for d, s in hits.items() if s}
|
||||
if not hits:
|
||||
return None, {}
|
||||
best = max(hits, key=lambda d: (hits[d], -DIMENSIONS.index(d)))
|
||||
return best, hits
|
||||
|
||||
|
||||
def classify_text(text):
|
||||
"""For a task row or a document: what is this about, or nothing."""
|
||||
best, _ = score(text)
|
||||
return (best, "text") if best else (UNASSIGNED, "none")
|
||||
|
||||
|
||||
def classify(repo, subject, paths=None):
|
||||
"""One dimension for a commit, and the basis that chose it.
|
||||
|
||||
Paths and subject are both read and weighted alike. Paths alone put
|
||||
`compaction: target the active conversation` under the server module it
|
||||
edits; the subject is the author saying which subsystem he thinks he moved.
|
||||
"""
|
||||
joined = " ".join(paths or [])
|
||||
_, from_paths = score(joined)
|
||||
_, from_subject = score(subject)
|
||||
total = {d: from_paths.get(d, 0) + from_subject.get(d, 0)
|
||||
for d in set(from_paths) | set(from_subject)}
|
||||
if total:
|
||||
best = max(total, key=lambda d: (total[d], -DIMENSIONS.index(d)))
|
||||
return best, "paths" if from_paths.get(best, 0) else "subject"
|
||||
for dim, pattern in PATH_RULES:
|
||||
if joined and re.search(pattern, joined, re.I):
|
||||
return dim, "paths"
|
||||
if repo in REPO_DEFAULT:
|
||||
return REPO_DEFAULT[repo], "repo"
|
||||
return UNASSIGNED, "none"
|
||||
|
||||
|
||||
# ---- weeks -----------------------------------------------------------------
|
||||
|
||||
def parse_week(spec, today=None):
|
||||
"""'2026-W33', 'current', 'last' -> (label, monday, sunday) as local dates."""
|
||||
today = today or dt.datetime.now(LOCAL).date()
|
||||
if spec in ("current", "last", None):
|
||||
monday = today - dt.timedelta(days=today.isoweekday() - 1)
|
||||
if spec == "last":
|
||||
monday -= dt.timedelta(days=7)
|
||||
else:
|
||||
m = re.fullmatch(r"(\d{4})-?W(\d{1,2})", spec.upper())
|
||||
if not m:
|
||||
raise ValueError(f"week must look like 2026-W33, got {spec!r}")
|
||||
monday = dt.date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1)
|
||||
year, week, _ = monday.isocalendar()
|
||||
return f"{year}-W{week:02d}", monday, monday + dt.timedelta(days=6)
|
||||
|
||||
|
||||
def week_bounds_utc(monday, sunday):
|
||||
"""Local week -> UTC instants, so a Sunday-evening commit stays in its week."""
|
||||
start = dt.datetime.combine(monday, dt.time.min, LOCAL).astimezone(dt.timezone.utc)
|
||||
end = dt.datetime.combine(sunday + dt.timedelta(days=1), dt.time.min, LOCAL).astimezone(dt.timezone.utc)
|
||||
return start, end
|
||||
|
||||
|
||||
def week_monday(label):
|
||||
year, week = label.split("-W")
|
||||
return dt.date.fromisocalendar(int(year), int(week), 1)
|
||||
|
||||
|
||||
def label_for(monday):
|
||||
y, w, _ = monday.isocalendar()
|
||||
return f"{y}-W{w:02d}"
|
||||
|
||||
|
||||
def previous_week(label):
|
||||
return label_for(week_monday(label) - dt.timedelta(days=7))
|
||||
|
||||
|
||||
def week_span(first, last):
|
||||
"""Every ISO week label from first to last inclusive, in order."""
|
||||
monday, end, out = week_monday(first), week_monday(last), []
|
||||
while monday <= end:
|
||||
out.append(label_for(monday))
|
||||
monday += dt.timedelta(days=7)
|
||||
return out
|
||||
|
||||
|
||||
def iso_z(when):
|
||||
return when.astimezone(dt.timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def parse_iso(value):
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
# ---- forge -----------------------------------------------------------------
|
||||
|
||||
class Forge:
|
||||
# Gitea clamps a page to MAX_RESPONSE_ITEMS (50) and ignores a larger limit
|
||||
# without saying so. The ledger lost 24 of 74 repos to this for months.
|
||||
PAGE = 50
|
||||
|
||||
def __init__(self, base=None, token=None):
|
||||
self.base = (base or os.environ.get("GITEA_URL") or "http://10.10.20.120:4455").rstrip("/")
|
||||
self.token = token or os.environ.get("GITEA_TOKEN") or self._stored_token()
|
||||
self.calls = 0
|
||||
|
||||
def _stored_token(self):
|
||||
path = os.path.expanduser("~/.git-credentials")
|
||||
host = urllib.parse.urlsplit(self.base).hostname or ""
|
||||
try:
|
||||
for line in open(path):
|
||||
parsed = urllib.parse.urlsplit(line.strip())
|
||||
if parsed.hostname and parsed.hostname.split("%")[0] == host:
|
||||
return parsed.password or ""
|
||||
except OSError:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def get(self, path):
|
||||
req = urllib.request.Request(f"{self.base}/api/v1/{path}")
|
||||
if self.token:
|
||||
req.add_header("Authorization", f"token {self.token}")
|
||||
self.calls += 1
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.load(r)
|
||||
|
||||
def pages(self, path, max_pages=40):
|
||||
sep = "&" if "?" in path else "?"
|
||||
out = []
|
||||
for page in range(1, max_pages + 1):
|
||||
chunk = self.get(f"{path}{sep}limit={self.PAGE}&page={page}")
|
||||
if isinstance(chunk, dict):
|
||||
chunk = chunk.get("workflow_runs") or chunk.get("data") or []
|
||||
if not chunk:
|
||||
return out
|
||||
out += chunk
|
||||
if len(chunk) < self.PAGE:
|
||||
return out
|
||||
raise RuntimeError(f"{path}: {max_pages} pages and still going")
|
||||
|
||||
def version(self):
|
||||
return self.get("version").get("version", "?")
|
||||
|
||||
|
||||
# ---- local checkouts -------------------------------------------------------
|
||||
|
||||
def checkout(repo):
|
||||
path = os.path.join(PROJECTS, repo)
|
||||
return path if os.path.isdir(os.path.join(path, ".git")) else None
|
||||
|
||||
|
||||
def git(path, *args):
|
||||
proc = subprocess.run(["git", "-C", path, *args], capture_output=True, text=True)
|
||||
return proc.stdout if proc.returncode == 0 else ""
|
||||
|
||||
|
||||
def commit_paths(repo, sha):
|
||||
"""Changed paths from the local clone, empty when it doesn't have the SHA."""
|
||||
path = checkout(repo)
|
||||
if not path:
|
||||
return []
|
||||
out = git(path, "show", "--name-only", "--format=", "--no-renames", sha)
|
||||
return [line for line in out.splitlines() if line.strip()]
|
||||
821
tools/project-state/synthesize.py
Normal file
821
tools/project-state/synthesize.py
Normal file
|
|
@ -0,0 +1,821 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Turn a week of evidence plus last week's state into this week's state.
|
||||
|
||||
Deterministic rules run first and produce every finding that can be computed.
|
||||
A model, when one is configured, only reads those findings and the rollup and
|
||||
adds cross-cutting observations; it is never asked to discover a fact. Model
|
||||
findings that cite an evidence id the week does not contain are dropped.
|
||||
|
||||
tools/project-state/synthesize.py --week 2026-W33
|
||||
tools/project-state/synthesize.py --week 2026-W33 --interpret
|
||||
tools/project-state/synthesize.py --explain f-0007
|
||||
|
||||
Env: ANALYST_URL, ANALYST_KEY, ANALYST_MODEL — any OpenAI-compatible endpoint.
|
||||
See docs/tasks/79-project-state.md.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import ipaddress
|
||||
import re
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import common as c
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
STORE = os.path.join(ROOT, "project-state")
|
||||
VERSION = 1
|
||||
STALE_DAYS = 14
|
||||
SAMPLE_PER_DIMENSION = 10
|
||||
MODEL_MAX = 6
|
||||
|
||||
MOVEMENTS = ["milestone_reached", "regressed", "blocked", "decision_required",
|
||||
"advanced", "unchanged"]
|
||||
# Rules whose findings survive a week that says nothing about them. The rest
|
||||
# describe one week and lapse with it.
|
||||
CARRIES = {"task-standing", "task-listed", "task-moved", "task-restated",
|
||||
"task-delisted", "pull-aging", "issue-aging", "issue-tracker-unused"}
|
||||
|
||||
CHARTER = """You read one week of evidence from the Souveraine project. Deterministic code has
|
||||
already counted everything countable and its findings are included so you do not repeat
|
||||
them. Your job is the three things code cannot do:
|
||||
|
||||
1. Work in one subsystem that changes what is true in another.
|
||||
2. A contradiction between what the repository declares about itself and what its
|
||||
commits, CI runs or pull requests show.
|
||||
3. An assumption this week invalidated.
|
||||
|
||||
Rules, and a finding that breaks one is discarded unread:
|
||||
- Write about the project, never about the findings list. "The deterministic finding
|
||||
says X" is not a finding. Never count anything.
|
||||
- Cite at least two evidence ids of different kinds — a commit id (c:) with a declared
|
||||
change (d:), a CI run (ci:) with a pull request (p:), and so on. One kind alone is a
|
||||
restatement, not a connection.
|
||||
- Never say that two commits share a dimension, or that one id appears in two samples.
|
||||
That is an artefact of how the evidence was sampled, not a fact about the project.
|
||||
- At most six findings. Fewer is better. An empty list is a legal answer.
|
||||
- One or two plain sentences. No hype, no praise, no summary of the week.
|
||||
- Do not invent releases, decisions, people, or outcomes.
|
||||
|
||||
Answer with JSON only: {"findings": [{"dimension": ..., "movement": ..., "title": ...,
|
||||
"statement": ..., "evidence": ["id", ...]}]}
|
||||
Movements: milestone_reached, regressed, blocked, decision_required, advanced, unchanged."""
|
||||
|
||||
|
||||
def load(path, default=None):
|
||||
try:
|
||||
return json.load(open(path))
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return default if default is not None else {}
|
||||
|
||||
|
||||
def finding(key, dimension, title, movement, statement, evidence, rule):
|
||||
return {
|
||||
"key": key,
|
||||
"dimension": dimension,
|
||||
"title": title,
|
||||
"movement": movement,
|
||||
"statement": statement,
|
||||
"evidence": evidence,
|
||||
"provenance": {"by": "deterministic", "rule": rule},
|
||||
}
|
||||
|
||||
|
||||
# ---- deterministic rules ---------------------------------------------------
|
||||
|
||||
def ci_findings(ev, prior_ev):
|
||||
out = []
|
||||
for repo, stat in sorted(ev["rollup"]["ci"].items()):
|
||||
runs, failed = stat["runs"], stat["failure"]
|
||||
was = (prior_ev.get("rollup", {}).get("ci", {}) or {}).get(repo)
|
||||
rate = failed / runs if runs else 0
|
||||
was_rate = (was["failure"] / was["runs"]) if was and was["runs"] else None
|
||||
newest = stat.get("newest") or {}
|
||||
cite = [newest["id"]] if newest.get("id") else []
|
||||
cite += [row["id"] for row in ev["ci"]
|
||||
if row["repo"] == repo and row["conclusion"] == "failure"
|
||||
and row["id"] not in cite][:5]
|
||||
tally = f"{failed} of {runs} runs failed"
|
||||
if stat["other"]:
|
||||
tally += f" and {stat['other']} never concluded"
|
||||
if newest.get("conclusion") == "failure":
|
||||
movement, title = "regressed", f"{repo} ends the week red"
|
||||
statement = f"The last run of the week on {repo} failed; {tally}."
|
||||
elif was_rate is not None and rate > was_rate + 0.15:
|
||||
movement, title = "regressed", f"{repo} CI failure rate rose"
|
||||
statement = f"{tally} on {repo} — {rate:.0%}, up from {was_rate:.0%}."
|
||||
elif failed == 0 and runs:
|
||||
movement, title = "advanced", f"{repo} CI clean"
|
||||
statement = f"{runs} runs on {repo}, none failed."
|
||||
else:
|
||||
movement, title = "unchanged", f"{repo} CI mixed"
|
||||
statement = f"On {repo}, {tally}."
|
||||
out.append(finding(f"ci/{repo}", "reliability", title, movement,
|
||||
statement, cite, "ci-week"))
|
||||
return out
|
||||
|
||||
|
||||
def dimension_findings(ev, prior_ev):
|
||||
out = []
|
||||
now = ev["rollup"]["by_dimension"]
|
||||
was = prior_ev.get("rollup", {}).get("by_dimension", {})
|
||||
for dim in c.DIMENSIONS:
|
||||
here = now.get(dim, {})
|
||||
commits = here.get("commits", 0)
|
||||
declared = [row for row in ev["declared"] if row["dimension"] == dim]
|
||||
before = was.get(dim, {}).get("commits", 0) if was else None
|
||||
cite = [row["id"] for row in ev["commits"] if row["dimension"] == dim][:8]
|
||||
cite += [row["id"] for row in declared][:4]
|
||||
if not commits and not declared:
|
||||
if before:
|
||||
statement = f"No commits, after {before} the week before."
|
||||
elif before == 0:
|
||||
statement = "A second quiet week."
|
||||
else:
|
||||
statement = "No commits, and no week before this one to compare against."
|
||||
out.append(finding(f"activity/{dim}", dim, f"{dim} did not move",
|
||||
"unchanged", statement, cite or [f"w:{ev['week']}"],
|
||||
"dimension-activity"))
|
||||
continue
|
||||
parts = [f"{commits} commit(s)"]
|
||||
if before is not None:
|
||||
parts.append(f"{before} the week before")
|
||||
statement = ", ".join(parts) + "."
|
||||
if declared:
|
||||
named = "; ".join(f"{row['change']} {row.get('title') or row['file']}"
|
||||
for row in declared[:4])
|
||||
statement += f" Declared: {named}."
|
||||
out.append(finding(f"activity/{dim}", dim, f"{dim} moved",
|
||||
"advanced" if declared else "unchanged", statement,
|
||||
cite, "dimension-activity"))
|
||||
return out
|
||||
|
||||
|
||||
BACKWARD = {"Live": 3, "In progress": 2, "Open": 1, "Blocked or gated": 0}
|
||||
# Casey writes the open question into the row itself. These are his phrasings,
|
||||
# read off the index on 2026-08-18 — not a general vocabulary of hesitation.
|
||||
DECISION = re.compile(
|
||||
r"verdict owed|decisions? (?:are|first|before)|needs a decision|open decision"
|
||||
r"|unanswered|undecided|scope and deliverable|answer it before|product call"
|
||||
r"|Casey'?s(?: call| own)?\b|owed\b", re.I)
|
||||
|
||||
|
||||
def movement_for(row):
|
||||
text = row.get("now") or row.get("was") or ""
|
||||
if row["section"] == "Blocked or gated":
|
||||
return "blocked"
|
||||
if DECISION.search(text):
|
||||
return "decision_required"
|
||||
return "unchanged" if row["section"] == "Open" else "advanced"
|
||||
|
||||
|
||||
def task_findings(ev):
|
||||
"""One finding per task standing in the index, moved by this week's diff.
|
||||
|
||||
Tracking only the diff made every restatement a permanent finding and the
|
||||
task itself invisible; the index is the state, the diff is the movement.
|
||||
"""
|
||||
changes = {row["task"]: row for row in ev["declared"] if row["kind"] == "task-index"}
|
||||
out = []
|
||||
for number, row in sorted(ev.get("index", {}).items(), key=lambda kv: int(kv[0])):
|
||||
change = changes.get(number)
|
||||
dim = c.classify_text(f"{row['title']} {row['file']} {row['left']}")[0]
|
||||
key, section, left = f"task/{number}", row["section"], row["left"][:240]
|
||||
cite = [change["id"]] if change else [f"d:task-{number}"]
|
||||
state = {"section": section, "left": row["left"], "file": row["file"]}
|
||||
if not change:
|
||||
f = finding(key, dim, f"task {number}: {row['title']}",
|
||||
"blocked" if section == "Blocked or gated" else "unchanged",
|
||||
f"Standing under “{section}”. {left}", cite, "task-standing")
|
||||
elif change["change"] == "listed":
|
||||
f = finding(key, dim, f"task {number} entered the index: {row['title']}",
|
||||
movement_for(change), f"Filed under “{section}”. {left}",
|
||||
cite, "task-listed")
|
||||
elif change["change"] == "moved":
|
||||
old, new = change["detail"].split(" → ")
|
||||
back = BACKWARD.get(new, 9) < BACKWARD.get(old, 9)
|
||||
f = finding(key, dim, f"task {number} moved: {row['title']}",
|
||||
"regressed" if back else movement_for(change),
|
||||
f"{old} → {new}. {left}", cite, "task-moved")
|
||||
else:
|
||||
f = finding(key, dim, f"task {number} restated: {row['title']}",
|
||||
movement_for(change),
|
||||
f"What's left was rewritten under “{section}”. {left}",
|
||||
cite, "task-restated")
|
||||
f["task_state"] = state
|
||||
out.append(f)
|
||||
for number, change in sorted(changes.items(), key=lambda kv: int(kv[0])):
|
||||
if change["change"] != "delisted":
|
||||
continue
|
||||
out.append(finding(f"task/{number}", change["dimension"],
|
||||
f"task {number} left the index: {change['title']}",
|
||||
"milestone_reached",
|
||||
f"Dropped from “{change['detail']}” — closed work moves to archive/.",
|
||||
[change["id"]], "task-delisted"))
|
||||
return out
|
||||
|
||||
|
||||
def aging_findings(ev):
|
||||
out = []
|
||||
for issue in ev["issues"]:
|
||||
if issue["state"] != "open" or (issue["age_days"] or 0) < STALE_DAYS:
|
||||
continue
|
||||
out.append(finding(f"issue/{issue['repo']}#{issue['number']}",
|
||||
c.classify_text(issue["title"])[0],
|
||||
f"{issue['repo']}#{issue['number']} open {issue['age_days']}d: {issue['title'][:70]}",
|
||||
"unchanged",
|
||||
f"Filed {issue['created'][:10]}, still open.",
|
||||
[issue["id"]], "issue-aging"))
|
||||
for pull in ev["pulls"]:
|
||||
if pull["state"] != "open" or (pull["age_days"] or 0) < STALE_DAYS:
|
||||
continue
|
||||
out.append(finding(f"pull/{pull['repo']}#{pull['number']}",
|
||||
c.classify_text(pull["title"])[0],
|
||||
f"{pull['repo']}#{pull['number']} unmerged {pull['age_days']}d: {pull['title'][:70]}",
|
||||
"decision_required",
|
||||
f"Opened {pull['created'][:10]}, never merged or closed.",
|
||||
[pull["id"]], "pull-aging"))
|
||||
for rel in ev["releases"]:
|
||||
if rel["cut_this_week"]:
|
||||
out.append(finding(f"release/{rel['repo']}", "release",
|
||||
f"{rel['repo']} released {rel['tag']}", "milestone_reached",
|
||||
f"{rel['assets']} assets published {rel['published'][:10]}.",
|
||||
[rel["id"]], "release-cut"))
|
||||
issues = ev["rollup"]["issues"]
|
||||
if issues["open"] and not issues["ever_closed"] and issues["oldest_open_days"] >= STALE_DAYS:
|
||||
oldest = next((i for i in ev["issues"] if i["id"] == issues["oldest_open"]), None)
|
||||
out.append(finding("issues/never-closed", "adoption",
|
||||
"No issue has ever been closed", "unchanged",
|
||||
f"{issues['open']} open, none closed since the tracker was first used"
|
||||
+ (f"; the oldest is {oldest['repo']}#{oldest['number']}, "
|
||||
f"{oldest['age_days']}d." if oldest else "."),
|
||||
[i["id"] for i in ev["issues"] if i["state"] == "open"][:8],
|
||||
"issue-tracker-unused"))
|
||||
return out
|
||||
|
||||
|
||||
def deterministic(ev, prior_ev):
|
||||
return (task_findings(ev) + ci_findings(ev, prior_ev)
|
||||
+ aging_findings(ev) + dimension_findings(ev, prior_ev))
|
||||
|
||||
|
||||
# ---- model pass ------------------------------------------------------------
|
||||
|
||||
def locality(url):
|
||||
host = urllib.parse.urlsplit(url).hostname or ""
|
||||
try:
|
||||
return "local" if ipaddress.ip_address(host).is_private else "remote"
|
||||
except ValueError:
|
||||
return "local" if host in ("localhost", "127.0.0.1") else "remote"
|
||||
|
||||
|
||||
def parse_findings(text):
|
||||
"""The findings, and whether they were salvaged from a truncated answer.
|
||||
|
||||
A reasoning model's answer arrives after its thinking, so a budget that runs
|
||||
out cuts the array mid-object. Every complete object before the cut is still
|
||||
a whole finding, and throwing them away because the closing bracket is
|
||||
missing loses real work.
|
||||
"""
|
||||
start, end = text.find("{"), text.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
try:
|
||||
return json.loads(text[start:end + 1]).get("findings", []), False
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
out, depth, begin = [], 0, None
|
||||
for i, ch in enumerate(text[text.find("[") + 1:] if "[" in text else ""):
|
||||
if ch == "{":
|
||||
if depth == 0:
|
||||
begin = i
|
||||
depth += 1
|
||||
elif ch == "}" and depth:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
fragment = text[text.find("[") + 1:][begin:i + 1]
|
||||
try:
|
||||
out.append(json.loads(fragment))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return (out, True) if out else (None, False)
|
||||
|
||||
|
||||
class AnalystUnavailable(Exception):
|
||||
"""The named model cannot be run. Never a reason to run a different one."""
|
||||
|
||||
|
||||
def require_model(url, model, note):
|
||||
"""Confirm the endpoint serves the exact model named, or refuse to proceed.
|
||||
|
||||
There is no default model and no fallback. A convenient default made the
|
||||
choice feel fungible, and on 2026-08-18 that ended in three other models
|
||||
being tried after one had been named.
|
||||
"""
|
||||
req = urllib.request.Request(f"{url.rstrip('/')}/v1/models")
|
||||
key = os.environ.get("ANALYST_KEY")
|
||||
if key:
|
||||
req.add_header("Authorization", f"Bearer {key}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
served = [m.get("id") for m in (json.load(r).get("data") or [])]
|
||||
except Exception as e:
|
||||
raise AnalystUnavailable(f"{url} did not answer /v1/models: {e}")
|
||||
if model not in served:
|
||||
near = [m for m in served if model.split("-")[0].lower() in (m or "").lower()][:4]
|
||||
raise AnalystUnavailable(
|
||||
f"{url} does not serve {model!r}"
|
||||
+ (f"; it does serve {near} — name one of those explicitly if you want it"
|
||||
if near else f"; {len(served)} models served, none of them that one"))
|
||||
note(f"analyst model {model} confirmed served by {url}")
|
||||
|
||||
|
||||
def interpret(ev, findings, prior_state, note):
|
||||
url = os.environ.get("ANALYST_URL")
|
||||
model = os.environ.get("ANALYST_MODEL")
|
||||
if not url or not model:
|
||||
raise AnalystUnavailable(
|
||||
"--interpret needs both ANALYST_URL and ANALYST_MODEL; there is no default model")
|
||||
require_model(url, model, note)
|
||||
ids = {row["id"] for group in ("commits", "ci", "issues", "pulls", "releases", "declared")
|
||||
for row in ev[group]}
|
||||
# The whole week is 39k characters of commit subjects. A local 9B spends its
|
||||
# entire budget reading that and never answers (measured 2026-08-18, 323s,
|
||||
# nothing returned). Sampling per dimension is what makes the pass affordable
|
||||
# — and reading every subject is a job for the deterministic half anyway.
|
||||
sample = {}
|
||||
for row in ev["commits"]:
|
||||
seen = sample.setdefault(row["dimension"], [])
|
||||
if len(seen) < SAMPLE_PER_DIMENSION:
|
||||
seen.append({"id": row["id"], "subject": row["subject"]})
|
||||
brief = {
|
||||
"week": ev["week"],
|
||||
"counts": {"by_dimension": ev["rollup"]["by_dimension"], "ci": ev["rollup"]["ci"],
|
||||
"issues": ev["rollup"]["issues"], "pulls": ev["rollup"]["pulls"]},
|
||||
"deterministic_findings": [
|
||||
{"dimension": f["dimension"], "movement": f["movement"], "title": f["title"]}
|
||||
for f in findings],
|
||||
"commit_sample": sample,
|
||||
"declared": [{"id": row["id"], "change": row["change"], "detail": row["detail"],
|
||||
"title": row.get("title") or row.get("file")} for row in ev["declared"]],
|
||||
"open_from_last_week": [
|
||||
{"key": f["key"], "title": f["title"], "weeks_open": f.get("weeks_open")}
|
||||
for f in prior_state.get("findings", []) if f.get("status") == "open"][:30],
|
||||
}
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "system", "content": CHARTER},
|
||||
{"role": "user", "content": json.dumps(brief, ensure_ascii=False)}],
|
||||
# Fable is a reasoning model: it fills `reasoning_content` first and
|
||||
# leaves `content` empty if the budget runs out before it concludes.
|
||||
# Measured 2026-08-18 — 3000 tokens went entirely to reasoning and
|
||||
# returned nothing, and a 250-token cap (what the ledger's narrator
|
||||
# uses) cannot ever answer.
|
||||
"max_tokens": int(os.environ.get("ANALYST_MAX_TOKENS", 6000)),
|
||||
"temperature": 0.3,
|
||||
}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
key = os.environ.get("ANALYST_KEY")
|
||||
if key:
|
||||
headers["Authorization"] = f"Bearer {key}"
|
||||
started = dt.datetime.now(dt.timezone.utc)
|
||||
try:
|
||||
req = urllib.request.Request(f"{url.rstrip('/')}/v1/chat/completions",
|
||||
data=json.dumps(payload).encode(), headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=900) as r:
|
||||
body = json.load(r)
|
||||
except Exception as e:
|
||||
raise AnalystUnavailable(
|
||||
f"{model} failed after {(dt.datetime.now(dt.timezone.utc) - started).seconds}s: {e}")
|
||||
choice = (body.get("choices") or [{}])[0]
|
||||
message = choice.get("message", {})
|
||||
text = (message.get("content") or "").strip()
|
||||
took = (dt.datetime.now(dt.timezone.utc) - started).seconds
|
||||
meta = {"model": model, "endpoint": locality(url), "ok": True, "seconds": took,
|
||||
"usage": body.get("usage"), "finish": choice.get("finish_reason"),
|
||||
"reasoning_chars": len(message.get("reasoning_content") or "")}
|
||||
# An empty findings list is a legal answer. No answer at all is a failure of
|
||||
# the model that was named, and stays that.
|
||||
if not text:
|
||||
raise AnalystUnavailable(
|
||||
f"{model} returned no answer in {took}s — {meta['reasoning_chars']} chars of "
|
||||
f"reasoning, finish={meta['finish']}; the budget went entirely to thinking")
|
||||
parsed, meta["salvaged"] = parse_findings(text)
|
||||
if parsed is None:
|
||||
raise AnalystUnavailable(
|
||||
f"{model} produced {len(text)} chars that are not findings JSON "
|
||||
f"(finish={meta['finish']})")
|
||||
out, dropped, seen = [], 0, set()
|
||||
for i, row in enumerate(parsed):
|
||||
cited = [e for e in (row.get("evidence") or []) if e in ids]
|
||||
kinds = {e.split(":", 1)[0] for e in cited}
|
||||
signature = frozenset(cited)
|
||||
# A connection spans two kinds of evidence. One kind, or a citation set
|
||||
# already used, is the model restating itself.
|
||||
if (len(cited) < 2 or len(kinds) < 2 or signature in seen
|
||||
or row.get("movement") not in MOVEMENTS or len(out) >= MODEL_MAX):
|
||||
dropped += 1
|
||||
continue
|
||||
seen.add(signature)
|
||||
title = str(row.get("title") or "")[:120]
|
||||
out.append({
|
||||
"key": f"model/{ev['week']}/{i}",
|
||||
"dimension": row.get("dimension") if row.get("dimension") in c.DIMENSIONS else c.UNASSIGNED,
|
||||
"title": title,
|
||||
"movement": row["movement"],
|
||||
"statement": str(row.get("statement") or "")[:600],
|
||||
"evidence": cited,
|
||||
"provenance": {"by": "model", "rule": "cross-cutting", **meta},
|
||||
})
|
||||
meta["accepted"], meta["dropped"] = len(out), dropped
|
||||
note(f"analyst: {len(out)} findings kept, {dropped} dropped for uncited or bad movement, {took}s")
|
||||
return out, meta
|
||||
|
||||
|
||||
# ---- carry forward ---------------------------------------------------------
|
||||
|
||||
def carry(prior, produced, week, analyst, closes):
|
||||
"""This week's findings over last week's, with the resolution rule explicit.
|
||||
|
||||
A finding the week says nothing about is still open. Only positive evidence
|
||||
closes one — a task leaving the index, a pull merging. Absence of evidence
|
||||
resolved four CI findings on the first two-week run, which is the difference
|
||||
between a state document and a diff.
|
||||
"""
|
||||
by_key = {f["key"]: f for f in prior.get("findings", [])}
|
||||
held = verdicts()
|
||||
seq = max((int(f["id"][2:]) for f in prior.get("findings", []) if f.get("id", "").startswith("f-")),
|
||||
default=0)
|
||||
stamp = c.iso_z(dt.datetime.now(dt.timezone.utc))
|
||||
weeks = prior.get("weeks", [])
|
||||
if week not in weeks:
|
||||
weeks = sorted(weeks + [week])
|
||||
out, seen = [], set()
|
||||
for new in produced:
|
||||
old = by_key.get(new["key"])
|
||||
seen.add(new["key"])
|
||||
if old:
|
||||
new["id"] = old["id"]
|
||||
new["first_seen"] = old.get("first_seen", week)
|
||||
new["was"] = {"movement": old.get("movement"), "week": old.get("last_seen")}
|
||||
else:
|
||||
seq += 1
|
||||
new["id"] = f"f-{seq:04d}"
|
||||
new["first_seen"] = week
|
||||
new["human"] = held.get(new["key"], {"verdict": None, "note": None, "edited": False})
|
||||
new["last_seen"] = week
|
||||
# A model reading belongs to the week that produced it and is never
|
||||
# carried as project state; it has no human verdict behind it.
|
||||
new["status"] = "reading" if new["provenance"]["by"] == "model" else "open"
|
||||
new["weeks_open"] = sum(1 for w in weeks if w >= new["first_seen"])
|
||||
if "task_state" in new:
|
||||
same_section = old and (old.get("task_state") or {}).get("section") == new["task_state"]["section"]
|
||||
new["section_since"] = old.get("section_since", week) if same_section else week
|
||||
new["weeks_in_section"] = sum(1 for w in weeks if w >= new["section_since"])
|
||||
new["provenance"].setdefault("at", stamp)
|
||||
new["provenance"].setdefault("evidence_week", week)
|
||||
out.append(new)
|
||||
for key, old in by_key.items():
|
||||
if key in seen:
|
||||
continue
|
||||
if old.get("status") == "open":
|
||||
if key in closes:
|
||||
old["status"] = "resolved"
|
||||
old["resolved_in"] = week
|
||||
elif old["provenance"]["rule"] not in CARRIES:
|
||||
# A CI week, a dimension's activity, a release cut: each states
|
||||
# something about one week and says nothing about the next.
|
||||
old["status"] = "lapsed"
|
||||
old["lapsed_in"] = week
|
||||
else:
|
||||
old["weeks_open"] = sum(1 for w in weeks if w >= old.get("first_seen", week))
|
||||
old["carried"] = True
|
||||
out.append(old)
|
||||
out.sort(key=lambda f: (f["status"] != "open", MOVEMENTS.index(f["movement"])
|
||||
if f["movement"] in MOVEMENTS else 9, f["dimension"], f["id"]))
|
||||
return {
|
||||
"version": VERSION,
|
||||
"updated": stamp,
|
||||
"latest_week": week,
|
||||
"weeks": weeks,
|
||||
"analysis": {"week": week, "analyst": analyst,
|
||||
"deterministic_rules": sorted({f["provenance"]["rule"] for f in produced
|
||||
if f["provenance"]["by"] == "deterministic"})},
|
||||
"findings": out,
|
||||
}
|
||||
|
||||
|
||||
# ---- report ----------------------------------------------------------------
|
||||
|
||||
BADGE = {"milestone_reached": "◆ milestone", "regressed": "▼ regressed",
|
||||
"blocked": "■ blocked", "decision_required": "? decision",
|
||||
"advanced": "▲ advanced", "unchanged": "· unchanged"}
|
||||
|
||||
|
||||
def report(state, ev, week):
|
||||
lines = [f"# Souveraine project state — {week}",
|
||||
"",
|
||||
f"{ev['range']['start']} to {ev['range']['end']} ({ev['range']['tz']}). "
|
||||
f"Internal. Generated from `project-state/evidence/{week}.json`; "
|
||||
f"the prose is disposable, the evidence and `state.json` are not.",
|
||||
""]
|
||||
r = ev["rollup"]
|
||||
lines += ["## The week, counted", "",
|
||||
f"- {len(ev['commits'])} commits across "
|
||||
f"{len({row['repo'] for row in ev['commits']})} repos, "
|
||||
f"{len(ev['ci'])} CI runs, {len(ev['declared'])} declared state changes.",
|
||||
f"- Issues: {r['issues']['open']} open, {r['issues']['opened_this_week']} filed, "
|
||||
f"{r['issues']['closed_this_week']} closed. "
|
||||
f"Pulls: {r['pulls']['open']} open, {r['pulls']['merged_this_week']} merged.",
|
||||
"- Dimension basis: "
|
||||
+ ", ".join(f"{n} by {b}" for b, n in sorted(r["dimension_basis"].items(),
|
||||
key=lambda kv: -kv[1]))
|
||||
+ " (`paths` read the diff, `subject` read the commit subject, "
|
||||
"`repo` fell back to the repository, `none` placed nothing).",
|
||||
""]
|
||||
|
||||
open_now = [f for f in state["findings"] if f["status"] == "open"]
|
||||
current = [f for f in open_now if f["last_seen"] == week]
|
||||
standing = [f for f in open_now if f["last_seen"] != week]
|
||||
resolved = [f for f in state["findings"] if f.get("resolved_in") == week]
|
||||
read = [f for f in state["findings"]
|
||||
if f["status"] == "reading" and f["last_seen"] == week]
|
||||
# A task that simply stands under its section is state, not news; it belongs
|
||||
# in Standing, not in a movement group that would be 60 rows long.
|
||||
quiet = [f for f in current if f["provenance"]["rule"] == "task-standing"]
|
||||
current = [f for f in current if f["provenance"]["rule"] != "task-standing"
|
||||
or f["movement"] == "blocked"]
|
||||
for movement in MOVEMENTS:
|
||||
group = [f for f in current if f["movement"] == movement]
|
||||
if not group:
|
||||
continue
|
||||
lines += [f"## {BADGE[movement]}", ""]
|
||||
for f in group:
|
||||
age = f" · open {f['weeks_open']} weeks since {f['first_seen']}" if f["weeks_open"] > 1 else ""
|
||||
was = f["was"]["movement"] if f.get("was") else None
|
||||
turn = f" · was {was.replace('_', ' ')}" if was and was != movement else ""
|
||||
mark = "model" if f["provenance"]["by"] == "model" else f["provenance"]["rule"]
|
||||
lines += [f"**{f['id']} · {f['dimension']} · {f['title']}** ",
|
||||
f"{f['statement']} ",
|
||||
f"<sub>{mark}{age}{turn} · evidence: "
|
||||
f"{', '.join('`' + e + '`' for e in f['evidence'][:8])}"
|
||||
f"{' …' if len(f['evidence']) > 8 else ''}</sub>", ""]
|
||||
if read:
|
||||
analyst = state["analysis"]["analyst"] or {}
|
||||
lines += ["## Read by the analyst — unreviewed", "",
|
||||
f"Written by {analyst.get('model', 'a model')}, not computed. Every id below "
|
||||
f"exists in the week's evidence and every finding spans two kinds of it; "
|
||||
f"neither check makes the reading correct. Nothing here has a human verdict.",
|
||||
""]
|
||||
for f in read:
|
||||
lines += [f"**{f['id']} · {f['dimension']} · {BADGE[f['movement']]} · {f['title']}** ",
|
||||
f"{f['statement']} ",
|
||||
f"<sub>evidence: {', '.join('`' + e + '`' for e in f['evidence'])}</sub>", ""]
|
||||
if resolved:
|
||||
lines += ["## Closed this week", ""]
|
||||
for f in resolved:
|
||||
lines += [f"- **{f['id']} · {f['title']}** — held from {f['first_seen']} "
|
||||
f"to {f['last_seen']}, {f['weeks_open']} week(s)."]
|
||||
lines.append("")
|
||||
held = sorted((f for f in quiet if f["movement"] != "blocked"),
|
||||
key=lambda f: (-f.get("weeks_in_section", 1), int(f["key"].split("/")[1])))
|
||||
if held:
|
||||
lines += ["## Standing longest without moving", "",
|
||||
f"{len(quiet)} task(s) sat in their section untouched this week. "
|
||||
f"The ten that have sat longest:", ""]
|
||||
for f in held[:10]:
|
||||
lines.append(f"- **{f['id']} · {f['dimension']} · {f['title']}** — "
|
||||
f"“{f['task_state']['section']}” for "
|
||||
f"{f.get('weeks_in_section', 1)} week(s), since {f.get('section_since')}.")
|
||||
lines.append("")
|
||||
if standing:
|
||||
lines += ["## Still open, unmentioned by this week's evidence", "",
|
||||
"Nothing closed these; the week simply said nothing about them.", ""]
|
||||
for f in sorted(standing, key=lambda f: -f["weeks_open"]):
|
||||
lines.append(f"- **{f['id']} · {f['dimension']} · {f['title']}** — "
|
||||
f"last evidence {f['last_seen']}, open {f['weeks_open']} week(s) "
|
||||
f"since {f['first_seen']}.")
|
||||
lines.append("")
|
||||
analyst = state["analysis"]["analyst"]
|
||||
lines += ["## Provenance", "",
|
||||
f"- Evidence: `project-state/evidence/{week}.json`, collected "
|
||||
f"{ev['collected_at']} by `{ev['collector']['tool']}` v{ev['collector']['version']} "
|
||||
f"against {ev['collector']['forge']} in {ev['collector']['api_calls']} API calls.",
|
||||
f"- Deterministic rules: {', '.join(state['analysis']['deterministic_rules'])}.",
|
||||
(f"- Analyst: {analyst['model']} ({analyst['endpoint']}), "
|
||||
f"{analyst.get('accepted', 0)} findings kept, {analyst.get('dropped', 0)} dropped."
|
||||
if analyst else "- Analyst: none run; every finding here is deterministic."),
|
||||
f"- Trace one: `tools/project-state/synthesize.py --explain <id>`.", ""]
|
||||
if ev["collector"]["notes"]:
|
||||
lines += ["Collector notes: " + "; ".join(ev["collector"]["notes"]), ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def explain(state, finding_id):
|
||||
f = next((x for x in state.get("findings", []) if x["id"] == finding_id or x["key"] == finding_id), None)
|
||||
if not f:
|
||||
print(f"no finding {finding_id}", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({k: v for k, v in f.items() if k != "evidence"}, indent=2, ensure_ascii=False))
|
||||
week = f["provenance"].get("evidence_week", f["last_seen"])
|
||||
ev = load(os.path.join(STORE, "evidence", f"{week}.json"))
|
||||
index = {row["id"]: (group, row)
|
||||
for group in ("commits", "ci", "issues", "pulls", "releases", "declared")
|
||||
for row in ev.get(group, [])}
|
||||
index[f"w:{week}"] = ("rollup", ev.get("rollup", {}))
|
||||
print(f"\nevidence ({len(f['evidence'])} rows from {week}):")
|
||||
for eid in f["evidence"]:
|
||||
group, row = index.get(eid, (None, None))
|
||||
if not row:
|
||||
print(f" {eid}: MISSING from {week} evidence")
|
||||
continue
|
||||
print(f"\n [{group}] {eid}")
|
||||
print(" " + json.dumps(row, indent=2, ensure_ascii=False).replace("\n", "\n "))
|
||||
return 0
|
||||
|
||||
|
||||
VERDICTS = ("confirm", "promote", "reject", "edit")
|
||||
VERDICT_FILE = "verdicts.json"
|
||||
|
||||
|
||||
def verdicts():
|
||||
"""A person's word on a finding, keyed by finding key.
|
||||
|
||||
Kept outside state.json because state is derived and gets regenerated — a
|
||||
verdict written into it vanished the next time the week was passed over.
|
||||
This file is the one thing here a machine does not write.
|
||||
"""
|
||||
return load(os.path.join(STORE, VERDICT_FILE), {})
|
||||
|
||||
|
||||
def write_verdict(spec, note, text=None, by=None):
|
||||
finding_id, _, verdict = spec.partition("=")
|
||||
if verdict not in VERDICTS:
|
||||
note(f"verdict must be one of {', '.join(VERDICTS)}, not {verdict!r}")
|
||||
return 2
|
||||
state = load(os.path.join(STORE, "state.json"))
|
||||
target = next((f for f in state.get("findings", [])
|
||||
if f["id"] == finding_id or f["key"] == finding_id), None)
|
||||
if not target:
|
||||
note(f"no finding {finding_id}")
|
||||
return 1
|
||||
held = verdicts()
|
||||
held[target["key"]] = {
|
||||
"verdict": verdict,
|
||||
"note": text,
|
||||
"by": by or os.environ.get("USER") or "unknown",
|
||||
"at": c.iso_z(dt.datetime.now(dt.timezone.utc)),
|
||||
"edited": verdict == "edit",
|
||||
"week": state.get("latest_week"),
|
||||
"finding_id": target["id"],
|
||||
"title": target["title"],
|
||||
}
|
||||
path = os.path.join(STORE, VERDICT_FILE)
|
||||
open(path, "w").write(json.dumps(held, indent=2, ensure_ascii=False) + "\n")
|
||||
target["human"] = held[target["key"]]
|
||||
open(os.path.join(STORE, "state.json"), "w").write(
|
||||
json.dumps(state, indent=2, ensure_ascii=False) + "\n")
|
||||
note(f"{target['id']} {verdict} by {held[target['key']]['by']} — {target['title'][:60]}")
|
||||
return 0
|
||||
|
||||
|
||||
def evidence_weeks():
|
||||
path = os.path.join(STORE, "evidence")
|
||||
return sorted(name[:-5] for name in os.listdir(path)) if os.path.isdir(path) else []
|
||||
|
||||
|
||||
def pending(state):
|
||||
"""Evidence weeks the state has not yet passed over, oldest first."""
|
||||
done = set(state.get("weeks", []))
|
||||
return [week for week in evidence_weeks() if week not in done]
|
||||
|
||||
|
||||
def run_week(label, prior, args, note):
|
||||
ev = load(os.path.join(STORE, "evidence", f"{label}.json"))
|
||||
if not ev:
|
||||
note(f"no evidence for {label} — run collect.py --week {label}")
|
||||
return None, None
|
||||
prior_ev = load(os.path.join(STORE, "evidence", f"{c.previous_week(label)}.json"))
|
||||
produced = deterministic(ev, prior_ev)
|
||||
model_rows, analyst = interpret(ev, produced, prior, note) if getattr(args, "interpret", False) else ([], None)
|
||||
# A task closes when it is gone from the index, not when it merely goes
|
||||
# unmentioned. The delisting week still shows its milestone; the week after
|
||||
# is when it stops being open state.
|
||||
listed = set(ev.get("index", {}))
|
||||
closes = {f["key"] for f in prior.get("findings", [])
|
||||
if f["key"].startswith("task/") and f["key"].split("/", 1)[1] not in listed}
|
||||
return carry(prior, produced + model_rows, label, analyst, closes), ev
|
||||
|
||||
|
||||
def persist(state, ev, label):
|
||||
text = report(state, ev, label)
|
||||
for folder in ("reports", "history"):
|
||||
os.makedirs(os.path.join(STORE, folder), exist_ok=True)
|
||||
open(os.path.join(STORE, "reports", f"{label}.md"), "w").write(text)
|
||||
blob = json.dumps(state, indent=2, ensure_ascii=False) + "\n"
|
||||
open(os.path.join(STORE, "history", f"{label}.json"), "w").write(blob)
|
||||
open(os.path.join(STORE, "state.json"), "w").write(blob)
|
||||
return text
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--week", default="last")
|
||||
ap.add_argument("--catch-up", action="store_true",
|
||||
help="pass over every collected week the state has not seen, in order")
|
||||
ap.add_argument("--pending", action="store_true", help="name those weeks and stop")
|
||||
ap.add_argument("--interpret", action="store_true",
|
||||
help="run the model pass; needs ANALYST_URL and ANALYST_MODEL, "
|
||||
"fails closed if that exact model is not served")
|
||||
ap.add_argument("--verdict", help="a person's word on a finding, as f-0069=promote")
|
||||
ap.add_argument("--note", help="text to store with --verdict")
|
||||
ap.add_argument("--by", help="who is giving the verdict (default $USER)")
|
||||
ap.add_argument("--explain", help="print a finding and every evidence row it cites")
|
||||
ap.add_argument("--dry-run", action="store_true", help="print the report, write nothing")
|
||||
ap.add_argument("--report-only", action="store_true",
|
||||
help="rewrite the report from the stored state, running nothing")
|
||||
args = ap.parse_args()
|
||||
|
||||
state_path = os.path.join(STORE, "state.json")
|
||||
prior = load(state_path, {"findings": []})
|
||||
if args.explain:
|
||||
return explain(prior, args.explain)
|
||||
|
||||
notes = []
|
||||
def note(message):
|
||||
notes.append(message)
|
||||
print(f"[project-state] {message}", file=sys.stderr)
|
||||
|
||||
if args.verdict:
|
||||
return write_verdict(args.verdict, note, args.note, args.by)
|
||||
|
||||
if args.pending:
|
||||
weeks = pending(prior)
|
||||
print("\n".join(weeks) if weeks else "", end="\n" if weeks else "")
|
||||
note(f"{len(weeks)} week(s) collected but not yet passed over" if weeks
|
||||
else f"state is current through {prior.get('latest_week', 'nothing')}")
|
||||
return 0
|
||||
|
||||
if args.catch_up:
|
||||
weeks = pending(prior)
|
||||
if not weeks:
|
||||
note(f"nothing pending; state is current through {prior.get('latest_week')}")
|
||||
return 0
|
||||
for label in weeks:
|
||||
state, ev = run_week(label, prior, args, note)
|
||||
if not state:
|
||||
return 2
|
||||
persist(state, ev, label)
|
||||
note(f"{label}: {sum(1 for f in state['findings'] if f['status'] == 'open')} open")
|
||||
prior = state
|
||||
return 0
|
||||
|
||||
label, _, _ = c.parse_week(args.week)
|
||||
ev = load(os.path.join(STORE, "evidence", f"{label}.json"))
|
||||
if not ev:
|
||||
note(f"no evidence for {label} — run collect.py --week {label}")
|
||||
return 2
|
||||
if args.report_only:
|
||||
state = load(os.path.join(STORE, "history", f"{label}.json")) or prior
|
||||
text = report(state, ev, label)
|
||||
if args.dry_run:
|
||||
sys.stdout.write(text)
|
||||
return 0
|
||||
open(os.path.join(STORE, "reports", f"{label}.md"), "w").write(text)
|
||||
note(f"{label}: report rewritten from stored state")
|
||||
return 0
|
||||
|
||||
# Re-running a week already in the state starts from the week before it, so
|
||||
# a second pass does not compound onto its own output.
|
||||
if label in prior.get("weeks", []):
|
||||
prior = load(os.path.join(STORE, "history", f"{c.previous_week(label)}.json"),
|
||||
{"findings": []})
|
||||
elif prior.get("latest_week") and c.previous_week(label) != prior["latest_week"]:
|
||||
note(f"state stops at {prior['latest_week']} and this is {label} — "
|
||||
f"{len(pending(prior))} week(s) unseen; --catch-up runs them in order")
|
||||
|
||||
state, ev = run_week(label, prior, args, note)
|
||||
if not state:
|
||||
return 2
|
||||
if args.dry_run:
|
||||
sys.stdout.write(report(state, ev, label))
|
||||
return 0
|
||||
persist(state, ev, label)
|
||||
produced = [f for f in state["findings"] if f["last_seen"] == label]
|
||||
note(f"{label}: {len(produced)} findings this week, "
|
||||
f"{sum(1 for f in state['findings'] if f['status'] == 'open')} open "
|
||||
f"-> project-state/reports/{label}.md")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except AnalystUnavailable as failure:
|
||||
# Nothing was written. The named model did not run, and no other one
|
||||
# stands in for it.
|
||||
print(f"[project-state] analyst unavailable: {failure}", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
Loading…
Reference in a new issue