Fimeg/SouveraineOS
Watch
1
0
Fork
You've already forked SouveraineOS
 0
Code Releases Activity

carry the publication gate on the candidate ref

Browse source
This commit is contained in:
Fimeg 2026-08-23 20:43:55 -04:00
commit 6bcc3085ad
11 changed files with 1321 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.gitea/workflows/distribution.yml
View file

@ -2,12 +2,12 @@ name: distribution-contract
on:
push:
branches: [main]
branches: [main, public]
paths:
- distribution/**
- tools/validate-distribution.py
pull_request:
branches: [main]
branches: [main, public]
paths:
- distribution/**
- tools/validate-distribution.py

4
.publication-mechanism.json Normal file
View file

@ -0,0 +1,4 @@
{
"manifest_sha256": "3e99da2194b599d29cf0a9a3814740b0e45628dac17fcc39e1ce831977500bc7",
"mechanism_version": 1
}

13
.publication/MANIFEST.json Normal file
View file

@ -0,0 +1,13 @@
{
"files": {
".publication/bin/verify-publication": "686947723311e864196eae1ea8151b678aa4814456d97b34070b8ef39516a453",
".publication/lib/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
".publication/lib/gates.py": "ee542bbb7a7277aaae1129ea3cbc2f9d45e64c7c2f09b5041c88fa62725c8caa",
".publication/lib/giteaapi.py": "74402d43081cd3fbe35058d83cd3e0fef4fcc6777efd85c45c8a72f12ec94cd5",
".publication/lib/history.py": "c1f64124d47324593e69f4dac0d4cd7654e121fc3e56ee5b13092f17ac76f24e",
".publication/lib/mechanism.py": "22f07e1e45be9d84c8dda40b186d67ba31a39918350b8f4eb7aee7c48773a43c",
".publication/lib/policy.py": "f6fa0dff1eedbe5d5451c96eb7fb7d91a52784de7acbef69c9df0e9c9976891c",
".publication/lib/scanner.py": "c57bba77ffbb4827d83e97c749350f39ea19ba6e5d27cd6ff5222114c48871e3"
},
"mechanism_version": 1
}

115
.publication/bin/verify-publication Executable file
View file

@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Prove a candidate SHA may be published. Exit 0 only if nothing objected.
Run inside the consuming repository, by CI, before any push to the destination.
Every failure path exits non-zero; there is no mode in which an error is treated
as permission.
"""
import argparse
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "..", "lib"))
import gates # noqa: E402
import history # noqa: E402
import mechanism # noqa: E402
import policy as policy_mod # noqa: E402
def main(argv=None):
ap = argparse.ArgumentParser(prog="verify-publication")
ap.add_argument("--policy-dir", required=True,
help="checkout of the policy repository (must not be inside --repo-root)")
ap.add_argument("--repo", required=True, help="internal repository, owner/name")
ap.add_argument("--sha", required=True, help="candidate commit, full lowercase hex")
ap.add_argument("--ref", required=True, help="candidate branch short name")
ap.add_argument("--repo-root", default=".")
ap.add_argument("--gitea-url", help="internal Gitea base URL; required unless --no-gates")
ap.add_argument("--token-env", default="GITEA_READ_TOKEN")
ap.add_argument("--no-gates", action="store_true",
help="check policy, mechanism and history only; NEVER sufficient to publish")
args = ap.parse_args(argv)
repo_root = os.path.abspath(args.repo_root)
reasons = []
try:
policy_set = policy_mod.PolicySet.from_directory(args.policy_dir)
pol = policy_set.authorize(args.repo)
policy_mod.assert_policy_is_external(pol, repo_root)
except policy_mod.PolicyError as exc:
_emit(["POLICY " + r for r in exc.reasons], allowed=False)
return 1
pinned_paths = ()
manifest_path = os.path.join(repo_root, ".publication", "MANIFEST.json")
if not os.path.isfile(manifest_path):
reasons.append("MECHANISM .publication/MANIFEST.json is absent")
else:
try:
manifest = mechanism.read_manifest(manifest_path)
pinned_paths = tuple(manifest["files"]) + (".publication/MANIFEST.json",)
reasons.extend("MECHANISM " + r for r in mechanism.verify(
repo_root, manifest, pol.mechanism_manifest_sha256, pol.mechanism_version))
except (json.JSONDecodeError, KeyError) as exc:
reasons.append(f"MECHANISM MANIFEST.json is unusable ({exc})")
cleanup = []
try:
reasons.extend("HISTORY " + r for r in history.verify(
args.ref, repo_root, pol, pinned_paths=pinned_paths, warnings=cleanup))
except history.GitError as exc:
reasons.append(f"HISTORY {exc}")
unproven = []
if args.no_gates:
reasons.append("GATES --no-gates was passed; gate proof was not attempted "
"and this run must not authorize a push")
else:
if not args.gitea_url:
reasons.append("GATES --gitea-url is required to prove gates")
else:
import giteaapi # noqa: E402
try:
client = giteaapi.GiteaClient(args.gitea_url, token_env=args.token_env)
except giteaapi.ApiError as exc:
client = None
reasons.append(f"GATES {exc}")
if client is not None:
request = gates.PublicationRequest(
internal_repository=args.repo,
destination_host=pol.destination_host,
destination_repository=pol.destination_repository,
sha=args.sha, ref=args.ref)
report = gates.evaluate(pol, request, client)
reasons.extend("GATES " + d for d in report.denials)
unproven = report.unproven
_emit(reasons, allowed=not reasons, unproven=unproven, policy=pol, cleanup=cleanup)
return 1 if reasons else 0
def _emit(reasons, allowed, unproven=(), policy=None, cleanup=()):
if policy is not None:
print(f"[publication] {policy.internal_repository} -> "
f"{policy.destination_host}/{policy.destination_repository} "
f"({policy.projection}, publisher {policy.publisher_identity})")
print(f"[publication] verdict: {'ALLOW' if allowed else 'DENY'}")
for r in reasons:
print(f"[publication] DENY {r}")
for u in unproven:
print(f"[publication] unproven: {u}")
if cleanup:
print(f"[publication] {len(cleanup)} cleanup finding(s) -- recorded, not blocking:")
for c in cleanup[:40]:
print(f"[publication] {c}")
if len(cleanup) > 40:
print(f"[publication] ... and {len(cleanup) - 40} more")
if __name__ == "__main__":
raise SystemExit(main())

0
.publication/lib/__init__.py Normal file
View file

299
.publication/lib/gates.py Normal file
View file

@ -0,0 +1,299 @@
"""Required-gate evaluator against the Gitea 1.25.5 Actions API shape.
Gate identity is (workflow_path, job_name). A gate passes only when exactly one
completed, successful job matches it at the exact candidate SHA, on the
permitted ref, under an allowed event, in the correct workflow.
Every other outcome denies. See docs/DESIGN.md for the attack list this shape
answers and the one it cannot (continue-on-error).
"""
RUN_FIELDS = ("id", "path", "head_sha", "head_branch", "event",
"status", "conclusion", "run_number", "run_attempt")
JOB_FIELDS = ("id", "run_id", "name", "head_sha", "head_branch",
"status", "conclusion", "run_attempt")
CONTINUE_ON_ERROR_CAVEAT = (
"a job failing under continue-on-error reports conclusion=success and is "
"indistinguishable from a genuine pass when the API returns no steps")
class Denied(Exception):
def __init__(self, reasons):
self.reasons = list(reasons)
super().__init__("; ".join(self.reasons))
class PublicationRequest:
__slots__ = ("internal_repository", "destination_host",
"destination_repository", "sha", "ref")
def __init__(self, internal_repository, destination_host,
destination_repository, sha, ref):
self.internal_repository = internal_repository
self.destination_host = destination_host
self.destination_repository = destination_repository
self.sha = sha
self.ref = ref
class Report:
def __init__(self):
self.denials = []
self.passed = []
self.unproven = []
@property
def allowed(self):
return not self.denials
def deny(self, code, detail):
self.denials.append(f"{code}: {detail}")
def note_unproven(self, detail):
if detail not in self.unproven:
self.unproven.append(detail)
def summary(self):
verdict = "ALLOW" if self.allowed else "DENY"
lines = [f"{verdict} ({len(self.passed)} gate(s) proven, "
f"{len(self.denials)} denial(s))"]
lines.extend(" proven: " + g for g in self.passed)
lines.extend(" DENY " + d for d in self.denials)
lines.extend(" unproven: " + u for u in self.unproven)
return "\n".join(lines)
def _require_fields(obj, fields, kind):
if not isinstance(obj, dict):
return [f"{kind} is not an object"]
return [f"{kind} is missing field '{f}'" for f in fields if obj.get(f) is None]
def _check_request(policy, request, report):
if request.internal_repository != policy.internal_repository:
report.deny("wrong-repository",
f"request names {request.internal_repository}, policy authorizes "
f"{policy.internal_repository}")
if request.destination_repository != policy.destination_repository:
report.deny("wrong-destination",
f"request targets {request.destination_repository}, policy permits "
f"{policy.destination_repository}")
if request.destination_host != policy.destination_host:
report.deny("wrong-destination-host",
f"request targets {request.destination_host}, policy permits "
f"{policy.destination_host}")
if request.ref not in policy.permitted_refs:
report.deny("wrong-ref",
f"ref {request.ref!r} is not in permitted_refs "
f"{list(policy.permitted_refs)}")
sha = request.sha
if not isinstance(sha, str) or len(sha) != 40 or any(
c not in "0123456789abcdef" for c in sha):
report.deny("malformed-sha", f"{sha!r} is not a full lowercase 40-hex SHA")
def _candidate_runs(policy, request, client, report):
try:
raw_runs = client.list_runs(policy.internal_repository, head_sha=request.sha)
except Exception as exc:
report.deny("api-unreachable", f"could not list runs ({exc})")
return []
runs = []
for r in raw_runs:
missing = _require_fields(r, RUN_FIELDS, "run")
if missing:
report.deny("malformed-run", "; ".join(missing))
continue
if r["head_sha"] != request.sha:
continue
if r["head_branch"] != request.ref:
continue
if r["event"] not in policy.allowed_events:
continue
runs.append(r)
return runs
def _select_runs(policy, gate, runs, report):
matching = [r for r in runs if r["path"] == gate.workflow_path]
if not matching:
report.deny("gate-missing",
f"{gate.workflow_path}::{gate.job_name} has no run at this "
f"SHA/ref/event")
return []
if policy.rerun_policy == "all-attempts-must-succeed":
return matching
top = max(r["run_number"] for r in matching)
latest = [r for r in matching if r["run_number"] == top]
if len(latest) > 1:
report.deny("ambiguous-run",
f"{gate.workflow_path} has {len(latest)} runs sharing run_number "
f"{top}; cannot choose")
return []
return latest
def _jobs_for(client, policy, run, report):
try:
raw = client.list_jobs(policy.internal_repository, run["id"])
except Exception as exc:
report.deny("api-unreachable", f"could not list jobs for run {run['id']} ({exc})")
return None
jobs = []
for j in raw:
missing = _require_fields(j, JOB_FIELDS, "job")
if missing:
report.deny("malformed-job", "; ".join(missing))
return None
jobs.append(j)
return jobs
def _judge_job(gate, job, report, label):
ident = f"{gate.workflow_path}::{gate.job_name} ({label})"
if job["status"] != "completed":
report.deny("gate-incomplete", f"{ident} status={job['status']}")
return False
if job["conclusion"] != "success":
report.deny(f"gate-{job['conclusion']}", f"{ident} conclusion={job['conclusion']}")
return False
steps = job.get("steps")
if not steps:
report.note_unproven(f"{ident}: no steps returned; {CONTINUE_ON_ERROR_CAVEAT}")
return True
failed = [s.get("name") for s in steps if s.get("conclusion") == "failure"]
if failed:
report.deny("gate-step-failed-under-success",
f"{ident} reports success but step(s) failed: {failed}")
return False
return True
def _matrix_drift(gate, jobs, report):
"""Tripwire, not a selector.
Selection is the exact list in matrix_jobs. This counts how many jobs in
the run look like legs of the same declaration and denies on a mismatch, so
a leg added upstream and never approved cannot pass unnoticed. It decides
nothing about which jobs are trusted -- only that the policy is stale.
"""
shaped = [j for j in jobs if j["name"].startswith(gate.job_name + " (")]
if len(shaped) != len(gate.matrix_jobs):
report.deny("matrix-drift",
f"{gate.workflow_path}::{gate.job_name} declares "
f"{len(gate.matrix_jobs)} legs, the run carries {len(shaped)}; "
"approve the change in policy rather than around it")
return False
return True
def _evaluate_matrix(policy, request, gate, runs, client, report):
selected = _select_runs(policy, gate, runs, report)
if not selected:
return
proven = True
for run in selected:
jobs = _jobs_for(client, policy, run, report)
if jobs is None:
return
at_sha = [j for j in jobs
if j["head_sha"] == request.sha and j["head_branch"] == request.ref]
if not _matrix_drift(gate, at_sha, report):
return
for leg in gate.matrix_jobs:
named = [j for j in at_sha if j["name"] == leg]
if not named:
report.deny("gate-missing",
f"{gate.workflow_path}::{leg} absent from run {run['id']}")
return
if policy.rerun_policy == "latest-attempt-must-succeed":
top = max(j["run_attempt"] for j in named)
candidates = [j for j in named if j["run_attempt"] == top]
label = f"run {run['id']} attempt {top}"
if len(candidates) != 1:
report.deny("gate-ambiguous",
f"{gate.workflow_path}::{leg} matched "
f"{len(candidates)} jobs at {label}; exactly one required")
return
else:
candidates = named
label = f"run {run['id']} all attempts"
for job in candidates:
if not _judge_job(gate, job, report, f"{leg}, {label}"):
proven = False
if proven:
report.passed.append(f"{gate.workflow_path}::{gate.job_name} "
f"({len(gate.matrix_jobs)} legs)")
def _evaluate_gate(policy, request, gate, runs, client, report):
if gate.cardinality == "matrix":
return _evaluate_matrix(policy, request, gate, runs, client, report)
if gate.cardinality != "exactly-one":
report.deny("unsupported-cardinality",
f"{gate.workflow_path}::{gate.job_name} declares "
f"{gate.cardinality!r}; not supported in this iteration")
return
selected = _select_runs(policy, gate, runs, report)
if not selected:
return
proven = True
for run in selected:
jobs = _jobs_for(client, policy, run, report)
if jobs is None:
return
named = [j for j in jobs
if j["name"] == gate.job_name
and j["head_sha"] == request.sha
and j["head_branch"] == request.ref]
if not named:
report.deny("gate-missing",
f"{gate.workflow_path}::{gate.job_name} absent from run "
f"{run['id']} (attempt {run['run_attempt']})")
return
if policy.rerun_policy == "latest-attempt-must-succeed":
top_attempt = max(j["run_attempt"] for j in named)
candidates = [j for j in named if j["run_attempt"] == top_attempt]
label = f"run {run['id']} attempt {top_attempt}"
else:
candidates = named
label = f"run {run['id']} all attempts"
if policy.rerun_policy == "latest-attempt-must-succeed" and len(candidates) != 1:
report.deny("gate-ambiguous",
f"{gate.workflow_path}::{gate.job_name} matched "
f"{len(candidates)} jobs at {label}; exactly one required")
return
for job in candidates:
if not _judge_job(gate, job, report, label):
proven = False
if proven:
report.passed.append(f"{gate.workflow_path}::{gate.job_name}")
def evaluate(policy, request, client):
"""Return a Report. Report.allowed is True only if every gate is proven."""
report = Report()
_check_request(policy, request, report)
if report.denials:
return report
runs = _candidate_runs(policy, request, client, report)
if report.denials:
return report
if not runs:
report.deny("no-runs",
f"no run at {request.sha} on {request.ref} under events "
f"{sorted(policy.allowed_events)}")
return report
for gate in policy.required_gates:
_evaluate_gate(policy, request, gate, runs, client, report)
report.note_unproven(
"gate proof shows a green run exists at this SHA on this ref; it does not "
"prove which push produced it")
return report

69
.publication/lib/giteaapi.py Normal file
View file

@ -0,0 +1,69 @@
"""Minimal read-only Gitea Actions client.
The token is read from the environment by name and never logged, echoed, or
placed in a URL. Only GET is implemented: this client exists to prove state,
never to change it.
"""
import json
import os
import urllib.error
import urllib.parse
import urllib.request
DEFAULT_PAGE_SIZE = 50
MAX_PAGES = 40
class ApiError(Exception):
pass
class GiteaClient:
def __init__(self, base_url, token_env="GITEA_READ_TOKEN", opener=None, timeout=20):
self.base_url = base_url.rstrip("/")
self._token = os.environ.get(token_env)
if not self._token:
raise ApiError(f"no read token in ${token_env}")
self._opener = opener or urllib.request.build_opener()
self._timeout = timeout
def _get(self, path, params=None):
url = f"{self.base_url}/api/v1{path}"
if params:
url += "?" + urllib.parse.urlencode(params)
req = urllib.request.Request(url, method="GET")
req.add_header("Authorization", f"token {self._token}")
req.add_header("Accept", "application/json")
try:
with self._opener.open(req, timeout=self._timeout) as resp:
return json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
raise ApiError(f"GET {path} -> HTTP {exc.code}")
except urllib.error.URLError as exc:
raise ApiError(f"GET {path} -> {exc.reason}")
def _paged(self, path, params, key):
out, page = [], 1
while page <= MAX_PAGES:
q = dict(params or {})
q.update({"page": page, "limit": DEFAULT_PAGE_SIZE})
body = self._get(path, q)
batch = body.get(key, []) if isinstance(body, dict) else body
if not batch:
break
out.extend(batch)
if len(batch) < DEFAULT_PAGE_SIZE:
break
page += 1
else:
raise ApiError(f"GET {path} exceeded {MAX_PAGES} pages; refusing to guess")
return out
def list_runs(self, repository, head_sha):
return self._paged(f"/repos/{repository}/actions/runs",
{"head_sha": head_sha}, "workflow_runs")
def list_jobs(self, repository, run_id):
return self._paged(f"/repos/{repository}/actions/runs/{run_id}/jobs",
None, "jobs")

213
.publication/lib/history.py Normal file
View file

@ -0,0 +1,213 @@
"""Git history driver for the house-leak scanner.
Replaces RedFlag's check-public-history.sh. Two differences that matter:
commit messages are scanned (the awk gate's --format replaced them, so bodies
were never read), and the allowlist is pinned by policy, so a repository cannot
bless its own history without a policy commit.
"""
import hashlib
import os
import subprocess
import scanner
ALLOWLIST_FILENAME = ".publication-history-allowlist.txt"
NUL = "\x00"
class GitError(Exception):
pass
def _git(args, cwd):
proc = subprocess.run(["git"] + args, cwd=cwd, capture_output=True, text=True)
if proc.returncode != 0:
raise GitError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
return proc.stdout
def require_full_history(cwd):
if _git(["rev-parse", "--is-shallow-repository"], cwd).strip() != "false":
raise GitError("full history is required; this is a shallow clone")
def scan_metadata_and_messages(ref, cwd, rules=scanner.HOUSE_RULES):
# git emits the separators; a literal NUL cannot travel in argv.
fmt = "%x00".join(["%H", "%an", "%ae", "%cn", "%ce", "%B"]) + "%x00%x01"
out = _git(["log", ref, f"--format={fmt}"], cwd)
findings = {}
for record in out.split("\x01"):
record = record.strip("\n")
if not record:
continue
parts = record.split(NUL)
if len(parts) < 6:
continue
sha, an, ae, cn, ce, body = parts[0].strip(), parts[1], parts[2], parts[3], parts[4], parts[5]
hits = scanner.scan_commit(message=body, identities=(an, ae, cn, ce), rules=rules)
if hits:
findings.setdefault(sha, []).extend(hits)
return findings
def scan_patches(ref, cwd, rules=scanner.HOUSE_RULES):
"""Stream the diff so a large history does not have to fit in memory."""
proc = subprocess.Popen(
["git", "log", ref, "-p", "--no-ext-diff", "--no-color",
"--format=@@COMMIT@@%H"],
cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, errors="replace")
findings = {}
current = None
seen_rules = set()
for line in proc.stdout:
if line.startswith("@@COMMIT@@"):
current = line[10:].strip()
seen_rules = set()
continue
if current is None:
continue
for hit in scanner.scan(line, scanner.CONTENT, rules):
if hit.rule_id in seen_rules:
continue
seen_rules.add(hit.rule_id)
findings.setdefault(current, []).append(hit)
proc.stdout.close()
err = proc.stderr.read()
proc.stderr.close()
if proc.wait() != 0:
raise GitError("git log -p failed: " + err.strip())
return findings
def scan_tree(ref, cwd, rules=scanner.HOUSE_RULES, pinned_paths=()):
"""Findings in the FILES at the candidate ref, not in its diffs.
These are never allowlistable. A reviewed historical disclosure says a
commit may exist; it never says the string it introduced is acceptable at
HEAD forever. souveraine-viewtop proved the difference: blessing eighteen
commits would have published a live internal dependency URL.
"""
# The mechanism carries the patterns it enforces, so it matches its own
# rules. Excluding exactly the manifest's paths is safe and no wider than
# safe: those files cannot vary without failing the drift check, and a file
# under .publication/ that the manifest does not list is still scanned.
pinned = frozenset(pinned_paths)
hits = []
for rule in rules:
if scanner.CONTENT not in rule.surfaces:
continue
proc = subprocess.run(
["git", "grep", "-I", "-i", "-n", "-E", "-e", rule.pattern.pattern, ref],
cwd=cwd, capture_output=True, text=True, errors="replace")
if proc.returncode not in (0, 1):
raise GitError("git grep failed: " + proc.stderr.strip())
for line in proc.stdout.splitlines():
parts = line.split(":", 3)
if len(parts) < 3:
continue
if parts[1] in pinned:
continue
hits.append((rule.rule_id, parts[1], parts[2]))
return sorted(set(hits))
def scan_history(ref, cwd, rules=scanner.HOUSE_RULES):
merged = {}
for source in (scan_patches(ref, cwd, rules), scan_metadata_and_messages(ref, cwd, rules)):
for sha, hits in source.items():
merged.setdefault(sha, []).extend(hits)
return merged
def canonical_allowlist(text):
shas = sorted({line.strip() for line in text.splitlines()
if line.strip() and not line.strip().startswith("#")})
return shas, hashlib.sha256(("\n".join(shas) + "\n").encode("utf-8")).hexdigest()
def load_allowlist(repo_root):
path = os.path.join(repo_root, ALLOWLIST_FILENAME)
if not os.path.isfile(path):
return [], hashlib.sha256(b"\n").hexdigest(), path
with open(path, "r", encoding="utf-8") as fh:
shas, digest = canonical_allowlist(fh.read())
return shas, digest, path
def verify(ref, repo_root, policy, pinned_paths=(), warnings=None):
"""Return failure reasons. Empty means this history may be published.
`warnings` collects nonsecret findings -- topology, home paths, ugly
defaults. They are recorded in the publication report and do not deny.
Secrets deny. Publication is not certification.
"""
reasons = []
warnings = [] if warnings is None else warnings
try:
require_full_history(repo_root)
_git(["rev-parse", "--verify", f"{ref}^{{commit}}"], repo_root)
except GitError as exc:
return [f"history: {exc}"]
allowed, digest, path = load_allowlist(repo_root)
if policy.history_allowlist_sha256 is None:
if allowed:
reasons.append(
f"history: policy pins no allowlist but {ALLOWLIST_FILENAME} lists "
f"{len(allowed)} commit(s)")
elif digest != policy.history_allowlist_sha256:
reasons.append(
"history: allowlist digest does not match the digest pinned in policy; "
"a repository cannot bless its own history")
for sha in allowed:
try:
_git(["merge-base", "--is-ancestor", sha, ref], repo_root)
except GitError:
reasons.append(f"history: allowlist commit {sha} is stale or unreachable from {ref}")
severity = {r.rule_id: r.severity for r in scanner.HOUSE_RULES}
findings = scan_history(ref, repo_root)
unreviewed = sorted(set(findings) - set(allowed))
for sha in unreviewed:
rules = sorted({f.rule_id for f in findings[sha]})
fatal = [r for r in rules if severity.get(r) == scanner.FATAL]
if fatal:
reasons.append(f"history: secret material in {sha} ({', '.join(fatal)})")
rest = [r for r in rules if severity.get(r) != scanner.FATAL]
if rest:
warnings.append(f"history {sha[:12]}: {', '.join(rest)}")
# Unallowlistable by construction: the allowlist is not consulted here.
for rule_id, path, line_no in scan_tree(ref, repo_root, pinned_paths=pinned_paths):
if severity.get(rule_id) == scanner.FATAL:
reasons.append(
f"candidate-tree: {rule_id} at {path}:{line_no} -- a secret in the tree "
f"at {ref} is a capability handed to whoever reads it")
else:
warnings.append(f"candidate-tree {path}:{line_no}: {rule_id}")
reasons.extend(scanner.check_repo_assertions(repo_root, policy.repo_assertions))
reasons.extend(_oversized(ref, repo_root, policy.repo_assertions["max_blob_bytes"]))
return reasons
def _oversized(ref, repo_root, max_bytes):
listing = _git(["rev-list", "--objects", ref], repo_root)
proc = subprocess.run(
["git", "cat-file", "--batch-check=%(objecttype) %(objectname) %(objectsize) %(rest)"],
cwd=repo_root, input=listing, capture_output=True, text=True)
if proc.returncode != 0:
raise GitError("git cat-file failed: " + proc.stderr.strip())
entries = []
for line in proc.stdout.splitlines():
parts = line.split(" ", 3)
if len(parts) < 3 or parts[0] != "blob":
continue
entries.append((parts[1], int(parts[2]), parts[3] if len(parts) > 3 else ""))
return [f"history: blob {path or name} is {size} bytes, over the {max_bytes} ceiling"
for name, size, path in scanner.oversized_blobs(entries, max_bytes)]

117
.publication/lib/mechanism.py Normal file
View file

@ -0,0 +1,117 @@
"""Mechanism distribution and drift detection.
Gitea 1.25.5 cannot read a reusable workflow out of a private repository
(that is PR #32562, milestone 1.26.0), so the invariant scripts are copied into
consuming repositories. Copying is only safe if drift is detectable, so every
copy is pinned three ways: file hashes against a manifest, the manifest against
its own digest, and that digest against the policy record, which the consuming
repository cannot write.
"""
import hashlib
import json
import os
import shutil
PIN_FILENAME = ".publication-mechanism.json"
def canonical_json(obj):
return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode("utf-8")
def sha256_file(path):
h = hashlib.sha256()
with open(path, "rb") as fh:
for chunk in iter(lambda: fh.read(65536), b""):
h.update(chunk)
return h.hexdigest()
def sha256_bytes(data):
return hashlib.sha256(data).hexdigest()
def compute_manifest(root, relpaths, version):
files = {}
for rel in sorted(relpaths):
path = os.path.join(root, rel)
if not os.path.isfile(path):
raise FileNotFoundError(f"mechanism: {rel} not found under {root}")
files[rel] = sha256_file(path)
return {"mechanism_version": version, "files": files}
def manifest_digest(manifest):
return sha256_bytes(canonical_json(manifest))
def write_manifest(path, manifest):
with open(path, "w", encoding="utf-8") as fh:
fh.write(json.dumps(manifest, sort_keys=True, indent=2) + "\n")
def read_manifest(path):
with open(path, "r", encoding="utf-8") as fh:
return json.load(fh)
def install(source_root, target_root, manifest, pin_dir=None):
"""Copy the manifest's files into target_root and write the pin file."""
for rel in sorted(manifest["files"]):
src = os.path.join(source_root, rel)
dst = os.path.join(target_root, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(src, dst)
os.chmod(dst, 0o755 if rel.endswith(".sh") or "/bin/" in rel else 0o644)
pin = {
"mechanism_version": manifest["mechanism_version"],
"manifest_sha256": manifest_digest(manifest),
}
pin_path = os.path.join(pin_dir or target_root, PIN_FILENAME)
with open(pin_path, "w", encoding="utf-8") as fh:
fh.write(json.dumps(pin, sort_keys=True, indent=2) + "\n")
return pin_path
def verify(target_root, manifest, policy_manifest_sha256, policy_mechanism_version):
"""Return failure reasons. Empty means the local copy is the pinned mechanism."""
reasons = []
digest = manifest_digest(manifest)
if digest != policy_manifest_sha256:
reasons.append(
"mechanism: canonical manifest digest does not match the digest pinned in "
"policy; the mechanism was changed without a policy change")
if manifest["mechanism_version"] != policy_mechanism_version:
reasons.append(
f"mechanism: manifest version {manifest['mechanism_version']} does not match "
f"policy version {policy_mechanism_version}")
pin_path = os.path.join(target_root, PIN_FILENAME)
if not os.path.isfile(pin_path):
reasons.append(f"mechanism: {PIN_FILENAME} is absent from {target_root}")
else:
try:
with open(pin_path, "r", encoding="utf-8") as fh:
pin = json.load(fh)
except json.JSONDecodeError as exc:
reasons.append(f"mechanism: {PIN_FILENAME} is not valid JSON ({exc})")
pin = None
if pin is not None:
if pin.get("manifest_sha256") != policy_manifest_sha256:
reasons.append(
"mechanism: in-repo pin disagrees with policy; a repository cannot "
"bless its own mechanism version")
if pin.get("mechanism_version") != policy_mechanism_version:
reasons.append("mechanism: in-repo pin version disagrees with policy")
for rel, expected in sorted(manifest["files"].items()):
path = os.path.join(target_root, rel)
if not os.path.isfile(path):
reasons.append(f"mechanism: {rel} is missing from the consuming repository")
continue
actual = sha256_file(path)
if actual != expected:
reasons.append(f"mechanism: {rel} has drifted from the canonical copy")
return reasons

321
.publication/lib/policy.py Normal file
View file

@ -0,0 +1,321 @@
"""Publication policy: the authorization oracle.
A policy record authorizes one internal repository to project one ref to one
destination. No record, or any defect in a record, denies publication.
Rationale and the trust argument live in docs/DESIGN.md.
"""
import json
import os
import re
SCHEMA_VERSION = 1
PROJECTIONS = frozenset({
"source-only",
"source+release-assets",
"source+rolling-archive",
})
KNOWN_EVENTS = frozenset({
"push", "workflow_dispatch", "pull_request", "schedule", "release",
})
# A matrix gate expands one declaration into many jobs. The selector is the
# explicit list in `matrix_jobs`, never a prefix: prefix matching decides which
# jobs count, and that is the ambiguity the (workflow_path, job_name) key exists
# to remove. Adding a leg upstream therefore requires a policy commit.
SUPPORTED_CARDINALITIES = frozenset({"exactly-one", "matrix"})
DECLARED_CARDINALITIES = frozenset({"exactly-one", "at-least-one", "all-of", "matrix"})
RERUN_POLICIES = frozenset({
"latest-attempt-must-succeed",
"all-attempts-must-succeed",
})
REPO_RE = re.compile(r"^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$")
HOST_RE = re.compile(r"^[a-z0-9.-]+$")
REF_RE = re.compile(r"^[A-Za-z0-9._/-]+$")
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
WORKFLOW_PATH_RE = re.compile(r"^\.(gitea|forgejo|github)/workflows/[A-Za-z0-9._-]+\.ya?ml$")
TOP_KEYS = {
"schema_version", "internal_repository", "destination_host",
"destination_repository", "permitted_refs", "projection",
"publisher_identity", "allowed_events", "rerun_policy",
"required_gates", "repo_assertions", "mechanism",
"history_allowlist_sha256",
}
GATE_KEYS = {"workflow_path", "job_name", "cardinality"}
MATRIX_GATE_KEYS = GATE_KEYS | {"matrix_jobs"}
ASSERT_KEYS = {"require_files", "readme_install_pattern", "max_blob_bytes"}
MECH_KEYS = {"version", "manifest_sha256"}
class PolicyError(Exception):
"""A policy record is absent, malformed, or declares something unsupported."""
def __init__(self, reasons):
self.reasons = list(reasons)
super().__init__("; ".join(self.reasons))
class Gate:
__slots__ = ("workflow_path", "job_name", "cardinality", "matrix_jobs")
def __init__(self, workflow_path, job_name, cardinality, matrix_jobs=()):
self.workflow_path = workflow_path
self.job_name = job_name
self.cardinality = cardinality
self.matrix_jobs = tuple(matrix_jobs)
def __repr__(self):
n = f", {len(self.matrix_jobs)} legs" if self.matrix_jobs else ""
return f"Gate({self.workflow_path}::{self.job_name}, {self.cardinality}{n})"
class Policy:
__slots__ = (
"schema_version", "internal_repository", "destination_host",
"destination_repository", "permitted_refs", "projection",
"publisher_identity", "allowed_events", "rerun_policy",
"required_gates", "repo_assertions", "mechanism_version",
"mechanism_manifest_sha256", "source_path", "history_allowlist_sha256",
)
def __init__(self, raw, source_path=None):
self.schema_version = raw["schema_version"]
self.internal_repository = raw["internal_repository"]
self.destination_host = raw["destination_host"]
self.destination_repository = raw["destination_repository"]
self.permitted_refs = tuple(raw["permitted_refs"])
self.projection = raw["projection"]
self.publisher_identity = raw["publisher_identity"]
self.allowed_events = frozenset(raw["allowed_events"])
self.rerun_policy = raw["rerun_policy"]
self.required_gates = tuple(
Gate(g["workflow_path"], g["job_name"], g["cardinality"],
g.get("matrix_jobs", ()))
for g in raw["required_gates"]
)
self.repo_assertions = dict(raw["repo_assertions"])
self.mechanism_version = raw["mechanism"]["version"]
self.mechanism_manifest_sha256 = raw["mechanism"]["manifest_sha256"]
self.history_allowlist_sha256 = raw["history_allowlist_sha256"]
self.source_path = source_path
def _strict_keys(obj, expected, where, reasons):
if not isinstance(obj, dict):
reasons.append(f"{where}: expected object")
return False
got = set(obj)
for k in sorted(expected - got):
reasons.append(f"{where}: missing key '{k}'")
for k in sorted(got - expected):
reasons.append(f"{where}: unknown key '{k}'")
return got == expected
def validate(raw):
"""Return a list of reasons the record is unusable. Empty list means valid."""
reasons = []
if not _strict_keys(raw, TOP_KEYS, "policy", reasons):
return reasons
if raw["schema_version"] != SCHEMA_VERSION:
reasons.append(
f"policy: schema_version {raw['schema_version']!r} is not {SCHEMA_VERSION}")
for field, pattern in (
("internal_repository", REPO_RE),
("destination_repository", REPO_RE),
("destination_host", HOST_RE),
):
v = raw[field]
if not isinstance(v, str) or not pattern.match(v):
reasons.append(f"policy: {field} {v!r} is malformed")
refs = raw["permitted_refs"]
if not isinstance(refs, list) or not refs:
reasons.append("policy: permitted_refs must be a non-empty list")
else:
for r in refs:
if not isinstance(r, str) or not REF_RE.match(r):
reasons.append(f"policy: permitted_ref {r!r} is malformed")
if len(set(refs)) != len(refs):
reasons.append("policy: permitted_refs contains duplicates")
if raw["projection"] not in PROJECTIONS:
reasons.append(f"policy: projection {raw['projection']!r} is not one of "
+ ", ".join(sorted(PROJECTIONS)))
if not isinstance(raw["publisher_identity"], str) or not raw["publisher_identity"]:
reasons.append("policy: publisher_identity must be a non-empty string")
events = raw["allowed_events"]
if not isinstance(events, list) or not events:
reasons.append("policy: allowed_events must be a non-empty list")
else:
for e in events:
if e not in KNOWN_EVENTS:
reasons.append(f"policy: allowed_event {e!r} is unknown")
if raw["rerun_policy"] not in RERUN_POLICIES:
reasons.append(f"policy: rerun_policy {raw['rerun_policy']!r} is not one of "
+ ", ".join(sorted(RERUN_POLICIES)))
gates = raw["required_gates"]
if not isinstance(gates, list) or not gates:
reasons.append("policy: required_gates must be a non-empty list")
else:
seen = set()
for i, g in enumerate(gates):
where = f"required_gates[{i}]"
expected = (MATRIX_GATE_KEYS
if isinstance(g, dict) and g.get("cardinality") == "matrix"
else GATE_KEYS)
if not _strict_keys(g, expected, where, reasons):
continue
wp, jn, card = g["workflow_path"], g["job_name"], g["cardinality"]
if not isinstance(wp, str) or not WORKFLOW_PATH_RE.match(wp):
reasons.append(f"{where}: workflow_path {wp!r} is malformed")
if not isinstance(jn, str) or not jn:
reasons.append(f"{where}: job_name must be a non-empty string")
if card not in DECLARED_CARDINALITIES:
reasons.append(f"{where}: cardinality {card!r} is not recognised")
elif card not in SUPPORTED_CARDINALITIES:
reasons.append(
f"{where}: gate {wp}::{jn} declares cardinality {card!r}, which is "
f"UNSUPPORTED in schema v{SCHEMA_VERSION}; publication is denied "
f"rather than guessed")
if card == "matrix":
legs = g["matrix_jobs"]
if not isinstance(legs, list) or not legs:
reasons.append(f"{where}: matrix_jobs must be a non-empty list "
"naming every expanded job exactly")
else:
if any(not isinstance(x, str) or not x for x in legs):
reasons.append(f"{where}: matrix_jobs must be non-empty strings")
if len(set(legs)) != len(legs):
reasons.append(f"{where}: matrix_jobs contains duplicates")
key = (wp, jn)
if key in seen:
reasons.append(f"{where}: duplicate gate {wp}::{jn}")
seen.add(key)
a = raw["repo_assertions"]
if _strict_keys(a, ASSERT_KEYS, "repo_assertions", reasons):
if not isinstance(a["require_files"], list) or any(
not isinstance(f, str) or not f for f in a["require_files"]):
reasons.append("repo_assertions: require_files must be a list of paths")
p = a["readme_install_pattern"]
if p is not None:
if not isinstance(p, str):
reasons.append("repo_assertions: readme_install_pattern must be a string or null")
else:
try:
re.compile(p)
except re.error as exc:
reasons.append(f"repo_assertions: readme_install_pattern is not a valid regex ({exc})")
n = a["max_blob_bytes"]
if not isinstance(n, int) or isinstance(n, bool) or n <= 0:
reasons.append("repo_assertions: max_blob_bytes must be a positive integer")
m = raw["mechanism"]
if _strict_keys(m, MECH_KEYS, "mechanism", reasons):
if not isinstance(m["version"], int) or isinstance(m["version"], bool) or m["version"] < 1:
reasons.append("mechanism: version must be a positive integer")
if not isinstance(m["manifest_sha256"], str) or not SHA256_RE.match(m["manifest_sha256"]):
reasons.append("mechanism: manifest_sha256 must be 64 lowercase hex characters")
h = raw["history_allowlist_sha256"]
if h is not None and (not isinstance(h, str) or not SHA256_RE.match(h)):
reasons.append(
"policy: history_allowlist_sha256 must be 64 lowercase hex characters or "
"null; pinning it here is what stops a repository blessing its own history")
return reasons
def load_record(path):
try:
with open(path, "r", encoding="utf-8") as fh:
raw = json.load(fh)
except FileNotFoundError:
raise PolicyError([f"policy: no record at {path}"])
except json.JSONDecodeError as exc:
raise PolicyError([f"policy: {path} is not valid JSON ({exc})"])
reasons = validate(raw)
if reasons:
raise PolicyError(reasons)
return Policy(raw, source_path=os.path.abspath(path))
class PolicySet:
"""Every record in a directory, indexed by internal repository.
A record whose filename does not match its internal_repository is rejected:
the filename is how a human finds it, and a mismatch hides one record behind
another's name.
"""
def __init__(self, records):
self._by_repo = records
@classmethod
def from_directory(cls, directory):
by_repo = {}
reasons = []
if not os.path.isdir(directory):
raise PolicyError([f"policy: no policy directory at {directory}"])
for name in sorted(os.listdir(directory)):
if not name.endswith(".json"):
continue
path = os.path.join(directory, name)
try:
pol = load_record(path)
except PolicyError as exc:
reasons.extend(f"{name}: {r}" for r in exc.reasons)
continue
expected = pol.internal_repository.replace("/", "-") + ".json"
if name != expected:
reasons.append(
f"{name}: record for {pol.internal_repository} must be named {expected}")
continue
if pol.internal_repository in by_repo:
reasons.append(f"{name}: duplicate record for {pol.internal_repository}")
continue
by_repo[pol.internal_repository] = pol
if reasons:
raise PolicyError(reasons)
return cls(by_repo)
def authorize(self, internal_repository):
pol = self._by_repo.get(internal_repository)
if pol is None:
raise PolicyError(
[f"policy: no record authorizes {internal_repository}; publication denied"])
return pol
def __len__(self):
return len(self._by_repo)
def repositories(self):
return tuple(sorted(self._by_repo))
def assert_policy_is_external(policy, consuming_repo_root):
"""The publishing repo must not be able to edit its own authorization.
Filesystem containment is the check available here; the durable control is
that the policy repository denies write to every publishing identity.
"""
if policy.source_path is None:
raise PolicyError(["policy: record has no source path to verify"])
root = os.path.realpath(consuming_repo_root)
src = os.path.realpath(policy.source_path)
if src == root or src.startswith(root + os.sep):
raise PolicyError([
f"policy: record {src} lives inside the repository it authorizes ({root}); "
"a repository must not be able to write its own policy"])

168
.publication/lib/scanner.py Normal file
View file

@ -0,0 +1,168 @@
"""House-leak scanner: Casey's private infrastructure, wherever it appears.
Universal rules live here and are identical for every repository. Anything that
varies per repository (which files must exist, what the README must contain,
how large a blob may be) is a policy assertion, not a rule -- see
check_repo_assertions.
Ported from RedFlag's awk gate. Python's regex engine removes the awk interval
portability hazard: {1,3} is not guaranteed in every awk a runner image ships.
"""
import os
import re
CONTENT = "content"
MESSAGE = "message"
METADATA = "metadata"
ALL_SURFACES = frozenset({CONTENT, MESSAGE, METADATA})
FATAL = "fatal"
WARN = "warn"
class Rule:
__slots__ = ("rule_id", "description", "pattern", "surfaces", "severity")
def __init__(self, rule_id, description, pattern, surfaces, severity=WARN):
self.rule_id = rule_id
self.description = description
self.pattern = re.compile(pattern, re.IGNORECASE)
self.surfaces = frozenset(surfaces)
self.severity = severity
# wiuf is matched as a bare token rather than as wiuf\.net so that one rule
# covers the domain, every depth of subdomain, every case, URLs, addresses,
# config values, and the host spellings wiufph and WIUF-Docker. Over-matching
# costs one review; under-matching publishes the house.
# Two severities, and the difference is what a finding costs someone.
#
# A private key or an embedded token is a capability: publishing it hands
# control to whoever reads it, and no amount of later cleanup takes it back.
# Those deny.
#
# A home path, a LAN address, a hostname or an ugly default is topology. It is
# worth removing and it is not worth withholding working software over. Those
# are recorded in the publication report and fixed afterwards.
#
# Publication is not certification.
SECRET_RULES = (
Rule("private-key", "PEM or OpenSSH private key block",
r"BEGIN (RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY( BLOCK)?-",
ALL_SURFACES, FATAL),
Rule("url-credential", "credential in a URL's userinfo",
r"://[A-Za-z0-9._%-]+:[A-Za-z0-9._%+/=-]{8,}@", ALL_SURFACES, FATAL),
Rule("oauth2-clone", "authenticated clone URL",
r"://oauth2:[^@\s\"']+@", ALL_SURFACES, FATAL),
Rule("forge-pat", "forge personal access token",
r"\b(gh[pousr]_[A-Za-z0-9]{36}|glpat-[A-Za-z0-9_-]{20,})\b",
ALL_SURFACES, FATAL),
Rule("aws-key", "AWS access key id",
r"\bAKIA[0-9A-Z]{16}\b", ALL_SURFACES, FATAL),
)
HOUSE_RULES = SECRET_RULES + (
Rule("wiuf", "canonical internal domain and host family (wiuf.net, any subdomain, wiufph, WIUF-Docker)",
r"wiuf", ALL_SURFACES),
Rule("net-10-10", "internal LAN 10.10.0.0/16",
r"\b10\.10\.\d{1,3}\.\d{1,3}\b", ALL_SURFACES),
Rule("net-10-8", "VPN peer range 10.8.0.0/16",
r"\b10\.8\.\d{1,3}\.\d{1,3}\b", ALL_SURFACES),
Rule("net-172-16-42", "phone USB network 172.16.42.0/24",
r"\b172\.16\.42\.\d{1,3}\b", ALL_SURFACES),
Rule("home-path", "developer home directory",
r"/home/casey", ALL_SURFACES),
Rule("archdev-host", "internal build host archdev",
r"\barchdev\b", ALL_SURFACES),
# 192.168/16 is excluded from patch bodies on purpose: a fleet-management
# tool documents private ranges legitimately. In a commit message or an
# author field it is the house.
Rule("net-192-168", "private range in prose or identity fields",
r"\b192\.168\.\d{1,3}\.\d{1,3}\b", {MESSAGE, METADATA}),
)
class Finding:
__slots__ = ("rule_id", "surface", "excerpt")
def __init__(self, rule_id, surface, excerpt):
self.rule_id = rule_id
self.surface = surface
self.excerpt = excerpt
def __repr__(self):
return f"Finding({self.rule_id}, {self.surface}, {self.excerpt!r})"
def __eq__(self, other):
return (isinstance(other, Finding)
and (self.rule_id, self.surface) == (other.rule_id, other.surface))
def __hash__(self):
return hash((self.rule_id, self.surface))
def _redact(text):
if len(text) <= 2:
return "*" * len(text)
return text[0] + "*" * (len(text) - 2) + text[-1]
def scan(text, surface, rules=HOUSE_RULES, redact=True):
"""Findings for one blob of text on one surface. Order follows HOUSE_RULES."""
if text is None:
return []
out = []
for rule in rules:
if surface not in rule.surfaces:
continue
m = rule.pattern.search(text)
if m:
hit = m.group(0)
out.append(Finding(rule.rule_id, surface, _redact(hit) if redact else hit))
return out
def scan_commit(message=None, patch=None, identities=(), rules=HOUSE_RULES, redact=True):
"""identities is the author/committer name and email strings."""
findings = []
findings.extend(scan(patch, CONTENT, rules, redact))
findings.extend(scan(message, MESSAGE, rules, redact))
for ident in identities:
findings.extend(scan(ident, METADATA, rules, redact))
seen, unique = set(), []
for f in findings:
key = (f.rule_id, f.surface)
if key not in seen:
seen.add(key)
unique.append(f)
return unique
def check_repo_assertions(root, assertions):
"""Repo-specific publication requirements, declared by policy not by code.
Returns a list of failure reasons; empty means the tree satisfies them.
"""
reasons = []
for rel in assertions.get("require_files", ()):
path = os.path.join(root, rel)
if not os.path.isfile(path) or os.path.getsize(path) == 0:
reasons.append(f"repo-assertion: {rel} is missing or empty")
pattern = assertions.get("readme_install_pattern")
if pattern:
readme = os.path.join(root, "README.md")
if not os.path.isfile(readme):
reasons.append("repo-assertion: readme_install_pattern declared but README.md is absent")
else:
with open(readme, "r", encoding="utf-8", errors="replace") as fh:
if not re.search(pattern, fh.read(), re.IGNORECASE | re.MULTILINE):
reasons.append("repo-assertion: README.md does not match readme_install_pattern")
return reasons
def oversized_blobs(entries, max_blob_bytes):
"""entries is an iterable of (object_name, size, path)."""
return [(name, size, path) for name, size, path in entries if size > max_blob_bytes]