213 lines
8.3 KiB
Python
213 lines
8.3 KiB
Python
"""Git history driver for the house-leak scanner.
|
|
|
|
Replaces RedFlag's check-public-history.sh. Two differences that matter:
|
|
commit messages are scanned (the awk gate's --format replaced them, so bodies
|
|
were never read), and the allowlist is pinned by policy, so a repository cannot
|
|
bless its own history without a policy commit.
|
|
"""
|
|
|
|
import hashlib
|
|
import os
|
|
import subprocess
|
|
|
|
import scanner
|
|
|
|
ALLOWLIST_FILENAME = ".publication-history-allowlist.txt"
|
|
NUL = "\x00"
|
|
|
|
|
|
class GitError(Exception):
|
|
pass
|
|
|
|
|
|
def _git(args, cwd):
|
|
proc = subprocess.run(["git"] + args, cwd=cwd, capture_output=True, text=True)
|
|
if proc.returncode != 0:
|
|
raise GitError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
|
|
return proc.stdout
|
|
|
|
|
|
def require_full_history(cwd):
|
|
if _git(["rev-parse", "--is-shallow-repository"], cwd).strip() != "false":
|
|
raise GitError("full history is required; this is a shallow clone")
|
|
|
|
|
|
def scan_metadata_and_messages(ref, cwd, rules=scanner.HOUSE_RULES):
|
|
# git emits the separators; a literal NUL cannot travel in argv.
|
|
fmt = "%x00".join(["%H", "%an", "%ae", "%cn", "%ce", "%B"]) + "%x00%x01"
|
|
out = _git(["log", ref, f"--format={fmt}"], cwd)
|
|
findings = {}
|
|
for record in out.split("\x01"):
|
|
record = record.strip("\n")
|
|
if not record:
|
|
continue
|
|
parts = record.split(NUL)
|
|
if len(parts) < 6:
|
|
continue
|
|
sha, an, ae, cn, ce, body = parts[0].strip(), parts[1], parts[2], parts[3], parts[4], parts[5]
|
|
hits = scanner.scan_commit(message=body, identities=(an, ae, cn, ce), rules=rules)
|
|
if hits:
|
|
findings.setdefault(sha, []).extend(hits)
|
|
return findings
|
|
|
|
|
|
def scan_patches(ref, cwd, rules=scanner.HOUSE_RULES):
|
|
"""Stream the diff so a large history does not have to fit in memory."""
|
|
proc = subprocess.Popen(
|
|
["git", "log", ref, "-p", "--no-ext-diff", "--no-color",
|
|
"--format=@@COMMIT@@%H"],
|
|
cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
text=True, errors="replace")
|
|
findings = {}
|
|
current = None
|
|
seen_rules = set()
|
|
for line in proc.stdout:
|
|
if line.startswith("@@COMMIT@@"):
|
|
current = line[10:].strip()
|
|
seen_rules = set()
|
|
continue
|
|
if current is None:
|
|
continue
|
|
for hit in scanner.scan(line, scanner.CONTENT, rules):
|
|
if hit.rule_id in seen_rules:
|
|
continue
|
|
seen_rules.add(hit.rule_id)
|
|
findings.setdefault(current, []).append(hit)
|
|
proc.stdout.close()
|
|
err = proc.stderr.read()
|
|
proc.stderr.close()
|
|
if proc.wait() != 0:
|
|
raise GitError("git log -p failed: " + err.strip())
|
|
return findings
|
|
|
|
|
|
def scan_tree(ref, cwd, rules=scanner.HOUSE_RULES, pinned_paths=()):
|
|
"""Findings in the FILES at the candidate ref, not in its diffs.
|
|
|
|
These are never allowlistable. A reviewed historical disclosure says a
|
|
commit may exist; it never says the string it introduced is acceptable at
|
|
HEAD forever. souveraine-viewtop proved the difference: blessing eighteen
|
|
commits would have published a live internal dependency URL.
|
|
"""
|
|
# The mechanism carries the patterns it enforces, so it matches its own
|
|
# rules. Excluding exactly the manifest's paths is safe and no wider than
|
|
# safe: those files cannot vary without failing the drift check, and a file
|
|
# under .publication/ that the manifest does not list is still scanned.
|
|
pinned = frozenset(pinned_paths)
|
|
hits = []
|
|
for rule in rules:
|
|
if scanner.CONTENT not in rule.surfaces:
|
|
continue
|
|
proc = subprocess.run(
|
|
["git", "grep", "-I", "-i", "-n", "-E", "-e", rule.pattern.pattern, ref],
|
|
cwd=cwd, capture_output=True, text=True, errors="replace")
|
|
if proc.returncode not in (0, 1):
|
|
raise GitError("git grep failed: " + proc.stderr.strip())
|
|
for line in proc.stdout.splitlines():
|
|
parts = line.split(":", 3)
|
|
if len(parts) < 3:
|
|
continue
|
|
if parts[1] in pinned:
|
|
continue
|
|
hits.append((rule.rule_id, parts[1], parts[2]))
|
|
return sorted(set(hits))
|
|
|
|
|
|
def scan_history(ref, cwd, rules=scanner.HOUSE_RULES):
|
|
merged = {}
|
|
for source in (scan_patches(ref, cwd, rules), scan_metadata_and_messages(ref, cwd, rules)):
|
|
for sha, hits in source.items():
|
|
merged.setdefault(sha, []).extend(hits)
|
|
return merged
|
|
|
|
|
|
def canonical_allowlist(text):
|
|
shas = sorted({line.strip() for line in text.splitlines()
|
|
if line.strip() and not line.strip().startswith("#")})
|
|
return shas, hashlib.sha256(("\n".join(shas) + "\n").encode("utf-8")).hexdigest()
|
|
|
|
|
|
def load_allowlist(repo_root):
|
|
path = os.path.join(repo_root, ALLOWLIST_FILENAME)
|
|
if not os.path.isfile(path):
|
|
return [], hashlib.sha256(b"\n").hexdigest(), path
|
|
with open(path, "r", encoding="utf-8") as fh:
|
|
shas, digest = canonical_allowlist(fh.read())
|
|
return shas, digest, path
|
|
|
|
|
|
def verify(ref, repo_root, policy, pinned_paths=(), warnings=None):
|
|
"""Return failure reasons. Empty means this history may be published.
|
|
|
|
`warnings` collects nonsecret findings -- topology, home paths, ugly
|
|
defaults. They are recorded in the publication report and do not deny.
|
|
Secrets deny. Publication is not certification.
|
|
"""
|
|
reasons = []
|
|
warnings = [] if warnings is None else warnings
|
|
try:
|
|
require_full_history(repo_root)
|
|
_git(["rev-parse", "--verify", f"{ref}^{{commit}}"], repo_root)
|
|
except GitError as exc:
|
|
return [f"history: {exc}"]
|
|
|
|
allowed, digest, path = load_allowlist(repo_root)
|
|
|
|
if policy.history_allowlist_sha256 is None:
|
|
if allowed:
|
|
reasons.append(
|
|
f"history: policy pins no allowlist but {ALLOWLIST_FILENAME} lists "
|
|
f"{len(allowed)} commit(s)")
|
|
elif digest != policy.history_allowlist_sha256:
|
|
reasons.append(
|
|
"history: allowlist digest does not match the digest pinned in policy; "
|
|
"a repository cannot bless its own history")
|
|
|
|
for sha in allowed:
|
|
try:
|
|
_git(["merge-base", "--is-ancestor", sha, ref], repo_root)
|
|
except GitError:
|
|
reasons.append(f"history: allowlist commit {sha} is stale or unreachable from {ref}")
|
|
|
|
severity = {r.rule_id: r.severity for r in scanner.HOUSE_RULES}
|
|
|
|
findings = scan_history(ref, repo_root)
|
|
unreviewed = sorted(set(findings) - set(allowed))
|
|
for sha in unreviewed:
|
|
rules = sorted({f.rule_id for f in findings[sha]})
|
|
fatal = [r for r in rules if severity.get(r) == scanner.FATAL]
|
|
if fatal:
|
|
reasons.append(f"history: secret material in {sha} ({', '.join(fatal)})")
|
|
rest = [r for r in rules if severity.get(r) != scanner.FATAL]
|
|
if rest:
|
|
warnings.append(f"history {sha[:12]}: {', '.join(rest)}")
|
|
|
|
# Unallowlistable by construction: the allowlist is not consulted here.
|
|
for rule_id, path, line_no in scan_tree(ref, repo_root, pinned_paths=pinned_paths):
|
|
if severity.get(rule_id) == scanner.FATAL:
|
|
reasons.append(
|
|
f"candidate-tree: {rule_id} at {path}:{line_no} -- a secret in the tree "
|
|
f"at {ref} is a capability handed to whoever reads it")
|
|
else:
|
|
warnings.append(f"candidate-tree {path}:{line_no}: {rule_id}")
|
|
|
|
reasons.extend(scanner.check_repo_assertions(repo_root, policy.repo_assertions))
|
|
reasons.extend(_oversized(ref, repo_root, policy.repo_assertions["max_blob_bytes"]))
|
|
return reasons
|
|
|
|
|
|
def _oversized(ref, repo_root, max_bytes):
|
|
listing = _git(["rev-list", "--objects", ref], repo_root)
|
|
proc = subprocess.run(
|
|
["git", "cat-file", "--batch-check=%(objecttype) %(objectname) %(objectsize) %(rest)"],
|
|
cwd=repo_root, input=listing, capture_output=True, text=True)
|
|
if proc.returncode != 0:
|
|
raise GitError("git cat-file failed: " + proc.stderr.strip())
|
|
entries = []
|
|
for line in proc.stdout.splitlines():
|
|
parts = line.split(" ", 3)
|
|
if len(parts) < 3 or parts[0] != "blob":
|
|
continue
|
|
entries.append((parts[1], int(parts[2]), parts[3] if len(parts) > 3 else ""))
|
|
return [f"history: blob {path or name} is {size} bytes, over the {max_bytes} ceiling"
|
|
for name, size, path in scanner.oversized_blobs(entries, max_bytes)]
|