168 lines
6.3 KiB
Python
168 lines
6.3 KiB
Python
"""House-leak scanner: Casey's private infrastructure, wherever it appears.
|
|
|
|
Universal rules live here and are identical for every repository. Anything that
|
|
varies per repository (which files must exist, what the README must contain,
|
|
how large a blob may be) is a policy assertion, not a rule -- see
|
|
check_repo_assertions.
|
|
|
|
Ported from RedFlag's awk gate. Python's regex engine removes the awk interval
|
|
portability hazard: {1,3} is not guaranteed in every awk a runner image ships.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
|
|
CONTENT = "content"
|
|
MESSAGE = "message"
|
|
METADATA = "metadata"
|
|
ALL_SURFACES = frozenset({CONTENT, MESSAGE, METADATA})
|
|
|
|
|
|
FATAL = "fatal"
|
|
WARN = "warn"
|
|
|
|
|
|
class Rule:
|
|
__slots__ = ("rule_id", "description", "pattern", "surfaces", "severity")
|
|
|
|
def __init__(self, rule_id, description, pattern, surfaces, severity=WARN):
|
|
self.rule_id = rule_id
|
|
self.description = description
|
|
self.pattern = re.compile(pattern, re.IGNORECASE)
|
|
self.surfaces = frozenset(surfaces)
|
|
self.severity = severity
|
|
|
|
|
|
# wiuf is matched as a bare token rather than as wiuf\.net so that one rule
|
|
# covers the domain, every depth of subdomain, every case, URLs, addresses,
|
|
# config values, and the host spellings wiufph and WIUF-Docker. Over-matching
|
|
# costs one review; under-matching publishes the house.
|
|
# Two severities, and the difference is what a finding costs someone.
|
|
#
|
|
# A private key or an embedded token is a capability: publishing it hands
|
|
# control to whoever reads it, and no amount of later cleanup takes it back.
|
|
# Those deny.
|
|
#
|
|
# A home path, a LAN address, a hostname or an ugly default is topology. It is
|
|
# worth removing and it is not worth withholding working software over. Those
|
|
# are recorded in the publication report and fixed afterwards.
|
|
#
|
|
# Publication is not certification.
|
|
SECRET_RULES = (
|
|
Rule("private-key", "PEM or OpenSSH private key block",
|
|
r"BEGIN (RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY( BLOCK)?-",
|
|
ALL_SURFACES, FATAL),
|
|
Rule("url-credential", "credential in a URL's userinfo",
|
|
r"://[A-Za-z0-9._%-]+:[A-Za-z0-9._%+/=-]{8,}@", ALL_SURFACES, FATAL),
|
|
Rule("oauth2-clone", "authenticated clone URL",
|
|
r"://oauth2:[^@\s\"']+@", ALL_SURFACES, FATAL),
|
|
Rule("forge-pat", "forge personal access token",
|
|
r"\b(gh[pousr]_[A-Za-z0-9]{36}|glpat-[A-Za-z0-9_-]{20,})\b",
|
|
ALL_SURFACES, FATAL),
|
|
Rule("aws-key", "AWS access key id",
|
|
r"\bAKIA[0-9A-Z]{16}\b", ALL_SURFACES, FATAL),
|
|
)
|
|
|
|
HOUSE_RULES = SECRET_RULES + (
|
|
Rule("wiuf", "canonical internal domain and host family (wiuf.net, any subdomain, wiufph, WIUF-Docker)",
|
|
r"wiuf", ALL_SURFACES),
|
|
Rule("net-10-10", "internal LAN 10.10.0.0/16",
|
|
r"\b10\.10\.\d{1,3}\.\d{1,3}\b", ALL_SURFACES),
|
|
Rule("net-10-8", "VPN peer range 10.8.0.0/16",
|
|
r"\b10\.8\.\d{1,3}\.\d{1,3}\b", ALL_SURFACES),
|
|
Rule("net-172-16-42", "phone USB network 172.16.42.0/24",
|
|
r"\b172\.16\.42\.\d{1,3}\b", ALL_SURFACES),
|
|
Rule("home-path", "developer home directory",
|
|
r"/home/casey", ALL_SURFACES),
|
|
Rule("archdev-host", "internal build host archdev",
|
|
r"\barchdev\b", ALL_SURFACES),
|
|
# 192.168/16 is excluded from patch bodies on purpose: a fleet-management
|
|
# tool documents private ranges legitimately. In a commit message or an
|
|
# author field it is the house.
|
|
Rule("net-192-168", "private range in prose or identity fields",
|
|
r"\b192\.168\.\d{1,3}\.\d{1,3}\b", {MESSAGE, METADATA}),
|
|
)
|
|
|
|
|
|
class Finding:
|
|
__slots__ = ("rule_id", "surface", "excerpt")
|
|
|
|
def __init__(self, rule_id, surface, excerpt):
|
|
self.rule_id = rule_id
|
|
self.surface = surface
|
|
self.excerpt = excerpt
|
|
|
|
def __repr__(self):
|
|
return f"Finding({self.rule_id}, {self.surface}, {self.excerpt!r})"
|
|
|
|
def __eq__(self, other):
|
|
return (isinstance(other, Finding)
|
|
and (self.rule_id, self.surface) == (other.rule_id, other.surface))
|
|
|
|
def __hash__(self):
|
|
return hash((self.rule_id, self.surface))
|
|
|
|
|
|
def _redact(text):
|
|
if len(text) <= 2:
|
|
return "*" * len(text)
|
|
return text[0] + "*" * (len(text) - 2) + text[-1]
|
|
|
|
|
|
def scan(text, surface, rules=HOUSE_RULES, redact=True):
|
|
"""Findings for one blob of text on one surface. Order follows HOUSE_RULES."""
|
|
if text is None:
|
|
return []
|
|
out = []
|
|
for rule in rules:
|
|
if surface not in rule.surfaces:
|
|
continue
|
|
m = rule.pattern.search(text)
|
|
if m:
|
|
hit = m.group(0)
|
|
out.append(Finding(rule.rule_id, surface, _redact(hit) if redact else hit))
|
|
return out
|
|
|
|
|
|
def scan_commit(message=None, patch=None, identities=(), rules=HOUSE_RULES, redact=True):
|
|
"""identities is the author/committer name and email strings."""
|
|
findings = []
|
|
findings.extend(scan(patch, CONTENT, rules, redact))
|
|
findings.extend(scan(message, MESSAGE, rules, redact))
|
|
for ident in identities:
|
|
findings.extend(scan(ident, METADATA, rules, redact))
|
|
seen, unique = set(), []
|
|
for f in findings:
|
|
key = (f.rule_id, f.surface)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique.append(f)
|
|
return unique
|
|
|
|
|
|
def check_repo_assertions(root, assertions):
|
|
"""Repo-specific publication requirements, declared by policy not by code.
|
|
|
|
Returns a list of failure reasons; empty means the tree satisfies them.
|
|
"""
|
|
reasons = []
|
|
for rel in assertions.get("require_files", ()):
|
|
path = os.path.join(root, rel)
|
|
if not os.path.isfile(path) or os.path.getsize(path) == 0:
|
|
reasons.append(f"repo-assertion: {rel} is missing or empty")
|
|
|
|
pattern = assertions.get("readme_install_pattern")
|
|
if pattern:
|
|
readme = os.path.join(root, "README.md")
|
|
if not os.path.isfile(readme):
|
|
reasons.append("repo-assertion: readme_install_pattern declared but README.md is absent")
|
|
else:
|
|
with open(readme, "r", encoding="utf-8", errors="replace") as fh:
|
|
if not re.search(pattern, fh.read(), re.IGNORECASE | re.MULTILINE):
|
|
reasons.append("repo-assertion: README.md does not match readme_install_pattern")
|
|
return reasons
|
|
|
|
|
|
def oversized_blobs(entries, max_blob_bytes):
|
|
"""entries is an iterable of (object_name, size, path)."""
|
|
return [(name, size, path) for name, size, path in entries if size > max_blob_bytes]
|