418 lines
18 KiB
Shell
Executable file
418 lines
18 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
# Additive publisher for the rolling `edge` pacman archive.
|
|
#
|
|
# The `edge` release on Fimeg/souveraine is a MULTI-PRODUCER archive: souveraine
|
|
# publishes its own binaries there, and so does every other repo that ships a
|
|
# package to the phone. Until 2026-07-25 souveraine's CI deleted and recreated
|
|
# the release on every push and uploaded only its own `pacman-repo`, so anything
|
|
# another job had published was erased — silently, with nothing erroring. This
|
|
# script is the additive replacement.
|
|
#
|
|
# Rules it enforces:
|
|
# * The release and its tag are NEVER deleted. Created once, patched after.
|
|
# * The per-arch database is read-modify-written: the live
|
|
# `souveraine-<arch>.db.tar.zst` is fetched, our packages are `repo-add`ed
|
|
# into it, and the result is uploaded back. Entries belonging to other
|
|
# producers survive because repo-add only touches the names it is given.
|
|
# * The ONLY assets deleted are ones this producer supersedes: for each
|
|
# package we publish, the `%FILENAME%` recorded in the live database under
|
|
# the same `%NAME%`. A producer can never delete another's package.
|
|
# * Everything runs under one exclusive flock on the archdev runner, so two
|
|
# jobs cannot interleave their read-modify-write of the same database.
|
|
#
|
|
# Usage:
|
|
# publish-edge.sh <producer> <repo-dir> [extra-asset ...]
|
|
#
|
|
# <producer> short name for this publisher (souveraine, souveraine-updater).
|
|
# Used for the manifest line in the release body and for the
|
|
# per-producer checksum asset. NOT used to decide deletions.
|
|
# <repo-dir> directory holding one subdirectory per architecture, each with
|
|
# `*.pkg.tar.zst` and matching `.sig` files. The databases are
|
|
# built here; do not pre-create them.
|
|
# extra-asset optional loose files to publish alongside (raw binaries,
|
|
# checksum manifests). Replaced in place by name.
|
|
#
|
|
# Environment:
|
|
# EDGE_TOKEN required. Gitea token with write access to the ARCHIVE
|
|
# repo — for a producer other than souveraine this must be
|
|
# a PAT, since a job's own GITHUB_TOKEN is scoped to its
|
|
# own repository and cannot upload here.
|
|
# ARCHIVE_KEY required. GPG key id used for `repo-add --sign`.
|
|
# GITHUB_SERVER_URL Gitea base URL. Default http://10.10.20.120:4455.
|
|
# ARCHIVE_REPO owner/name holding the release. Default Fimeg/souveraine.
|
|
# EDGE_TARGET_SHA commitish for the tag, used only when the release does
|
|
# not exist yet. Default: the archive repo's default branch.
|
|
# PRODUCER_VERSION version string recorded in the release body manifest.
|
|
# EDGE_LOCK lock file path. Default ~/.cache/souveraine-archive/edge.lock.
|
|
|
|
set -euo pipefail
|
|
|
|
PRODUCER="${1:?usage: publish-edge.sh <producer> <repo-dir> [extra-asset ...]}"
|
|
REPO_DIR="${2:?usage: publish-edge.sh <producer> <repo-dir> [extra-asset ...]}"
|
|
shift 2
|
|
EXTRA_ASSETS=("$@")
|
|
|
|
: "${EDGE_TOKEN:?EDGE_TOKEN must be set (Gitea token with write access to the archive repo)}"
|
|
: "${ARCHIVE_KEY:?ARCHIVE_KEY must be set (gpg key id for repo-add --sign)}"
|
|
|
|
SERVER="${GITHUB_SERVER_URL:-http://10.10.20.120:4455}"
|
|
ARCHIVE_REPO="${ARCHIVE_REPO:-Fimeg/souveraine}"
|
|
PRODUCER_VERSION="${PRODUCER_VERSION:-unknown}"
|
|
LOCK="${EDGE_LOCK:-$HOME/.cache/souveraine-archive/edge.lock}"
|
|
# Overridable so a change to this script can be rehearsed end-to-end against a
|
|
# throwaway tag before it is pointed at the archive the phone actually installs
|
|
# from. Leave it alone in CI.
|
|
EDGE_TAG="${EDGE_TAG:-edge}"
|
|
|
|
# Re-exec under an exclusive lock. The runner is a single host-mode box shared
|
|
# by every repo's jobs, so this one file serialises all producers. Without it
|
|
# two concurrent read-modify-writes of the same database both start from the
|
|
# same base and the second upload silently drops the first's entries — the same
|
|
# lost-update the delete-and-recreate shape caused, just narrower.
|
|
if [ "${EDGE_LOCK_HELD:-}" != 1 ]; then
|
|
mkdir -p "$(dirname "$LOCK")"
|
|
export EDGE_LOCK_HELD=1
|
|
exec flock "$LOCK" "$0" "$PRODUCER" "$REPO_DIR" "${EXTRA_ASSETS[@]+"${EXTRA_ASSETS[@]}"}"
|
|
fi
|
|
|
|
API="$SERVER/api/v1"
|
|
AUTH="Authorization: token $EDGE_TOKEN"
|
|
|
|
api() { curl -sf -H "$AUTH" "$@"; }
|
|
|
|
log() { echo "[publish-edge] $*"; }
|
|
|
|
# --- locate or create the release -------------------------------------------
|
|
# Note the asymmetry with the old code: there is no DELETE anywhere in this
|
|
# script for the release or the tag. `edge` outlives every individual push.
|
|
REL_JSON=$(curl -s -H "$AUTH" "$API/repos/$ARCHIVE_REPO/releases/tags/$EDGE_TAG" || true)
|
|
REL_ID=$(printf '%s' "$REL_JSON" | python3 -c "
|
|
import json,sys
|
|
try: print(json.load(sys.stdin).get('id',''))
|
|
except Exception: print('')")
|
|
|
|
if [ -z "$REL_ID" ]; then
|
|
TARGET="${EDGE_TARGET_SHA:-}"
|
|
if [ -z "$TARGET" ]; then
|
|
TARGET=$(api "$API/repos/$ARCHIVE_REPO" | python3 -c "
|
|
import json,sys; print(json.load(sys.stdin)['default_branch'])")
|
|
fi
|
|
log "no $EDGE_TAG release yet — creating it at $TARGET"
|
|
REL_ID=$(api -X POST -H "Content-Type: application/json" \
|
|
"$API/repos/$ARCHIVE_REPO/releases" \
|
|
-d "$(python3 -c "
|
|
import json,sys
|
|
print(json.dumps({
|
|
'tag_name': sys.argv[2],
|
|
'target_commitish': sys.argv[1],
|
|
'name': sys.argv[2],
|
|
'body': 'Rolling multi-producer pacman archive. Each repo publishes its own '
|
|
'signed packages into the shared souveraine-<arch> databases; the '
|
|
'release is never deleted. Producers are listed below.\n',
|
|
'prerelease': True,
|
|
}))" "$TARGET" "$EDGE_TAG")" | python3 -c "import json,sys; print(json.load(sys.stdin)['id'])")
|
|
else
|
|
log "publishing into existing $EDGE_TAG release $REL_ID"
|
|
fi
|
|
|
|
ASSETS_JSON=$(api "$API/repos/$ARCHIVE_REPO/releases/$REL_ID/assets")
|
|
|
|
asset_id() {
|
|
printf '%s' "$ASSETS_JSON" | python3 -c "
|
|
import json,sys
|
|
name = sys.argv[1]
|
|
for a in json.load(sys.stdin):
|
|
if a['name'] == name:
|
|
print(a['id']); break
|
|
" "$1"
|
|
}
|
|
|
|
delete_asset() {
|
|
local name="$1" id
|
|
id=$(asset_id "$name")
|
|
[ -n "$id" ] || return 0
|
|
curl -sf -X DELETE -H "$AUTH" \
|
|
"$API/repos/$ARCHIVE_REPO/releases/$REL_ID/assets/$id" -o /dev/null
|
|
log "deleted stale asset $name"
|
|
}
|
|
|
|
# Gitea happily stores two assets with the same name, which would leave pacman
|
|
# fetching whichever the API returned first. Always delete before uploading.
|
|
upload_asset() {
|
|
local path="$1" name
|
|
name="$(basename "$path")"
|
|
delete_asset "$name"
|
|
curl -sf -X POST -H "$AUTH" \
|
|
"$API/repos/$ARCHIVE_REPO/releases/$REL_ID/assets?name=$name" \
|
|
-F "attachment=@$path" -o /dev/null
|
|
log "uploaded $name"
|
|
}
|
|
|
|
# Fetch by the SAME URL shape pacman uses on the phone
|
|
# (`releases/download/<tag>/<name>`), not by asset id. The by-id API endpoint
|
|
# returns attachment metadata as JSON in some Gitea versions, and writing that
|
|
# into a .db.tar.zst would look like a corrupt database rather than an error.
|
|
# This path is proven: it is exactly what souveraine-pacman-fetch already pulls.
|
|
download_asset() {
|
|
local name="$1" dest="$2"
|
|
curl -sfL -H "$AUTH" \
|
|
"$SERVER/$ARCHIVE_REPO/releases/download/$EDGE_TAG/$name" -o "$dest"
|
|
}
|
|
|
|
# --- per-arch read-modify-write ---------------------------------------------
|
|
STAGE=$(mktemp -d)
|
|
trap 'rm -rf "$STAGE"' EXIT
|
|
|
|
# Dump `NAME<TAB>FILENAME` for every entry in a pacman database. Used three
|
|
# times: to find our own superseded builds, to snapshot what was in the archive
|
|
# before we touched it, and to prove after upload that the snapshot survived.
|
|
cat > "$STAGE/dbentries.py" <<'PY'
|
|
import subprocess, sys
|
|
|
|
# `bsdtar -xO` streams every member concatenated; entry `desc` files are the
|
|
# only ones we want. Each is a sequence of `%KEY%` header lines followed by
|
|
# their values, and a `%FILENAME%` header is what starts a new entry.
|
|
out = subprocess.run(
|
|
["bsdtar", "-xOf", sys.argv[1], "--include=*/desc"],
|
|
capture_output=True, text=True, check=True).stdout
|
|
|
|
entries, cur, key = [], {}, None
|
|
for line in out.splitlines():
|
|
line = line.strip()
|
|
if line.startswith("%") and line.endswith("%") and len(line) > 2:
|
|
if line == "%FILENAME%" and cur:
|
|
entries.append(cur)
|
|
cur = {}
|
|
key = line.strip("%")
|
|
elif line and key and key not in cur:
|
|
cur[key] = line
|
|
if cur:
|
|
entries.append(cur)
|
|
|
|
for e in entries:
|
|
if "NAME" in e and "FILENAME" in e:
|
|
print(f"{e['NAME']}\t{e['FILENAME']}")
|
|
PY
|
|
|
|
db_entries() { python3 "$STAGE/dbentries.py" "$1"; }
|
|
|
|
# Package name is everything before the trailing -pkgver-pkgrel-arch. pkgver and
|
|
# pkgrel may not contain a hyphen, so the last three fields are always known.
|
|
pkg_name_of() {
|
|
local base
|
|
base="$(basename "$1")"
|
|
base="${base%.pkg.tar.*}"
|
|
printf '%s\n' "${base%-*-*-*}"
|
|
}
|
|
|
|
for ARCH_DIR in "$REPO_DIR"/*/; do
|
|
[ -d "$ARCH_DIR" ] || continue
|
|
ARCH="$(basename "$ARCH_DIR")"
|
|
shopt -s nullglob
|
|
PKGS=("$ARCH_DIR"*.pkg.tar.zst)
|
|
shopt -u nullglob
|
|
if [ ${#PKGS[@]} -eq 0 ]; then
|
|
log "$ARCH: no packages, skipping"
|
|
continue
|
|
fi
|
|
|
|
DB="souveraine-$ARCH"
|
|
WORK="$STAGE/$ARCH"
|
|
mkdir -p "$WORK"
|
|
|
|
# Pull the LIVE database. Every other producer's entries live in here, and
|
|
# they survive only because we hand this file to repo-add rather than
|
|
# building a fresh one.
|
|
#
|
|
# The distinction below is the whole safety property of this script. Starting
|
|
# a fresh database is correct ONLY when the archive genuinely has none yet. If
|
|
# the asset is listed but we cannot fetch or read it — a transient 500, a
|
|
# truncated body, an HTML error page — falling through to a fresh database
|
|
# would erase every other producer on upload. That is precisely the clobber
|
|
# this script exists to remove, so it is a hard failure instead.
|
|
if [ -n "$(asset_id "$DB.db.tar.zst")" ]; then
|
|
download_asset "$DB.db.tar.zst" "$WORK/$DB.db.tar.zst" \
|
|
|| { echo "FATAL: $DB.db.tar.zst is published but could not be fetched;" \
|
|
"refusing to publish a database that would drop other producers" >&2
|
|
exit 1; }
|
|
bsdtar -tf "$WORK/$DB.db.tar.zst" >/dev/null 2>&1 \
|
|
|| { echo "FATAL: fetched $DB.db.tar.zst is not a readable archive;" \
|
|
"refusing to overwrite the live database" >&2
|
|
exit 1; }
|
|
log "$ARCH: fetched live database ($(bsdtar -tf "$WORK/$DB.db.tar.zst" | grep -c '/desc$') entries)"
|
|
else
|
|
log "$ARCH: archive has no $DB database yet — creating the first one"
|
|
fi
|
|
|
|
# The files database is published too (pacman -F). repo-add maintains it
|
|
# beside the .db automatically, but only merges into one that is already
|
|
# there — without this fetch it would be rebuilt from just our packages and
|
|
# every other producer would vanish from `pacman -F` while still installing
|
|
# fine, which is the kind of half-broken that goes unnoticed for months.
|
|
if [ -n "$(asset_id "$DB.files.tar.zst")" ]; then
|
|
download_asset "$DB.files.tar.zst" "$WORK/$DB.files.tar.zst" \
|
|
|| { echo "FATAL: $DB.files.tar.zst is published but could not be fetched" >&2; exit 1; }
|
|
bsdtar -tf "$WORK/$DB.files.tar.zst" >/dev/null 2>&1 \
|
|
|| { echo "FATAL: fetched $DB.files.tar.zst is not a readable archive" >&2; exit 1; }
|
|
fi
|
|
|
|
# Snapshot the archive as it stands BEFORE we touch it. Everything in here
|
|
# that we do not deliberately supersede must still be in the database we
|
|
# upload; the check at the end of this loop enforces exactly that.
|
|
: > "$WORK/before.tsv"
|
|
if [ -f "$WORK/$DB.db.tar.zst" ]; then
|
|
db_entries "$WORK/$DB.db.tar.zst" > "$WORK/before.tsv"
|
|
fi
|
|
|
|
# The package NAMES we are publishing. Any entry in the live database under
|
|
# one of these names is a previous build of OURS — those filenames are the
|
|
# only assets this script is ever entitled to delete.
|
|
: > "$WORK/ours.txt"
|
|
: > "$WORK/newfiles.txt"
|
|
for p in "${PKGS[@]}"; do
|
|
pkg_name_of "$p" >> "$WORK/ours.txt"
|
|
basename "$p" >> "$WORK/newfiles.txt"
|
|
done
|
|
|
|
SUPERSEDED=$(awk -F'\t' '
|
|
NR==FNR && FILENAME==ARGV[1] { ours[$0]=1; next }
|
|
FILENAME==ARGV[2] { newf[$0]=1; next }
|
|
($1 in ours) && !($2 in newf) { print $2 }
|
|
' "$WORK/ours.txt" "$WORK/newfiles.txt" "$WORK/before.tsv")
|
|
|
|
cp "${PKGS[@]}" "$WORK/"
|
|
for p in "${PKGS[@]}"; do
|
|
if [ -f "$p.sig" ]; then cp "$p.sig" "$WORK/"; fi
|
|
done
|
|
|
|
shopt -s nullglob
|
|
STAGED=("$WORK"/*.pkg.tar.zst)
|
|
shopt -u nullglob
|
|
repo-add --include-sigs --sign --key "$ARCHIVE_KEY" \
|
|
"$WORK/$DB.db.tar.zst" "${STAGED[@]}"
|
|
|
|
# Before anything is uploaded: the merged database must still carry every
|
|
# entry the archive had, except the ones we deliberately replaced. A failure
|
|
# here means repo-add did not merge the way this script assumes, and the
|
|
# upload is abandoned with the live archive untouched.
|
|
db_entries "$WORK/$DB.db.tar.zst" > "$WORK/after.tsv"
|
|
MISSING=$(awk -F'\t' '
|
|
NR==FNR && FILENAME==ARGV[1] { ours[$0]=1; next }
|
|
FILENAME==ARGV[2] { kept[$1]=1; next }
|
|
!($1 in ours) && !($1 in kept) { print $1 }
|
|
' "$WORK/ours.txt" "$WORK/after.tsv" "$WORK/before.tsv")
|
|
if [ -n "$MISSING" ]; then
|
|
echo "FATAL: merging into $DB dropped entries this producer does not own:" >&2
|
|
printf ' %s\n' $MISSING >&2
|
|
echo "nothing was uploaded; the live archive is unchanged" >&2
|
|
exit 1
|
|
fi
|
|
log "$ARCH: merged — $(wc -l < "$WORK/before.tsv") entries in, $(wc -l < "$WORK/after.tsv") out"
|
|
|
|
# repo-add leaves .db/.files and their .sig as symlinks; release assets cannot
|
|
# carry those, so publish real files. pacman fetches the BARE names — `-Sy`
|
|
# asks for `$DB.db`, `-Fy` for `$DB.files` — so the .tar.zst copies alone are
|
|
# not enough. The files db was missing its bare name until 2026-07-25, which
|
|
# is why `pacman -Fy` 404'd on this repo while `-Sy` worked.
|
|
rm -f "$WORK/$DB.db" "$WORK/$DB.db.sig"
|
|
cp "$WORK/$DB.db.tar.zst" "$WORK/$DB.db"
|
|
cp "$WORK/$DB.db.tar.zst.sig" "$WORK/$DB.db.sig"
|
|
if [ -f "$WORK/$DB.files.tar.zst" ]; then
|
|
rm -f "$WORK/$DB.files" "$WORK/$DB.files.sig"
|
|
cp "$WORK/$DB.files.tar.zst" "$WORK/$DB.files"
|
|
if [ -f "$WORK/$DB.files.tar.zst.sig" ]; then
|
|
cp "$WORK/$DB.files.tar.zst.sig" "$WORK/$DB.files.sig"
|
|
fi
|
|
fi
|
|
|
|
# Producer-scoped checksums: a whole-archive manifest would go stale the
|
|
# moment another producer published, and claim coverage it does not have.
|
|
( cd "$WORK" && sha256sum ./*.pkg.tar.zst > "$PRODUCER-$ARCH.sha256" )
|
|
|
|
# Packages first, database last. A reader that races us then sees either the
|
|
# old database (pointing at packages that are all still present) or the new
|
|
# one (pointing at packages already uploaded) — never a database naming a
|
|
# package that has not landed yet.
|
|
for f in "${STAGED[@]}"; do
|
|
upload_asset "$f"
|
|
if [ -f "$f.sig" ]; then upload_asset "$f.sig"; fi
|
|
done
|
|
upload_asset "$WORK/$PRODUCER-$ARCH.sha256"
|
|
if [ -f "$WORK/$DB.files.tar.zst" ]; then
|
|
upload_asset "$WORK/$DB.files.tar.zst"
|
|
upload_asset "$WORK/$DB.files"
|
|
if [ -f "$WORK/$DB.files.tar.zst.sig" ]; then
|
|
upload_asset "$WORK/$DB.files.tar.zst.sig"
|
|
upload_asset "$WORK/$DB.files.sig"
|
|
fi
|
|
fi
|
|
upload_asset "$WORK/$DB.db.tar.zst"
|
|
upload_asset "$WORK/$DB.db.tar.zst.sig"
|
|
upload_asset "$WORK/$DB.db"
|
|
upload_asset "$WORK/$DB.db.sig"
|
|
|
|
# Read the database back over the SAME URL pacman uses and confirm the archive
|
|
# really holds what we just merged. Publishing is two dozen HTTP calls against
|
|
# a box on the far side of a LAN; "the upload returned 201" is not the same
|
|
# claim as "the phone will see this".
|
|
if download_asset "$DB.db.tar.zst" "$WORK/live.db.tar.zst"; then
|
|
db_entries "$WORK/live.db.tar.zst" | sort > "$WORK/live.tsv"
|
|
sort "$WORK/after.tsv" > "$WORK/expect.tsv"
|
|
if ! diff -q "$WORK/expect.tsv" "$WORK/live.tsv" >/dev/null; then
|
|
echo "FATAL: the published $DB database does not match what was uploaded:" >&2
|
|
diff "$WORK/expect.tsv" "$WORK/live.tsv" >&2 || true
|
|
exit 1
|
|
fi
|
|
log "$ARCH: verified live database matches ($(wc -l < "$WORK/live.tsv") entries)"
|
|
else
|
|
echo "FATAL: could not read back $DB.db.tar.zst after publishing it" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Now — and only now, with the new database live and verified — drop our own
|
|
# superseded builds. Doing it earlier would leave a window where the published
|
|
# database referenced a package that no longer existed.
|
|
if [ -n "$SUPERSEDED" ]; then
|
|
while IFS= read -r old; do
|
|
[ -n "$old" ] || continue
|
|
delete_asset "$old"
|
|
delete_asset "$old.sig"
|
|
done <<< "$SUPERSEDED"
|
|
fi
|
|
|
|
# Legacy whole-archive manifest from the delete-and-recreate era. Harmless but
|
|
# misleading — it claims to cover packages this producer never built.
|
|
delete_asset "$DB-repo.sha256"
|
|
done
|
|
|
|
for f in "${EXTRA_ASSETS[@]+"${EXTRA_ASSETS[@]}"}"; do
|
|
[ -f "$f" ] || { log "extra asset $f missing, skipping"; continue; }
|
|
upload_asset "$f"
|
|
done
|
|
|
|
# --- manifest line in the release body ---------------------------------------
|
|
# The body is the only place that records WHO put what in the archive. With the
|
|
# tag pinned at creation it is also the only place a reader can see that the
|
|
# archive moved at all.
|
|
BODY=$(printf '%s' "$REL_JSON" | python3 -c "
|
|
import json,sys
|
|
try: print(json.load(sys.stdin).get('body',''))
|
|
except Exception: print('')")
|
|
|
|
NEW_BODY=$(python3 - "$BODY" "$PRODUCER" "$PRODUCER_VERSION" <<'PY'
|
|
import sys
|
|
body, producer, version = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
line = f"- {producer}: {version}"
|
|
kept = [l for l in body.splitlines() if not l.startswith(f"- {producer}: ")]
|
|
while kept and not kept[-1].strip():
|
|
kept.pop()
|
|
print("\n".join(kept + [line]))
|
|
PY
|
|
)
|
|
|
|
curl -sf -X PATCH -H "$AUTH" -H "Content-Type: application/json" \
|
|
"$API/repos/$ARCHIVE_REPO/releases/$REL_ID" \
|
|
-d "$(python3 -c "import json,sys; print(json.dumps({'body': sys.argv[1]}))" "$NEW_BODY")" \
|
|
-o /dev/null
|
|
|
|
log "$PRODUCER $PRODUCER_VERSION published into edge additively"
|