Fimeg/RedFlag
Watch
1
0
Fork
You've already forked RedFlag
 0
Code Releases 1 Activity

projection: begin the exact-path public epoch

Browse source 
The public tree and its history contain only the listed paths. Earlier projection history remains preserved internally.

Source-Sha: 913fde029b935671833254797f0f20f1eb9fabba

Policy-Sha: 913fde029b935671833254797f0f20f1eb9fabba

Tree-Digest: 180ae530c1058a2a5c89837bdce2d323ae83e669e38590ca72e75b8d92b7262f
This commit is contained in:
Fimeg 2026-09-08 21:59:33 -04:00
commit 67e26be2d9
838 changed files with 157302 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

82
server/internal/orchestrator/interfaces.go Normal file
View file

@ -0,0 +1,82 @@
// Package orchestrator advances the package-update lifecycle server-side. It
// auto-approves packages by operator policy and recovers packages stuck in
// active states, holding no state of its own — every decision is read from the
// database on each sweep, so a server restart loses nothing and the next tick
// reconciles. The DB-level guarded UPDATE from LIFECYCLE-001 makes every
// transition idempotent, so a redundant advance is a no-op.
//
// The dependencies below are declared as consumer interfaces so the package is
// unit-testable with fakes and imports neither the queries nor the handlers
// layer. The concrete wiring lives in cmd/server/main.go.
package orchestrator
import (
"time"
"github.com/Fimeg/RedFlag/server/internal/models"
"github.com/gofrs/uuid/v5"
)
// Store is the persistence surface the orchestrator drives. Satisfied by
// *queries.UpdateQueries.
type Store interface {
GetUpdateByID(id uuid.UUID) (*models.UpdateState, error)
GetPackagesInStatus(status models.PackageStatus) ([]models.UpdateState, error)
TransitionByID(id uuid.UUID, to models.PackageStatus, historyMeta models.JSONB) error
BumpRetryCounter(id uuid.UUID, key string) (int, error)
}
// DryRunEnqueuer creates and signs the dry_run_update command that moves an
// approved package into checking_dependencies. Satisfied by *handlers.UpdateHandler.
type DryRunEnqueuer interface {
EnqueueDryRun(update *models.UpdateState) error
}
// DependencyConfirmer performs the capability-path confirm step headlessly: it
// transitions a package waiting in pending_dependencies into installing and
// mints its capability token — the equivalent of an operator's
// ConfirmDependencies dashboard click. Satisfied by *handlers.UpdateHandler.
//
// It returns handled=false when the package is not capability-gated (legacy
// command path: docker/winget/Windows). Those are left for the operator, since
// the server cannot verify a legacy dependency was satisfied. An error means a
// genuine failure (the confirmer has already marked the package failed); the
// orchestrator only logs it. The caller (orchestrator) has already checked the
// maintenance window and auto-approval policy before invoking.
type DependencyConfirmer interface {
ConfirmDependenciesAuto(update *models.UpdateState) (handled bool, err error)
}
// MaintenanceWindow reports whether installs are currently permitted. The
// orchestrator consults it before auto-confirming a dependency closure: dry-run
// is read-only and safe anytime, but advancing to installing must respect the
// operator's window (design Q1). Satisfied by *queries.MaintenanceWindowQueries.
type MaintenanceWindow interface {
IsWithinMaintenanceWindow(now time.Time) (bool, error)
}
// Settings supplies live, operator-tunable policy and timing values. Satisfied
// by *services.SecuritySettingsService. A nil/missing value falls back to the
// supplied default, so a misconfigured deployment degrades to the safe default
// (auto-approval off) rather than failing.
type Settings interface {
GetPolicyString(key, fallback string) string
GetPolicyBool(key string, fallback bool) bool
GetOperationalInt(key string, fallback int) int
}
// EventLogger persists server-side system events (orchestrator timeouts,
// workflow state changes) to the system_events table. Satisfied by
// *services.SystemEventLogger.
type EventLogger interface {
LogEvent(agentID *uuid.UUID, eventType, eventSubtype, severity, component, message string, metadata map[string]interface{})
}
// Clock is injected so timeout logic is deterministic under test.
type Clock interface {
Now() time.Time
}
type systemClock struct{}
func (systemClock) Now() time.Time { return time.Now().UTC() }

127
server/internal/orchestrator/orchestrator.go Normal file
View file

@ -0,0 +1,127 @@
package orchestrator
import (
"log"
"sync"
"time"
"github.com/gofrs/uuid/v5"
)
// Orchestrator advances the package update lifecycle on a server-side timer and
// via synchronous discovery triggers. See the package doc for the design stance.
type Orchestrator struct {
store Store
enqueuer DryRunEnqueuer
confirmer DependencyConfirmer // optional; nil disables auto-confirm
window MaintenanceWindow // optional; nil disables auto-confirm
settings Settings
events EventLogger // optional; nil disables event emission
clock Clock
interval time.Duration
ticker *time.Ticker
stopChan chan struct{}
// sweepLock serializes the auto-advance passes (auto-approve and
// auto-confirm) so the timer tick and a synchronous discovery trigger
// cannot run them concurrently.
sweepLock sync.Mutex
}
// New builds an Orchestrator. interval <= 0 selects the 60s default. confirmer,
// window, and events may be nil; if confirmer or window is, auto-confirmation of
// resolved closures is disabled. If events is nil, system_events are not emitted.
func New(store Store, enqueuer DryRunEnqueuer, confirmer DependencyConfirmer, window MaintenanceWindow, settings Settings, events EventLogger, interval time.Duration) *Orchestrator {
if interval <= 0 {
interval = 60 * time.Second
}
return &Orchestrator{
store: store,
enqueuer: enqueuer,
confirmer: confirmer,
window: window,
settings: settings,
events: events,
clock: systemClock{},
interval: interval,
stopChan: make(chan struct{}),
}
}
// Start launches the timer sweep goroutine.
func (o *Orchestrator) Start() {
log.Printf("[INFO] [server] [orchestrator] service_started interval=%s", o.interval)
o.ticker = time.NewTicker(o.interval)
go func() {
for {
select {
case <-o.ticker.C:
o.sweep()
case <-o.stopChan:
o.ticker.Stop()
log.Printf("[INFO] [server] [orchestrator] service_stopped")
return
}
}
}()
}
// Stop halts the timer sweep.
func (o *Orchestrator) Stop() { close(o.stopChan) }
// emitEvent writes a system event via the injected logger, if present. No-op
// when the logger is nil so the orchestrator degrades gracefully without it.
func (o *Orchestrator) emitEvent(agentID *uuid.UUID, eventType, eventSubtype, severity, component, message string, metadata map[string]interface{}) {
if o.events == nil {
return
}
o.events.LogEvent(agentID, eventType, eventSubtype, severity, component, message, metadata)
}
// sweep runs one full reconciliation pass: auto-approve eligible pending
// packages, auto-confirm resolved closures whose dependencies were reported,
// then recover anything stuck in an active state.
func (o *Orchestrator) sweep() {
now := o.clock.Now()
o.runAutoAdvance(now)
o.sweepCheckingDependencies(now)
o.sweepInstalling(now)
o.sweepPendingDependencies(now)
}
// runAutoAdvance runs the forward-advance passes (auto-approve, then
// auto-confirm) under the sweep lock so the timer tick and a synchronous
// discovery trigger cannot pile onto each other. If a pass is already running,
// the caller skips — the next tick will catch up.
func (o *Orchestrator) runAutoAdvance(now time.Time) {
if !o.sweepLock.TryLock() {
return
}
defer o.sweepLock.Unlock()
o.sweepAutoApprove()
o.sweepAutoConfirm(now)
}
// OnPackagesDiscovered is the synchronous trigger the update handler fires after
// a scan report lands, so newly-discovered pending packages are auto-approved
// without waiting for the next timer tick. Non-blocking.
func (o *Orchestrator) OnPackagesDiscovered() {
go o.runAutoAdvance(o.clock.Now())
}
// OnDependenciesReported is the synchronous trigger the update handler fires
// after an agent reports a dependency closure (the package has just entered
// pending_dependencies). It runs the auto-confirm pass immediately so an
// eligible package mints its capability token without waiting for the next
// timer tick. Non-blocking; the pass re-reads the package from the DB, so a
// status that changed in between is handled idempotently.
func (o *Orchestrator) OnDependenciesReported(updateID uuid.UUID) {
go func() {
if !o.sweepLock.TryLock() {
return
}
defer o.sweepLock.Unlock()
o.sweepAutoConfirm(o.clock.Now())
}()
}

432
server/internal/orchestrator/orchestrator_test.go Normal file
View file

@ -0,0 +1,432 @@
package orchestrator
import (
"errors"
"testing"
"time"
"github.com/Fimeg/RedFlag/server/internal/models"
"github.com/gofrs/uuid/v5"
)
// --- fakes -----------------------------------------------------------------
type transitionCall struct {
id uuid.UUID
to models.PackageStatus
meta models.JSONB
}
type fakeStore struct {
byStatus map[models.PackageStatus][]models.UpdateState
byID map[uuid.UUID]*models.UpdateState
counters map[uuid.UUID]int
transitions []transitionCall
failBump bool
}
func newFakeStore() *fakeStore {
return &fakeStore{
byStatus: map[models.PackageStatus][]models.UpdateState{},
byID: map[uuid.UUID]*models.UpdateState{},
counters: map[uuid.UUID]int{},
}
}
func (f *fakeStore) add(p models.UpdateState) {
f.byStatus[p.Status] = append(f.byStatus[p.Status], p)
cp := p
f.byID[p.ID] = &cp
}
func (f *fakeStore) GetUpdateByID(id uuid.UUID) (*models.UpdateState, error) {
if p, ok := f.byID[id]; ok {
return p, nil
}
return nil, errors.New("not found")
}
func (f *fakeStore) GetPackagesInStatus(status models.PackageStatus) ([]models.UpdateState, error) {
return f.byStatus[status], nil
}
func (f *fakeStore) TransitionByID(id uuid.UUID, to models.PackageStatus, meta models.JSONB) error {
f.transitions = append(f.transitions, transitionCall{id: id, to: to, meta: meta})
if p, ok := f.byID[id]; ok {
p.Status = to
}
return nil
}
func (f *fakeStore) BumpRetryCounter(id uuid.UUID, key string) (int, error) {
if f.failBump {
return 0, errors.New("bump failed")
}
f.counters[id]++
return f.counters[id], nil
}
func (f *fakeStore) transitionsTo(to models.PackageStatus) int {
n := 0
for _, t := range f.transitions {
if t.to == to {
n++
}
}
return n
}
type fakeEnqueuer struct {
calls int
err error
}
func (f *fakeEnqueuer) EnqueueDryRun(update *models.UpdateState) error {
f.calls++
return f.err
}
type fakeConfirmer struct {
calls int
handled bool
err error
seen []uuid.UUID
}
func (f *fakeConfirmer) ConfirmDependenciesAuto(update *models.UpdateState) (bool, error) {
f.calls++
f.seen = append(f.seen, update.ID)
return f.handled, f.err
}
type fakeWindow struct {
open bool
err error
}
func (f fakeWindow) IsWithinMaintenanceWindow(time.Time) (bool, error) { return f.open, f.err }
type fakeSettings struct {
policyStr map[string]string
policyBool map[string]bool
opInt map[string]int
}
func (f fakeSettings) GetPolicyString(k, d string) string {
if v, ok := f.policyStr[k]; ok {
return v
}
return d
}
func (f fakeSettings) GetPolicyBool(k string, d bool) bool {
if v, ok := f.policyBool[k]; ok {
return v
}
return d
}
func (f fakeSettings) GetOperationalInt(k string, d int) int {
if v, ok := f.opInt[k]; ok {
return v
}
return d
}
func newOrch(store Store, enq DryRunEnqueuer, set Settings, now time.Time) *Orchestrator {
return &Orchestrator{store: store, enqueuer: enq, settings: set, clock: fixedClock{now}, interval: time.Minute, stopChan: make(chan struct{})}
}
func newConfirmOrch(store Store, conf DependencyConfirmer, win MaintenanceWindow, set Settings, now time.Time) *Orchestrator {
return &Orchestrator{store: store, confirmer: conf, window: win, settings: set, clock: fixedClock{now}, interval: time.Minute, stopChan: make(chan struct{})}
}
type fixedClock struct{ t time.Time }
func (c fixedClock) Now() time.Time { return c.t }
func pkg(status models.PackageStatus, severity string, age time.Duration, now time.Time) models.UpdateState {
return models.UpdateState{
ID: uuid.Must(uuid.NewV4()),
AgentID: uuid.Must(uuid.NewV4()),
PackageType: "dnf",
PackageName: "demo",
Severity: severity,
Status: status,
LastUpdatedAt: now.Add(-age),
}
}
// --- policy ----------------------------------------------------------------
func TestAutoApproveCeiling(t *testing.T) {
cases := map[string]int{
"": 0,
"off": 0,
"none": 0,
"disabled": 0,
"garbage": 0, // unrecognized must fail safe to disabled
"low": 1,
"high": 4,
"critical": 5,
"CRITICAL": 5,
}
for in, want := range cases {
if got := autoApproveCeiling(in); got != want {
t.Errorf("autoApproveCeiling(%q) = %d, want %d", in, got, want)
}
}
}
func TestShouldAutoApprove(t *testing.T) {
now := time.Now().UTC()
base := pkg(models.StatusPending, "moderate", 0, now)
if !shouldAutoApprove(base, "high") {
t.Error("moderate package should auto-approve under high ceiling")
}
if shouldAutoApprove(base, "low") {
t.Error("moderate package must not auto-approve under low ceiling")
}
if shouldAutoApprove(base, "off") {
t.Error("nothing auto-approves when policy is off")
}
withVulns := base
withVulns.Metadata = models.JSONB{"supply_chain_vulns": []interface{}{map[string]interface{}{"id": "CVE-1"}}}
if shouldAutoApprove(withVulns, "critical") {
t.Error("package with vulns must never auto-approve regardless of ceiling")
}
unknownSev := pkg(models.StatusPending, "spicy", 0, now)
if shouldAutoApprove(unknownSev, "critical") {
t.Error("unknown severity must not auto-approve")
}
}
// --- auto-approve sweep ----------------------------------------------------
func TestSweepAutoApprove_ApprovesEligibleAndEnqueues(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPending, "low", 0, now)) // eligible
store.add(pkg(models.StatusPending, "critical", 0, now)) // too severe for "high"
enq := &fakeEnqueuer{}
set := fakeSettings{policyStr: map[string]string{"auto_approve_max_severity": "high"}}
o := newOrch(store, enq, set, now)
o.sweepAutoApprove()
if got := store.transitionsTo(models.StatusApproved); got != 1 {
t.Errorf("approved transitions = %d, want 1", got)
}
if enq.calls != 1 {
t.Errorf("dry-run enqueues = %d, want 1", enq.calls)
}
}
func TestSweepAutoApprove_DisabledByDefault(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPending, "low", 0, now))
enq := &fakeEnqueuer{}
set := fakeSettings{} // no policy set → default "off"
o := newOrch(store, enq, set, now)
o.sweepAutoApprove()
if len(store.transitions) != 0 || enq.calls != 0 {
t.Errorf("auto-approve must be a no-op by default: transitions=%d enqueues=%d", len(store.transitions), enq.calls)
}
}
func TestSweepAutoApprove_RespectsAllowDryRunsFalse(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPending, "low", 0, now))
enq := &fakeEnqueuer{}
set := fakeSettings{
policyStr: map[string]string{"auto_approve_max_severity": "high"},
policyBool: map[string]bool{"allow_dry_runs": false},
}
o := newOrch(store, enq, set, now)
o.sweepAutoApprove()
if len(store.transitions) != 0 || enq.calls != 0 {
t.Errorf("auto-approve must not run when dry-runs are disabled: transitions=%d enqueues=%d", len(store.transitions), enq.calls)
}
}
// --- auto-confirm sweep ----------------------------------------------------
func confirmSettings() fakeSettings {
return fakeSettings{policyStr: map[string]string{"auto_approve_max_severity": "high"}}
}
// clearedPkg is a pending_dependencies package whose closure has been OSV-checked
// and came back clean — the precondition for auto-confirmation.
func clearedPkg(severity string, now time.Time) models.UpdateState {
p := pkg(models.StatusPendingDependencies, severity, 0, now)
p.Metadata = models.JSONB{
"closure_checked_at": now.Format(time.RFC3339),
"closure_vulns": "[]",
}
return p
}
func TestSweepAutoConfirm_ConfirmsEligibleInsideWindow(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(clearedPkg("low", now)) // eligible: cleared closure, under ceiling
store.add(clearedPkg("critical", now)) // too severe for "high"
conf := &fakeConfirmer{handled: true}
o := newConfirmOrch(store, conf, fakeWindow{open: true}, confirmSettings(), now)
o.sweepAutoConfirm(now)
if conf.calls != 1 {
t.Errorf("confirm calls = %d, want 1 (only the eligible package)", conf.calls)
}
}
func TestSweepAutoConfirm_HoldsUncheckedClosure(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
// Eligible by severity/window, but the closure was never OSV-checked (no
// closure_checked_at) — fail-closed, must not auto-mint.
store.add(pkg(models.StatusPendingDependencies, "low", 0, now))
conf := &fakeConfirmer{handled: true}
o := newConfirmOrch(store, conf, fakeWindow{open: true}, confirmSettings(), now)
o.sweepAutoConfirm(now)
if conf.calls != 0 {
t.Errorf("must not auto-confirm an unchecked closure, got %d calls", conf.calls)
}
}
func TestSweepAutoConfirm_BlocksVulnerableClosure(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
p := clearedPkg("low", now)
p.Metadata["closure_vulns"] = `[{"name":"libxml2","version":"2.9.0","vulns":[{"id":"CVE-9"}]}]`
store.add(p)
conf := &fakeConfirmer{handled: true}
o := newConfirmOrch(store, conf, fakeWindow{open: true}, confirmSettings(), now)
o.sweepAutoConfirm(now)
if conf.calls != 0 {
t.Errorf("must not auto-confirm a closure with vulns, got %d calls", conf.calls)
}
}
func TestSweepAutoConfirm_SkipsOutsideWindow(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPendingDependencies, "low", 0, now))
conf := &fakeConfirmer{handled: true}
o := newConfirmOrch(store, conf, fakeWindow{open: false}, confirmSettings(), now)
o.sweepAutoConfirm(now)
if conf.calls != 0 {
t.Errorf("must not confirm outside the maintenance window, got %d calls", conf.calls)
}
}
func TestSweepAutoConfirm_FailClosedOnWindowError(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPendingDependencies, "low", 0, now))
conf := &fakeConfirmer{handled: true}
o := newConfirmOrch(store, conf, fakeWindow{err: errors.New("db down")}, confirmSettings(), now)
o.sweepAutoConfirm(now)
if conf.calls != 0 {
t.Errorf("must not confirm when the window check errors, got %d calls", conf.calls)
}
}
func TestSweepAutoConfirm_DisabledByDefault(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPendingDependencies, "low", 0, now))
conf := &fakeConfirmer{handled: true}
o := newConfirmOrch(store, conf, fakeWindow{open: true}, fakeSettings{}, now) // policy off
o.sweepAutoConfirm(now)
if conf.calls != 0 {
t.Errorf("auto-confirm must be a no-op when auto-approval is off, got %d calls", conf.calls)
}
}
func TestSweepAutoConfirm_NoConfirmerIsNoOp(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusPendingDependencies, "low", 0, now))
// confirmer + window nil (unwired, e.g. minter disabled)
o := &Orchestrator{store: store, settings: confirmSettings(), clock: fixedClock{now}, interval: time.Minute, stopChan: make(chan struct{})}
o.sweepAutoConfirm(now) // must not panic
}
// --- stuck-state recovery --------------------------------------------------
func TestSweepCheckingDependencies_RequeueThenFail(t *testing.T) {
now := time.Now().UTC()
stuck := pkg(models.StatusCheckingDependencies, "high", 45*time.Minute, now) // past 30m threshold
store := newFakeStore()
store.add(stuck)
enq := &fakeEnqueuer{}
set := fakeSettings{} // defaults: 30m threshold, 2 retries
o := newOrch(store, enq, set, now)
// Attempts 1 and 2 re-enqueue the dry-run; no failure yet.
o.sweepCheckingDependencies(now)
o.sweepCheckingDependencies(now)
if enq.calls != 2 {
t.Fatalf("expected 2 re-enqueues, got %d", enq.calls)
}
if store.transitionsTo(models.StatusFailed) != 0 {
t.Fatalf("must not fail before retry budget is spent")
}
// Attempt 3 exceeds the budget → fail.
o.sweepCheckingDependencies(now)
if enq.calls != 2 {
t.Errorf("must not re-enqueue after budget spent, got %d calls", enq.calls)
}
if store.transitionsTo(models.StatusFailed) != 1 {
t.Errorf("expected package failed after budget, got %d failed transitions", store.transitionsTo(models.StatusFailed))
}
}
func TestSweepCheckingDependencies_SkipsFreshPackages(t *testing.T) {
now := time.Now().UTC()
fresh := pkg(models.StatusCheckingDependencies, "high", 5*time.Minute, now) // under 30m
store := newFakeStore()
store.add(fresh)
enq := &fakeEnqueuer{}
o := newOrch(store, enq, fakeSettings{}, now)
o.sweepCheckingDependencies(now)
if enq.calls != 0 || len(store.transitions) != 0 {
t.Errorf("fresh package must be untouched: enqueues=%d transitions=%d", enq.calls, len(store.transitions))
}
}
func TestSweepInstalling_FailsStuck(t *testing.T) {
now := time.Now().UTC()
store := newFakeStore()
store.add(pkg(models.StatusInstalling, "high", 90*time.Minute, now)) // past 60m
store.add(pkg(models.StatusInstalling, "high", 10*time.Minute, now)) // fresh
o := newOrch(store, &fakeEnqueuer{}, fakeSettings{}, now)
o.sweepInstalling(now)
if got := store.transitionsTo(models.StatusFailed); got != 1 {
t.Errorf("expected exactly the stuck install to fail, got %d", got)
}
}

84
server/internal/orchestrator/policy.go Normal file
View file

@ -0,0 +1,84 @@
package orchestrator
import (
"strings"
"github.com/Fimeg/RedFlag/server/internal/models"
)
// severityRank orders package severities from least to most urgent. A package is
// auto-approvable only when its severity is at or below the configured ceiling.
var severityRank = map[string]int{
"low": 1,
"moderate": 2,
"important": 3,
"high": 4,
"critical": 5,
}
// severityUnknown ranks above critical so an unrecognized severity is never
// auto-approved — unknown means "a human looks at it".
const severityUnknown = 99
func rankOf(severity string) int {
if r, ok := severityRank[strings.ToLower(strings.TrimSpace(severity))]; ok {
return r
}
return severityUnknown
}
// autoApproveCeiling resolves policy.auto_approve_max_severity into a rank. The
// sentinels "off" / "none" / "disabled" / "" return 0 — nothing is at or below
// rank 0, so auto-approval is disabled. This is the safe default.
func autoApproveCeiling(setting string) int {
switch strings.ToLower(strings.TrimSpace(setting)) {
case "", "off", "none", "disabled":
return 0
}
r := rankOf(setting)
if r == severityUnknown {
// An unrecognized policy value must not silently approve everything;
// fail safe to disabled.
return 0
}
return r
}
// shouldAutoApprove decides whether a pending package may be advanced without an
// operator. It is intentionally conservative: severity must be at or below the
// configured ceiling, and the package must not carry supply-chain vulnerabilities.
// It never bypasses dry-run or the capability gate — it only collapses the
// pending -> approved -> checking_dependencies wait that an operator would
// otherwise click through.
func shouldAutoApprove(pkg models.UpdateState, maxSeverity string) bool {
ceiling := autoApproveCeiling(maxSeverity)
if ceiling == 0 {
return false
}
if rankOf(pkg.Severity) > ceiling {
return false
}
if hasBlockingVulns(pkg) {
return false
}
return true
}
// hasBlockingVulns reports whether the package metadata records supply-chain
// vulnerabilities on the top-level package. Auto-approval defers to the operator
// whenever vulns are present, regardless of the severity ceiling — the vuln
// review is a human call. Delegates to the shared gate predicate so auto-confirm
// and the manual approval path enforce the same rule.
func hasBlockingVulns(pkg models.UpdateState) bool {
return models.MetadataHasVulns(pkg, "supply_chain_vulns")
}
// closureCleared reports whether the resolved dependency closure has been
// OSV-checked AND came back clean. Auto-confirmation requires both. A closure
// that was never checked (closure_checked_at absent — e.g. OSV was unreachable
// at report time) is treated as NOT cleared: fail-closed, so the capability
// token is never minted over transitive artifacts OSV did not vet. A closure
// carrying vulns is left for an operator regardless of the severity ceiling.
func closureCleared(pkg models.UpdateState) bool {
return models.ClosureCleared(pkg)
}

127
server/internal/orchestrator/timeouts.go Normal file
View file

@ -0,0 +1,127 @@
package orchestrator
import (
"fmt"
"log"
"time"
"github.com/Fimeg/RedFlag/server/internal/models"
)
// dryRunRetryKey is the metadata counter the orchestrator uses to bound dry-run
// re-enqueue attempts on a stuck package, without holding state of its own.
const dryRunRetryKey = "orchestrator_dry_run_retries"
// sweepCheckingDependencies recovers packages stuck waiting on a dry-run report.
// A package that has sat in checking_dependencies past the threshold had its
// dry_run_update command lost or its agent-side dry-run fail silently. Re-enqueue
// the dry-run up to the retry budget; once spent, fail the package so it stops
// occupying an active state (ETHOS §3 — every active state has a timeout path).
func (o *Orchestrator) sweepCheckingDependencies(now time.Time) {
threshold := time.Duration(o.settings.GetOperationalInt("orchestrator_checking_timeout_minutes", 30)) * time.Minute
maxRetries := o.settings.GetOperationalInt("orchestrator_dry_run_max_retries", 2)
stuck, err := o.store.GetPackagesInStatus(models.StatusCheckingDependencies)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] sweep_checking_failed error=%v", err)
return
}
for _, pkg := range stuck {
if now.Sub(pkg.LastUpdatedAt) < threshold {
continue
}
attempts, err := o.store.BumpRetryCounter(pkg.ID, dryRunRetryKey)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] dry_run_retry_bump_failed update_id=%s error=%v", pkg.ID, err)
continue
}
if attempts > maxRetries {
meta := models.JSONB{
"failure_reason": "dry-run never reported; retry budget exhausted",
"failed_by": "orchestrator",
"dry_run_attempts": attempts - 1,
}
if err := o.store.TransitionByID(pkg.ID, models.StatusFailed, meta); err != nil {
log.Printf("[ERROR] [server] [orchestrator] checking_fail_transition_failed update_id=%s error=%v", pkg.ID, err)
continue
}
log.Printf("[INFO] [server] [orchestrator] checking_dependencies_failed update_id=%s package=%s/%s attempts=%d",
pkg.ID, pkg.PackageType, pkg.PackageName, attempts-1)
aid := pkg.AgentID
o.emitEvent(&aid, "orchestrator", "checking_dependencies_timeout", "error", "orchestrator",
"Dry-run never reported; retry budget exhausted", map[string]interface{}{
"update_id": pkg.ID.String(),
"package_type": pkg.PackageType,
"package_name": pkg.PackageName,
"dry_run_attempts": attempts - 1,
})
continue
}
p := pkg
if err := o.enqueuer.EnqueueDryRun(&p); err != nil {
log.Printf("[ERROR] [server] [orchestrator] dry_run_requeue_failed update_id=%s error=%v", pkg.ID, err)
continue
}
log.Printf("[INFO] [server] [orchestrator] dry_run_requeued update_id=%s package=%s/%s attempt=%d",
pkg.ID, pkg.PackageType, pkg.PackageName, attempts)
}
}
// sweepInstalling fails packages that entered installing but never reported a
// terminal result within the window (helper crashed mid-install, agent died, or
// the token was consumed but the result was lost). The 1h default matches the
// command-layer sent-timeout so the two layers don't fight over the same row.
func (o *Orchestrator) sweepInstalling(now time.Time) {
threshold := time.Duration(o.settings.GetOperationalInt("orchestrator_installing_timeout_minutes", 60)) * time.Minute
stuck, err := o.store.GetPackagesInStatus(models.StatusInstalling)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] sweep_installing_failed error=%v", err)
return
}
for _, pkg := range stuck {
if now.Sub(pkg.LastUpdatedAt) < threshold {
continue
}
meta := models.JSONB{
"failure_reason": fmt.Sprintf("install did not report a result within %s", threshold),
"failed_by": "orchestrator",
}
if err := o.store.TransitionByID(pkg.ID, models.StatusFailed, meta); err != nil {
log.Printf("[ERROR] [server] [orchestrator] installing_fail_transition_failed update_id=%s error=%v", pkg.ID, err)
continue
}
log.Printf("[INFO] [server] [orchestrator] installing_timed_out update_id=%s package=%s/%s threshold=%s",
pkg.ID, pkg.PackageType, pkg.PackageName, threshold)
aid := pkg.AgentID
o.emitEvent(&aid, "orchestrator", "installing_timeout", "error", "orchestrator",
"Install did not report a result within threshold",
map[string]interface{}{
"update_id": pkg.ID.String(),
"package_type": pkg.PackageType,
"package_name": pkg.PackageName,
"threshold": threshold.String(),
})
}
}
// sweepPendingDependencies surfaces packages an operator left unconfirmed for a
// long time. Per design this is NOT a state change — the operator deliberately
// deferred, and the package ages out on the next scan. We only log the count so
// the condition is visible; the operator-facing warning indicator is LIFECYCLE-004.
func (o *Orchestrator) sweepPendingDependencies(now time.Time) {
threshold := time.Duration(o.settings.GetOperationalInt("orchestrator_pending_deps_warn_hours", 24)) * time.Hour
stuck, err := o.store.GetPackagesInStatus(models.StatusPendingDependencies)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] sweep_pending_deps_failed error=%v", err)
return
}
stale := 0
for _, pkg := range stuck {
if now.Sub(pkg.LastUpdatedAt) >= threshold {
stale++
}
}
if stale > 0 {
log.Printf("[INFO] [server] [orchestrator] pending_dependencies_stale count=%d threshold=%s", stale, threshold)
}
}

190
server/internal/orchestrator/workflow.go Normal file
View file

@ -0,0 +1,190 @@
package orchestrator
import (
"fmt"
"log"
"time"
"github.com/Fimeg/RedFlag/server/internal/models"
"github.com/gofrs/uuid/v5"
)
// sweepAutoApprove advances eligible pending packages without an operator. Policy
// is consulted per package; a qualifying package is approved and its dry-run is
// enqueued (approved -> checking_dependencies). Auto-approval stops there: the
// install phase still requires the dry-run result and the maintenance window
// (enforced at InstallUpdate), and on the capability path the supply-chain gate
// still mints and verifies the token. Auto-approval never reaches installing on
// its own, and never runs when dry-runs are disabled by policy.
func (o *Orchestrator) sweepAutoApprove() {
maxSeverity := o.settings.GetPolicyString("auto_approve_max_severity", "off")
if autoApproveCeiling(maxSeverity) == 0 {
return // disabled — the safe default
}
if !o.settings.GetPolicyBool("allow_dry_runs", true) {
// The operator has taken manual control of the dry-run step; auto-approval
// would strand packages in approved with no way to advance. Stay out.
return
}
pending, err := o.store.GetPackagesInStatus(models.StatusPending)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] sweep_auto_approve_failed error=%v", err)
return
}
approved := 0
for _, pkg := range pending {
if !shouldAutoApprove(pkg, maxSeverity) {
continue
}
if err := o.advanceApproved(pkg); err != nil {
log.Printf("[ERROR] [server] [orchestrator] auto_approve_failed update_id=%s error=%v", pkg.ID, err)
continue
}
approved++
}
if approved > 0 {
log.Printf("[INFO] [server] [orchestrator] auto_approved count=%d max_severity=%s", approved, maxSeverity)
}
}
// advanceApproved approves a package and enqueues its dry-run. Both steps are
// individually idempotent: the guarded transition no-ops if the row already
// moved, and a duplicate dry_run_update command is harmless (the agent reports
// against update_id, and DryRun is read-only).
func (o *Orchestrator) advanceApproved(pkg models.UpdateState) error {
meta := models.JSONB{"approved_by": "orchestrator", "auto_approved": true}
if err := o.store.TransitionByID(pkg.ID, models.StatusApproved, meta); err != nil {
return fmt.Errorf("approve: %w", err)
}
p := pkg
p.Status = models.StatusApproved
if err := o.enqueuer.EnqueueDryRun(&p); err != nil {
return fmt.Errorf("enqueue dry-run: %w", err)
}
aid := pkg.AgentID
o.emitEvent(&aid, "orchestrator", "auto_approved", "info", "orchestrator",
"Package auto-approved by orchestrator",
map[string]interface{}{
"update_id": pkg.ID.String(),
"package_type": pkg.PackageType,
"package_name": pkg.PackageName,
})
return nil
}
// sweepAutoConfirm advances capability-gated packages waiting in
// pending_dependencies to installing and mints their capability token, without
// an operator click — the dependency-confirmation half of the auto-approve
// lane. It is intentionally stateless: rather than relying on a persisted
// "auto-approved" marker (the approve-time history metadata is not carried on
// the row), it re-applies the same auto-approval policy here. A package is
// auto-confirmed only when:
//
// - auto-approval policy is enabled and the package is at or below the
// severity ceiling with no top-level supply-chain vulns (shouldAutoApprove), and
// - the resolved dependency closure has been OSV-checked and came back clean
// (closureCleared) — fail-closed: an unchecked or vulnerable closure is
// never auto-minted, since the closure is the exact artifact set the token
// authorizes, and
// - dry-runs are not under manual operator control (allow_dry_runs), and
// - the current time is inside a maintenance window (installs respect the
// window even though the read-only dry-run that preceded this did not), and
// - the package is capability-gated (dnf/apt) — legacy command packages are
// left for the operator, since the server can't verify a legacy dependency.
//
// The mint itself is idempotent (an active token short-circuits the mint), and
// the install transition is guarded, so a redundant pass is a no-op.
func (o *Orchestrator) sweepAutoConfirm(now time.Time) {
if o.confirmer == nil || o.window == nil {
return // not wired (e.g. minter disabled) — closures wait for an operator
}
maxSeverity := o.settings.GetPolicyString("auto_approve_max_severity", "off")
if autoApproveCeiling(maxSeverity) == 0 {
return // auto-approval disabled — the safe default
}
if !o.settings.GetPolicyBool("allow_dry_runs", true) {
return // operator has taken manual control of the dry-run/confirm lane
}
open, err := o.window.IsWithinMaintenanceWindow(now)
if err != nil {
// Fail closed: do not auto-confirm installs when the window can't be evaluated.
log.Printf("[ERROR] [server] [orchestrator] auto_confirm_window_check_failed error=%v", err)
return
}
if !open {
return // outside the maintenance window — installs wait, dry-run already done
}
waiting, err := o.store.GetPackagesInStatus(models.StatusPendingDependencies)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] sweep_auto_confirm_failed error=%v", err)
return
}
confirmed := 0
for _, pkg := range waiting {
if !shouldAutoApprove(pkg, maxSeverity) {
continue // too severe, or top-level vulns — a human confirms this one
}
if !closureCleared(pkg) {
// Closure not yet OSV-checked (will retry), or it carries vulns in a
// transitive artifact — never auto-mint over an unvetted/vulnerable
// closure. The dependency vulns are recorded on the row for the operator.
continue
}
p := pkg
handled, err := o.confirmer.ConfirmDependenciesAuto(&p)
if err != nil {
// The confirmer has already marked the package failed; just record it.
log.Printf("[ERROR] [server] [orchestrator] auto_confirm_failed update_id=%s package=%s/%s error=%v",
pkg.ID, pkg.PackageType, pkg.PackageName, err)
continue
}
if !handled {
continue // legacy command path — left for the operator
}
confirmed++
log.Printf("[INFO] [server] [orchestrator] auto_confirmed update_id=%s package=%s/%s",
pkg.ID, pkg.PackageType, pkg.PackageName)
aid := pkg.AgentID
o.emitEvent(&aid, "orchestrator", "auto_confirmed", "info", "orchestrator",
"Dependencies auto-confirmed; capability token minted",
map[string]interface{}{
"update_id": pkg.ID.String(),
"package_type": pkg.PackageType,
"package_name": pkg.PackageName,
})
}
if confirmed > 0 {
log.Printf("[INFO] [server] [orchestrator] auto_confirm_minted count=%d max_severity=%s", confirmed, maxSeverity)
}
}
// AdvanceLifecycle is a synchronous, targeted entry point: given a freshly
// discovered package id, it auto-approves immediately if policy allows, instead
// of waiting for the next timer sweep. Best-effort and idempotent — it no-ops
// unless the package is still pending and qualifies. Report-driven transitions
// (dependency results, capability results) remain in their handlers, which the
// LIFECYCLE-001 guarded transitions already make safe; folding those into the
// orchestrator is tracked LIFECYCLE-003 follow-up.
func (o *Orchestrator) AdvanceLifecycle(updateID uuid.UUID) {
pkg, err := o.store.GetUpdateByID(updateID)
if err != nil {
log.Printf("[ERROR] [server] [orchestrator] advance_load_failed update_id=%s error=%v", updateID, err)
return
}
if pkg.Status != models.StatusPending {
return
}
if !o.settings.GetPolicyBool("allow_dry_runs", true) {
return
}
maxSeverity := o.settings.GetPolicyString("auto_approve_max_severity", "off")
if !shouldAutoApprove(*pkg, maxSeverity) {
return
}
if err := o.advanceApproved(*pkg); err != nil {
log.Printf("[ERROR] [server] [orchestrator] advance_auto_approve_failed update_id=%s error=%v", updateID, err)
}
}