Server becomes self-contained: web/dist embedded via go:embed (server/internal/webui), SPA served from the binary with JSON-404 guard on /api paths, nginx web container removed from compose (31336 now maps to the server). Clean checkouts without the UI copy build API-only. Agent local API gains its first write endpoint, POST /v1/actions/trigger-scan (FEAT-002 write path): group-ACL authorized, single-flight, 202/409/503 semantics. Registered agents run the same HandleScanUpdates path as a signed scan command (empty command_id, no ack tracking); standalone agents scan through the orchestrator into the local read model only. Also repairs localapi tests left uncompilable by the desktop-provider parameter.
813 lines
32 KiB
Go
813 lines
32 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math/rand"
|
|
"os"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/Fimeg/RedFlag/agent/internal/acknowledgment"
|
|
"github.com/Fimeg/RedFlag/agent/internal/cache"
|
|
"github.com/Fimeg/RedFlag/agent/internal/circuitbreaker"
|
|
"github.com/Fimeg/RedFlag/agent/internal/client"
|
|
"github.com/Fimeg/RedFlag/agent/internal/config"
|
|
"github.com/Fimeg/RedFlag/agent/internal/constants"
|
|
"github.com/Fimeg/RedFlag/agent/internal/desktop"
|
|
"github.com/Fimeg/RedFlag/agent/internal/handlers"
|
|
"github.com/Fimeg/RedFlag/agent/internal/integrations"
|
|
"github.com/Fimeg/RedFlag/agent/internal/kernel"
|
|
"github.com/Fimeg/RedFlag/agent/internal/localapi"
|
|
"github.com/Fimeg/RedFlag/agent/internal/logging"
|
|
"github.com/Fimeg/RedFlag/agent/internal/models"
|
|
"github.com/Fimeg/RedFlag/agent/internal/orchestrator"
|
|
"github.com/Fimeg/RedFlag/agent/internal/receipt"
|
|
"github.com/Fimeg/RedFlag/agent/internal/recovery"
|
|
"github.com/Fimeg/RedFlag/agent/internal/scanner"
|
|
"github.com/Fimeg/RedFlag/agent/internal/startup"
|
|
"github.com/Fimeg/RedFlag/agent/internal/supplychain"
|
|
"github.com/Fimeg/RedFlag/agent/internal/system"
|
|
"github.com/Fimeg/RedFlag/agent/internal/version"
|
|
)
|
|
|
|
// newCircuitBreaker creates a circuit breaker from config
|
|
func newCircuitBreaker(name string, cfg config.CircuitBreakerConfig) *circuitbreaker.CircuitBreaker {
|
|
return circuitbreaker.New(name, circuitbreaker.Config{
|
|
FailureThreshold: cfg.FailureThreshold,
|
|
FailureWindow: cfg.FailureWindow,
|
|
OpenDuration: cfg.OpenDuration,
|
|
HalfOpenAttempts: cfg.HalfOpenAttempts,
|
|
})
|
|
}
|
|
|
|
// RunAgentLoop runs the main agent polling loop
|
|
func RunAgentLoop(cfg *config.Config) error {
|
|
// Panic recovery for the main agent loop [TD-002]
|
|
defer recovery.Recover("agent_main_loop")
|
|
|
|
// Initialize startup logger
|
|
startupLogger := startup.NewLogger(constants.GetAgentStateDir(), version.Version)
|
|
startupLogger.LogEvent(startup.EventTypeStartup, true, nil, map[string]interface{}{
|
|
"agent_id": cfg.AgentID.String(),
|
|
"server": cfg.ServerURL,
|
|
})
|
|
|
|
log.Printf("RedFlag Agent v%s starting...", version.Version)
|
|
log.Printf("Agent ID: %s Server: %s Interval: %ds",
|
|
cfg.AgentID, cfg.ServerURL, cfg.CheckInInterval)
|
|
|
|
apiClient := client.NewClient(cfg.ServerURL, cfg.Token)
|
|
|
|
// Initialize scanners
|
|
aptScanner := scanner.NewAPTScanner()
|
|
dnfScanner := scanner.NewDNFScanner()
|
|
windowsUpdateScanner := scanner.NewWindowsUpdateScanner()
|
|
wingetScanner := scanner.NewWingetScanner()
|
|
storageScanner := orchestrator.NewStorageScanner(version.Version)
|
|
systemScanner := orchestrator.NewSystemScanner(version.Version)
|
|
dockerScanner, _ := orchestrator.NewDockerScanner()
|
|
|
|
// Initialize circuit breakers
|
|
aptCB := newCircuitBreaker("APT", cfg.Subsystems.APT.CircuitBreaker)
|
|
dnfCB := newCircuitBreaker("DNF", cfg.Subsystems.DNF.CircuitBreaker)
|
|
windowsCB := newCircuitBreaker("Windows Update", cfg.Subsystems.Windows.CircuitBreaker)
|
|
wingetCB := newCircuitBreaker("Winget", cfg.Subsystems.Winget.CircuitBreaker)
|
|
storageCB := newCircuitBreaker("Storage", cfg.Subsystems.Storage.CircuitBreaker)
|
|
systemCB := newCircuitBreaker("System", cfg.Subsystems.System.CircuitBreaker)
|
|
dockerCB := newCircuitBreaker("Docker", cfg.Subsystems.Docker.CircuitBreaker)
|
|
|
|
// Initialize orchestrator
|
|
scanOrchestrator := orchestrator.NewOrchestrator()
|
|
|
|
// Register all scanners
|
|
scanOrchestrator.RegisterScanner("apt", aptScanner, aptCB, cfg.Subsystems.APT.Timeout, cfg.Subsystems.APT.Enabled)
|
|
scanOrchestrator.RegisterScanner("dnf", dnfScanner, dnfCB, cfg.Subsystems.DNF.Timeout, cfg.Subsystems.DNF.Enabled)
|
|
scanOrchestrator.RegisterScanner("windows", windowsUpdateScanner, windowsCB, cfg.Subsystems.Windows.Timeout, cfg.Subsystems.Windows.Enabled)
|
|
scanOrchestrator.RegisterScanner("winget", wingetScanner, wingetCB, cfg.Subsystems.Winget.Timeout, cfg.Subsystems.Winget.Enabled)
|
|
scanOrchestrator.RegisterScanner("storage", storageScanner, storageCB, cfg.Subsystems.Storage.Timeout, cfg.Subsystems.Storage.Enabled)
|
|
scanOrchestrator.RegisterScanner("system", systemScanner, systemCB, cfg.Subsystems.System.Timeout, cfg.Subsystems.System.Enabled)
|
|
scanOrchestrator.RegisterScanner("docker", dockerScanner, dockerCB, cfg.Subsystems.Docker.Timeout, cfg.Subsystems.Docker.Enabled)
|
|
|
|
// Initialize acknowledgment tracker (result acks — pending_acks.json)
|
|
ackTracker := acknowledgment.NewTracker(constants.GetAgentStateDir())
|
|
if err := ackTracker.Load(); err != nil {
|
|
log.Printf("[WARNING] [agent] [acknowledgment] load_pending_acks_failed error=%v", err)
|
|
}
|
|
|
|
// Initialize kernel enforcement enforcer
|
|
kernelEnforcer, err := kernel.NewEnforcer(cfg)
|
|
if err != nil {
|
|
log.Printf("[WARNING] [agent] [kernel] enforcer_init_failed error=%v", err)
|
|
} else {
|
|
log.Printf("[INFO] [agent] [kernel] %s_enforcer_started", kernelEnforcer.GetPackageType())
|
|
}
|
|
|
|
// Initialize receipt tracker (command-receipt confirmation — pending_receipts.json)
|
|
// Migration 033 §2: doctrine TODO-full-command-lifecycle.md
|
|
receiptTracker := receipt.NewTracker(constants.GetAgentStateDir())
|
|
if err := receiptTracker.Load(); err != nil {
|
|
log.Printf("[WARNING] [agent] [receipt] load_pending_receipts_failed error=%v", err)
|
|
}
|
|
|
|
// Initialize confirmed tracker (command-completion confirmation from server)
|
|
// This tracks commands the server has acknowledged via ReportLog, preventing
|
|
// false-positive duplicate rejections when server hasn't processed the log yet.
|
|
confirmedTracker := orchestrator.NewConfirmedTracker(constants.GetAgentStateDir())
|
|
if err := confirmedTracker.Load(); err != nil {
|
|
log.Printf("[WARNING] [agent] [confirmed] load_confirmed_completed_failed error=%v", err)
|
|
}
|
|
|
|
// Initialize command handler
|
|
securityLogger, _ := logging.NewSecurityLogger(cfg, constants.GetAgentStateDir())
|
|
commandHandler, err := orchestrator.NewCommandHandler(cfg, constants.GetAgentStateDir(), securityLogger, log.New(os.Stdout, "", log.LstdFlags))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to initialize command handler: %w", err)
|
|
}
|
|
|
|
// Initialize desktop manager (spawns Tauri system tray + local UI)
|
|
desktopMgr := desktop.NewManager(
|
|
"", // auto-detect binary alongside agent
|
|
cfg.Desktop.Enabled,
|
|
cfg.Desktop.MaxRestarts,
|
|
cfg.Desktop.RestartDelaySec,
|
|
)
|
|
|
|
// Start the main loop
|
|
return RunPollingLoop(&LoopContext{
|
|
Ctx: context.Background(),
|
|
Cfg: cfg,
|
|
APIClient: apiClient,
|
|
AckTracker: ackTracker,
|
|
ReceiptTracker: receiptTracker,
|
|
ConfirmedTracker: confirmedTracker,
|
|
CommandHandler: commandHandler,
|
|
ScanOrchestrator: scanOrchestrator,
|
|
DesktopManager: desktopMgr,
|
|
CircuitBreakers: map[string]*circuitbreaker.CircuitBreaker{
|
|
"apt": aptCB,
|
|
"dnf": dnfCB,
|
|
"windows": windowsCB,
|
|
"winget": wingetCB,
|
|
"storage": storageCB,
|
|
"system": systemCB,
|
|
"docker": dockerCB,
|
|
},
|
|
})
|
|
}
|
|
|
|
// LoopContext holds all dependencies for the polling loop.
|
|
// Fields are exported so the Windows service can fully construct one.
|
|
type LoopContext struct {
|
|
Cfg *config.Config
|
|
APIClient *client.Client
|
|
AckTracker *acknowledgment.Tracker
|
|
ReceiptTracker *receipt.Tracker
|
|
ConfirmedTracker *orchestrator.ConfirmedTracker // tracks commands server confirmed as completed
|
|
CommandHandler *orchestrator.CommandHandler
|
|
ScanOrchestrator *orchestrator.Orchestrator
|
|
CircuitBreakers map[string]*circuitbreaker.CircuitBreaker
|
|
KernelEnforcer kernel.Enforcer
|
|
DesktopManager *desktop.Manager
|
|
Ctx context.Context
|
|
StopCh <-chan struct{} // non-nil causes loop to exit cleanly when closed
|
|
}
|
|
|
|
// RunPollingLoop runs the main agent polling loop.
|
|
// It is called by RunAgentLoop and may also be called by the Windows service
|
|
// with a stop channel. When stopCh is non-nil, the loop selects on it and exits cleanly.
|
|
func RunPollingLoop(loopCtx *LoopContext) error {
|
|
// Panic recovery for the polling loop [TD-002]
|
|
defer recovery.Recover("agent_polling_loop")
|
|
|
|
ctx := loopCtx
|
|
|
|
// FEAT-002 write path: single-flight scan trigger for the local API.
|
|
// Authorization is the socket/pipe group ACL; the scan itself runs through
|
|
// the same handler primitives a signed server scan command uses.
|
|
var scanInFlight atomic.Bool
|
|
triggerScan := func(source string) error {
|
|
if !scanInFlight.CompareAndSwap(false, true) {
|
|
return localapi.ErrScanInFlight
|
|
}
|
|
go func() {
|
|
defer recovery.Recover("local_triggered_scan")
|
|
defer scanInFlight.Store(false)
|
|
if err := handlers.HandleLocalTriggeredScan(ctx.APIClient, ctx.Cfg, ctx.AckTracker, ctx.ScanOrchestrator, source); err != nil {
|
|
log.Printf("[ERROR] [agent] [localapi] local_scan_failed source=%s error=%v", source, err)
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
localAPIServer, err := localapi.Start(localapi.Options{
|
|
Config: ctx.Cfg,
|
|
DesktopProvider: ctx.DesktopManager,
|
|
TriggerScan: triggerScan,
|
|
})
|
|
if err != nil {
|
|
log.Printf("[ERROR] [agent] [localapi] start_failed error=%v", err)
|
|
} else {
|
|
defer localAPIServer.Stop()
|
|
}
|
|
|
|
// Start desktop app (connects to the local API socket above)
|
|
if ctx.DesktopManager != nil {
|
|
go ctx.DesktopManager.Start(ctx.Ctx)
|
|
defer ctx.DesktopManager.Stop()
|
|
}
|
|
|
|
// Start kernel enforcement enforcer
|
|
if ctx.KernelEnforcer != nil {
|
|
if err := ctx.KernelEnforcer.Start(ctx.Ctx); err != nil {
|
|
log.Printf("[ERROR] [agent] [kernel] enforcer_start_failed error=%v", err)
|
|
}
|
|
defer func() {
|
|
if err := ctx.KernelEnforcer.Stop(); err != nil {
|
|
log.Printf("[ERROR] [agent] [kernel] enforcer_stop_failed error=%v", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
consecutiveFailures := 0
|
|
lastSystemInfoUpdate := time.Time{}
|
|
lastConfigRefresh := time.Time{} // zero → refresh on first successful check-in
|
|
|
|
for {
|
|
// Stop-channel check before each iteration
|
|
if ctx.StopCh != nil {
|
|
select {
|
|
case <-ctx.StopCh:
|
|
log.Printf("[INFO] [agent] [loop] stop_signal_received")
|
|
return nil
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Calculate jitter — always use the base check-in interval for
|
|
// pre-fetch jitter; rapid-polling acceleration applies to the
|
|
// post-processing sleep (recalculated after commands are handled).
|
|
baseInterval := time.Duration(ctx.Cfg.CheckInInterval) * time.Second
|
|
maxJitter := baseInterval / 2
|
|
jitterCap := time.Duration(resolveJitterMaxSeconds(ctx.Cfg)) * time.Second
|
|
if maxJitter > jitterCap {
|
|
maxJitter = jitterCap
|
|
}
|
|
if maxJitter < 1*time.Second {
|
|
maxJitter = 1 * time.Second
|
|
}
|
|
|
|
jitter := time.Duration(rand.Intn(int(maxJitter.Seconds())+1)) * time.Second
|
|
time.Sleep(jitter)
|
|
|
|
// Check for system info update
|
|
if time.Since(lastSystemInfoUpdate) >= 1*time.Hour {
|
|
if err := reportSystemInfo(ctx.APIClient, ctx.Cfg); err != nil {
|
|
log.Printf("Failed to report system info: %v", err)
|
|
} else {
|
|
lastSystemInfoUpdate = time.Now()
|
|
}
|
|
}
|
|
|
|
// Refresh server public key if needed
|
|
if ctx.CommandHandler.ShouldRefreshKey() {
|
|
if err := ctx.CommandHandler.RefreshPrimaryKey(ctx.Cfg.ServerURL); err != nil {
|
|
log.Printf("[WARNING] Failed to refresh public key: %v", err)
|
|
}
|
|
ctx.CommandHandler.CleanupExecutedIDs()
|
|
}
|
|
|
|
log.Printf("Checking in with server... (Agent v%s)", version.Version)
|
|
|
|
// Collect system metrics, then bolt on the tracked IDs so the server can both
|
|
// (a) acknowledge command results we're still buffering (pending_acks.json) and
|
|
// (b) confirm receipt of commands we received last round but haven't completed
|
|
// (pending_receipts.json, Migration 033 §2).
|
|
metrics := collectMetrics(ctx.Cfg)
|
|
if metrics != nil {
|
|
metrics.PendingAcknowledgments = ctx.AckTracker.GetPending()
|
|
metrics.ReceivedCommandIDs = ctx.ReceiptTracker.GetPending()
|
|
}
|
|
|
|
// Get commands from server
|
|
response, err := ctx.APIClient.GetCommands(ctx.Cfg.AgentID, metrics)
|
|
if err != nil {
|
|
terminalBackoff := false
|
|
if errors.Is(err, client.ErrMachineMismatch) {
|
|
// Terminal: the server no longer recognizes this host as the one the
|
|
// agent registered on — config moved or copied. Renewal can't fix it
|
|
// (renewal is now machine-bound too), so don't even try. Surface loudly;
|
|
// the client already buffered a critical machine_binding_rejected event.
|
|
// We keep polling rather than exit, so the agent stays visible and
|
|
// self-heals the moment an operator rebinds it server-side.
|
|
log.Printf("[ERROR] [agent] [auth] machine_id_mismatch identity_moved_or_copied re_registration_required agent_id=%s", ctx.Cfg.AgentID)
|
|
terminalBackoff = true
|
|
} else if errors.Is(err, client.ErrUnauthorized) && ctx.Cfg.RefreshToken != "" {
|
|
log.Printf("[INFO] [agent] [auth] jwt_expired attempting_renewal agent_id=%s", ctx.Cfg.AgentID)
|
|
renewErr := ctx.APIClient.RenewToken(ctx.Cfg.AgentID, ctx.Cfg.RefreshToken, version.Version)
|
|
switch {
|
|
case renewErr == nil:
|
|
ctx.Cfg.Token = ctx.APIClient.GetToken()
|
|
// The refresh token rotates on each renewal (migration 045) — persist
|
|
// the new one or the next renewal will look like a replay. If the Save
|
|
// fails here, the server's accept-previous-once grace recovers us on
|
|
// the next attempt with the old token still on disk.
|
|
if rt := ctx.APIClient.GetRefreshToken(); rt != "" {
|
|
ctx.Cfg.RefreshToken = rt
|
|
}
|
|
if saveErr := ctx.Cfg.Save(constants.GetAgentConfigPath()); saveErr != nil {
|
|
log.Printf("[WARNING] [agent] [auth] token_persist_failed error=%v", saveErr)
|
|
}
|
|
log.Printf("[INFO] [agent] [auth] token_renewed_successfully")
|
|
consecutiveFailures = 0
|
|
continue
|
|
case errors.Is(renewErr, client.ErrRefreshTokenInvalid):
|
|
// Terminal: the refresh token is dead. Backing off won't help —
|
|
// the agent needs re-registration. Surface it loudly; the client
|
|
// already buffered a critical refresh_token_invalid event.
|
|
log.Printf("[ERROR] [agent] [auth] refresh_token_invalid re_registration_required agent_id=%s error=%v", ctx.Cfg.AgentID, renewErr)
|
|
terminalBackoff = true
|
|
default:
|
|
// Transient renewal failure (network, 502). Fall through to backoff and retry.
|
|
log.Printf("[ERROR] [agent] [auth] token_renewal_failed error=%v", renewErr)
|
|
}
|
|
}
|
|
consecutiveFailures++
|
|
recordLocalAgentStatus(ctx.Cfg, "backoff", false)
|
|
var backoffDelay time.Duration
|
|
if terminalBackoff {
|
|
backoffDelay = 10 * time.Minute
|
|
log.Printf("[ERROR] [agent] [auth] terminal_state waiting_for_operator_intervention delay=%s agent_id=%s", backoffDelay, ctx.Cfg.AgentID)
|
|
} else {
|
|
backoffDelay = calculateBackoff(consecutiveFailures, resolveBackoffBase(ctx.Cfg), resolveBackoffMax(ctx.Cfg))
|
|
log.Printf("[WARNING] Server unavailable (attempt %d), retrying in %s: %v", consecutiveFailures, backoffDelay, err)
|
|
}
|
|
// Non-blocking stop check during backoff
|
|
if ctx.StopCh != nil {
|
|
select {
|
|
case <-ctx.StopCh:
|
|
log.Printf("[INFO] [agent] [loop] stop_signal_received_during_backoff")
|
|
return nil
|
|
default:
|
|
}
|
|
}
|
|
time.Sleep(backoffDelay)
|
|
continue
|
|
}
|
|
|
|
consecutiveFailures = 0
|
|
recordLocalAgentStatus(ctx.Cfg, "online", true)
|
|
|
|
// Refresh fleet-wide operational config (polling resilience tuning) on
|
|
// first check-in and periodically thereafter. Server is the source of
|
|
// the fleet default; non-zero values are merged into the local config so
|
|
// an operator change in the dashboard propagates without touching hosts.
|
|
// Failure here is non-fatal — the agent keeps its current tuning.
|
|
if time.Since(lastConfigRefresh) >= 15*time.Minute {
|
|
if cfgResp, err := ctx.APIClient.GetConfig(ctx.Cfg.AgentID); err != nil {
|
|
log.Printf("[WARNING] [agent] [config] config_refresh_failed error=%v", err)
|
|
} else {
|
|
if applyServerPolling(ctx.Cfg, cfgResp) {
|
|
if saveErr := ctx.Cfg.Save(constants.GetAgentConfigPath()); saveErr != nil {
|
|
log.Printf("[ERROR] [agent] [config] polling_persist_failed error=%v", saveErr)
|
|
} else {
|
|
log.Printf("[INFO] [agent] [config] polling_updated jitter=%d backoff_base=%d backoff_max=%d",
|
|
ctx.Cfg.Polling.JitterMaxSeconds, ctx.Cfg.Polling.BackoffBaseSeconds, ctx.Cfg.Polling.BackoffMaxSeconds)
|
|
}
|
|
}
|
|
lastConfigRefresh = time.Now()
|
|
}
|
|
}
|
|
|
|
// Log check-in success event [TD-003]
|
|
ctx.APIClient.BufferEvent(models.EventTypeAgentCheckIn, models.SubtypeSuccess, models.SeverityInfo,
|
|
models.ComponentAgent, "Agent checked in successfully", map[string]interface{}{
|
|
"commands_received": len(response.Commands),
|
|
"rapid_polling": ctx.Cfg.RapidPollingEnabled && time.Now().Before(ctx.Cfg.RapidPollingUntil),
|
|
})
|
|
|
|
// Drop result-acks the server confirmed.
|
|
if response != nil && len(response.AcknowledgedIDs) > 0 {
|
|
ctx.AckTracker.Acknowledge(response.AcknowledgedIDs)
|
|
log.Printf("[INFO] [agent] [acknowledgment] results_acknowledged count=%d", len(response.AcknowledgedIDs))
|
|
if err := ctx.AckTracker.Save(); err != nil {
|
|
log.Printf("[ERROR] [agent] [acknowledgment] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
// Drop receipts the server confirmed (sent→received transition successful).
|
|
if response != nil && len(response.ReceiptConfirmedIDs) > 0 {
|
|
ctx.ReceiptTracker.Confirm(response.ReceiptConfirmedIDs)
|
|
log.Printf("[INFO] [agent] [receipt] receipts_confirmed count=%d", len(response.ReceiptConfirmedIDs))
|
|
if err := ctx.ReceiptTracker.Save(); err != nil {
|
|
log.Printf("[ERROR] [agent] [receipt] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
// Bound the delivery trackers and journal whatever they abandon. A dropped
|
|
// result-ack or receipt is the silent loss of an auditable event — the agent
|
|
// only ever redelivers the command ID, never the payload, so a result the
|
|
// server never recorded is unrecoverable once it ages out. ETHOS #1: it
|
|
// becomes history, not a stdout line. BufferEvent persists to disk and flushes
|
|
// later, so the record survives the same network loss that caused the drop.
|
|
if dropped := ctx.AckTracker.Cleanup(); len(dropped) > 0 {
|
|
for _, d := range dropped {
|
|
log.Printf("[WARNING] [agent] [acknowledgment] result_ack_dropped command_id=%s reason=%s retries=%d age_s=%d",
|
|
d.CommandID, d.Reason, d.RetryCount, d.AgeSeconds)
|
|
ctx.APIClient.BufferEvent(models.EventTypeError, models.SubtypeFailed, models.SeverityWarning,
|
|
models.ComponentAgent,
|
|
fmt.Sprintf("Abandoned delivery of command result %s (%s) — server never confirmed receipt", d.CommandID, d.Reason),
|
|
map[string]interface{}{
|
|
"kind": "result_ack",
|
|
"command_id": d.CommandID,
|
|
"reason": d.Reason,
|
|
"retry_count": d.RetryCount,
|
|
"age_seconds": d.AgeSeconds,
|
|
})
|
|
}
|
|
if err := ctx.AckTracker.Save(); err != nil {
|
|
log.Printf("[ERROR] [agent] [acknowledgment] save_failed error=%v", err)
|
|
}
|
|
}
|
|
if dropped := ctx.ReceiptTracker.Cleanup(); len(dropped) > 0 {
|
|
for _, d := range dropped {
|
|
log.Printf("[WARNING] [agent] [receipt] receipt_dropped command_id=%s age_s=%d", d.CommandID, d.AgeSeconds)
|
|
ctx.APIClient.BufferEvent(models.EventTypeError, models.SubtypeFailed, models.SeverityWarning,
|
|
models.ComponentAgent,
|
|
fmt.Sprintf("Abandoned receipt confirmation for command %s — server never acknowledged receipt before max-age", d.CommandID),
|
|
map[string]interface{}{
|
|
"kind": "receipt",
|
|
"command_id": d.CommandID,
|
|
"age_seconds": d.AgeSeconds,
|
|
})
|
|
}
|
|
if err := ctx.ReceiptTracker.Save(); err != nil {
|
|
log.Printf("[ERROR] [agent] [receipt] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
// Drop confirmed completions the server acknowledged via ReportLog.
|
|
// This is the key fix for duplicate command rejections: if the server
|
|
// has confirmed a command as completed, the agent should NOT reject it
|
|
// as a duplicate even if it's in the executed set. The race was:
|
|
// 1. Server sends command -> agent executes -> agent reports log
|
|
// 2. Server hasn't processed log yet -> sends command again on next poll
|
|
// 3. Agent rejects as duplicate (BUG)
|
|
// With this fix: agent checks if server confirmed -> if yes, allow.
|
|
if response != nil && len(response.ConfirmedCommandIDs) > 0 {
|
|
ctx.ConfirmedTracker.Confirm(response.ConfirmedCommandIDs)
|
|
log.Printf("[INFO] [agent] [confirmed] completions_confirmed count=%d", len(response.ConfirmedCommandIDs))
|
|
if err := ctx.ConfirmedTracker.Save(); err != nil {
|
|
log.Printf("[ERROR] [agent] [confirmed] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
// Report circuit breaker health
|
|
go reportCircuitBreakerHealth(ctx)
|
|
|
|
// Report buffered events [TD-003]
|
|
go reportBufferedEvents(ctx)
|
|
|
|
// Process commands — record receipt BEFORE verification/dispatch so the server
|
|
// stops re-issuing immediately, even for commands that turn out malformed.
|
|
if len(response.Commands) > 0 {
|
|
log.Printf("Received %d command(s)", len(response.Commands))
|
|
for _, cmd := range response.Commands {
|
|
ctx.ReceiptTracker.Add(cmd.ID)
|
|
}
|
|
if err := ctx.ReceiptTracker.Save(); err != nil {
|
|
log.Printf("[ERROR] [agent] [receipt] save_after_receive_failed error=%v", err)
|
|
}
|
|
processCommands(ctx, response.Commands)
|
|
}
|
|
|
|
// Supply Chain Gate — pull and execute any signed capability tokens the
|
|
// server minted for this host. Independent of the command path: tokens
|
|
// authorize package operations directly, verified by the privileged
|
|
// executor. A gate that is not enabled server-side returns no tokens.
|
|
processCapabilityTokens(ctx)
|
|
|
|
// Recalculate polling interval AFTER processing commands — a freshly
|
|
// enabled heartbeat takes effect this cycle, not next time.
|
|
pollingInterval := time.Duration(ctx.Cfg.CheckInInterval) * time.Second
|
|
if ctx.Cfg.RapidPollingEnabled && time.Now().Before(ctx.Cfg.RapidPollingUntil) {
|
|
pollingInterval = 5 * time.Second
|
|
}
|
|
|
|
// Sleep until next poll (or stop signal)
|
|
if ctx.StopCh != nil {
|
|
select {
|
|
case <-ctx.StopCh:
|
|
log.Printf("[INFO] [agent] [loop] stop_signal_received")
|
|
return nil
|
|
case <-time.After(pollingInterval):
|
|
}
|
|
} else {
|
|
time.Sleep(pollingInterval)
|
|
}
|
|
}
|
|
}
|
|
|
|
// processCapabilityTokens fetches and processes this host's capability tokens.
|
|
// Best-effort per poll: errors are logged and the loop continues. The executor
|
|
// binary path is taken from REDFLAG_HELPER_BIN, defaulting inside the consumer.
|
|
func processCapabilityTokens(ctx *LoopContext) {
|
|
tokens, err := ctx.APIClient.GetCapabilityTokens(ctx.Cfg.AgentID)
|
|
if err != nil {
|
|
log.Printf("[WARNING] [agent] [supplychain] token_fetch_failed error=%v", err)
|
|
recordLocalCapabilityTokenFetch(ctx.Cfg, 0, err)
|
|
return
|
|
}
|
|
recordLocalCapabilityTokenFetch(ctx.Cfg, len(tokens), nil)
|
|
if len(tokens) == 0 {
|
|
return
|
|
}
|
|
|
|
log.Printf("[INFO] [agent] [supplychain] tokens_received count=%d", len(tokens))
|
|
executor := supplychain.NewExecutor(os.Getenv("REDFLAG_HELPER_BIN"))
|
|
consumer := supplychain.NewConsumer(ctx.Cfg.AgentID, executor, ctx.APIClient)
|
|
summary := consumer.ProcessTokensWithSummary(ctx.Ctx, tokens)
|
|
recordLocalCapabilityTokenProcess(ctx.Cfg, summary.Processed, summary.Failed)
|
|
}
|
|
|
|
// collectMetrics collects system metrics for the check-in
|
|
func collectMetrics(cfg *config.Config) *client.SystemMetrics {
|
|
sysMetrics, err := system.GetLightweightMetrics()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
metrics := &client.SystemMetrics{
|
|
CPUPercent: sysMetrics.CPUPercent,
|
|
MemoryPercent: sysMetrics.MemoryPercent,
|
|
MemoryUsedGB: sysMetrics.MemoryUsedGB,
|
|
MemoryTotalGB: sysMetrics.MemoryTotalGB,
|
|
DiskUsedGB: sysMetrics.DiskUsedGB,
|
|
DiskTotalGB: sysMetrics.DiskTotalGB,
|
|
DiskPercent: sysMetrics.DiskPercent,
|
|
Uptime: sysMetrics.Uptime,
|
|
Version: version.Version,
|
|
// ARC-001: re-advertise scanner capabilities every check-in so the
|
|
// server can pick up scanners installed/removed after registration.
|
|
// Stateless detection from the scanner package — does NOT re-run
|
|
// registration (that's a one-time TOFU flow).
|
|
AvailableScanners: scanner.DetectAvailable(),
|
|
}
|
|
|
|
if cfg.RapidPollingEnabled && time.Now().Before(cfg.RapidPollingUntil) {
|
|
metrics.Metadata = map[string]interface{}{
|
|
"rapid_polling_enabled": true,
|
|
"rapid_polling_until": cfg.RapidPollingUntil.Format(time.RFC3339),
|
|
"rapid_polling_duration_minutes": int(time.Until(cfg.RapidPollingUntil).Minutes()),
|
|
}
|
|
}
|
|
|
|
return metrics
|
|
}
|
|
|
|
// reportCircuitBreakerHealth reports circuit breaker status to server
|
|
func reportCircuitBreakerHealth(ctx *LoopContext) {
|
|
cbReport := client.CircuitBreakerReport{
|
|
Timestamp: time.Now().UTC(),
|
|
}
|
|
|
|
for name, cb := range ctx.CircuitBreakers {
|
|
stats := cb.GetStats()
|
|
cbReport.Subsystems = append(cbReport.Subsystems, client.CircuitBreakerStatus{
|
|
Name: name,
|
|
State: cb.State().String(),
|
|
RecentFailures: stats.RecentFailures,
|
|
ConsecutiveSuccess: stats.ConsecutiveSuccess,
|
|
})
|
|
}
|
|
|
|
if err := ctx.APIClient.ReportCircuitBreakerStats(ctx.Cfg.AgentID, cbReport); err != nil {
|
|
log.Printf("[WARNING] Failed to report circuit breaker stats: %v", err)
|
|
}
|
|
}
|
|
|
|
// reportBufferedEvents sends buffered events to the server [TD-003]
|
|
func reportBufferedEvents(ctx *LoopContext) {
|
|
events, err := ctx.APIClient.GetBufferedEvents()
|
|
if err != nil {
|
|
log.Printf("[WARNING] Failed to get buffered events: %v", err)
|
|
return
|
|
}
|
|
|
|
if len(events) == 0 {
|
|
return // No events to report
|
|
}
|
|
|
|
accepted, rejected, err := ctx.APIClient.ReportEvents(ctx.Cfg.AgentID, events)
|
|
if err != nil {
|
|
log.Printf("[WARNING] Failed to report %d buffered events: %v", len(events), err)
|
|
return
|
|
}
|
|
|
|
if rejected > 0 {
|
|
log.Printf("[WARNING] Server rejected %d/%d events", rejected, len(events))
|
|
}
|
|
|
|
if accepted > 0 {
|
|
log.Printf("[INFO] Successfully reported %d buffered event(s)", accepted)
|
|
}
|
|
}
|
|
|
|
func recordLocalAgentStatus(cfg *config.Config, status string, checkedIn bool) {
|
|
localCache, err := cache.Load()
|
|
if err != nil {
|
|
log.Printf("[WARNING] [agent] [local_state] load_failed error=%v", err)
|
|
localCache = &cache.LocalCache{}
|
|
}
|
|
if cfg != nil && cfg.IsRegistered() {
|
|
localCache.SetAgentInfo(cfg.AgentID, cfg.ServerURL)
|
|
}
|
|
localCache.SetAgentStatus(status)
|
|
if checkedIn {
|
|
localCache.UpdateCheckIn()
|
|
}
|
|
if err := localCache.Save(); err != nil {
|
|
log.Printf("[WARNING] [agent] [local_state] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
func recordLocalCapabilityTokenFetch(cfg *config.Config, fetched int, fetchErr error) {
|
|
localCache, err := cache.Load()
|
|
if err != nil {
|
|
log.Printf("[WARNING] [agent] [local_state] load_failed error=%v", err)
|
|
localCache = &cache.LocalCache{}
|
|
}
|
|
if cfg != nil && cfg.IsRegistered() {
|
|
localCache.SetAgentInfo(cfg.AgentID, cfg.ServerURL)
|
|
}
|
|
localCache.RecordCapabilityTokenFetch(fetched, fetchErr)
|
|
if err := localCache.Save(); err != nil {
|
|
log.Printf("[WARNING] [agent] [local_state] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
func recordLocalCapabilityTokenProcess(cfg *config.Config, processed, failed int) {
|
|
localCache, err := cache.Load()
|
|
if err != nil {
|
|
log.Printf("[WARNING] [agent] [local_state] load_failed error=%v", err)
|
|
localCache = &cache.LocalCache{}
|
|
}
|
|
if cfg != nil && cfg.IsRegistered() {
|
|
localCache.SetAgentInfo(cfg.AgentID, cfg.ServerURL)
|
|
}
|
|
localCache.RecordCapabilityTokenProcess(processed, failed)
|
|
if err := localCache.Save(); err != nil {
|
|
log.Printf("[WARNING] [agent] [local_state] save_failed error=%v", err)
|
|
}
|
|
}
|
|
|
|
// processCommands processes commands from the server
|
|
func processCommands(ctx *LoopContext, commands []client.Command) {
|
|
for _, cmd := range commands {
|
|
log.Printf("Processing command: %s (%s)", cmd.Type, cmd.ID)
|
|
|
|
// Panic recovery for individual commands
|
|
func() {
|
|
defer recovery.RecoverWithCallback(fmt.Sprintf("command_%s", cmd.Type), func(err interface{}, stack []byte) {
|
|
logReport := client.LogReport{
|
|
CommandID: cmd.ID,
|
|
Action: cmd.Type,
|
|
Result: "panic",
|
|
Stderr: fmt.Sprintf("Command panic: %v\nStack: %s", err, string(stack)),
|
|
ExitCode: -1,
|
|
}
|
|
ctx.APIClient.ReportLog(ctx.Cfg.AgentID, logReport)
|
|
ctx.AckTracker.Add(cmd.ID)
|
|
})
|
|
|
|
// Verify command signature
|
|
if err := ctx.CommandHandler.ProcessCommand(cmd, ctx.Cfg, ctx.Cfg.AgentID); err != nil {
|
|
log.Printf("[ERROR] Command rejected: %s", err)
|
|
logReport := client.LogReport{
|
|
CommandID: cmd.ID,
|
|
Action: "verify_command",
|
|
Result: "failed",
|
|
Stderr: fmt.Sprintf("Command verification failed: %s", err),
|
|
ExitCode: 1,
|
|
}
|
|
ctx.APIClient.ReportLog(ctx.Cfg.AgentID, logReport)
|
|
ctx.AckTracker.Add(cmd.ID)
|
|
return
|
|
}
|
|
|
|
// Dispatch cross-platform commands (scans + update_agent) via the
|
|
// shared dispatcher so the cross-platform agent loop and the
|
|
// Windows service path stay aligned.
|
|
if !handlers.DispatchCrossPlatformCommand(ctx.APIClient, ctx.Cfg, ctx.AckTracker, ctx.ScanOrchestrator, cmd) {
|
|
log.Printf("Command type %s has no registered handler", cmd.Type)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// Polling resilience defaults. Applied when the config carries no override
|
|
// (zero value), so older config files and unset server policy keep working.
|
|
const (
|
|
defaultJitterMaxSeconds = 30
|
|
defaultBackoffBaseSeconds = 10
|
|
defaultBackoffMaxSeconds = 300
|
|
)
|
|
|
|
// resolveJitterMaxSeconds returns the operator-configured jitter cap, or the
|
|
// built-in default when unset.
|
|
func resolveJitterMaxSeconds(cfg *config.Config) int {
|
|
if cfg != nil && cfg.Polling.JitterMaxSeconds > 0 {
|
|
return cfg.Polling.JitterMaxSeconds
|
|
}
|
|
return defaultJitterMaxSeconds
|
|
}
|
|
|
|
func resolveBackoffBase(cfg *config.Config) time.Duration {
|
|
if cfg != nil && cfg.Polling.BackoffBaseSeconds > 0 {
|
|
return time.Duration(cfg.Polling.BackoffBaseSeconds) * time.Second
|
|
}
|
|
return defaultBackoffBaseSeconds * time.Second
|
|
}
|
|
|
|
func resolveBackoffMax(cfg *config.Config) time.Duration {
|
|
if cfg != nil && cfg.Polling.BackoffMaxSeconds > 0 {
|
|
return time.Duration(cfg.Polling.BackoffMaxSeconds) * time.Second
|
|
}
|
|
return defaultBackoffMaxSeconds * time.Second
|
|
}
|
|
|
|
// applyServerPolling merges server-delivered polling tuning into cfg.Polling.
|
|
// It returns true if any value changed. Zero/absent server values are ignored
|
|
// so the agent keeps its current (possibly locally-configured) tuning rather
|
|
// than being reset to zero. The polling loop reads cfg.Polling through the
|
|
// resolve* helpers on every iteration, so a merged change takes effect next
|
|
// loop without a restart.
|
|
func applyServerPolling(cfg *config.Config, resp *client.AgentConfigResponse) bool {
|
|
if cfg == nil || resp == nil || resp.Polling == nil {
|
|
return false
|
|
}
|
|
changed := false
|
|
if v := resp.Polling.JitterMaxSeconds; v > 0 && v != cfg.Polling.JitterMaxSeconds {
|
|
cfg.Polling.JitterMaxSeconds = v
|
|
changed = true
|
|
}
|
|
if v := resp.Polling.BackoffBaseSeconds; v > 0 && v != cfg.Polling.BackoffBaseSeconds {
|
|
cfg.Polling.BackoffBaseSeconds = v
|
|
changed = true
|
|
}
|
|
if v := resp.Polling.BackoffMaxSeconds; v > 0 && v != cfg.Polling.BackoffMaxSeconds {
|
|
cfg.Polling.BackoffMaxSeconds = v
|
|
changed = true
|
|
}
|
|
return changed
|
|
}
|
|
|
|
// calculateBackoff returns an exponential backoff delay with full jitter,
|
|
// bounded by the admin-adjustable base (floor) and maxDelay (ceiling).
|
|
func calculateBackoff(attempt int, base, maxDelay time.Duration) time.Duration {
|
|
ceiling := base * time.Duration(1<<uint(attempt))
|
|
if ceiling > maxDelay || ceiling <= 0 {
|
|
ceiling = maxDelay
|
|
}
|
|
|
|
delay := time.Duration(rand.Int63n(int64(ceiling)))
|
|
if delay < base {
|
|
delay = base
|
|
}
|
|
return delay
|
|
}
|
|
|
|
// reportSystemInfo reports detailed system information to server
|
|
func reportSystemInfo(apiClient *client.Client, cfg *config.Config) error {
|
|
sysInfo, err := system.GetSystemInfo(version.Version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
report := client.SystemInfoReport{
|
|
Timestamp: time.Now().UTC(),
|
|
CPUModel: sysInfo.CPUInfo.ModelName,
|
|
CPUCores: sysInfo.CPUInfo.Cores,
|
|
CPUThreads: sysInfo.CPUInfo.Threads,
|
|
MemoryTotal: uint64(sysInfo.MemoryInfo.Total),
|
|
IPAddress: sysInfo.IPAddress,
|
|
Processes: sysInfo.RunningProcesses,
|
|
Uptime: sysInfo.Uptime,
|
|
}
|
|
|
|
if len(sysInfo.DiskInfo) > 0 {
|
|
report.DiskTotal = uint64(sysInfo.DiskInfo[0].Total)
|
|
report.DiskUsed = uint64(sysInfo.DiskInfo[0].Used)
|
|
}
|
|
|
|
// Fold in any locally-observed integrations (Sunshine, etc.). Observe-only:
|
|
// the server merges this under agent.metadata["integrations"], which the
|
|
// dashboard renders. Empty when nothing is detected.
|
|
if detected := integrations.Detect(sysInfo.IPAddress); len(detected) > 0 {
|
|
report.Metadata = map[string]interface{}{"integrations": detected}
|
|
}
|
|
|
|
return apiClient.ReportSystemInfo(cfg.AgentID, report)
|
|
}
|