refactor: unified backoff policy with failure classes (BUG-014)
classifyFailure is now the single source of truth for which polling failures are terminal (ErrRefreshTokenInvalid, ErrMachineMismatch — wrapped or bare) vs transient. delayForFailure picks the curve: flat 10-minute delay for terminal states awaiting operator intervention, jittered exponential (calculateBackoff) for everything else. The terminalBackoff bool is gone from the polling loop. Task file said to delete itself when this landed — done.
This commit is contained in:
parent
1c4b363375
commit
6ac937bfe5
2 changed files with 96 additions and 7 deletions
54
agent/internal/agent/backoff_test.go
Normal file
54
agent/internal/agent/backoff_test.go
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Fimeg/RedFlag/agent/internal/client"
|
||||
)
|
||||
|
||||
// TestClassifyFailure locks in the BUG-014 policy: dead credentials and
|
||||
// machine-binding mismatches are terminal; everything else is transient.
|
||||
func TestClassifyFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want failureClass
|
||||
}{
|
||||
{"machine mismatch", client.ErrMachineMismatch, failureTerminal},
|
||||
{"refresh token invalid", client.ErrRefreshTokenInvalid, failureTerminal},
|
||||
{"wrapped machine mismatch", fmt.Errorf("get commands: %w", client.ErrMachineMismatch), failureTerminal},
|
||||
{"wrapped refresh invalid", fmt.Errorf("renew: %w", client.ErrRefreshTokenInvalid), failureTerminal},
|
||||
{"unauthorized alone is not terminal (renewal may fix it)", client.ErrUnauthorized, failureTransient},
|
||||
{"plain network error", errors.New("dial tcp: connection refused"), failureTransient},
|
||||
{"nil-adjacent generic error", errors.New("502 bad gateway"), failureTransient},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := classifyFailure(tc.err); got != tc.want {
|
||||
t.Errorf("%s: classifyFailure() = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelayForFailure verifies the policy curves: terminal is a long flat
|
||||
// delay independent of attempt count; transient follows the jittered
|
||||
// exponential bounded by base and max.
|
||||
func TestDelayForFailure(t *testing.T) {
|
||||
base := 5 * time.Second
|
||||
max := 5 * time.Minute
|
||||
|
||||
for _, attempt := range []int{1, 3, 50} {
|
||||
if got := delayForFailure(failureTerminal, attempt, base, max); got != terminalRetryDelay {
|
||||
t.Errorf("terminal attempt %d: delay = %s, want flat %s", attempt, got, terminalRetryDelay)
|
||||
}
|
||||
}
|
||||
|
||||
for attempt := 1; attempt <= 30; attempt++ {
|
||||
got := delayForFailure(failureTransient, attempt, base, max)
|
||||
if got < base || got > max {
|
||||
t.Errorf("transient attempt %d: delay %s outside [%s, %s]", attempt, got, base, max)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -353,7 +353,7 @@ func RunPollingLoop(loopCtx *LoopContext) error {
|
|||
// Get commands from server
|
||||
response, err := ctx.APIClient.GetCommands(ctx.Cfg.AgentID, metrics)
|
||||
if err != nil {
|
||||
terminalBackoff := false
|
||||
class := classifyFailure(err)
|
||||
if errors.Is(err, client.ErrMachineMismatch) {
|
||||
// Terminal: the server no longer recognizes this host as the one the
|
||||
// agent registered on — config moved or copied. Renewal can't fix it
|
||||
|
|
@ -362,7 +362,6 @@ func RunPollingLoop(loopCtx *LoopContext) error {
|
|||
// We keep polling rather than exit, so the agent stays visible and
|
||||
// self-heals the moment an operator rebinds it server-side.
|
||||
log.Printf("[ERROR] [agent] [auth] machine_id_mismatch identity_moved_or_copied re_registration_required agent_id=%s", ctx.Cfg.AgentID)
|
||||
terminalBackoff = true
|
||||
} else if errors.Is(err, client.ErrUnauthorized) && ctx.Cfg.RefreshToken != "" {
|
||||
log.Printf("[INFO] [agent] [auth] jwt_expired attempting_renewal agent_id=%s", ctx.Cfg.AgentID)
|
||||
renewErr := ctx.APIClient.RenewToken(ctx.Cfg.AgentID, ctx.Cfg.RefreshToken, version.Version)
|
||||
|
|
@ -387,7 +386,7 @@ func RunPollingLoop(loopCtx *LoopContext) error {
|
|||
// the agent needs re-registration. Surface it loudly; the client
|
||||
// already buffered a critical refresh_token_invalid event.
|
||||
log.Printf("[ERROR] [agent] [auth] refresh_token_invalid re_registration_required agent_id=%s error=%v", ctx.Cfg.AgentID, renewErr)
|
||||
terminalBackoff = true
|
||||
class = classifyFailure(renewErr)
|
||||
default:
|
||||
// Transient renewal failure (network, 502). Fall through to backoff and retry.
|
||||
log.Printf("[ERROR] [agent] [auth] token_renewal_failed error=%v", renewErr)
|
||||
|
|
@ -395,12 +394,10 @@ func RunPollingLoop(loopCtx *LoopContext) error {
|
|||
}
|
||||
consecutiveFailures++
|
||||
recordLocalAgentStatus(ctx.Cfg, "backoff", false)
|
||||
var backoffDelay time.Duration
|
||||
if terminalBackoff {
|
||||
backoffDelay = 10 * time.Minute
|
||||
backoffDelay := delayForFailure(class, consecutiveFailures, resolveBackoffBase(ctx.Cfg), resolveBackoffMax(ctx.Cfg))
|
||||
if class == failureTerminal {
|
||||
ctx.TeeLogger.Error("agent", "auth", "auth", fmt.Sprintf("terminal_state waiting_for_operator_intervention delay=%s agent_id=%s", backoffDelay, ctx.Cfg.AgentID), map[string]interface{}{"delay": backoffDelay.String(), "agent_id": ctx.Cfg.AgentID.String()})
|
||||
} else {
|
||||
backoffDelay = calculateBackoff(consecutiveFailures, resolveBackoffBase(ctx.Cfg), resolveBackoffMax(ctx.Cfg))
|
||||
ctx.TeeLogger.Warning("agent", "loop", "agent_loop", fmt.Sprintf("server_unavailable attempt=%d retrying_in=%s error=%v", consecutiveFailures, backoffDelay, err), map[string]interface{}{"attempt": consecutiveFailures, "retry_delay": backoffDelay.String(), "error": err.Error()})
|
||||
}
|
||||
// Non-blocking stop check during backoff
|
||||
|
|
@ -865,8 +862,46 @@ func applyServerPolling(cfg *config.Config, resp *client.AgentConfigResponse) bo
|
|||
return changed
|
||||
}
|
||||
|
||||
// failureClass partitions polling failures by how they recover (BUG-014).
|
||||
type failureClass int
|
||||
|
||||
const (
|
||||
// failureTransient — network blips, 5xx, DNS, transient renewal failures.
|
||||
// Self-healing: exponential backoff with full jitter.
|
||||
failureTransient failureClass = iota
|
||||
// failureTerminal — dead credentials (refresh-token reuse/revocation) or
|
||||
// machine-binding mismatch. Retrying cannot fix these; an operator must
|
||||
// rebind or re-register. Long flat delay keeps the agent visible without
|
||||
// hammering the server with requests that can only fail.
|
||||
failureTerminal
|
||||
)
|
||||
|
||||
// terminalRetryDelay is the flat poll interval in a terminal credential state —
|
||||
// long enough not to hammer the server, short enough that an operator-side
|
||||
// rebind is picked up within the same working session.
|
||||
const terminalRetryDelay = 10 * time.Minute
|
||||
|
||||
// classifyFailure maps a polling or renewal error to its failure class. This is
|
||||
// the single source of truth for which states are terminal.
|
||||
func classifyFailure(err error) failureClass {
|
||||
if errors.Is(err, client.ErrMachineMismatch) || errors.Is(err, client.ErrRefreshTokenInvalid) {
|
||||
return failureTerminal
|
||||
}
|
||||
return failureTransient
|
||||
}
|
||||
|
||||
// delayForFailure is the backoff policy: it returns the wait before the next
|
||||
// poll attempt for the given failure class.
|
||||
func delayForFailure(class failureClass, attempt int, base, maxDelay time.Duration) time.Duration {
|
||||
if class == failureTerminal {
|
||||
return terminalRetryDelay
|
||||
}
|
||||
return calculateBackoff(attempt, base, maxDelay)
|
||||
}
|
||||
|
||||
// calculateBackoff returns an exponential backoff delay with full jitter,
|
||||
// bounded by the admin-adjustable base (floor) and maxDelay (ceiling).
|
||||
// It is the transient curve behind delayForFailure.
|
||||
func calculateBackoff(attempt int, base, maxDelay time.Duration) time.Duration {
|
||||
ceiling := base * time.Duration(1<<uint(attempt))
|
||||
if ceiling > maxDelay || ceiling <= 0 {
|
||||
|
|
|
|||
Loading…
Reference in a new issue