Watch
1
0
Fork
You've already forked RedFlag
0
RedFlag/server/internal/circuitbreaker/circuitbreaker.go
Fimeg e99ad6e8c7 gave the server a breaker so a sulking upstream can't drag it down
osv.dev or repology going dark used to mean every check sat there burning its
30s timeout, one after another. now there's a breaker (ported the agent's, it's
already proven) wrapping both:

- osv: one breaker over the batch + single-query paths. trips after 5 fails in
  a minute, fails open while tripped — an unreachable advisory feed never blocks
  a patch. that's the whole sovereignty bet
- repology: same deal, best-effort, 404 doesn't count against it
- both visible at /health/tasks so you can see them trip and heal

db-pool shedding (503 + retry-after) is the other half — left it for later, the
pool bump + bounded background already took most of that pressure off.

race detector's clean.
2026-06-07 19:45:36 -04:00

200 lines
5.1 KiB
Go

// Package circuitbreaker is the server-side circuit breaker (SCALE-001 S8).
// It mirrors the agent's breaker (agent/internal/circuitbreaker) — same states,
// config, and semantics — so the two halves of RedFlag behave identically. The
// server uses it to wrap fragile outbound dependencies (OSV.dev, repology.org):
// when an upstream is failing or slow, the breaker opens and calls fail fast
// instead of piling goroutines and timing out one by one. Callers decide the
// fail direction; the supply-chain path fails OPEN (sovereignty) on an open
// breaker, exactly as it does on a transport error.
package circuitbreaker
import (
"fmt"
"sync"
"time"
)
// State represents the circuit breaker state.
type State int
const (
StateClosed State = iota // Normal operation
StateOpen // Failing fast
StateHalfOpen // Probing recovery
)
func (s State) String() string {
switch s {
case StateClosed:
return "closed"
case StateOpen:
return "open"
case StateHalfOpen:
return "half-open"
default:
return "unknown"
}
}
// Config holds circuit breaker configuration.
type Config struct {
FailureThreshold int // Failures within the window before opening
FailureWindow time.Duration // Window over which failures are counted
OpenDuration time.Duration // How long the circuit stays open before probing
HalfOpenAttempts int // Consecutive successes needed to close from half-open
}
// CircuitBreaker implements the breaker pattern for a single dependency.
type CircuitBreaker struct {
name string
config Config
mu sync.RWMutex
state State
failures []time.Time
consecutiveSuccess int
openedAt time.Time
}
// New creates a closed circuit breaker.
func New(name string, config Config) *CircuitBreaker {
return &CircuitBreaker{
name: name,
config: config,
state: StateClosed,
failures: make([]time.Time, 0),
}
}
// Call runs fn under breaker protection. When the breaker is open it returns the
// breaker error without invoking fn.
func (cb *CircuitBreaker) Call(fn func() error) error {
if err := cb.beforeCall(); err != nil {
return err
}
err := fn()
cb.afterCall(err)
return err
}
func (cb *CircuitBreaker) beforeCall() error {
cb.mu.Lock()
defer cb.mu.Unlock()
switch cb.state {
case StateClosed:
return nil
case StateOpen:
if time.Since(cb.openedAt) >= cb.config.OpenDuration {
cb.state = StateHalfOpen
cb.consecutiveSuccess = 0
return nil
}
return fmt.Errorf("circuit breaker [%s] is OPEN (will retry at %s)",
cb.name, cb.openedAt.Add(cb.config.OpenDuration).Format("15:04:05"))
case StateHalfOpen:
return nil
default:
return fmt.Errorf("unknown circuit breaker state")
}
}
func (cb *CircuitBreaker) afterCall(err error) {
cb.mu.Lock()
defer cb.mu.Unlock()
now := time.Now()
if err != nil {
cb.recordFailure(now)
if cb.state == StateHalfOpen {
cb.state = StateOpen
cb.openedAt = now
cb.consecutiveSuccess = 0
return
}
if cb.shouldOpen(now) {
cb.state = StateOpen
cb.openedAt = now
cb.consecutiveSuccess = 0
}
return
}
switch cb.state {
case StateHalfOpen:
cb.consecutiveSuccess++
if cb.consecutiveSuccess >= cb.config.HalfOpenAttempts {
cb.state = StateClosed
cb.failures = make([]time.Time, 0)
cb.consecutiveSuccess = 0
}
case StateClosed:
cb.cleanupOldFailures(now)
}
}
func (cb *CircuitBreaker) recordFailure(now time.Time) {
cb.failures = append(cb.failures, now)
cb.cleanupOldFailures(now)
}
func (cb *CircuitBreaker) cleanupOldFailures(now time.Time) {
cutoff := now.Add(-cb.config.FailureWindow)
valid := make([]time.Time, 0, len(cb.failures))
for _, t := range cb.failures {
if t.After(cutoff) {
valid = append(valid, t)
}
}
cb.failures = valid
}
func (cb *CircuitBreaker) shouldOpen(now time.Time) bool {
cb.cleanupOldFailures(now)
return len(cb.failures) >= cb.config.FailureThreshold
}
// State returns the current state (thread-safe).
func (cb *CircuitBreaker) State() State {
cb.mu.RLock()
defer cb.mu.RUnlock()
return cb.state
}
// Stats is a point-in-time view for health/metrics (OBS-001).
type Stats struct {
Name string `json:"name"`
State string `json:"state"`
RecentFailures int `json:"recent_failures"`
ConsecutiveSuccess int `json:"consecutive_success"`
NextAttempt *time.Time `json:"next_attempt,omitempty"`
}
// GetStats returns the current breaker statistics (thread-safe).
func (cb *CircuitBreaker) GetStats() Stats {
cb.mu.RLock()
defer cb.mu.RUnlock()
stats := Stats{
Name: cb.name,
State: cb.state.String(),
RecentFailures: len(cb.failures),
ConsecutiveSuccess: cb.consecutiveSuccess,
}
if cb.state == StateOpen && !cb.openedAt.IsZero() {
next := cb.openedAt.Add(cb.config.OpenDuration)
stats.NextAttempt = &next
}
return stats
}
// Reset forces the breaker back to closed (manual recovery / tests).
func (cb *CircuitBreaker) Reset() {
cb.mu.Lock()
defer cb.mu.Unlock()
cb.state = StateClosed
cb.failures = make([]time.Time, 0)
cb.consecutiveSuccess = 0
cb.openedAt = time.Time{}
}