projection: begin the exact-path public epoch
The public tree and its history contain only the listed paths. Earlier projection history remains preserved internally. Source-Sha: 913fde029b935671833254797f0f20f1eb9fabba Policy-Sha: 913fde029b935671833254797f0f20f1eb9fabba Tree-Digest: 180ae530c1058a2a5c89837bdce2d323ae83e669e38590ca72e75b8d92b7262f
This commit is contained in:
commit
67e26be2d9
838 changed files with 157302 additions and 0 deletions
200
server/internal/circuitbreaker/circuitbreaker.go
Normal file
200
server/internal/circuitbreaker/circuitbreaker.go
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
// Package circuitbreaker is the server-side circuit breaker (SCALE-001 S8).
|
||||
// It mirrors the agent's breaker (agent/internal/circuitbreaker) — same states,
|
||||
// config, and semantics — so the two halves of RedFlag behave identically. The
|
||||
// server uses it to wrap fragile outbound dependencies (OSV.dev, repology.org):
|
||||
// when an upstream is failing or slow, the breaker opens and calls fail fast
|
||||
// instead of piling goroutines and timing out one by one. Callers decide the
|
||||
// fail direction; the supply-chain path fails OPEN (sovereignty) on an open
|
||||
// breaker, exactly as it does on a transport error.
|
||||
package circuitbreaker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// State represents the circuit breaker state.
|
||||
type State int
|
||||
|
||||
const (
|
||||
StateClosed State = iota // Normal operation
|
||||
StateOpen // Failing fast
|
||||
StateHalfOpen // Probing recovery
|
||||
)
|
||||
|
||||
func (s State) String() string {
|
||||
switch s {
|
||||
case StateClosed:
|
||||
return "closed"
|
||||
case StateOpen:
|
||||
return "open"
|
||||
case StateHalfOpen:
|
||||
return "half-open"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// Config holds circuit breaker configuration.
|
||||
type Config struct {
|
||||
FailureThreshold int // Failures within the window before opening
|
||||
FailureWindow time.Duration // Window over which failures are counted
|
||||
OpenDuration time.Duration // How long the circuit stays open before probing
|
||||
HalfOpenAttempts int // Consecutive successes needed to close from half-open
|
||||
}
|
||||
|
||||
// CircuitBreaker implements the breaker pattern for a single dependency.
|
||||
type CircuitBreaker struct {
|
||||
name string
|
||||
config Config
|
||||
|
||||
mu sync.RWMutex
|
||||
state State
|
||||
failures []time.Time
|
||||
consecutiveSuccess int
|
||||
openedAt time.Time
|
||||
}
|
||||
|
||||
// New creates a closed circuit breaker.
|
||||
func New(name string, config Config) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
name: name,
|
||||
config: config,
|
||||
state: StateClosed,
|
||||
failures: make([]time.Time, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Call runs fn under breaker protection. When the breaker is open it returns the
|
||||
// breaker error without invoking fn.
|
||||
func (cb *CircuitBreaker) Call(fn func() error) error {
|
||||
if err := cb.beforeCall(); err != nil {
|
||||
return err
|
||||
}
|
||||
err := fn()
|
||||
cb.afterCall(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) beforeCall() error {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
switch cb.state {
|
||||
case StateClosed:
|
||||
return nil
|
||||
case StateOpen:
|
||||
if time.Since(cb.openedAt) >= cb.config.OpenDuration {
|
||||
cb.state = StateHalfOpen
|
||||
cb.consecutiveSuccess = 0
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("circuit breaker [%s] is OPEN (will retry at %s)",
|
||||
cb.name, cb.openedAt.Add(cb.config.OpenDuration).Format("15:04:05"))
|
||||
case StateHalfOpen:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unknown circuit breaker state")
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) afterCall(err error) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
if err != nil {
|
||||
cb.recordFailure(now)
|
||||
if cb.state == StateHalfOpen {
|
||||
cb.state = StateOpen
|
||||
cb.openedAt = now
|
||||
cb.consecutiveSuccess = 0
|
||||
return
|
||||
}
|
||||
if cb.shouldOpen(now) {
|
||||
cb.state = StateOpen
|
||||
cb.openedAt = now
|
||||
cb.consecutiveSuccess = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
switch cb.state {
|
||||
case StateHalfOpen:
|
||||
cb.consecutiveSuccess++
|
||||
if cb.consecutiveSuccess >= cb.config.HalfOpenAttempts {
|
||||
cb.state = StateClosed
|
||||
cb.failures = make([]time.Time, 0)
|
||||
cb.consecutiveSuccess = 0
|
||||
}
|
||||
case StateClosed:
|
||||
cb.cleanupOldFailures(now)
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) recordFailure(now time.Time) {
|
||||
cb.failures = append(cb.failures, now)
|
||||
cb.cleanupOldFailures(now)
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) cleanupOldFailures(now time.Time) {
|
||||
cutoff := now.Add(-cb.config.FailureWindow)
|
||||
valid := make([]time.Time, 0, len(cb.failures))
|
||||
for _, t := range cb.failures {
|
||||
if t.After(cutoff) {
|
||||
valid = append(valid, t)
|
||||
}
|
||||
}
|
||||
cb.failures = valid
|
||||
}
|
||||
|
||||
func (cb *CircuitBreaker) shouldOpen(now time.Time) bool {
|
||||
cb.cleanupOldFailures(now)
|
||||
return len(cb.failures) >= cb.config.FailureThreshold
|
||||
}
|
||||
|
||||
// State returns the current state (thread-safe).
|
||||
func (cb *CircuitBreaker) State() State {
|
||||
cb.mu.RLock()
|
||||
defer cb.mu.RUnlock()
|
||||
return cb.state
|
||||
}
|
||||
|
||||
// Stats is a point-in-time view for health/metrics (OBS-001).
|
||||
type Stats struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
RecentFailures int `json:"recent_failures"`
|
||||
ConsecutiveSuccess int `json:"consecutive_success"`
|
||||
NextAttempt *time.Time `json:"next_attempt,omitempty"`
|
||||
}
|
||||
|
||||
// GetStats returns the current breaker statistics (thread-safe).
|
||||
func (cb *CircuitBreaker) GetStats() Stats {
|
||||
cb.mu.RLock()
|
||||
defer cb.mu.RUnlock()
|
||||
|
||||
stats := Stats{
|
||||
Name: cb.name,
|
||||
State: cb.state.String(),
|
||||
RecentFailures: len(cb.failures),
|
||||
ConsecutiveSuccess: cb.consecutiveSuccess,
|
||||
}
|
||||
if cb.state == StateOpen && !cb.openedAt.IsZero() {
|
||||
next := cb.openedAt.Add(cb.config.OpenDuration)
|
||||
stats.NextAttempt = &next
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
// Reset forces the breaker back to closed (manual recovery / tests).
|
||||
func (cb *CircuitBreaker) Reset() {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
cb.state = StateClosed
|
||||
cb.failures = make([]time.Time, 0)
|
||||
cb.consecutiveSuccess = 0
|
||||
cb.openedAt = time.Time{}
|
||||
}
|
||||
91
server/internal/circuitbreaker/circuitbreaker_test.go
Normal file
91
server/internal/circuitbreaker/circuitbreaker_test.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package circuitbreaker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testConfig() Config {
|
||||
return Config{
|
||||
FailureThreshold: 3,
|
||||
FailureWindow: time.Second,
|
||||
OpenDuration: 50 * time.Millisecond,
|
||||
HalfOpenAttempts: 2,
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedStaysClosedOnSuccess(t *testing.T) {
|
||||
cb := New("t", testConfig())
|
||||
for i := 0; i < 10; i++ {
|
||||
if err := cb.Call(func() error { return nil }); err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
}
|
||||
if cb.State() != StateClosed {
|
||||
t.Fatalf("state=%s, want closed", cb.State())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpensAfterThresholdAndFailsFast(t *testing.T) {
|
||||
cb := New("t", testConfig())
|
||||
boom := errors.New("boom")
|
||||
for i := 0; i < 3; i++ {
|
||||
_ = cb.Call(func() error { return boom })
|
||||
}
|
||||
if cb.State() != StateOpen {
|
||||
t.Fatalf("state=%s, want open after %d failures", cb.State(), 3)
|
||||
}
|
||||
|
||||
// While open, fn must not run and Call returns the breaker error.
|
||||
ran := false
|
||||
err := cb.Call(func() error { ran = true; return nil })
|
||||
if ran {
|
||||
t.Error("fn ran while breaker open — should fail fast")
|
||||
}
|
||||
if err == nil {
|
||||
t.Error("expected breaker error while open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHalfOpenRecoversToClosed(t *testing.T) {
|
||||
cb := New("t", testConfig())
|
||||
boom := errors.New("boom")
|
||||
for i := 0; i < 3; i++ {
|
||||
_ = cb.Call(func() error { return boom })
|
||||
}
|
||||
if cb.State() != StateOpen {
|
||||
t.Fatalf("precondition: want open, got %s", cb.State())
|
||||
}
|
||||
|
||||
time.Sleep(60 * time.Millisecond) // past OpenDuration → next call probes (half-open)
|
||||
|
||||
// HalfOpenAttempts=2 consecutive successes needed to close.
|
||||
if err := cb.Call(func() error { return nil }); err != nil {
|
||||
t.Fatalf("probe call err: %v", err)
|
||||
}
|
||||
if err := cb.Call(func() error { return nil }); err != nil {
|
||||
t.Fatalf("second probe err: %v", err)
|
||||
}
|
||||
if cb.State() != StateClosed {
|
||||
t.Fatalf("state=%s, want closed after recovery", cb.State())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHalfOpenReopensOnFailure(t *testing.T) {
|
||||
cb := New("t", testConfig())
|
||||
boom := errors.New("boom")
|
||||
for i := 0; i < 3; i++ {
|
||||
_ = cb.Call(func() error { return boom })
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond) // → half-open on next call
|
||||
|
||||
_ = cb.Call(func() error { return boom }) // a failure in half-open reopens
|
||||
if cb.State() != StateOpen {
|
||||
t.Fatalf("state=%s, want open (half-open failure must reopen)", cb.State())
|
||||
}
|
||||
st := cb.GetStats()
|
||||
if st.State != "open" || st.NextAttempt == nil {
|
||||
t.Errorf("stats=%+v, want open with NextAttempt set", st)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue