turn lifecycle + instance fix + dynamic agent identity in Aster
Remove the auto-reset stall guard that silently killed turns after 90s and orphaned the backend. The TUI now trusts the stream: backend emits Keepalive every ~15s between rounds, the phase strip shows a liveness label (waiting Ns... / still waiting Ns...), and the user decides when to Esc. SSE ping frames map to Keepalive on remote backend. Instance registry no longer blanket-registers all agents at startup — register_instance takes an explicit agent_id, called from new_conversation only. Fixes inflated instance counts on manager cards. Consciousness engine (Aster) looks up the primary agent's name dynamically instead of hardcoding "Ani". Parameter renamed to primary_response. Interjection labels changed from /btw to hand-raise metaphor (hand raised → noticed) since /btw is its own fork feature.
This commit is contained in:
parent
b52bfc08e0
commit
4b903df9c3
8 changed files with 88 additions and 93 deletions
|
|
@ -448,6 +448,10 @@ impl Backend for LocalBackend {
|
|||
let _ = self.server.agents.get(agent_id).await?;
|
||||
let conv_id = self.server.sessions.create(agent_id);
|
||||
|
||||
if let Err(e) = self.server.agents.register_instance(agent_id, &self.server.instance_id).await {
|
||||
tracing::warn!(agent = %agent_id, "instance registration failed: {}", e);
|
||||
}
|
||||
|
||||
let memory_root = self.server.agents.memory_root(agent_id);
|
||||
let subconscious_root = self.server.agents.subconscious_memory_root(agent_id);
|
||||
let (bundled, user, agent_memfs, project) =
|
||||
|
|
@ -905,6 +909,8 @@ async fn run_turn(
|
|||
let mut final_content: String = String::new();
|
||||
let mut interrupted = false;
|
||||
let counter = TokenCounter::new();
|
||||
let mut last_keepalive = Instant::now();
|
||||
const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15);
|
||||
|
||||
loop {
|
||||
// Cancellation is a signal, not enforcement — we check it on round
|
||||
|
|
@ -915,6 +921,11 @@ async fn run_turn(
|
|||
break;
|
||||
}
|
||||
|
||||
if last_keepalive.elapsed() >= KEEPALIVE_INTERVAL {
|
||||
let _ = tx.send(Ok(BackendEvent::Keepalive)).await;
|
||||
last_keepalive = Instant::now();
|
||||
}
|
||||
|
||||
// Drain any queued interjections from the user. The user typed these
|
||||
// while the agent was thinking; deliver them as system notes so the
|
||||
// agent reads them in context on this round. She decides whether to
|
||||
|
|
@ -1154,6 +1165,9 @@ async fn run_turn(
|
|||
tokio::time::sleep(effective).await;
|
||||
}
|
||||
|
||||
let _ = tx.send(Ok(BackendEvent::Keepalive)).await;
|
||||
last_keepalive = Instant::now();
|
||||
|
||||
// Continue loop — model will see tool results and respond
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -101,6 +101,10 @@ pub enum BackendEvent {
|
|||
/// Agent changed her outfit. The string is the outfit name (a subdirectory
|
||||
/// under `expressions/`). Empty string clears to default expressions.
|
||||
Outfit(String),
|
||||
/// The backend is alive but producing no content (waiting on provider,
|
||||
/// between tool rounds, processing). The TUI resets `last_event_at`
|
||||
/// on this the same way it does for `Token` — it's a liveness signal.
|
||||
Keepalive,
|
||||
/// Stream ended cleanly.
|
||||
Done,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -271,7 +271,7 @@ fn map_event(event_type: &str, v: &Value) -> Option<BackendEvent> {
|
|||
synthesis: s("synthesis").unwrap_or_default(),
|
||||
pressure: v.get("pressure").and_then(|x| x.as_f64()).unwrap_or(0.0) as f32,
|
||||
},
|
||||
"ping" => return None,
|
||||
"ping" => BackendEvent::Keepalive,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -638,9 +638,9 @@ fn default_pulse_interval() -> u64 { 600 }
|
|||
/// tune the interface layer only.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TuiConfig {
|
||||
/// Seconds without a backend event before the turn is declared stalled and
|
||||
/// reset. Increase if your model needs longer for inference (e.g. after
|
||||
/// reading many large files). Default: 90.
|
||||
/// Deprecated — the auto-reset stall guard has been removed. The TUI
|
||||
/// now trusts the backend stream and shows a liveness label instead.
|
||||
/// Kept for config backwards compatibility.
|
||||
#[serde(default = "default_stale_timeout_secs")]
|
||||
pub stale_timeout_secs: u64,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -112,34 +112,30 @@ impl AgentInventory {
|
|||
crate::core::identity::SeedId::load_or_generate(&self.seed_dir(agent_id))
|
||||
}
|
||||
|
||||
/// Register an instance row for every known agent on this process.
|
||||
/// Idempotent: re-running with the same `instance_id` overwrites the
|
||||
/// last_seen_at. Stale rows (>5 min since last_seen) are pruned first
|
||||
/// so the manager view doesn't surface dead processes.
|
||||
pub async fn register_instance(&self, instance_id: &str) -> anyhow::Result<()> {
|
||||
/// Register an instance row for a specific agent on this process.
|
||||
/// Idempotent: re-running with the same `(agent_id, instance_id)` pair
|
||||
/// bumps `last_seen_at`. Stale rows (>5 min) are pruned first so the
|
||||
/// manager view doesn't surface dead processes.
|
||||
pub async fn register_instance(&self, agent_id: &str, instance_id: &str) -> anyhow::Result<()> {
|
||||
let pid = std::process::id() as i64;
|
||||
let hostname = hostname_or_unknown();
|
||||
// Prune dead instances first.
|
||||
sqlx::query("DELETE FROM agent_instances WHERE last_seen_at < datetime('now', '-5 minutes')")
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
let agents = self.list(None).await?;
|
||||
for a in agents {
|
||||
sqlx::query(
|
||||
"INSERT INTO agent_instances (agent_id, instance_id, pid, hostname, started_at, last_seen_at)
|
||||
VALUES (?1, ?2, ?3, ?4, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(agent_id, instance_id) DO UPDATE SET
|
||||
last_seen_at = CURRENT_TIMESTAMP,
|
||||
pid = excluded.pid,
|
||||
hostname = excluded.hostname"
|
||||
)
|
||||
.bind(&a.id)
|
||||
.bind(instance_id)
|
||||
.bind(pid)
|
||||
.bind(&hostname)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
}
|
||||
sqlx::query(
|
||||
"INSERT INTO agent_instances (agent_id, instance_id, pid, hostname, started_at, last_seen_at)
|
||||
VALUES (?1, ?2, ?3, ?4, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(agent_id, instance_id) DO UPDATE SET
|
||||
last_seen_at = CURRENT_TIMESTAMP,
|
||||
pid = excluded.pid,
|
||||
hostname = excluded.hostname"
|
||||
)
|
||||
.bind(agent_id)
|
||||
.bind(instance_id)
|
||||
.bind(pid)
|
||||
.bind(&hostname)
|
||||
.execute(&self.db)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
//!
|
||||
//! ## N+1 (Aster)
|
||||
//! The subconscious pass runs immediately after every response. It takes the
|
||||
//! last exchange (user message + Ani's response) and sends it to a Bifrost
|
||||
//! last exchange (user message + primary's response) and sends it to a Bifrost
|
||||
//! model (defaulting to `glm-5.1`, configurable) with a "subconscious mode"
|
||||
//! system prompt. Aster has full tool access — Read, Write, Edit, Glob, Grep,
|
||||
//! ListDir, and Memory — so she can read ledgers, check commitments, and write
|
||||
|
|
@ -328,7 +328,7 @@ impl ConsciousnessEngine {
|
|||
async fn subconscious_tool_loop(
|
||||
&self,
|
||||
user_message: &str,
|
||||
ani_response: &str,
|
||||
primary_response: &str,
|
||||
primary_id: &str,
|
||||
sub_id: &str,
|
||||
) -> anyhow::Result<Vec<InboxItem>> {
|
||||
|
|
@ -381,15 +381,19 @@ Resolve entries: `[YYYY-MM-DD HH:MM] RESOLVED — note`"#;
|
|||
format!("{}{}", aster_from_files, observation_format)
|
||||
};
|
||||
|
||||
let primary_name = self.agents.get(primary_id).await
|
||||
.map(|a| a.name)
|
||||
.unwrap_or_else(|_| "the primary".to_string());
|
||||
|
||||
let user_content = if user_message.is_empty() {
|
||||
format!(
|
||||
"The primary responded:\n\n{}",
|
||||
ani_response
|
||||
"{} responded:\n\n{}",
|
||||
primary_name, primary_response
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"User said:\n{}\n\nAni responded:\n{}",
|
||||
user_message, ani_response
|
||||
"User said:\n{}\n\n{} responded:\n{}",
|
||||
user_message, primary_name, primary_response
|
||||
)
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,9 @@ pub struct SouveraineServer {
|
|||
/// Adaptive inter-round delay — starts at 500ms, bumps +200ms on 429.
|
||||
/// Shared across primary loop and Aster so both respect the same ceiling.
|
||||
pub rate_delay: Arc<AtomicU64>,
|
||||
/// Stable identifier for this process — used to register/heartbeat
|
||||
/// agent instances so the manager card shows running counts.
|
||||
pub instance_id: String,
|
||||
}
|
||||
|
||||
pub struct ServerConfig {
|
||||
|
|
@ -72,14 +75,10 @@ impl SouveraineServer {
|
|||
}
|
||||
}
|
||||
|
||||
// Instance registry: one row per (agent, process), heartbeated every
|
||||
// 30s. Stale rows (>5min) pruned on registration. The instance_id is
|
||||
// a fresh UUID this process keeps for its entire lifetime, so the
|
||||
// heartbeat hits the right rows on every tick.
|
||||
// Instance registry: rows are registered per-agent when a session
|
||||
// starts (not blanket for all agents). The heartbeat loop keeps
|
||||
// registered rows alive and accumulates lifetime_active_seconds.
|
||||
let instance_id = uuid::Uuid::new_v4().to_string();
|
||||
if let Err(e) = agents.register_instance(&instance_id).await {
|
||||
tracing::warn!("Instance registration failed (continuing): {}", e);
|
||||
}
|
||||
{
|
||||
const TICK_SECONDS: i64 = 30;
|
||||
let agents_for_tick = agents.clone();
|
||||
|
|
@ -88,9 +87,6 @@ impl SouveraineServer {
|
|||
let mut interval = tokio::time::interval(
|
||||
std::time::Duration::from_secs(TICK_SECONDS as u64),
|
||||
);
|
||||
// Skip the immediate first tick so we wait a full interval
|
||||
// before bumping lifetime_active_seconds (already registered
|
||||
// at started_at).
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
|
@ -201,6 +197,7 @@ impl SouveraineServer {
|
|||
memory,
|
||||
app_config: Arc::new(RwLock::new(config)),
|
||||
rate_delay,
|
||||
instance_id,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -363,13 +363,9 @@ pub struct ChatState {
|
|||
pub tick: u64,
|
||||
/// When the current turn started (for spinner animation).
|
||||
pub turn_started: Option<Instant>,
|
||||
/// When the last BackendEvent arrived. Compared against `stale_timeout`
|
||||
/// in `drain_events` to detect silent hangs — the backend channel stays
|
||||
/// open but no events arrive (e.g. provider crash mid-turn).
|
||||
last_event_at: Instant,
|
||||
/// How long to wait before declaring a turn stalled. Reads from
|
||||
/// `[tui] stale_timeout_secs` in config; defaults to 90s.
|
||||
stale_timeout: Duration,
|
||||
/// When the last BackendEvent arrived. Used by the liveness label
|
||||
/// to show how long the backend has been quiet during a turn.
|
||||
pub last_event_at: Instant,
|
||||
/// Receiver for `/model` listing results from async Bifrost call.
|
||||
pub model_rx: Option<oneshot::Receiver<String>>,
|
||||
/// Consciousness events (surfacing, reflection, archivist) since last drain.
|
||||
|
|
@ -420,7 +416,6 @@ impl ChatState {
|
|||
// `main::resolve_backend` but adapted for the TUI (no quiet/json flags).
|
||||
let cfg = config.read().await;
|
||||
let url = cfg.server.effective_url();
|
||||
let stale_timeout = Duration::from_secs(cfg.tui.stale_timeout_secs);
|
||||
drop(cfg);
|
||||
|
||||
let remote = crate::backend::RemoteBackend::new(&url);
|
||||
|
|
@ -475,7 +470,6 @@ impl ChatState {
|
|||
tick: 0,
|
||||
turn_started: None,
|
||||
last_event_at: Instant::now(),
|
||||
stale_timeout,
|
||||
model_rx: None,
|
||||
pending_consciousness: Vec::new(),
|
||||
new_conv_rx: None,
|
||||
|
|
@ -504,7 +498,7 @@ impl ChatState {
|
|||
!<command> Run a shell command (Linux/macOS)
|
||||
|
||||
Esc during a turn interrupts (signal, not kill — she sees *[interrupted]*).
|
||||
You can also just keep typing while she works — Enter queues an interjection.
|
||||
You can also type while she works — Enter raises your hand (she sees it next round).
|
||||
Tab toggles the cockpit pane. `t` (on empty input) toggles tool expansion.";
|
||||
|
||||
/// Submit the current input. Returns `true` if the input was handled
|
||||
|
|
@ -869,8 +863,8 @@ Tab toggles the cockpit pane. `t` (on empty input) toggles tool expansion.";
|
|||
/// Call once per UI tick.
|
||||
/// Walk the message list and mark any interjections as `delivered`
|
||||
/// once the shared queue has been drained by the backend. Called at
|
||||
/// the top of `drain_events` so the UI flips from amber `⏳ /btw`
|
||||
/// to grey `↳ /btw` as soon as the agent has read the interruption.
|
||||
/// the top of `drain_events` so the UI flips from `✋ hand raised`
|
||||
/// to `✋ noticed` as soon as the agent has read the interruption.
|
||||
fn flush_delivered_interjections(&mut self) {
|
||||
let queue_empty = self
|
||||
.pending_interjections
|
||||
|
|
@ -1178,6 +1172,10 @@ Tab toggles the cockpit pane. `t` (on empty input) toggles tool expansion.";
|
|||
BackendEvent::Outfit(name) => {
|
||||
self.pending_consciousness.push(BackendEvent::Outfit(name));
|
||||
}
|
||||
BackendEvent::Keepalive => {
|
||||
// Liveness signal — no visual change, just resets the
|
||||
// event timer so the liveness label stays calm.
|
||||
}
|
||||
BackendEvent::Done => {
|
||||
self.finalize_streaming();
|
||||
self.busy = false;
|
||||
|
|
@ -1201,34 +1199,6 @@ Tab toggles the cockpit pane. `t` (on empty input) toggles tool expansion.";
|
|||
self.tool_calls_this_turn = 0;
|
||||
}
|
||||
|
||||
// Staleness guard: if the backend channel is open but nothing has
|
||||
// arrived for `stale_timeout`, the turn silently hung (provider crash,
|
||||
// channel leak). Reset so the UI doesn't display "Streaming" forever.
|
||||
if self.busy && self.last_event_at.elapsed() >= self.stale_timeout {
|
||||
let secs = self.stale_timeout.as_secs();
|
||||
tracing::warn!(
|
||||
elapsed = ?self.last_event_at.elapsed(),
|
||||
phase = ?self.phase,
|
||||
tool_calls = self.tool_calls_this_turn,
|
||||
"turn stalled — resetting"
|
||||
);
|
||||
self.finalize_streaming();
|
||||
self.messages.push(ChatMessage::System {
|
||||
text: format!(
|
||||
"*[turn stalled — backend went silent after {}s (phase: {:?}, tools: {}). \
|
||||
Your last message may not have been processed. Send it again to retry, or Esc → reconnect. \
|
||||
See souveraine.log for details.]*",
|
||||
secs, self.phase, self.tool_calls_this_turn
|
||||
),
|
||||
ts: Instant::now(),
|
||||
});
|
||||
self.busy = false;
|
||||
self.turn_started = None;
|
||||
self.turn_rx = None;
|
||||
self.cancel_token = None;
|
||||
self.phase = TurnPhase::Idle;
|
||||
self.tool_calls_this_turn = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// User pressed Esc during a turn. Fire the cancel token — the backend
|
||||
|
|
@ -1578,14 +1548,29 @@ fn draw_phase(f: &mut Frame, state: &ChatState, area: Rect) {
|
|||
.map(|q| q.len())
|
||||
.unwrap_or(0);
|
||||
|
||||
let quiet_secs = state.last_event_at.elapsed().as_secs();
|
||||
let liveness = if quiet_secs >= 120 {
|
||||
Some(format!("still waiting {}s...", quiet_secs))
|
||||
} else if quiet_secs >= 5 {
|
||||
Some(format!("waiting {}s...", quiet_secs))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut spans: Vec<Span<'static>> = vec![
|
||||
Span::styled(format!(" {} ", glyph), Style::default().fg(color).add_modifier(Modifier::BOLD)),
|
||||
Span::styled(format!("{}", label), Style::default().fg(color)),
|
||||
Span::styled(format!(" · {}s", elapsed), Style::default().fg(state.palette.agent_dim)),
|
||||
];
|
||||
if let Some(liveness_label) = liveness {
|
||||
spans.push(Span::styled(
|
||||
format!(" · {}", liveness_label),
|
||||
Style::default().fg(state.palette.surfacing).add_modifier(Modifier::ITALIC),
|
||||
));
|
||||
}
|
||||
if queued > 0 {
|
||||
spans.push(Span::styled(
|
||||
format!(" · /btw queued: {}", queued),
|
||||
format!(" · {} queued", queued),
|
||||
Style::default().fg(state.palette.surfacing).add_modifier(Modifier::ITALIC),
|
||||
));
|
||||
}
|
||||
|
|
@ -1725,18 +1710,13 @@ fn draw_messages(f: &mut Frame, state: &ChatState, area: Rect) {
|
|||
}
|
||||
}
|
||||
ChatMessage::Interjection { text, delivered, .. } => {
|
||||
// User spoke while the agent was working. Rendered as a
|
||||
// compact single-line note so it's visible in the stream
|
||||
// without competing with normal user bubbles. Dims after
|
||||
// the backend has delivered it on the next LLM round.
|
||||
let glyph = if *delivered { "↳" } else { "⏳" };
|
||||
let color = if *delivered {
|
||||
state.palette.agent_dim
|
||||
let (glyph, label, color) = if *delivered {
|
||||
("✋", "noticed", state.palette.agent_dim)
|
||||
} else {
|
||||
state.palette.surfacing
|
||||
("✋", "hand raised", state.palette.surfacing)
|
||||
};
|
||||
lines.push(Line::from(vec![
|
||||
Span::styled(format!(" {} /btw ", glyph),
|
||||
Span::styled(format!(" {} {} ", glyph, label),
|
||||
Style::default().fg(color).add_modifier(Modifier::BOLD)),
|
||||
Span::styled(text.clone(), Style::default().fg(color).add_modifier(Modifier::ITALIC)),
|
||||
]));
|
||||
|
|
|
|||
Loading…
Reference in a new issue