Watch
1
0
Fork
You've already forked souveraine
0

server: announce restarts and resume with history hydrated

POST /v1/server/restart writes a reason marker and asks systemd; boot
reads it, announces resumed (or started) on the event bus and the new
status endpoint, and the panel says why it is back instead of
pretending. Conversations hydrate at boot and on a lazy miss, so a
held conversation id resolves after a restart instead of 404ing.
This commit is contained in:
Fimeg 2026-08-18 10:54:48 -04:00
commit 7089dfb824
9 changed files with 276 additions and 38 deletions

View file

@ -1,5 +1,39 @@
use crate::api::models::*;
use crate::server::SouveraineServer;
/// Resolve a session, hydrating from disk when a freshly booted server has
/// not loaded this conversation yet. Surfaces hold conversation ids, not a
/// map; a restart must answer a held id without a prior listing.
async fn session_or_hydrate(
server: &SouveraineServer,
id: &str,
) -> Result<Arc<crate::server::session_manager::Session>, ApiError> {
if let Some(session) = server.sessions.get(id) {
return Ok(session);
}
server
.sessions
.hydrate_containing(id)
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(ErrorResponse {
error: "conversation_hydrate_failed".to_string(),
message: e.to_string(),
}),
)
})?;
server.sessions.get(id).ok_or_else(|| {
(
StatusCode::NOT_FOUND,
Json(ErrorResponse {
error: "conversation_not_found".to_string(),
message: format!("Conversation {} not found", id),
}),
)
})
}
use axum::{
body::Bytes,
extract::{ws::WebSocket, Path, Query, State, WebSocketUpgrade},
@ -249,15 +283,7 @@ pub async fn get_conversation(
State(server): State<Arc<SouveraineServer>>,
Path(id): Path<String>,
) -> Result<Json<Conversation>, ApiError> {
let session = server.sessions.get(&id).ok_or_else(|| {
(
StatusCode::NOT_FOUND,
Json(ErrorResponse {
error: "conversation_not_found".to_string(),
message: format!("Conversation {} not found", id),
}),
)
})?;
let session = session_or_hydrate(&server, &id).await?;
let conversation = Conversation {
id: session.conversation_id.clone(),
@ -278,15 +304,7 @@ pub async fn stream_messages(
ApiError,
> {
let agent_id = {
let session = server.sessions.get(&conversation_id).ok_or_else(|| {
(
StatusCode::NOT_FOUND,
Json(ErrorResponse {
error: "conversation_not_found".to_string(),
message: format!("Conversation {} not found", conversation_id),
}),
)
})?;
let session = session_or_hydrate(&server, &conversation_id).await?;
session.agent_id.clone()
};
@ -585,15 +603,7 @@ pub async fn get_conversation_messages(
State(server): State<Arc<SouveraineServer>>,
Path(id): Path<String>,
) -> Result<Json<Vec<crate::core::session::ConversationMessage>>, ApiError> {
let session = server.sessions.get(&id).ok_or_else(|| {
(
StatusCode::NOT_FOUND,
Json(ErrorResponse {
error: "conversation_not_found".to_string(),
message: format!("Conversation {} not found", id),
}),
)
})?;
let session = session_or_hydrate(&server, &id).await?;
Ok(Json(session.messages.clone()))
}
@ -677,6 +687,56 @@ pub async fn interject(
Ok(StatusCode::ACCEPTED)
}
/// POST /v1/server/restart — an intentional restart: marker, `restarting`
/// event, then systemd takes the unit down. The next boot announces
/// `resumed` with the reason instead of a bare `started`. Responds before
/// the restart lands so the caller sees the confirmation.
pub async fn restart_server(
State(server): State<Arc<SouveraineServer>>,
Json(request): Json<RestartRequest>,
) -> Result<Json<RestartResponse>, ApiError> {
let marker = crate::server::restart::RestartMarker {
reason: request.reason,
at: chrono::Utc::now(),
by: request.by.unwrap_or_else(|| "api".to_string()),
};
crate::server::restart::write(&server.data_dir, &marker).map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(ErrorResponse {
error: "restart_marker_failed".to_string(),
message: e.to_string(),
}),
)
})?;
server.event_bus.send(crate::core::nervous::SensorEvent {
sensor_name: "server".to_string(),
timestamp: marker.at,
event_type: "restarting".to_string(),
target: None,
urgency: 1.0,
payload: serde_json::to_value(&marker).ok(),
seed_id: None,
reply_to: None,
});
let at = marker.at;
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
let _ = std::process::Command::new("systemctl")
.args(["--user", "restart", "souveraine.service"])
.spawn();
});
Ok(Json(RestartResponse { scheduled: true, at }))
}
/// GET /v1/server/status — how this process came up. `resumed` carries the
/// marker's reason; surfaces print it so a restart is announced, not hidden.
pub async fn server_status(
State(server): State<Arc<SouveraineServer>>,
) -> Result<Json<crate::server::BootInfo>, ApiError> {
Ok(Json(server.boot.clone()))
}
// ─── Memory (memfs HTTP write path) ───────────────────────────────────────
//
// Routes:
@ -996,15 +1056,7 @@ pub async fn get_conversation_tokens(
State(server): State<Arc<SouveraineServer>>,
Path(conversation_id): Path<String>,
) -> Result<Json<serde_json::Value>, ApiError> {
let session = server.sessions.get(&conversation_id).ok_or_else(|| {
(
StatusCode::NOT_FOUND,
Json(ErrorResponse {
error: "conversation_not_found".to_string(),
message: format!("Conversation {} not found", conversation_id),
}),
)
})?;
let session = session_or_hydrate(&server, &conversation_id).await?;
let counter = crate::bridge::model_router::TokenCounter::new();

View file

@ -33,7 +33,9 @@ pub fn create_routes(state: Arc<SouveraineServer>) -> Router {
"/v1/conversations/:id/tokens",
get(handlers::get_conversation_tokens),
)
.route("/health", get(health_check));
.route("/health", get(health_check))
.route("/v1/server/status", get(handlers::server_status))
.route("/v1/server/restart", post(handlers::restart_server));
// Protected agent routes — require per-agent bearer token.
let agent_routes = Router::new()

View file

@ -512,6 +512,22 @@ pub struct InterjectRequest {
pub text: String,
}
#[derive(Debug, Deserialize)]
pub struct RestartRequest {
/// Why the server is being restarted. Travels with the marker and is
/// announced as a `resumed` event on the next boot.
pub reason: String,
/// Who asked. Defaults to "api" when omitted.
#[serde(default)]
pub by: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct RestartResponse {
pub scheduled: bool,
pub at: chrono::DateTime<chrono::Utc>,
}
/// The full wire mirror of [`crate::backend::BackendEvent`].
///
/// Every engine event crosses the SSE boundary — no silent skips. The

View file

@ -302,4 +302,12 @@ pub trait Backend: Send + Sync {
) -> Vec<crate::core::nervous::pending::PendingSurfacing> {
Vec::new()
}
/// How the server this backend talks to came up. `resumed` means an
/// intentional restart with a reason attached; the surface announces it.
/// Default: None — a local in-process server reports through its own
/// state, not through this channel.
async fn server_status(&self) -> Option<crate::server::BootInfo> {
None
}
}

View file

@ -155,6 +155,20 @@ impl Backend for RemoteBackend {
}
}
async fn server_status(&self) -> Option<crate::server::BootInfo> {
self.client
.get(self.url("/v1/server/status"))
.timeout(Duration::from_millis(800))
.send()
.await
.ok()?
.error_for_status()
.ok()?
.json()
.await
.ok()
}
async fn list_agents(&self) -> Result<Vec<AgentInfo>> {
#[derive(Deserialize)]
struct Wire {

View file

@ -3,6 +3,7 @@ use crate::bridge::{build_registry, ProviderRegistry};
use crate::core::compact::{CompactionEngine, DefaultCompactionEngine, UtcClock};
use crate::core::config::ConsciousnessConfig;
use crate::server::gitea_memory::GiteaMemory;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::AtomicU64;
@ -19,6 +20,7 @@ pub mod federation;
pub mod gitea_client;
pub mod gitea_memory;
pub mod listener;
pub mod restart;
pub mod session_manager;
pub mod subagent;
pub mod summon_handler;
@ -34,6 +36,19 @@ pub use session_manager::SessionManager;
// is rebuilt in Stage 5 against the `Memory` trait from `crates/memory`.
pub type ServerMemory = GiteaMemory;
/// How this process came up. `resumed` carries the reason from an
/// intentional restart (POST /v1/server/restart); `started` is a cold boot.
/// Surfaces read this so a restarted server says why it is here instead of
/// pretending nothing happened.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BootInfo {
pub instance_id: String,
pub event_type: String,
pub at: chrono::DateTime<chrono::Utc>,
pub reason: Option<String>,
pub by: Option<String>,
}
#[derive(Clone)]
pub struct SouveraineServer {
pub agents: Arc<AgentInventory>,
@ -55,6 +70,9 @@ pub struct SouveraineServer {
/// todo, energy, posture. Firehose subscribers (EventLog, WebSocket
/// bridge, desktop overlay) listen on this bus.
pub event_bus: crate::core::nervous::EventBus,
/// How this process came up — set at construction from the restart
/// marker, and read by surfaces to announce a resumption.
pub boot: BootInfo,
/// Tracks known federated peers. Updated by `device_announce`/`device_leave`
/// events on the bus. Persisted to disk for CLI access.
pub device_registry: Option<Arc<DeviceRegistry>>,
@ -163,6 +181,51 @@ impl SouveraineServer {
let event_bus = crate::core::nervous::EventBus::default();
let sessions = Arc::new(SessionManager::with_persistence(data_dir.join("agents")));
// ── Boot: resume with history, and say why we are here ──
// Hydrate every agent's persisted conversations before serving: the
// surfaces hold conversation ids, not a map, so a restart that answers
// a held id with 404 until someone lists reads as amnesia. An
// intentional restart (POST /v1/server/restart) leaves a marker; boot
// announces `resumed` with its reason on the event bus and clears it.
// A boot with no marker announces `started`.
if let Ok(existing) = agents.list(None).await {
for summary in &existing {
if let Err(e) = sessions.load_persisted(&summary.id).await {
tracing::warn!("boot hydration failed for {}: {}", summary.id, e);
}
}
}
let boot_marker = crate::server::restart::read(&data_dir);
let boot_event_type = if boot_marker.is_some() { "resumed" } else { "started" };
let boot = BootInfo {
instance_id: instance_id.clone(),
event_type: boot_event_type.to_string(),
at: boot_marker
.as_ref()
.map(|m| m.at)
.unwrap_or_else(chrono::Utc::now),
reason: boot_marker.as_ref().map(|m| m.reason.clone()),
by: boot_marker.as_ref().map(|m| m.by.clone()),
};
event_bus.send(crate::core::nervous::SensorEvent {
sensor_name: "server".to_string(),
timestamp: boot.at,
event_type: boot.event_type.clone(),
target: None,
urgency: 1.0,
payload: boot_marker
.as_ref()
.and_then(|m| serde_json::to_value(m).ok()),
seed_id: None,
reply_to: None,
});
tracing::info!(
event_type = boot.event_type,
reason = boot.reason,
"server boot"
);
crate::server::restart::clear(&data_dir);
// Build the per-agent provider registry from config. z.ai, bifrost, etc.
let providers = Arc::new(build_registry(&config)?);
@ -396,6 +459,7 @@ impl SouveraineServer {
rate_delay,
instance_id,
event_bus,
boot,
device_registry,
summon_handler,
local_seed_id,

36
src/server/restart.rs Normal file
View file

@ -0,0 +1,36 @@
//! Intentional restarts carry their reason across the reboot. The API writes
//! a marker before asking systemd for a restart; boot reads it, announces
//! `resumed` on the event bus (firehose + federation peers), and clears it.
//! A boot with no marker reads as a cold `started` and stays quiet about why.
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
const MARKER_FILE: &str = "restart.json";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RestartMarker {
pub reason: String,
pub at: DateTime<Utc>,
pub by: String,
}
pub fn marker_path(data_dir: &Path) -> PathBuf {
data_dir.join(MARKER_FILE)
}
pub fn read(data_dir: &Path) -> Option<RestartMarker> {
std::fs::read_to_string(marker_path(data_dir))
.ok()
.and_then(|raw| serde_json::from_str(&raw).ok())
}
pub fn write(data_dir: &Path, marker: &RestartMarker) -> anyhow::Result<()> {
std::fs::write(marker_path(data_dir), serde_json::to_string_pretty(marker)?)?;
Ok(())
}
pub fn clear(data_dir: &Path) {
let _ = std::fs::remove_file(marker_path(data_dir));
}

View file

@ -360,6 +360,36 @@ impl SessionManager {
Ok(records)
}
/// Resolve a conversation id the index does not know yet by hydrating
/// the agent whose persisted conversations contain it. A restarted
/// server must answer a held conversation id without the client having
/// to list first. Returns true when the session is live afterwards.
pub async fn hydrate_containing(&self, conversation_id: &str) -> anyhow::Result<bool> {
if self.sessions.contains_key(conversation_id) {
return Ok(true);
}
let Some(handle) = &self.store else {
return Ok(false);
};
let mut entries = tokio::fs::read_dir(&handle.agents_dir).await?;
while let Some(entry) = entries.next_entry().await? {
if !entry.file_type().await?.is_dir() {
continue;
}
let agent_id = entry.file_name().to_string_lossy().into_owned();
let conv_dir = handle
.agents_dir
.join(&agent_id)
.join("conversations")
.join(conversation_id);
if tokio::fs::try_exists(&conv_dir).await.unwrap_or(false) {
self.load_persisted(&agent_id).await?;
return Ok(self.sessions.contains_key(conversation_id));
}
}
Ok(false)
}
pub fn conversation_store_for(&self, agent_id: &str) -> Option<ConversationStore> {
self.store.as_ref().map(|h| h.store_for(agent_id))
}

View file

@ -504,8 +504,24 @@ impl ChatState {
};
let pending = backend.take_pending_surfacings(&agent.id).await;
let resumed = backend
.server_status()
.await
.filter(|s| s.event_type == "resumed");
let ready_line = match &resumed {
Some(status) => format!(
"Souveraine resumed after restart{} — {}",
status
.by
.as_deref()
.map(|by| format!(" (by {by})"))
.unwrap_or_default(),
status.reason.as_deref().unwrap_or("no reason given")
),
None => "Souveraine ready. Type to begin.".to_string(),
};
let mut messages: Vec<ChatMessage> = vec![ChatMessage::System {
text: "Souveraine ready. Type to begin.".to_string(),
text: ready_line,
ts: Instant::now(),
}];
let mut cockpit_log: Vec<CockpitEntry> = Vec::new();