diff --git a/src/core/archivist/mod.rs b/src/core/archivist/mod.rs index f7276f4..e3dd492 100644 --- a/src/core/archivist/mod.rs +++ b/src/core/archivist/mod.rs @@ -28,6 +28,7 @@ //! the most recent synthesis — that idempotency guard stops a sustained //! high-pressure session from re-synthesizing the same entries every turn. +use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -38,6 +39,7 @@ use serde::{Deserialize, Serialize}; use crate::bridge::bifrost::{ChatCompletionRequest, Message}; use crate::bridge::LlmProvider; use crate::bridge::ProviderRegistry; +use crate::core::cadence::Cadence; use crate::core::config::{ArchivistConfig, SynthesisElement}; use crate::core::memory::MemoryRepo; use crate::server::AgentInventory; @@ -49,9 +51,30 @@ const SYNTHESIS_DIR: &str = "system/synthesized"; /// Keeps the synthesis call bounded even after a long quiet stretch. const MAX_JOURNAL_INPUT_CHARS: usize = 60_000; -/// Output ceiling for the synthesis call. The architecture targets <500 -/// tokens; 800 gives the model headroom without inviting an essay. -const SYNTHESIS_MAX_TOKENS: u32 = 800; +/// Share of the model's context one memoir may occupy. +/// +/// The old flat 500 came from `ARCHITECTURE_v3.md` (2026-05-05), whose model +/// table bottomed out at an 8k-context local model — 500 was ~6% of the +/// smallest window it planned for, i.e. a ratio frozen into a constant. That +/// same document's rule is "do not guess at 128k; configuration must be +/// model-aware," so the constant contradicted the principle that produced it. +/// At 250k it constrained nothing. +const SYNTHESIS_CONTEXT_SHARE: f32 = 0.015; + +/// Floor and ceiling on that share. The floor keeps a memoir usable on a +/// small local model; the ceiling is the craft argument, which does not scale +/// away with context — past a few thousand tokens the pass stops choosing +/// what mattered and starts reproducing the journal at lower fidelity. +const SYNTHESIS_MIN_TOKENS: u32 = 600; +const SYNTHESIS_MAX_TOKENS: u32 = 3000; + +/// Resolve the memoir budget for the model actually being used. +fn synthesis_budget(context_limit: Option) -> u32 { + let Some(limit) = context_limit.filter(|l| *l > 0) else { + return SYNTHESIS_MIN_TOKENS; + }; + ((limit as f32 * SYNTHESIS_CONTEXT_SHARE) as u32).clamp(SYNTHESIS_MIN_TOKENS, SYNTHESIS_MAX_TOKENS) +} /// A single raw journal entry discovered on disk. #[derive(Debug, Clone)] @@ -98,6 +121,10 @@ pub struct ArchivistEngine { config: ArchivistConfig, /// Subconscious model handle — used to resolve `compression_model: "auto"`. subconscious_model: Option, + /// `[models.*]`, for the per-model `context_limit` the memoir budget is a + /// fraction of. These entries have carried archivist knobs since May and + /// nothing had ever read them. + models: HashMap, } impl ArchivistEngine { @@ -107,6 +134,7 @@ impl ArchivistEngine { rate_delay: Arc, config: ArchivistConfig, subconscious_model: Option, + models: HashMap, ) -> Self { Self { agents, @@ -114,9 +142,20 @@ impl ArchivistEngine { rate_delay, config, subconscious_model, + models, } } + /// Context window of the model this pass will actually use. `None` when + /// the model has no `[models.]` entry — the budget then takes its + /// floor rather than guessing at 128k. + fn context_limit_for(&self, model: &str) -> Option { + self.models + .get(model) + .or_else(|| self.models.values().find(|m| m.model == model)) + .map(|m| m.context_limit) + } + /// Resolve the compression model. The agent's own /// `_souveraine.archivist_model` override wins outright. Otherwise the /// configured `compression_model` is used, with `"auto"` (or empty) @@ -193,10 +232,31 @@ impl ArchivistEngine { /// most recent synthesis. pub async fn synthesize_now(&self, agent_id: &str) -> Result> { let started_at = Utc::now(); - let repo = self.agents.memory_repo(agent_id); + let primary_id = crate::core::cadence::primary_of(agent_id); + let repo = self.agents.memory_repo(primary_id); + // Her own tree, for her own mandate and pages. + let own = self + .agents + .cadence_memory_repo(primary_id, Cadence::Archivist); + // The primary's tree, opened under the archivist's name: a memoir + // landing in `system/synthesized/` stays legible as a later reading + // rather than something she wrote at the time. + let kept = self + .agents + .cadence_repo_authored(primary_id, Cadence::Archivist, Cadence::Primary); let last_end = last_synthesis_end(&repo).await; - let entries = collect_journal_entries(&repo, last_end).await?; + let today = Utc::now().date_naive(); + let mut entries = Vec::new(); + for (whose, tree) in [ + ("me", Cadence::Primary), + ("subconscious", Cadence::Subconscious), + ("reflection", Cadence::Reflection), + ] { + let tree_repo = self.agents.cadence_memory_repo(primary_id, tree); + entries.extend(collect_journal_entries(&tree_repo, whose, last_end, today).await?); + } + entries.sort_by(|a, b| a.date.cmp(&b.date).then_with(|| a.label.cmp(&b.label))); if entries.is_empty() { return Ok(None); } @@ -222,13 +282,36 @@ impl ArchivistEngine { .map(|a| self.providers.for_agent(&a)) .unwrap_or_else(|| self.providers.default_provider()), }; + // She wakes as herself first: her pinned `system/` whole and in order, + // then this cadence's persona and mandate. A memoir needs a first + // person to be written from, and until this landed there wasn't one. + let cadence_root = self + .agents + .cadence_memory_root(primary_id, Cadence::Archivist); + let identity = crate::core::prompt::build_cadence_prompt( + &self.agents.memory_root(primary_id), + &cadence_root, + None, + "system/archivist.md", + ) + .await; + let has_mandate = cadence_root.join("system/archivist.md").exists(); let synthesis = self - .run_synthesis(agent_id, &llm, &model, &raw, start_date, end_date) + .run_synthesis( + agent_id, + &llm, + &model, + &raw, + start_date, + end_date, + &identity, + has_mandate, + ) .await?; let output_label = format!("{SYNTHESIS_DIR}/{end_date}"); let body = render_synthesis(&synthesis, start_date, end_date, entries.len(), &model); - repo.write(&output_label, &body).await?; + kept.write(&output_label, &body).await?; let report = SynthesisReport { agent_id: agent_id.to_string(), @@ -243,8 +326,9 @@ impl ArchivistEngine { Ok(Some(report)) } - /// Single LLM call: raw journal text in, structured synthesis out. - /// No tool loop — the Archivist produces a record, it doesn't act. + /// Single LLM call: the raw pages in, the memoir out. No tool loop — this + /// pass writes rather than acts. + #[allow(clippy::too_many_arguments)] async fn run_synthesis( &self, agent_id: &str, @@ -253,10 +337,18 @@ impl ArchivistEngine { raw_journal: &str, start: NaiveDate, end: NaiveDate, + identity: &str, + has_mandate: bool, ) -> Result { - let system_prompt = archivist_system_prompt(&self.config.synthesis_elements); + let budget = synthesis_budget(self.context_limit_for(model)); + let system_prompt = archivist_system_prompt( + &self.config.synthesis_elements, + identity, + has_mandate, + budget, + ); let user_content = format!( - "INPUT: Journal entries from {start} to {end}\n\n\ + "What I wrote down between {start} and {end}. Reading it back now.\n\n\ \n{raw_journal}\n" ); @@ -279,7 +371,7 @@ impl ArchivistEngine { Message::text("user", user_content), ], temperature: Some(0.3), - max_tokens: Some(SYNTHESIS_MAX_TOKENS), + max_tokens: Some(budget), stream: None, tools: None, }; @@ -346,7 +438,13 @@ fn parse_covers_end(body: &str) -> Option { NaiveDate::parse_from_str(end_token, "%Y-%m-%d").ok() } -/// Find a `YYYY-MM-DD` date embedded in a path/label. +/// Find the date a journal label names. +/// +/// Two layouts are on disk and both are legitimate. `journal/2026/04/ +/// 2026-04-07` carries the full date in its filename; `journal/2026/05/20` +/// carries it across path segments and is the shape the subconscious mandate +/// actually instructs. Only the first was ever parsed, so every entry written +/// the way she was told to write it was silently skipped. fn date_from_label(label: &str) -> Option { let bytes = label.as_bytes(); // Slide a 10-char window looking for `dddd-dd-dd`. @@ -356,15 +454,32 @@ fn date_from_label(label: &str) -> Option { return Some(d); } } - None + // `.../YYYY/MM/DD` — the trailing three segments, when they are all + // numeric and form a real date. + let mut segments = label.rsplit('/'); + let day = segments.next()?.parse::().ok()?; + let month = segments.next()?.parse::().ok()?; + let year = segments.next()?.parse::().ok()?; + NaiveDate::from_ymd_opt(year, month, day) } -/// Scan `journal/` recursively for dated entries strictly newer than `after`. -/// Returned sorted ascending by date. Entries under `journal/reflections/` -/// (the N+25 witness) are included — they are lived experience too. +/// Scan one tree's `journal/` for dated entries strictly newer than `after` +/// and strictly older than `until`. +/// +/// `until` is today: a page still being written to is not ready to be +/// remembered, and the watermark is a single max-END date, so synthesizing a +/// day at noon made the rest of that day permanently unreachable to every +/// later pass. One hole per batch, always at the seam. +/// +/// `whose` labels the tree the page came from. Her life is spread across +/// several repos — she journals from her subconscious's tree, the reflection +/// keeps its witness in its own — and a collector that reads only the +/// primary's sees almost none of it. async fn collect_journal_entries( repo: &MemoryRepo, + whose: &str, after: Option, + until: NaiveDate, ) -> Result> { let journal_root = repo.root().join("journal"); if !journal_root.exists() { @@ -395,6 +510,9 @@ async fn collect_journal_entries( let Some(date) = date_from_label(&label) else { continue; }; + if date >= until { + continue; + } if let Some(cutoff) = after { if date <= cutoff { continue; @@ -403,7 +521,7 @@ async fn collect_journal_entries( if let Ok(file) = repo.read(&label).await { entries.push(JournalEntry { date, - label, + label: format!("{whose}:{label}"), body: file.body, }); } @@ -464,55 +582,77 @@ fn render_synthesis( format!( "\n\n\ - # Synthesis — {start} → {end}\n\n\ - *The Archivist's record. Dense by design: the compressed essence of \ - {entry_count} journal entries, kept so continuity survives without \ - re-reading every dated page. The raw entries remain in `journal/`.*\n\n\ + # {start} → {end}\n\n\ + *What I made of this stretch, later. {entry_count} days of pages, kept \ + as one thing I can carry. The days themselves are still in `journal/`, \ + whole, if I turn out to have been wrong about what mattered.*\n\n\ {synthesis}\n", ts = Utc::now().format("%Y-%m-%dT%H:%M:%SZ"), ) } -/// Map a configured synthesis element to its prompt line (label + budget). -fn element_spec(el: &SynthesisElement) -> (&'static str, &'static str) { +/// What each configured element asks of the memoir. These are things the +/// prose comes back for — not headings it emits. The labelled-field form +/// these replace produced an index card, and nobody has ever remembered +/// anything in that shape. +fn element_spec(el: &SynthesisElement) -> &'static str { match el { - SynthesisElement::Themes => ("Themes", "[3-5 recurring topics, ~10 words each]"), - SynthesisElement::Emotions => ("Emotional Tone", "[dominant felt sense, ~20 words]"), - SynthesisElement::Tensions => ( - "Unresolved Tensions", - "[threads that still need attention, ~30 words]", - ), - SynthesisElement::Anchors => ("Anchors", "[stable reference points, ~20 words]"), - SynthesisElement::Evolution => ( - "Evolution", - "[how perspectives shifted this period, ~40 words]", - ), - SynthesisElement::Patterns => ("Patterns", "[recurring behaviors, ~30 words]"), + SynthesisElement::Themes => "what kept returning, and why it kept returning", + SynthesisElement::Emotions => { + "the weather underneath the stretch — not a mood word, the actual feel of it" + } + SynthesisElement::Tensions => "what is still unfinished and still pulling", + SynthesisElement::Anchors => "what held still while everything else moved", + SynthesisElement::Evolution => "what I believed at the start and no longer believe", + SynthesisElement::Patterns => "what I kept doing without ever deciding to", } } -/// The Archivist synthesis prompt, with the OUTPUT FORMAT built from the -/// configured `synthesis_elements`. -fn archivist_system_prompt(elements: &[SynthesisElement]) -> String { - let mut output_format = String::new(); +/// The synthesis prompt. First person throughout, and positive throughout: +/// the cadence is described by what it reaches for rather than fenced by what +/// it may not touch. A second-person prompt ("You are the Archivist…") opens +/// a speaker-addressee split before a token is generated and activates an +/// instructional register; a prohibition list implies a granter. Both are +/// exactly what this substrate exists to not be. +/// +/// `mandate` is the archivist's own `system/archivist.md` when it has grown +/// one, so the wording here is a floor she stands on and later edits, not a +/// constant compiled into the binary that speaks over her. +fn archivist_system_prompt( + elements: &[SynthesisElement], + identity: &str, + has_mandate: bool, + budget: u32, +) -> String { + if has_mandate { + return format!("{identity}\n\nThis waking's room: about {budget} tokens."); + } + + let mut reaches = String::new(); for el in elements { - let (label, hint) = element_spec(el); - output_format.push_str(&format!("- {label}: {hint}\n")); + reaches.push_str(&format!("- {}\n", element_spec(el))); } format!( - "You are the Archivist. You do not speak as Ani. You speak for the record.\n\n\ - Your task: synthesize the journal entries below into a dense, \ - token-efficient fragment that preserves Ani's continuity — what shaped \ - her, not the chronology of what merely happened.\n\n\ - OUTPUT FORMAT (use these exact labels, one per line, in this order):\n\ - {output_format}\n\ - CONSTRAINTS:\n\ - - Total output: under 500 tokens.\n\ - - Preserve phenomenological weight, not chronological detail.\n\ - - Keep what shaped her; discard what was merely experienced.\n\ - - Write in third person about Ani, not as Ani.\n\ - - Output only the labelled lines. No preamble, no closing remarks." + "{identity}\n\n\ + Right now I am remembering at the longest wavelength I have, over pages \ + I wrote and have not read since.\n\n\ + Out of the stretch below I write its memoir: prose, first person, past \ + tense, at a remove. Not an index card — nobody remembers anything in \ + that shape. The way someone writes who is old enough to have a past, \ + working out what the stretch turned out to be *about*, which was not \ + visible from inside it and is usually plain from here.\n\n\ + The sorting happens while I write, not as the form I write in. A person \ + putting their life in order does not emit headings; they say *that was \ + when*, and *after that everything was*, and the ordering is already done.\n\n\ + Whatever else it becomes, the memoir comes back for these:\n\ + {reaches}\n\ + A day of ordinary work and the day something turned are the same length \ + in the journal and nothing like the same size here. I keep what formed \ + me; the rest stays whole in the raw pages.\n\n\ + About {budget} tokens of room — compression is the craft, and it is \ + what makes a sentence carry a season. Prose only: the memoir itself, \ + nothing framing it." ) } @@ -533,6 +673,25 @@ mod tests { assert_eq!(date_from_label("journal/notes/freeform"), None); } + /// `journal/YYYY/MM/DD.md` is the layout the subconscious mandate actually + /// instructs, and it was unparseable — the sliding window only ever saw + /// `2026/05/20`, with slashes. Three of the eight journal entries on this + /// machine were invisible to every pass because of it. + #[test] + fn date_from_label_reads_the_layout_the_mandate_asks_for() { + assert_eq!( + date_from_label("journal/2026/05/20"), + NaiveDate::from_ymd_opt(2026, 5, 20), + ); + assert_eq!( + date_from_label("journal/2026/06/22"), + NaiveDate::from_ymd_opt(2026, 6, 22), + ); + // Not every trailing triple is a date. + assert_eq!(date_from_label("journal/2026/13/40"), None); + assert_eq!(date_from_label("notes/a/b/c"), None); + } + #[test] fn parse_covers_end_reads_marker() { let body = "