//! Independently verifying root-ledger reconciliation. This module never executes //! an operation or grants authority from an agent-provided ready/unlock flag. //! Both general and APT listeners must use the SAME executor.lock and SQLite DB. use anyhow::{Context, Result, ensure}; use chrono::{DateTime, Utc}; use openssl::{ pkey::{Id, PKey}, sign::Verifier, }; use rmm_domain::{ jobs::{Action, ExecutionPermitPayload, hash}, reconciliation::{ AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes, }, }; use rmm_privileged_helper::{Binding, Request}; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeMap, BTreeSet}, fs::File, path::{Path, PathBuf}, }; use uuid::Uuid; pub trait Clock { fn now(&self) -> Result>; } #[derive(Default)] pub struct NativeClock(rmm_privileged_helper::TrustedClock); impl Clock for NativeClock { fn now(&self) -> Result> { ensure!(self.0.trusted(), "CLOCK_UNTRUSTED"); Ok(Utc::now()) } } pub trait BootSource { fn current_boot(&self) -> Result; } pub struct NativeBoot; impl BootSource for NativeBoot { fn current_boot(&self) -> Result { Ok(Uuid::parse_str(&rmm_agent_linux::boot_id()?)?) } } /// Acquire instead of, not on top of, the existing listener's lock_state call. /// Holding this guard is mandatory through snapshot/apply/can_start/consume and /// actual effect. Production uses owner_uid=0; tests use their own private UID. pub struct ExecutorGuard { _file: File, directory: PathBuf, _not_sync: std::marker::PhantomData>, } impl ExecutorGuard { pub fn acquire(directory: &Path, owner_uid: u32) -> Result { ensure!(directory.is_absolute(), "ABSOLUTE_STATE_DIRECTORY_REQUIRED"); let file = rmm_privileged_helper::lock_state(directory, owner_uid)?; Ok(Self { _file: file, directory: directory.canonicalize()?, _not_sync: std::marker::PhantomData, }) } pub(crate) fn check(&self, db: &Connection) -> Result<()> { let path: String = db.query_row( "SELECT file FROM pragma_database_list WHERE name='main'", [], |r| r.get(0), )?; ensure!( Path::new(&path).canonicalize()? == self.directory.join("consumption.sqlite").canonicalize()?, "NOT_THE_SHARED_EXECUTOR_LEDGER" ); Ok(()) } } /// Same two fields as public SyncResponse, without adding a new dependency or /// wire type. RPC glue copies the ORIGINAL bytes/signature, never reserializes. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct SignedResponse { pub decision_bytes: Vec, pub signature: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct LocalAttempt { pub job_id: Uuid, pub attempt_id: Uuid, pub job_payload_hash: String, pub grant_id: Uuid, pub authorization_id: Uuid, pub plan_hash: String, pub action: String, pub execution_identity: String, pub revision: u64, pub original_boot_id: Uuid, pub consumption: String, pub state: String, pub outcome_sha256: String, pub outcome_bytes: Vec, /// Hash of the exact root-owned Request containing original job/permit bytes. pub original_request_sha256: String, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Snapshot { pub journal_id: Uuid, pub checkpoint: u64, pub previous_decision_id: Option, pub boot_id: Uuid, pub captured_at: DateTime, pub attempts: Vec, } pub struct Applied { pub snapshot: Snapshot, pub unlocked_grants: Vec, pub ready: bool, } struct Capture { snapshot: Snapshot, fingerprint: String, generation: u64, } struct Lease { checkpoint: u64, decision_id: Uuid, boot_id: Uuid, generation: u64, until: DateTime, fingerprint: String, } pub struct HelperSync { binding: Binding, born: DateTime, capture: Option, lease: Option, } impl HelperSync { /// No persisted ready bit can open a newly started general/APT helper. pub fn new(binding: Binding, clock: &impl Clock) -> Result { pinned_key(&binding)?; Ok(Self { binding, born: clock.now()?, capture: None, lease: None, }) } pub fn close(&mut self) { self.lease = None; self.capture = None; } pub fn active_generation(&self) -> Option { self.lease.as_ref().map(|lease| lease.generation) } /// Every call creates a NEW durable checkpoint challenge and closes the old /// lease. Use apply's returned snapshot for its ACK, not another snapshot(). pub fn snapshot( &mut self, db: &mut Connection, guard: &ExecutorGuard, generation: u64, clock: &impl Clock, boot: &impl BootSource, ) -> Result { self.close(); let result = self.snapshot_inner(db, guard, generation, clock, boot); if let Err(error) = &result { persist_fault(db, error); } result } fn snapshot_inner( &mut self, db: &mut Connection, guard: &ExecutorGuard, generation: u64, clock: &impl Clock, boot: &impl BootSource, ) -> Result { guard.check(db)?; let now = clock.now()?; ensure!( now >= self.born && generation > 0, "CLOCK_OR_CHANNEL_REGRESSION" ); let boot_id = boot.current_boot()?; ensure!(!boot_id.is_nil(), "BOOT_ID_REQUIRED"); initialize(db, &self.binding)?; let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; let meta = metadata(&tx)?; ensure!(meta.fault.is_none(), "HELPER_MANUAL_RECOVERY_REQUIRED"); let highest: u64 = tx.query_row( "SELECT COALESCE(MAX(checkpoint),0) FROM helper_sync_captures", [], |r| r.get(0), )?; ensure!(meta.checkpoint == highest, "HELPER_CHECKPOINT_ROLLBACK"); ensure!(generation >= meta.generation, "STALE_HELPER_CONNECTION"); validate_history(&tx, &self.binding, &meta, generation)?; let attempts = load_rows(&tx, &self.binding, true)?; ensure!(attempts.len() <= 256, "HELPER_SYNC_CAPACITY"); let fingerprint = ledger_fingerprint(&tx)?; // Two independently gated profiles share one signed helper checkpoint. // Join only a still-unapplied root-owned capture created after THIS // process was born, on the same fenced channel and unchanged ledger. if meta.checkpoint > 0 && meta.generation == generation { let applied: bool = tx.query_row( "SELECT EXISTS(SELECT 1 FROM helper_sync_decisions WHERE checkpoint=?1)", [meta.checkpoint], |r| r.get(0), )?; let old: Option<(Vec, String)> = tx .query_row( "SELECT snapshot,fingerprint FROM helper_sync_captures WHERE checkpoint=?1", [meta.checkpoint], |r| Ok((r.get(0)?, r.get(1)?)), ) .optional()?; if !applied && let Some((bytes, old_fingerprint)) = old { let shared: Snapshot = serde_json::from_slice(&bytes)?; if shared.captured_at >= self.born && shared.captured_at <= now && now - shared.captured_at < chrono::Duration::seconds(30) && shared.boot_id == boot_id && shared.previous_decision_id == meta.decision && old_fingerprint == fingerprint && serde_json::to_vec(&shared.attempts)? == serde_json::to_vec(&attempts)? { tx.commit()?; self.capture = Some(Capture { snapshot: shared.clone(), fingerprint, generation, }); return Ok(shared); } } } let checkpoint = meta .checkpoint .checked_add(1) .context("CHECKPOINT_OVERFLOW")?; i64::try_from(checkpoint)?; let snapshot = Snapshot { journal_id: meta.journal_id, checkpoint, previous_decision_id: meta.decision, boot_id, captured_at: now, attempts, }; let bytes = serde_json::to_vec(&snapshot)?; ensure!(bytes.len() <= 2 * 1024 * 1024, "HELPER_SNAPSHOT_SIZE"); tx.execute("INSERT INTO helper_sync_captures(checkpoint,snapshot,fingerprint,captured_at) VALUES(?1,?2,?3,?4)",params![checkpoint,bytes,fingerprint,now.to_rfc3339()])?; tx.execute( "UPDATE helper_sync_meta SET checkpoint=?1,generation=?2 WHERE id=1", params![checkpoint, generation], )?; tx.commit()?; self.capture = Some(Capture { snapshot: snapshot.clone(), fingerprint, generation, }); Ok(snapshot) } /// raw_request is SyncRequest::encode_to_vec(), NOT the outer Envelope or a /// JSON view. Verify signature before decoding fields. No unsigned unlocks. pub fn apply( &mut self, db: &mut Connection, guard: &ExecutorGuard, response: &SignedResponse, raw_request: &[u8], clock: &impl Clock, boot: &impl BootSource, ) -> Result { self.lease = None; let result = self.apply_inner(db, guard, response, raw_request, clock, boot); if let Err(error) = &result { self.lease = None; persist_fault(db, error); } result } fn apply_inner( &mut self, db: &mut Connection, guard: &ExecutorGuard, response: &SignedResponse, raw_request: &[u8], clock: &impl Clock, boot: &impl BootSource, ) -> Result { guard.check(db)?; let now = clock.now()?; let capture = self .capture .as_ref() .context("FRESH_PROCESS_SNAPSHOT_REQUIRED")?; let decision = verify(&self.binding, response, now)?; let request = RequestView::decode(raw_request)?; let current_boot = boot.current_boot()?; ensure!( now >= self.born && decision.issued_at >= self.born && decision.issued_at >= capture.snapshot.captured_at && current_boot == capture.snapshot.boot_id && decision.boot_id == current_boot, "OLD_PROCESS_DECISION_OR_BOOT" ); ensure!( decision.connection_generation == capture.generation && decision.helper_journal_id == Some(capture.snapshot.journal_id) && decision.helper_checkpoint == capture.snapshot.checkpoint && request.helper_journal_id == capture.snapshot.journal_id.to_string() && request.helper_checkpoint == capture.snapshot.checkpoint && request.helper_previous_decision_id == capture .snapshot .previous_decision_id .map(|id| id.to_string()) .unwrap_or_default() && decision.request_id.to_string() == request.request_id && decision.request_hash == hash(raw_request) && decision.journal_id.to_string() == request.journal_id && decision.checkpoint == request.checkpoint && request.assignment_version == self.binding.assignment_version && request.journal_complete, "HELPER_SYNC_REQUEST_BINDING" ); let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; let meta = metadata(&tx)?; ensure!( meta.fault.is_none() && meta.journal_id == capture.snapshot.journal_id && meta.checkpoint >= capture.snapshot.checkpoint && meta.generation >= capture.generation, "HELPER_CHECKPOINT_ROLLBACK" ); ensure!( meta.checkpoint == capture.snapshot.checkpoint && meta.generation == capture.generation, "STALE_HELPER_CAPTURE" ); ensure!( meta.agent_journal .is_none_or(|id| id == decision.journal_id) && decision.checkpoint >= meta.agent_checkpoint, "AGENT_JOURNAL_ROLLBACK_OR_REPLACEMENT" ); validate_history(&tx, &self.binding, &meta, capture.generation)?; if let Some(old)=tx.query_row("SELECT response,request,fingerprint FROM helper_sync_decisions WHERE decision_id=?1",[decision.decision_id.to_string()],|r|Ok((r.get::<_,Vec>(0)?,r.get::<_,Vec>(1)?,r.get::<_,String>(2)?))).optional()? { ensure!(old.0==serde_json::to_vec(response)? && old.1==raw_request && meta.decision==Some(decision.decision_id) && ledger_fingerprint(&tx)?==old.2,"HELPER_REPLAY_OR_LEDGER_CHANGED"); let rows=load_rows(&tx,&self.binding,false)?; let ready=decision.ready && decision.helper_ready && rows.iter().all(|row|definitive(row) && decision.attempts.iter().any(|d|d.job_id==row.job_id && d.attempt_id==row.attempt_id && d.outcome_state==row.state && d.outcome_sha256.as_deref()==Some(row.outcome_sha256.as_str()))) && tx.query_row("SELECT count(*) FROM consumed WHERE result IS NULL",[],|r|r.get::<_,i64>(0))?==0; let snapshot=Snapshot {journal_id:meta.journal_id,checkpoint:meta.checkpoint,previous_decision_id:meta.decision, boot_id:current_boot,captured_at:capture.snapshot.captured_at,attempts:rows}; tx.commit()?; if ready {self.lease=Some(Lease {checkpoint:meta.checkpoint,decision_id:decision.decision_id,boot_id:current_boot, generation:capture.generation,until:decision.valid_until,fingerprint:old.2});} return Ok(Applied {snapshot,unlocked_grants:vec![],ready}); } ensure!( meta.decision == capture.snapshot.previous_decision_id, "HELPER_PREVIOUS_DECISION_CHANGED" ); ensure!( ledger_fingerprint(&tx)? == capture.fingerprint, "LOCAL_LEDGER_CHANGED_DURING_SYNC" ); let rows = load_rows(&tx, &self.binding, false)?; ensure!( serde_json::to_vec(&rows)? == serde_json::to_vec(&capture.snapshot.attempts)?, "LOCAL_SNAPSHOT_CHANGED_DURING_SYNC" ); let summaries = unique_summaries(&request.attempts)?; let mut decisions = BTreeMap::new(); for d in &decision.attempts { ensure!( decisions.insert((d.job_id, d.attempt_id), d).is_none(), "DUPLICATE_SIGNED_ATTEMPT" ); } let known: BTreeSet<_> = rows.iter().map(|r| (r.job_id, r.attempt_id)).collect(); for d in &decision.attempts { ensure!( d.execution_identity != "root" || d.grant_id.is_none() || known.contains(&(d.job_id, d.attempt_id)), "SIGNED_ROOT_ATTEMPT_MISSING_LOCALLY" ); } let mut unlocked = Vec::new(); let mut safe = true; for row in &rows { let id = (row.job_id, row.attempt_id); let local = summaries .get(&id) .context("ROOT_ATTEMPT_OMITTED_FROM_REQUEST")?; let d = decisions .get(&id) .context("ROOT_ATTEMPT_OMITTED_FROM_DECISION")?; validate_summary(row, local)?; validate_decision(row, local, d)?; if d.unlock { let bytes = unlock_outcome(row, d, current_boot)?; ensure!( tx.execute( "UPDATE consumed SET result=?2 WHERE grant_id=?1", params![row.grant_id.to_string(), bytes] )? == 1, "ROOT_CONSUMPTION_MISSING" ); tx.execute( "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1", params![row.grant_id.to_string(), bytes], )?; unlocked.push(row.grant_id); } else { safe &= definitive(row) && d.outcome_state == row.state && d.outcome_sha256.as_deref() == Some(row.outcome_sha256.as_str()); // A definitive local state with an unresolved consumed lock is // not globally ready merely because the server says ready. let locked: bool = tx.query_row( "SELECT result IS NULL FROM consumed WHERE grant_id=?1", [row.grant_id.to_string()], |r| r.get(0), )?; safe &= !locked; } } let fault = recovery_reason(&decision.reason).then(|| decision.reason.clone()); ensure!( !decision.attempts.iter().any(|d| recovery_reason(&d.reason)), "SIGNED_LEDGER_RECOVERY_REQUIRED" ); let after = load_rows(&tx, &self.binding, true)?; let fingerprint = ledger_fingerprint(&tx)?; tx.execute("INSERT INTO helper_sync_decisions(decision_id,checkpoint,response,request,fingerprint) VALUES(?1,?2,?3,?4,?5)",params![decision.decision_id.to_string(),decision.helper_checkpoint,serde_json::to_vec(response)?,raw_request,fingerprint])?; tx.execute("UPDATE helper_sync_meta SET last_decision=?1,fault=COALESCE(fault,?2),agent_journal_id=?3,agent_checkpoint=?4 WHERE id=1",params![decision.decision_id.to_string(),fault,decision.journal_id.to_string(),decision.checkpoint])?; tx.commit()?; let ready = safe && fault.is_none() && decision.ready && decision.helper_ready && after.iter().all(definitive); if ready { self.lease = Some(Lease { checkpoint: decision.helper_checkpoint, decision_id: decision.decision_id, boot_id: current_boot, generation: capture.generation, until: decision.valid_until, fingerprint, }); } Ok(Applied { snapshot: Snapshot { journal_id: meta.journal_id, checkpoint: meta.checkpoint, previous_decision_id: Some(decision.decision_id), boot_id: current_boot, captured_at: capture.snapshot.captured_at, attempts: after, }, unlocked_grants: unlocked, ready, }) } /// Call under the same executor guard immediately BEFORE fresh grant /// consumption. Once that operation changes the ledger, resync is required. pub fn can_start( &self, db: &Connection, guard: &ExecutorGuard, generation: u64, clock: &impl Clock, boot: &impl BootSource, ) -> bool { let result = (|| -> Result { guard.check(db)?; let now = clock.now()?; let Some(lease) = &self.lease else { return Ok(false); }; let meta = metadata(db)?; if meta.fault.is_some() || meta.checkpoint != lease.checkpoint || meta.decision != Some(lease.decision_id) || generation != lease.generation { return Ok(false); } validate_history(db, &self.binding, &meta, generation)?; Ok(meta.fault.is_none() && meta.checkpoint == lease.checkpoint && meta.decision == Some(lease.decision_id) && generation == lease.generation && now < lease.until && now >= self.born && boot.current_boot()? == lease.boot_id && ledger_fingerprint(db)? == lease.fingerprint && db.query_row( "SELECT count(*) FROM consumed WHERE result IS NULL", [], |r| r.get::<_, i64>(0), )? == 0) })(); if let Err(error) = &result { persist_fault(db, error); } result.unwrap_or(false) } } fn pinned_key(binding: &Binding) -> Result> { let raw = hex::decode(&binding.public_key_hex)?; ensure!(raw.len() == 32, "PINNED_HELPER_KEY_SIZE"); PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?; Ok(raw) } fn verify( binding: &Binding, response: &SignedResponse, now: DateTime, ) -> Result { ensure!( response.signature.len() == 64 && response.decision_bytes.len() <= 512 * 1024, "SYNC_SIGNATURE_SIZE" ); let raw = pinned_key(binding)?; let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?; ensure!( Verifier::new_without_digest(&key)? .verify_oneshot(&response.signature, &response.decision_bytes)?, "INVALID_SYNC_SIGNATURE" ); let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; ensure!( d.request_hash.len() == 64 && d.request_hash .bytes() .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), "SIGNED_REQUEST_HASH_MISSING_OR_INVALID" ); i64::try_from(d.checkpoint)?; i64::try_from(d.helper_checkpoint)?; i64::try_from(d.connection_generation)?; for attempt in &d.attempts { i64::try_from(attempt.revision)?; } ensure!( d.valid_profile(now) && d.valid_until - d.issued_at <= chrono::Duration::seconds(60) && d.key_id == hash(&raw) && d.tenant_id == binding.tenant_id && d.asset_id == binding.asset_id && d.agent_id == binding.agent_id && d.enrollment_generation == binding.enrollment_generation && d.assignment_version == binding.assignment_version, "HELPER_DECISION_PROFILE_OR_IDENTITY" ); Ok(d) } struct Meta { journal_id: Uuid, checkpoint: u64, generation: u64, decision: Option, fault: Option, agent_journal: Option, agent_checkpoint: u64, } fn initialize(db: &Connection, binding: &Binding) -> Result<()> { let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; ensure!(check == "ok", "ROOT_LEDGER_INTEGRITY"); db.execute_batch("PRAGMA synchronous=FULL; CREATE TABLE IF NOT EXISTS helper_sync_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,binding_sha256 TEXT NOT NULL,last_decision TEXT,fault TEXT,agent_journal_id TEXT,agent_checkpoint INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS helper_sync_rows(grant_id TEXT PRIMARY KEY,revision INTEGER NOT NULL,fingerprint TEXT NOT NULL); CREATE TABLE IF NOT EXISTS helper_sync_captures(checkpoint INTEGER PRIMARY KEY,snapshot BLOB NOT NULL,fingerprint TEXT NOT NULL,captured_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS helper_sync_decisions(decision_id TEXT PRIMARY KEY,checkpoint INTEGER NOT NULL,response BLOB NOT NULL,request BLOB NOT NULL,fingerprint TEXT NOT NULL);")?; let binding_hash = hash(&serde_json::to_vec(binding)?); db.execute( "INSERT OR IGNORE INTO helper_sync_meta(id,journal_id,binding_sha256) VALUES(1,?1,?2)", params![Uuid::new_v4().to_string(), binding_hash], )?; let old: String = db.query_row( "SELECT binding_sha256 FROM helper_sync_meta WHERE id=1", [], |r| r.get(0), )?; ensure!( old == binding_hash, "INDEPENDENT_HELPER_BINDING_OR_KEY_CHANGED" ); Ok(()) } fn metadata(db: &Connection) -> Result { type MetaRow = ( String, u64, u64, Option, Option, Option, u64, ); let (id,checkpoint,generation,decision,fault,agent,agent_checkpoint):MetaRow=db.query_row("SELECT journal_id,checkpoint,generation,last_decision,fault,agent_journal_id,agent_checkpoint FROM helper_sync_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?)))?; Ok(Meta { journal_id: Uuid::parse_str(&id)?, checkpoint, generation, decision: decision.map(|s| Uuid::parse_str(&s)).transpose()?, fault, agent_journal: agent.map(|s| Uuid::parse_str(&s)).transpose()?, agent_checkpoint, }) } fn validate_history( db: &Connection, binding: &Binding, meta: &Meta, generation: u64, ) -> Result<()> { let highest: u64 = db.query_row( "SELECT COALESCE(MAX(checkpoint),0) FROM helper_sync_captures", [], |r| r.get(0), )?; ensure!( highest == meta.checkpoint, "HELPER_CAPTURE_HISTORY_ROLLBACK" ); let decision_count: u64 = db.query_row("SELECT count(*) FROM helper_sync_decisions", [], |r| { r.get(0) })?; if let Some(id) = meta.decision { let stored: Option> = db .query_row( "SELECT response FROM helper_sync_decisions WHERE decision_id=?1", [id.to_string()], |r| r.get(0), ) .optional()?; let response: SignedResponse = serde_json::from_slice(&stored.context("HELPER_DECISION_HISTORY_MISSING_LOCALLY")?)?; // Historical already-applied evidence is checked at its ORIGINAL issue // time; it never creates a lease or bypasses the fresh-process capture. let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; let previous = verify(binding, &response, parsed.issued_at)?; let highest_decision: u64 = db.query_row( "SELECT COALESCE(MAX(checkpoint),0) FROM helper_sync_decisions", [], |r| r.get(0), )?; ensure!( previous.decision_id == id && previous.helper_journal_id == Some(meta.journal_id) && previous.helper_checkpoint == highest_decision && previous.helper_checkpoint <= meta.checkpoint && generation >= previous.connection_generation && meta.agent_journal == Some(previous.journal_id) && meta.agent_checkpoint >= previous.checkpoint, "HELPER_DECISION_HISTORY_ROLLBACK" ); } else { ensure!( decision_count == 0 && meta.agent_journal.is_none() && meta.agent_checkpoint == 0, "HELPER_METADATA_HISTORY_ROLLBACK" ); } Ok(()) } /// Bootstrap-only historical verification; this never creates a lease. pub fn verify_checkpoint_repair_history( db: &Connection, binding: &Binding, links: Option<( &rmm_domain::checkpoint_repair::Receipt, &rmm_domain::checkpoint_repair::Evidence, )>, ) -> Result<()> { let meta = metadata(db)?; validate_history(db, binding, &meta, meta.generation)?; let mut statement = db.prepare( "SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint", )?; let entries = statement .query_map([], |r| { Ok(( r.get::<_, Vec>(0)?, r.get::<_, Vec>(1)?, r.get::<_, u64>(2)?, )) })? .collect::>>()?; ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT"); let mut previous = None; let mut previous_checkpoint = 0; let mut previous_agent_checkpoint = 0; for (signed, raw, checkpoint) in entries { let response: SignedResponse = serde_json::from_slice(&signed)?; let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; let decision = verify(binding, &response, parsed.issued_at)?; let request = RequestView::decode(&raw)?; let captured: Vec = db.query_row( "SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1", [checkpoint], |r| r.get(0), )?; let capture: Snapshot = serde_json::from_slice(&captured)?; ensure!( decision.helper_journal_id == Some(meta.journal_id) && decision.helper_checkpoint == checkpoint && request.helper_journal_id == meta.journal_id.to_string() && request.helper_checkpoint == checkpoint && request.assignment_version == binding.assignment_version && request.journal_complete && decision.request_hash == hash(&raw) && decision.request_id.to_string() == request.request_id && decision.journal_id.to_string() == request.journal_id && decision.checkpoint == request.checkpoint && capture.journal_id == meta.journal_id && capture.checkpoint == checkpoint && capture.boot_id == decision.boot_id && capture.captured_at <= decision.issued_at && capture.previous_decision_id == previous && checkpoint > previous_checkpoint && decision.checkpoint >= previous_agent_checkpoint && decision.connection_generation <= meta.generation && request.helper_previous_decision_id == capture .previous_decision_id .map(|v| v.to_string()) .unwrap_or_default(), "REPAIR_HELPER_HISTORY_BINDING" ); previous = Some(decision.decision_id); previous_checkpoint = checkpoint; previous_agent_checkpoint = decision.checkpoint; } let _ = load_rows(db, binding, false)?; if let Some((r, e)) = links { verify_checkpoint_repair_links(db, binding, r, e)?; } Ok(()) } fn verify_checkpoint_repair_links( db: &Connection, binding: &Binding, r: &rmm_domain::checkpoint_repair::Receipt, e: &rmm_domain::checkpoint_repair::Evidence, ) -> Result<()> { let x = RequestView::decode(&e.uncommitted_request)?; let y = RequestView::decode(&e.rejected_request)?; let a: ReconcileDecisionPayload = serde_json::from_slice(&e.previous_valid_response.decision_bytes)?; let a = verify( binding, &SignedResponse { decision_bytes: e.previous_valid_response.decision_bytes.clone(), signature: e.previous_valid_response.signature.clone(), }, a.issued_at, )?; let old: Vec = db.query_row( "SELECT response FROM helper_sync_decisions WHERE decision_id=?1", [a.decision_id.to_string()], |v| v.get(0), )?; let old: SignedResponse = serde_json::from_slice(&old)?; ensure!( old.decision_bytes == e.previous_valid_response.decision_bytes && old.signature == e.previous_valid_response.signature && a.decision_id == r.previous_valid_decision_id && hash(&old.decision_bytes) == r.previous_valid_decision_sha256 && a.reason == "ATTEMPT_RECONCILIATION_REQUIRED" && !a.ready && !a.helper_ready && x.request_id == r.uncommitted_request_id.to_string() && hash(&e.uncommitted_request) == r.uncommitted_request_sha256 && y.request_id == r.rejected_request_id.to_string() && hash(&e.rejected_request) == r.rejected_request_sha256 && x.previous_decision_id == a.decision_id.to_string() && x.previous_request_id.is_empty() && x.helper_previous_decision_id == a.decision_id.to_string() && y.helper_previous_decision_id == a.decision_id.to_string() && y.previous_decision_id.is_empty() && y.previous_request_id == x.request_id && x.journal_id == r.agent_journal_id.to_string() && y.journal_id == x.journal_id && a.journal_id == r.agent_journal_id && x.helper_journal_id == r.helper_journal_id.to_string() && y.helper_journal_id == x.helper_journal_id && a.helper_journal_id == Some(r.helper_journal_id) && a.checkpoint < x.checkpoint && x.checkpoint < y.checkpoint && y.checkpoint == r.agent_checkpoint && a.helper_checkpoint < x.helper_checkpoint && x.helper_checkpoint < y.helper_checkpoint && y.helper_checkpoint == r.helper_checkpoint && x.assignment_version == binding.assignment_version && y.assignment_version == binding.assignment_version && x.journal_complete && y.journal_complete, "REPAIR_EXACT_UNSENT_CHAIN_REQUIRED" ); Ok(()) } fn persist_fault(db: &Connection, error: &anyhow::Error) { let text = error.to_string(); if [ "OMITTED", "ROLLBACK", "LEDGER_CHANGED", "SNAPSHOT_CHANGED", "MISSING_LOCALLY", "ROOT_CONSUMPTION_MISSING", "UNKNOWN_LOCAL_STATE", "RECOVERY_REQUIRED", "BINDING_OR_KEY_CHANGED", "LOCAL_START_MISSING", "LOCAL_REQUEST_BINDING", "LOCAL_OUTCOME_CONFLICT", "LOCAL_SUCCESS_PROOF_MISSING", "SIGNED_REQUEST_HASH_MISSING_OR_INVALID", "ROOT_SIGNED_ATTEMPT_BINDING", ] .iter() .any(|s| text.contains(s)) { let _ = db.execute( "UPDATE helper_sync_meta SET fault=COALESCE(fault,?1) WHERE id=1", [text], ); } } fn recovery_reason(reason: &str) -> bool { reason.contains("ROLLBACK") || reason.contains("RECOVERY_REQUIRED") || matches!( reason, "MISSING_SERVER_JOB" | "ATTEMPT_BINDING_MISMATCH" | "ASSIGNMENT_MISMATCH" | "JOURNAL_SEQUENCE_REGRESSION" | "SERVER_CHECKPOINT_MISSING" | "CHECKPOINT_CHAIN_MISMATCH" | "JOURNAL_HISTORY_MISSING" | "LOCAL_ATTEMPT_MISSING" ) } type LedgerRow = ( String, String, u64, String, String, Option>, Option>, Option>, ); fn ledger_rows(db: &Connection) -> Result> { let mut statement=db.prepare("SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e ON e.grant_id=c.grant_id ORDER BY c.grant_id")?; Ok(statement .query_map([], |r| { Ok(( r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?, r.get(7)?, )) })? .collect::>()?) } fn ledger_fingerprint(db: &Connection) -> Result { Ok(hash(&serde_json::to_vec(&ledger_rows(db)?)?)) } fn canonical(bytes: &[u8]) -> Result> { let o: ReconciledOutcome = serde_json::from_slice(bytes)?; Ok(canonical_outcome_bytes(&o)?) } fn load_rows(db: &Connection, binding: &Binding, refresh: bool) -> Result> { let mut result = Vec::new(); for (grant, agent, generation, job_id, attempt_id, finished, start, last) in ledger_rows(db)? { let start = start.context("LOCAL_START_MISSING_POSSIBLE_EFFECT")?; let start: serde_json::Value = serde_json::from_slice(&start)?; let original_boot_id = Uuid::parse_str( start["previous_boot_id"] .as_str() .context("LOCAL_ORIGINAL_BOOT_MISSING")?, )?; let request: Request = serde_json::from_value(start["request"].clone())?; let permit: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?; // Historical verification checks the original valid grant window, NOT // authority to start now. Root consumption remains permanently recorded. let (job, verified) = binding.verify(&request, permit.not_before)?; ensure!( verified == permit && job.execution_identity == "root" && grant == permit.grant_id.to_string() && agent == binding.agent_id.to_string() && generation == binding.enrollment_generation && job_id == job.job_id.to_string() && attempt_id == job.attempt_id.to_string(), "LOCAL_REQUEST_BINDING" ); if let (Some(a), Some(b)) = (&finished, &last) { ensure!(canonical(a)? == canonical(b)?, "LOCAL_OUTCOME_CONFLICT"); } let outcome_bytes = last .as_deref() .or(finished.as_deref()) .map(canonical) .transpose()? .unwrap_or_default(); ensure!(outcome_bytes.len() <= 512 * 1024, "LOCAL_OUTCOME_SIZE"); let state = if outcome_bytes.is_empty() { "unknown".to_string() } else { let o: ReconciledOutcome = serde_json::from_slice(&outcome_bytes)?; o.state }; ensure!( matches!( state.as_str(), "unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired" ), "UNKNOWN_LOCAL_STATE" ); ensure!(!original_boot_id.is_nil(), "LOCAL_ORIGINAL_BOOT_MISSING"); if state == "succeeded" { let o: ReconciledOutcome = serde_json::from_slice(&outcome_bytes)?; ensure!( o.evidence["postconditions_passed"] == true, "LOCAL_SUCCESS_PROOF_MISSING" ); if let Action::RunScript { expected_stdout, .. } = &job.action { ensure!( o.exit_code == Some(0) && expected_stdout .as_ref() .is_none_or(|s| !o.output_truncated && o.stdout == s.as_bytes()), "LOCAL_SUCCESS_PROOF_MISSING" ); } } let mut row = LocalAttempt { job_id: job.job_id, attempt_id: job.attempt_id, job_payload_hash: hash(&request.job_bytes), grant_id: permit.grant_id, authorization_id: permit.authorization_id, plan_hash: permit.plan_hash, action: job.action.name().into(), execution_identity: job.execution_identity, revision: 0, original_boot_id, consumption: "consumed".into(), state, outcome_sha256: if outcome_bytes.is_empty() { String::new() } else { hash(&outcome_bytes) }, outcome_bytes, original_request_sha256: hash(&serde_json::to_vec(&request)?), }; let fingerprint = hash(&serde_json::to_vec(&row)?); let old: Option<(u64, String)> = db .query_row( "SELECT revision,fingerprint FROM helper_sync_rows WHERE grant_id=?1", [&grant], |r| Ok((r.get(0)?, r.get(1)?)), ) .optional()?; row.revision = match old { Some((revision, old)) => { if old == fingerprint { revision } else { ensure!(refresh, "ROOT_ROW_CHANGED_WITHOUT_SNAPSHOT"); revision.checked_add(1).context("REVISION_OVERFLOW")? } } None => { ensure!(refresh, "ROOT_ROW_MISSING_WITHOUT_SNAPSHOT"); 1 } }; i64::try_from(row.revision)?; if refresh { db.execute("INSERT INTO helper_sync_rows(grant_id,revision,fingerprint) VALUES(?1,?2,?3) ON CONFLICT(grant_id) DO UPDATE SET revision=excluded.revision,fingerprint=excluded.fingerprint",params![grant,row.revision,fingerprint])?; } result.push(row); } let retained: u64 = db.query_row("SELECT count(*) FROM helper_sync_rows", [], |r| r.get(0))?; ensure!( retained == result.len() as u64, "ROOT_CONSUMPTION_MISSING_LOCALLY" ); Ok(result) } fn definitive(row: &LocalAttempt) -> bool { matches!( row.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired" ) && !row.outcome_bytes.is_empty() } fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<()> { ensure!( a.job_payload_hash == row.job_payload_hash && a.grant_id == row.grant_id.to_string() && a.consumption == "consumed" && a.state == row.state && a.revision >= row.revision && a.original_boot_id == row.original_boot_id.to_string() && a.outcome_sha256 == row.outcome_sha256 && (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes), "ROOT_SNAPSHOT_REQUEST_MISMATCH" ); Ok(()) } fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -> Result<()> { ensure!( d.job_payload_hash == row.job_payload_hash && d.grant_id == Some(row.grant_id) && d.authorization_id == Some(row.authorization_id) && d.plan_hash.as_deref() == Some(row.plan_hash.as_str()) && d.action == row.action && d.execution_identity == "root" && d.original_boot_id == Some(row.original_boot_id) && (d.revision >= a.revision || (!d.unlock && d.outcome_state == "unknown")) && d.previous_outcome_sha256.as_deref() == (!row.outcome_sha256.is_empty()).then_some(row.outcome_sha256.as_str()), "ROOT_SIGNED_ATTEMPT_BINDING" ); ensure!( !d.unlock || matches!( d.outcome_state.as_str(), "succeeded" | "failed" | "cancelled" | "expired" ), "UNKNOWN_CANNOT_UNLOCK" ); Ok(()) } fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -> Result> { let bytes = if d.outcome_bytes.is_empty() { row.outcome_bytes.clone() } else { d.outcome_bytes.clone() }; let o: ReconciledOutcome = serde_json::from_slice(&bytes)?; ensure!( canonical_outcome_bytes(&o)? == bytes && d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()) && o.state == d.outcome_state, "ROOT_CANONICAL_UNLOCK_OUTCOME" ); if bytes != row.outcome_bytes { ensure!( row.action == "reboot" && matches!(row.state.as_str(), "unknown" | "awaiting_reboot") && o.state == "succeeded" && row.original_boot_id != current_boot && o.evidence["postconditions_passed"] == true && o.evidence["reconciled_after_reboot"] == true && o.evidence["new_boot_id"].as_str() == Some(current_boot.to_string().as_str()) && o.evidence["inventory_message_id"] .as_str() .is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())), "ROOT_REBOOT_NATIVE_BOOT_PROOF_REQUIRED" ); } else { ensure!(definitive(row), "UNKNOWN_LOCAL_EFFECT_CANNOT_UNLOCK"); if o.state == "succeeded" { ensure!( o.evidence["postconditions_passed"] == true, "ROOT_POSTCONDITION_MISSING" ); } if matches!(o.state.as_str(), "cancelled" | "expired") { ensure!( o.evidence["not_started"] == true || (o.state == "cancelled" && o.evidence["abort_confirmed"] == true), "ROOT_ABORT_NOT_PROVEN" ); } if o.state == "failed" { ensure!(!o.error_code.is_empty(), "ROOT_FAILURE_NOT_PROVEN"); } } Ok(bytes) } // The helper manifest deliberately has no prost/protocol dependency. This small // bounded parser reads the EXISTING SyncRequest tags (1..12) and SyncAttempt // tags (1..13), not a parallel public protocol. Glue passes the actual canonical // prost bytes. Unknown/duplicate fields, wrong wire types and overlong varints // fail closed. Nested JobResult/Envelope are not trusted as root outcome proof. #[derive(Default)] struct RequestView { request_id: String, journal_id: String, checkpoint: u64, assignment_version: u64, attempts: Vec, journal_complete: bool, helper_journal_id: String, helper_checkpoint: u64, helper_previous_decision_id: String, previous_decision_id: String, previous_request_id: String, } #[derive(Default)] struct AttemptView { job_id: String, attempt_id: String, job_payload_hash: String, grant_id: String, consumption: String, state: String, revision: u64, original_boot_id: String, outcome_sha256: String, outcome_bytes: Vec, } enum Field<'a> { Number(u64), Bytes(&'a [u8]), } fn varint(bytes: &[u8], offset: &mut usize) -> Result { let mut n = 0u64; for i in 0..10 { let b = *bytes.get(*offset).context("PROTOBUF_TRUNCATED")?; *offset += 1; ensure!(i != 9 || b <= 1, "PROTOBUF_VARINT_OVERFLOW"); n |= u64::from(b & 127) << (7 * i); if b & 128 == 0 { ensure!(i == 0 || b != 0, "PROTOBUF_NONCANONICAL_VARINT"); return Ok(n); } } anyhow::bail!("PROTOBUF_VARINT_OVERFLOW") } fn fields(bytes: &[u8]) -> Result)>> { let mut offset = 0; let mut result = Vec::new(); while offset < bytes.len() { ensure!(result.len() < 2048, "PROTOBUF_FIELD_LIMIT"); let tag = varint(bytes, &mut offset)?; let number = u32::try_from(tag >> 3)?; ensure!(number > 0, "PROTOBUF_ZERO_TAG"); let value = match tag & 7 { 0 => Field::Number(varint(bytes, &mut offset)?), 2 => { let len = usize::try_from(varint(bytes, &mut offset)?)?; let end = offset .checked_add(len) .context("PROTOBUF_LENGTH_OVERFLOW")?; let value = bytes.get(offset..end).context("PROTOBUF_TRUNCATED")?; offset = end; Field::Bytes(value) } _ => anyhow::bail!("PROTOBUF_WIRE_TYPE"), }; result.push((number, value)); } Ok(result) } fn text(field: Field<'_>) -> Result { match field { Field::Bytes(b) => { ensure!(b.len() <= 1024, "PROTOBUF_STRING_SIZE"); Ok(std::str::from_utf8(b)?.to_string()) } _ => anyhow::bail!("PROTOBUF_EXPECTED_STRING"), } } fn number(field: Field<'_>) -> Result { match field { Field::Number(n) => Ok(n), _ => anyhow::bail!("PROTOBUF_EXPECTED_NUMBER"), } } fn bytes(field: Field<'_>) -> Result> { match field { Field::Bytes(b) => Ok(b.to_vec()), _ => anyhow::bail!("PROTOBUF_EXPECTED_BYTES"), } } impl RequestView { fn decode(raw: &[u8]) -> Result { ensure!(raw.len() <= 1024 * 1024, "SYNC_REQUEST_SIZE"); let mut r = Self::default(); let mut seen = BTreeSet::new(); let mut receipts = 0usize; for (tag, value) in fields(raw)? { ensure!( (1..=12).contains(&tag) && (matches!(tag, 6 | 7) || seen.insert(tag)), "PROTOBUF_UNKNOWN_OR_DUPLICATE_FIELD" ); match tag { 1 => r.request_id = text(value)?, 2 => r.journal_id = text(value)?, 3 => r.checkpoint = number(value)?, 4 | 12 => { let s = text(value)?; if !s.is_empty() { Uuid::parse_str(&s)?; } if tag == 4 { r.previous_decision_id = s; } else { r.previous_request_id = s; } } 5 => r.assignment_version = number(value)?, 6 => { ensure!(r.attempts.len() < 256, "SYNC_ATTEMPT_LIMIT"); r.attempts.push(AttemptView::decode(&bytes(value)?)?); } 7 => { receipts += 1; ensure!(receipts <= 256, "SYNC_RECEIPT_LIMIT"); let s = text(value)?; ensure!(!Uuid::parse_str(&s)?.is_nil(), "SYNC_RECEIPT_ID"); } 8 => { let n = number(value)?; ensure!(n <= 1, "PROTOBUF_BOOL"); r.journal_complete = n == 1; } 9 => r.helper_journal_id = text(value)?, 10 => r.helper_checkpoint = number(value)?, 11 => r.helper_previous_decision_id = text(value)?, _ => unreachable!(), } } ensure!( !Uuid::parse_str(&r.request_id)?.is_nil() && !Uuid::parse_str(&r.journal_id)?.is_nil(), "SYNC_REQUEST_IDENTITIES" ); i64::try_from(r.checkpoint)?; i64::try_from(r.helper_checkpoint)?; Ok(r) } } impl AttemptView { fn decode(raw: &[u8]) -> Result { let mut r = Self::default(); let mut seen = BTreeSet::new(); for (tag, value) in fields(raw)? { ensure!( (1..=13).contains(&tag) && seen.insert(tag), "PROTOBUF_ATTEMPT_FIELD" ); match tag { 1 => r.job_id = text(value)?, 2 => r.attempt_id = text(value)?, 3 => r.job_payload_hash = text(value)?, 4 => r.grant_id = text(value)?, 5 => r.consumption = text(value)?, 6 => r.state = text(value)?, 7 => r.revision = number(value)?, 8 => r.original_boot_id = text(value)?, 9 | 13 => { ensure!(bytes(value)?.len() <= 1024 * 1024, "SYNC_RESULT_SIZE"); } 10 => { let _ = text(value)?; } 11 => r.outcome_sha256 = text(value)?, 12 => { r.outcome_bytes = bytes(value)?; ensure!(r.outcome_bytes.len() <= 512 * 1024, "SYNC_OUTCOME_SIZE"); } _ => unreachable!(), } } i64::try_from(r.revision)?; Ok(r) } } fn unique_summaries(rows: &[AttemptView]) -> Result> { let mut map = BTreeMap::new(); for row in rows { let id = ( Uuid::parse_str(&row.job_id)?, Uuid::parse_str(&row.attempt_id)?, ); ensure!( !id.0.is_nil() && !id.1.is_nil() && map.insert(id, row).is_none(), "DUPLICATE_REQUEST_ATTEMPT" ); } Ok(map) }