//! Reconnect reconciliation, deliberately independent of main/jobs and transport. //! No method executes a job, deletes consumption tombstones or issues a permit. //! Call only while holding the existing agent.lock. HelperSnapshot must come //! from the independently authenticated root-helper IPC, never from the server. use crate::auth::Binding; pub use crate::auth::HelperIdentity; use crate::crypto::{DefaultVerifier, Ed25519Verifier}; use anyhow::{Context, Result, ensure}; use chrono::{DateTime, Utc}; use prost::Message; use rmm_domain::{ jobs::{JobPayload, hash}, reconciliation::{ AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes, }, }; use rmm_protocol::{ Envelope, JobOffer, JobResult, Payload, SyncAttempt, SyncRequest, SyncResponse, }; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use std::collections::{BTreeMap, BTreeSet}; use uuid::Uuid; /// A fresh observation returned by the trusted local clock implementation, /// never an IPC DTO or an execution permit. Callers must not retain it as a lease. pub struct ClockReading { pub own_wall: DateTime, pub upper: DateTime, } pub trait SyncClock { fn now(&self) -> Result>; /// Existing providers retain their existing point-clock contract. Providers /// with an independently verified interval override this atomically. fn reading(&self) -> Result { let own_wall = self.now()?; Ok(ClockReading { own_wall, upper: own_wall, }) } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct Channel { pub connection_generation: u64, pub boot_id: Uuid, } #[derive(Clone, Debug)] pub struct HelperSnapshot { pub journal_id: Uuid, pub checkpoint: u64, pub previous_decision_id: Option, /// Complete consumed/unsettled summaries from the root-owned ledger. pub attempts: Vec, } #[derive(Clone)] pub struct PreparedSync { pub envelope: Envelope, pub request: SyncRequest, } /// Pass the ORIGINAL signed response to the helper. It must independently verify /// it, apply targeted unlocks and commit its checkpoint before acknowledging. #[derive(Clone)] pub struct HelperInstruction { pub response: SyncResponse, pub decision_id: Uuid, pub journal_id: Uuid, pub checkpoint: u64, } pub struct Applied { pub decision_id: Uuid, pub acknowledged_message_ids: Vec, pub helper_instruction: Option, pub reason: String, } struct Gate { channel: Channel, until: DateTime, ready: bool, helper_ready: bool, helper: Option<(Uuid, u64, Uuid)>, local_applied: bool, request: SyncRequest, response: SyncResponse, } pub struct ClientSync { binding: Binding, verifier: V, helper_identity: HelperIdentity, pending: Option, channel: Option, gate: Option, } impl ClientSync { /// Every new process starts closed, independently of any persisted ready bit. pub fn new(binding: Binding) -> Self { Self::with_verifier(binding, DefaultVerifier::default()) } } impl ClientSync { pub fn with_verifier(binding: Binding, verifier: V) -> Self { Self { binding, verifier, helper_identity: HelperIdentity::Root, pending: None, channel: None, gate: None, } } /// Configure one explicit helper identity; changing it discards all leases. pub fn with_helper_identity(mut self, identity: HelperIdentity) -> Self { self.close(); self.helper_identity = identity; self } /// Invoke on every authenticated/fenced ServerHello or transport loss. pub fn close(&mut self) { self.gate = None; self.channel = None; self.pending = None; } pub fn validity_deadline(&self) -> Option> { self.gate.as_ref().map(|gate| gate.until) } pub fn can_start(&self, channel: &Channel, root: bool, clock: &impl SyncClock) -> bool { let Ok(reading) = clock.reading() else { return false; }; self.gate.as_ref().is_some_and(|g| { g.channel == *channel && g.ready && g.local_applied && reading.own_wall < g.until && reading.upper < g.until && (!root || (g.helper_ready && g.helper.is_none())) }) } /// Only call with a fresh independently authenticated helper snapshot. There /// is no unsigned ready/unlock flag: the exact signed decision/checkpoint /// must already be durably applied by that helper. pub fn confirm_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> { let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?; let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?; ensure!( snapshot.journal_id == expected.0 && snapshot.checkpoint == expected.1 && snapshot.previous_decision_id == Some(expected.2), "HELPER_CHECKPOINT_NOT_APPLIED" ); ensure!( snapshot .attempts .iter() .all(|a| terminal(&a.state) || (a.consumption == "unconsumed" && a.state == "accepted")), "HELPER_EFFECT_UNRESOLVED" ); gate.helper = None; Ok(()) } /// Typed read-only ACK for exactly the own original update handoff. It /// clears only the control delivery acknowledgement; both start flags stay /// false and can_start remains false for ALL jobs, including this attempt. pub fn confirm_update_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> { let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?; let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?; ensure!( !gate.ready && !gate.helper_ready && snapshot.journal_id == expected.0 && snapshot.checkpoint == expected.1 && snapshot.previous_decision_id == Some(expected.2), "UPDATE_CONTROL_NOT_APPLIED" ); let updates: Vec<_> = snapshot .attempts .iter() .filter(|a| !a.update_handoff_bytes.is_empty()) .collect(); ensure!(updates.len() == 1, "UPDATE_CONTROL_EXACTLY_ONE_REQUIRED"); let local = updates[0]; let requested = gate .request .attempts .iter() .find(|a| a.job_id == local.job_id && a.attempt_id == local.attempt_id) .context("UPDATE_CONTROL_NOT_REQUESTED")?; ensure!( local.state == "running" && local.consumption == "consumed" && local.outcome_bytes.is_empty() && local.update_handoff_bytes == requested.update_handoff_bytes && local.grant_id == requested.grant_id && local.job_payload_hash == requested.job_payload_hash, "UPDATE_CONTROL_CHANGED" ); let decision: ReconcileDecisionPayload = serde_json::from_slice(&gate.response.decision_bytes)?; ensure!( decision .attempts .iter() .any(|d| d.job_id.to_string() == local.job_id && d.attempt_id.to_string() == local.attempt_id && d.update_inflight.is_some() && !d.unlock), "UPDATE_CONTROL_NOT_SIGNED" ); ensure!( snapshot .attempts .iter() .filter(|a| a.job_id != local.job_id || a.attempt_id != local.attempt_id) .all(|a| terminal(&a.state) || (a.state == "accepted" && a.consumption == "unconsumed")), "UPDATE_OTHER_EFFECT_UNRESOLVED" ); gate.helper = None; Ok(()) } pub fn prepare( &mut self, db: &mut Connection, ledger: &Connection, channel: Channel, helper: Option<&HelperSnapshot>, clock: &impl SyncClock, ) -> Result { let result = self.prepare_inner(db, ledger, channel, helper, clock); if let Err(error) = &result { let text = error.to_string(); if [ "ROLLBACK", "REGRESSION", "DISAPPEARED", "MISSING_FROM_AGENT_JOURNAL", "MISSING_FROM_HELPER_JOURNAL", ] .iter() .any(|s| text.contains(s)) { let _ = db.execute( "UPDATE reconcile_meta SET fault=COALESCE(fault,?1) WHERE id=1", [text], ); } } result } fn prepare_inner( &mut self, db: &mut Connection, ledger: &Connection, channel: Channel, helper: Option<&HelperSnapshot>, clock: &impl SyncClock, ) -> Result { self.close(); let now = clock.now()?; ensure!( channel.connection_generation > 0 && !channel.boot_id.is_nil(), "UNAUTHENTICATED_CHANNEL" ); initialize(db)?; let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; let meta = metadata(&tx)?; ensure!(meta.fault.is_none(), "SYNC_MANUAL_RECOVERY_REQUIRED"); let sequence: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?; ensure!( sequence >= meta.checkpoint && sequence >= meta.high_sequence, "LOCAL_CHECKPOINT_ROLLBACK" ); ensure!( channel.connection_generation >= meta.generation, "CONNECTION_GENERATION_REGRESSION" ); if let Some(h) = helper { ensure!( !h.journal_id.is_nil() && h.checkpoint <= i64::MAX as u64, "HELPER_CHECKPOINT" ); if let Some(old) = meta.helper_id { ensure!( old == h.journal_id && h.checkpoint >= meta.helper_checkpoint, "HELPER_JOURNAL_ROLLBACK" ); } } else { ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED"); } if let Some(h) = helper { ingest_helper( &tx, ledger, (&self.binding, &self.verifier), &channel, h, now, self.helper_identity, )?; } archive_events(&tx)?; tx.execute( "UPDATE job_attempts SET state='accepted' WHERE state='queued' AND grant_id=''", [], )?; // A possibly started attempt is NEVER returned to accepted. Preserve // revisions and results; only ambiguous no-result states become unknown. let inflight: Vec<_> = helper .into_iter() .flat_map(|h| h.attempts.iter()) .filter(|a| !a.update_handoff_bytes.is_empty()) .collect(); ensure!(inflight.len() <= 1, "UPDATE_MULTIPLE_INFLIGHT"); if let Some(update) = inflight.first() { tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT (job=?1 AND attempt=?2) AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",params![update.job_id,update.attempt_id])?; } else { tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?; } let mut attempts = attempts(&tx, ledger, &self.binding)?; if let Some(h) = helper { merge_helper(&mut attempts, h)?; } ensure!(attempts.len() <= 256, "SYNC_ATTEMPT_CAPACITY_NO_COMPACTION"); let mut ids = pending_ids(&tx)?; if let Some(update) = inflight.first() { let grant = Uuid::parse_str(&update.grant_id)?; ids.extend(crate::update_health_inventory::pending_ids(&tx, grant)?); ids.sort(); ids.dedup(); } ensure!(ids.len() <= 256, "SYNC_RECEIPT_CAPACITY"); let next: i64 = tx.query_row( "UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq", [], |r| r.get(0), )?; let request = SyncRequest { request_id: Uuid::new_v4().to_string(), journal_id: meta.journal_id.to_string(), checkpoint: u64::try_from(next)?, // A pending request may have committed with a lost reply, or may // never have reached the server. Preserve both witnesses: the // server resolves the pending request first and otherwise checks // this last applied decision against its authoritative head. previous_decision_id: meta.decision.map(|id| id.to_string()).unwrap_or_default(), assignment_version: self.binding.assignment_version, attempts, durable_message_ids: ids, journal_complete: true, helper_journal_id: helper.map(|h| h.journal_id.to_string()).unwrap_or_default(), helper_checkpoint: helper.map_or(0, |h| h.checkpoint), helper_previous_decision_id: helper .and_then(|h| h.previous_decision_id) .map(|id| id.to_string()) .unwrap_or_default(), previous_request_id: meta .pending_request .map(|id| id.to_string()) .unwrap_or_default(), }; let envelope = Envelope { protocol_major: 1, protocol_minor: 0, message_id: Uuid::new_v4().to_string(), agent_id: self.binding.agent_id.to_string(), enrollment_generation: self.binding.enrollment_generation, connection_generation: channel.connection_generation, boot_id: channel.boot_id.to_string(), sequence: u64::try_from(next)?, observed_at: now.timestamp_millis(), payload: Some(Payload::SyncRequest(request.clone())), }; ensure!( envelope.encoded_len() <= rmm_domain::MAX_MESSAGE, "SYNC_ENVELOPE_SIZE" ); tx.execute( "INSERT INTO reconcile_requests(id,request,envelope,generation) VALUES(?1,?2,?3,?4)", params![ request.request_id, request.encode_to_vec(), envelope.encode_to_vec(), channel.connection_generation ], )?; tx.execute("UPDATE reconcile_meta SET checkpoint=?1,high_sequence=?1,generation=?2,pending_request=?3 WHERE id=1",params![next,channel.connection_generation,request.request_id])?; tx.commit()?; let prepared = PreparedSync { envelope, request }; self.pending = Some(prepared.clone()); self.channel = Some(channel); Ok(prepared) } /// All changes, concrete ACKs, result archives and checkpoint chain commit /// together. A rejection closes only mutation authority; telemetry may run. pub fn apply( &mut self, db: &mut Connection, response: &SyncResponse, clock: &impl SyncClock, ) -> Result { if self.pending.is_none() { let result = self.replay_applied(db, response, clock); if result.is_err() { self.gate = None; } return result; } self.gate = None; let result = self.apply_inner(db, response, clock); if result.is_err() { self.gate = None; } result } fn replay_applied( &self, db: &Connection, response: &SyncResponse, clock: &impl SyncClock, ) -> Result { let gate = self.gate.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?; ensure!(gate.response == *response, "REPLAYED_DIFFERENT_DECISION"); let d = verify_response_with( &self.verifier, &self.binding, &gate.request, &gate.channel, response, clock.now()?, )?; let meta = metadata(db)?; let sequence: i64 = db.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?; ensure!( meta.decision == Some(d.decision_id) && meta.pending_request.is_none() && meta.checkpoint == i64::try_from(d.checkpoint)? && sequence >= meta.high_sequence, "APPLIED_CHECKPOINT_REGRESSION" ); Ok(Applied { decision_id: d.decision_id, acknowledged_message_ids: d.acknowledged_message_ids, helper_instruction: d.helper_journal_id.map(|journal_id| HelperInstruction { response: response.clone(), decision_id: d.decision_id, journal_id, checkpoint: d.helper_checkpoint, }), reason: d.reason, }) } fn apply_inner( &mut self, db: &mut Connection, response: &SyncResponse, clock: &impl SyncClock, ) -> Result { let now = clock.now()?; let prepared = self.pending.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?; let channel = self.channel.as_ref().context("NO_AUTHENTICATED_CHANNEL")?; let decision = verify_response_with( &self.verifier, &self.binding, &prepared.request, channel, response, now, )?; let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; let meta = metadata(&tx)?; ensure!( meta.fault.is_none() && meta.pending_request == Some(decision.request_id) && meta.checkpoint == i64::try_from(decision.checkpoint)? && meta.generation == channel.connection_generation, "SYNC_CHECKPOINT_CHANGED" ); let seq: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?; ensure!( seq >= meta.high_sequence && seq >= meta.checkpoint, "LOCAL_SEQUENCE_ROLLBACK" ); let current = tx.query_row( "SELECT request FROM reconcile_requests WHERE id=?1", [decision.request_id.to_string()], |r| r.get::<_, Vec>(0), )?; ensure!( current == prepared.request.encode_to_vec(), "DURABLE_CHALLENGE_CHANGED" ); let summaries: BTreeMap<_, _> = prepared .request .attempts .iter() .map(|a| ((a.job_id.clone(), a.attempt_id.clone()), a)) .collect(); let actual: BTreeSet<_> = decision .attempts .iter() .map(|a| (a.job_id.to_string(), a.attempt_id.to_string())) .collect(); ensure!( actual.len() == decision.attempts.len() && actual == summaries.keys().cloned().collect(), "SYNC_ATTEMPT_OMITTED_OR_ADDED" ); let mut safe = true; let mut fault = recovery_reason(&decision.reason).then(|| decision.reason.clone()); for d in &decision.attempts { let local = summaries .get(&(d.job_id.to_string(), d.attempt_id.to_string())) .context("LOCAL_ATTEMPT_MISSING")?; if recovery_reason(&d.reason) { fault = Some(d.reason.clone()); } let offer_bytes: Vec = tx.query_row( "SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2", params![local.job_id, local.attempt_id], |r| r.get(0), )?; let offer = JobOffer::decode(offer_bytes.as_slice())?; let job = self.binding.job(&offer.payload_bytes)?; // Missing-server decisions intentionally have empty action fields; // persist their sticky denial without treating it as start authority. if recovery_reason(&d.reason) { safe = false; continue; } let outcome = validate_attempt(&job, &offer, local, d, channel.boot_id, &self.binding, now)?; safe &= (d.outcome_state == "accepted" && local.consumption == "unconsumed" && local.grant_id.is_empty()) || (terminal(&d.outcome_state) && (local.consumption == "unconsumed" || d.unlock)); let (stored_state, revision): (String, u64) = tx.query_row( "SELECT state,revision FROM job_attempts WHERE job=?1 AND attempt=?2", params![local.job_id, local.attempt_id], |r| Ok((r.get(0)?, r.get(1)?)), )?; ensure!( revision == local.revision && stored_state == local.state, "LOCAL_ATTEMPT_CHANGED_DURING_SYNC" ); let stored = if d.outcome_state == "unknown" && local.state == "awaiting_reboot" { "awaiting_reboot" } else { d.outcome_state.as_str() }; tx.execute( "UPDATE job_attempts SET state=?3,revision=?4 WHERE job=?1 AND attempt=?2", params![ local.job_id, local.attempt_id, stored, d.revision.max(local.revision) ], )?; if let Some(bytes) = outcome { tx.execute("INSERT INTO reconcile_outcomes(job,attempt,outcome,decision_id) VALUES(?1,?2,?3,?4) ON CONFLICT(job,attempt) DO UPDATE SET outcome=excluded.outcome,decision_id=excluded.decision_id",params![local.job_id,local.attempt_id,bytes,decision.decision_id.to_string()])?; } } let requested: BTreeSet<_> = prepared .request .durable_message_ids .iter() .map(String::as_str) .collect(); let mut receipts = BTreeSet::new(); for id in &decision.acknowledged_message_ids { ensure!( receipts.insert(*id) && requested.contains(id.to_string().as_str()), "ACK_NOT_IN_REQUEST" ); archive_events(&tx)?; tx.execute("DELETE FROM job_events WHERE id=?1", [id.to_string()])?; tx.execute( "INSERT OR IGNORE INTO reconcile_receipts(id,decision_id) VALUES(?1,?2)", params![id.to_string(), decision.decision_id.to_string()], )?; } tx.execute( "UPDATE reconcile_requests SET response=?2 WHERE id=?1 AND response IS NULL", params![decision.request_id.to_string(), response.encode_to_vec()], )?; for local in prepared .request .attempts .iter() .filter(|a| !a.update_handoff_bytes.is_empty()) { crate::update_health_inventory::acknowledge_signed( &tx, Uuid::parse_str(&local.grant_id)?, &decision, response, )?; } tx.execute("UPDATE reconcile_meta SET last_decision=?1,pending_request=NULL,helper_id=?2,helper_checkpoint=?3,fault=COALESCE(fault,?4),high_sequence=?5 WHERE id=1", params![decision.decision_id.to_string(),decision.helper_journal_id.map(|id|id.to_string()),decision.helper_checkpoint,fault,seq])?; tx.commit()?; let helper_instruction = decision .helper_journal_id .map(|journal_id| HelperInstruction { response: response.clone(), decision_id: decision.decision_id, journal_id, checkpoint: decision.helper_checkpoint, }); self.gate = Some(Gate { channel: channel.clone(), until: decision.valid_until, ready: decision.ready && safe && fault.is_none(), helper_ready: decision.helper_ready, helper: helper_instruction .as_ref() .map(|h| (h.journal_id, h.checkpoint, h.decision_id)), local_applied: !decision.attempts.iter().any(|d| d.unlock), request: prepared.request.clone(), response: response.clone(), }); self.pending = None; Ok(Applied { decision_id: decision.decision_id, acknowledged_message_ids: decision.acknowledged_message_ids, helper_instruction, reason: decision.reason, }) } /// Apply ONLY this process's independently owned consumption ledger. Root /// helper unlocks use HelperInstruction over authenticated IPC instead. /// A crash between journal and ledger transactions leaves the gate closed; /// resync/reapply never deletes the consumed attempt or repeats an effect. pub fn apply_consumption( &mut self, ledger: &mut Connection, clock: &impl SyncClock, ) -> Result<()> { let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?; gate.local_applied = false; let decision = verify_response_with( &self.verifier, &self.binding, &gate.request, &gate.channel, &gate.response, clock.now()?, )?; let tx = ledger.transaction_with_behavior(TransactionBehavior::Immediate)?; tx.execute_batch("CREATE TABLE IF NOT EXISTS reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB NOT NULL);")?; let applied: Option> = tx .query_row( "SELECT response FROM reconcile_consumption WHERE decision_id=?1", [decision.decision_id.to_string()], |r| r.get(0), ) .optional()?; if let Some(old) = applied { ensure!( old == gate.response.encode_to_vec(), "CONSUMPTION_DECISION_CONFLICT" ); } else { for d in decision.attempts.iter().filter(|d| d.unlock) { let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?; let (agent,generation,job,attempt,previous):(String,u64,String,String,Option>)=tx.query_row( "SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1", [grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?; ensure!( agent == self.binding.agent_id.to_string() && generation == self.binding.enrollment_generation && job == d.job_id.to_string() && attempt == d.attempt_id.to_string(), "LOCAL_UNLOCK_CONSUMPTION_BINDING" ); let previous = previous.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?; let prior = canonical_outcome_bytes(&serde_json::from_slice::( &previous, )?)?; ensure!( d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()), "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH" ); let bytes = if d.outcome_bytes.is_empty() { prior } else { d.outcome_bytes.clone() }; let outcome: ReconciledOutcome = serde_json::from_slice(&bytes)?; ensure!( terminal(&outcome.state) && outcome.state == d.outcome_state && canonical_outcome_bytes(&outcome)? == bytes && d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()), "LOCAL_UNLOCK_CANONICAL_OUTCOME" ); tx.execute( "UPDATE consumed SET result=?2 WHERE grant_id=?1", params![grant.to_string(), bytes], )?; tx.execute( "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1", params![grant.to_string(), bytes], )?; } tx.execute( "INSERT INTO reconcile_consumption(decision_id,response) VALUES(?1,?2)", params![ decision.decision_id.to_string(), gate.response.encode_to_vec() ], )?; } tx.commit()?; gate.local_applied = true; Ok(()) } } pub fn initialize(db: &Connection) -> Result<()> { let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; ensure!(check == "ok", "JOURNAL_INTEGRITY"); db.execute_batch("PRAGMA synchronous=FULL; CREATE TABLE IF NOT EXISTS reconcile_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,high_sequence INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,last_decision TEXT,pending_request TEXT,helper_id TEXT,helper_checkpoint INTEGER NOT NULL DEFAULT 0,fault TEXT); CREATE TABLE IF NOT EXISTS reconcile_requests(id TEXT PRIMARY KEY,request BLOB NOT NULL,envelope BLOB NOT NULL,generation INTEGER NOT NULL,response BLOB); CREATE TABLE IF NOT EXISTS reconcile_results(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,revision INTEGER NOT NULL,envelope BLOB NOT NULL); CREATE TABLE IF NOT EXISTS reconcile_receipts(id TEXT PRIMARY KEY,decision_id TEXT NOT NULL); CREATE TABLE IF NOT EXISTS reconcile_outcomes(job TEXT NOT NULL,attempt TEXT NOT NULL,outcome BLOB NOT NULL,decision_id TEXT NOT NULL,PRIMARY KEY(job,attempt));")?; crate::update_health_inventory::initialize(db)?; db.execute( "INSERT OR IGNORE INTO reconcile_meta(id,journal_id) VALUES(1,?1)", [Uuid::new_v4().to_string()], )?; Ok(()) } struct Meta { journal_id: Uuid, checkpoint: i64, high_sequence: i64, generation: u64, decision: Option, pending_request: Option, helper_id: Option, helper_checkpoint: u64, fault: Option, } type MetadataRow = ( String, i64, i64, u64, Option, Option, Option, u64, Option, ); fn metadata(db: &Connection) -> Result { let row:MetadataRow=db.query_row("SELECT journal_id,checkpoint,high_sequence,generation,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?)))?; Ok(Meta { journal_id: Uuid::parse_str(&row.0)?, checkpoint: row.1, high_sequence: row.2, generation: row.3, decision: row.4.map(|s| Uuid::parse_str(&s)).transpose()?, pending_request: row.5.map(|s| Uuid::parse_str(&s)).transpose()?, helper_id: row.6.map(|s| Uuid::parse_str(&s)).transpose()?, helper_checkpoint: row.7, fault: row.8, }) } /// Call inside the same transaction BEFORE deleting an ACKed job event. This /// retains the original result identity/output for future reconciliation. pub fn archive_events(db: &Connection) -> Result<()> { let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY rowid")?; for row in statement.query_map([], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, Vec>(1)?)) })? { let (id, bytes) = row?; let envelope = Envelope::decode(bytes.as_slice())?; ensure!(id == envelope.message_id, "EVENT_IDENTITY_CONFLICT"); if let Some(Payload::JobResult(r)) = &envelope.payload { let existing: Option> = db .query_row( "SELECT envelope FROM reconcile_results WHERE id=?1", [&id], |r| r.get(0), ) .optional()?; ensure!( existing.as_ref().is_none_or(|b| *b == bytes), "RESULT_IDENTITY_CONFLICT" ); db.execute("INSERT OR IGNORE INTO reconcile_results(id,job,attempt,revision,envelope) VALUES(?1,?2,?3,?4,?5)",params![id,r.job_id,r.attempt_id,r.revision,bytes])?; } } Ok(()) } fn pending_ids(db: &Connection) -> Result> { let mut statement = db.prepare("SELECT id FROM job_events ORDER BY id")?; let ids = statement .query_map([], |r| r.get::<_, String>(0))? .collect::>>()?; for id in &ids { ensure!(!Uuid::parse_str(id)?.is_nil(), "INVALID_DURABLE_MESSAGE_ID"); } Ok(ids) } fn outcome(r: &JobResult) -> Result { Ok(ReconciledOutcome { state: r.state.clone(), stdout: r.stdout.clone(), stderr: r.stderr.clone(), exit_code: r.exit_code, output_truncated: r.output_truncated, reboot_required: r.reboot_required, evidence: serde_json::from_str(&r.evidence_json)?, error_code: r.error_code.clone(), }) } fn terminal(state: &str) -> bool { matches!(state, "succeeded" | "failed" | "cancelled" | "expired") } fn optional_uuid(s: &str) -> Result> { if s.is_empty() { Ok(None) } else { let id = Uuid::parse_str(s)?; ensure!(!id.is_nil(), "NIL_ID"); Ok(Some(id)) } } fn recovery_reason(reason: &str) -> bool { matches!( reason, "MISSING_SERVER_JOB" | "ATTEMPT_BINDING_MISMATCH" | "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED" | "SERVER_STATE_ROLLBACK" | "ASSIGNMENT_MISMATCH" | "JOURNAL_SEQUENCE_REGRESSION" | "SERVER_CHECKPOINT_MISSING" | "JOURNAL_ROLLBACK_OR_REPLACEMENT" | "CHECKPOINT_CHAIN_MISMATCH" | "HELPER_JOURNAL_ROLLBACK" | "HELPER_CHECKPOINT_MISMATCH" | "HELPER_CHECKPOINT_MISSING" | "HELPER_CHECKPOINT_REQUIRED" | "JOURNAL_HISTORY_MISSING" | "LOCAL_ATTEMPT_MISSING" ) } fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result> { let mut statement = db.prepare("SELECT offer,state,revision,grant_id FROM job_attempts ORDER BY job,attempt")?; let rows = statement.query_map([], |r| { Ok(( r.get::<_, Vec>(0)?, r.get::<_, String>(1)?, r.get::<_, u64>(2)?, r.get::<_, String>(3)?, )) })?; let mut summaries = Vec::new(); let mut known = BTreeSet::new(); for row in rows { let (bytes, state, revision, mut grant_id) = row?; let offer = JobOffer::decode(bytes.as_slice())?; let job = binding.job(&offer.payload_bytes)?; ensure!( offer.job_id == job.job_id.to_string() && offer.attempt_id == job.attempt_id.to_string() && offer.job_payload_hash == hash(&offer.payload_bytes), "LOCAL_OFFER_BINDING" ); known.insert((offer.job_id.clone(), offer.attempt_id.clone())); let consumed: Option<(String, String, u64, Option>)> = ledger .query_row( "SELECT grant_id,agent,generation,result FROM consumed WHERE job=?1 AND attempt=?2", params![offer.job_id, offer.attempt_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), ) .optional()?; // A signed definitive nonstart can make an accepted no-grant row // terminal without a JobResult. Preserve that fact on the next Sync; // requesting/running/unknown still cannot claim non-consumption. An // actual durable consumption row below always overrides this label. let mut consumption = if grant_id.is_empty() && matches!(state.as_str(), "accepted" | "expired" | "cancelled") { "unconsumed" } else { "uncertain" }; let mut original_boot_id = String::new(); let mut canonical = Vec::new(); if let Some((id, agent, generation, result)) = consumed { ensure!( agent == binding.agent_id.to_string() && generation == binding.enrollment_generation && (grant_id.is_empty() || grant_id == id), "CONSUMPTION_BINDING" ); grant_id = id; consumption = "consumed"; let evidence: Option<(Vec, Option>)> = ledger .query_row( "SELECT start,outcome FROM execution_evidence WHERE grant_id=?1", [&grant_id], |r| Ok((r.get(0)?, r.get(1)?)), ) .optional()?; let mut stored = result; if let Some((start, last)) = evidence { let start: serde_json::Value = serde_json::from_slice(&start)?; original_boot_id = start["previous_boot_id"] .as_str() .context("ORIGINAL_BOOT_MISSING")? .to_string(); optional_uuid(&original_boot_id)?; let request: crate::auth::Request = serde_json::from_value(start["request"].clone())?; ensure!( request.job_bytes == offer.payload_bytes && request.plan_hash == offer.plan_hash && request.authorization_id.to_string() == offer.authorization_id, "START_EVIDENCE_BINDING" ); if last.is_some() { stored = last; } } if let Some(stored) = stored { let o: ReconciledOutcome = serde_json::from_slice(&stored)?; canonical = canonical_outcome_bytes(&o)?; } } let original:Option>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",params![offer.job_id,offer.attempt_id],|r|r.get(0)).optional()?; let mut result_envelope = None; let mut result = None; let mut result_message_id = String::new(); if let Some(bytes) = original { let envelope = Envelope::decode(bytes.as_slice())?; let Some(Payload::JobResult(r)) = &envelope.payload else { anyhow::bail!("INVALID_ARCHIVED_RESULT") }; if r.revision == revision && r.state == state { ensure!( envelope.agent_id == binding.agent_id.to_string() && envelope.enrollment_generation == binding.enrollment_generation && r.grant_id == grant_id, "RESULT_BINDING" ); let bytes = canonical_outcome_bytes(&outcome(r)?)?; ensure!( canonical.is_empty() || canonical == bytes, "LOCAL_OUTCOME_LEDGER_MISMATCH" ); canonical = bytes; result_message_id = envelope.message_id.clone(); result = Some(r.clone()); result_envelope = Some(envelope); } } if !canonical.is_empty() { let o: ReconciledOutcome = serde_json::from_slice(&canonical)?; ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH"); ensure!(canonical.len() <= 512 * 1024, "SYNC_CANONICAL_OUTCOME_SIZE"); if grant_id.is_empty() && terminal(&state) && o.evidence["not_started"] == true { consumption = "unconsumed"; } } summaries.push(SyncAttempt { update_handoff_bytes: Vec::new(), job_id: offer.job_id, attempt_id: offer.attempt_id, job_payload_hash: offer.job_payload_hash, grant_id, consumption: consumption.into(), state, revision, original_boot_id, result, result_message_id, outcome_sha256: if canonical.is_empty() { String::new() } else { hash(&canonical) }, outcome_bytes: canonical, result_envelope, }); } let mut statement = ledger.prepare("SELECT job,attempt FROM consumed")?; for row in statement.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { ensure!( known.contains(&row?), "CONSUMED_ATTEMPT_MISSING_FROM_AGENT_JOURNAL" ); } Ok(summaries) } fn ingest_helper( db: &Connection, ledger: &Connection, binding_and_verifier: (&Binding, &impl Ed25519Verifier), channel: &Channel, helper: &HelperSnapshot, now: DateTime, identity: HelperIdentity, ) -> Result<()> { let (binding, verifier) = binding_and_verifier; let mut statement = db.prepare("SELECT job,attempt,offer FROM job_attempts")?; for row in statement.query_map([], |r| { Ok(( r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Vec>(2)?, )) })? { let (job, attempt, bytes) = row?; let offer = JobOffer::decode(bytes.as_slice())?; if binding.job(&offer.payload_bytes)?.execution_identity == identity.as_str() { let consumed: bool = ledger.query_row( "SELECT EXISTS(SELECT 1 FROM consumed WHERE job=?1 AND attempt=?2)", params![job, attempt], |r| r.get(0), )?; ensure!( !consumed || helper .attempts .iter() .any(|a| a.job_id == job && a.attempt_id == attempt), "ROOT_CONSUMED_ATTEMPT_MISSING_FROM_HELPER_JOURNAL" ); } } let mut seen = BTreeSet::new(); for h in &helper.attempts { ensure!( seen.insert((&h.job_id, &h.attempt_id)), "DUPLICATE_HELPER_ATTEMPT" ); let row: Option<(Vec, u64, String)> = db .query_row( "SELECT offer,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), ) .optional()?; let (bytes, revision, grant) = row.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?; let offer = JobOffer::decode(bytes.as_slice())?; let job = binding.job(&offer.payload_bytes)?; ensure!( job.execution_identity == identity.as_str() && h.job_payload_hash == offer.job_payload_hash && (grant.is_empty() || grant == h.grant_id), "HELPER_ATTEMPT_BINDING" ); if h.outcome_bytes.is_empty() { if !h.update_handoff_bytes.is_empty() { let header = validate_update_summary(binding, &job, &offer, h, channel.boot_id, now)?; ensure!( header.helper_journal_id == helper.journal_id && header.agent_journal_id == metadata(db)?.journal_id, "UPDATE_HELPER_JOURNAL_BINDING" ); let state: String = db.query_row( "SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id], |r| r.get(0), )?; ensure!( matches!( state.as_str(), "accepted" | "requesting" | "running" | "verifying" ), "UPDATE_CANNOT_HEAL_UNKNOWN" ); type OriginalEvidence = ( String, u64, String, String, Vec, Option>, Option>, ); let original:Option=ledger.query_row( "SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM execution_evidence e JOIN consumed c USING(grant_id) WHERE grant_id=?1",[&h.grant_id], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?; let (agent, generation, job_id, attempt_id, start, outcome, result) = original.context("UPDATE_LOCAL_ORIGINAL_MISSING")?; ensure!( agent == binding.agent_id.to_string() && generation == binding.enrollment_generation && job_id == h.job_id && attempt_id == h.attempt_id, "UPDATE_LOCAL_CONSUMPTION_BINDING" ); ensure!( outcome.is_none() && result.is_none(), "UPDATE_CANNOT_REPLACE_OUTCOME" ); let start: serde_json::Value = serde_json::from_slice(&start)?; let request: crate::auth::Request = serde_json::from_value(start["request"].clone())?; let permit: rmm_domain::jobs::ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?; binding.verify_with(&request, permit.not_before, verifier)?; ensure!( hash(&serde_json::to_vec(&request)?) == header.attempt.original_request_sha256 && permit.grant_id == header.attempt.grant_id && permit.agent_update_plan_id == Some(header.attempt.plan_id), "UPDATE_LOCAL_ORIGINAL_CHANGED" ); db.execute("UPDATE job_attempts SET state='running',revision=MAX(revision,?3) WHERE job=?1 AND attempt=?2",params![h.job_id,h.attempt_id,h.revision])?; } continue; } let o: ReconciledOutcome = serde_json::from_slice(&h.outcome_bytes)?; ensure!( canonical_outcome_bytes(&o)? == h.outcome_bytes && hash(&h.outcome_bytes) == h.outcome_sha256 && o.state == h.state && (terminal(&o.state) || matches!(o.state.as_str(), "unknown" | "awaiting_reboot")) && h.consumption == "consumed" && optional_uuid(&h.grant_id)?.is_some(), "HELPER_CANONICAL_OUTCOME" ); let existing: Option> = ledger .query_row( "SELECT outcome FROM execution_evidence WHERE grant_id=?1", [&h.grant_id], |r| r.get(0), ) .optional()? .flatten(); let same = existing .as_ref() .map(|b| { serde_json::from_slice::(b) .and_then(|o| canonical_outcome_bytes(&o)) }) .transpose()? .is_some_and(|b| b == h.outcome_bytes); let state: String = db.query_row( "SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id], |r| r.get(0), )?; if same && state == h.state { continue; } let consumed: Option<(String, u64, String, String)> = ledger .query_row( "SELECT agent,generation,job,attempt FROM consumed WHERE grant_id=?1", [&h.grant_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), ) .optional()?; ensure!( consumed == Some(( binding.agent_id.to_string(), binding.enrollment_generation, h.job_id.clone(), h.attempt_id.clone() )), "HELPER_CONSUMPTION_NOT_KNOWN_LOCALLY" ); // Store actual helper evidence without unlocking result/consumption. A // crash here still leaves both gates closed and the original helper // result available; the next sync imports it again idempotently. ensure!( ledger.execute( "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1", params![h.grant_id, h.outcome_bytes] )? == 1, "LOCAL_START_EVIDENCE_MISSING" ); let next_revision = revision .max(h.revision) .checked_add(1) .context("REVISION_OVERFLOW")?; i64::try_from(next_revision)?; let seq: i64 = db.query_row( "UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq", [], |r| r.get(0), )?; let result = JobResult { job_id: h.job_id.clone(), attempt_id: h.attempt_id.clone(), revision: next_revision, state: o.state.clone(), grant_id: h.grant_id.clone(), stdout: o.stdout, stderr: o.stderr, output_truncated: o.output_truncated, reboot_required: o.reboot_required, exit_code: o.exit_code, evidence_json: o.evidence.to_string(), error_code: o.error_code, }; let envelope = Envelope { protocol_major: 1, protocol_minor: 0, message_id: Uuid::new_v4().to_string(), agent_id: binding.agent_id.to_string(), enrollment_generation: binding.enrollment_generation, connection_generation: channel.connection_generation, boot_id: channel.boot_id.to_string(), sequence: u64::try_from(seq)?, observed_at: now.timestamp_millis(), payload: Some(Payload::JobResult(result)), }; ensure!( envelope.encoded_len() <= rmm_domain::MAX_MESSAGE, "HELPER_RESULT_SIZE" ); db.execute( "INSERT INTO job_events(id,payload) VALUES(?1,?2)", params![envelope.message_id, envelope.encode_to_vec()], )?; db.execute( "UPDATE job_attempts SET state=?3,revision=?4,grant_id=?5 WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id, o.state, next_revision, h.grant_id], )?; } Ok(()) } fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()> { let mut seen = BTreeSet::new(); for h in &helper.attempts { ensure!( seen.insert((&h.job_id, &h.attempt_id)), "DUPLICATE_HELPER_ATTEMPT" ); let a = local .iter_mut() .find(|a| a.job_id == h.job_id && a.attempt_id == h.attempt_id) .context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?; ensure!( a.job_payload_hash == h.job_payload_hash && (a.grant_id.is_empty() || a.grant_id == h.grant_id), "HELPER_ATTEMPT_BINDING" ); if h.consumption != "unconsumed" { if !h.update_handoff_bytes.is_empty() { ensure!( h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(), "UPDATE_CANNOT_HEAL_UNKNOWN" ); a.update_handoff_bytes.clone_from(&h.update_handoff_bytes); } a.grant_id = h.grant_id.clone(); a.consumption = h.consumption.clone(); ensure!( a.outcome_sha256 == h.outcome_sha256, "HELPER_OUTCOME_MISMATCH" ); if a.original_boot_id.is_empty() { a.original_boot_id = h.original_boot_id.clone(); } ensure!( a.original_boot_id == h.original_boot_id, "HELPER_ORIGINAL_BOOT_MISMATCH" ); } } Ok(()) } fn validate_update_summary( _binding: &Binding, job: &JobPayload, offer: &JobOffer, local: &SyncAttempt, boot: Uuid, now: DateTime, ) -> Result { let proof = rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes) .map_err(anyhow::Error::msg)?; let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?; let (h, _) = rmm_domain::agent_update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?; let now = u64::try_from(now.timestamp())?; ensure!( rmm_domain::agent_update::handoff_matches_job(&h, job) && h.boot_id == boot && h.attempt.grant_id.to_string() == local.grant_id && h.attempt.authorization_id.to_string() == offer.authorization_id && h.attempt.plan_sha256 == offer.plan_hash && h.attempt.job_payload_sha256 == offer.job_payload_hash && local.original_boot_id == boot.to_string() && local.consumption == "consumed" && local.state == "running" && local.outcome_bytes.is_empty() && local.outcome_sha256.is_empty() && local.result.is_none() && local.result_envelope.is_none() && h.consumed_at <= now && now < h.finish_before, "UPDATE_INFLIGHT_SUMMARY_BINDING" ); Ok(h) } pub fn verify_response( binding: &Binding, request: &SyncRequest, channel: &Channel, response: &SyncResponse, now: DateTime, ) -> Result { verify_response_with( &DefaultVerifier::default(), binding, request, channel, response, now, ) } pub fn verify_response_with( verifier: &impl Ed25519Verifier, binding: &Binding, request: &SyncRequest, channel: &Channel, response: &SyncResponse, now: DateTime, ) -> Result { ensure!( response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64, "SYNC_SIGNATURE_SIZE" ); let raw = decode_key(&binding.public_key_hex)?; ensure!( verifier.verify(&raw, &response.decision_bytes, &response.signature)?, "INVALID_SYNC_SIGNATURE" ); let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; ensure!( d.valid_profile(now) && d.valid_until - d.issued_at <= chrono::Duration::seconds(60) && d.key_id == hash(&raw), "SYNC_PROFILE_OR_CLOCK" ); ensure!( d.tenant_id == binding.tenant_id && d.asset_id == binding.asset_id && d.agent_id == binding.agent_id && d.enrollment_generation == binding.enrollment_generation && d.assignment_version == binding.assignment_version && d.connection_generation == channel.connection_generation && d.boot_id == channel.boot_id && d.request_id.to_string() == request.request_id && d.request_hash == hash(&request.encode_to_vec()) && d.journal_id.to_string() == request.journal_id && d.checkpoint == request.checkpoint && d.helper_journal_id == optional_uuid(&request.helper_journal_id)? && d.helper_checkpoint == request.helper_checkpoint, "SYNC_DECISION_BINDING" ); Ok(d) } fn decode_key(hex: &str) -> Result> { ensure!(hex.len() == 64, "PINNED_KEY_SIZE"); (0..32) .map(|i| { Ok(u8::from_str_radix( hex.get(i * 2..i * 2 + 2).context("PINNED_KEY_ENCODING")?, 16, )?) }) .collect() } fn validate_attempt( job: &JobPayload, offer: &JobOffer, local: &SyncAttempt, d: &AttemptDecision, boot: Uuid, binding: &Binding, now: DateTime, ) -> Result>> { ensure!( d.job_payload_hash == offer.job_payload_hash && d.grant_id == optional_uuid(&local.grant_id)? && d.action == job.action.name() && d.execution_identity == job.execution_identity && (d.revision >= local.revision || d.outcome_state == "unknown") && d.original_boot_id == optional_uuid(&local.original_boot_id)? && d.previous_outcome_sha256.as_deref() == (!local.outcome_sha256.is_empty()).then_some(local.outcome_sha256.as_str()), "SYNC_ATTEMPT_BINDING" ); if d.grant_id.is_some() { ensure!( d.authorization_id == Some(Uuid::parse_str(&offer.authorization_id)?) && d.plan_hash.as_deref() == Some(offer.plan_hash.as_str()), "SYNC_AUTHORIZATION_BINDING" ); } ensure!( !terminal(&local.state) || d.outcome_state == local.state, "TERMINAL_STATE_REGRESSION" ); if d.update_inflight.is_some() || !local.update_handoff_bytes.is_empty() { validate_update_summary(binding, job, offer, local, boot, now)?; let proof = rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes) .map_err(anyhow::Error::msg)?; let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?; let expected = rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?; ensure!( d.update_inflight.as_ref() == Some(&expected) && d.outcome_state == "running" && !d.unlock && d.outcome_bytes.is_empty() && d.outcome_sha256.is_none(), "UPDATE_INFLIGHT_DECISION_CHANGED" ); return Ok(None); } if d.outcome_state == "accepted" { ensure!( local.state == "accepted" && local.consumption == "unconsumed" && d.grant_id.is_none() && !d.unlock && d.outcome_bytes.is_empty(), "SYNC_CANNOT_RESTART_ATTEMPT" ); return Ok(None); } if d.outcome_state == "unknown" { ensure!(!d.unlock, "UNKNOWN_CANNOT_UNLOCK"); return Ok(None); } ensure!(terminal(&d.outcome_state), "UNSUPPORTED_SYNC_STATE"); if d.outcome_bytes.is_empty() && d.outcome_sha256.is_none() { ensure!( local.consumption == "unconsumed" && d.grant_id.is_none() && !d.unlock && d.outcome_state != "succeeded", "TERMINAL_WITHOUT_EVIDENCE" ); return Ok(None); } let bytes = if d.outcome_bytes.is_empty() { local.outcome_bytes.clone() } else { d.outcome_bytes.clone() }; let o: ReconciledOutcome = serde_json::from_slice(&bytes)?; ensure!( canonical_outcome_bytes(&o)? == bytes && d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()) && o.state == d.outcome_state, "SYNC_CANONICAL_OUTCOME" ); if bytes != local.outcome_bytes { ensure!( matches!(job.action, rmm_domain::jobs::Action::Reboot { .. }) && d.outcome_state == "succeeded" && d.original_boot_id.is_some_and(|id| id != boot) && o.evidence["postconditions_passed"] == true && o.evidence["reconciled_after_reboot"] == true && o.evidence["new_boot_id"].as_str() == Some(boot.to_string().as_str()) && o.evidence["inventory_message_id"] .as_str() .is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())), "SYNC_REBOOT_REAUTH_EVIDENCE" ); } ensure!( !d.unlock || (local.consumption == "consumed" && d.grant_id.is_some()), "UNLOCK_WITHOUT_CONSUMPTION" ); Ok(Some(bytes)) }