//! Reconnect reconciliation, deliberately independent of main/jobs and transport. //! No method executes a job, deletes consumption tombstones or issues a permit. //! Call only while holding the existing agent.lock. HelperSnapshot must come //! from the independently authenticated root-helper IPC, never from the server. use crate::auth::Binding; pub use crate::auth::HelperIdentity; use crate::crypto::{DefaultVerifier, Ed25519Verifier}; use anyhow::{Context, Result, ensure}; use chrono::{DateTime, Utc}; use prost::Message; use rmm_domain::{ jobs::{JobPayload, hash}, reconciliation::{ AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes, }, }; use rmm_protocol::{ Envelope, JobOffer, JobResult, Payload, SyncAttempt, SyncRequest, SyncResponse, }; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use std::collections::{BTreeMap, BTreeSet}; use uuid::Uuid; /// A fresh observation returned by the trusted local clock implementation, /// never an IPC DTO or an execution permit. Callers must not retain it as a lease. pub struct ClockReading { pub own_wall: DateTime, pub upper: DateTime, } pub trait SyncClock { fn now(&self) -> Result>; /// Existing providers retain their existing point-clock contract. Providers /// with an independently verified interval override this atomically. fn reading(&self) -> Result { let own_wall = self.now()?; Ok(ClockReading { own_wall, upper: own_wall, }) } /// Full trusted UTC interval for crypto validity. Existing point-clock /// providers retain their contract; interval providers override atomically. fn bounds(&self) -> Result<(DateTime, DateTime)> { let reading = self.reading()?; ensure!(reading.own_wall <= reading.upper, "CLOCK_INTERVAL_ORDER"); Ok((reading.own_wall, reading.upper)) } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct Channel { pub connection_generation: u64, pub boot_id: Uuid, } #[derive(Clone, Debug)] pub struct HelperSnapshot { pub journal_id: Uuid, pub checkpoint: u64, pub previous_decision_id: Option, /// Complete consumed/unsettled summaries from the root-owned ledger. pub attempts: Vec, } #[derive(Clone)] pub struct PreparedSync { pub envelope: Envelope, pub request: SyncRequest, } /// Pass the ORIGINAL signed response to the helper. It must independently verify /// it, apply targeted unlocks and commit its checkpoint before acknowledging. #[derive(Clone)] pub struct HelperInstruction { pub response: SyncResponse, pub decision_id: Uuid, pub journal_id: Uuid, pub checkpoint: u64, } pub struct Applied { pub decision_id: Uuid, pub acknowledged_message_ids: Vec, pub helper_instruction: Option, pub reason: String, } /// Diagnostics only; never accepted as an authorization or reusable lease. #[derive(Clone, Copy)] pub struct GateObservation { pub signed_ready: bool, pub local_applied: bool, pub helper_ack_pending: bool, } struct Gate { channel: Channel, until: DateTime, ready: bool, helper_ready: bool, helper: Option<(Uuid, u64, Uuid)>, local_applied: bool, request: SyncRequest, response: SyncResponse, } pub struct ClientSync { binding: Binding, verifier: V, helper_identity: HelperIdentity, pending: Option, channel: Option, gate: Option, } impl ClientSync { /// Every new process starts closed, independently of any persisted ready bit. pub fn new(binding: Binding) -> Self { Self::with_verifier(binding, DefaultVerifier::default()) } } impl ClientSync { pub fn with_verifier(binding: Binding, verifier: V) -> Self { Self { binding, verifier, helper_identity: HelperIdentity::Root, pending: None, channel: None, gate: None, } } /// Configure one explicit helper identity; changing it discards all leases. pub fn with_helper_identity(mut self, identity: HelperIdentity) -> Self { self.close(); self.helper_identity = identity; self } /// Invoke on every authenticated/fenced ServerHello or transport loss. pub fn close(&mut self) { self.gate = None; self.channel = None; self.pending = None; } pub fn gate_observation(&self) -> Option { self.gate.as_ref().map(|g| GateObservation { signed_ready: g.ready, local_applied: g.local_applied, helper_ack_pending: g.helper.is_some(), }) } pub fn validity_deadline(&self) -> Option> { self.gate.as_ref().map(|gate| gate.until) } pub fn can_start(&self, channel: &Channel, root: bool, clock: &impl SyncClock) -> bool { let Ok(reading) = clock.reading() else { return false; }; self.gate.as_ref().is_some_and(|g| { g.channel == *channel && g.ready && g.local_applied && reading.own_wall < g.until && reading.upper < g.until && (!root || (g.helper_ready && g.helper.is_none())) }) } /// Only call with a fresh independently authenticated helper snapshot. There /// is no unsigned ready/unlock flag: the exact signed decision/checkpoint /// must already be durably applied by that helper. pub fn confirm_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> { let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?; let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?; ensure!( snapshot.journal_id == expected.0 && snapshot.checkpoint == expected.1 && snapshot.previous_decision_id == Some(expected.2), "HELPER_CHECKPOINT_NOT_APPLIED" ); ensure!( snapshot .attempts .iter() .all(|a| terminal(&a.state) || (a.consumption == "unconsumed" && a.state == "accepted")), "HELPER_EFFECT_UNRESOLVED" ); gate.helper = None; Ok(()) } /// Typed read-only ACK for exactly the own original update handoff. It /// clears only the control delivery acknowledgement; both start flags stay /// false and can_start remains false for ALL jobs, including this attempt. pub fn confirm_update_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> { let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?; let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?; ensure!( !gate.ready && !gate.helper_ready && snapshot.journal_id == expected.0 && snapshot.checkpoint == expected.1 && snapshot.previous_decision_id == Some(expected.2), "UPDATE_CONTROL_NOT_APPLIED" ); let updates: Vec<_> = snapshot .attempts .iter() .filter(|a| !a.update_handoff_bytes.is_empty()) .collect(); ensure!(updates.len() == 1, "UPDATE_CONTROL_EXACTLY_ONE_REQUIRED"); let local = updates[0]; let requested = gate .request .attempts .iter() .find(|a| a.job_id == local.job_id && a.attempt_id == local.attempt_id) .context("UPDATE_CONTROL_NOT_REQUESTED")?; ensure!( local.state == "running" && local.consumption == "consumed" && local.outcome_bytes.is_empty() && local.update_handoff_bytes == requested.update_handoff_bytes && local.grant_id == requested.grant_id && local.job_payload_hash == requested.job_payload_hash, "UPDATE_CONTROL_CHANGED" ); let decision: ReconcileDecisionPayload = serde_json::from_slice(&gate.response.decision_bytes)?; ensure!( decision .attempts .iter() .any(|d| d.job_id.to_string() == local.job_id && d.attempt_id.to_string() == local.attempt_id && d.update_inflight.is_some() && !d.unlock), "UPDATE_CONTROL_NOT_SIGNED" ); ensure!( snapshot .attempts .iter() .filter(|a| a.job_id != local.job_id || a.attempt_id != local.attempt_id) .all(|a| terminal(&a.state) || (a.state == "accepted" && a.consumption == "unconsumed")), "UPDATE_OTHER_EFFECT_UNRESOLVED" ); gate.helper = None; Ok(()) } /// Historical authentication only. Never apply an old response or create a /// lease: the returned bytes prove a previously durable local control step. fn stored_control( &self, db: &Connection, ledger: &Connection, id: Uuid, boot: Uuid, ) -> Result<(SyncRequest, ReconcileDecisionPayload)> { let ack: Vec = ledger.query_row( "SELECT response FROM reconcile_consumption WHERE decision_id=?1", [id.to_string()], |r| r.get(0), )?; ensure!( ack.len() <= rmm_domain::MAX_MESSAGE, "CLOSED_CONTROL_ACK_BOUND" ); let response = SyncResponse::decode(ack.as_slice())?; ensure!( response.encode_to_vec() == ack, "CLOSED_CONTROL_CANONICAL_ACK" ); let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; let (raw, envelope, generation, saved): (Vec, Vec, u64, Vec) = db.query_row( "SELECT request,envelope,generation,response FROM reconcile_requests WHERE id=?1", [parsed.request_id.to_string()], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), )?; ensure!( raw.len() <= rmm_domain::MAX_MESSAGE && envelope.len() <= rmm_domain::MAX_MESSAGE && saved == ack, "CLOSED_CONTROL_ORIGINAL_BYTES" ); let request = SyncRequest::decode(raw.as_slice())?; let env = Envelope::decode(envelope.as_slice())?; ensure!( raw == request.encode_to_vec() && env.encode_to_vec() == envelope && env.payload == Some(Payload::SyncRequest(request.clone())) && rmm_protocol::valid_envelope( &env, &self.binding.agent_id.to_string(), generation ) && env.boot_id == boot.to_string() && env.sequence == request.checkpoint, "CLOSED_CONTROL_ORIGINAL_ENVELOPE" ); let decision = verify_response_with( &self.verifier, &self.binding, &request, &Channel { connection_generation: generation, boot_id: boot, }, &response, parsed.issued_at, )?; ensure!( decision.decision_id == id && decision.boot_id == boot && !recovery_reason(&decision.reason), "CLOSED_CONTROL_SIGNED_ORIGINAL" ); Ok((request, decision)) } /// Admit only a new, ordinary online reconciliation of an exact missing /// helper ACK with already definitive, identical outcomes. The caller owns /// agent.lock, has no live executor/deferred work, and obtained `helper` /// from the serialized, independently authenticated current-boot helper. /// This predicate is read-only; fresh server/own-clock/helper ACK gates are /// unchanged and no flag or authority from either old response is reused. fn closed_control_continuation( &self, db: &Connection, ledger: &Connection, channel: &Channel, helper: &HelperSnapshot, meta: &Meta, now: DateTime, ) -> Result<()> { ensure!( self.helper_identity == HelperIdentity::System && self.pending.is_none() && self.gate.is_none() && meta.fault.is_none() && meta.pending_request.is_none(), "CLOSED_CONTROL_IDLE_ONLY" ); let a = meta .decision .context("CLOSED_CONTROL_AGENT_HEAD_REQUIRED")?; let h = helper .previous_decision_id .context("CLOSED_CONTROL_HELPER_HEAD_REQUIRED")?; ensure!( a != h && Some(helper.journal_id) == meta.helper_id && helper.checkpoint >= meta.helper_checkpoint && pending_ids(db)?.is_empty(), "CLOSED_CONTROL_CURRENT_HEADS" ); let legacy: bool = db.query_row( "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='events')", [], |r| r.get(0), )?; ensure!( !legacy || db.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, u64>(0))? == 0, "CLOSED_CONTROL_NO_PENDING_EVENTS" ); let (original, decision) = self.stored_control(db, ledger, a, channel.boot_id)?; let (_, previous) = self.stored_control(db, ledger, h, channel.boot_id)?; ensure!( decision.journal_id == meta.journal_id && decision.checkpoint == u64::try_from(meta.checkpoint)? && original.journal_id == meta.journal_id.to_string() && original.checkpoint == decision.checkpoint && original.previous_decision_id == h.to_string() && original.helper_previous_decision_id == h.to_string() && original.previous_request_id.is_empty() && original.journal_complete && decision.issued_at <= now && decision.connection_generation <= channel.connection_generation && previous.connection_generation <= decision.connection_generation && previous.checkpoint < decision.checkpoint && previous.journal_id == decision.journal_id && previous.helper_journal_id == Some(helper.journal_id) && decision.helper_journal_id == Some(helper.journal_id) && original.helper_journal_id == helper.journal_id.to_string() && previous.helper_checkpoint < decision.helper_checkpoint && decision.helper_checkpoint == meta.helper_checkpoint && helper.checkpoint >= decision.helper_checkpoint, "CLOSED_CONTROL_EXACT_PARTIAL_EDGE" ); verify_closed_result_messages(db, &original, &decision)?; let current = attempts(db, ledger, &self.binding)?; ensure!( current.len() == original.attempts.len() && current.len() == decision.attempts.len() && current.len() <= 256, "CLOSED_CONTROL_EXACT_ATTEMPT_SET" ); let expected_consumed = current .iter() .filter(|r| r.consumption == "consumed") .count() as u64; let ledger_counts: (u64, u64, u64) = ledger.query_row( "SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM execution_evidence),(SELECT count(*) FROM consumed WHERE result IS NULL)", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; ensure!( ledger_counts == (expected_consumed, expected_consumed, 0), "CLOSED_CONTROL_EXACT_COMPLETE_LEDGER" ); for row in ¤t { ensure!( matches!(row.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired") && row.update_handoff_bytes.is_empty(), "CLOSED_CONTROL_DEFINITIVE_ONLY" ); let old = original .attempts .iter() .find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id) .context("CLOSED_CONTROL_ORIGINAL_ATTEMPT")?; let signed = decision .attempts .iter() .find(|r| { r.job_id.to_string() == row.job_id && r.attempt_id.to_string() == row.attempt_id }) .context("CLOSED_CONTROL_SIGNED_ATTEMPT")?; ensure!( old.state == row.state && old.job_payload_hash == row.job_payload_hash && old.grant_id == row.grant_id && old.consumption == row.consumption && old.original_boot_id == row.original_boot_id && old.outcome_bytes == row.outcome_bytes && old.outcome_sha256 == row.outcome_sha256 && old.update_handoff_bytes.is_empty() && row.revision == old.revision.max(signed.revision) && signed.outcome_state == row.state && signed.update_inflight.is_none(), "CLOSED_CONTROL_RESULT_UNCHANGED" ); // An ungranted expiry has no fabricated outcome. Its exact // definitive nonstart is already authenticated in the original // request/decision; it can only permit a NEW online request. if row.consumption == "unconsumed" { let consumed: u64 = ledger.query_row( "SELECT count(*) FROM consumed WHERE job=?1 AND attempt=?2", params![row.job_id, row.attempt_id], |r| r.get(0), )?; ensure!( matches!(row.state.as_str(), "expired" | "cancelled") && row.grant_id.is_empty() && row.original_boot_id.is_empty() && row.outcome_bytes.is_empty() && row.outcome_sha256.is_empty() && signed.grant_id.is_none() && !signed.unlock && signed.reason == "DEFINITIVE_NONSTART" && signed.previous_outcome_sha256.is_none() && signed.outcome_sha256.is_none() && signed.outcome_bytes.is_empty() && consumed == 0 && !helper .attempts .iter() .any(|h| h.job_id == row.job_id && h.attempt_id == row.attempt_id), "CLOSED_CONTROL_SIGNED_UNGRANTED_NONSTART" ); continue; } ensure!( !row.outcome_bytes.is_empty() && signed.grant_id.map(|id| id.to_string()).as_deref() == Some(row.grant_id.as_str()) && hash(&row.outcome_bytes) == row.outcome_sha256 && signed.outcome_sha256.as_deref() == Some(row.outcome_sha256.as_str()) && (signed.outcome_bytes.is_empty() || signed.outcome_bytes == row.outcome_bytes), "CLOSED_CONTROL_SIGNED_CANONICAL_RESULT" ); let outcome: ReconciledOutcome = serde_json::from_slice(&row.outcome_bytes)?; ensure!( canonical_outcome_bytes(&outcome)? == row.outcome_bytes && outcome.state == row.state, "CLOSED_CONTROL_CANONICAL_RESULT" ); match row.consumption.as_str() { "consumed" => { let (start,finished,last):(Vec,Vec,Vec)=ledger.query_row("SELECT e.start,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1 AND c.job=?2 AND c.attempt=?3 AND c.agent=?4 AND c.generation=?5",params![row.grant_id,row.job_id,row.attempt_id,self.binding.agent_id.to_string(),self.binding.enrollment_generation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?; let saved: serde_json::Value = serde_json::from_slice(&start)?; let request: crate::auth::Request = serde_json::from_value(saved["request"].clone())?; let permit: rmm_domain::jobs::ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?; let (job, permit) = self.binding .verify_with(&request, permit.not_before, &self.verifier)?; ensure!( job.job_id.to_string() == row.job_id && job.attempt_id.to_string() == row.attempt_id && hash(&request.job_bytes) == row.job_payload_hash && permit.grant_id.to_string() == row.grant_id && !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) && canonical_outcome_bytes(&serde_json::from_slice::< ReconciledOutcome, >(&finished)?)? == row.outcome_bytes, "CLOSED_CONTROL_ORIGINAL_CONSUMPTION" ); if crate::windows_system_nonstart::effective_outcome( ledger, Some(&self.binding), permit.grant_id, )? .is_none() { ensure!( canonical_outcome_bytes(&serde_json::from_slice::( &last )?)? == row.outcome_bytes, "CLOSED_CONTROL_ORIGINAL_RESULT_CONFLICT" ); } if job.execution_identity == self.helper_identity.as_str() { let r = helper .attempts .iter() .find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id) .context("CLOSED_CONTROL_HELPER_ATTEMPT_REQUIRED")?; ensure!( r.state == row.state && r.grant_id == row.grant_id && r.job_payload_hash == row.job_payload_hash && r.consumption == "consumed" && r.outcome_bytes == row.outcome_bytes && r.outcome_sha256 == row.outcome_sha256 && r.original_boot_id == row.original_boot_id && r.revision <= row.revision && r.update_handoff_bytes.is_empty(), "CLOSED_CONTROL_HELPER_RESULT_UNCHANGED" ); } } _ => anyhow::bail!("CLOSED_CONTROL_UNCERTAIN_CONSUMPTION"), } } for row in &helper.attempts { ensure!( terminal(&row.state) && row.update_handoff_bytes.is_empty() && current.iter().any(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id && r.grant_id == row.grant_id && r.outcome_bytes == row.outcome_bytes), "CLOSED_CONTROL_NO_EXTRA_HELPER_ATTEMPT" ); } Ok(()) } pub fn prepare( &mut self, db: &mut Connection, ledger: &Connection, channel: Channel, helper: Option<&HelperSnapshot>, clock: &impl SyncClock, ) -> Result { self.prepare_with_closed_control_idle(db, ledger, channel, helper, clock, false) } /// The OS actor may attest quiescence only after its execution/deferred /// workers and protected intent store are all settled. Ordinary same-head /// synchronization does not need this attestation, including while busy. pub fn prepare_with_closed_control_idle( &mut self, db: &mut Connection, ledger: &Connection, channel: Channel, helper: Option<&HelperSnapshot>, clock: &impl SyncClock, closed_control_idle: bool, ) -> Result { let result = self.prepare_inner(db, ledger, channel, helper, clock, closed_control_idle); if let Err(error) = &result { let text = error.to_string(); if [ "ROLLBACK", "REGRESSION", "DISAPPEARED", "MISSING_FROM_AGENT_JOURNAL", "MISSING_FROM_HELPER_JOURNAL", ] .iter() .any(|s| text.contains(s)) { let _ = db.execute( "UPDATE reconcile_meta SET fault=COALESCE(fault,?1) WHERE id=1", [text], ); } } result } fn prepare_inner( &mut self, db: &mut Connection, ledger: &Connection, channel: Channel, helper: Option<&HelperSnapshot>, clock: &impl SyncClock, closed_control_idle: bool, ) -> Result { self.close(); let now = clock.now()?; ensure!( channel.connection_generation > 0 && !channel.boot_id.is_nil(), "UNAUTHENTICATED_CHANNEL" ); initialize(db)?; let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; let meta = metadata(&tx)?; ensure!(meta.fault.is_none(), "SYNC_MANUAL_RECOVERY_REQUIRED"); let sequence: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?; ensure!( sequence >= meta.checkpoint && sequence >= meta.high_sequence, "LOCAL_CHECKPOINT_ROLLBACK" ); ensure!( channel.connection_generation >= meta.generation, "CONNECTION_GENERATION_REGRESSION" ); if let Some(h) = helper { ensure!( !h.journal_id.is_nil() && h.checkpoint <= i64::MAX as u64, "HELPER_CHECKPOINT" ); if let Some(old) = meta.helper_id { ensure!( old == h.journal_id && h.checkpoint >= meta.helper_checkpoint, "HELPER_JOURNAL_ROLLBACK" ); if self.helper_identity == HelperIdentity::System { // A locally committed Windows decision is not proof of its // application by SYSTEM. Never advance either checkpoint // from an older helper head, including after process loss. if meta.decision.is_none() || h.previous_decision_id != meta.decision { ensure!(closed_control_idle, "SYSTEM_HELPER_APPLY_UNCONFIRMED"); self.closed_control_continuation(&tx, ledger, &channel, h, &meta, now) .context("SYSTEM_HELPER_APPLY_UNCONFIRMED")?; } } } } else { ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED"); } if let Some(h) = helper { ingest_helper( &tx, ledger, (&self.binding, &self.verifier), &channel, h, now, self.helper_identity, )?; } archive_events(&tx)?; tx.execute( "UPDATE job_attempts SET state='accepted' WHERE state='queued' AND grant_id=''", [], )?; // A possibly started attempt is NEVER returned to accepted. Preserve // revisions and results; only ambiguous no-result states become unknown. let inflight: Vec<_> = helper .into_iter() .flat_map(|h| h.attempts.iter()) .filter(|a| !a.update_handoff_bytes.is_empty()) .collect(); ensure!(inflight.len() <= 1, "UPDATE_MULTIPLE_INFLIGHT"); if let Some(update) = inflight.first() { tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT (job=?1 AND attempt=?2) AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",params![update.job_id,update.attempt_id])?; } else { tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?; } let mut attempts = attempts(&tx, ledger, &self.binding)?; if let Some(h) = helper { merge_helper(&mut attempts, h)?; } ensure!(attempts.len() <= 256, "SYNC_ATTEMPT_CAPACITY_NO_COMPACTION"); let mut ids = pending_ids(&tx)?; if let Some(update) = inflight.first() { let grant = Uuid::parse_str(&update.grant_id)?; ids.extend(crate::update_health_inventory::pending_ids(&tx, grant)?); ids.sort(); ids.dedup(); } ensure!(ids.len() <= 256, "SYNC_RECEIPT_CAPACITY"); let next: i64 = tx.query_row( "UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq", [], |r| r.get(0), )?; let request = SyncRequest { request_id: Uuid::new_v4().to_string(), journal_id: meta.journal_id.to_string(), checkpoint: u64::try_from(next)?, // A pending request may have committed with a lost reply, or may // never have reached the server. Preserve both witnesses: the // server resolves the pending request first and otherwise checks // this last applied decision against its authoritative head. previous_decision_id: meta.decision.map(|id| id.to_string()).unwrap_or_default(), assignment_version: self.binding.assignment_version, attempts, durable_message_ids: ids, journal_complete: true, helper_journal_id: helper.map(|h| h.journal_id.to_string()).unwrap_or_default(), helper_checkpoint: helper.map_or(0, |h| h.checkpoint), helper_previous_decision_id: helper .and_then(|h| h.previous_decision_id) .map(|id| id.to_string()) .unwrap_or_default(), previous_request_id: meta .pending_request .map(|id| id.to_string()) .unwrap_or_default(), }; let envelope = Envelope { protocol_major: 1, protocol_minor: 0, message_id: Uuid::new_v4().to_string(), agent_id: self.binding.agent_id.to_string(), enrollment_generation: self.binding.enrollment_generation, connection_generation: channel.connection_generation, boot_id: channel.boot_id.to_string(), sequence: u64::try_from(next)?, observed_at: now.timestamp_millis(), payload: Some(Payload::SyncRequest(request.clone())), }; ensure!( envelope.encoded_len() <= rmm_domain::MAX_MESSAGE, "SYNC_ENVELOPE_SIZE" ); tx.execute( "INSERT INTO reconcile_requests(id,request,envelope,generation) VALUES(?1,?2,?3,?4)", params![ request.request_id, request.encode_to_vec(), envelope.encode_to_vec(), channel.connection_generation ], )?; tx.execute("UPDATE reconcile_meta SET checkpoint=?1,high_sequence=?1,generation=?2,pending_request=?3 WHERE id=1",params![next,channel.connection_generation,request.request_id])?; tx.commit()?; let prepared = PreparedSync { envelope, request }; self.pending = Some(prepared.clone()); self.channel = Some(channel); Ok(prepared) } /// All changes, concrete ACKs, result archives and checkpoint chain commit /// together. A rejection closes only mutation authority; telemetry may run. pub fn apply( &mut self, db: &mut Connection, response: &SyncResponse, clock: &impl SyncClock, ) -> Result { if self.pending.is_none() { let result = self.replay_applied(db, response, clock); if result.is_err() { self.gate = None; } return result; } self.gate = None; let result = self.apply_inner(db, response, clock); if result.is_err() { self.gate = None; } result } fn replay_applied( &self, db: &Connection, response: &SyncResponse, clock: &impl SyncClock, ) -> Result { let gate = self.gate.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?; ensure!(gate.response == *response, "REPLAYED_DIFFERENT_DECISION"); let d = verify_response_with( &self.verifier, &self.binding, &gate.request, &gate.channel, response, clock.now()?, )?; let meta = metadata(db)?; let sequence: i64 = db.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?; ensure!( meta.decision == Some(d.decision_id) && meta.pending_request.is_none() && meta.checkpoint == i64::try_from(d.checkpoint)? && sequence >= meta.high_sequence, "APPLIED_CHECKPOINT_REGRESSION" ); Ok(Applied { decision_id: d.decision_id, acknowledged_message_ids: d.acknowledged_message_ids, helper_instruction: d.helper_journal_id.map(|journal_id| HelperInstruction { response: response.clone(), decision_id: d.decision_id, journal_id, checkpoint: d.helper_checkpoint, }), reason: d.reason, }) } fn apply_inner( &mut self, db: &mut Connection, response: &SyncResponse, clock: &impl SyncClock, ) -> Result { let now = clock.now()?; let prepared = self.pending.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?; let channel = self.channel.as_ref().context("NO_AUTHENTICATED_CHANNEL")?; let decision = verify_response_with( &self.verifier, &self.binding, &prepared.request, channel, response, now, )?; let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; let meta = metadata(&tx)?; ensure!( meta.fault.is_none() && meta.pending_request == Some(decision.request_id) && meta.checkpoint == i64::try_from(decision.checkpoint)? && meta.generation == channel.connection_generation, "SYNC_CHECKPOINT_CHANGED" ); let seq: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?; ensure!( seq >= meta.high_sequence && seq >= meta.checkpoint, "LOCAL_SEQUENCE_ROLLBACK" ); let current = tx.query_row( "SELECT request FROM reconcile_requests WHERE id=?1", [decision.request_id.to_string()], |r| r.get::<_, Vec>(0), )?; ensure!( current == prepared.request.encode_to_vec(), "DURABLE_CHALLENGE_CHANGED" ); let summaries: BTreeMap<_, _> = prepared .request .attempts .iter() .map(|a| ((a.job_id.clone(), a.attempt_id.clone()), a)) .collect(); let actual: BTreeSet<_> = decision .attempts .iter() .map(|a| (a.job_id.to_string(), a.attempt_id.to_string())) .collect(); ensure!( actual.len() == decision.attempts.len() && actual == summaries.keys().cloned().collect(), "SYNC_ATTEMPT_OMITTED_OR_ADDED" ); let mut safe = true; let mut fault = recovery_reason(&decision.reason).then(|| decision.reason.clone()); for d in &decision.attempts { let local = summaries .get(&(d.job_id.to_string(), d.attempt_id.to_string())) .context("LOCAL_ATTEMPT_MISSING")?; if recovery_reason(&d.reason) { fault = Some(d.reason.clone()); } let offer_bytes: Vec = tx.query_row( "SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2", params![local.job_id, local.attempt_id], |r| r.get(0), )?; let offer = JobOffer::decode(offer_bytes.as_slice())?; let job = self.binding.job(&offer.payload_bytes)?; // Missing-server decisions intentionally have empty action fields; // persist their sticky denial without treating it as start authority. if recovery_reason(&d.reason) { safe = false; continue; } let outcome = validate_attempt(&job, &offer, local, d, channel.boot_id, &self.binding, now)?; safe &= (d.outcome_state == "accepted" && local.consumption == "unconsumed" && local.grant_id.is_empty()) || (terminal(&d.outcome_state) && (local.consumption == "unconsumed" || d.unlock)); let (stored_state, revision): (String, u64) = tx.query_row( "SELECT state,revision FROM job_attempts WHERE job=?1 AND attempt=?2", params![local.job_id, local.attempt_id], |r| Ok((r.get(0)?, r.get(1)?)), )?; ensure!( revision == local.revision && stored_state == local.state, "LOCAL_ATTEMPT_CHANGED_DURING_SYNC" ); let stored = if d.outcome_state == "unknown" && local.state == "awaiting_reboot" { "awaiting_reboot" } else { d.outcome_state.as_str() }; tx.execute( "UPDATE job_attempts SET state=?3,revision=?4 WHERE job=?1 AND attempt=?2", params![ local.job_id, local.attempt_id, stored, d.revision.max(local.revision) ], )?; if let Some(bytes) = outcome { tx.execute("INSERT INTO reconcile_outcomes(job,attempt,outcome,decision_id) VALUES(?1,?2,?3,?4) ON CONFLICT(job,attempt) DO UPDATE SET outcome=excluded.outcome,decision_id=excluded.decision_id",params![local.job_id,local.attempt_id,bytes,decision.decision_id.to_string()])?; } } let requested: BTreeSet<_> = prepared .request .durable_message_ids .iter() .map(String::as_str) .collect(); let mut receipts = BTreeSet::new(); for id in &decision.acknowledged_message_ids { ensure!( receipts.insert(*id) && requested.contains(id.to_string().as_str()), "ACK_NOT_IN_REQUEST" ); archive_events(&tx)?; tx.execute("DELETE FROM job_events WHERE id=?1", [id.to_string()])?; tx.execute( "INSERT OR IGNORE INTO reconcile_receipts(id,decision_id) VALUES(?1,?2)", params![id.to_string(), decision.decision_id.to_string()], )?; } tx.execute( "UPDATE reconcile_requests SET response=?2 WHERE id=?1 AND response IS NULL", params![decision.request_id.to_string(), response.encode_to_vec()], )?; for local in prepared .request .attempts .iter() .filter(|a| !a.update_handoff_bytes.is_empty()) { crate::update_health_inventory::acknowledge_signed( &tx, Uuid::parse_str(&local.grant_id)?, &decision, response, )?; } tx.execute("UPDATE reconcile_meta SET last_decision=?1,pending_request=NULL,helper_id=?2,helper_checkpoint=?3,fault=COALESCE(fault,?4),high_sequence=?5 WHERE id=1", params![decision.decision_id.to_string(),decision.helper_journal_id.map(|id|id.to_string()),decision.helper_checkpoint,fault,seq])?; tx.commit()?; let helper_instruction = decision .helper_journal_id .map(|journal_id| HelperInstruction { response: response.clone(), decision_id: decision.decision_id, journal_id, checkpoint: decision.helper_checkpoint, }); self.gate = Some(Gate { channel: channel.clone(), until: decision.valid_until, ready: decision.ready && safe && fault.is_none(), helper_ready: decision.helper_ready, helper: helper_instruction .as_ref() .map(|h| (h.journal_id, h.checkpoint, h.decision_id)), local_applied: !decision.attempts.iter().any(|d| d.unlock), request: prepared.request.clone(), response: response.clone(), }); self.pending = None; Ok(Applied { decision_id: decision.decision_id, acknowledged_message_ids: decision.acknowledged_message_ids, helper_instruction, reason: decision.reason, }) } /// Apply ONLY this process's independently owned consumption ledger. Root /// helper unlocks use HelperInstruction over authenticated IPC instead. /// A crash between journal and ledger transactions leaves the gate closed; /// resync/reapply never deletes the consumed attempt or repeats an effect. pub fn apply_consumption( &mut self, ledger: &mut Connection, clock: &impl SyncClock, ) -> Result<()> { let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?; gate.local_applied = false; let decision = verify_response_with( &self.verifier, &self.binding, &gate.request, &gate.channel, &gate.response, clock.now()?, )?; let tx = ledger.transaction_with_behavior(TransactionBehavior::Immediate)?; tx.execute_batch("CREATE TABLE IF NOT EXISTS reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB NOT NULL);")?; let applied: Option> = tx .query_row( "SELECT response FROM reconcile_consumption WHERE decision_id=?1", [decision.decision_id.to_string()], |r| r.get(0), ) .optional()?; if let Some(old) = applied { ensure!( old == gate.response.encode_to_vec(), "CONSUMPTION_DECISION_CONFLICT" ); } else { for d in decision.attempts.iter().filter(|d| d.unlock) { let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?; let (agent,generation,job,attempt,previous):(String,u64,String,String,Option>)=tx.query_row( "SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1", [grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?; ensure!( agent == self.binding.agent_id.to_string() && generation == self.binding.enrollment_generation && job == d.job_id.to_string() && attempt == d.attempt_id.to_string(), "LOCAL_UNLOCK_CONSUMPTION_BINDING" ); let resolved = crate::windows_system_nonstart::effective_outcome( &tx, Some(&self.binding), grant, )?; let previous = resolved .or(previous) .context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?; let original = previous.clone(); let prior = canonical_outcome_bytes(&serde_json::from_slice::( &previous, )?)?; ensure!( d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()), "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH" ); let bytes = if d.outcome_bytes.is_empty() { prior } else { d.outcome_bytes.clone() }; let outcome: ReconciledOutcome = serde_json::from_slice(&bytes)?; ensure!( terminal(&outcome.state) && outcome.state == d.outcome_state && canonical_outcome_bytes(&outcome)? == bytes && d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()), "LOCAL_UNLOCK_CANONICAL_OUTCOME" ); let preserve = crate::windows_system_nonstart::preserve_original_unlock( &tx, &self.binding, grant, &bytes, )?; if outcome.state == "closed_unknown" { crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?; } tx.execute( "UPDATE consumed SET result=?2 WHERE grant_id=?1", params![grant.to_string(), bytes], )?; if !preserve { tx.execute( "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1", params![grant.to_string(), bytes], )?; } } tx.execute( "INSERT INTO reconcile_consumption(decision_id,response) VALUES(?1,?2)", params![ decision.decision_id.to_string(), gate.response.encode_to_vec() ], )?; } tx.commit()?; gate.local_applied = true; Ok(()) } } pub fn initialize(db: &Connection) -> Result<()> { let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; ensure!(check == "ok", "JOURNAL_INTEGRITY"); db.execute_batch("PRAGMA synchronous=FULL; CREATE TABLE IF NOT EXISTS reconcile_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,high_sequence INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,last_decision TEXT,pending_request TEXT,helper_id TEXT,helper_checkpoint INTEGER NOT NULL DEFAULT 0,fault TEXT); CREATE TABLE IF NOT EXISTS reconcile_requests(id TEXT PRIMARY KEY,request BLOB NOT NULL,envelope BLOB NOT NULL,generation INTEGER NOT NULL,response BLOB); CREATE TABLE IF NOT EXISTS reconcile_results(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,revision INTEGER NOT NULL,envelope BLOB NOT NULL); CREATE TABLE IF NOT EXISTS reconcile_receipts(id TEXT PRIMARY KEY,decision_id TEXT NOT NULL); CREATE TABLE IF NOT EXISTS reconcile_outcomes(job TEXT NOT NULL,attempt TEXT NOT NULL,outcome BLOB NOT NULL,decision_id TEXT NOT NULL,PRIMARY KEY(job,attempt));")?; crate::update_health_inventory::initialize(db)?; db.execute( "INSERT OR IGNORE INTO reconcile_meta(id,journal_id) VALUES(1,?1)", [Uuid::new_v4().to_string()], )?; Ok(()) } struct Meta { journal_id: Uuid, checkpoint: i64, high_sequence: i64, generation: u64, decision: Option, pending_request: Option, helper_id: Option, helper_checkpoint: u64, fault: Option, } type MetadataRow = ( String, i64, i64, u64, Option, Option, Option, u64, Option, ); fn metadata(db: &Connection) -> Result { let row:MetadataRow=db.query_row("SELECT journal_id,checkpoint,high_sequence,generation,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?)))?; Ok(Meta { journal_id: Uuid::parse_str(&row.0)?, checkpoint: row.1, high_sequence: row.2, generation: row.3, decision: row.4.map(|s| Uuid::parse_str(&s)).transpose()?, pending_request: row.5.map(|s| Uuid::parse_str(&s)).transpose()?, helper_id: row.6.map(|s| Uuid::parse_str(&s)).transpose()?, helper_checkpoint: row.7, fault: row.8, }) } /// Call inside the same transaction BEFORE deleting an ACKed job event. This /// retains the original result identity/output for future reconciliation. pub fn archive_events(db: &Connection) -> Result<()> { let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY rowid")?; for row in statement.query_map([], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, Vec>(1)?)) })? { let (id, bytes) = row?; let envelope = Envelope::decode(bytes.as_slice())?; ensure!(id == envelope.message_id, "EVENT_IDENTITY_CONFLICT"); if let Some(Payload::JobResult(r)) = &envelope.payload { let existing: Option> = db .query_row( "SELECT envelope FROM reconcile_results WHERE id=?1", [&id], |r| r.get(0), ) .optional()?; ensure!( existing.as_ref().is_none_or(|b| *b == bytes), "RESULT_IDENTITY_CONFLICT" ); db.execute("INSERT OR IGNORE INTO reconcile_results(id,job,attempt,revision,envelope) VALUES(?1,?2,?3,?4,?5)",params![id,r.job_id,r.attempt_id,r.revision,bytes])?; } } Ok(()) } /// A partial control edge can contain the result that closed an attempt, and /// an earlier unknown report for that same grant. Only account for archived /// messages here. Never ACK an event, change its state or replay a result. fn verify_closed_result_messages( db: &Connection, request: &SyncRequest, decision: &ReconcileDecisionPayload, ) -> Result<()> { for id in &request.durable_message_ids { let raw: Vec = db.query_row( "SELECT envelope FROM reconcile_results WHERE id=?1", [id], |r| r.get(0), )?; let envelope = Envelope::decode(raw.as_slice())?; let Some(Payload::JobResult(result)) = &envelope.payload else { anyhow::bail!("CLOSED_CONTROL_EXPECTED_RESULT"); }; let attempt = request .attempts .iter() .find(|a| { a.job_id == result.job_id && a.attempt_id == result.attempt_id && a.grant_id == result.grant_id }) .context("CLOSED_CONTROL_ARCHIVED_ATTEMPT")?; let final_envelope = attempt .result_envelope .as_ref() .context("CLOSED_CONTROL_FINAL_ENVELOPE")?; let final_id = Uuid::parse_str(&attempt.result_message_id)?; let persisted: bool = db.query_row( "SELECT EXISTS(SELECT 1 FROM reconcile_receipts WHERE id=?1 AND decision_id=?2)", params![attempt.result_message_id, decision.decision_id.to_string()], |r| r.get(0), )?; ensure!( persisted && decision.acknowledged_message_ids.contains(&final_id) && request .durable_message_ids .contains(&attempt.result_message_id) && final_envelope.message_id == attempt.result_message_id && envelope.message_id == *id && envelope.encode_to_vec() == raw && rmm_protocol::valid_envelope( &envelope, &decision.agent_id.to_string(), final_envelope.connection_generation ) && envelope.boot_id == final_envelope.boot_id && matches!( attempt.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired" ) && !result.grant_id.is_empty(), "CLOSED_CONTROL_FINAL_RESULT_RECEIPT" ); ensure!( if *id == attempt.result_message_id { envelope == *final_envelope && result.revision == attempt.revision && result.state == attempt.state && canonical_outcome_bytes(&outcome(result)?)? == attempt.outcome_bytes } else { result.state == "unknown" && result.revision < attempt.revision && envelope.sequence < final_envelope.sequence }, "CLOSED_CONTROL_CONFLICTING_ARCHIVED_RESULT" ); } Ok(()) } fn pending_ids(db: &Connection) -> Result> { let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?; let rows = statement .query_map([], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, Vec>(1)?)) })? .collect::>>()?; let mut ids = vec![]; for (id, bytes) in rows { ensure!( !Uuid::parse_str(&id)?.is_nil(), "INVALID_DURABLE_MESSAGE_ID" ); let envelope = Envelope::decode(bytes.as_slice())?; ensure!(envelope.message_id == id, "EVENT_IDENTITY_CONFLICT"); if !crate::windows_system_nonstart::retired_event(db, &envelope)? { ids.push(id); } } Ok(ids) } pub(crate) fn outcome(r: &JobResult) -> Result { Ok(ReconciledOutcome { state: r.state.clone(), stdout: r.stdout.clone(), stderr: r.stderr.clone(), exit_code: r.exit_code, output_truncated: r.output_truncated, reboot_required: r.reboot_required, evidence: serde_json::from_str(&r.evidence_json)?, error_code: r.error_code.clone(), }) } fn terminal(state: &str) -> bool { matches!( state, "succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown" ) } /// Server-signed final bytes of a human WebUI closure for an attempt whose /// effect stayed unknown. They never claim success, failure or nonstart. pub(crate) fn operator_closure( reason: &str, local_state: &str, outcome: &ReconciledOutcome, ) -> bool { reason == "OPERATOR_CLOSED_UNKNOWN" && local_state == "unknown" && outcome.state == "closed_unknown" && outcome.error_code == "OPERATOR_CLOSED_UNKNOWN" && outcome.exit_code.is_none() && outcome.stdout.is_empty() && outcome.stderr.is_empty() && outcome.evidence["postconditions_passed"] == false && outcome.evidence["effect"] == "unknown" && outcome.evidence["closed_by_human"] == true && outcome.evidence["retry"] == false } fn optional_uuid(s: &str) -> Result> { if s.is_empty() { Ok(None) } else { let id = Uuid::parse_str(s)?; ensure!(!id.is_nil(), "NIL_ID"); Ok(Some(id)) } } fn recovery_reason(reason: &str) -> bool { matches!( reason, "MISSING_SERVER_JOB" | "ATTEMPT_BINDING_MISMATCH" | "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED" | "SERVER_STATE_ROLLBACK" | "ASSIGNMENT_MISMATCH" | "JOURNAL_SEQUENCE_REGRESSION" | "SERVER_CHECKPOINT_MISSING" | "JOURNAL_ROLLBACK_OR_REPLACEMENT" | "CHECKPOINT_CHAIN_MISMATCH" | "HELPER_JOURNAL_ROLLBACK" | "HELPER_CHECKPOINT_MISMATCH" | "HELPER_CHECKPOINT_MISSING" | "HELPER_CHECKPOINT_REQUIRED" | "JOURNAL_HISTORY_MISSING" | "LOCAL_ATTEMPT_MISSING" ) } fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result> { let mut statement = db.prepare("SELECT offer,state,revision,grant_id FROM job_attempts ORDER BY job,attempt")?; let rows = statement.query_map([], |r| { Ok(( r.get::<_, Vec>(0)?, r.get::<_, String>(1)?, r.get::<_, u64>(2)?, r.get::<_, String>(3)?, )) })?; let mut summaries = Vec::new(); let mut known = BTreeSet::new(); for row in rows { let (bytes, state, revision, mut grant_id) = row?; let offer = JobOffer::decode(bytes.as_slice())?; let job = binding.job(&offer.payload_bytes)?; ensure!( offer.job_id == job.job_id.to_string() && offer.attempt_id == job.attempt_id.to_string() && offer.job_payload_hash == hash(&offer.payload_bytes), "LOCAL_OFFER_BINDING" ); known.insert((offer.job_id.clone(), offer.attempt_id.clone())); let consumed: Option<(String, String, u64, Option>)> = ledger .query_row( "SELECT grant_id,agent,generation,result FROM consumed WHERE job=?1 AND attempt=?2", params![offer.job_id, offer.attempt_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), ) .optional()?; // A signed definitive nonstart can make an accepted no-grant row // terminal without a JobResult. Preserve that fact on the next Sync; // requesting/running/unknown still cannot claim non-consumption. An // actual durable consumption row below always overrides this label. let mut consumption = if grant_id.is_empty() && matches!(state.as_str(), "accepted" | "expired" | "cancelled") { "unconsumed" } else { "uncertain" }; let mut original_boot_id = String::new(); let mut canonical = Vec::new(); if let Some((id, agent, generation, result)) = consumed { ensure!( agent == binding.agent_id.to_string() && generation == binding.enrollment_generation && (grant_id.is_empty() || grant_id == id), "CONSUMPTION_BINDING" ); grant_id = id; consumption = "consumed"; let evidence: Option<(Vec, Option>)> = ledger .query_row( "SELECT start,outcome FROM execution_evidence WHERE grant_id=?1", [&grant_id], |r| Ok((r.get(0)?, r.get(1)?)), ) .optional()?; let mut stored = result; if let Some((start, last)) = evidence { let start: serde_json::Value = serde_json::from_slice(&start)?; original_boot_id = start["previous_boot_id"] .as_str() .context("ORIGINAL_BOOT_MISSING")? .to_string(); optional_uuid(&original_boot_id)?; let request: crate::auth::Request = serde_json::from_value(start["request"].clone())?; ensure!( request.job_bytes == offer.payload_bytes && request.plan_hash == offer.plan_hash && request.authorization_id.to_string() == offer.authorization_id, "START_EVIDENCE_BINDING" ); if let Some(resolved) = crate::windows_system_nonstart::effective_outcome( ledger, Some(binding), Uuid::parse_str(&grant_id)?, )? { stored = Some(resolved); } else if last.is_some() { stored = last; } } if let Some(stored) = stored { let o: ReconciledOutcome = serde_json::from_slice(&stored)?; canonical = canonical_outcome_bytes(&o)?; } } let original:Option>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",params![offer.job_id,offer.attempt_id],|r|r.get(0)).optional()?; let mut result_envelope = None; let mut result = None; let mut result_message_id = String::new(); if let Some(bytes) = original { let envelope = Envelope::decode(bytes.as_slice())?; let Some(Payload::JobResult(r)) = &envelope.payload else { anyhow::bail!("INVALID_ARCHIVED_RESULT") }; if r.revision == revision && r.state == state { ensure!( envelope.agent_id == binding.agent_id.to_string() && envelope.enrollment_generation == binding.enrollment_generation && r.grant_id == grant_id, "RESULT_BINDING" ); let bytes = canonical_outcome_bytes(&outcome(r)?)?; ensure!( canonical.is_empty() || canonical == bytes, "LOCAL_OUTCOME_LEDGER_MISMATCH" ); canonical = bytes; result_message_id = envelope.message_id.clone(); result = Some(r.clone()); result_envelope = Some(envelope); } } if !canonical.is_empty() { let o: ReconciledOutcome = serde_json::from_slice(&canonical)?; ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH"); ensure!(canonical.len() <= 512 * 1024, "SYNC_CANONICAL_OUTCOME_SIZE"); if grant_id.is_empty() && terminal(&state) && o.evidence["not_started"] == true { consumption = "unconsumed"; } } summaries.push(SyncAttempt { update_handoff_bytes: Vec::new(), job_id: offer.job_id, attempt_id: offer.attempt_id, job_payload_hash: offer.job_payload_hash, grant_id, consumption: consumption.into(), state, revision, original_boot_id, result, result_message_id, outcome_sha256: if canonical.is_empty() { String::new() } else { hash(&canonical) }, outcome_bytes: canonical, result_envelope, }); } let mut statement = ledger.prepare("SELECT job,attempt FROM consumed")?; for row in statement.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { ensure!( known.contains(&row?), "CONSUMED_ATTEMPT_MISSING_FROM_AGENT_JOURNAL" ); } Ok(summaries) } fn ingest_helper( db: &Connection, ledger: &Connection, binding_and_verifier: (&Binding, &impl Ed25519Verifier), channel: &Channel, helper: &HelperSnapshot, now: DateTime, identity: HelperIdentity, ) -> Result<()> { let (binding, verifier) = binding_and_verifier; let mut statement = db.prepare("SELECT job,attempt,offer FROM job_attempts")?; for row in statement.query_map([], |r| { Ok(( r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Vec>(2)?, )) })? { let (job, attempt, bytes) = row?; let offer = JobOffer::decode(bytes.as_slice())?; if binding.job(&offer.payload_bytes)?.execution_identity == identity.as_str() { let consumed: bool = ledger.query_row( "SELECT EXISTS(SELECT 1 FROM consumed WHERE job=?1 AND attempt=?2)", params![job, attempt], |r| r.get(0), )?; ensure!( !consumed || helper .attempts .iter() .any(|a| a.job_id == job && a.attempt_id == attempt), "ROOT_CONSUMED_ATTEMPT_MISSING_FROM_HELPER_JOURNAL" ); } } let mut seen = BTreeSet::new(); for h in &helper.attempts { ensure!( seen.insert((&h.job_id, &h.attempt_id)), "DUPLICATE_HELPER_ATTEMPT" ); let row: Option<(Vec, u64, String)> = db .query_row( "SELECT offer,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), ) .optional()?; let (bytes, revision, grant) = row.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?; let offer = JobOffer::decode(bytes.as_slice())?; let job = binding.job(&offer.payload_bytes)?; ensure!( job.execution_identity == identity.as_str() && h.job_payload_hash == offer.job_payload_hash && (grant.is_empty() || grant == h.grant_id), "HELPER_ATTEMPT_BINDING" ); if h.outcome_bytes.is_empty() { if !h.update_handoff_bytes.is_empty() { let header = validate_update_summary(binding, &job, &offer, h, channel.boot_id, now)?; ensure!( header.helper_journal_id == helper.journal_id && header.agent_journal_id == metadata(db)?.journal_id, "UPDATE_HELPER_JOURNAL_BINDING" ); let state: String = db.query_row( "SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id], |r| r.get(0), )?; ensure!( matches!( state.as_str(), "accepted" | "requesting" | "running" | "verifying" ), "UPDATE_CANNOT_HEAL_UNKNOWN" ); type OriginalEvidence = ( String, u64, String, String, Vec, Option>, Option>, ); let original:Option=ledger.query_row( "SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM execution_evidence e JOIN consumed c USING(grant_id) WHERE grant_id=?1",[&h.grant_id], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?; let (agent, generation, job_id, attempt_id, start, outcome, result) = original.context("UPDATE_LOCAL_ORIGINAL_MISSING")?; ensure!( agent == binding.agent_id.to_string() && generation == binding.enrollment_generation && job_id == h.job_id && attempt_id == h.attempt_id, "UPDATE_LOCAL_CONSUMPTION_BINDING" ); ensure!( outcome.is_none() && result.is_none(), "UPDATE_CANNOT_REPLACE_OUTCOME" ); let start: serde_json::Value = serde_json::from_slice(&start)?; let request: crate::auth::Request = serde_json::from_value(start["request"].clone())?; let permit: rmm_domain::jobs::ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?; binding.verify_with(&request, permit.not_before, verifier)?; ensure!( hash(&serde_json::to_vec(&request)?) == header.attempt.original_request_sha256 && permit.grant_id == header.attempt.grant_id && permit.agent_update_plan_id == Some(header.attempt.plan_id), "UPDATE_LOCAL_ORIGINAL_CHANGED" ); db.execute("UPDATE job_attempts SET state='running',revision=MAX(revision,?3) WHERE job=?1 AND attempt=?2",params![h.job_id,h.attempt_id,h.revision])?; } continue; } let o: ReconciledOutcome = serde_json::from_slice(&h.outcome_bytes)?; ensure!( canonical_outcome_bytes(&o)? == h.outcome_bytes && hash(&h.outcome_bytes) == h.outcome_sha256 && o.state == h.state && (terminal(&o.state) || matches!(o.state.as_str(), "unknown" | "awaiting_reboot")) && h.consumption == "consumed" && optional_uuid(&h.grant_id)?.is_some(), "HELPER_CANONICAL_OUTCOME" ); let existing: Option> = ledger .query_row( "SELECT outcome FROM execution_evidence WHERE grant_id=?1", [&h.grant_id], |r| r.get(0), ) .optional()? .flatten(); let resolved = crate::windows_system_nonstart::effective_outcome( ledger, Some(binding), Uuid::parse_str(&h.grant_id)?, )?; let same = resolved .as_ref() .or(existing.as_ref()) .map(|b| { serde_json::from_slice::(b) .and_then(|o| canonical_outcome_bytes(&o)) }) .transpose()? .is_some_and(|b| b == h.outcome_bytes); let state: String = db.query_row( "SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id], |r| r.get(0), )?; if same && state == h.state { continue; } let consumed: Option<(String, u64, String, String)> = ledger .query_row( "SELECT agent,generation,job,attempt FROM consumed WHERE grant_id=?1", [&h.grant_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), ) .optional()?; ensure!( consumed == Some(( binding.agent_id.to_string(), binding.enrollment_generation, h.job_id.clone(), h.attempt_id.clone() )), "HELPER_CONSUMPTION_NOT_KNOWN_LOCALLY" ); // Store actual helper evidence without unlocking result/consumption. A // crash here still leaves both gates closed and the original helper // result available; the next sync imports it again idempotently. ensure!( ledger.execute( "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1", params![h.grant_id, h.outcome_bytes] )? == 1, "LOCAL_START_EVIDENCE_MISSING" ); let next_revision = revision .max(h.revision) .checked_add(1) .context("REVISION_OVERFLOW")?; i64::try_from(next_revision)?; let seq: i64 = db.query_row( "UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq", [], |r| r.get(0), )?; let result = JobResult { job_id: h.job_id.clone(), attempt_id: h.attempt_id.clone(), revision: next_revision, state: o.state.clone(), grant_id: h.grant_id.clone(), stdout: o.stdout, stderr: o.stderr, output_truncated: o.output_truncated, reboot_required: o.reboot_required, exit_code: o.exit_code, evidence_json: o.evidence.to_string(), error_code: o.error_code, }; let envelope = Envelope { protocol_major: 1, protocol_minor: 0, message_id: Uuid::new_v4().to_string(), agent_id: binding.agent_id.to_string(), enrollment_generation: binding.enrollment_generation, connection_generation: channel.connection_generation, boot_id: channel.boot_id.to_string(), sequence: u64::try_from(seq)?, observed_at: now.timestamp_millis(), payload: Some(Payload::JobResult(result)), }; ensure!( envelope.encoded_len() <= rmm_domain::MAX_MESSAGE, "HELPER_RESULT_SIZE" ); db.execute( "INSERT INTO job_events(id,payload) VALUES(?1,?2)", params![envelope.message_id, envelope.encode_to_vec()], )?; db.execute( "UPDATE job_attempts SET state=?3,revision=?4,grant_id=?5 WHERE job=?1 AND attempt=?2", params![h.job_id, h.attempt_id, o.state, next_revision, h.grant_id], )?; } Ok(()) } fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()> { let mut seen = BTreeSet::new(); for h in &helper.attempts { ensure!( seen.insert((&h.job_id, &h.attempt_id)), "DUPLICATE_HELPER_ATTEMPT" ); let a = local .iter_mut() .find(|a| a.job_id == h.job_id && a.attempt_id == h.attempt_id) .context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?; ensure!( a.job_payload_hash == h.job_payload_hash && (a.grant_id.is_empty() || a.grant_id == h.grant_id), "HELPER_ATTEMPT_BINDING" ); if h.consumption != "unconsumed" { if !h.update_handoff_bytes.is_empty() { ensure!( h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(), "UPDATE_CANNOT_HEAL_UNKNOWN" ); a.update_handoff_bytes.clone_from(&h.update_handoff_bytes); } a.grant_id = h.grant_id.clone(); a.consumption = h.consumption.clone(); ensure!( a.outcome_sha256 == h.outcome_sha256, "HELPER_OUTCOME_MISMATCH" ); if a.original_boot_id.is_empty() { a.original_boot_id = h.original_boot_id.clone(); } ensure!( a.original_boot_id == h.original_boot_id, "HELPER_ORIGINAL_BOOT_MISMATCH" ); } } Ok(()) } fn validate_update_summary( _binding: &Binding, job: &JobPayload, offer: &JobOffer, local: &SyncAttempt, boot: Uuid, now: DateTime, ) -> Result { let proof = rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes) .map_err(anyhow::Error::msg)?; let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?; let (h, _) = rmm_domain::agent_update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?; let now = u64::try_from(now.timestamp())?; ensure!( rmm_domain::agent_update::handoff_matches_job(&h, job) && h.boot_id == boot && h.attempt.grant_id.to_string() == local.grant_id && h.attempt.authorization_id.to_string() == offer.authorization_id && h.attempt.plan_sha256 == offer.plan_hash && h.attempt.job_payload_sha256 == offer.job_payload_hash && local.original_boot_id == boot.to_string() && local.consumption == "consumed" && local.state == "running" && local.outcome_bytes.is_empty() && local.outcome_sha256.is_empty() && local.result.is_none() && local.result_envelope.is_none() && h.consumed_at <= now && now < h.finish_before, "UPDATE_INFLIGHT_SUMMARY_BINDING" ); Ok(h) } pub fn verify_response( binding: &Binding, request: &SyncRequest, channel: &Channel, response: &SyncResponse, now: DateTime, ) -> Result { verify_response_with( &DefaultVerifier::default(), binding, request, channel, response, now, ) } pub fn verify_response_with( verifier: &impl Ed25519Verifier, binding: &Binding, request: &SyncRequest, channel: &Channel, response: &SyncResponse, now: DateTime, ) -> Result { ensure!( response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64, "SYNC_SIGNATURE_SIZE" ); let raw = decode_key(&binding.public_key_hex)?; ensure!( verifier.verify(&raw, &response.decision_bytes, &response.signature)?, "INVALID_SYNC_SIGNATURE" ); let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; ensure!( d.valid_profile(now) && d.valid_until - d.issued_at <= chrono::Duration::seconds(60) && d.key_id == hash(&raw), "SYNC_PROFILE_OR_CLOCK" ); ensure!( d.tenant_id == binding.tenant_id && d.asset_id == binding.asset_id && d.agent_id == binding.agent_id && d.enrollment_generation == binding.enrollment_generation && d.assignment_version == binding.assignment_version && d.connection_generation == channel.connection_generation && d.boot_id == channel.boot_id && d.request_id.to_string() == request.request_id && d.request_hash == hash(&request.encode_to_vec()) && d.journal_id.to_string() == request.journal_id && d.checkpoint == request.checkpoint && d.helper_journal_id == optional_uuid(&request.helper_journal_id)? && d.helper_checkpoint == request.helper_checkpoint, "SYNC_DECISION_BINDING" ); Ok(d) } fn decode_key(hex: &str) -> Result> { ensure!(hex.len() == 64, "PINNED_KEY_SIZE"); (0..32) .map(|i| { Ok(u8::from_str_radix( hex.get(i * 2..i * 2 + 2).context("PINNED_KEY_ENCODING")?, 16, )?) }) .collect() } fn validate_attempt( job: &JobPayload, offer: &JobOffer, local: &SyncAttempt, d: &AttemptDecision, boot: Uuid, binding: &Binding, now: DateTime, ) -> Result>> { ensure!( d.job_payload_hash == offer.job_payload_hash && d.grant_id == optional_uuid(&local.grant_id)? && d.action == job.action.name() && d.execution_identity == job.execution_identity && (d.revision >= local.revision || d.outcome_state == "unknown") && d.original_boot_id == optional_uuid(&local.original_boot_id)? && d.previous_outcome_sha256.as_deref() == (!local.outcome_sha256.is_empty()).then_some(local.outcome_sha256.as_str()), "SYNC_ATTEMPT_BINDING" ); if d.grant_id.is_some() { ensure!( d.authorization_id == Some(Uuid::parse_str(&offer.authorization_id)?) && d.plan_hash.as_deref() == Some(offer.plan_hash.as_str()), "SYNC_AUTHORIZATION_BINDING" ); } ensure!( !terminal(&local.state) || d.outcome_state == local.state, "TERMINAL_STATE_REGRESSION" ); if d.update_inflight.is_some() || !local.update_handoff_bytes.is_empty() { validate_update_summary(binding, job, offer, local, boot, now)?; let proof = rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes) .map_err(anyhow::Error::msg)?; let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?; let expected = rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?; ensure!( d.update_inflight.as_ref() == Some(&expected) && d.outcome_state == "running" && !d.unlock && d.outcome_bytes.is_empty() && d.outcome_sha256.is_none(), "UPDATE_INFLIGHT_DECISION_CHANGED" ); return Ok(None); } if d.outcome_state == "accepted" { ensure!( local.state == "accepted" && local.consumption == "unconsumed" && d.grant_id.is_none() && !d.unlock && d.outcome_bytes.is_empty(), "SYNC_CANNOT_RESTART_ATTEMPT" ); return Ok(None); } if d.outcome_state == "unknown" { ensure!(!d.unlock, "UNKNOWN_CANNOT_UNLOCK"); return Ok(None); } ensure!(terminal(&d.outcome_state), "UNSUPPORTED_SYNC_STATE"); if d.outcome_bytes.is_empty() && d.outcome_sha256.is_none() { ensure!( local.consumption == "unconsumed" && d.grant_id.is_none() && !d.unlock && d.outcome_state != "succeeded", "TERMINAL_WITHOUT_EVIDENCE" ); return Ok(None); } let bytes = if d.outcome_bytes.is_empty() { local.outcome_bytes.clone() } else { d.outcome_bytes.clone() }; let o: ReconciledOutcome = serde_json::from_slice(&bytes)?; ensure!( canonical_outcome_bytes(&o)? == bytes && d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()) && o.state == d.outcome_state, "SYNC_CANONICAL_OUTCOME" ); if bytes != local.outcome_bytes && !operator_closure(&d.reason, &local.state, &o) { ensure!( matches!(job.action, rmm_domain::jobs::Action::Reboot { .. }) && d.outcome_state == "succeeded" && d.original_boot_id.is_some_and(|id| id != boot) && o.evidence["postconditions_passed"] == true && o.evidence["reconciled_after_reboot"] == true && o.evidence["new_boot_id"].as_str() == Some(boot.to_string().as_str()) && o.evidence["inventory_message_id"] .as_str() .is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())), "SYNC_REBOOT_REAUTH_EVIDENCE" ); } ensure!( !d.unlock || (local.consumption == "consumed" && d.grant_id.is_some()), "UNLOCK_WITHOUT_CONSUMPTION" ); Ok(Some(bytes)) } /// Operator regression only: imports the bounded, selected actual public /// fixture into two IN-MEMORY databases. No production path/network/executor. #[doc(hidden)] pub fn closed_control_contract(raw: &[u8]) -> Result { use rusqlite::types::Value as SqlValue; use serde_json::{Value, json}; ensure!(raw.len() <= 4 * 1024 * 1024, "CONTROL_FIXTURE_BOUND"); let f: Value = serde_json::from_slice(raw)?; ensure!( f["schema_version"] == 1 && f["read_only"] == true && f["database_writes"] == false && f["private_keys_read"] == false, "CONTROL_FIXTURE_SOURCE" ); let binding: Binding = serde_json::from_value(f["binding"].clone())?; let mut db = Connection::open_in_memory()?; let ledger = Connection::open_in_memory()?; initialize(&db)?; crate::ledger::initialize_ledger(&ledger)?; db.execute_batch("DELETE FROM reconcile_meta; CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT,PRIMARY KEY(job,attempt));CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE events(id TEXT PRIMARY KEY,payload BLOB)")?; ledger.execute_batch( "CREATE TABLE reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB)", )?; const RESOLUTION: &str = "CREATE TABLE windows_system_nonstart_221_resolution(grant_id TEXT PRIMARY KEY,role TEXT,receipt BLOB,receipt_sha256 TEXT,derived_outcome BLOB)"; db.execute_batch(RESOLUTION)?; ledger.execute_batch(RESOLUTION)?; fn cell(v: &Value) -> Result { let a = v.as_array().context("CONTROL_FIXTURE_CELL")?; Ok(match a.first().and_then(Value::as_str) { Some("null") if a.len() == 1 => SqlValue::Null, Some("integer") if a.len() == 2 => { SqlValue::Integer(a[1].as_i64().context("CONTROL_FIXTURE_INTEGER")?) } Some("text") if a.len() == 2 => { SqlValue::Text(a[1].as_str().context("CONTROL_FIXTURE_TEXT")?.into()) } Some("blob") if a.len() == 2 => SqlValue::Blob(serde_json::from_value(a[1].clone())?), _ => anyhow::bail!("CONTROL_FIXTURE_CELL_TYPE"), }) } fn import(db: &Connection, rows: &Value, sql: &str) -> Result<()> { let rows = rows.as_array().context("CONTROL_FIXTURE_ROWS")?; ensure!(rows.len() <= 8, "CONTROL_FIXTURE_ROW_BOUND"); for row in rows { let values = row .as_array() .context("CONTROL_FIXTURE_ROW")? .iter() .map(cell) .collect::>>()?; db.execute(sql, rusqlite::params_from_iter(values))?; } Ok(()) } for (key, sql) in [ ( "meta", "INSERT INTO reconcile_meta VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)", ), ("counter", "INSERT INTO counter VALUES(?1,?2)"), ( "requests", "INSERT INTO reconcile_requests VALUES(?1,?2,?3,?4,?5)", ), ("jobs", "INSERT INTO job_attempts VALUES(?1,?2,?3,?4,?5,?6)"), ( "outcomes", "INSERT INTO reconcile_outcomes VALUES(?1,?2,?3,?4)", ), ( "results", "INSERT INTO reconcile_results VALUES(?1,?2,?3,?4,?5)", ), ("events", "INSERT INTO job_events VALUES(?1,?2)"), ( "resolution", "INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)", ), ] { import(&db, &f["agent"][key], sql)?; } ensure!( f["agent"]["legacy_count"] == 0, "CONTROL_FIXTURE_PENDING_LEGACY" ); for (key, sql) in [ ("consumed", "INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,?6)"), ( "evidence", "INSERT INTO execution_evidence VALUES(?1,?2,?3)", ), ("acks", "INSERT INTO reconcile_consumption VALUES(?1,?2)"), ( "resolution", "INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)", ), ] { import(&ledger, &f["ledger"][key], sql)?; } let captures = f["helper"]["capture"] .as_array() .context("CONTROL_FIXTURE_CAPTURE")?; ensure!(captures.len() == 1, "CONTROL_FIXTURE_CAPTURE_COUNT"); let SqlValue::Blob(raw_snapshot) = cell(&captures[0][1])? else { anyhow::bail!("CONTROL_FIXTURE_CAPTURE_BYTES") }; let snapshot: crate::helper_reconcile::Snapshot = serde_json::from_slice(&raw_snapshot)?; let m = f["helper"]["meta"] .as_array() .context("CONTROL_FIXTURE_HELPER_META")?; ensure!(m.len() == 1, "CONTROL_FIXTURE_HELPER_META_COUNT"); ensure!( cell(&m[0][0])? == SqlValue::Text(snapshot.journal_id.to_string()) && cell(&m[0][1])? == SqlValue::Integer(i64::try_from(snapshot.checkpoint)?) && cell(&m[0][3])? == SqlValue::Text( snapshot .previous_decision_id .context("CONTROL_FIXTURE_HELPER_HEAD")? .to_string() ) && cell(&m[0][6])? == SqlValue::Null, "CONTROL_FIXTURE_ACTUAL_HELPER_CAPTURE" ); crate::helper_reconcile::precommit_clock_contract(binding.clone(), snapshot.clone())?; let helper = HelperSnapshot { journal_id: snapshot.journal_id, checkpoint: snapshot.checkpoint, previous_decision_id: snapshot.previous_decision_id, attempts: snapshot .attempts .iter() .map(|a| SyncAttempt { job_id: a.job_id.to_string(), attempt_id: a.attempt_id.to_string(), job_payload_hash: a.job_payload_hash.clone(), grant_id: a.grant_id.to_string(), consumption: a.consumption.clone(), state: a.state.clone(), revision: a.revision, original_boot_id: a.original_boot_id.to_string(), outcome_sha256: a.outcome_sha256.clone(), outcome_bytes: a.outcome_bytes.clone(), update_handoff_bytes: a.update_handoff_bytes.clone(), ..Default::default() }) .collect(), }; let meta = metadata(&db)?; let helper_generation = match cell(&m[0][2])? { SqlValue::Integer(v) => u64::try_from(v)?, _ => anyhow::bail!("CONTROL_FIXTURE_HELPER_GENERATION"), }; let channel = Channel { connection_generation: meta .generation .max(helper_generation) .checked_add(1) .context("CONTROL_FIXTURE_GENERATION_OVERFLOW")?, boot_id: snapshot.boot_id, }; let now: DateTime = serde_json::from_value(f["captured_at_utc"].clone())?; let client = ClientSync::with_verifier(binding.clone(), crate::crypto::DalekVerifier) .with_helper_identity(HelperIdentity::System); let before = (db.total_changes(), ledger.total_changes()); client.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)?; ensure!( (db.total_changes(), ledger.total_changes()) == before, "CONTROL_ADMISSION_WROTE_HISTORY" ); ledger.execute( "INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,NULL)", params![ Uuid::from_u128(2).to_string(), binding.agent_id.to_string(), binding.enrollment_generation, Uuid::from_u128(3).to_string(), Uuid::from_u128(4).to_string() ], )?; ensure!( client .closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now) .is_err(), "CONTROL_HIDDEN_OPEN_GRANT_NOT_REJECTED" ); ledger.execute( "DELETE FROM consumed WHERE grant_id=?1", [Uuid::from_u128(2).to_string()], )?; let a = meta.decision.context("CONTROL_FIXTURE_A")?; let (request, _) = client.stored_control(&db, &ledger, a, channel.boot_id)?; let (original_raw, original_env, original_response): (Vec, Vec, Vec) = db .query_row( "SELECT request,envelope,response FROM reconcile_requests WHERE id=?1", [request.request_id.clone()], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), )?; let mut changed = request.clone(); changed.helper_previous_decision_id = Uuid::from_u128(1).to_string(); let mut env = Envelope::decode(original_env.as_slice())?; env.payload = Some(Payload::SyncRequest(changed.clone())); db.execute( "UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3", params![ changed.encode_to_vec(), env.encode_to_vec(), request.request_id ], )?; ensure!( client .closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now) .is_err(), "CONTROL_MUTATED_A_NOT_REJECTED" ); db.execute( "UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3", params![original_raw, original_env, request.request_id], )?; let mut changed = SyncResponse::decode(original_response.as_slice())?; changed.signature[0] ^= 1; let bytes = changed.encode_to_vec(); db.execute( "UPDATE reconcile_requests SET response=?1 WHERE id=?2", params![bytes, request.request_id], )?; ledger.execute( "UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2", params![bytes, a.to_string()], )?; ensure!( client .closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now) .is_err(), "CONTROL_MUTATED_SERVER_NOT_REJECTED" ); db.execute( "UPDATE reconcile_requests SET response=?1 WHERE id=?2", params![original_response, request.request_id], )?; ledger.execute( "UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2", params![original_response, a.to_string()], )?; struct CurrentClock(DateTime); impl SyncClock for CurrentClock { fn now(&self) -> Result> { Ok(self.0) } } let clock = CurrentClock(now); let mut client = ClientSync::with_verifier(binding, crate::crypto::DalekVerifier) .with_helper_identity(HelperIdentity::System); ensure!( client .prepare(&mut db, &ledger, channel.clone(), Some(&helper), &clock) .is_err(), "CONTROL_IDLE_ATTESTATION_REQUIRED" ); let fresh = client.prepare_with_closed_control_idle( &mut db, &ledger, channel.clone(), Some(&helper), &clock, true, )?; ensure!( fresh.request.previous_decision_id == a.to_string() && fresh.request.helper_previous_decision_id == helper.previous_decision_id.unwrap().to_string() && fresh.envelope.connection_generation == channel.connection_generation && fresh.envelope.observed_at == now.timestamp_millis() && !client.can_start(&channel, false, &clock) && !client.can_start(&channel, true, &clock) && client.gate_observation().is_none(), "CONTROL_FRESH_REQUEST_MUST_NOT_AUTHORIZE" ); Ok( json!({"actual_closed_partial_edge_accepted":true,"mutated_agent_request_rejected":true,"mutated_server_signature_and_ack_rejected":true,"admission_read_only":true,"hidden_open_grant_rejected":true,"explicit_idle_attestation_required":true,"typed_precommit_clock_keeps_capture_without_writes":true,"fresh_request_keeps_both_heads":true,"old_authority_reused":false,"new_ready_or_unlock":false,"product_databases_opened":false,"effects":false}), ) } #[cfg(test)] mod closure_tests { use super::*; #[test] fn only_a_human_closure_of_a_locally_unknown_attempt_is_accepted() { let closed = ReconciledOutcome { state: "closed_unknown".into(), stdout: vec![], stderr: vec![], exit_code: None, output_truncated: false, reboot_required: false, evidence: serde_json::json!({"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"retry":false}), error_code: "OPERATOR_CLOSED_UNKNOWN".into(), }; assert!(operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &closed)); assert!(!operator_closure("DEFINITIVE_OUTCOME_RECONCILED", "unknown", &closed)); assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "running", &closed)); assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "succeeded", &closed)); for case in 0..6 { let mut other = closed.clone(); match case { 0 => other.state = "failed".into(), 1 => other.error_code = "SCRIPT_FAILED".into(), 2 => other.exit_code = Some(0), 3 => other.stdout = b"x".to_vec(), 4 => other.evidence["postconditions_passed"] = true.into(), _ => other.evidence["retry"] = true.into(), } assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &other)); } assert!(terminal("closed_unknown")); } }