Files
rust-discovery/crates/agent-core/src/reconcile.rs
T

2307 lines
95 KiB
Rust

//! Reconnect reconciliation, deliberately independent of main/jobs and transport.
//! No method executes a job, deletes consumption tombstones or issues a permit.
//! Call only while holding the existing agent.lock. HelperSnapshot must come
//! from the independently authenticated root-helper IPC, never from the server.
use crate::auth::Binding;
pub use crate::auth::HelperIdentity;
use crate::crypto::{DefaultVerifier, Ed25519Verifier};
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Utc};
use prost::Message;
use rmm_domain::{
jobs::{JobPayload, hash},
reconciliation::{
AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes,
},
};
use rmm_protocol::{
Envelope, JobOffer, JobResult, Payload, SyncAttempt, SyncRequest, SyncResponse,
};
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
use std::collections::{BTreeMap, BTreeSet};
use uuid::Uuid;
/// A fresh observation returned by the trusted local clock implementation,
/// never an IPC DTO or an execution permit. Callers must not retain it as a lease.
pub struct ClockReading {
pub own_wall: DateTime<Utc>,
pub upper: DateTime<Utc>,
}
pub trait SyncClock {
fn now(&self) -> Result<DateTime<Utc>>;
/// Existing providers retain their existing point-clock contract. Providers
/// with an independently verified interval override this atomically.
fn reading(&self) -> Result<ClockReading> {
let own_wall = self.now()?;
Ok(ClockReading {
own_wall,
upper: own_wall,
})
}
/// Full trusted UTC interval for crypto validity. Existing point-clock
/// providers retain their contract; interval providers override atomically.
fn bounds(&self) -> Result<(DateTime<Utc>, DateTime<Utc>)> {
let reading = self.reading()?;
ensure!(reading.own_wall <= reading.upper, "CLOCK_INTERVAL_ORDER");
Ok((reading.own_wall, reading.upper))
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Channel {
pub connection_generation: u64,
pub boot_id: Uuid,
}
#[derive(Clone, Debug)]
pub struct HelperSnapshot {
pub journal_id: Uuid,
pub checkpoint: u64,
pub previous_decision_id: Option<Uuid>,
/// Complete consumed/unsettled summaries from the root-owned ledger.
pub attempts: Vec<SyncAttempt>,
}
#[derive(Clone)]
pub struct PreparedSync {
pub envelope: Envelope,
pub request: SyncRequest,
}
/// Pass the ORIGINAL signed response to the helper. It must independently verify
/// it, apply targeted unlocks and commit its checkpoint before acknowledging.
#[derive(Clone)]
pub struct HelperInstruction {
pub response: SyncResponse,
pub decision_id: Uuid,
pub journal_id: Uuid,
pub checkpoint: u64,
}
pub struct Applied {
pub decision_id: Uuid,
pub acknowledged_message_ids: Vec<Uuid>,
pub helper_instruction: Option<HelperInstruction>,
pub reason: String,
}
/// Diagnostics only; never accepted as an authorization or reusable lease.
#[derive(Clone, Copy)]
pub struct GateObservation {
pub signed_ready: bool,
pub local_applied: bool,
pub helper_ack_pending: bool,
}
struct Gate {
channel: Channel,
until: DateTime<Utc>,
ready: bool,
helper_ready: bool,
helper: Option<(Uuid, u64, Uuid)>,
local_applied: bool,
request: SyncRequest,
response: SyncResponse,
}
pub struct ClientSync<V = DefaultVerifier> {
binding: Binding,
verifier: V,
helper_identity: HelperIdentity,
pending: Option<PreparedSync>,
channel: Option<Channel>,
gate: Option<Gate>,
}
impl ClientSync<DefaultVerifier> {
/// Every new process starts closed, independently of any persisted ready bit.
pub fn new(binding: Binding) -> Self {
Self::with_verifier(binding, DefaultVerifier::default())
}
}
impl<V: Ed25519Verifier> ClientSync<V> {
pub fn with_verifier(binding: Binding, verifier: V) -> Self {
Self {
binding,
verifier,
helper_identity: HelperIdentity::Root,
pending: None,
channel: None,
gate: None,
}
}
/// Configure one explicit helper identity; changing it discards all leases.
pub fn with_helper_identity(mut self, identity: HelperIdentity) -> Self {
self.close();
self.helper_identity = identity;
self
}
/// Invoke on every authenticated/fenced ServerHello or transport loss.
pub fn close(&mut self) {
self.gate = None;
self.channel = None;
self.pending = None;
}
pub fn gate_observation(&self) -> Option<GateObservation> {
self.gate.as_ref().map(|g| GateObservation {
signed_ready: g.ready,
local_applied: g.local_applied,
helper_ack_pending: g.helper.is_some(),
})
}
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
self.gate.as_ref().map(|gate| gate.until)
}
pub fn can_start(&self, channel: &Channel, root: bool, clock: &impl SyncClock) -> bool {
let Ok(reading) = clock.reading() else {
return false;
};
self.gate.as_ref().is_some_and(|g| {
g.channel == *channel
&& g.ready
&& g.local_applied
&& reading.own_wall < g.until
&& reading.upper < g.until
&& (!root || (g.helper_ready && g.helper.is_none()))
})
}
/// Only call with a fresh independently authenticated helper snapshot. There
/// is no unsigned ready/unlock flag: the exact signed decision/checkpoint
/// must already be durably applied by that helper.
pub fn confirm_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
ensure!(
snapshot.journal_id == expected.0
&& snapshot.checkpoint == expected.1
&& snapshot.previous_decision_id == Some(expected.2),
"HELPER_CHECKPOINT_NOT_APPLIED"
);
ensure!(
snapshot
.attempts
.iter()
.all(|a| terminal(&a.state)
|| (a.consumption == "unconsumed" && a.state == "accepted")),
"HELPER_EFFECT_UNRESOLVED"
);
gate.helper = None;
Ok(())
}
/// Typed read-only ACK for exactly the own original update handoff. It
/// clears only the control delivery acknowledgement; both start flags stay
/// false and can_start remains false for ALL jobs, including this attempt.
pub fn confirm_update_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
ensure!(
!gate.ready
&& !gate.helper_ready
&& snapshot.journal_id == expected.0
&& snapshot.checkpoint == expected.1
&& snapshot.previous_decision_id == Some(expected.2),
"UPDATE_CONTROL_NOT_APPLIED"
);
let updates: Vec<_> = snapshot
.attempts
.iter()
.filter(|a| !a.update_handoff_bytes.is_empty())
.collect();
ensure!(updates.len() == 1, "UPDATE_CONTROL_EXACTLY_ONE_REQUIRED");
let local = updates[0];
let requested = gate
.request
.attempts
.iter()
.find(|a| a.job_id == local.job_id && a.attempt_id == local.attempt_id)
.context("UPDATE_CONTROL_NOT_REQUESTED")?;
ensure!(
local.state == "running"
&& local.consumption == "consumed"
&& local.outcome_bytes.is_empty()
&& local.update_handoff_bytes == requested.update_handoff_bytes
&& local.grant_id == requested.grant_id
&& local.job_payload_hash == requested.job_payload_hash,
"UPDATE_CONTROL_CHANGED"
);
let decision: ReconcileDecisionPayload =
serde_json::from_slice(&gate.response.decision_bytes)?;
ensure!(
decision
.attempts
.iter()
.any(|d| d.job_id.to_string() == local.job_id
&& d.attempt_id.to_string() == local.attempt_id
&& d.update_inflight.is_some()
&& !d.unlock),
"UPDATE_CONTROL_NOT_SIGNED"
);
ensure!(
snapshot
.attempts
.iter()
.filter(|a| a.job_id != local.job_id || a.attempt_id != local.attempt_id)
.all(|a| terminal(&a.state)
|| (a.state == "accepted" && a.consumption == "unconsumed")),
"UPDATE_OTHER_EFFECT_UNRESOLVED"
);
gate.helper = None;
Ok(())
}
/// Historical authentication only. Never apply an old response or create a
/// lease: the returned bytes prove a previously durable local control step.
fn stored_control(
&self,
db: &Connection,
ledger: &Connection,
id: Uuid,
boot: Uuid,
) -> Result<(SyncRequest, ReconcileDecisionPayload)> {
let ack: Vec<u8> = ledger.query_row(
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
[id.to_string()],
|r| r.get(0),
)?;
ensure!(
ack.len() <= rmm_domain::MAX_MESSAGE,
"CLOSED_CONTROL_ACK_BOUND"
);
let response = SyncResponse::decode(ack.as_slice())?;
ensure!(
response.encode_to_vec() == ack,
"CLOSED_CONTROL_CANONICAL_ACK"
);
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
let (raw, envelope, generation, saved): (Vec<u8>, Vec<u8>, u64, Vec<u8>) = db.query_row(
"SELECT request,envelope,generation,response FROM reconcile_requests WHERE id=?1",
[parsed.request_id.to_string()],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)?;
ensure!(
raw.len() <= rmm_domain::MAX_MESSAGE
&& envelope.len() <= rmm_domain::MAX_MESSAGE
&& saved == ack,
"CLOSED_CONTROL_ORIGINAL_BYTES"
);
let request = SyncRequest::decode(raw.as_slice())?;
let env = Envelope::decode(envelope.as_slice())?;
ensure!(
raw == request.encode_to_vec()
&& env.encode_to_vec() == envelope
&& env.payload == Some(Payload::SyncRequest(request.clone()))
&& rmm_protocol::valid_envelope(
&env,
&self.binding.agent_id.to_string(),
generation
)
&& env.boot_id == boot.to_string()
&& env.sequence == request.checkpoint,
"CLOSED_CONTROL_ORIGINAL_ENVELOPE"
);
let decision = verify_response_with(
&self.verifier,
&self.binding,
&request,
&Channel {
connection_generation: generation,
boot_id: boot,
},
&response,
parsed.issued_at,
)?;
ensure!(
decision.decision_id == id
&& decision.boot_id == boot
&& !recovery_reason(&decision.reason),
"CLOSED_CONTROL_SIGNED_ORIGINAL"
);
Ok((request, decision))
}
/// Admit only a new, ordinary online reconciliation of an exact missing
/// helper ACK with already definitive, identical outcomes. The caller owns
/// agent.lock, has no live executor/deferred work, and obtained `helper`
/// from the serialized, independently authenticated current-boot helper.
/// This predicate is read-only; fresh server/own-clock/helper ACK gates are
/// unchanged and no flag or authority from either old response is reused.
fn closed_control_continuation(
&self,
db: &Connection,
ledger: &Connection,
channel: &Channel,
helper: &HelperSnapshot,
meta: &Meta,
now: DateTime<Utc>,
) -> Result<()> {
ensure!(
self.helper_identity == HelperIdentity::System
&& self.pending.is_none()
&& self.gate.is_none()
&& meta.fault.is_none()
&& meta.pending_request.is_none(),
"CLOSED_CONTROL_IDLE_ONLY"
);
let a = meta
.decision
.context("CLOSED_CONTROL_AGENT_HEAD_REQUIRED")?;
let h = helper
.previous_decision_id
.context("CLOSED_CONTROL_HELPER_HEAD_REQUIRED")?;
ensure!(
a != h
&& Some(helper.journal_id) == meta.helper_id
&& helper.checkpoint >= meta.helper_checkpoint
&& pending_ids(db)?.is_empty(),
"CLOSED_CONTROL_CURRENT_HEADS"
);
let legacy: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='events')",
[],
|r| r.get(0),
)?;
ensure!(
!legacy
|| db.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, u64>(0))? == 0,
"CLOSED_CONTROL_NO_PENDING_EVENTS"
);
let (original, decision) = self.stored_control(db, ledger, a, channel.boot_id)?;
let (_, previous) = self.stored_control(db, ledger, h, channel.boot_id)?;
ensure!(
decision.journal_id == meta.journal_id
&& decision.checkpoint == u64::try_from(meta.checkpoint)?
&& original.journal_id == meta.journal_id.to_string()
&& original.checkpoint == decision.checkpoint
&& original.previous_decision_id == h.to_string()
&& original.helper_previous_decision_id == h.to_string()
&& original.previous_request_id.is_empty()
&& original.journal_complete
&& decision.issued_at <= now
&& decision.connection_generation <= channel.connection_generation
&& previous.connection_generation <= decision.connection_generation
&& previous.checkpoint < decision.checkpoint
&& previous.journal_id == decision.journal_id
&& previous.helper_journal_id == Some(helper.journal_id)
&& decision.helper_journal_id == Some(helper.journal_id)
&& original.helper_journal_id == helper.journal_id.to_string()
&& previous.helper_checkpoint < decision.helper_checkpoint
&& decision.helper_checkpoint == meta.helper_checkpoint
&& helper.checkpoint >= decision.helper_checkpoint,
"CLOSED_CONTROL_EXACT_PARTIAL_EDGE"
);
verify_closed_result_messages(db, &original, &decision)?;
let current = attempts(db, ledger, &self.binding)?;
ensure!(
current.len() == original.attempts.len()
&& current.len() == decision.attempts.len()
&& current.len() <= 256,
"CLOSED_CONTROL_EXACT_ATTEMPT_SET"
);
let expected_consumed = current
.iter()
.filter(|r| r.consumption == "consumed")
.count() as u64;
let ledger_counts: (u64, u64, u64) = ledger.query_row(
"SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM execution_evidence),(SELECT count(*) FROM consumed WHERE result IS NULL)",
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
ensure!(
ledger_counts == (expected_consumed, expected_consumed, 0),
"CLOSED_CONTROL_EXACT_COMPLETE_LEDGER"
);
for row in &current {
ensure!(
matches!(row.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired")
&& row.update_handoff_bytes.is_empty(),
"CLOSED_CONTROL_DEFINITIVE_ONLY"
);
let old = original
.attempts
.iter()
.find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id)
.context("CLOSED_CONTROL_ORIGINAL_ATTEMPT")?;
let signed = decision
.attempts
.iter()
.find(|r| {
r.job_id.to_string() == row.job_id && r.attempt_id.to_string() == row.attempt_id
})
.context("CLOSED_CONTROL_SIGNED_ATTEMPT")?;
ensure!(
old.state == row.state
&& old.job_payload_hash == row.job_payload_hash
&& old.grant_id == row.grant_id
&& old.consumption == row.consumption
&& old.original_boot_id == row.original_boot_id
&& old.outcome_bytes == row.outcome_bytes
&& old.outcome_sha256 == row.outcome_sha256
&& old.update_handoff_bytes.is_empty()
&& row.revision == old.revision.max(signed.revision)
&& signed.outcome_state == row.state
&& signed.update_inflight.is_none(),
"CLOSED_CONTROL_RESULT_UNCHANGED"
);
// An ungranted expiry has no fabricated outcome. Its exact
// definitive nonstart is already authenticated in the original
// request/decision; it can only permit a NEW online request.
if row.consumption == "unconsumed" {
let consumed: u64 = ledger.query_row(
"SELECT count(*) FROM consumed WHERE job=?1 AND attempt=?2",
params![row.job_id, row.attempt_id],
|r| r.get(0),
)?;
ensure!(
matches!(row.state.as_str(), "expired" | "cancelled")
&& row.grant_id.is_empty()
&& row.original_boot_id.is_empty()
&& row.outcome_bytes.is_empty()
&& row.outcome_sha256.is_empty()
&& signed.grant_id.is_none()
&& !signed.unlock
&& signed.reason == "DEFINITIVE_NONSTART"
&& signed.previous_outcome_sha256.is_none()
&& signed.outcome_sha256.is_none()
&& signed.outcome_bytes.is_empty()
&& consumed == 0
&& !helper
.attempts
.iter()
.any(|h| h.job_id == row.job_id && h.attempt_id == row.attempt_id),
"CLOSED_CONTROL_SIGNED_UNGRANTED_NONSTART"
);
continue;
}
ensure!(
!row.outcome_bytes.is_empty()
&& signed.grant_id.map(|id| id.to_string()).as_deref()
== Some(row.grant_id.as_str())
&& hash(&row.outcome_bytes) == row.outcome_sha256
&& signed.outcome_sha256.as_deref() == Some(row.outcome_sha256.as_str())
&& (signed.outcome_bytes.is_empty()
|| signed.outcome_bytes == row.outcome_bytes),
"CLOSED_CONTROL_SIGNED_CANONICAL_RESULT"
);
let outcome: ReconciledOutcome = serde_json::from_slice(&row.outcome_bytes)?;
ensure!(
canonical_outcome_bytes(&outcome)? == row.outcome_bytes
&& outcome.state == row.state,
"CLOSED_CONTROL_CANONICAL_RESULT"
);
match row.consumption.as_str() {
"consumed" => {
let (start,finished,last):(Vec<u8>,Vec<u8>,Vec<u8>)=ledger.query_row("SELECT e.start,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1 AND c.job=?2 AND c.attempt=?3 AND c.agent=?4 AND c.generation=?5",params![row.grant_id,row.job_id,row.attempt_id,self.binding.agent_id.to_string(),self.binding.enrollment_generation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
let saved: serde_json::Value = serde_json::from_slice(&start)?;
let request: crate::auth::Request =
serde_json::from_value(saved["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
let (job, permit) =
self.binding
.verify_with(&request, permit.not_before, &self.verifier)?;
ensure!(
job.job_id.to_string() == row.job_id
&& job.attempt_id.to_string() == row.attempt_id
&& hash(&request.job_bytes) == row.job_payload_hash
&& permit.grant_id.to_string() == row.grant_id
&& !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. })
&& canonical_outcome_bytes(&serde_json::from_slice::<
ReconciledOutcome,
>(&finished)?)?
== row.outcome_bytes,
"CLOSED_CONTROL_ORIGINAL_CONSUMPTION"
);
if crate::windows_system_nonstart::effective_outcome(
ledger,
Some(&self.binding),
permit.grant_id,
)?
.is_none()
{
ensure!(
canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&last
)?)? == row.outcome_bytes,
"CLOSED_CONTROL_ORIGINAL_RESULT_CONFLICT"
);
}
if job.execution_identity == self.helper_identity.as_str() {
let r = helper
.attempts
.iter()
.find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id)
.context("CLOSED_CONTROL_HELPER_ATTEMPT_REQUIRED")?;
ensure!(
r.state == row.state
&& r.grant_id == row.grant_id
&& r.job_payload_hash == row.job_payload_hash
&& r.consumption == "consumed"
&& r.outcome_bytes == row.outcome_bytes
&& r.outcome_sha256 == row.outcome_sha256
&& r.original_boot_id == row.original_boot_id
&& r.revision <= row.revision
&& r.update_handoff_bytes.is_empty(),
"CLOSED_CONTROL_HELPER_RESULT_UNCHANGED"
);
}
}
_ => anyhow::bail!("CLOSED_CONTROL_UNCERTAIN_CONSUMPTION"),
}
}
for row in &helper.attempts {
ensure!(
terminal(&row.state)
&& row.update_handoff_bytes.is_empty()
&& current.iter().any(|r| r.job_id == row.job_id
&& r.attempt_id == row.attempt_id
&& r.grant_id == row.grant_id
&& r.outcome_bytes == row.outcome_bytes),
"CLOSED_CONTROL_NO_EXTRA_HELPER_ATTEMPT"
);
}
Ok(())
}
pub fn prepare(
&mut self,
db: &mut Connection,
ledger: &Connection,
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
) -> Result<PreparedSync> {
self.prepare_with_closed_control_idle(db, ledger, channel, helper, clock, false)
}
/// The OS actor may attest quiescence only after its execution/deferred
/// workers and protected intent store are all settled. Ordinary same-head
/// synchronization does not need this attestation, including while busy.
pub fn prepare_with_closed_control_idle(
&mut self,
db: &mut Connection,
ledger: &Connection,
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
closed_control_idle: bool,
) -> Result<PreparedSync> {
let result = self.prepare_inner(db, ledger, channel, helper, clock, closed_control_idle);
if let Err(error) = &result {
let text = error.to_string();
if [
"ROLLBACK",
"REGRESSION",
"DISAPPEARED",
"MISSING_FROM_AGENT_JOURNAL",
"MISSING_FROM_HELPER_JOURNAL",
]
.iter()
.any(|s| text.contains(s))
{
let _ = db.execute(
"UPDATE reconcile_meta SET fault=COALESCE(fault,?1) WHERE id=1",
[text],
);
}
}
result
}
fn prepare_inner(
&mut self,
db: &mut Connection,
ledger: &Connection,
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
closed_control_idle: bool,
) -> Result<PreparedSync> {
self.close();
let now = clock.now()?;
ensure!(
channel.connection_generation > 0 && !channel.boot_id.is_nil(),
"UNAUTHENTICATED_CHANNEL"
);
initialize(db)?;
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let meta = metadata(&tx)?;
ensure!(meta.fault.is_none(), "SYNC_MANUAL_RECOVERY_REQUIRED");
let sequence: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
ensure!(
sequence >= meta.checkpoint && sequence >= meta.high_sequence,
"LOCAL_CHECKPOINT_ROLLBACK"
);
ensure!(
channel.connection_generation >= meta.generation,
"CONNECTION_GENERATION_REGRESSION"
);
if let Some(h) = helper {
ensure!(
!h.journal_id.is_nil() && h.checkpoint <= i64::MAX as u64,
"HELPER_CHECKPOINT"
);
if let Some(old) = meta.helper_id {
ensure!(
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
"HELPER_JOURNAL_ROLLBACK"
);
if self.helper_identity == HelperIdentity::System {
// A locally committed Windows decision is not proof of its
// application by SYSTEM. Never advance either checkpoint
// from an older helper head, including after process loss.
if meta.decision.is_none() || h.previous_decision_id != meta.decision {
ensure!(closed_control_idle, "SYSTEM_HELPER_APPLY_UNCONFIRMED");
self.closed_control_continuation(&tx, ledger, &channel, h, &meta, now)
.context("SYSTEM_HELPER_APPLY_UNCONFIRMED")?;
}
}
}
} else {
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
}
if let Some(h) = helper {
ingest_helper(
&tx,
ledger,
(&self.binding, &self.verifier),
&channel,
h,
now,
self.helper_identity,
)?;
}
archive_events(&tx)?;
tx.execute(
"UPDATE job_attempts SET state='accepted' WHERE state='queued' AND grant_id=''",
[],
)?;
// A possibly started attempt is NEVER returned to accepted. Preserve
// revisions and results; only ambiguous no-result states become unknown.
let inflight: Vec<_> = helper
.into_iter()
.flat_map(|h| h.attempts.iter())
.filter(|a| !a.update_handoff_bytes.is_empty())
.collect();
ensure!(inflight.len() <= 1, "UPDATE_MULTIPLE_INFLIGHT");
if let Some(update) = inflight.first() {
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT (job=?1 AND attempt=?2) AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",params![update.job_id,update.attempt_id])?;
} else {
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?;
}
let mut attempts = attempts(&tx, ledger, &self.binding)?;
if let Some(h) = helper {
merge_helper(&mut attempts, h)?;
}
ensure!(attempts.len() <= 256, "SYNC_ATTEMPT_CAPACITY_NO_COMPACTION");
let mut ids = pending_ids(&tx)?;
if let Some(update) = inflight.first() {
let grant = Uuid::parse_str(&update.grant_id)?;
ids.extend(crate::update_health_inventory::pending_ids(&tx, grant)?);
ids.sort();
ids.dedup();
}
ensure!(ids.len() <= 256, "SYNC_RECEIPT_CAPACITY");
let next: i64 = tx.query_row(
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
[],
|r| r.get(0),
)?;
let request = SyncRequest {
request_id: Uuid::new_v4().to_string(),
journal_id: meta.journal_id.to_string(),
checkpoint: u64::try_from(next)?,
// A pending request may have committed with a lost reply, or may
// never have reached the server. Preserve both witnesses: the
// server resolves the pending request first and otherwise checks
// this last applied decision against its authoritative head.
previous_decision_id: meta.decision.map(|id| id.to_string()).unwrap_or_default(),
assignment_version: self.binding.assignment_version,
attempts,
durable_message_ids: ids,
journal_complete: true,
helper_journal_id: helper.map(|h| h.journal_id.to_string()).unwrap_or_default(),
helper_checkpoint: helper.map_or(0, |h| h.checkpoint),
helper_previous_decision_id: helper
.and_then(|h| h.previous_decision_id)
.map(|id| id.to_string())
.unwrap_or_default(),
previous_request_id: meta
.pending_request
.map(|id| id.to_string())
.unwrap_or_default(),
};
let envelope = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: self.binding.agent_id.to_string(),
enrollment_generation: self.binding.enrollment_generation,
connection_generation: channel.connection_generation,
boot_id: channel.boot_id.to_string(),
sequence: u64::try_from(next)?,
observed_at: now.timestamp_millis(),
payload: Some(Payload::SyncRequest(request.clone())),
};
ensure!(
envelope.encoded_len() <= rmm_domain::MAX_MESSAGE,
"SYNC_ENVELOPE_SIZE"
);
tx.execute(
"INSERT INTO reconcile_requests(id,request,envelope,generation) VALUES(?1,?2,?3,?4)",
params![
request.request_id,
request.encode_to_vec(),
envelope.encode_to_vec(),
channel.connection_generation
],
)?;
tx.execute("UPDATE reconcile_meta SET checkpoint=?1,high_sequence=?1,generation=?2,pending_request=?3 WHERE id=1",params![next,channel.connection_generation,request.request_id])?;
tx.commit()?;
let prepared = PreparedSync { envelope, request };
self.pending = Some(prepared.clone());
self.channel = Some(channel);
Ok(prepared)
}
/// All changes, concrete ACKs, result archives and checkpoint chain commit
/// together. A rejection closes only mutation authority; telemetry may run.
pub fn apply(
&mut self,
db: &mut Connection,
response: &SyncResponse,
clock: &impl SyncClock,
) -> Result<Applied> {
if self.pending.is_none() {
let result = self.replay_applied(db, response, clock);
if result.is_err() {
self.gate = None;
}
return result;
}
self.gate = None;
let result = self.apply_inner(db, response, clock);
if result.is_err() {
self.gate = None;
}
result
}
fn replay_applied(
&self,
db: &Connection,
response: &SyncResponse,
clock: &impl SyncClock,
) -> Result<Applied> {
let gate = self.gate.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?;
ensure!(gate.response == *response, "REPLAYED_DIFFERENT_DECISION");
let d = verify_response_with(
&self.verifier,
&self.binding,
&gate.request,
&gate.channel,
response,
clock.now()?,
)?;
let meta = metadata(db)?;
let sequence: i64 = db.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
ensure!(
meta.decision == Some(d.decision_id)
&& meta.pending_request.is_none()
&& meta.checkpoint == i64::try_from(d.checkpoint)?
&& sequence >= meta.high_sequence,
"APPLIED_CHECKPOINT_REGRESSION"
);
Ok(Applied {
decision_id: d.decision_id,
acknowledged_message_ids: d.acknowledged_message_ids,
helper_instruction: d.helper_journal_id.map(|journal_id| HelperInstruction {
response: response.clone(),
decision_id: d.decision_id,
journal_id,
checkpoint: d.helper_checkpoint,
}),
reason: d.reason,
})
}
fn apply_inner(
&mut self,
db: &mut Connection,
response: &SyncResponse,
clock: &impl SyncClock,
) -> Result<Applied> {
let now = clock.now()?;
let prepared = self.pending.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?;
let channel = self.channel.as_ref().context("NO_AUTHENTICATED_CHANNEL")?;
let decision = verify_response_with(
&self.verifier,
&self.binding,
&prepared.request,
channel,
response,
now,
)?;
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let meta = metadata(&tx)?;
ensure!(
meta.fault.is_none()
&& meta.pending_request == Some(decision.request_id)
&& meta.checkpoint == i64::try_from(decision.checkpoint)?
&& meta.generation == channel.connection_generation,
"SYNC_CHECKPOINT_CHANGED"
);
let seq: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
ensure!(
seq >= meta.high_sequence && seq >= meta.checkpoint,
"LOCAL_SEQUENCE_ROLLBACK"
);
let current = tx.query_row(
"SELECT request FROM reconcile_requests WHERE id=?1",
[decision.request_id.to_string()],
|r| r.get::<_, Vec<u8>>(0),
)?;
ensure!(
current == prepared.request.encode_to_vec(),
"DURABLE_CHALLENGE_CHANGED"
);
let summaries: BTreeMap<_, _> = prepared
.request
.attempts
.iter()
.map(|a| ((a.job_id.clone(), a.attempt_id.clone()), a))
.collect();
let actual: BTreeSet<_> = decision
.attempts
.iter()
.map(|a| (a.job_id.to_string(), a.attempt_id.to_string()))
.collect();
ensure!(
actual.len() == decision.attempts.len()
&& actual == summaries.keys().cloned().collect(),
"SYNC_ATTEMPT_OMITTED_OR_ADDED"
);
let mut safe = true;
let mut fault = recovery_reason(&decision.reason).then(|| decision.reason.clone());
for d in &decision.attempts {
let local = summaries
.get(&(d.job_id.to_string(), d.attempt_id.to_string()))
.context("LOCAL_ATTEMPT_MISSING")?;
if recovery_reason(&d.reason) {
fault = Some(d.reason.clone());
}
let offer_bytes: Vec<u8> = tx.query_row(
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
params![local.job_id, local.attempt_id],
|r| r.get(0),
)?;
let offer = JobOffer::decode(offer_bytes.as_slice())?;
let job = self.binding.job(&offer.payload_bytes)?;
// Missing-server decisions intentionally have empty action fields;
// persist their sticky denial without treating it as start authority.
if recovery_reason(&d.reason) {
safe = false;
continue;
}
let outcome =
validate_attempt(&job, &offer, local, d, channel.boot_id, &self.binding, now)?;
safe &= (d.outcome_state == "accepted"
&& local.consumption == "unconsumed"
&& local.grant_id.is_empty())
|| (terminal(&d.outcome_state) && (local.consumption == "unconsumed" || d.unlock));
let (stored_state, revision): (String, u64) = tx.query_row(
"SELECT state,revision FROM job_attempts WHERE job=?1 AND attempt=?2",
params![local.job_id, local.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
ensure!(
revision == local.revision && stored_state == local.state,
"LOCAL_ATTEMPT_CHANGED_DURING_SYNC"
);
let stored = if d.outcome_state == "unknown" && local.state == "awaiting_reboot" {
"awaiting_reboot"
} else {
d.outcome_state.as_str()
};
tx.execute(
"UPDATE job_attempts SET state=?3,revision=?4 WHERE job=?1 AND attempt=?2",
params![
local.job_id,
local.attempt_id,
stored,
d.revision.max(local.revision)
],
)?;
if let Some(bytes) = outcome {
tx.execute("INSERT INTO reconcile_outcomes(job,attempt,outcome,decision_id) VALUES(?1,?2,?3,?4) ON CONFLICT(job,attempt) DO UPDATE SET outcome=excluded.outcome,decision_id=excluded.decision_id",params![local.job_id,local.attempt_id,bytes,decision.decision_id.to_string()])?;
}
}
let requested: BTreeSet<_> = prepared
.request
.durable_message_ids
.iter()
.map(String::as_str)
.collect();
let mut receipts = BTreeSet::new();
for id in &decision.acknowledged_message_ids {
ensure!(
receipts.insert(*id) && requested.contains(id.to_string().as_str()),
"ACK_NOT_IN_REQUEST"
);
archive_events(&tx)?;
tx.execute("DELETE FROM job_events WHERE id=?1", [id.to_string()])?;
tx.execute(
"INSERT OR IGNORE INTO reconcile_receipts(id,decision_id) VALUES(?1,?2)",
params![id.to_string(), decision.decision_id.to_string()],
)?;
}
tx.execute(
"UPDATE reconcile_requests SET response=?2 WHERE id=?1 AND response IS NULL",
params![decision.request_id.to_string(), response.encode_to_vec()],
)?;
for local in prepared
.request
.attempts
.iter()
.filter(|a| !a.update_handoff_bytes.is_empty())
{
crate::update_health_inventory::acknowledge_signed(
&tx,
Uuid::parse_str(&local.grant_id)?,
&decision,
response,
)?;
}
tx.execute("UPDATE reconcile_meta SET last_decision=?1,pending_request=NULL,helper_id=?2,helper_checkpoint=?3,fault=COALESCE(fault,?4),high_sequence=?5 WHERE id=1",
params![decision.decision_id.to_string(),decision.helper_journal_id.map(|id|id.to_string()),decision.helper_checkpoint,fault,seq])?;
tx.commit()?;
let helper_instruction = decision
.helper_journal_id
.map(|journal_id| HelperInstruction {
response: response.clone(),
decision_id: decision.decision_id,
journal_id,
checkpoint: decision.helper_checkpoint,
});
self.gate = Some(Gate {
channel: channel.clone(),
until: decision.valid_until,
ready: decision.ready && safe && fault.is_none(),
helper_ready: decision.helper_ready,
helper: helper_instruction
.as_ref()
.map(|h| (h.journal_id, h.checkpoint, h.decision_id)),
local_applied: !decision.attempts.iter().any(|d| d.unlock),
request: prepared.request.clone(),
response: response.clone(),
});
self.pending = None;
Ok(Applied {
decision_id: decision.decision_id,
acknowledged_message_ids: decision.acknowledged_message_ids,
helper_instruction,
reason: decision.reason,
})
}
/// Apply ONLY this process's independently owned consumption ledger. Root
/// helper unlocks use HelperInstruction over authenticated IPC instead.
/// A crash between journal and ledger transactions leaves the gate closed;
/// resync/reapply never deletes the consumed attempt or repeats an effect.
pub fn apply_consumption(
&mut self,
ledger: &mut Connection,
clock: &impl SyncClock,
) -> Result<()> {
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
gate.local_applied = false;
let decision = verify_response_with(
&self.verifier,
&self.binding,
&gate.request,
&gate.channel,
&gate.response,
clock.now()?,
)?;
let tx = ledger.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute_batch("CREATE TABLE IF NOT EXISTS reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB NOT NULL);")?;
let applied: Option<Vec<u8>> = tx
.query_row(
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
[decision.decision_id.to_string()],
|r| r.get(0),
)
.optional()?;
if let Some(old) = applied {
ensure!(
old == gate.response.encode_to_vec(),
"CONSUMPTION_DECISION_CONFLICT"
);
} else {
for d in decision.attempts.iter().filter(|d| d.unlock) {
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
let (agent,generation,job,attempt,previous):(String,u64,String,String,Option<Vec<u8>>)=tx.query_row(
"SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1",
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?;
ensure!(
agent == self.binding.agent_id.to_string()
&& generation == self.binding.enrollment_generation
&& job == d.job_id.to_string()
&& attempt == d.attempt_id.to_string(),
"LOCAL_UNLOCK_CONSUMPTION_BINDING"
);
let resolved = crate::windows_system_nonstart::effective_outcome(
&tx,
Some(&self.binding),
grant,
)?;
let previous = resolved
.or(previous)
.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
let original = previous.clone();
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&previous,
)?)?;
ensure!(
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
);
let bytes = if d.outcome_bytes.is_empty() {
prior
} else {
d.outcome_bytes.clone()
};
let outcome: ReconciledOutcome = serde_json::from_slice(&bytes)?;
ensure!(
terminal(&outcome.state)
&& outcome.state == d.outcome_state
&& canonical_outcome_bytes(&outcome)? == bytes
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()),
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
);
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
&tx,
&self.binding,
grant,
&bytes,
)?;
if outcome.state == "closed_unknown" {
crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?;
}
tx.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
params![grant.to_string(), bytes],
)?;
if !preserve {
tx.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
params![grant.to_string(), bytes],
)?;
}
}
tx.execute(
"INSERT INTO reconcile_consumption(decision_id,response) VALUES(?1,?2)",
params![
decision.decision_id.to_string(),
gate.response.encode_to_vec()
],
)?;
}
tx.commit()?;
gate.local_applied = true;
Ok(())
}
}
pub fn initialize(db: &Connection) -> Result<()> {
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "JOURNAL_INTEGRITY");
db.execute_batch("PRAGMA synchronous=FULL;
CREATE TABLE IF NOT EXISTS reconcile_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,high_sequence INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,last_decision TEXT,pending_request TEXT,helper_id TEXT,helper_checkpoint INTEGER NOT NULL DEFAULT 0,fault TEXT);
CREATE TABLE IF NOT EXISTS reconcile_requests(id TEXT PRIMARY KEY,request BLOB NOT NULL,envelope BLOB NOT NULL,generation INTEGER NOT NULL,response BLOB);
CREATE TABLE IF NOT EXISTS reconcile_results(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,revision INTEGER NOT NULL,envelope BLOB NOT NULL);
CREATE TABLE IF NOT EXISTS reconcile_receipts(id TEXT PRIMARY KEY,decision_id TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS reconcile_outcomes(job TEXT NOT NULL,attempt TEXT NOT NULL,outcome BLOB NOT NULL,decision_id TEXT NOT NULL,PRIMARY KEY(job,attempt));")?;
crate::update_health_inventory::initialize(db)?;
db.execute(
"INSERT OR IGNORE INTO reconcile_meta(id,journal_id) VALUES(1,?1)",
[Uuid::new_v4().to_string()],
)?;
Ok(())
}
struct Meta {
journal_id: Uuid,
checkpoint: i64,
high_sequence: i64,
generation: u64,
decision: Option<Uuid>,
pending_request: Option<Uuid>,
helper_id: Option<Uuid>,
helper_checkpoint: u64,
fault: Option<String>,
}
type MetadataRow = (
String,
i64,
i64,
u64,
Option<String>,
Option<String>,
Option<String>,
u64,
Option<String>,
);
fn metadata(db: &Connection) -> Result<Meta> {
let row:MetadataRow=db.query_row("SELECT journal_id,checkpoint,high_sequence,generation,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?)))?;
Ok(Meta {
journal_id: Uuid::parse_str(&row.0)?,
checkpoint: row.1,
high_sequence: row.2,
generation: row.3,
decision: row.4.map(|s| Uuid::parse_str(&s)).transpose()?,
pending_request: row.5.map(|s| Uuid::parse_str(&s)).transpose()?,
helper_id: row.6.map(|s| Uuid::parse_str(&s)).transpose()?,
helper_checkpoint: row.7,
fault: row.8,
})
}
/// Call inside the same transaction BEFORE deleting an ACKed job event. This
/// retains the original result identity/output for future reconciliation.
pub fn archive_events(db: &Connection) -> Result<()> {
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY rowid")?;
for row in statement.query_map([], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
})? {
let (id, bytes) = row?;
let envelope = Envelope::decode(bytes.as_slice())?;
ensure!(id == envelope.message_id, "EVENT_IDENTITY_CONFLICT");
if let Some(Payload::JobResult(r)) = &envelope.payload {
let existing: Option<Vec<u8>> = db
.query_row(
"SELECT envelope FROM reconcile_results WHERE id=?1",
[&id],
|r| r.get(0),
)
.optional()?;
ensure!(
existing.as_ref().is_none_or(|b| *b == bytes),
"RESULT_IDENTITY_CONFLICT"
);
db.execute("INSERT OR IGNORE INTO reconcile_results(id,job,attempt,revision,envelope) VALUES(?1,?2,?3,?4,?5)",params![id,r.job_id,r.attempt_id,r.revision,bytes])?;
}
}
Ok(())
}
/// A partial control edge can contain the result that closed an attempt, and
/// an earlier unknown report for that same grant. Only account for archived
/// messages here. Never ACK an event, change its state or replay a result.
fn verify_closed_result_messages(
db: &Connection,
request: &SyncRequest,
decision: &ReconcileDecisionPayload,
) -> Result<()> {
for id in &request.durable_message_ids {
let raw: Vec<u8> = db.query_row(
"SELECT envelope FROM reconcile_results WHERE id=?1",
[id],
|r| r.get(0),
)?;
let envelope = Envelope::decode(raw.as_slice())?;
let Some(Payload::JobResult(result)) = &envelope.payload else {
anyhow::bail!("CLOSED_CONTROL_EXPECTED_RESULT");
};
let attempt = request
.attempts
.iter()
.find(|a| {
a.job_id == result.job_id
&& a.attempt_id == result.attempt_id
&& a.grant_id == result.grant_id
})
.context("CLOSED_CONTROL_ARCHIVED_ATTEMPT")?;
let final_envelope = attempt
.result_envelope
.as_ref()
.context("CLOSED_CONTROL_FINAL_ENVELOPE")?;
let final_id = Uuid::parse_str(&attempt.result_message_id)?;
let persisted: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM reconcile_receipts WHERE id=?1 AND decision_id=?2)",
params![attempt.result_message_id, decision.decision_id.to_string()],
|r| r.get(0),
)?;
ensure!(
persisted
&& decision.acknowledged_message_ids.contains(&final_id)
&& request
.durable_message_ids
.contains(&attempt.result_message_id)
&& final_envelope.message_id == attempt.result_message_id
&& envelope.message_id == *id
&& envelope.encode_to_vec() == raw
&& rmm_protocol::valid_envelope(
&envelope,
&decision.agent_id.to_string(),
final_envelope.connection_generation
)
&& envelope.boot_id == final_envelope.boot_id
&& matches!(
attempt.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
)
&& !result.grant_id.is_empty(),
"CLOSED_CONTROL_FINAL_RESULT_RECEIPT"
);
ensure!(
if *id == attempt.result_message_id {
envelope == *final_envelope
&& result.revision == attempt.revision
&& result.state == attempt.state
&& canonical_outcome_bytes(&outcome(result)?)? == attempt.outcome_bytes
} else {
result.state == "unknown"
&& result.revision < attempt.revision
&& envelope.sequence < final_envelope.sequence
},
"CLOSED_CONTROL_CONFLICTING_ARCHIVED_RESULT"
);
}
Ok(())
}
fn pending_ids(db: &Connection) -> Result<Vec<String>> {
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?;
let rows = statement
.query_map([], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut ids = vec![];
for (id, bytes) in rows {
ensure!(
!Uuid::parse_str(&id)?.is_nil(),
"INVALID_DURABLE_MESSAGE_ID"
);
let envelope = Envelope::decode(bytes.as_slice())?;
ensure!(envelope.message_id == id, "EVENT_IDENTITY_CONFLICT");
if !crate::windows_system_nonstart::retired_event(db, &envelope)? {
ids.push(id);
}
}
Ok(ids)
}
pub(crate) fn outcome(r: &JobResult) -> Result<ReconciledOutcome> {
Ok(ReconciledOutcome {
state: r.state.clone(),
stdout: r.stdout.clone(),
stderr: r.stderr.clone(),
exit_code: r.exit_code,
output_truncated: r.output_truncated,
reboot_required: r.reboot_required,
evidence: serde_json::from_str(&r.evidence_json)?,
error_code: r.error_code.clone(),
})
}
fn terminal(state: &str) -> bool {
matches!(
state,
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
)
}
/// Server-signed final bytes of a human WebUI closure for an attempt whose
/// effect stayed unknown. They never claim success, failure or nonstart.
pub(crate) fn operator_closure(
reason: &str,
local_state: &str,
outcome: &ReconciledOutcome,
) -> bool {
reason == "OPERATOR_CLOSED_UNKNOWN"
&& local_state == "unknown"
&& outcome.state == "closed_unknown"
&& outcome.error_code == "OPERATOR_CLOSED_UNKNOWN"
&& outcome.exit_code.is_none()
&& outcome.stdout.is_empty()
&& outcome.stderr.is_empty()
&& outcome.evidence["postconditions_passed"] == false
&& outcome.evidence["effect"] == "unknown"
&& outcome.evidence["closed_by_human"] == true
&& outcome.evidence["retry"] == false
}
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
if s.is_empty() {
Ok(None)
} else {
let id = Uuid::parse_str(s)?;
ensure!(!id.is_nil(), "NIL_ID");
Ok(Some(id))
}
}
fn recovery_reason(reason: &str) -> bool {
matches!(
reason,
"MISSING_SERVER_JOB"
| "ATTEMPT_BINDING_MISMATCH"
| "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED"
| "SERVER_STATE_ROLLBACK"
| "ASSIGNMENT_MISMATCH"
| "JOURNAL_SEQUENCE_REGRESSION"
| "SERVER_CHECKPOINT_MISSING"
| "JOURNAL_ROLLBACK_OR_REPLACEMENT"
| "CHECKPOINT_CHAIN_MISMATCH"
| "HELPER_JOURNAL_ROLLBACK"
| "HELPER_CHECKPOINT_MISMATCH"
| "HELPER_CHECKPOINT_MISSING"
| "HELPER_CHECKPOINT_REQUIRED"
| "JOURNAL_HISTORY_MISSING"
| "LOCAL_ATTEMPT_MISSING"
)
}
fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<Vec<SyncAttempt>> {
let mut statement =
db.prepare("SELECT offer,state,revision,grant_id FROM job_attempts ORDER BY job,attempt")?;
let rows = statement.query_map([], |r| {
Ok((
r.get::<_, Vec<u8>>(0)?,
r.get::<_, String>(1)?,
r.get::<_, u64>(2)?,
r.get::<_, String>(3)?,
))
})?;
let mut summaries = Vec::new();
let mut known = BTreeSet::new();
for row in rows {
let (bytes, state, revision, mut grant_id) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes),
"LOCAL_OFFER_BINDING"
);
known.insert((offer.job_id.clone(), offer.attempt_id.clone()));
let consumed: Option<(String, String, u64, Option<Vec<u8>>)> = ledger
.query_row(
"SELECT grant_id,agent,generation,result FROM consumed WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)
.optional()?;
// A signed definitive nonstart can make an accepted no-grant row
// terminal without a JobResult. Preserve that fact on the next Sync;
// requesting/running/unknown still cannot claim non-consumption. An
// actual durable consumption row below always overrides this label.
let mut consumption = if grant_id.is_empty()
&& matches!(state.as_str(), "accepted" | "expired" | "cancelled")
{
"unconsumed"
} else {
"uncertain"
};
let mut original_boot_id = String::new();
let mut canonical = Vec::new();
if let Some((id, agent, generation, result)) = consumed {
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& (grant_id.is_empty() || grant_id == id),
"CONSUMPTION_BINDING"
);
grant_id = id;
consumption = "consumed";
let evidence: Option<(Vec<u8>, Option<Vec<u8>>)> = ledger
.query_row(
"SELECT start,outcome FROM execution_evidence WHERE grant_id=?1",
[&grant_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()?;
let mut stored = result;
if let Some((start, last)) = evidence {
let start: serde_json::Value = serde_json::from_slice(&start)?;
original_boot_id = start["previous_boot_id"]
.as_str()
.context("ORIGINAL_BOOT_MISSING")?
.to_string();
optional_uuid(&original_boot_id)?;
let request: crate::auth::Request =
serde_json::from_value(start["request"].clone())?;
ensure!(
request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"START_EVIDENCE_BINDING"
);
if let Some(resolved) = crate::windows_system_nonstart::effective_outcome(
ledger,
Some(binding),
Uuid::parse_str(&grant_id)?,
)? {
stored = Some(resolved);
} else if last.is_some() {
stored = last;
}
}
if let Some(stored) = stored {
let o: ReconciledOutcome = serde_json::from_slice(&stored)?;
canonical = canonical_outcome_bytes(&o)?;
}
}
let original:Option<Vec<u8>>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",params![offer.job_id,offer.attempt_id],|r|r.get(0)).optional()?;
let mut result_envelope = None;
let mut result = None;
let mut result_message_id = String::new();
if let Some(bytes) = original {
let envelope = Envelope::decode(bytes.as_slice())?;
let Some(Payload::JobResult(r)) = &envelope.payload else {
anyhow::bail!("INVALID_ARCHIVED_RESULT")
};
if r.revision == revision && r.state == state {
ensure!(
envelope.agent_id == binding.agent_id.to_string()
&& envelope.enrollment_generation == binding.enrollment_generation
&& r.grant_id == grant_id,
"RESULT_BINDING"
);
let bytes = canonical_outcome_bytes(&outcome(r)?)?;
ensure!(
canonical.is_empty() || canonical == bytes,
"LOCAL_OUTCOME_LEDGER_MISMATCH"
);
canonical = bytes;
result_message_id = envelope.message_id.clone();
result = Some(r.clone());
result_envelope = Some(envelope);
}
}
if !canonical.is_empty() {
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
ensure!(canonical.len() <= 512 * 1024, "SYNC_CANONICAL_OUTCOME_SIZE");
if grant_id.is_empty() && terminal(&state) && o.evidence["not_started"] == true {
consumption = "unconsumed";
}
}
summaries.push(SyncAttempt {
update_handoff_bytes: Vec::new(),
job_id: offer.job_id,
attempt_id: offer.attempt_id,
job_payload_hash: offer.job_payload_hash,
grant_id,
consumption: consumption.into(),
state,
revision,
original_boot_id,
result,
result_message_id,
outcome_sha256: if canonical.is_empty() {
String::new()
} else {
hash(&canonical)
},
outcome_bytes: canonical,
result_envelope,
});
}
let mut statement = ledger.prepare("SELECT job,attempt FROM consumed")?;
for row in statement.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
ensure!(
known.contains(&row?),
"CONSUMED_ATTEMPT_MISSING_FROM_AGENT_JOURNAL"
);
}
Ok(summaries)
}
fn ingest_helper(
db: &Connection,
ledger: &Connection,
binding_and_verifier: (&Binding, &impl Ed25519Verifier),
channel: &Channel,
helper: &HelperSnapshot,
now: DateTime<Utc>,
identity: HelperIdentity,
) -> Result<()> {
let (binding, verifier) = binding_and_verifier;
let mut statement = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
for row in statement.query_map([], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, String>(1)?,
r.get::<_, Vec<u8>>(2)?,
))
})? {
let (job, attempt, bytes) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
if binding.job(&offer.payload_bytes)?.execution_identity == identity.as_str() {
let consumed: bool = ledger.query_row(
"SELECT EXISTS(SELECT 1 FROM consumed WHERE job=?1 AND attempt=?2)",
params![job, attempt],
|r| r.get(0),
)?;
ensure!(
!consumed
|| helper
.attempts
.iter()
.any(|a| a.job_id == job && a.attempt_id == attempt),
"ROOT_CONSUMED_ATTEMPT_MISSING_FROM_HELPER_JOURNAL"
);
}
}
let mut seen = BTreeSet::new();
for h in &helper.attempts {
ensure!(
seen.insert((&h.job_id, &h.attempt_id)),
"DUPLICATE_HELPER_ATTEMPT"
);
let row: Option<(Vec<u8>, u64, String)> = db
.query_row(
"SELECT offer,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.optional()?;
let (bytes, revision, grant) = row.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
job.execution_identity == identity.as_str()
&& h.job_payload_hash == offer.job_payload_hash
&& (grant.is_empty() || grant == h.grant_id),
"HELPER_ATTEMPT_BINDING"
);
if h.outcome_bytes.is_empty() {
if !h.update_handoff_bytes.is_empty() {
let header =
validate_update_summary(binding, &job, &offer, h, channel.boot_id, now)?;
ensure!(
header.helper_journal_id == helper.journal_id
&& header.agent_journal_id == metadata(db)?.journal_id,
"UPDATE_HELPER_JOURNAL_BINDING"
);
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id],
|r| r.get(0),
)?;
ensure!(
matches!(
state.as_str(),
"accepted" | "requesting" | "running" | "verifying"
),
"UPDATE_CANNOT_HEAL_UNKNOWN"
);
type OriginalEvidence = (
String,
u64,
String,
String,
Vec<u8>,
Option<Vec<u8>>,
Option<Vec<u8>>,
);
let original:Option<OriginalEvidence>=ledger.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM execution_evidence e JOIN consumed c USING(grant_id) WHERE grant_id=?1",[&h.grant_id],
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
let (agent, generation, job_id, attempt_id, start, outcome, result) =
original.context("UPDATE_LOCAL_ORIGINAL_MISSING")?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == h.job_id
&& attempt_id == h.attempt_id,
"UPDATE_LOCAL_CONSUMPTION_BINDING"
);
ensure!(
outcome.is_none() && result.is_none(),
"UPDATE_CANNOT_REPLACE_OUTCOME"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: crate::auth::Request =
serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
binding.verify_with(&request, permit.not_before, verifier)?;
ensure!(
hash(&serde_json::to_vec(&request)?) == header.attempt.original_request_sha256
&& permit.grant_id == header.attempt.grant_id
&& permit.agent_update_plan_id == Some(header.attempt.plan_id),
"UPDATE_LOCAL_ORIGINAL_CHANGED"
);
db.execute("UPDATE job_attempts SET state='running',revision=MAX(revision,?3) WHERE job=?1 AND attempt=?2",params![h.job_id,h.attempt_id,h.revision])?;
}
continue;
}
let o: ReconciledOutcome = serde_json::from_slice(&h.outcome_bytes)?;
ensure!(
canonical_outcome_bytes(&o)? == h.outcome_bytes
&& hash(&h.outcome_bytes) == h.outcome_sha256
&& o.state == h.state
&& (terminal(&o.state)
|| matches!(o.state.as_str(), "unknown" | "awaiting_reboot"))
&& h.consumption == "consumed"
&& optional_uuid(&h.grant_id)?.is_some(),
"HELPER_CANONICAL_OUTCOME"
);
let existing: Option<Vec<u8>> = ledger
.query_row(
"SELECT outcome FROM execution_evidence WHERE grant_id=?1",
[&h.grant_id],
|r| r.get(0),
)
.optional()?
.flatten();
let resolved = crate::windows_system_nonstart::effective_outcome(
ledger,
Some(binding),
Uuid::parse_str(&h.grant_id)?,
)?;
let same = resolved
.as_ref()
.or(existing.as_ref())
.map(|b| {
serde_json::from_slice::<ReconciledOutcome>(b)
.and_then(|o| canonical_outcome_bytes(&o))
})
.transpose()?
.is_some_and(|b| b == h.outcome_bytes);
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id],
|r| r.get(0),
)?;
if same && state == h.state {
continue;
}
let consumed: Option<(String, u64, String, String)> = ledger
.query_row(
"SELECT agent,generation,job,attempt FROM consumed WHERE grant_id=?1",
[&h.grant_id],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)
.optional()?;
ensure!(
consumed
== Some((
binding.agent_id.to_string(),
binding.enrollment_generation,
h.job_id.clone(),
h.attempt_id.clone()
)),
"HELPER_CONSUMPTION_NOT_KNOWN_LOCALLY"
);
// Store actual helper evidence without unlocking result/consumption. A
// crash here still leaves both gates closed and the original helper
// result available; the next sync imports it again idempotently.
ensure!(
ledger.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
params![h.grant_id, h.outcome_bytes]
)? == 1,
"LOCAL_START_EVIDENCE_MISSING"
);
let next_revision = revision
.max(h.revision)
.checked_add(1)
.context("REVISION_OVERFLOW")?;
i64::try_from(next_revision)?;
let seq: i64 = db.query_row(
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
[],
|r| r.get(0),
)?;
let result = JobResult {
job_id: h.job_id.clone(),
attempt_id: h.attempt_id.clone(),
revision: next_revision,
state: o.state.clone(),
grant_id: h.grant_id.clone(),
stdout: o.stdout,
stderr: o.stderr,
output_truncated: o.output_truncated,
reboot_required: o.reboot_required,
exit_code: o.exit_code,
evidence_json: o.evidence.to_string(),
error_code: o.error_code,
};
let envelope = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: binding.agent_id.to_string(),
enrollment_generation: binding.enrollment_generation,
connection_generation: channel.connection_generation,
boot_id: channel.boot_id.to_string(),
sequence: u64::try_from(seq)?,
observed_at: now.timestamp_millis(),
payload: Some(Payload::JobResult(result)),
};
ensure!(
envelope.encoded_len() <= rmm_domain::MAX_MESSAGE,
"HELPER_RESULT_SIZE"
);
db.execute(
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
params![envelope.message_id, envelope.encode_to_vec()],
)?;
db.execute(
"UPDATE job_attempts SET state=?3,revision=?4,grant_id=?5 WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id, o.state, next_revision, h.grant_id],
)?;
}
Ok(())
}
fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()> {
let mut seen = BTreeSet::new();
for h in &helper.attempts {
ensure!(
seen.insert((&h.job_id, &h.attempt_id)),
"DUPLICATE_HELPER_ATTEMPT"
);
let a = local
.iter_mut()
.find(|a| a.job_id == h.job_id && a.attempt_id == h.attempt_id)
.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?;
ensure!(
a.job_payload_hash == h.job_payload_hash
&& (a.grant_id.is_empty() || a.grant_id == h.grant_id),
"HELPER_ATTEMPT_BINDING"
);
if h.consumption != "unconsumed" {
if !h.update_handoff_bytes.is_empty() {
ensure!(
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
"UPDATE_CANNOT_HEAL_UNKNOWN"
);
a.update_handoff_bytes.clone_from(&h.update_handoff_bytes);
}
a.grant_id = h.grant_id.clone();
a.consumption = h.consumption.clone();
ensure!(
a.outcome_sha256 == h.outcome_sha256,
"HELPER_OUTCOME_MISMATCH"
);
if a.original_boot_id.is_empty() {
a.original_boot_id = h.original_boot_id.clone();
}
ensure!(
a.original_boot_id == h.original_boot_id,
"HELPER_ORIGINAL_BOOT_MISMATCH"
);
}
}
Ok(())
}
fn validate_update_summary(
_binding: &Binding,
job: &JobPayload,
offer: &JobOffer,
local: &SyncAttempt,
boot: Uuid,
now: DateTime<Utc>,
) -> Result<rmm_updater_handoff::Handoff> {
let proof =
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
let (h, _) =
rmm_domain::agent_update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
let now = u64::try_from(now.timestamp())?;
ensure!(
rmm_domain::agent_update::handoff_matches_job(&h, job)
&& h.boot_id == boot
&& h.attempt.grant_id.to_string() == local.grant_id
&& h.attempt.authorization_id.to_string() == offer.authorization_id
&& h.attempt.plan_sha256 == offer.plan_hash
&& h.attempt.job_payload_sha256 == offer.job_payload_hash
&& local.original_boot_id == boot.to_string()
&& local.consumption == "consumed"
&& local.state == "running"
&& local.outcome_bytes.is_empty()
&& local.outcome_sha256.is_empty()
&& local.result.is_none()
&& local.result_envelope.is_none()
&& h.consumed_at <= now
&& now < h.finish_before,
"UPDATE_INFLIGHT_SUMMARY_BINDING"
);
Ok(h)
}
pub fn verify_response(
binding: &Binding,
request: &SyncRequest,
channel: &Channel,
response: &SyncResponse,
now: DateTime<Utc>,
) -> Result<ReconcileDecisionPayload> {
verify_response_with(
&DefaultVerifier::default(),
binding,
request,
channel,
response,
now,
)
}
pub fn verify_response_with(
verifier: &impl Ed25519Verifier,
binding: &Binding,
request: &SyncRequest,
channel: &Channel,
response: &SyncResponse,
now: DateTime<Utc>,
) -> Result<ReconcileDecisionPayload> {
ensure!(
response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64,
"SYNC_SIGNATURE_SIZE"
);
let raw = decode_key(&binding.public_key_hex)?;
ensure!(
verifier.verify(&raw, &response.decision_bytes, &response.signature)?,
"INVALID_SYNC_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
ensure!(
d.valid_profile(now)
&& d.valid_until - d.issued_at <= chrono::Duration::seconds(60)
&& d.key_id == hash(&raw),
"SYNC_PROFILE_OR_CLOCK"
);
ensure!(
d.tenant_id == binding.tenant_id
&& d.asset_id == binding.asset_id
&& d.agent_id == binding.agent_id
&& d.enrollment_generation == binding.enrollment_generation
&& d.assignment_version == binding.assignment_version
&& d.connection_generation == channel.connection_generation
&& d.boot_id == channel.boot_id
&& d.request_id.to_string() == request.request_id
&& d.request_hash == hash(&request.encode_to_vec())
&& d.journal_id.to_string() == request.journal_id
&& d.checkpoint == request.checkpoint
&& d.helper_journal_id == optional_uuid(&request.helper_journal_id)?
&& d.helper_checkpoint == request.helper_checkpoint,
"SYNC_DECISION_BINDING"
);
Ok(d)
}
fn decode_key(hex: &str) -> Result<Vec<u8>> {
ensure!(hex.len() == 64, "PINNED_KEY_SIZE");
(0..32)
.map(|i| {
Ok(u8::from_str_radix(
hex.get(i * 2..i * 2 + 2).context("PINNED_KEY_ENCODING")?,
16,
)?)
})
.collect()
}
fn validate_attempt(
job: &JobPayload,
offer: &JobOffer,
local: &SyncAttempt,
d: &AttemptDecision,
boot: Uuid,
binding: &Binding,
now: DateTime<Utc>,
) -> Result<Option<Vec<u8>>> {
ensure!(
d.job_payload_hash == offer.job_payload_hash
&& d.grant_id == optional_uuid(&local.grant_id)?
&& d.action == job.action.name()
&& d.execution_identity == job.execution_identity
&& (d.revision >= local.revision || d.outcome_state == "unknown")
&& d.original_boot_id == optional_uuid(&local.original_boot_id)?
&& d.previous_outcome_sha256.as_deref()
== (!local.outcome_sha256.is_empty()).then_some(local.outcome_sha256.as_str()),
"SYNC_ATTEMPT_BINDING"
);
if d.grant_id.is_some() {
ensure!(
d.authorization_id == Some(Uuid::parse_str(&offer.authorization_id)?)
&& d.plan_hash.as_deref() == Some(offer.plan_hash.as_str()),
"SYNC_AUTHORIZATION_BINDING"
);
}
ensure!(
!terminal(&local.state) || d.outcome_state == local.state,
"TERMINAL_STATE_REGRESSION"
);
if d.update_inflight.is_some() || !local.update_handoff_bytes.is_empty() {
validate_update_summary(binding, job, offer, local, boot, now)?;
let proof =
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let key =
rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
let expected =
rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
ensure!(
d.update_inflight.as_ref() == Some(&expected)
&& d.outcome_state == "running"
&& !d.unlock
&& d.outcome_bytes.is_empty()
&& d.outcome_sha256.is_none(),
"UPDATE_INFLIGHT_DECISION_CHANGED"
);
return Ok(None);
}
if d.outcome_state == "accepted" {
ensure!(
local.state == "accepted"
&& local.consumption == "unconsumed"
&& d.grant_id.is_none()
&& !d.unlock
&& d.outcome_bytes.is_empty(),
"SYNC_CANNOT_RESTART_ATTEMPT"
);
return Ok(None);
}
if d.outcome_state == "unknown" {
ensure!(!d.unlock, "UNKNOWN_CANNOT_UNLOCK");
return Ok(None);
}
ensure!(terminal(&d.outcome_state), "UNSUPPORTED_SYNC_STATE");
if d.outcome_bytes.is_empty() && d.outcome_sha256.is_none() {
ensure!(
local.consumption == "unconsumed"
&& d.grant_id.is_none()
&& !d.unlock
&& d.outcome_state != "succeeded",
"TERMINAL_WITHOUT_EVIDENCE"
);
return Ok(None);
}
let bytes = if d.outcome_bytes.is_empty() {
local.outcome_bytes.clone()
} else {
d.outcome_bytes.clone()
};
let o: ReconciledOutcome = serde_json::from_slice(&bytes)?;
ensure!(
canonical_outcome_bytes(&o)? == bytes
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
&& o.state == d.outcome_state,
"SYNC_CANONICAL_OUTCOME"
);
if bytes != local.outcome_bytes && !operator_closure(&d.reason, &local.state, &o) {
ensure!(
matches!(job.action, rmm_domain::jobs::Action::Reboot { .. })
&& d.outcome_state == "succeeded"
&& d.original_boot_id.is_some_and(|id| id != boot)
&& o.evidence["postconditions_passed"] == true
&& o.evidence["reconciled_after_reboot"] == true
&& o.evidence["new_boot_id"].as_str() == Some(boot.to_string().as_str())
&& o.evidence["inventory_message_id"]
.as_str()
.is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())),
"SYNC_REBOOT_REAUTH_EVIDENCE"
);
}
ensure!(
!d.unlock || (local.consumption == "consumed" && d.grant_id.is_some()),
"UNLOCK_WITHOUT_CONSUMPTION"
);
Ok(Some(bytes))
}
/// Operator regression only: imports the bounded, selected actual public
/// fixture into two IN-MEMORY databases. No production path/network/executor.
#[doc(hidden)]
pub fn closed_control_contract(raw: &[u8]) -> Result<serde_json::Value> {
use rusqlite::types::Value as SqlValue;
use serde_json::{Value, json};
ensure!(raw.len() <= 4 * 1024 * 1024, "CONTROL_FIXTURE_BOUND");
let f: Value = serde_json::from_slice(raw)?;
ensure!(
f["schema_version"] == 1
&& f["read_only"] == true
&& f["database_writes"] == false
&& f["private_keys_read"] == false,
"CONTROL_FIXTURE_SOURCE"
);
let binding: Binding = serde_json::from_value(f["binding"].clone())?;
let mut db = Connection::open_in_memory()?;
let ledger = Connection::open_in_memory()?;
initialize(&db)?;
crate::ledger::initialize_ledger(&ledger)?;
db.execute_batch("DELETE FROM reconcile_meta; CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT,PRIMARY KEY(job,attempt));CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE events(id TEXT PRIMARY KEY,payload BLOB)")?;
ledger.execute_batch(
"CREATE TABLE reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB)",
)?;
const RESOLUTION: &str = "CREATE TABLE windows_system_nonstart_221_resolution(grant_id TEXT PRIMARY KEY,role TEXT,receipt BLOB,receipt_sha256 TEXT,derived_outcome BLOB)";
db.execute_batch(RESOLUTION)?;
ledger.execute_batch(RESOLUTION)?;
fn cell(v: &Value) -> Result<SqlValue> {
let a = v.as_array().context("CONTROL_FIXTURE_CELL")?;
Ok(match a.first().and_then(Value::as_str) {
Some("null") if a.len() == 1 => SqlValue::Null,
Some("integer") if a.len() == 2 => {
SqlValue::Integer(a[1].as_i64().context("CONTROL_FIXTURE_INTEGER")?)
}
Some("text") if a.len() == 2 => {
SqlValue::Text(a[1].as_str().context("CONTROL_FIXTURE_TEXT")?.into())
}
Some("blob") if a.len() == 2 => SqlValue::Blob(serde_json::from_value(a[1].clone())?),
_ => anyhow::bail!("CONTROL_FIXTURE_CELL_TYPE"),
})
}
fn import(db: &Connection, rows: &Value, sql: &str) -> Result<()> {
let rows = rows.as_array().context("CONTROL_FIXTURE_ROWS")?;
ensure!(rows.len() <= 8, "CONTROL_FIXTURE_ROW_BOUND");
for row in rows {
let values = row
.as_array()
.context("CONTROL_FIXTURE_ROW")?
.iter()
.map(cell)
.collect::<Result<Vec<_>>>()?;
db.execute(sql, rusqlite::params_from_iter(values))?;
}
Ok(())
}
for (key, sql) in [
(
"meta",
"INSERT INTO reconcile_meta VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",
),
("counter", "INSERT INTO counter VALUES(?1,?2)"),
(
"requests",
"INSERT INTO reconcile_requests VALUES(?1,?2,?3,?4,?5)",
),
("jobs", "INSERT INTO job_attempts VALUES(?1,?2,?3,?4,?5,?6)"),
(
"outcomes",
"INSERT INTO reconcile_outcomes VALUES(?1,?2,?3,?4)",
),
(
"results",
"INSERT INTO reconcile_results VALUES(?1,?2,?3,?4,?5)",
),
("events", "INSERT INTO job_events VALUES(?1,?2)"),
(
"resolution",
"INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)",
),
] {
import(&db, &f["agent"][key], sql)?;
}
ensure!(
f["agent"]["legacy_count"] == 0,
"CONTROL_FIXTURE_PENDING_LEGACY"
);
for (key, sql) in [
("consumed", "INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,?6)"),
(
"evidence",
"INSERT INTO execution_evidence VALUES(?1,?2,?3)",
),
("acks", "INSERT INTO reconcile_consumption VALUES(?1,?2)"),
(
"resolution",
"INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)",
),
] {
import(&ledger, &f["ledger"][key], sql)?;
}
let captures = f["helper"]["capture"]
.as_array()
.context("CONTROL_FIXTURE_CAPTURE")?;
ensure!(captures.len() == 1, "CONTROL_FIXTURE_CAPTURE_COUNT");
let SqlValue::Blob(raw_snapshot) = cell(&captures[0][1])? else {
anyhow::bail!("CONTROL_FIXTURE_CAPTURE_BYTES")
};
let snapshot: crate::helper_reconcile::Snapshot = serde_json::from_slice(&raw_snapshot)?;
let m = f["helper"]["meta"]
.as_array()
.context("CONTROL_FIXTURE_HELPER_META")?;
ensure!(m.len() == 1, "CONTROL_FIXTURE_HELPER_META_COUNT");
ensure!(
cell(&m[0][0])? == SqlValue::Text(snapshot.journal_id.to_string())
&& cell(&m[0][1])? == SqlValue::Integer(i64::try_from(snapshot.checkpoint)?)
&& cell(&m[0][3])?
== SqlValue::Text(
snapshot
.previous_decision_id
.context("CONTROL_FIXTURE_HELPER_HEAD")?
.to_string()
)
&& cell(&m[0][6])? == SqlValue::Null,
"CONTROL_FIXTURE_ACTUAL_HELPER_CAPTURE"
);
crate::helper_reconcile::precommit_clock_contract(binding.clone(), snapshot.clone())?;
let helper = HelperSnapshot {
journal_id: snapshot.journal_id,
checkpoint: snapshot.checkpoint,
previous_decision_id: snapshot.previous_decision_id,
attempts: snapshot
.attempts
.iter()
.map(|a| SyncAttempt {
job_id: a.job_id.to_string(),
attempt_id: a.attempt_id.to_string(),
job_payload_hash: a.job_payload_hash.clone(),
grant_id: a.grant_id.to_string(),
consumption: a.consumption.clone(),
state: a.state.clone(),
revision: a.revision,
original_boot_id: a.original_boot_id.to_string(),
outcome_sha256: a.outcome_sha256.clone(),
outcome_bytes: a.outcome_bytes.clone(),
update_handoff_bytes: a.update_handoff_bytes.clone(),
..Default::default()
})
.collect(),
};
let meta = metadata(&db)?;
let helper_generation = match cell(&m[0][2])? {
SqlValue::Integer(v) => u64::try_from(v)?,
_ => anyhow::bail!("CONTROL_FIXTURE_HELPER_GENERATION"),
};
let channel = Channel {
connection_generation: meta
.generation
.max(helper_generation)
.checked_add(1)
.context("CONTROL_FIXTURE_GENERATION_OVERFLOW")?,
boot_id: snapshot.boot_id,
};
let now: DateTime<Utc> = serde_json::from_value(f["captured_at_utc"].clone())?;
let client = ClientSync::with_verifier(binding.clone(), crate::crypto::DalekVerifier)
.with_helper_identity(HelperIdentity::System);
let before = (db.total_changes(), ledger.total_changes());
client.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)?;
ensure!(
(db.total_changes(), ledger.total_changes()) == before,
"CONTROL_ADMISSION_WROTE_HISTORY"
);
ledger.execute(
"INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,NULL)",
params![
Uuid::from_u128(2).to_string(),
binding.agent_id.to_string(),
binding.enrollment_generation,
Uuid::from_u128(3).to_string(),
Uuid::from_u128(4).to_string()
],
)?;
ensure!(
client
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
.is_err(),
"CONTROL_HIDDEN_OPEN_GRANT_NOT_REJECTED"
);
ledger.execute(
"DELETE FROM consumed WHERE grant_id=?1",
[Uuid::from_u128(2).to_string()],
)?;
let a = meta.decision.context("CONTROL_FIXTURE_A")?;
let (request, _) = client.stored_control(&db, &ledger, a, channel.boot_id)?;
let (original_raw, original_env, original_response): (Vec<u8>, Vec<u8>, Vec<u8>) = db
.query_row(
"SELECT request,envelope,response FROM reconcile_requests WHERE id=?1",
[request.request_id.clone()],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)?;
let mut changed = request.clone();
changed.helper_previous_decision_id = Uuid::from_u128(1).to_string();
let mut env = Envelope::decode(original_env.as_slice())?;
env.payload = Some(Payload::SyncRequest(changed.clone()));
db.execute(
"UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3",
params![
changed.encode_to_vec(),
env.encode_to_vec(),
request.request_id
],
)?;
ensure!(
client
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
.is_err(),
"CONTROL_MUTATED_A_NOT_REJECTED"
);
db.execute(
"UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3",
params![original_raw, original_env, request.request_id],
)?;
let mut changed = SyncResponse::decode(original_response.as_slice())?;
changed.signature[0] ^= 1;
let bytes = changed.encode_to_vec();
db.execute(
"UPDATE reconcile_requests SET response=?1 WHERE id=?2",
params![bytes, request.request_id],
)?;
ledger.execute(
"UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2",
params![bytes, a.to_string()],
)?;
ensure!(
client
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
.is_err(),
"CONTROL_MUTATED_SERVER_NOT_REJECTED"
);
db.execute(
"UPDATE reconcile_requests SET response=?1 WHERE id=?2",
params![original_response, request.request_id],
)?;
ledger.execute(
"UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2",
params![original_response, a.to_string()],
)?;
struct CurrentClock(DateTime<Utc>);
impl SyncClock for CurrentClock {
fn now(&self) -> Result<DateTime<Utc>> {
Ok(self.0)
}
}
let clock = CurrentClock(now);
let mut client = ClientSync::with_verifier(binding, crate::crypto::DalekVerifier)
.with_helper_identity(HelperIdentity::System);
ensure!(
client
.prepare(&mut db, &ledger, channel.clone(), Some(&helper), &clock)
.is_err(),
"CONTROL_IDLE_ATTESTATION_REQUIRED"
);
let fresh = client.prepare_with_closed_control_idle(
&mut db,
&ledger,
channel.clone(),
Some(&helper),
&clock,
true,
)?;
ensure!(
fresh.request.previous_decision_id == a.to_string()
&& fresh.request.helper_previous_decision_id
== helper.previous_decision_id.unwrap().to_string()
&& fresh.envelope.connection_generation == channel.connection_generation
&& fresh.envelope.observed_at == now.timestamp_millis()
&& !client.can_start(&channel, false, &clock)
&& !client.can_start(&channel, true, &clock)
&& client.gate_observation().is_none(),
"CONTROL_FRESH_REQUEST_MUST_NOT_AUTHORIZE"
);
Ok(
json!({"actual_closed_partial_edge_accepted":true,"mutated_agent_request_rejected":true,"mutated_server_signature_and_ack_rejected":true,"admission_read_only":true,"hidden_open_grant_rejected":true,"explicit_idle_attestation_required":true,"typed_precommit_clock_keeps_capture_without_writes":true,"fresh_request_keeps_both_heads":true,"old_authority_reused":false,"new_ready_or_unlock":false,"product_databases_opened":false,"effects":false}),
)
}
#[cfg(test)]
mod closure_tests {
use super::*;
#[test]
fn only_a_human_closure_of_a_locally_unknown_attempt_is_accepted() {
let closed = ReconciledOutcome {
state: "closed_unknown".into(),
stdout: vec![],
stderr: vec![],
exit_code: None,
output_truncated: false,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"retry":false}),
error_code: "OPERATOR_CLOSED_UNKNOWN".into(),
};
assert!(operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &closed));
assert!(!operator_closure("DEFINITIVE_OUTCOME_RECONCILED", "unknown", &closed));
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "running", &closed));
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "succeeded", &closed));
for case in 0..6 {
let mut other = closed.clone();
match case {
0 => other.state = "failed".into(),
1 => other.error_code = "SCRIPT_FAILED".into(),
2 => other.exit_code = Some(0),
3 => other.stdout = b"x".to_vec(),
4 => other.evidence["postconditions_passed"] = true.into(),
_ => other.evidence["retry"] = true.into(),
}
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &other));
}
assert!(terminal("closed_unknown"));
}
}