Files
rust-discovery/crates/agent-core/src/reconcile.rs
T
2026-11-15 10:26:24 +01:00

1492 lines
60 KiB
Rust

//! Reconnect reconciliation, deliberately independent of main/jobs and transport.
//! No method executes a job, deletes consumption tombstones or issues a permit.
//! Call only while holding the existing agent.lock. HelperSnapshot must come
//! from the independently authenticated root-helper IPC, never from the server.
use crate::auth::Binding;
pub use crate::auth::HelperIdentity;
use crate::crypto::{DefaultVerifier, Ed25519Verifier};
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Utc};
use prost::Message;
use rmm_domain::{
jobs::{JobPayload, hash},
reconciliation::{
AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes,
},
};
use rmm_protocol::{
Envelope, JobOffer, JobResult, Payload, SyncAttempt, SyncRequest, SyncResponse,
};
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
use std::collections::{BTreeMap, BTreeSet};
use uuid::Uuid;
/// A fresh observation returned by the trusted local clock implementation,
/// never an IPC DTO or an execution permit. Callers must not retain it as a lease.
pub struct ClockReading {
pub own_wall: DateTime<Utc>,
pub upper: DateTime<Utc>,
}
pub trait SyncClock {
fn now(&self) -> Result<DateTime<Utc>>;
/// Existing providers retain their existing point-clock contract. Providers
/// with an independently verified interval override this atomically.
fn reading(&self) -> Result<ClockReading> {
let own_wall = self.now()?;
Ok(ClockReading {
own_wall,
upper: own_wall,
})
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Channel {
pub connection_generation: u64,
pub boot_id: Uuid,
}
#[derive(Clone, Debug)]
pub struct HelperSnapshot {
pub journal_id: Uuid,
pub checkpoint: u64,
pub previous_decision_id: Option<Uuid>,
/// Complete consumed/unsettled summaries from the root-owned ledger.
pub attempts: Vec<SyncAttempt>,
}
#[derive(Clone)]
pub struct PreparedSync {
pub envelope: Envelope,
pub request: SyncRequest,
}
/// Pass the ORIGINAL signed response to the helper. It must independently verify
/// it, apply targeted unlocks and commit its checkpoint before acknowledging.
#[derive(Clone)]
pub struct HelperInstruction {
pub response: SyncResponse,
pub decision_id: Uuid,
pub journal_id: Uuid,
pub checkpoint: u64,
}
pub struct Applied {
pub decision_id: Uuid,
pub acknowledged_message_ids: Vec<Uuid>,
pub helper_instruction: Option<HelperInstruction>,
pub reason: String,
}
struct Gate {
channel: Channel,
until: DateTime<Utc>,
ready: bool,
helper_ready: bool,
helper: Option<(Uuid, u64, Uuid)>,
local_applied: bool,
request: SyncRequest,
response: SyncResponse,
}
pub struct ClientSync<V = DefaultVerifier> {
binding: Binding,
verifier: V,
helper_identity: HelperIdentity,
pending: Option<PreparedSync>,
channel: Option<Channel>,
gate: Option<Gate>,
}
impl ClientSync<DefaultVerifier> {
/// Every new process starts closed, independently of any persisted ready bit.
pub fn new(binding: Binding) -> Self {
Self::with_verifier(binding, DefaultVerifier::default())
}
}
impl<V: Ed25519Verifier> ClientSync<V> {
pub fn with_verifier(binding: Binding, verifier: V) -> Self {
Self {
binding,
verifier,
helper_identity: HelperIdentity::Root,
pending: None,
channel: None,
gate: None,
}
}
/// Configure one explicit helper identity; changing it discards all leases.
pub fn with_helper_identity(mut self, identity: HelperIdentity) -> Self {
self.close();
self.helper_identity = identity;
self
}
/// Invoke on every authenticated/fenced ServerHello or transport loss.
pub fn close(&mut self) {
self.gate = None;
self.channel = None;
self.pending = None;
}
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
self.gate.as_ref().map(|gate| gate.until)
}
pub fn can_start(&self, channel: &Channel, root: bool, clock: &impl SyncClock) -> bool {
let Ok(reading) = clock.reading() else {
return false;
};
self.gate.as_ref().is_some_and(|g| {
g.channel == *channel
&& g.ready
&& g.local_applied
&& reading.own_wall < g.until
&& reading.upper < g.until
&& (!root || (g.helper_ready && g.helper.is_none()))
})
}
/// Only call with a fresh independently authenticated helper snapshot. There
/// is no unsigned ready/unlock flag: the exact signed decision/checkpoint
/// must already be durably applied by that helper.
pub fn confirm_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
ensure!(
snapshot.journal_id == expected.0
&& snapshot.checkpoint == expected.1
&& snapshot.previous_decision_id == Some(expected.2),
"HELPER_CHECKPOINT_NOT_APPLIED"
);
ensure!(
snapshot
.attempts
.iter()
.all(|a| terminal(&a.state)
|| (a.consumption == "unconsumed" && a.state == "accepted")),
"HELPER_EFFECT_UNRESOLVED"
);
gate.helper = None;
Ok(())
}
/// Typed read-only ACK for exactly the own original update handoff. It
/// clears only the control delivery acknowledgement; both start flags stay
/// false and can_start remains false for ALL jobs, including this attempt.
pub fn confirm_update_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
ensure!(
!gate.ready
&& !gate.helper_ready
&& snapshot.journal_id == expected.0
&& snapshot.checkpoint == expected.1
&& snapshot.previous_decision_id == Some(expected.2),
"UPDATE_CONTROL_NOT_APPLIED"
);
let updates: Vec<_> = snapshot
.attempts
.iter()
.filter(|a| !a.update_handoff_bytes.is_empty())
.collect();
ensure!(updates.len() == 1, "UPDATE_CONTROL_EXACTLY_ONE_REQUIRED");
let local = updates[0];
let requested = gate
.request
.attempts
.iter()
.find(|a| a.job_id == local.job_id && a.attempt_id == local.attempt_id)
.context("UPDATE_CONTROL_NOT_REQUESTED")?;
ensure!(
local.state == "running"
&& local.consumption == "consumed"
&& local.outcome_bytes.is_empty()
&& local.update_handoff_bytes == requested.update_handoff_bytes
&& local.grant_id == requested.grant_id
&& local.job_payload_hash == requested.job_payload_hash,
"UPDATE_CONTROL_CHANGED"
);
let decision: ReconcileDecisionPayload =
serde_json::from_slice(&gate.response.decision_bytes)?;
ensure!(
decision
.attempts
.iter()
.any(|d| d.job_id.to_string() == local.job_id
&& d.attempt_id.to_string() == local.attempt_id
&& d.update_inflight.is_some()
&& !d.unlock),
"UPDATE_CONTROL_NOT_SIGNED"
);
ensure!(
snapshot
.attempts
.iter()
.filter(|a| a.job_id != local.job_id || a.attempt_id != local.attempt_id)
.all(|a| terminal(&a.state)
|| (a.state == "accepted" && a.consumption == "unconsumed")),
"UPDATE_OTHER_EFFECT_UNRESOLVED"
);
gate.helper = None;
Ok(())
}
pub fn prepare(
&mut self,
db: &mut Connection,
ledger: &Connection,
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
) -> Result<PreparedSync> {
let result = self.prepare_inner(db, ledger, channel, helper, clock);
if let Err(error) = &result {
let text = error.to_string();
if [
"ROLLBACK",
"REGRESSION",
"DISAPPEARED",
"MISSING_FROM_AGENT_JOURNAL",
"MISSING_FROM_HELPER_JOURNAL",
]
.iter()
.any(|s| text.contains(s))
{
let _ = db.execute(
"UPDATE reconcile_meta SET fault=COALESCE(fault,?1) WHERE id=1",
[text],
);
}
}
result
}
fn prepare_inner(
&mut self,
db: &mut Connection,
ledger: &Connection,
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
) -> Result<PreparedSync> {
self.close();
let now = clock.now()?;
ensure!(
channel.connection_generation > 0 && !channel.boot_id.is_nil(),
"UNAUTHENTICATED_CHANNEL"
);
initialize(db)?;
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let meta = metadata(&tx)?;
ensure!(meta.fault.is_none(), "SYNC_MANUAL_RECOVERY_REQUIRED");
let sequence: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
ensure!(
sequence >= meta.checkpoint && sequence >= meta.high_sequence,
"LOCAL_CHECKPOINT_ROLLBACK"
);
ensure!(
channel.connection_generation >= meta.generation,
"CONNECTION_GENERATION_REGRESSION"
);
if let Some(h) = helper {
ensure!(
!h.journal_id.is_nil() && h.checkpoint <= i64::MAX as u64,
"HELPER_CHECKPOINT"
);
if let Some(old) = meta.helper_id {
ensure!(
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
"HELPER_JOURNAL_ROLLBACK"
);
}
} else {
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
}
if let Some(h) = helper {
ingest_helper(
&tx,
ledger,
(&self.binding, &self.verifier),
&channel,
h,
now,
self.helper_identity,
)?;
}
archive_events(&tx)?;
tx.execute(
"UPDATE job_attempts SET state='accepted' WHERE state='queued' AND grant_id=''",
[],
)?;
// A possibly started attempt is NEVER returned to accepted. Preserve
// revisions and results; only ambiguous no-result states become unknown.
let inflight: Vec<_> = helper
.into_iter()
.flat_map(|h| h.attempts.iter())
.filter(|a| !a.update_handoff_bytes.is_empty())
.collect();
ensure!(inflight.len() <= 1, "UPDATE_MULTIPLE_INFLIGHT");
if let Some(update) = inflight.first() {
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT (job=?1 AND attempt=?2) AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",params![update.job_id,update.attempt_id])?;
} else {
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?;
}
let mut attempts = attempts(&tx, ledger, &self.binding)?;
if let Some(h) = helper {
merge_helper(&mut attempts, h)?;
}
ensure!(attempts.len() <= 256, "SYNC_ATTEMPT_CAPACITY_NO_COMPACTION");
let mut ids = pending_ids(&tx)?;
if let Some(update) = inflight.first() {
let grant = Uuid::parse_str(&update.grant_id)?;
ids.extend(crate::update_health_inventory::pending_ids(&tx, grant)?);
ids.sort();
ids.dedup();
}
ensure!(ids.len() <= 256, "SYNC_RECEIPT_CAPACITY");
let next: i64 = tx.query_row(
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
[],
|r| r.get(0),
)?;
let request = SyncRequest {
request_id: Uuid::new_v4().to_string(),
journal_id: meta.journal_id.to_string(),
checkpoint: u64::try_from(next)?,
// A pending request may have committed with a lost reply, or may
// never have reached the server. Preserve both witnesses: the
// server resolves the pending request first and otherwise checks
// this last applied decision against its authoritative head.
previous_decision_id: meta.decision.map(|id| id.to_string()).unwrap_or_default(),
assignment_version: self.binding.assignment_version,
attempts,
durable_message_ids: ids,
journal_complete: true,
helper_journal_id: helper.map(|h| h.journal_id.to_string()).unwrap_or_default(),
helper_checkpoint: helper.map_or(0, |h| h.checkpoint),
helper_previous_decision_id: helper
.and_then(|h| h.previous_decision_id)
.map(|id| id.to_string())
.unwrap_or_default(),
previous_request_id: meta
.pending_request
.map(|id| id.to_string())
.unwrap_or_default(),
};
let envelope = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: self.binding.agent_id.to_string(),
enrollment_generation: self.binding.enrollment_generation,
connection_generation: channel.connection_generation,
boot_id: channel.boot_id.to_string(),
sequence: u64::try_from(next)?,
observed_at: now.timestamp_millis(),
payload: Some(Payload::SyncRequest(request.clone())),
};
ensure!(
envelope.encoded_len() <= rmm_domain::MAX_MESSAGE,
"SYNC_ENVELOPE_SIZE"
);
tx.execute(
"INSERT INTO reconcile_requests(id,request,envelope,generation) VALUES(?1,?2,?3,?4)",
params![
request.request_id,
request.encode_to_vec(),
envelope.encode_to_vec(),
channel.connection_generation
],
)?;
tx.execute("UPDATE reconcile_meta SET checkpoint=?1,high_sequence=?1,generation=?2,pending_request=?3 WHERE id=1",params![next,channel.connection_generation,request.request_id])?;
tx.commit()?;
let prepared = PreparedSync { envelope, request };
self.pending = Some(prepared.clone());
self.channel = Some(channel);
Ok(prepared)
}
/// All changes, concrete ACKs, result archives and checkpoint chain commit
/// together. A rejection closes only mutation authority; telemetry may run.
pub fn apply(
&mut self,
db: &mut Connection,
response: &SyncResponse,
clock: &impl SyncClock,
) -> Result<Applied> {
if self.pending.is_none() {
let result = self.replay_applied(db, response, clock);
if result.is_err() {
self.gate = None;
}
return result;
}
self.gate = None;
let result = self.apply_inner(db, response, clock);
if result.is_err() {
self.gate = None;
}
result
}
fn replay_applied(
&self,
db: &Connection,
response: &SyncResponse,
clock: &impl SyncClock,
) -> Result<Applied> {
let gate = self.gate.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?;
ensure!(gate.response == *response, "REPLAYED_DIFFERENT_DECISION");
let d = verify_response_with(
&self.verifier,
&self.binding,
&gate.request,
&gate.channel,
response,
clock.now()?,
)?;
let meta = metadata(db)?;
let sequence: i64 = db.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
ensure!(
meta.decision == Some(d.decision_id)
&& meta.pending_request.is_none()
&& meta.checkpoint == i64::try_from(d.checkpoint)?
&& sequence >= meta.high_sequence,
"APPLIED_CHECKPOINT_REGRESSION"
);
Ok(Applied {
decision_id: d.decision_id,
acknowledged_message_ids: d.acknowledged_message_ids,
helper_instruction: d.helper_journal_id.map(|journal_id| HelperInstruction {
response: response.clone(),
decision_id: d.decision_id,
journal_id,
checkpoint: d.helper_checkpoint,
}),
reason: d.reason,
})
}
fn apply_inner(
&mut self,
db: &mut Connection,
response: &SyncResponse,
clock: &impl SyncClock,
) -> Result<Applied> {
let now = clock.now()?;
let prepared = self.pending.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?;
let channel = self.channel.as_ref().context("NO_AUTHENTICATED_CHANNEL")?;
let decision = verify_response_with(
&self.verifier,
&self.binding,
&prepared.request,
channel,
response,
now,
)?;
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let meta = metadata(&tx)?;
ensure!(
meta.fault.is_none()
&& meta.pending_request == Some(decision.request_id)
&& meta.checkpoint == i64::try_from(decision.checkpoint)?
&& meta.generation == channel.connection_generation,
"SYNC_CHECKPOINT_CHANGED"
);
let seq: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
ensure!(
seq >= meta.high_sequence && seq >= meta.checkpoint,
"LOCAL_SEQUENCE_ROLLBACK"
);
let current = tx.query_row(
"SELECT request FROM reconcile_requests WHERE id=?1",
[decision.request_id.to_string()],
|r| r.get::<_, Vec<u8>>(0),
)?;
ensure!(
current == prepared.request.encode_to_vec(),
"DURABLE_CHALLENGE_CHANGED"
);
let summaries: BTreeMap<_, _> = prepared
.request
.attempts
.iter()
.map(|a| ((a.job_id.clone(), a.attempt_id.clone()), a))
.collect();
let actual: BTreeSet<_> = decision
.attempts
.iter()
.map(|a| (a.job_id.to_string(), a.attempt_id.to_string()))
.collect();
ensure!(
actual.len() == decision.attempts.len()
&& actual == summaries.keys().cloned().collect(),
"SYNC_ATTEMPT_OMITTED_OR_ADDED"
);
let mut safe = true;
let mut fault = recovery_reason(&decision.reason).then(|| decision.reason.clone());
for d in &decision.attempts {
let local = summaries
.get(&(d.job_id.to_string(), d.attempt_id.to_string()))
.context("LOCAL_ATTEMPT_MISSING")?;
if recovery_reason(&d.reason) {
fault = Some(d.reason.clone());
}
let offer_bytes: Vec<u8> = tx.query_row(
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
params![local.job_id, local.attempt_id],
|r| r.get(0),
)?;
let offer = JobOffer::decode(offer_bytes.as_slice())?;
let job = self.binding.job(&offer.payload_bytes)?;
// Missing-server decisions intentionally have empty action fields;
// persist their sticky denial without treating it as start authority.
if recovery_reason(&d.reason) {
safe = false;
continue;
}
let outcome =
validate_attempt(&job, &offer, local, d, channel.boot_id, &self.binding, now)?;
safe &= (d.outcome_state == "accepted"
&& local.consumption == "unconsumed"
&& local.grant_id.is_empty())
|| (terminal(&d.outcome_state) && (local.consumption == "unconsumed" || d.unlock));
let (stored_state, revision): (String, u64) = tx.query_row(
"SELECT state,revision FROM job_attempts WHERE job=?1 AND attempt=?2",
params![local.job_id, local.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
ensure!(
revision == local.revision && stored_state == local.state,
"LOCAL_ATTEMPT_CHANGED_DURING_SYNC"
);
let stored = if d.outcome_state == "unknown" && local.state == "awaiting_reboot" {
"awaiting_reboot"
} else {
d.outcome_state.as_str()
};
tx.execute(
"UPDATE job_attempts SET state=?3,revision=?4 WHERE job=?1 AND attempt=?2",
params![
local.job_id,
local.attempt_id,
stored,
d.revision.max(local.revision)
],
)?;
if let Some(bytes) = outcome {
tx.execute("INSERT INTO reconcile_outcomes(job,attempt,outcome,decision_id) VALUES(?1,?2,?3,?4) ON CONFLICT(job,attempt) DO UPDATE SET outcome=excluded.outcome,decision_id=excluded.decision_id",params![local.job_id,local.attempt_id,bytes,decision.decision_id.to_string()])?;
}
}
let requested: BTreeSet<_> = prepared
.request
.durable_message_ids
.iter()
.map(String::as_str)
.collect();
let mut receipts = BTreeSet::new();
for id in &decision.acknowledged_message_ids {
ensure!(
receipts.insert(*id) && requested.contains(id.to_string().as_str()),
"ACK_NOT_IN_REQUEST"
);
archive_events(&tx)?;
tx.execute("DELETE FROM job_events WHERE id=?1", [id.to_string()])?;
tx.execute(
"INSERT OR IGNORE INTO reconcile_receipts(id,decision_id) VALUES(?1,?2)",
params![id.to_string(), decision.decision_id.to_string()],
)?;
}
tx.execute(
"UPDATE reconcile_requests SET response=?2 WHERE id=?1 AND response IS NULL",
params![decision.request_id.to_string(), response.encode_to_vec()],
)?;
for local in prepared
.request
.attempts
.iter()
.filter(|a| !a.update_handoff_bytes.is_empty())
{
crate::update_health_inventory::acknowledge_signed(
&tx,
Uuid::parse_str(&local.grant_id)?,
&decision,
response,
)?;
}
tx.execute("UPDATE reconcile_meta SET last_decision=?1,pending_request=NULL,helper_id=?2,helper_checkpoint=?3,fault=COALESCE(fault,?4),high_sequence=?5 WHERE id=1",
params![decision.decision_id.to_string(),decision.helper_journal_id.map(|id|id.to_string()),decision.helper_checkpoint,fault,seq])?;
tx.commit()?;
let helper_instruction = decision
.helper_journal_id
.map(|journal_id| HelperInstruction {
response: response.clone(),
decision_id: decision.decision_id,
journal_id,
checkpoint: decision.helper_checkpoint,
});
self.gate = Some(Gate {
channel: channel.clone(),
until: decision.valid_until,
ready: decision.ready && safe && fault.is_none(),
helper_ready: decision.helper_ready,
helper: helper_instruction
.as_ref()
.map(|h| (h.journal_id, h.checkpoint, h.decision_id)),
local_applied: !decision.attempts.iter().any(|d| d.unlock),
request: prepared.request.clone(),
response: response.clone(),
});
self.pending = None;
Ok(Applied {
decision_id: decision.decision_id,
acknowledged_message_ids: decision.acknowledged_message_ids,
helper_instruction,
reason: decision.reason,
})
}
/// Apply ONLY this process's independently owned consumption ledger. Root
/// helper unlocks use HelperInstruction over authenticated IPC instead.
/// A crash between journal and ledger transactions leaves the gate closed;
/// resync/reapply never deletes the consumed attempt or repeats an effect.
pub fn apply_consumption(
&mut self,
ledger: &mut Connection,
clock: &impl SyncClock,
) -> Result<()> {
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
gate.local_applied = false;
let decision = verify_response_with(
&self.verifier,
&self.binding,
&gate.request,
&gate.channel,
&gate.response,
clock.now()?,
)?;
let tx = ledger.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute_batch("CREATE TABLE IF NOT EXISTS reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB NOT NULL);")?;
let applied: Option<Vec<u8>> = tx
.query_row(
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
[decision.decision_id.to_string()],
|r| r.get(0),
)
.optional()?;
if let Some(old) = applied {
ensure!(
old == gate.response.encode_to_vec(),
"CONSUMPTION_DECISION_CONFLICT"
);
} else {
for d in decision.attempts.iter().filter(|d| d.unlock) {
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
let (agent,generation,job,attempt,previous):(String,u64,String,String,Option<Vec<u8>>)=tx.query_row(
"SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1",
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?;
ensure!(
agent == self.binding.agent_id.to_string()
&& generation == self.binding.enrollment_generation
&& job == d.job_id.to_string()
&& attempt == d.attempt_id.to_string(),
"LOCAL_UNLOCK_CONSUMPTION_BINDING"
);
let previous = previous.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&previous,
)?)?;
ensure!(
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
);
let bytes = if d.outcome_bytes.is_empty() {
prior
} else {
d.outcome_bytes.clone()
};
let outcome: ReconciledOutcome = serde_json::from_slice(&bytes)?;
ensure!(
terminal(&outcome.state)
&& outcome.state == d.outcome_state
&& canonical_outcome_bytes(&outcome)? == bytes
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()),
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
);
tx.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
params![grant.to_string(), bytes],
)?;
tx.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
params![grant.to_string(), bytes],
)?;
}
tx.execute(
"INSERT INTO reconcile_consumption(decision_id,response) VALUES(?1,?2)",
params![
decision.decision_id.to_string(),
gate.response.encode_to_vec()
],
)?;
}
tx.commit()?;
gate.local_applied = true;
Ok(())
}
}
pub fn initialize(db: &Connection) -> Result<()> {
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "JOURNAL_INTEGRITY");
db.execute_batch("PRAGMA synchronous=FULL;
CREATE TABLE IF NOT EXISTS reconcile_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,high_sequence INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,last_decision TEXT,pending_request TEXT,helper_id TEXT,helper_checkpoint INTEGER NOT NULL DEFAULT 0,fault TEXT);
CREATE TABLE IF NOT EXISTS reconcile_requests(id TEXT PRIMARY KEY,request BLOB NOT NULL,envelope BLOB NOT NULL,generation INTEGER NOT NULL,response BLOB);
CREATE TABLE IF NOT EXISTS reconcile_results(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,revision INTEGER NOT NULL,envelope BLOB NOT NULL);
CREATE TABLE IF NOT EXISTS reconcile_receipts(id TEXT PRIMARY KEY,decision_id TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS reconcile_outcomes(job TEXT NOT NULL,attempt TEXT NOT NULL,outcome BLOB NOT NULL,decision_id TEXT NOT NULL,PRIMARY KEY(job,attempt));")?;
crate::update_health_inventory::initialize(db)?;
db.execute(
"INSERT OR IGNORE INTO reconcile_meta(id,journal_id) VALUES(1,?1)",
[Uuid::new_v4().to_string()],
)?;
Ok(())
}
struct Meta {
journal_id: Uuid,
checkpoint: i64,
high_sequence: i64,
generation: u64,
decision: Option<Uuid>,
pending_request: Option<Uuid>,
helper_id: Option<Uuid>,
helper_checkpoint: u64,
fault: Option<String>,
}
type MetadataRow = (
String,
i64,
i64,
u64,
Option<String>,
Option<String>,
Option<String>,
u64,
Option<String>,
);
fn metadata(db: &Connection) -> Result<Meta> {
let row:MetadataRow=db.query_row("SELECT journal_id,checkpoint,high_sequence,generation,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?)))?;
Ok(Meta {
journal_id: Uuid::parse_str(&row.0)?,
checkpoint: row.1,
high_sequence: row.2,
generation: row.3,
decision: row.4.map(|s| Uuid::parse_str(&s)).transpose()?,
pending_request: row.5.map(|s| Uuid::parse_str(&s)).transpose()?,
helper_id: row.6.map(|s| Uuid::parse_str(&s)).transpose()?,
helper_checkpoint: row.7,
fault: row.8,
})
}
/// Call inside the same transaction BEFORE deleting an ACKed job event. This
/// retains the original result identity/output for future reconciliation.
pub fn archive_events(db: &Connection) -> Result<()> {
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY rowid")?;
for row in statement.query_map([], |r| {
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
})? {
let (id, bytes) = row?;
let envelope = Envelope::decode(bytes.as_slice())?;
ensure!(id == envelope.message_id, "EVENT_IDENTITY_CONFLICT");
if let Some(Payload::JobResult(r)) = &envelope.payload {
let existing: Option<Vec<u8>> = db
.query_row(
"SELECT envelope FROM reconcile_results WHERE id=?1",
[&id],
|r| r.get(0),
)
.optional()?;
ensure!(
existing.as_ref().is_none_or(|b| *b == bytes),
"RESULT_IDENTITY_CONFLICT"
);
db.execute("INSERT OR IGNORE INTO reconcile_results(id,job,attempt,revision,envelope) VALUES(?1,?2,?3,?4,?5)",params![id,r.job_id,r.attempt_id,r.revision,bytes])?;
}
}
Ok(())
}
fn pending_ids(db: &Connection) -> Result<Vec<String>> {
let mut statement = db.prepare("SELECT id FROM job_events ORDER BY id")?;
let ids = statement
.query_map([], |r| r.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
for id in &ids {
ensure!(!Uuid::parse_str(id)?.is_nil(), "INVALID_DURABLE_MESSAGE_ID");
}
Ok(ids)
}
fn outcome(r: &JobResult) -> Result<ReconciledOutcome> {
Ok(ReconciledOutcome {
state: r.state.clone(),
stdout: r.stdout.clone(),
stderr: r.stderr.clone(),
exit_code: r.exit_code,
output_truncated: r.output_truncated,
reboot_required: r.reboot_required,
evidence: serde_json::from_str(&r.evidence_json)?,
error_code: r.error_code.clone(),
})
}
fn terminal(state: &str) -> bool {
matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
}
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
if s.is_empty() {
Ok(None)
} else {
let id = Uuid::parse_str(s)?;
ensure!(!id.is_nil(), "NIL_ID");
Ok(Some(id))
}
}
fn recovery_reason(reason: &str) -> bool {
matches!(
reason,
"MISSING_SERVER_JOB"
| "ATTEMPT_BINDING_MISMATCH"
| "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED"
| "SERVER_STATE_ROLLBACK"
| "ASSIGNMENT_MISMATCH"
| "JOURNAL_SEQUENCE_REGRESSION"
| "SERVER_CHECKPOINT_MISSING"
| "JOURNAL_ROLLBACK_OR_REPLACEMENT"
| "CHECKPOINT_CHAIN_MISMATCH"
| "HELPER_JOURNAL_ROLLBACK"
| "HELPER_CHECKPOINT_MISMATCH"
| "HELPER_CHECKPOINT_MISSING"
| "HELPER_CHECKPOINT_REQUIRED"
| "JOURNAL_HISTORY_MISSING"
| "LOCAL_ATTEMPT_MISSING"
)
}
fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<Vec<SyncAttempt>> {
let mut statement =
db.prepare("SELECT offer,state,revision,grant_id FROM job_attempts ORDER BY job,attempt")?;
let rows = statement.query_map([], |r| {
Ok((
r.get::<_, Vec<u8>>(0)?,
r.get::<_, String>(1)?,
r.get::<_, u64>(2)?,
r.get::<_, String>(3)?,
))
})?;
let mut summaries = Vec::new();
let mut known = BTreeSet::new();
for row in rows {
let (bytes, state, revision, mut grant_id) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes),
"LOCAL_OFFER_BINDING"
);
known.insert((offer.job_id.clone(), offer.attempt_id.clone()));
let consumed: Option<(String, String, u64, Option<Vec<u8>>)> = ledger
.query_row(
"SELECT grant_id,agent,generation,result FROM consumed WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)
.optional()?;
// A signed definitive nonstart can make an accepted no-grant row
// terminal without a JobResult. Preserve that fact on the next Sync;
// requesting/running/unknown still cannot claim non-consumption. An
// actual durable consumption row below always overrides this label.
let mut consumption = if grant_id.is_empty()
&& matches!(state.as_str(), "accepted" | "expired" | "cancelled")
{
"unconsumed"
} else {
"uncertain"
};
let mut original_boot_id = String::new();
let mut canonical = Vec::new();
if let Some((id, agent, generation, result)) = consumed {
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& (grant_id.is_empty() || grant_id == id),
"CONSUMPTION_BINDING"
);
grant_id = id;
consumption = "consumed";
let evidence: Option<(Vec<u8>, Option<Vec<u8>>)> = ledger
.query_row(
"SELECT start,outcome FROM execution_evidence WHERE grant_id=?1",
[&grant_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()?;
let mut stored = result;
if let Some((start, last)) = evidence {
let start: serde_json::Value = serde_json::from_slice(&start)?;
original_boot_id = start["previous_boot_id"]
.as_str()
.context("ORIGINAL_BOOT_MISSING")?
.to_string();
optional_uuid(&original_boot_id)?;
let request: crate::auth::Request =
serde_json::from_value(start["request"].clone())?;
ensure!(
request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"START_EVIDENCE_BINDING"
);
if last.is_some() {
stored = last;
}
}
if let Some(stored) = stored {
let o: ReconciledOutcome = serde_json::from_slice(&stored)?;
canonical = canonical_outcome_bytes(&o)?;
}
}
let original:Option<Vec<u8>>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",params![offer.job_id,offer.attempt_id],|r|r.get(0)).optional()?;
let mut result_envelope = None;
let mut result = None;
let mut result_message_id = String::new();
if let Some(bytes) = original {
let envelope = Envelope::decode(bytes.as_slice())?;
let Some(Payload::JobResult(r)) = &envelope.payload else {
anyhow::bail!("INVALID_ARCHIVED_RESULT")
};
if r.revision == revision && r.state == state {
ensure!(
envelope.agent_id == binding.agent_id.to_string()
&& envelope.enrollment_generation == binding.enrollment_generation
&& r.grant_id == grant_id,
"RESULT_BINDING"
);
let bytes = canonical_outcome_bytes(&outcome(r)?)?;
ensure!(
canonical.is_empty() || canonical == bytes,
"LOCAL_OUTCOME_LEDGER_MISMATCH"
);
canonical = bytes;
result_message_id = envelope.message_id.clone();
result = Some(r.clone());
result_envelope = Some(envelope);
}
}
if !canonical.is_empty() {
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
ensure!(canonical.len() <= 512 * 1024, "SYNC_CANONICAL_OUTCOME_SIZE");
if grant_id.is_empty() && terminal(&state) && o.evidence["not_started"] == true {
consumption = "unconsumed";
}
}
summaries.push(SyncAttempt {
update_handoff_bytes: Vec::new(),
job_id: offer.job_id,
attempt_id: offer.attempt_id,
job_payload_hash: offer.job_payload_hash,
grant_id,
consumption: consumption.into(),
state,
revision,
original_boot_id,
result,
result_message_id,
outcome_sha256: if canonical.is_empty() {
String::new()
} else {
hash(&canonical)
},
outcome_bytes: canonical,
result_envelope,
});
}
let mut statement = ledger.prepare("SELECT job,attempt FROM consumed")?;
for row in statement.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
ensure!(
known.contains(&row?),
"CONSUMED_ATTEMPT_MISSING_FROM_AGENT_JOURNAL"
);
}
Ok(summaries)
}
fn ingest_helper(
db: &Connection,
ledger: &Connection,
binding_and_verifier: (&Binding, &impl Ed25519Verifier),
channel: &Channel,
helper: &HelperSnapshot,
now: DateTime<Utc>,
identity: HelperIdentity,
) -> Result<()> {
let (binding, verifier) = binding_and_verifier;
let mut statement = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
for row in statement.query_map([], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, String>(1)?,
r.get::<_, Vec<u8>>(2)?,
))
})? {
let (job, attempt, bytes) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
if binding.job(&offer.payload_bytes)?.execution_identity == identity.as_str() {
let consumed: bool = ledger.query_row(
"SELECT EXISTS(SELECT 1 FROM consumed WHERE job=?1 AND attempt=?2)",
params![job, attempt],
|r| r.get(0),
)?;
ensure!(
!consumed
|| helper
.attempts
.iter()
.any(|a| a.job_id == job && a.attempt_id == attempt),
"ROOT_CONSUMED_ATTEMPT_MISSING_FROM_HELPER_JOURNAL"
);
}
}
let mut seen = BTreeSet::new();
for h in &helper.attempts {
ensure!(
seen.insert((&h.job_id, &h.attempt_id)),
"DUPLICATE_HELPER_ATTEMPT"
);
let row: Option<(Vec<u8>, u64, String)> = db
.query_row(
"SELECT offer,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.optional()?;
let (bytes, revision, grant) = row.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
job.execution_identity == identity.as_str()
&& h.job_payload_hash == offer.job_payload_hash
&& (grant.is_empty() || grant == h.grant_id),
"HELPER_ATTEMPT_BINDING"
);
if h.outcome_bytes.is_empty() {
if !h.update_handoff_bytes.is_empty() {
let header =
validate_update_summary(binding, &job, &offer, h, channel.boot_id, now)?;
ensure!(
header.helper_journal_id == helper.journal_id
&& header.agent_journal_id == metadata(db)?.journal_id,
"UPDATE_HELPER_JOURNAL_BINDING"
);
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id],
|r| r.get(0),
)?;
ensure!(
matches!(
state.as_str(),
"accepted" | "requesting" | "running" | "verifying"
),
"UPDATE_CANNOT_HEAL_UNKNOWN"
);
type OriginalEvidence = (
String,
u64,
String,
String,
Vec<u8>,
Option<Vec<u8>>,
Option<Vec<u8>>,
);
let original:Option<OriginalEvidence>=ledger.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM execution_evidence e JOIN consumed c USING(grant_id) WHERE grant_id=?1",[&h.grant_id],
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
let (agent, generation, job_id, attempt_id, start, outcome, result) =
original.context("UPDATE_LOCAL_ORIGINAL_MISSING")?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == h.job_id
&& attempt_id == h.attempt_id,
"UPDATE_LOCAL_CONSUMPTION_BINDING"
);
ensure!(
outcome.is_none() && result.is_none(),
"UPDATE_CANNOT_REPLACE_OUTCOME"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: crate::auth::Request =
serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
binding.verify_with(&request, permit.not_before, verifier)?;
ensure!(
hash(&serde_json::to_vec(&request)?) == header.attempt.original_request_sha256
&& permit.grant_id == header.attempt.grant_id
&& permit.agent_update_plan_id == Some(header.attempt.plan_id),
"UPDATE_LOCAL_ORIGINAL_CHANGED"
);
db.execute("UPDATE job_attempts SET state='running',revision=MAX(revision,?3) WHERE job=?1 AND attempt=?2",params![h.job_id,h.attempt_id,h.revision])?;
}
continue;
}
let o: ReconciledOutcome = serde_json::from_slice(&h.outcome_bytes)?;
ensure!(
canonical_outcome_bytes(&o)? == h.outcome_bytes
&& hash(&h.outcome_bytes) == h.outcome_sha256
&& o.state == h.state
&& (terminal(&o.state)
|| matches!(o.state.as_str(), "unknown" | "awaiting_reboot"))
&& h.consumption == "consumed"
&& optional_uuid(&h.grant_id)?.is_some(),
"HELPER_CANONICAL_OUTCOME"
);
let existing: Option<Vec<u8>> = ledger
.query_row(
"SELECT outcome FROM execution_evidence WHERE grant_id=?1",
[&h.grant_id],
|r| r.get(0),
)
.optional()?
.flatten();
let same = existing
.as_ref()
.map(|b| {
serde_json::from_slice::<ReconciledOutcome>(b)
.and_then(|o| canonical_outcome_bytes(&o))
})
.transpose()?
.is_some_and(|b| b == h.outcome_bytes);
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id],
|r| r.get(0),
)?;
if same && state == h.state {
continue;
}
let consumed: Option<(String, u64, String, String)> = ledger
.query_row(
"SELECT agent,generation,job,attempt FROM consumed WHERE grant_id=?1",
[&h.grant_id],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)
.optional()?;
ensure!(
consumed
== Some((
binding.agent_id.to_string(),
binding.enrollment_generation,
h.job_id.clone(),
h.attempt_id.clone()
)),
"HELPER_CONSUMPTION_NOT_KNOWN_LOCALLY"
);
// Store actual helper evidence without unlocking result/consumption. A
// crash here still leaves both gates closed and the original helper
// result available; the next sync imports it again idempotently.
ensure!(
ledger.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
params![h.grant_id, h.outcome_bytes]
)? == 1,
"LOCAL_START_EVIDENCE_MISSING"
);
let next_revision = revision
.max(h.revision)
.checked_add(1)
.context("REVISION_OVERFLOW")?;
i64::try_from(next_revision)?;
let seq: i64 = db.query_row(
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
[],
|r| r.get(0),
)?;
let result = JobResult {
job_id: h.job_id.clone(),
attempt_id: h.attempt_id.clone(),
revision: next_revision,
state: o.state.clone(),
grant_id: h.grant_id.clone(),
stdout: o.stdout,
stderr: o.stderr,
output_truncated: o.output_truncated,
reboot_required: o.reboot_required,
exit_code: o.exit_code,
evidence_json: o.evidence.to_string(),
error_code: o.error_code,
};
let envelope = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: binding.agent_id.to_string(),
enrollment_generation: binding.enrollment_generation,
connection_generation: channel.connection_generation,
boot_id: channel.boot_id.to_string(),
sequence: u64::try_from(seq)?,
observed_at: now.timestamp_millis(),
payload: Some(Payload::JobResult(result)),
};
ensure!(
envelope.encoded_len() <= rmm_domain::MAX_MESSAGE,
"HELPER_RESULT_SIZE"
);
db.execute(
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
params![envelope.message_id, envelope.encode_to_vec()],
)?;
db.execute(
"UPDATE job_attempts SET state=?3,revision=?4,grant_id=?5 WHERE job=?1 AND attempt=?2",
params![h.job_id, h.attempt_id, o.state, next_revision, h.grant_id],
)?;
}
Ok(())
}
fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()> {
let mut seen = BTreeSet::new();
for h in &helper.attempts {
ensure!(
seen.insert((&h.job_id, &h.attempt_id)),
"DUPLICATE_HELPER_ATTEMPT"
);
let a = local
.iter_mut()
.find(|a| a.job_id == h.job_id && a.attempt_id == h.attempt_id)
.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?;
ensure!(
a.job_payload_hash == h.job_payload_hash
&& (a.grant_id.is_empty() || a.grant_id == h.grant_id),
"HELPER_ATTEMPT_BINDING"
);
if h.consumption != "unconsumed" {
if !h.update_handoff_bytes.is_empty() {
ensure!(
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
"UPDATE_CANNOT_HEAL_UNKNOWN"
);
a.update_handoff_bytes.clone_from(&h.update_handoff_bytes);
}
a.grant_id = h.grant_id.clone();
a.consumption = h.consumption.clone();
ensure!(
a.outcome_sha256 == h.outcome_sha256,
"HELPER_OUTCOME_MISMATCH"
);
if a.original_boot_id.is_empty() {
a.original_boot_id = h.original_boot_id.clone();
}
ensure!(
a.original_boot_id == h.original_boot_id,
"HELPER_ORIGINAL_BOOT_MISMATCH"
);
}
}
Ok(())
}
fn validate_update_summary(
_binding: &Binding,
job: &JobPayload,
offer: &JobOffer,
local: &SyncAttempt,
boot: Uuid,
now: DateTime<Utc>,
) -> Result<rmm_updater_handoff::Handoff> {
let proof =
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
let (h, _) =
rmm_domain::agent_update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
let now = u64::try_from(now.timestamp())?;
ensure!(
rmm_domain::agent_update::handoff_matches_job(&h, job)
&& h.boot_id == boot
&& h.attempt.grant_id.to_string() == local.grant_id
&& h.attempt.authorization_id.to_string() == offer.authorization_id
&& h.attempt.plan_sha256 == offer.plan_hash
&& h.attempt.job_payload_sha256 == offer.job_payload_hash
&& local.original_boot_id == boot.to_string()
&& local.consumption == "consumed"
&& local.state == "running"
&& local.outcome_bytes.is_empty()
&& local.outcome_sha256.is_empty()
&& local.result.is_none()
&& local.result_envelope.is_none()
&& h.consumed_at <= now
&& now < h.finish_before,
"UPDATE_INFLIGHT_SUMMARY_BINDING"
);
Ok(h)
}
pub fn verify_response(
binding: &Binding,
request: &SyncRequest,
channel: &Channel,
response: &SyncResponse,
now: DateTime<Utc>,
) -> Result<ReconcileDecisionPayload> {
verify_response_with(
&DefaultVerifier::default(),
binding,
request,
channel,
response,
now,
)
}
pub fn verify_response_with(
verifier: &impl Ed25519Verifier,
binding: &Binding,
request: &SyncRequest,
channel: &Channel,
response: &SyncResponse,
now: DateTime<Utc>,
) -> Result<ReconcileDecisionPayload> {
ensure!(
response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64,
"SYNC_SIGNATURE_SIZE"
);
let raw = decode_key(&binding.public_key_hex)?;
ensure!(
verifier.verify(&raw, &response.decision_bytes, &response.signature)?,
"INVALID_SYNC_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
ensure!(
d.valid_profile(now)
&& d.valid_until - d.issued_at <= chrono::Duration::seconds(60)
&& d.key_id == hash(&raw),
"SYNC_PROFILE_OR_CLOCK"
);
ensure!(
d.tenant_id == binding.tenant_id
&& d.asset_id == binding.asset_id
&& d.agent_id == binding.agent_id
&& d.enrollment_generation == binding.enrollment_generation
&& d.assignment_version == binding.assignment_version
&& d.connection_generation == channel.connection_generation
&& d.boot_id == channel.boot_id
&& d.request_id.to_string() == request.request_id
&& d.request_hash == hash(&request.encode_to_vec())
&& d.journal_id.to_string() == request.journal_id
&& d.checkpoint == request.checkpoint
&& d.helper_journal_id == optional_uuid(&request.helper_journal_id)?
&& d.helper_checkpoint == request.helper_checkpoint,
"SYNC_DECISION_BINDING"
);
Ok(d)
}
fn decode_key(hex: &str) -> Result<Vec<u8>> {
ensure!(hex.len() == 64, "PINNED_KEY_SIZE");
(0..32)
.map(|i| {
Ok(u8::from_str_radix(
hex.get(i * 2..i * 2 + 2).context("PINNED_KEY_ENCODING")?,
16,
)?)
})
.collect()
}
fn validate_attempt(
job: &JobPayload,
offer: &JobOffer,
local: &SyncAttempt,
d: &AttemptDecision,
boot: Uuid,
binding: &Binding,
now: DateTime<Utc>,
) -> Result<Option<Vec<u8>>> {
ensure!(
d.job_payload_hash == offer.job_payload_hash
&& d.grant_id == optional_uuid(&local.grant_id)?
&& d.action == job.action.name()
&& d.execution_identity == job.execution_identity
&& (d.revision >= local.revision || d.outcome_state == "unknown")
&& d.original_boot_id == optional_uuid(&local.original_boot_id)?
&& d.previous_outcome_sha256.as_deref()
== (!local.outcome_sha256.is_empty()).then_some(local.outcome_sha256.as_str()),
"SYNC_ATTEMPT_BINDING"
);
if d.grant_id.is_some() {
ensure!(
d.authorization_id == Some(Uuid::parse_str(&offer.authorization_id)?)
&& d.plan_hash.as_deref() == Some(offer.plan_hash.as_str()),
"SYNC_AUTHORIZATION_BINDING"
);
}
ensure!(
!terminal(&local.state) || d.outcome_state == local.state,
"TERMINAL_STATE_REGRESSION"
);
if d.update_inflight.is_some() || !local.update_handoff_bytes.is_empty() {
validate_update_summary(binding, job, offer, local, boot, now)?;
let proof =
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let key =
rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
let expected =
rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
ensure!(
d.update_inflight.as_ref() == Some(&expected)
&& d.outcome_state == "running"
&& !d.unlock
&& d.outcome_bytes.is_empty()
&& d.outcome_sha256.is_none(),
"UPDATE_INFLIGHT_DECISION_CHANGED"
);
return Ok(None);
}
if d.outcome_state == "accepted" {
ensure!(
local.state == "accepted"
&& local.consumption == "unconsumed"
&& d.grant_id.is_none()
&& !d.unlock
&& d.outcome_bytes.is_empty(),
"SYNC_CANNOT_RESTART_ATTEMPT"
);
return Ok(None);
}
if d.outcome_state == "unknown" {
ensure!(!d.unlock, "UNKNOWN_CANNOT_UNLOCK");
return Ok(None);
}
ensure!(terminal(&d.outcome_state), "UNSUPPORTED_SYNC_STATE");
if d.outcome_bytes.is_empty() && d.outcome_sha256.is_none() {
ensure!(
local.consumption == "unconsumed"
&& d.grant_id.is_none()
&& !d.unlock
&& d.outcome_state != "succeeded",
"TERMINAL_WITHOUT_EVIDENCE"
);
return Ok(None);
}
let bytes = if d.outcome_bytes.is_empty() {
local.outcome_bytes.clone()
} else {
d.outcome_bytes.clone()
};
let o: ReconciledOutcome = serde_json::from_slice(&bytes)?;
ensure!(
canonical_outcome_bytes(&o)? == bytes
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
&& o.state == d.outcome_state,
"SYNC_CANONICAL_OUTCOME"
);
if bytes != local.outcome_bytes {
ensure!(
matches!(job.action, rmm_domain::jobs::Action::Reboot { .. })
&& d.outcome_state == "succeeded"
&& d.original_boot_id.is_some_and(|id| id != boot)
&& o.evidence["postconditions_passed"] == true
&& o.evidence["reconciled_after_reboot"] == true
&& o.evidence["new_boot_id"].as_str() == Some(boot.to_string().as_str())
&& o.evidence["inventory_message_id"]
.as_str()
.is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())),
"SYNC_REBOOT_REAUTH_EVIDENCE"
);
}
ensure!(
!d.unlock || (local.consumption == "consumed" && d.grant_id.is_some()),
"UNLOCK_WITHOUT_CONSUMPTION"
);
Ok(Some(bytes))
}