2414 lines
100 KiB
Rust
2414 lines
100 KiB
Rust
//! Reconnect reconciliation, deliberately independent of main/jobs and transport.
|
|
//! No method executes a job, deletes consumption tombstones or issues a permit.
|
|
//! Call only while holding the existing agent.lock. HelperSnapshot must come
|
|
//! from the independently authenticated root-helper IPC, never from the server.
|
|
use crate::auth::Binding;
|
|
pub use crate::auth::HelperIdentity;
|
|
use crate::crypto::{DefaultVerifier, Ed25519Verifier};
|
|
use anyhow::{Context, Result, ensure};
|
|
use chrono::{DateTime, Utc};
|
|
use prost::Message;
|
|
use rmm_domain::{
|
|
jobs::{JobPayload, hash},
|
|
reconciliation::{
|
|
AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes,
|
|
},
|
|
};
|
|
use rmm_protocol::{
|
|
Envelope, JobOffer, JobResult, Payload, SyncAttempt, SyncRequest, SyncResponse,
|
|
};
|
|
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use uuid::Uuid;
|
|
|
|
/// A fresh observation returned by the trusted local clock implementation,
|
|
/// never an IPC DTO or an execution permit. Callers must not retain it as a lease.
|
|
pub struct ClockReading {
|
|
pub own_wall: DateTime<Utc>,
|
|
pub upper: DateTime<Utc>,
|
|
}
|
|
pub trait SyncClock {
|
|
fn now(&self) -> Result<DateTime<Utc>>;
|
|
/// Existing providers retain their existing point-clock contract. Providers
|
|
/// with an independently verified interval override this atomically.
|
|
fn reading(&self) -> Result<ClockReading> {
|
|
let own_wall = self.now()?;
|
|
Ok(ClockReading {
|
|
own_wall,
|
|
upper: own_wall,
|
|
})
|
|
}
|
|
/// Full trusted UTC interval for crypto validity. Existing point-clock
|
|
/// providers retain their contract; interval providers override atomically.
|
|
fn bounds(&self) -> Result<(DateTime<Utc>, DateTime<Utc>)> {
|
|
let reading = self.reading()?;
|
|
ensure!(reading.own_wall <= reading.upper, "CLOCK_INTERVAL_ORDER");
|
|
Ok((reading.own_wall, reading.upper))
|
|
}
|
|
}
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct Channel {
|
|
pub connection_generation: u64,
|
|
pub boot_id: Uuid,
|
|
}
|
|
#[derive(Clone, Debug)]
|
|
pub struct HelperSnapshot {
|
|
pub journal_id: Uuid,
|
|
pub checkpoint: u64,
|
|
pub previous_decision_id: Option<Uuid>,
|
|
/// Complete consumed/unsettled summaries from the root-owned ledger.
|
|
pub attempts: Vec<SyncAttempt>,
|
|
}
|
|
#[derive(Clone)]
|
|
pub struct PreparedSync {
|
|
pub envelope: Envelope,
|
|
pub request: SyncRequest,
|
|
}
|
|
/// Pass the ORIGINAL signed response to the helper. It must independently verify
|
|
/// it, apply targeted unlocks and commit its checkpoint before acknowledging.
|
|
#[derive(Clone)]
|
|
pub struct HelperInstruction {
|
|
pub response: SyncResponse,
|
|
pub decision_id: Uuid,
|
|
pub journal_id: Uuid,
|
|
pub checkpoint: u64,
|
|
}
|
|
pub struct Applied {
|
|
pub decision_id: Uuid,
|
|
pub acknowledged_message_ids: Vec<Uuid>,
|
|
pub helper_instruction: Option<HelperInstruction>,
|
|
pub reason: String,
|
|
}
|
|
/// Diagnostics only; never accepted as an authorization or reusable lease.
|
|
#[derive(Clone, Copy)]
|
|
pub struct GateObservation {
|
|
pub signed_ready: bool,
|
|
pub local_applied: bool,
|
|
pub helper_ack_pending: bool,
|
|
}
|
|
struct Gate {
|
|
channel: Channel,
|
|
until: DateTime<Utc>,
|
|
ready: bool,
|
|
helper_ready: bool,
|
|
helper: Option<(Uuid, u64, Uuid)>,
|
|
local_applied: bool,
|
|
request: SyncRequest,
|
|
response: SyncResponse,
|
|
}
|
|
pub struct ClientSync<V = DefaultVerifier> {
|
|
binding: Binding,
|
|
verifier: V,
|
|
helper_identity: HelperIdentity,
|
|
pending: Option<PreparedSync>,
|
|
channel: Option<Channel>,
|
|
gate: Option<Gate>,
|
|
}
|
|
impl ClientSync<DefaultVerifier> {
|
|
/// Every new process starts closed, independently of any persisted ready bit.
|
|
pub fn new(binding: Binding) -> Self {
|
|
Self::with_verifier(binding, DefaultVerifier::default())
|
|
}
|
|
}
|
|
impl<V: Ed25519Verifier> ClientSync<V> {
|
|
pub fn with_verifier(binding: Binding, verifier: V) -> Self {
|
|
Self {
|
|
binding,
|
|
verifier,
|
|
helper_identity: HelperIdentity::Root,
|
|
pending: None,
|
|
channel: None,
|
|
gate: None,
|
|
}
|
|
}
|
|
/// Configure one explicit helper identity; changing it discards all leases.
|
|
pub fn with_helper_identity(mut self, identity: HelperIdentity) -> Self {
|
|
self.close();
|
|
self.helper_identity = identity;
|
|
self
|
|
}
|
|
/// Invoke on every authenticated/fenced ServerHello or transport loss.
|
|
pub fn close(&mut self) {
|
|
self.gate = None;
|
|
self.channel = None;
|
|
self.pending = None;
|
|
}
|
|
pub fn gate_observation(&self) -> Option<GateObservation> {
|
|
self.gate.as_ref().map(|g| GateObservation {
|
|
signed_ready: g.ready,
|
|
local_applied: g.local_applied,
|
|
helper_ack_pending: g.helper.is_some(),
|
|
})
|
|
}
|
|
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
|
|
self.gate.as_ref().map(|gate| gate.until)
|
|
}
|
|
pub fn can_start(&self, channel: &Channel, root: bool, clock: &impl SyncClock) -> bool {
|
|
let Ok(reading) = clock.reading() else {
|
|
return false;
|
|
};
|
|
self.gate.as_ref().is_some_and(|g| {
|
|
g.channel == *channel
|
|
&& g.ready
|
|
&& g.local_applied
|
|
&& reading.own_wall < g.until
|
|
&& reading.upper < g.until
|
|
&& (!root || (g.helper_ready && g.helper.is_none()))
|
|
})
|
|
}
|
|
/// Only call with a fresh independently authenticated helper snapshot. There
|
|
/// is no unsigned ready/unlock flag: the exact signed decision/checkpoint
|
|
/// must already be durably applied by that helper.
|
|
pub fn confirm_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
|
|
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
|
|
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
|
|
ensure!(
|
|
snapshot.journal_id == expected.0
|
|
&& snapshot.checkpoint == expected.1
|
|
&& snapshot.previous_decision_id == Some(expected.2),
|
|
"HELPER_CHECKPOINT_NOT_APPLIED"
|
|
);
|
|
ensure!(
|
|
snapshot
|
|
.attempts
|
|
.iter()
|
|
.all(|a| terminal(&a.state)
|
|
|| (a.consumption == "unconsumed" && a.state == "accepted")),
|
|
"HELPER_EFFECT_UNRESOLVED"
|
|
);
|
|
gate.helper = None;
|
|
Ok(())
|
|
}
|
|
/// Typed read-only ACK for exactly the own original update handoff. It
|
|
/// clears only the control delivery acknowledgement; both start flags stay
|
|
/// false and can_start remains false for ALL jobs, including this attempt.
|
|
pub fn confirm_update_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
|
|
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
|
|
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
|
|
ensure!(
|
|
!gate.ready
|
|
&& !gate.helper_ready
|
|
&& snapshot.journal_id == expected.0
|
|
&& snapshot.checkpoint == expected.1
|
|
&& snapshot.previous_decision_id == Some(expected.2),
|
|
"UPDATE_CONTROL_NOT_APPLIED"
|
|
);
|
|
let updates: Vec<_> = snapshot
|
|
.attempts
|
|
.iter()
|
|
.filter(|a| !a.update_handoff_bytes.is_empty())
|
|
.collect();
|
|
ensure!(updates.len() == 1, "UPDATE_CONTROL_EXACTLY_ONE_REQUIRED");
|
|
let local = updates[0];
|
|
let requested = gate
|
|
.request
|
|
.attempts
|
|
.iter()
|
|
.find(|a| a.job_id == local.job_id && a.attempt_id == local.attempt_id)
|
|
.context("UPDATE_CONTROL_NOT_REQUESTED")?;
|
|
ensure!(
|
|
local.state == "running"
|
|
&& local.consumption == "consumed"
|
|
&& local.outcome_bytes.is_empty()
|
|
&& local.update_handoff_bytes == requested.update_handoff_bytes
|
|
&& local.grant_id == requested.grant_id
|
|
&& local.job_payload_hash == requested.job_payload_hash,
|
|
"UPDATE_CONTROL_CHANGED"
|
|
);
|
|
let decision: ReconcileDecisionPayload =
|
|
serde_json::from_slice(&gate.response.decision_bytes)?;
|
|
ensure!(
|
|
decision
|
|
.attempts
|
|
.iter()
|
|
.any(|d| d.job_id.to_string() == local.job_id
|
|
&& d.attempt_id.to_string() == local.attempt_id
|
|
&& d.update_inflight.is_some()
|
|
&& !d.unlock),
|
|
"UPDATE_CONTROL_NOT_SIGNED"
|
|
);
|
|
ensure!(
|
|
snapshot
|
|
.attempts
|
|
.iter()
|
|
.filter(|a| a.job_id != local.job_id || a.attempt_id != local.attempt_id)
|
|
.all(|a| terminal(&a.state)
|
|
|| (a.state == "accepted" && a.consumption == "unconsumed")),
|
|
"UPDATE_OTHER_EFFECT_UNRESOLVED"
|
|
);
|
|
gate.helper = None;
|
|
Ok(())
|
|
}
|
|
/// Historical authentication only. Never apply an old response or create a
|
|
/// lease: the returned bytes prove a previously durable local control step.
|
|
fn stored_control(
|
|
&self,
|
|
db: &Connection,
|
|
ledger: &Connection,
|
|
id: Uuid,
|
|
boot: Uuid,
|
|
) -> Result<(SyncRequest, ReconcileDecisionPayload)> {
|
|
let ack: Vec<u8> = ledger.query_row(
|
|
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
|
|
[id.to_string()],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
ack.len() <= rmm_domain::MAX_MESSAGE,
|
|
"CLOSED_CONTROL_ACK_BOUND"
|
|
);
|
|
let response = SyncResponse::decode(ack.as_slice())?;
|
|
ensure!(
|
|
response.encode_to_vec() == ack,
|
|
"CLOSED_CONTROL_CANONICAL_ACK"
|
|
);
|
|
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
|
let (raw, envelope, generation, saved): (Vec<u8>, Vec<u8>, u64, Vec<u8>) = db.query_row(
|
|
"SELECT request,envelope,generation,response FROM reconcile_requests WHERE id=?1",
|
|
[parsed.request_id.to_string()],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
|
)?;
|
|
ensure!(
|
|
raw.len() <= rmm_domain::MAX_MESSAGE
|
|
&& envelope.len() <= rmm_domain::MAX_MESSAGE
|
|
&& saved == ack,
|
|
"CLOSED_CONTROL_ORIGINAL_BYTES"
|
|
);
|
|
let request = SyncRequest::decode(raw.as_slice())?;
|
|
let env = Envelope::decode(envelope.as_slice())?;
|
|
ensure!(
|
|
raw == request.encode_to_vec()
|
|
&& env.encode_to_vec() == envelope
|
|
&& env.payload == Some(Payload::SyncRequest(request.clone()))
|
|
&& rmm_protocol::valid_envelope(
|
|
&env,
|
|
&self.binding.agent_id.to_string(),
|
|
generation
|
|
)
|
|
&& env.boot_id == boot.to_string()
|
|
&& env.sequence == request.checkpoint,
|
|
"CLOSED_CONTROL_ORIGINAL_ENVELOPE"
|
|
);
|
|
let decision = verify_response_with(
|
|
&self.verifier,
|
|
&self.binding,
|
|
&request,
|
|
&Channel {
|
|
connection_generation: generation,
|
|
boot_id: boot,
|
|
},
|
|
&response,
|
|
parsed.issued_at,
|
|
)?;
|
|
ensure!(
|
|
decision.decision_id == id
|
|
&& decision.boot_id == boot
|
|
&& !recovery_reason(&decision.reason),
|
|
"CLOSED_CONTROL_SIGNED_ORIGINAL"
|
|
);
|
|
Ok((request, decision))
|
|
}
|
|
/// Admit only a new, ordinary online reconciliation of an exact missing
|
|
/// helper ACK with already definitive, identical outcomes. The caller owns
|
|
/// agent.lock, has no live executor/deferred work, and obtained `helper`
|
|
/// from the serialized, independently authenticated current-boot helper.
|
|
/// This predicate is read-only; fresh server/own-clock/helper ACK gates are
|
|
/// unchanged and no flag or authority from either old response is reused.
|
|
fn closed_control_continuation(
|
|
&self,
|
|
db: &Connection,
|
|
ledger: &Connection,
|
|
channel: &Channel,
|
|
helper: &HelperSnapshot,
|
|
meta: &Meta,
|
|
now: DateTime<Utc>,
|
|
) -> Result<()> {
|
|
ensure!(
|
|
self.helper_identity == HelperIdentity::System
|
|
&& self.pending.is_none()
|
|
&& self.gate.is_none()
|
|
&& meta.fault.is_none()
|
|
&& meta.pending_request.is_none(),
|
|
"CLOSED_CONTROL_IDLE_ONLY"
|
|
);
|
|
let a = meta
|
|
.decision
|
|
.context("CLOSED_CONTROL_AGENT_HEAD_REQUIRED")?;
|
|
let h = helper
|
|
.previous_decision_id
|
|
.context("CLOSED_CONTROL_HELPER_HEAD_REQUIRED")?;
|
|
ensure!(
|
|
a != h
|
|
&& Some(helper.journal_id) == meta.helper_id
|
|
&& helper.checkpoint >= meta.helper_checkpoint
|
|
&& pending_ids(db)?.is_empty(),
|
|
"CLOSED_CONTROL_CURRENT_HEADS"
|
|
);
|
|
let legacy: bool = db.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='events')",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
!legacy
|
|
|| db.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, u64>(0))? == 0,
|
|
"CLOSED_CONTROL_NO_PENDING_EVENTS"
|
|
);
|
|
let (original, decision) = self.stored_control(db, ledger, a, channel.boot_id)?;
|
|
let (_, previous) = self.stored_control(db, ledger, h, channel.boot_id)?;
|
|
ensure!(
|
|
decision.journal_id == meta.journal_id
|
|
&& decision.checkpoint == u64::try_from(meta.checkpoint)?
|
|
&& original.journal_id == meta.journal_id.to_string()
|
|
&& original.checkpoint == decision.checkpoint
|
|
&& original.previous_decision_id == h.to_string()
|
|
&& original.helper_previous_decision_id == h.to_string()
|
|
&& original.previous_request_id.is_empty()
|
|
&& original.journal_complete
|
|
&& decision.issued_at <= now
|
|
&& decision.connection_generation <= channel.connection_generation
|
|
&& previous.connection_generation <= decision.connection_generation
|
|
&& previous.checkpoint < decision.checkpoint
|
|
&& previous.journal_id == decision.journal_id
|
|
&& previous.helper_journal_id == Some(helper.journal_id)
|
|
&& decision.helper_journal_id == Some(helper.journal_id)
|
|
&& original.helper_journal_id == helper.journal_id.to_string()
|
|
&& previous.helper_checkpoint < decision.helper_checkpoint
|
|
&& decision.helper_checkpoint == meta.helper_checkpoint
|
|
&& helper.checkpoint >= decision.helper_checkpoint,
|
|
"CLOSED_CONTROL_EXACT_PARTIAL_EDGE"
|
|
);
|
|
verify_closed_result_messages(db, &original, &decision)?;
|
|
let current = attempts(db, ledger, &self.binding)?;
|
|
ensure!(
|
|
current.len() == original.attempts.len()
|
|
&& current.len() == decision.attempts.len()
|
|
&& current.len() <= 256,
|
|
"CLOSED_CONTROL_EXACT_ATTEMPT_SET"
|
|
);
|
|
let expected_consumed = current
|
|
.iter()
|
|
.filter(|r| r.consumption == "consumed")
|
|
.count() as u64;
|
|
let ledger_counts: (u64, u64, u64) = ledger.query_row(
|
|
"SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM execution_evidence),(SELECT count(*) FROM consumed WHERE result IS NULL)",
|
|
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
|
|
ensure!(
|
|
ledger_counts == (expected_consumed, expected_consumed, 0),
|
|
"CLOSED_CONTROL_EXACT_COMPLETE_LEDGER"
|
|
);
|
|
for row in ¤t {
|
|
ensure!(
|
|
matches!(row.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired")
|
|
&& row.update_handoff_bytes.is_empty(),
|
|
"CLOSED_CONTROL_DEFINITIVE_ONLY"
|
|
);
|
|
let old = original
|
|
.attempts
|
|
.iter()
|
|
.find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id)
|
|
.context("CLOSED_CONTROL_ORIGINAL_ATTEMPT")?;
|
|
let signed = decision
|
|
.attempts
|
|
.iter()
|
|
.find(|r| {
|
|
r.job_id.to_string() == row.job_id && r.attempt_id.to_string() == row.attempt_id
|
|
})
|
|
.context("CLOSED_CONTROL_SIGNED_ATTEMPT")?;
|
|
ensure!(
|
|
old.state == row.state
|
|
&& old.job_payload_hash == row.job_payload_hash
|
|
&& old.grant_id == row.grant_id
|
|
&& old.consumption == row.consumption
|
|
&& old.original_boot_id == row.original_boot_id
|
|
&& old.outcome_bytes == row.outcome_bytes
|
|
&& old.outcome_sha256 == row.outcome_sha256
|
|
&& old.update_handoff_bytes.is_empty()
|
|
&& row.revision == old.revision.max(signed.revision)
|
|
&& signed.outcome_state == row.state
|
|
&& signed.update_inflight.is_none(),
|
|
"CLOSED_CONTROL_RESULT_UNCHANGED"
|
|
);
|
|
// An ungranted expiry has no fabricated outcome. Its exact
|
|
// definitive nonstart is already authenticated in the original
|
|
// request/decision; it can only permit a NEW online request.
|
|
if row.consumption == "unconsumed" {
|
|
let consumed: u64 = ledger.query_row(
|
|
"SELECT count(*) FROM consumed WHERE job=?1 AND attempt=?2",
|
|
params![row.job_id, row.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
matches!(row.state.as_str(), "expired" | "cancelled")
|
|
&& row.grant_id.is_empty()
|
|
&& row.original_boot_id.is_empty()
|
|
&& row.outcome_bytes.is_empty()
|
|
&& row.outcome_sha256.is_empty()
|
|
&& signed.grant_id.is_none()
|
|
&& !signed.unlock
|
|
&& signed.reason == "DEFINITIVE_NONSTART"
|
|
&& signed.previous_outcome_sha256.is_none()
|
|
&& signed.outcome_sha256.is_none()
|
|
&& signed.outcome_bytes.is_empty()
|
|
&& consumed == 0
|
|
&& !helper
|
|
.attempts
|
|
.iter()
|
|
.any(|h| h.job_id == row.job_id && h.attempt_id == row.attempt_id),
|
|
"CLOSED_CONTROL_SIGNED_UNGRANTED_NONSTART"
|
|
);
|
|
continue;
|
|
}
|
|
ensure!(
|
|
!row.outcome_bytes.is_empty()
|
|
&& signed.grant_id.map(|id| id.to_string()).as_deref()
|
|
== Some(row.grant_id.as_str())
|
|
&& hash(&row.outcome_bytes) == row.outcome_sha256
|
|
&& signed.outcome_sha256.as_deref() == Some(row.outcome_sha256.as_str())
|
|
&& (signed.outcome_bytes.is_empty()
|
|
|| signed.outcome_bytes == row.outcome_bytes),
|
|
"CLOSED_CONTROL_SIGNED_CANONICAL_RESULT"
|
|
);
|
|
let outcome: ReconciledOutcome = serde_json::from_slice(&row.outcome_bytes)?;
|
|
ensure!(
|
|
canonical_outcome_bytes(&outcome)? == row.outcome_bytes
|
|
&& outcome.state == row.state,
|
|
"CLOSED_CONTROL_CANONICAL_RESULT"
|
|
);
|
|
match row.consumption.as_str() {
|
|
"consumed" => {
|
|
let (start,finished,last):(Vec<u8>,Vec<u8>,Vec<u8>)=ledger.query_row("SELECT e.start,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1 AND c.job=?2 AND c.attempt=?3 AND c.agent=?4 AND c.generation=?5",params![row.grant_id,row.job_id,row.attempt_id,self.binding.agent_id.to_string(),self.binding.enrollment_generation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
|
|
let saved: serde_json::Value = serde_json::from_slice(&start)?;
|
|
let request: crate::auth::Request =
|
|
serde_json::from_value(saved["request"].clone())?;
|
|
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
|
serde_json::from_slice(&request.permit_bytes)?;
|
|
let (job, permit) =
|
|
self.binding
|
|
.verify_with(&request, permit.not_before, &self.verifier)?;
|
|
ensure!(
|
|
job.job_id.to_string() == row.job_id
|
|
&& job.attempt_id.to_string() == row.attempt_id
|
|
&& hash(&request.job_bytes) == row.job_payload_hash
|
|
&& permit.grant_id.to_string() == row.grant_id
|
|
&& !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. })
|
|
&& canonical_outcome_bytes(&serde_json::from_slice::<
|
|
ReconciledOutcome,
|
|
>(&finished)?)?
|
|
== row.outcome_bytes,
|
|
"CLOSED_CONTROL_ORIGINAL_CONSUMPTION"
|
|
);
|
|
if crate::windows_system_nonstart::effective_outcome(
|
|
ledger,
|
|
Some(&self.binding),
|
|
permit.grant_id,
|
|
)?
|
|
.is_none()
|
|
{
|
|
ensure!(
|
|
canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
|
|
&last
|
|
)?)? == row.outcome_bytes,
|
|
"CLOSED_CONTROL_ORIGINAL_RESULT_CONFLICT"
|
|
);
|
|
}
|
|
if job.execution_identity == self.helper_identity.as_str() {
|
|
let r = helper
|
|
.attempts
|
|
.iter()
|
|
.find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id)
|
|
.context("CLOSED_CONTROL_HELPER_ATTEMPT_REQUIRED")?;
|
|
ensure!(
|
|
r.state == row.state
|
|
&& r.grant_id == row.grant_id
|
|
&& r.job_payload_hash == row.job_payload_hash
|
|
&& r.consumption == "consumed"
|
|
&& r.outcome_bytes == row.outcome_bytes
|
|
&& r.outcome_sha256 == row.outcome_sha256
|
|
&& r.original_boot_id == row.original_boot_id
|
|
&& r.revision <= row.revision
|
|
&& r.update_handoff_bytes.is_empty(),
|
|
"CLOSED_CONTROL_HELPER_RESULT_UNCHANGED"
|
|
);
|
|
}
|
|
}
|
|
_ => anyhow::bail!("CLOSED_CONTROL_UNCERTAIN_CONSUMPTION"),
|
|
}
|
|
}
|
|
for row in &helper.attempts {
|
|
ensure!(
|
|
terminal(&row.state)
|
|
&& row.update_handoff_bytes.is_empty()
|
|
&& current.iter().any(|r| r.job_id == row.job_id
|
|
&& r.attempt_id == row.attempt_id
|
|
&& r.grant_id == row.grant_id
|
|
&& r.outcome_bytes == row.outcome_bytes),
|
|
"CLOSED_CONTROL_NO_EXTRA_HELPER_ATTEMPT"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
pub fn prepare(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
ledger: &Connection,
|
|
channel: Channel,
|
|
helper: Option<&HelperSnapshot>,
|
|
clock: &impl SyncClock,
|
|
) -> Result<PreparedSync> {
|
|
self.prepare_with_closed_control_idle(db, ledger, channel, helper, clock, false)
|
|
}
|
|
/// The OS actor may attest quiescence only after its execution/deferred
|
|
/// workers and protected intent store are all settled. Ordinary same-head
|
|
/// synchronization does not need this attestation, including while busy.
|
|
pub fn prepare_with_closed_control_idle(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
ledger: &Connection,
|
|
channel: Channel,
|
|
helper: Option<&HelperSnapshot>,
|
|
clock: &impl SyncClock,
|
|
closed_control_idle: bool,
|
|
) -> Result<PreparedSync> {
|
|
let result = self.prepare_inner(db, ledger, channel, helper, clock, closed_control_idle);
|
|
if let Err(error) = &result {
|
|
let text = error.to_string();
|
|
if [
|
|
"ROLLBACK",
|
|
"REGRESSION",
|
|
"DISAPPEARED",
|
|
"MISSING_FROM_AGENT_JOURNAL",
|
|
"MISSING_FROM_HELPER_JOURNAL",
|
|
]
|
|
.iter()
|
|
.any(|s| text.contains(s))
|
|
{
|
|
let _ = db.execute(
|
|
"UPDATE reconcile_meta SET fault=COALESCE(fault,?1) WHERE id=1",
|
|
[text],
|
|
);
|
|
}
|
|
}
|
|
result
|
|
}
|
|
fn prepare_inner(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
ledger: &Connection,
|
|
channel: Channel,
|
|
helper: Option<&HelperSnapshot>,
|
|
clock: &impl SyncClock,
|
|
closed_control_idle: bool,
|
|
) -> Result<PreparedSync> {
|
|
self.close();
|
|
let now = clock.now()?;
|
|
ensure!(
|
|
channel.connection_generation > 0 && !channel.boot_id.is_nil(),
|
|
"UNAUTHENTICATED_CHANNEL"
|
|
);
|
|
initialize(db)?;
|
|
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
|
let meta = metadata(&tx)?;
|
|
ensure!(meta.fault.is_none(), "SYNC_MANUAL_RECOVERY_REQUIRED");
|
|
let sequence: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
|
|
ensure!(
|
|
sequence >= meta.checkpoint && sequence >= meta.high_sequence,
|
|
"LOCAL_CHECKPOINT_ROLLBACK"
|
|
);
|
|
ensure!(
|
|
channel.connection_generation >= meta.generation,
|
|
"CONNECTION_GENERATION_REGRESSION"
|
|
);
|
|
if let Some(h) = helper {
|
|
ensure!(
|
|
!h.journal_id.is_nil() && h.checkpoint <= i64::MAX as u64,
|
|
"HELPER_CHECKPOINT"
|
|
);
|
|
if let Some(old) = meta.helper_id {
|
|
ensure!(
|
|
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
|
|
"HELPER_JOURNAL_ROLLBACK"
|
|
);
|
|
if self.helper_identity == HelperIdentity::System {
|
|
// A locally committed Windows decision is not proof of its
|
|
// application by SYSTEM. Never advance either checkpoint
|
|
// from an older helper head, including after process loss.
|
|
if meta.decision.is_none() || h.previous_decision_id != meta.decision {
|
|
ensure!(closed_control_idle, "SYSTEM_HELPER_APPLY_UNCONFIRMED");
|
|
self.closed_control_continuation(&tx, ledger, &channel, h, &meta, now)
|
|
.context("SYSTEM_HELPER_APPLY_UNCONFIRMED")?;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
|
|
}
|
|
if let Some(h) = helper {
|
|
ingest_helper(
|
|
&tx,
|
|
ledger,
|
|
(&self.binding, &self.verifier),
|
|
&channel,
|
|
h,
|
|
now,
|
|
self.helper_identity,
|
|
)?;
|
|
}
|
|
archive_events(&tx)?;
|
|
tx.execute(
|
|
"UPDATE job_attempts SET state='accepted' WHERE state='queued' AND grant_id=''",
|
|
[],
|
|
)?;
|
|
// A possibly started attempt is NEVER returned to accepted. Preserve
|
|
// revisions and results; only ambiguous no-result states become unknown.
|
|
let inflight: Vec<_> = helper
|
|
.into_iter()
|
|
.flat_map(|h| h.attempts.iter())
|
|
.filter(|a| !a.update_handoff_bytes.is_empty())
|
|
.collect();
|
|
ensure!(inflight.len() <= 1, "UPDATE_MULTIPLE_INFLIGHT");
|
|
if let Some(update) = inflight.first() {
|
|
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT (job=?1 AND attempt=?2) AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",params![update.job_id,update.attempt_id])?;
|
|
} else {
|
|
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?;
|
|
}
|
|
let mut attempts = attempts(&tx, ledger, &self.binding)?;
|
|
if let Some(h) = helper {
|
|
merge_helper(&mut attempts, h)?;
|
|
}
|
|
ensure!(attempts.len() <= 256, "SYNC_ATTEMPT_CAPACITY_NO_COMPACTION");
|
|
let mut ids = pending_ids(&tx)?;
|
|
if let Some(update) = inflight.first() {
|
|
let grant = Uuid::parse_str(&update.grant_id)?;
|
|
ids.extend(crate::update_health_inventory::pending_ids(&tx, grant)?);
|
|
ids.sort();
|
|
ids.dedup();
|
|
}
|
|
ensure!(ids.len() <= 256, "SYNC_RECEIPT_CAPACITY");
|
|
let next: i64 = tx.query_row(
|
|
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
let request = SyncRequest {
|
|
request_id: Uuid::new_v4().to_string(),
|
|
journal_id: meta.journal_id.to_string(),
|
|
checkpoint: u64::try_from(next)?,
|
|
// A pending request may have committed with a lost reply, or may
|
|
// never have reached the server. Preserve both witnesses: the
|
|
// server resolves the pending request first and otherwise checks
|
|
// this last applied decision against its authoritative head.
|
|
previous_decision_id: meta.decision.map(|id| id.to_string()).unwrap_or_default(),
|
|
assignment_version: self.binding.assignment_version,
|
|
attempts,
|
|
durable_message_ids: ids,
|
|
journal_complete: true,
|
|
helper_journal_id: helper.map(|h| h.journal_id.to_string()).unwrap_or_default(),
|
|
helper_checkpoint: helper.map_or(0, |h| h.checkpoint),
|
|
helper_previous_decision_id: helper
|
|
.and_then(|h| h.previous_decision_id)
|
|
.map(|id| id.to_string())
|
|
.unwrap_or_default(),
|
|
previous_request_id: meta
|
|
.pending_request
|
|
.map(|id| id.to_string())
|
|
.unwrap_or_default(),
|
|
};
|
|
let envelope = Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: self.binding.agent_id.to_string(),
|
|
enrollment_generation: self.binding.enrollment_generation,
|
|
connection_generation: channel.connection_generation,
|
|
boot_id: channel.boot_id.to_string(),
|
|
sequence: u64::try_from(next)?,
|
|
observed_at: now.timestamp_millis(),
|
|
payload: Some(Payload::SyncRequest(request.clone())),
|
|
};
|
|
ensure!(
|
|
envelope.encoded_len() <= rmm_domain::MAX_MESSAGE,
|
|
"SYNC_ENVELOPE_SIZE"
|
|
);
|
|
tx.execute(
|
|
"INSERT INTO reconcile_requests(id,request,envelope,generation) VALUES(?1,?2,?3,?4)",
|
|
params![
|
|
request.request_id,
|
|
request.encode_to_vec(),
|
|
envelope.encode_to_vec(),
|
|
channel.connection_generation
|
|
],
|
|
)?;
|
|
tx.execute("UPDATE reconcile_meta SET checkpoint=?1,high_sequence=?1,generation=?2,pending_request=?3 WHERE id=1",params![next,channel.connection_generation,request.request_id])?;
|
|
tx.commit()?;
|
|
let prepared = PreparedSync { envelope, request };
|
|
self.pending = Some(prepared.clone());
|
|
self.channel = Some(channel);
|
|
Ok(prepared)
|
|
}
|
|
/// All changes, concrete ACKs, result archives and checkpoint chain commit
|
|
/// together. A rejection closes only mutation authority; telemetry may run.
|
|
pub fn apply(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
response: &SyncResponse,
|
|
clock: &impl SyncClock,
|
|
) -> Result<Applied> {
|
|
if self.pending.is_none() {
|
|
let result = self.replay_applied(db, response, clock);
|
|
if result.is_err() {
|
|
self.gate = None;
|
|
}
|
|
return result;
|
|
}
|
|
self.gate = None;
|
|
let result = self.apply_inner(db, response, clock);
|
|
if result.is_err() {
|
|
self.gate = None;
|
|
}
|
|
result
|
|
}
|
|
fn replay_applied(
|
|
&self,
|
|
db: &Connection,
|
|
response: &SyncResponse,
|
|
clock: &impl SyncClock,
|
|
) -> Result<Applied> {
|
|
let gate = self.gate.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?;
|
|
ensure!(gate.response == *response, "REPLAYED_DIFFERENT_DECISION");
|
|
let d = verify_response_with(
|
|
&self.verifier,
|
|
&self.binding,
|
|
&gate.request,
|
|
&gate.channel,
|
|
response,
|
|
clock.now()?,
|
|
)?;
|
|
let meta = metadata(db)?;
|
|
let sequence: i64 = db.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
|
|
ensure!(
|
|
meta.decision == Some(d.decision_id)
|
|
&& meta.pending_request.is_none()
|
|
&& meta.checkpoint == i64::try_from(d.checkpoint)?
|
|
&& sequence >= meta.high_sequence,
|
|
"APPLIED_CHECKPOINT_REGRESSION"
|
|
);
|
|
Ok(Applied {
|
|
decision_id: d.decision_id,
|
|
acknowledged_message_ids: d.acknowledged_message_ids,
|
|
helper_instruction: d.helper_journal_id.map(|journal_id| HelperInstruction {
|
|
response: response.clone(),
|
|
decision_id: d.decision_id,
|
|
journal_id,
|
|
checkpoint: d.helper_checkpoint,
|
|
}),
|
|
reason: d.reason,
|
|
})
|
|
}
|
|
fn apply_inner(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
response: &SyncResponse,
|
|
clock: &impl SyncClock,
|
|
) -> Result<Applied> {
|
|
let now = clock.now()?;
|
|
let prepared = self.pending.as_ref().context("NO_CURRENT_SYNC_CHALLENGE")?;
|
|
let channel = self.channel.as_ref().context("NO_AUTHENTICATED_CHANNEL")?;
|
|
let decision = verify_response_with(
|
|
&self.verifier,
|
|
&self.binding,
|
|
&prepared.request,
|
|
channel,
|
|
response,
|
|
now,
|
|
)?;
|
|
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
|
let meta = metadata(&tx)?;
|
|
ensure!(
|
|
meta.fault.is_none()
|
|
&& meta.pending_request == Some(decision.request_id)
|
|
&& meta.checkpoint == i64::try_from(decision.checkpoint)?
|
|
&& meta.generation == channel.connection_generation,
|
|
"SYNC_CHECKPOINT_CHANGED"
|
|
);
|
|
let seq: i64 = tx.query_row("SELECT seq FROM counter WHERE id=1", [], |r| r.get(0))?;
|
|
ensure!(
|
|
seq >= meta.high_sequence && seq >= meta.checkpoint,
|
|
"LOCAL_SEQUENCE_ROLLBACK"
|
|
);
|
|
let current = tx.query_row(
|
|
"SELECT request FROM reconcile_requests WHERE id=?1",
|
|
[decision.request_id.to_string()],
|
|
|r| r.get::<_, Vec<u8>>(0),
|
|
)?;
|
|
ensure!(
|
|
current == prepared.request.encode_to_vec(),
|
|
"DURABLE_CHALLENGE_CHANGED"
|
|
);
|
|
let summaries: BTreeMap<_, _> = prepared
|
|
.request
|
|
.attempts
|
|
.iter()
|
|
.map(|a| ((a.job_id.clone(), a.attempt_id.clone()), a))
|
|
.collect();
|
|
let actual: BTreeSet<_> = decision
|
|
.attempts
|
|
.iter()
|
|
.map(|a| (a.job_id.to_string(), a.attempt_id.to_string()))
|
|
.collect();
|
|
ensure!(
|
|
actual.len() == decision.attempts.len()
|
|
&& actual == summaries.keys().cloned().collect(),
|
|
"SYNC_ATTEMPT_OMITTED_OR_ADDED"
|
|
);
|
|
let mut safe = true;
|
|
let mut fault = recovery_reason(&decision.reason).then(|| decision.reason.clone());
|
|
for d in &decision.attempts {
|
|
let local = summaries
|
|
.get(&(d.job_id.to_string(), d.attempt_id.to_string()))
|
|
.context("LOCAL_ATTEMPT_MISSING")?;
|
|
if recovery_reason(&d.reason) {
|
|
fault = Some(d.reason.clone());
|
|
}
|
|
let offer_bytes: Vec<u8> = tx.query_row(
|
|
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![local.job_id, local.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
let offer = JobOffer::decode(offer_bytes.as_slice())?;
|
|
let job = self.binding.job(&offer.payload_bytes)?;
|
|
// Missing-server decisions intentionally have empty action fields;
|
|
// persist their sticky denial without treating it as start authority.
|
|
if recovery_reason(&d.reason) {
|
|
safe = false;
|
|
continue;
|
|
}
|
|
let outcome =
|
|
validate_attempt(&job, &offer, local, d, channel.boot_id, &self.binding, now)?;
|
|
safe &= (d.outcome_state == "accepted"
|
|
&& local.consumption == "unconsumed"
|
|
&& local.grant_id.is_empty())
|
|
|| (terminal(&d.outcome_state) && (local.consumption == "unconsumed" || d.unlock));
|
|
let (stored_state, revision): (String, u64) = tx.query_row(
|
|
"SELECT state,revision FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![local.job_id, local.attempt_id],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)?;
|
|
ensure!(
|
|
revision == local.revision && stored_state == local.state,
|
|
"LOCAL_ATTEMPT_CHANGED_DURING_SYNC"
|
|
);
|
|
let stored = if d.outcome_state == "unknown" && local.state == "awaiting_reboot" {
|
|
"awaiting_reboot"
|
|
} else {
|
|
d.outcome_state.as_str()
|
|
};
|
|
tx.execute(
|
|
"UPDATE job_attempts SET state=?3,revision=?4 WHERE job=?1 AND attempt=?2",
|
|
params![
|
|
local.job_id,
|
|
local.attempt_id,
|
|
stored,
|
|
d.revision.max(local.revision)
|
|
],
|
|
)?;
|
|
if let Some(bytes) = outcome {
|
|
tx.execute("INSERT INTO reconcile_outcomes(job,attempt,outcome,decision_id) VALUES(?1,?2,?3,?4) ON CONFLICT(job,attempt) DO UPDATE SET outcome=excluded.outcome,decision_id=excluded.decision_id",params![local.job_id,local.attempt_id,bytes,decision.decision_id.to_string()])?;
|
|
}
|
|
}
|
|
let requested: BTreeSet<_> = prepared
|
|
.request
|
|
.durable_message_ids
|
|
.iter()
|
|
.map(String::as_str)
|
|
.collect();
|
|
let mut receipts = BTreeSet::new();
|
|
for id in &decision.acknowledged_message_ids {
|
|
ensure!(
|
|
receipts.insert(*id) && requested.contains(id.to_string().as_str()),
|
|
"ACK_NOT_IN_REQUEST"
|
|
);
|
|
archive_events(&tx)?;
|
|
tx.execute("DELETE FROM job_events WHERE id=?1", [id.to_string()])?;
|
|
tx.execute(
|
|
"INSERT OR IGNORE INTO reconcile_receipts(id,decision_id) VALUES(?1,?2)",
|
|
params![id.to_string(), decision.decision_id.to_string()],
|
|
)?;
|
|
}
|
|
tx.execute(
|
|
"UPDATE reconcile_requests SET response=?2 WHERE id=?1 AND response IS NULL",
|
|
params![decision.request_id.to_string(), response.encode_to_vec()],
|
|
)?;
|
|
for local in prepared
|
|
.request
|
|
.attempts
|
|
.iter()
|
|
.filter(|a| !a.update_handoff_bytes.is_empty())
|
|
{
|
|
crate::update_health_inventory::acknowledge_signed(
|
|
&tx,
|
|
Uuid::parse_str(&local.grant_id)?,
|
|
&decision,
|
|
response,
|
|
)?;
|
|
}
|
|
tx.execute("UPDATE reconcile_meta SET last_decision=?1,pending_request=NULL,helper_id=?2,helper_checkpoint=?3,fault=COALESCE(fault,?4),high_sequence=?5 WHERE id=1",
|
|
params![decision.decision_id.to_string(),decision.helper_journal_id.map(|id|id.to_string()),decision.helper_checkpoint,fault,seq])?;
|
|
tx.commit()?;
|
|
let helper_instruction = decision
|
|
.helper_journal_id
|
|
.map(|journal_id| HelperInstruction {
|
|
response: response.clone(),
|
|
decision_id: decision.decision_id,
|
|
journal_id,
|
|
checkpoint: decision.helper_checkpoint,
|
|
});
|
|
self.gate = Some(Gate {
|
|
channel: channel.clone(),
|
|
until: decision.valid_until,
|
|
ready: decision.ready && safe && fault.is_none(),
|
|
helper_ready: decision.helper_ready,
|
|
helper: helper_instruction
|
|
.as_ref()
|
|
.map(|h| (h.journal_id, h.checkpoint, h.decision_id)),
|
|
local_applied: !decision.attempts.iter().any(|d| d.unlock),
|
|
request: prepared.request.clone(),
|
|
response: response.clone(),
|
|
});
|
|
self.pending = None;
|
|
Ok(Applied {
|
|
decision_id: decision.decision_id,
|
|
acknowledged_message_ids: decision.acknowledged_message_ids,
|
|
helper_instruction,
|
|
reason: decision.reason,
|
|
})
|
|
}
|
|
/// Apply ONLY this process's independently owned consumption ledger. Root
|
|
/// helper unlocks use HelperInstruction over authenticated IPC instead.
|
|
/// A crash between journal and ledger transactions leaves the gate closed;
|
|
/// resync/reapply never deletes the consumed attempt or repeats an effect.
|
|
pub fn apply_consumption(
|
|
&mut self,
|
|
ledger: &mut Connection,
|
|
clock: &impl SyncClock,
|
|
) -> Result<()> {
|
|
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
|
|
gate.local_applied = false;
|
|
let decision = verify_response_with(
|
|
&self.verifier,
|
|
&self.binding,
|
|
&gate.request,
|
|
&gate.channel,
|
|
&gate.response,
|
|
clock.now()?,
|
|
)?;
|
|
let tx = ledger.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
|
tx.execute_batch("CREATE TABLE IF NOT EXISTS reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB NOT NULL);")?;
|
|
let applied: Option<Vec<u8>> = tx
|
|
.query_row(
|
|
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
|
|
[decision.decision_id.to_string()],
|
|
|r| r.get(0),
|
|
)
|
|
.optional()?;
|
|
if let Some(old) = applied {
|
|
ensure!(
|
|
old == gate.response.encode_to_vec(),
|
|
"CONSUMPTION_DECISION_CONFLICT"
|
|
);
|
|
} else {
|
|
for d in decision.attempts.iter().filter(|d| d.unlock) {
|
|
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
|
|
let (agent,generation,job,attempt,previous,start):(String,u64,String,String,Option<Vec<u8>>,Option<Vec<u8>>)=tx.query_row(
|
|
"SELECT c.agent,c.generation,c.job,c.attempt,COALESCE(e.outcome,c.result),e.start FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
|
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?;
|
|
ensure!(
|
|
agent == self.binding.agent_id.to_string()
|
|
&& generation == self.binding.enrollment_generation
|
|
&& job == d.job_id.to_string()
|
|
&& attempt == d.attempt_id.to_string(),
|
|
"LOCAL_UNLOCK_CONSUMPTION_BINDING"
|
|
);
|
|
let resolved = crate::windows_system_nonstart::effective_outcome(
|
|
&tx,
|
|
Some(&self.binding),
|
|
grant,
|
|
)?;
|
|
let previous = resolved.or(previous);
|
|
let summary = gate
|
|
.request
|
|
.attempts
|
|
.iter()
|
|
.find(|a| a.job_id == job && a.attempt_id == attempt)
|
|
.context("LOCAL_UNLOCK_ATTEMPT_MISSING")?;
|
|
let closure = d.reason == "OPERATOR_CLOSED_UNKNOWN"
|
|
&& d.outcome_state == "closed_unknown"
|
|
&& summary.consumption == "consumed"
|
|
&& uncertain_agent_outcome(
|
|
&summary.state,
|
|
&summary.outcome_bytes,
|
|
&summary.outcome_sha256,
|
|
);
|
|
let prior = match &previous {
|
|
Some(bytes) => {
|
|
let canonical = canonical_outcome_bytes(&serde_json::from_slice(bytes)?)?;
|
|
if closure
|
|
&& summary.state == "closed_unknown"
|
|
&& uncertain_agent_outcome("unknown", &canonical, &hash(&canonical))
|
|
{
|
|
summary.outcome_bytes.clone()
|
|
} else {
|
|
canonical
|
|
}
|
|
}
|
|
None => {
|
|
ensure!(closure, "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING");
|
|
let start: serde_json::Value = serde_json::from_slice(
|
|
&start.context("LOCAL_UNLOCK_ORIGINAL_START_MISSING")?,
|
|
)?;
|
|
let request: crate::auth::Request =
|
|
serde_json::from_value(start["request"].clone())?;
|
|
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
|
serde_json::from_slice(&request.permit_bytes)?;
|
|
let (original, permit) = self.binding.verify_with(
|
|
&request,
|
|
permit.not_before,
|
|
&self.verifier,
|
|
)?;
|
|
ensure!(
|
|
permit.grant_id == grant
|
|
&& original.job_id == d.job_id
|
|
&& original.attempt_id == d.attempt_id
|
|
&& hash(&request.job_bytes) == d.job_payload_hash
|
|
&& Some(permit.authorization_id) == d.authorization_id
|
|
&& Some(permit.plan_hash.as_str()) == d.plan_hash.as_deref()
|
|
&& start["previous_boot_id"].as_str()
|
|
== d.original_boot_id.map(|id| id.to_string()).as_deref(),
|
|
"LOCAL_UNLOCK_ORIGINAL_START_BINDING"
|
|
);
|
|
summary.outcome_bytes.clone()
|
|
}
|
|
};
|
|
ensure!(
|
|
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
|
|
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
|
|
);
|
|
let bytes = if d.outcome_bytes.is_empty() {
|
|
prior
|
|
} else {
|
|
d.outcome_bytes.clone()
|
|
};
|
|
let outcome: ReconciledOutcome = serde_json::from_slice(&bytes)?;
|
|
ensure!(
|
|
terminal(&outcome.state)
|
|
&& outcome.state == d.outcome_state
|
|
&& canonical_outcome_bytes(&outcome)? == bytes
|
|
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
|
|
&& (previous.is_some() || operator_closure(&d.reason, "unknown", &outcome)),
|
|
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
|
|
);
|
|
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
|
|
&tx,
|
|
&self.binding,
|
|
grant,
|
|
&bytes,
|
|
)?;
|
|
if outcome.state == "closed_unknown" {
|
|
crate::ledger::retain_closure_original(
|
|
&tx,
|
|
grant,
|
|
previous.as_deref().unwrap_or_default(),
|
|
&bytes,
|
|
)?;
|
|
}
|
|
tx.execute(
|
|
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
|
|
params![grant.to_string(), bytes],
|
|
)?;
|
|
if !preserve {
|
|
tx.execute(
|
|
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
|
|
params![grant.to_string(), bytes],
|
|
)?;
|
|
}
|
|
}
|
|
tx.execute(
|
|
"INSERT INTO reconcile_consumption(decision_id,response) VALUES(?1,?2)",
|
|
params![
|
|
decision.decision_id.to_string(),
|
|
gate.response.encode_to_vec()
|
|
],
|
|
)?;
|
|
}
|
|
tx.commit()?;
|
|
gate.local_applied = true;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
pub fn initialize(db: &Connection) -> Result<()> {
|
|
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
|
ensure!(check == "ok", "JOURNAL_INTEGRITY");
|
|
db.execute_batch("PRAGMA synchronous=FULL;
|
|
CREATE TABLE IF NOT EXISTS reconcile_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,high_sequence INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,last_decision TEXT,pending_request TEXT,helper_id TEXT,helper_checkpoint INTEGER NOT NULL DEFAULT 0,fault TEXT);
|
|
CREATE TABLE IF NOT EXISTS reconcile_requests(id TEXT PRIMARY KEY,request BLOB NOT NULL,envelope BLOB NOT NULL,generation INTEGER NOT NULL,response BLOB);
|
|
CREATE TABLE IF NOT EXISTS reconcile_results(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,revision INTEGER NOT NULL,envelope BLOB NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS reconcile_receipts(id TEXT PRIMARY KEY,decision_id TEXT NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS reconcile_outcomes(job TEXT NOT NULL,attempt TEXT NOT NULL,outcome BLOB NOT NULL,decision_id TEXT NOT NULL,PRIMARY KEY(job,attempt));")?;
|
|
crate::update_health_inventory::initialize(db)?;
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO reconcile_meta(id,journal_id) VALUES(1,?1)",
|
|
[Uuid::new_v4().to_string()],
|
|
)?;
|
|
Ok(())
|
|
}
|
|
struct Meta {
|
|
journal_id: Uuid,
|
|
checkpoint: i64,
|
|
high_sequence: i64,
|
|
generation: u64,
|
|
decision: Option<Uuid>,
|
|
pending_request: Option<Uuid>,
|
|
helper_id: Option<Uuid>,
|
|
helper_checkpoint: u64,
|
|
fault: Option<String>,
|
|
}
|
|
type MetadataRow = (
|
|
String,
|
|
i64,
|
|
i64,
|
|
u64,
|
|
Option<String>,
|
|
Option<String>,
|
|
Option<String>,
|
|
u64,
|
|
Option<String>,
|
|
);
|
|
fn metadata(db: &Connection) -> Result<Meta> {
|
|
let row:MetadataRow=db.query_row("SELECT journal_id,checkpoint,high_sequence,generation,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?)))?;
|
|
Ok(Meta {
|
|
journal_id: Uuid::parse_str(&row.0)?,
|
|
checkpoint: row.1,
|
|
high_sequence: row.2,
|
|
generation: row.3,
|
|
decision: row.4.map(|s| Uuid::parse_str(&s)).transpose()?,
|
|
pending_request: row.5.map(|s| Uuid::parse_str(&s)).transpose()?,
|
|
helper_id: row.6.map(|s| Uuid::parse_str(&s)).transpose()?,
|
|
helper_checkpoint: row.7,
|
|
fault: row.8,
|
|
})
|
|
}
|
|
/// Call inside the same transaction BEFORE deleting an ACKed job event. This
|
|
/// retains the original result identity/output for future reconciliation.
|
|
pub fn archive_events(db: &Connection) -> Result<()> {
|
|
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY rowid")?;
|
|
for row in statement.query_map([], |r| {
|
|
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
|
|
})? {
|
|
let (id, bytes) = row?;
|
|
let envelope = Envelope::decode(bytes.as_slice())?;
|
|
ensure!(id == envelope.message_id, "EVENT_IDENTITY_CONFLICT");
|
|
if let Some(Payload::JobResult(r)) = &envelope.payload {
|
|
let existing: Option<Vec<u8>> = db
|
|
.query_row(
|
|
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
|
[&id],
|
|
|r| r.get(0),
|
|
)
|
|
.optional()?;
|
|
ensure!(
|
|
existing.as_ref().is_none_or(|b| *b == bytes),
|
|
"RESULT_IDENTITY_CONFLICT"
|
|
);
|
|
db.execute("INSERT OR IGNORE INTO reconcile_results(id,job,attempt,revision,envelope) VALUES(?1,?2,?3,?4,?5)",params![id,r.job_id,r.attempt_id,r.revision,bytes])?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
/// A partial control edge can contain the result that closed an attempt, and
|
|
/// an earlier unknown report for that same grant. Only account for archived
|
|
/// messages here. Never ACK an event, change its state or replay a result.
|
|
fn verify_closed_result_messages(
|
|
db: &Connection,
|
|
request: &SyncRequest,
|
|
decision: &ReconcileDecisionPayload,
|
|
) -> Result<()> {
|
|
for id in &request.durable_message_ids {
|
|
let raw: Vec<u8> = db.query_row(
|
|
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
|
[id],
|
|
|r| r.get(0),
|
|
)?;
|
|
let envelope = Envelope::decode(raw.as_slice())?;
|
|
let Some(Payload::JobResult(result)) = &envelope.payload else {
|
|
anyhow::bail!("CLOSED_CONTROL_EXPECTED_RESULT");
|
|
};
|
|
let attempt = request
|
|
.attempts
|
|
.iter()
|
|
.find(|a| {
|
|
a.job_id == result.job_id
|
|
&& a.attempt_id == result.attempt_id
|
|
&& a.grant_id == result.grant_id
|
|
})
|
|
.context("CLOSED_CONTROL_ARCHIVED_ATTEMPT")?;
|
|
let final_envelope = attempt
|
|
.result_envelope
|
|
.as_ref()
|
|
.context("CLOSED_CONTROL_FINAL_ENVELOPE")?;
|
|
let final_id = Uuid::parse_str(&attempt.result_message_id)?;
|
|
let persisted: bool = db.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM reconcile_receipts WHERE id=?1 AND decision_id=?2)",
|
|
params![attempt.result_message_id, decision.decision_id.to_string()],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
persisted
|
|
&& decision.acknowledged_message_ids.contains(&final_id)
|
|
&& request
|
|
.durable_message_ids
|
|
.contains(&attempt.result_message_id)
|
|
&& final_envelope.message_id == attempt.result_message_id
|
|
&& envelope.message_id == *id
|
|
&& envelope.encode_to_vec() == raw
|
|
&& rmm_protocol::valid_envelope(
|
|
&envelope,
|
|
&decision.agent_id.to_string(),
|
|
final_envelope.connection_generation
|
|
)
|
|
&& envelope.boot_id == final_envelope.boot_id
|
|
&& matches!(
|
|
attempt.state.as_str(),
|
|
"succeeded" | "failed" | "cancelled" | "expired"
|
|
)
|
|
&& !result.grant_id.is_empty(),
|
|
"CLOSED_CONTROL_FINAL_RESULT_RECEIPT"
|
|
);
|
|
ensure!(
|
|
if *id == attempt.result_message_id {
|
|
envelope == *final_envelope
|
|
&& result.revision == attempt.revision
|
|
&& result.state == attempt.state
|
|
&& canonical_outcome_bytes(&outcome(result)?)? == attempt.outcome_bytes
|
|
} else {
|
|
result.state == "unknown"
|
|
&& result.revision < attempt.revision
|
|
&& envelope.sequence < final_envelope.sequence
|
|
},
|
|
"CLOSED_CONTROL_CONFLICTING_ARCHIVED_RESULT"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
fn pending_ids(db: &Connection) -> Result<Vec<String>> {
|
|
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?;
|
|
let rows = statement
|
|
.query_map([], |r| {
|
|
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
|
|
})?
|
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
|
let mut ids = vec![];
|
|
for (id, bytes) in rows {
|
|
ensure!(
|
|
!Uuid::parse_str(&id)?.is_nil(),
|
|
"INVALID_DURABLE_MESSAGE_ID"
|
|
);
|
|
let envelope = Envelope::decode(bytes.as_slice())?;
|
|
ensure!(envelope.message_id == id, "EVENT_IDENTITY_CONFLICT");
|
|
if !crate::windows_system_nonstart::retired_event(db, &envelope)? {
|
|
ids.push(id);
|
|
}
|
|
}
|
|
Ok(ids)
|
|
}
|
|
pub(crate) fn outcome(r: &JobResult) -> Result<ReconciledOutcome> {
|
|
Ok(ReconciledOutcome {
|
|
state: r.state.clone(),
|
|
stdout: r.stdout.clone(),
|
|
stderr: r.stderr.clone(),
|
|
exit_code: r.exit_code,
|
|
output_truncated: r.output_truncated,
|
|
reboot_required: r.reboot_required,
|
|
evidence: serde_json::from_str(&r.evidence_json)?,
|
|
error_code: r.error_code.clone(),
|
|
})
|
|
}
|
|
fn terminal(state: &str) -> bool {
|
|
matches!(
|
|
state,
|
|
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
|
|
)
|
|
}
|
|
/// Server-signed final bytes of a human WebUI closure for an attempt whose
|
|
/// effect stayed unknown. They never claim success, failure or nonstart.
|
|
pub(crate) fn operator_closure(
|
|
reason: &str,
|
|
local_state: &str,
|
|
outcome: &ReconciledOutcome,
|
|
) -> bool {
|
|
reason == "OPERATOR_CLOSED_UNKNOWN"
|
|
&& local_state == "unknown"
|
|
&& outcome.state == "closed_unknown"
|
|
&& outcome.error_code == "OPERATOR_CLOSED_UNKNOWN"
|
|
&& outcome.exit_code.is_none()
|
|
&& outcome.stdout.is_empty()
|
|
&& outcome.stderr.is_empty()
|
|
&& outcome.evidence["postconditions_passed"] == false
|
|
&& outcome.evidence["effect"] == "unknown"
|
|
&& outcome.evidence["closed_by_human"] == true
|
|
&& outcome.evidence["retry"] == false
|
|
}
|
|
|
|
/// An interrupted agent can record its own uncertainty while the independently
|
|
/// consumed helper grant has no result. Neither witness establishes an effect.
|
|
pub(crate) fn uncertain_agent_outcome(state: &str, bytes: &[u8], digest: &str) -> bool {
|
|
if bytes.is_empty() || hash(bytes) != digest {
|
|
return false;
|
|
}
|
|
let Ok(outcome) = serde_json::from_slice::<ReconciledOutcome>(bytes) else {
|
|
return false;
|
|
};
|
|
canonical_outcome_bytes(&outcome).is_ok_and(|canonical| canonical == bytes)
|
|
&& outcome.state == state
|
|
&& !outcome.output_truncated
|
|
&& !outcome.reboot_required
|
|
&& (operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &outcome)
|
|
|| (state == "unknown"
|
|
&& matches!(
|
|
outcome.error_code.as_str(),
|
|
"AGENT_PROCESS_INTERRUPTED" | "HELPER_TRANSPORT_EFFECT_UNKNOWN"
|
|
)
|
|
&& outcome.exit_code.is_none()
|
|
&& outcome.stdout.is_empty()
|
|
&& outcome.stderr.is_empty()
|
|
&& outcome.evidence["postconditions_passed"] == false))
|
|
}
|
|
|
|
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
|
|
if s.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
let id = Uuid::parse_str(s)?;
|
|
ensure!(!id.is_nil(), "NIL_ID");
|
|
Ok(Some(id))
|
|
}
|
|
}
|
|
fn recovery_reason(reason: &str) -> bool {
|
|
matches!(
|
|
reason,
|
|
"MISSING_SERVER_JOB"
|
|
| "ATTEMPT_BINDING_MISMATCH"
|
|
| "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED"
|
|
| "SERVER_STATE_ROLLBACK"
|
|
| "ASSIGNMENT_MISMATCH"
|
|
| "JOURNAL_SEQUENCE_REGRESSION"
|
|
| "SERVER_CHECKPOINT_MISSING"
|
|
| "JOURNAL_ROLLBACK_OR_REPLACEMENT"
|
|
| "CHECKPOINT_CHAIN_MISMATCH"
|
|
| "HELPER_JOURNAL_ROLLBACK"
|
|
| "HELPER_CHECKPOINT_MISMATCH"
|
|
| "HELPER_CHECKPOINT_MISSING"
|
|
| "HELPER_CHECKPOINT_REQUIRED"
|
|
| "JOURNAL_HISTORY_MISSING"
|
|
| "LOCAL_ATTEMPT_MISSING"
|
|
)
|
|
}
|
|
|
|
fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<Vec<SyncAttempt>> {
|
|
let mut statement =
|
|
db.prepare("SELECT offer,state,revision,grant_id FROM job_attempts ORDER BY job,attempt")?;
|
|
let rows = statement.query_map([], |r| {
|
|
Ok((
|
|
r.get::<_, Vec<u8>>(0)?,
|
|
r.get::<_, String>(1)?,
|
|
r.get::<_, u64>(2)?,
|
|
r.get::<_, String>(3)?,
|
|
))
|
|
})?;
|
|
let mut summaries = Vec::new();
|
|
let mut known = BTreeSet::new();
|
|
for row in rows {
|
|
let (bytes, state, revision, mut grant_id) = row?;
|
|
let offer = JobOffer::decode(bytes.as_slice())?;
|
|
let job = binding.job(&offer.payload_bytes)?;
|
|
ensure!(
|
|
offer.job_id == job.job_id.to_string()
|
|
&& offer.attempt_id == job.attempt_id.to_string()
|
|
&& offer.job_payload_hash == hash(&offer.payload_bytes),
|
|
"LOCAL_OFFER_BINDING"
|
|
);
|
|
known.insert((offer.job_id.clone(), offer.attempt_id.clone()));
|
|
let consumed: Option<(String, String, u64, Option<Vec<u8>>)> = ledger
|
|
.query_row(
|
|
"SELECT grant_id,agent,generation,result FROM consumed WHERE job=?1 AND attempt=?2",
|
|
params![offer.job_id, offer.attempt_id],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
|
)
|
|
.optional()?;
|
|
// A signed definitive nonstart can make an accepted no-grant row
|
|
// terminal without a JobResult. Preserve that fact on the next Sync;
|
|
// requesting/running/unknown still cannot claim non-consumption. An
|
|
// actual durable consumption row below always overrides this label.
|
|
let mut consumption = if grant_id.is_empty()
|
|
&& matches!(state.as_str(), "accepted" | "expired" | "cancelled")
|
|
{
|
|
"unconsumed"
|
|
} else {
|
|
"uncertain"
|
|
};
|
|
let mut original_boot_id = String::new();
|
|
let mut canonical = Vec::new();
|
|
if let Some((id, agent, generation, result)) = consumed {
|
|
ensure!(
|
|
agent == binding.agent_id.to_string()
|
|
&& generation == binding.enrollment_generation
|
|
&& (grant_id.is_empty() || grant_id == id),
|
|
"CONSUMPTION_BINDING"
|
|
);
|
|
grant_id = id;
|
|
consumption = "consumed";
|
|
let evidence: Option<(Vec<u8>, Option<Vec<u8>>)> = ledger
|
|
.query_row(
|
|
"SELECT start,outcome FROM execution_evidence WHERE grant_id=?1",
|
|
[&grant_id],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)
|
|
.optional()?;
|
|
let mut stored = result;
|
|
if let Some((start, last)) = evidence {
|
|
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
|
original_boot_id = start["previous_boot_id"]
|
|
.as_str()
|
|
.context("ORIGINAL_BOOT_MISSING")?
|
|
.to_string();
|
|
optional_uuid(&original_boot_id)?;
|
|
let request: crate::auth::Request =
|
|
serde_json::from_value(start["request"].clone())?;
|
|
ensure!(
|
|
request.job_bytes == offer.payload_bytes
|
|
&& request.plan_hash == offer.plan_hash
|
|
&& request.authorization_id.to_string() == offer.authorization_id,
|
|
"START_EVIDENCE_BINDING"
|
|
);
|
|
if let Some(resolved) = crate::windows_system_nonstart::effective_outcome(
|
|
ledger,
|
|
Some(binding),
|
|
Uuid::parse_str(&grant_id)?,
|
|
)? {
|
|
stored = Some(resolved);
|
|
} else if last.is_some() {
|
|
stored = last;
|
|
}
|
|
}
|
|
if let Some(stored) = stored {
|
|
let o: ReconciledOutcome = serde_json::from_slice(&stored)?;
|
|
canonical = canonical_outcome_bytes(&o)?;
|
|
}
|
|
}
|
|
let original:Option<Vec<u8>>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",params![offer.job_id,offer.attempt_id],|r|r.get(0)).optional()?;
|
|
let mut result_envelope = None;
|
|
let mut result = None;
|
|
let mut result_message_id = String::new();
|
|
if let Some(bytes) = original {
|
|
let envelope = Envelope::decode(bytes.as_slice())?;
|
|
let Some(Payload::JobResult(r)) = &envelope.payload else {
|
|
anyhow::bail!("INVALID_ARCHIVED_RESULT")
|
|
};
|
|
if r.revision == revision && r.state == state {
|
|
ensure!(
|
|
envelope.agent_id == binding.agent_id.to_string()
|
|
&& envelope.enrollment_generation == binding.enrollment_generation
|
|
&& r.grant_id == grant_id,
|
|
"RESULT_BINDING"
|
|
);
|
|
let bytes = canonical_outcome_bytes(&outcome(r)?)?;
|
|
ensure!(
|
|
canonical.is_empty() || canonical == bytes,
|
|
"LOCAL_OUTCOME_LEDGER_MISMATCH"
|
|
);
|
|
canonical = bytes;
|
|
result_message_id = envelope.message_id.clone();
|
|
result = Some(r.clone());
|
|
result_envelope = Some(envelope);
|
|
}
|
|
}
|
|
// The signed closure projection can commit before either consumption
|
|
// ledger. A fresh signed check may then finish the remaining writes.
|
|
if state == "closed_unknown"
|
|
&& consumption == "consumed"
|
|
&& (canonical.is_empty()
|
|
|| uncertain_agent_outcome("unknown", &canonical, &hash(&canonical)))
|
|
{
|
|
let closed: Vec<u8> = db.query_row(
|
|
"SELECT outcome FROM reconcile_outcomes WHERE job=?1 AND attempt=?2",
|
|
params![offer.job_id, offer.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
uncertain_agent_outcome(&state, &closed, &hash(&closed)),
|
|
"LOCAL_CLOSURE_OUTCOME_MISMATCH"
|
|
);
|
|
canonical = closed;
|
|
}
|
|
if !canonical.is_empty() {
|
|
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
|
|
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
|
|
ensure!(canonical.len() <= 512 * 1024, "SYNC_CANONICAL_OUTCOME_SIZE");
|
|
if grant_id.is_empty() && terminal(&state) && o.evidence["not_started"] == true {
|
|
consumption = "unconsumed";
|
|
}
|
|
}
|
|
summaries.push(SyncAttempt {
|
|
update_handoff_bytes: Vec::new(),
|
|
job_id: offer.job_id,
|
|
attempt_id: offer.attempt_id,
|
|
job_payload_hash: offer.job_payload_hash,
|
|
grant_id,
|
|
consumption: consumption.into(),
|
|
state,
|
|
revision,
|
|
original_boot_id,
|
|
result,
|
|
result_message_id,
|
|
outcome_sha256: if canonical.is_empty() {
|
|
String::new()
|
|
} else {
|
|
hash(&canonical)
|
|
},
|
|
outcome_bytes: canonical,
|
|
result_envelope,
|
|
});
|
|
}
|
|
let mut statement = ledger.prepare("SELECT job,attempt FROM consumed")?;
|
|
for row in statement.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
|
ensure!(
|
|
known.contains(&row?),
|
|
"CONSUMED_ATTEMPT_MISSING_FROM_AGENT_JOURNAL"
|
|
);
|
|
}
|
|
Ok(summaries)
|
|
}
|
|
fn ingest_helper(
|
|
db: &Connection,
|
|
ledger: &Connection,
|
|
binding_and_verifier: (&Binding, &impl Ed25519Verifier),
|
|
channel: &Channel,
|
|
helper: &HelperSnapshot,
|
|
now: DateTime<Utc>,
|
|
identity: HelperIdentity,
|
|
) -> Result<()> {
|
|
let (binding, verifier) = binding_and_verifier;
|
|
let mut statement = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
|
|
for row in statement.query_map([], |r| {
|
|
Ok((
|
|
r.get::<_, String>(0)?,
|
|
r.get::<_, String>(1)?,
|
|
r.get::<_, Vec<u8>>(2)?,
|
|
))
|
|
})? {
|
|
let (job, attempt, bytes) = row?;
|
|
let offer = JobOffer::decode(bytes.as_slice())?;
|
|
if binding.job(&offer.payload_bytes)?.execution_identity == identity.as_str() {
|
|
let consumed: bool = ledger.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM consumed WHERE job=?1 AND attempt=?2)",
|
|
params![job, attempt],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
!consumed
|
|
|| helper
|
|
.attempts
|
|
.iter()
|
|
.any(|a| a.job_id == job && a.attempt_id == attempt),
|
|
"ROOT_CONSUMED_ATTEMPT_MISSING_FROM_HELPER_JOURNAL"
|
|
);
|
|
}
|
|
}
|
|
let mut seen = BTreeSet::new();
|
|
for h in &helper.attempts {
|
|
ensure!(
|
|
seen.insert((&h.job_id, &h.attempt_id)),
|
|
"DUPLICATE_HELPER_ATTEMPT"
|
|
);
|
|
let row: Option<(Vec<u8>, u64, String)> = db
|
|
.query_row(
|
|
"SELECT offer,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![h.job_id, h.attempt_id],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
|
)
|
|
.optional()?;
|
|
let (bytes, revision, grant) = row.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?;
|
|
let offer = JobOffer::decode(bytes.as_slice())?;
|
|
let job = binding.job(&offer.payload_bytes)?;
|
|
ensure!(
|
|
job.execution_identity == identity.as_str()
|
|
&& h.job_payload_hash == offer.job_payload_hash
|
|
&& (grant.is_empty() || grant == h.grant_id),
|
|
"HELPER_ATTEMPT_BINDING"
|
|
);
|
|
if h.outcome_bytes.is_empty() {
|
|
if !h.update_handoff_bytes.is_empty() {
|
|
let header =
|
|
validate_update_summary(binding, &job, &offer, h, channel.boot_id, now)?;
|
|
ensure!(
|
|
header.helper_journal_id == helper.journal_id
|
|
&& header.agent_journal_id == metadata(db)?.journal_id,
|
|
"UPDATE_HELPER_JOURNAL_BINDING"
|
|
);
|
|
let state: String = db.query_row(
|
|
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![h.job_id, h.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
matches!(
|
|
state.as_str(),
|
|
"accepted" | "requesting" | "running" | "verifying"
|
|
),
|
|
"UPDATE_CANNOT_HEAL_UNKNOWN"
|
|
);
|
|
type OriginalEvidence = (
|
|
String,
|
|
u64,
|
|
String,
|
|
String,
|
|
Vec<u8>,
|
|
Option<Vec<u8>>,
|
|
Option<Vec<u8>>,
|
|
);
|
|
let original:Option<OriginalEvidence>=ledger.query_row(
|
|
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM execution_evidence e JOIN consumed c USING(grant_id) WHERE grant_id=?1",[&h.grant_id],
|
|
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
|
let (agent, generation, job_id, attempt_id, start, outcome, result) =
|
|
original.context("UPDATE_LOCAL_ORIGINAL_MISSING")?;
|
|
ensure!(
|
|
agent == binding.agent_id.to_string()
|
|
&& generation == binding.enrollment_generation
|
|
&& job_id == h.job_id
|
|
&& attempt_id == h.attempt_id,
|
|
"UPDATE_LOCAL_CONSUMPTION_BINDING"
|
|
);
|
|
ensure!(
|
|
outcome.is_none() && result.is_none(),
|
|
"UPDATE_CANNOT_REPLACE_OUTCOME"
|
|
);
|
|
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
|
let request: crate::auth::Request =
|
|
serde_json::from_value(start["request"].clone())?;
|
|
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
|
serde_json::from_slice(&request.permit_bytes)?;
|
|
binding.verify_with(&request, permit.not_before, verifier)?;
|
|
ensure!(
|
|
hash(&serde_json::to_vec(&request)?) == header.attempt.original_request_sha256
|
|
&& permit.grant_id == header.attempt.grant_id
|
|
&& permit.agent_update_plan_id == Some(header.attempt.plan_id),
|
|
"UPDATE_LOCAL_ORIGINAL_CHANGED"
|
|
);
|
|
db.execute("UPDATE job_attempts SET state='running',revision=MAX(revision,?3) WHERE job=?1 AND attempt=?2",params![h.job_id,h.attempt_id,h.revision])?;
|
|
}
|
|
continue;
|
|
}
|
|
let o: ReconciledOutcome = serde_json::from_slice(&h.outcome_bytes)?;
|
|
ensure!(
|
|
canonical_outcome_bytes(&o)? == h.outcome_bytes
|
|
&& hash(&h.outcome_bytes) == h.outcome_sha256
|
|
&& o.state == h.state
|
|
&& (terminal(&o.state)
|
|
|| matches!(o.state.as_str(), "unknown" | "awaiting_reboot"))
|
|
&& h.consumption == "consumed"
|
|
&& optional_uuid(&h.grant_id)?.is_some(),
|
|
"HELPER_CANONICAL_OUTCOME"
|
|
);
|
|
let existing: Option<Vec<u8>> = ledger
|
|
.query_row(
|
|
"SELECT outcome FROM execution_evidence WHERE grant_id=?1",
|
|
[&h.grant_id],
|
|
|r| r.get(0),
|
|
)
|
|
.optional()?
|
|
.flatten();
|
|
let resolved = crate::windows_system_nonstart::effective_outcome(
|
|
ledger,
|
|
Some(binding),
|
|
Uuid::parse_str(&h.grant_id)?,
|
|
)?;
|
|
let same = resolved
|
|
.as_ref()
|
|
.or(existing.as_ref())
|
|
.map(|b| {
|
|
serde_json::from_slice::<ReconciledOutcome>(b)
|
|
.and_then(|o| canonical_outcome_bytes(&o))
|
|
})
|
|
.transpose()?
|
|
.is_some_and(|b| b == h.outcome_bytes);
|
|
let state: String = db.query_row(
|
|
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![h.job_id, h.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
if same && state == h.state {
|
|
continue;
|
|
}
|
|
let consumed: Option<(String, u64, String, String)> = ledger
|
|
.query_row(
|
|
"SELECT agent,generation,job,attempt FROM consumed WHERE grant_id=?1",
|
|
[&h.grant_id],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
|
)
|
|
.optional()?;
|
|
ensure!(
|
|
consumed
|
|
== Some((
|
|
binding.agent_id.to_string(),
|
|
binding.enrollment_generation,
|
|
h.job_id.clone(),
|
|
h.attempt_id.clone()
|
|
)),
|
|
"HELPER_CONSUMPTION_NOT_KNOWN_LOCALLY"
|
|
);
|
|
// Store actual helper evidence without unlocking result/consumption. A
|
|
// crash here still leaves both gates closed and the original helper
|
|
// result available; the next sync imports it again idempotently.
|
|
ensure!(
|
|
ledger.execute(
|
|
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
|
|
params![h.grant_id, h.outcome_bytes]
|
|
)? == 1,
|
|
"LOCAL_START_EVIDENCE_MISSING"
|
|
);
|
|
let next_revision = revision
|
|
.max(h.revision)
|
|
.checked_add(1)
|
|
.context("REVISION_OVERFLOW")?;
|
|
i64::try_from(next_revision)?;
|
|
let seq: i64 = db.query_row(
|
|
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
let result = JobResult {
|
|
job_id: h.job_id.clone(),
|
|
attempt_id: h.attempt_id.clone(),
|
|
revision: next_revision,
|
|
state: o.state.clone(),
|
|
grant_id: h.grant_id.clone(),
|
|
stdout: o.stdout,
|
|
stderr: o.stderr,
|
|
output_truncated: o.output_truncated,
|
|
reboot_required: o.reboot_required,
|
|
exit_code: o.exit_code,
|
|
evidence_json: o.evidence.to_string(),
|
|
error_code: o.error_code,
|
|
};
|
|
let envelope = Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: binding.agent_id.to_string(),
|
|
enrollment_generation: binding.enrollment_generation,
|
|
connection_generation: channel.connection_generation,
|
|
boot_id: channel.boot_id.to_string(),
|
|
sequence: u64::try_from(seq)?,
|
|
observed_at: now.timestamp_millis(),
|
|
payload: Some(Payload::JobResult(result)),
|
|
};
|
|
ensure!(
|
|
envelope.encoded_len() <= rmm_domain::MAX_MESSAGE,
|
|
"HELPER_RESULT_SIZE"
|
|
);
|
|
db.execute(
|
|
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
|
|
params![envelope.message_id, envelope.encode_to_vec()],
|
|
)?;
|
|
db.execute(
|
|
"UPDATE job_attempts SET state=?3,revision=?4,grant_id=?5 WHERE job=?1 AND attempt=?2",
|
|
params![h.job_id, h.attempt_id, o.state, next_revision, h.grant_id],
|
|
)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()> {
|
|
let mut seen = BTreeSet::new();
|
|
for h in &helper.attempts {
|
|
ensure!(
|
|
seen.insert((&h.job_id, &h.attempt_id)),
|
|
"DUPLICATE_HELPER_ATTEMPT"
|
|
);
|
|
let a = local
|
|
.iter_mut()
|
|
.find(|a| a.job_id == h.job_id && a.attempt_id == h.attempt_id)
|
|
.context("HELPER_ATTEMPT_MISSING_FROM_AGENT_JOURNAL")?;
|
|
ensure!(
|
|
a.job_payload_hash == h.job_payload_hash
|
|
&& (a.grant_id.is_empty() || a.grant_id == h.grant_id),
|
|
"HELPER_ATTEMPT_BINDING"
|
|
);
|
|
if h.consumption != "unconsumed" {
|
|
let unreported = h.state == "unknown"
|
|
&& h.consumption == "consumed"
|
|
&& h.outcome_bytes.is_empty()
|
|
&& h.outcome_sha256.is_empty()
|
|
&& h.update_handoff_bytes.is_empty()
|
|
&& a.consumption == "consumed"
|
|
&& a.grant_id == h.grant_id
|
|
&& a.original_boot_id == h.original_boot_id
|
|
&& uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
|
|
if !h.update_handoff_bytes.is_empty() {
|
|
ensure!(
|
|
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
|
|
"UPDATE_CANNOT_HEAL_UNKNOWN"
|
|
);
|
|
a.update_handoff_bytes.clone_from(&h.update_handoff_bytes);
|
|
}
|
|
a.grant_id = h.grant_id.clone();
|
|
a.consumption = h.consumption.clone();
|
|
ensure!(
|
|
a.outcome_sha256 == h.outcome_sha256 || unreported,
|
|
"HELPER_OUTCOME_MISMATCH"
|
|
);
|
|
if a.original_boot_id.is_empty() {
|
|
a.original_boot_id = h.original_boot_id.clone();
|
|
}
|
|
ensure!(
|
|
a.original_boot_id == h.original_boot_id,
|
|
"HELPER_ORIGINAL_BOOT_MISMATCH"
|
|
);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
fn validate_update_summary(
|
|
_binding: &Binding,
|
|
job: &JobPayload,
|
|
offer: &JobOffer,
|
|
local: &SyncAttempt,
|
|
boot: Uuid,
|
|
now: DateTime<Utc>,
|
|
) -> Result<rmm_updater_handoff::Handoff> {
|
|
let proof =
|
|
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
|
.map_err(anyhow::Error::msg)?;
|
|
let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
|
|
let (h, _) =
|
|
rmm_domain::agent_update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
|
|
let now = u64::try_from(now.timestamp())?;
|
|
ensure!(
|
|
rmm_domain::agent_update::handoff_matches_job(&h, job)
|
|
&& h.boot_id == boot
|
|
&& h.attempt.grant_id.to_string() == local.grant_id
|
|
&& h.attempt.authorization_id.to_string() == offer.authorization_id
|
|
&& h.attempt.plan_sha256 == offer.plan_hash
|
|
&& h.attempt.job_payload_sha256 == offer.job_payload_hash
|
|
&& local.original_boot_id == boot.to_string()
|
|
&& local.consumption == "consumed"
|
|
&& local.state == "running"
|
|
&& local.outcome_bytes.is_empty()
|
|
&& local.outcome_sha256.is_empty()
|
|
&& local.result.is_none()
|
|
&& local.result_envelope.is_none()
|
|
&& h.consumed_at <= now
|
|
&& now < h.finish_before,
|
|
"UPDATE_INFLIGHT_SUMMARY_BINDING"
|
|
);
|
|
Ok(h)
|
|
}
|
|
|
|
pub fn verify_response(
|
|
binding: &Binding,
|
|
request: &SyncRequest,
|
|
channel: &Channel,
|
|
response: &SyncResponse,
|
|
now: DateTime<Utc>,
|
|
) -> Result<ReconcileDecisionPayload> {
|
|
verify_response_with(
|
|
&DefaultVerifier::default(),
|
|
binding,
|
|
request,
|
|
channel,
|
|
response,
|
|
now,
|
|
)
|
|
}
|
|
pub fn verify_response_with(
|
|
verifier: &impl Ed25519Verifier,
|
|
binding: &Binding,
|
|
request: &SyncRequest,
|
|
channel: &Channel,
|
|
response: &SyncResponse,
|
|
now: DateTime<Utc>,
|
|
) -> Result<ReconcileDecisionPayload> {
|
|
ensure!(
|
|
response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64,
|
|
"SYNC_SIGNATURE_SIZE"
|
|
);
|
|
let raw = decode_key(&binding.public_key_hex)?;
|
|
ensure!(
|
|
verifier.verify(&raw, &response.decision_bytes, &response.signature)?,
|
|
"INVALID_SYNC_SIGNATURE"
|
|
);
|
|
let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
|
ensure!(
|
|
d.valid_profile(now)
|
|
&& d.valid_until - d.issued_at <= chrono::Duration::seconds(60)
|
|
&& d.key_id == hash(&raw),
|
|
"SYNC_PROFILE_OR_CLOCK"
|
|
);
|
|
ensure!(
|
|
d.tenant_id == binding.tenant_id
|
|
&& d.asset_id == binding.asset_id
|
|
&& d.agent_id == binding.agent_id
|
|
&& d.enrollment_generation == binding.enrollment_generation
|
|
&& d.assignment_version == binding.assignment_version
|
|
&& d.connection_generation == channel.connection_generation
|
|
&& d.boot_id == channel.boot_id
|
|
&& d.request_id.to_string() == request.request_id
|
|
&& d.request_hash == hash(&request.encode_to_vec())
|
|
&& d.journal_id.to_string() == request.journal_id
|
|
&& d.checkpoint == request.checkpoint
|
|
&& d.helper_journal_id == optional_uuid(&request.helper_journal_id)?
|
|
&& d.helper_checkpoint == request.helper_checkpoint,
|
|
"SYNC_DECISION_BINDING"
|
|
);
|
|
Ok(d)
|
|
}
|
|
fn decode_key(hex: &str) -> Result<Vec<u8>> {
|
|
ensure!(hex.len() == 64, "PINNED_KEY_SIZE");
|
|
(0..32)
|
|
.map(|i| {
|
|
Ok(u8::from_str_radix(
|
|
hex.get(i * 2..i * 2 + 2).context("PINNED_KEY_ENCODING")?,
|
|
16,
|
|
)?)
|
|
})
|
|
.collect()
|
|
}
|
|
fn validate_attempt(
|
|
job: &JobPayload,
|
|
offer: &JobOffer,
|
|
local: &SyncAttempt,
|
|
d: &AttemptDecision,
|
|
boot: Uuid,
|
|
binding: &Binding,
|
|
now: DateTime<Utc>,
|
|
) -> Result<Option<Vec<u8>>> {
|
|
ensure!(
|
|
d.job_payload_hash == offer.job_payload_hash
|
|
&& d.grant_id == optional_uuid(&local.grant_id)?
|
|
&& d.action == job.action.name()
|
|
&& d.execution_identity == job.execution_identity
|
|
&& (d.revision >= local.revision || d.outcome_state == "unknown")
|
|
&& d.original_boot_id == optional_uuid(&local.original_boot_id)?
|
|
&& d.previous_outcome_sha256.as_deref()
|
|
== (!local.outcome_sha256.is_empty()).then_some(local.outcome_sha256.as_str()),
|
|
"SYNC_ATTEMPT_BINDING"
|
|
);
|
|
if d.grant_id.is_some() {
|
|
ensure!(
|
|
d.authorization_id == Some(Uuid::parse_str(&offer.authorization_id)?)
|
|
&& d.plan_hash.as_deref() == Some(offer.plan_hash.as_str()),
|
|
"SYNC_AUTHORIZATION_BINDING"
|
|
);
|
|
}
|
|
ensure!(
|
|
!terminal(&local.state) || d.outcome_state == local.state,
|
|
"TERMINAL_STATE_REGRESSION"
|
|
);
|
|
if d.update_inflight.is_some() || !local.update_handoff_bytes.is_empty() {
|
|
validate_update_summary(binding, job, offer, local, boot, now)?;
|
|
let proof =
|
|
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
|
.map_err(anyhow::Error::msg)?;
|
|
let key =
|
|
rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
|
|
let expected =
|
|
rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
|
|
ensure!(
|
|
d.update_inflight.as_ref() == Some(&expected)
|
|
&& d.outcome_state == "running"
|
|
&& !d.unlock
|
|
&& d.outcome_bytes.is_empty()
|
|
&& d.outcome_sha256.is_none(),
|
|
"UPDATE_INFLIGHT_DECISION_CHANGED"
|
|
);
|
|
return Ok(None);
|
|
}
|
|
if d.outcome_state == "accepted" {
|
|
ensure!(
|
|
local.state == "accepted"
|
|
&& local.consumption == "unconsumed"
|
|
&& d.grant_id.is_none()
|
|
&& !d.unlock
|
|
&& d.outcome_bytes.is_empty(),
|
|
"SYNC_CANNOT_RESTART_ATTEMPT"
|
|
);
|
|
return Ok(None);
|
|
}
|
|
if d.outcome_state == "unknown" {
|
|
ensure!(!d.unlock, "UNKNOWN_CANNOT_UNLOCK");
|
|
return Ok(None);
|
|
}
|
|
ensure!(terminal(&d.outcome_state), "UNSUPPORTED_SYNC_STATE");
|
|
if d.outcome_bytes.is_empty() && d.outcome_sha256.is_none() {
|
|
ensure!(
|
|
local.consumption == "unconsumed"
|
|
&& d.grant_id.is_none()
|
|
&& !d.unlock
|
|
&& d.outcome_state != "succeeded",
|
|
"TERMINAL_WITHOUT_EVIDENCE"
|
|
);
|
|
return Ok(None);
|
|
}
|
|
let bytes = if d.outcome_bytes.is_empty() {
|
|
local.outcome_bytes.clone()
|
|
} else {
|
|
d.outcome_bytes.clone()
|
|
};
|
|
let o: ReconciledOutcome = serde_json::from_slice(&bytes)?;
|
|
ensure!(
|
|
canonical_outcome_bytes(&o)? == bytes
|
|
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
|
|
&& o.state == d.outcome_state,
|
|
"SYNC_CANONICAL_OUTCOME"
|
|
);
|
|
if bytes != local.outcome_bytes && !operator_closure(&d.reason, &local.state, &o) {
|
|
ensure!(
|
|
matches!(job.action, rmm_domain::jobs::Action::Reboot { .. })
|
|
&& d.outcome_state == "succeeded"
|
|
&& d.original_boot_id.is_some_and(|id| id != boot)
|
|
&& o.evidence["postconditions_passed"] == true
|
|
&& o.evidence["reconciled_after_reboot"] == true
|
|
&& o.evidence["new_boot_id"].as_str() == Some(boot.to_string().as_str())
|
|
&& o.evidence["inventory_message_id"]
|
|
.as_str()
|
|
.is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())),
|
|
"SYNC_REBOOT_REAUTH_EVIDENCE"
|
|
);
|
|
}
|
|
ensure!(
|
|
!d.unlock || (local.consumption == "consumed" && d.grant_id.is_some()),
|
|
"UNLOCK_WITHOUT_CONSUMPTION"
|
|
);
|
|
Ok(Some(bytes))
|
|
}
|
|
|
|
/// Operator regression only: imports the bounded, selected actual public
|
|
/// fixture into two IN-MEMORY databases. No production path/network/executor.
|
|
#[doc(hidden)]
|
|
pub fn closed_control_contract(raw: &[u8]) -> Result<serde_json::Value> {
|
|
use rusqlite::types::Value as SqlValue;
|
|
use serde_json::{Value, json};
|
|
ensure!(raw.len() <= 4 * 1024 * 1024, "CONTROL_FIXTURE_BOUND");
|
|
let f: Value = serde_json::from_slice(raw)?;
|
|
ensure!(
|
|
f["schema_version"] == 1
|
|
&& f["read_only"] == true
|
|
&& f["database_writes"] == false
|
|
&& f["private_keys_read"] == false,
|
|
"CONTROL_FIXTURE_SOURCE"
|
|
);
|
|
let binding: Binding = serde_json::from_value(f["binding"].clone())?;
|
|
let mut db = Connection::open_in_memory()?;
|
|
let ledger = Connection::open_in_memory()?;
|
|
initialize(&db)?;
|
|
crate::ledger::initialize_ledger(&ledger)?;
|
|
db.execute_batch("DELETE FROM reconcile_meta; CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT,PRIMARY KEY(job,attempt));CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE events(id TEXT PRIMARY KEY,payload BLOB)")?;
|
|
ledger.execute_batch(
|
|
"CREATE TABLE reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB)",
|
|
)?;
|
|
const RESOLUTION: &str = "CREATE TABLE windows_system_nonstart_221_resolution(grant_id TEXT PRIMARY KEY,role TEXT,receipt BLOB,receipt_sha256 TEXT,derived_outcome BLOB)";
|
|
db.execute_batch(RESOLUTION)?;
|
|
ledger.execute_batch(RESOLUTION)?;
|
|
fn cell(v: &Value) -> Result<SqlValue> {
|
|
let a = v.as_array().context("CONTROL_FIXTURE_CELL")?;
|
|
Ok(match a.first().and_then(Value::as_str) {
|
|
Some("null") if a.len() == 1 => SqlValue::Null,
|
|
Some("integer") if a.len() == 2 => {
|
|
SqlValue::Integer(a[1].as_i64().context("CONTROL_FIXTURE_INTEGER")?)
|
|
}
|
|
Some("text") if a.len() == 2 => {
|
|
SqlValue::Text(a[1].as_str().context("CONTROL_FIXTURE_TEXT")?.into())
|
|
}
|
|
Some("blob") if a.len() == 2 => SqlValue::Blob(serde_json::from_value(a[1].clone())?),
|
|
_ => anyhow::bail!("CONTROL_FIXTURE_CELL_TYPE"),
|
|
})
|
|
}
|
|
fn import(db: &Connection, rows: &Value, sql: &str) -> Result<()> {
|
|
let rows = rows.as_array().context("CONTROL_FIXTURE_ROWS")?;
|
|
ensure!(rows.len() <= 8, "CONTROL_FIXTURE_ROW_BOUND");
|
|
for row in rows {
|
|
let values = row
|
|
.as_array()
|
|
.context("CONTROL_FIXTURE_ROW")?
|
|
.iter()
|
|
.map(cell)
|
|
.collect::<Result<Vec<_>>>()?;
|
|
db.execute(sql, rusqlite::params_from_iter(values))?;
|
|
}
|
|
Ok(())
|
|
}
|
|
for (key, sql) in [
|
|
(
|
|
"meta",
|
|
"INSERT INTO reconcile_meta VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",
|
|
),
|
|
("counter", "INSERT INTO counter VALUES(?1,?2)"),
|
|
(
|
|
"requests",
|
|
"INSERT INTO reconcile_requests VALUES(?1,?2,?3,?4,?5)",
|
|
),
|
|
("jobs", "INSERT INTO job_attempts VALUES(?1,?2,?3,?4,?5,?6)"),
|
|
(
|
|
"outcomes",
|
|
"INSERT INTO reconcile_outcomes VALUES(?1,?2,?3,?4)",
|
|
),
|
|
(
|
|
"results",
|
|
"INSERT INTO reconcile_results VALUES(?1,?2,?3,?4,?5)",
|
|
),
|
|
("events", "INSERT INTO job_events VALUES(?1,?2)"),
|
|
(
|
|
"resolution",
|
|
"INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)",
|
|
),
|
|
] {
|
|
import(&db, &f["agent"][key], sql)?;
|
|
}
|
|
ensure!(
|
|
f["agent"]["legacy_count"] == 0,
|
|
"CONTROL_FIXTURE_PENDING_LEGACY"
|
|
);
|
|
for (key, sql) in [
|
|
("consumed", "INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,?6)"),
|
|
(
|
|
"evidence",
|
|
"INSERT INTO execution_evidence VALUES(?1,?2,?3)",
|
|
),
|
|
("acks", "INSERT INTO reconcile_consumption VALUES(?1,?2)"),
|
|
(
|
|
"resolution",
|
|
"INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)",
|
|
),
|
|
] {
|
|
import(&ledger, &f["ledger"][key], sql)?;
|
|
}
|
|
let captures = f["helper"]["capture"]
|
|
.as_array()
|
|
.context("CONTROL_FIXTURE_CAPTURE")?;
|
|
ensure!(captures.len() == 1, "CONTROL_FIXTURE_CAPTURE_COUNT");
|
|
let SqlValue::Blob(raw_snapshot) = cell(&captures[0][1])? else {
|
|
anyhow::bail!("CONTROL_FIXTURE_CAPTURE_BYTES")
|
|
};
|
|
let snapshot: crate::helper_reconcile::Snapshot = serde_json::from_slice(&raw_snapshot)?;
|
|
let m = f["helper"]["meta"]
|
|
.as_array()
|
|
.context("CONTROL_FIXTURE_HELPER_META")?;
|
|
ensure!(m.len() == 1, "CONTROL_FIXTURE_HELPER_META_COUNT");
|
|
ensure!(
|
|
cell(&m[0][0])? == SqlValue::Text(snapshot.journal_id.to_string())
|
|
&& cell(&m[0][1])? == SqlValue::Integer(i64::try_from(snapshot.checkpoint)?)
|
|
&& cell(&m[0][3])?
|
|
== SqlValue::Text(
|
|
snapshot
|
|
.previous_decision_id
|
|
.context("CONTROL_FIXTURE_HELPER_HEAD")?
|
|
.to_string()
|
|
)
|
|
&& cell(&m[0][6])? == SqlValue::Null,
|
|
"CONTROL_FIXTURE_ACTUAL_HELPER_CAPTURE"
|
|
);
|
|
crate::helper_reconcile::precommit_clock_contract(binding.clone(), snapshot.clone())?;
|
|
let helper = HelperSnapshot {
|
|
journal_id: snapshot.journal_id,
|
|
checkpoint: snapshot.checkpoint,
|
|
previous_decision_id: snapshot.previous_decision_id,
|
|
attempts: snapshot
|
|
.attempts
|
|
.iter()
|
|
.map(|a| SyncAttempt {
|
|
job_id: a.job_id.to_string(),
|
|
attempt_id: a.attempt_id.to_string(),
|
|
job_payload_hash: a.job_payload_hash.clone(),
|
|
grant_id: a.grant_id.to_string(),
|
|
consumption: a.consumption.clone(),
|
|
state: a.state.clone(),
|
|
revision: a.revision,
|
|
original_boot_id: a.original_boot_id.to_string(),
|
|
outcome_sha256: a.outcome_sha256.clone(),
|
|
outcome_bytes: a.outcome_bytes.clone(),
|
|
update_handoff_bytes: a.update_handoff_bytes.clone(),
|
|
..Default::default()
|
|
})
|
|
.collect(),
|
|
};
|
|
let meta = metadata(&db)?;
|
|
let helper_generation = match cell(&m[0][2])? {
|
|
SqlValue::Integer(v) => u64::try_from(v)?,
|
|
_ => anyhow::bail!("CONTROL_FIXTURE_HELPER_GENERATION"),
|
|
};
|
|
let channel = Channel {
|
|
connection_generation: meta
|
|
.generation
|
|
.max(helper_generation)
|
|
.checked_add(1)
|
|
.context("CONTROL_FIXTURE_GENERATION_OVERFLOW")?,
|
|
boot_id: snapshot.boot_id,
|
|
};
|
|
let now: DateTime<Utc> = serde_json::from_value(f["captured_at_utc"].clone())?;
|
|
let client = ClientSync::with_verifier(binding.clone(), crate::crypto::DalekVerifier)
|
|
.with_helper_identity(HelperIdentity::System);
|
|
let before = (db.total_changes(), ledger.total_changes());
|
|
client.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)?;
|
|
ensure!(
|
|
(db.total_changes(), ledger.total_changes()) == before,
|
|
"CONTROL_ADMISSION_WROTE_HISTORY"
|
|
);
|
|
ledger.execute(
|
|
"INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,NULL)",
|
|
params![
|
|
Uuid::from_u128(2).to_string(),
|
|
binding.agent_id.to_string(),
|
|
binding.enrollment_generation,
|
|
Uuid::from_u128(3).to_string(),
|
|
Uuid::from_u128(4).to_string()
|
|
],
|
|
)?;
|
|
ensure!(
|
|
client
|
|
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
|
|
.is_err(),
|
|
"CONTROL_HIDDEN_OPEN_GRANT_NOT_REJECTED"
|
|
);
|
|
ledger.execute(
|
|
"DELETE FROM consumed WHERE grant_id=?1",
|
|
[Uuid::from_u128(2).to_string()],
|
|
)?;
|
|
let a = meta.decision.context("CONTROL_FIXTURE_A")?;
|
|
let (request, _) = client.stored_control(&db, &ledger, a, channel.boot_id)?;
|
|
let (original_raw, original_env, original_response): (Vec<u8>, Vec<u8>, Vec<u8>) = db
|
|
.query_row(
|
|
"SELECT request,envelope,response FROM reconcile_requests WHERE id=?1",
|
|
[request.request_id.clone()],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
|
)?;
|
|
let mut changed = request.clone();
|
|
changed.helper_previous_decision_id = Uuid::from_u128(1).to_string();
|
|
let mut env = Envelope::decode(original_env.as_slice())?;
|
|
env.payload = Some(Payload::SyncRequest(changed.clone()));
|
|
db.execute(
|
|
"UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3",
|
|
params![
|
|
changed.encode_to_vec(),
|
|
env.encode_to_vec(),
|
|
request.request_id
|
|
],
|
|
)?;
|
|
ensure!(
|
|
client
|
|
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
|
|
.is_err(),
|
|
"CONTROL_MUTATED_A_NOT_REJECTED"
|
|
);
|
|
db.execute(
|
|
"UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3",
|
|
params![original_raw, original_env, request.request_id],
|
|
)?;
|
|
let mut changed = SyncResponse::decode(original_response.as_slice())?;
|
|
changed.signature[0] ^= 1;
|
|
let bytes = changed.encode_to_vec();
|
|
db.execute(
|
|
"UPDATE reconcile_requests SET response=?1 WHERE id=?2",
|
|
params![bytes, request.request_id],
|
|
)?;
|
|
ledger.execute(
|
|
"UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2",
|
|
params![bytes, a.to_string()],
|
|
)?;
|
|
ensure!(
|
|
client
|
|
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
|
|
.is_err(),
|
|
"CONTROL_MUTATED_SERVER_NOT_REJECTED"
|
|
);
|
|
db.execute(
|
|
"UPDATE reconcile_requests SET response=?1 WHERE id=?2",
|
|
params![original_response, request.request_id],
|
|
)?;
|
|
ledger.execute(
|
|
"UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2",
|
|
params![original_response, a.to_string()],
|
|
)?;
|
|
struct CurrentClock(DateTime<Utc>);
|
|
impl SyncClock for CurrentClock {
|
|
fn now(&self) -> Result<DateTime<Utc>> {
|
|
Ok(self.0)
|
|
}
|
|
}
|
|
let clock = CurrentClock(now);
|
|
let mut client = ClientSync::with_verifier(binding, crate::crypto::DalekVerifier)
|
|
.with_helper_identity(HelperIdentity::System);
|
|
ensure!(
|
|
client
|
|
.prepare(&mut db, &ledger, channel.clone(), Some(&helper), &clock)
|
|
.is_err(),
|
|
"CONTROL_IDLE_ATTESTATION_REQUIRED"
|
|
);
|
|
let fresh = client.prepare_with_closed_control_idle(
|
|
&mut db,
|
|
&ledger,
|
|
channel.clone(),
|
|
Some(&helper),
|
|
&clock,
|
|
true,
|
|
)?;
|
|
ensure!(
|
|
fresh.request.previous_decision_id == a.to_string()
|
|
&& fresh.request.helper_previous_decision_id
|
|
== helper.previous_decision_id.unwrap().to_string()
|
|
&& fresh.envelope.connection_generation == channel.connection_generation
|
|
&& fresh.envelope.observed_at == now.timestamp_millis()
|
|
&& !client.can_start(&channel, false, &clock)
|
|
&& !client.can_start(&channel, true, &clock)
|
|
&& client.gate_observation().is_none(),
|
|
"CONTROL_FRESH_REQUEST_MUST_NOT_AUTHORIZE"
|
|
);
|
|
Ok(
|
|
json!({"actual_closed_partial_edge_accepted":true,"mutated_agent_request_rejected":true,"mutated_server_signature_and_ack_rejected":true,"admission_read_only":true,"hidden_open_grant_rejected":true,"explicit_idle_attestation_required":true,"typed_precommit_clock_keeps_capture_without_writes":true,"fresh_request_keeps_both_heads":true,"old_authority_reused":false,"new_ready_or_unlock":false,"product_databases_opened":false,"effects":false}),
|
|
)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod closure_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn only_a_human_closure_of_a_locally_unknown_attempt_is_accepted() {
|
|
let closed = ReconciledOutcome {
|
|
state: "closed_unknown".into(),
|
|
stdout: vec![],
|
|
stderr: vec![],
|
|
exit_code: None,
|
|
output_truncated: false,
|
|
reboot_required: false,
|
|
evidence: serde_json::json!({"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"retry":false}),
|
|
error_code: "OPERATOR_CLOSED_UNKNOWN".into(),
|
|
};
|
|
assert!(operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &closed));
|
|
assert!(!operator_closure("DEFINITIVE_OUTCOME_RECONCILED", "unknown", &closed));
|
|
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "running", &closed));
|
|
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "succeeded", &closed));
|
|
for case in 0..6 {
|
|
let mut other = closed.clone();
|
|
match case {
|
|
0 => other.state = "failed".into(),
|
|
1 => other.error_code = "SCRIPT_FAILED".into(),
|
|
2 => other.exit_code = Some(0),
|
|
3 => other.stdout = b"x".to_vec(),
|
|
4 => other.evidence["postconditions_passed"] = true.into(),
|
|
_ => other.evidence["retry"] = true.into(),
|
|
}
|
|
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &other));
|
|
}
|
|
assert!(terminal("closed_unknown"));
|
|
}
|
|
}
|