1893 lines
74 KiB
Rust
1893 lines
74 KiB
Rust
//! Independently verifying root-ledger reconciliation. This module never executes
|
|
//! an operation or grants authority from an agent-provided ready/unlock flag.
|
|
//! Both general and APT listeners must use the SAME executor.lock and SQLite DB.
|
|
use crate::auth::HelperIdentity;
|
|
use crate::auth::{Binding, Outcome, Request};
|
|
use crate::crypto::{DefaultVerifier, Ed25519Verifier};
|
|
use anyhow::{Context, Result, ensure};
|
|
use chrono::{DateTime, Utc};
|
|
use rmm_domain::{
|
|
jobs::{Action, ExecutionPermitPayload, hash},
|
|
reconciliation::{
|
|
AttemptDecision, ReconcileDecisionPayload, ReconciledOutcome, canonical_outcome_bytes,
|
|
},
|
|
};
|
|
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use uuid::Uuid;
|
|
|
|
pub use crate::reconcile::SyncClock as Clock;
|
|
pub trait BootSource {
|
|
fn current_boot(&self) -> Result<Uuid>;
|
|
}
|
|
/// Must be backed by a held executor lock and an independently authenticated
|
|
/// ledger file identity. Windows adapters use Volume/File-ID, never Unix inodes.
|
|
/// The guard remains held through snapshot/apply/admission/consumption/effect.
|
|
pub trait ExecutorLedgerGuard {
|
|
fn check(&self, db: &Connection) -> Result<()>;
|
|
/// Only the specifically provisioned update authorizer may judge a live
|
|
/// owner. General readers stay closed without changing the original attempt.
|
|
fn is_update_owner_authority(&self) -> bool {
|
|
false
|
|
}
|
|
/// Default-deny live owner attestation for the one typed update handoff.
|
|
/// Implement only after qualifying fixed service/PID/birth/image and the
|
|
/// original immutable delegation under this SAME actual executor guard.
|
|
fn check_update_owner(
|
|
&self,
|
|
_handoff: &rmm_updater_handoff::Handoff,
|
|
_delegation: &rmm_domain::agent_update::UpdateDelegation,
|
|
_phase: rmm_updater_handoff::Phase,
|
|
_events: &[rmm_updater_handoff::Event],
|
|
) -> Result<()> {
|
|
anyhow::bail!("UPDATE_OWNER_NOT_QUALIFIED")
|
|
}
|
|
}
|
|
/// OS adapter contract, not an IPC assertion. The implementation must pin the
|
|
/// live helper PID/creation and ledger/lock file identities, and independently
|
|
/// read the fixed protected intent store's immutable known_no_dispatch CAS.
|
|
/// The same executor guard stays held throughout this check and the commit.
|
|
pub trait NoStartGuard: ExecutorLedgerGuard {
|
|
fn live_instance_sha256(&self) -> Result<String>;
|
|
fn verify_no_dispatch(
|
|
&self,
|
|
original: &Request,
|
|
anchor: &NoStartAnchor,
|
|
) -> Result<NoDispatchObservation>;
|
|
}
|
|
/// Trusted adapter return value only: deliberately has no IPC deserializer.
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
pub struct NoDispatchObservation {
|
|
pub intent_id: Uuid,
|
|
pub intent_sha256: String,
|
|
pub intent_recorded_at: DateTime<Utc>,
|
|
pub original_boot_id: Uuid,
|
|
}
|
|
/// Historical ready context, never a new lease or execution authorization.
|
|
/// Deserialization alone conveys no authority: every field is checked against
|
|
/// the still-held helper instance and the original independently stored history.
|
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct NoStartAnchor {
|
|
journal_id: Uuid,
|
|
checkpoint: u64,
|
|
decision_id: Uuid,
|
|
connection_generation: u64,
|
|
boot_id: Uuid,
|
|
binding_sha256: String,
|
|
execution_identity: String,
|
|
helper_instance_sha256: String,
|
|
helper_born_at: DateTime<Utc>,
|
|
sync_request_sha256: String,
|
|
signed_response_sha256: String,
|
|
ledger_fingerprint: String,
|
|
captured_at: DateTime<Utc>,
|
|
}
|
|
impl NoStartAnchor {
|
|
pub fn journal_id(&self) -> Uuid {
|
|
self.journal_id
|
|
}
|
|
pub fn checkpoint(&self) -> u64 {
|
|
self.checkpoint
|
|
}
|
|
pub fn decision_id(&self) -> Uuid {
|
|
self.decision_id
|
|
}
|
|
pub fn connection_generation(&self) -> u64 {
|
|
self.connection_generation
|
|
}
|
|
pub fn boot_id(&self) -> Uuid {
|
|
self.boot_id
|
|
}
|
|
pub fn helper_instance_sha256(&self) -> &str {
|
|
&self.helper_instance_sha256
|
|
}
|
|
pub fn sync_request_sha256(&self) -> &str {
|
|
&self.sync_request_sha256
|
|
}
|
|
pub fn signed_response_sha256(&self) -> &str {
|
|
&self.signed_response_sha256
|
|
}
|
|
}
|
|
/// Same two fields as public SyncResponse, without adding a new dependency or
|
|
/// wire type. RPC glue copies the ORIGINAL bytes/signature, never reserializes.
|
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct SignedResponse {
|
|
pub decision_bytes: Vec<u8>,
|
|
pub signature: Vec<u8>,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct LocalAttempt {
|
|
pub job_id: Uuid,
|
|
pub attempt_id: Uuid,
|
|
pub job_payload_hash: String,
|
|
pub grant_id: Uuid,
|
|
pub authorization_id: Uuid,
|
|
pub plan_hash: String,
|
|
pub action: String,
|
|
pub execution_identity: String,
|
|
pub revision: u64,
|
|
pub original_boot_id: Uuid,
|
|
pub consumption: String,
|
|
pub state: String,
|
|
pub outcome_sha256: String,
|
|
pub outcome_bytes: Vec<u8>,
|
|
/// Hash of the exact root-owned Request containing original job/permit bytes.
|
|
pub original_request_sha256: String,
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub update_handoff_bytes: Vec<u8>,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct Snapshot {
|
|
pub journal_id: Uuid,
|
|
pub checkpoint: u64,
|
|
pub previous_decision_id: Option<Uuid>,
|
|
pub boot_id: Uuid,
|
|
pub captured_at: DateTime<Utc>,
|
|
pub attempts: Vec<LocalAttempt>,
|
|
}
|
|
pub struct Applied {
|
|
pub snapshot: Snapshot,
|
|
pub unlocked_grants: Vec<Uuid>,
|
|
pub ready: bool,
|
|
}
|
|
struct Capture {
|
|
snapshot: Snapshot,
|
|
fingerprint: String,
|
|
generation: u64,
|
|
}
|
|
struct Lease {
|
|
checkpoint: u64,
|
|
decision_id: Uuid,
|
|
boot_id: Uuid,
|
|
generation: u64,
|
|
until: DateTime<Utc>,
|
|
fingerprint: String,
|
|
}
|
|
pub struct HelperSync<V = DefaultVerifier> {
|
|
binding: Binding,
|
|
verifier: V,
|
|
identity: HelperIdentity,
|
|
born: DateTime<Utc>,
|
|
capture: Option<Capture>,
|
|
lease: Option<Lease>,
|
|
}
|
|
impl HelperSync<DefaultVerifier> {
|
|
/// No persisted ready bit can open a newly started helper.
|
|
pub fn new(binding: Binding, clock: &impl Clock) -> Result<Self> {
|
|
Self::with_verifier(binding, DefaultVerifier::default(), clock)
|
|
}
|
|
}
|
|
impl<V: Ed25519Verifier> HelperSync<V> {
|
|
pub fn with_verifier(binding: Binding, verifier: V, clock: &impl Clock) -> Result<Self> {
|
|
pinned_key(&binding, &verifier)?;
|
|
Ok(Self {
|
|
binding,
|
|
verifier,
|
|
identity: HelperIdentity::Root,
|
|
born: clock.now()?,
|
|
capture: None,
|
|
lease: None,
|
|
})
|
|
}
|
|
/// Select exactly one protected helper execution identity, closing leases.
|
|
pub fn with_helper_identity(mut self, identity: HelperIdentity) -> Self {
|
|
self.close();
|
|
self.identity = identity;
|
|
self
|
|
}
|
|
pub fn close(&mut self) {
|
|
self.lease = None;
|
|
self.capture = None;
|
|
}
|
|
/// Read-only bound of the existing confirmed lease; creates no readiness.
|
|
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
|
|
self.lease.as_ref().map(|lease| lease.until)
|
|
}
|
|
pub fn active_generation(&self) -> Option<u64> {
|
|
self.lease.as_ref().map(|lease| lease.generation)
|
|
}
|
|
/// Capture only an actually confirmed live ready lease, before an intent
|
|
/// or fence closes it. Retaining this anchor cannot keep that lease alive.
|
|
pub fn no_start_anchor(
|
|
&self,
|
|
db: &Connection,
|
|
guard: &impl NoStartGuard,
|
|
generation: u64,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<NoStartAnchor> {
|
|
ensure!(
|
|
self.can_start(db, guard, generation, clock, boot),
|
|
"DENIAL_READY_ANCHOR_REQUIRED"
|
|
);
|
|
let lease = self
|
|
.lease
|
|
.as_ref()
|
|
.context("DENIAL_READY_ANCHOR_REQUIRED")?;
|
|
let meta = metadata(db)?;
|
|
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
|
|
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
|
|
[lease.decision_id.to_string()],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)?;
|
|
let instance = guard.live_instance_sha256()?;
|
|
ensure!(digest(&instance), "DENIAL_HELPER_INSTANCE_INVALID");
|
|
let capture: Vec<u8> = db.query_row(
|
|
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
|
|
[meta.checkpoint],
|
|
|r| r.get(0),
|
|
)?;
|
|
let capture: Snapshot = serde_json::from_slice(&capture)?;
|
|
let anchor = NoStartAnchor {
|
|
journal_id: meta.journal_id,
|
|
checkpoint: meta.checkpoint,
|
|
decision_id: lease.decision_id,
|
|
connection_generation: generation,
|
|
boot_id: lease.boot_id,
|
|
binding_sha256: hash(&serde_json::to_vec(&self.binding)?),
|
|
execution_identity: self.identity.as_str().into(),
|
|
helper_instance_sha256: instance,
|
|
helper_born_at: self.born,
|
|
sync_request_sha256: hash(&request),
|
|
signed_response_sha256: hash(&response),
|
|
ledger_fingerprint: lease.fingerprint.clone(),
|
|
captured_at: capture.captured_at,
|
|
};
|
|
self.verify_no_start_anchor(db, guard, &anchor, clock, boot)?;
|
|
Ok(anchor)
|
|
}
|
|
/// Consume exactly the approved update attempt and delegate its ORIGINAL
|
|
/// effect once. This is the only entry into update handoff persistence;
|
|
/// the independent live helper gate and all existing checks remain intact.
|
|
pub fn begin_update_handoff(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
guard: &impl crate::update_handoff::UpdateAdmissionGuard,
|
|
generation: u64,
|
|
prepared: &crate::update_handoff::PreparedHandoff<'_>,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<()> {
|
|
let crate::update_handoff::PreparedHandoff {
|
|
request,
|
|
handoff,
|
|
recorded_at,
|
|
..
|
|
} = *prepared;
|
|
ensure!(
|
|
self.can_start(db, guard, generation, clock, boot),
|
|
"UPDATE_HELPER_READY_REQUIRED"
|
|
);
|
|
let now = clock.now()?;
|
|
ensure!(
|
|
recorded_at <= now && now - recorded_at <= chrono::Duration::seconds(1),
|
|
"UPDATE_STALE_PREPARED_HANDOFF"
|
|
);
|
|
let (job, permit) = self.binding.verify_with(request, now, &self.verifier)?;
|
|
ensure!(
|
|
matches!(job.action, Action::UpdateAgent { .. })
|
|
&& job.execution_identity == self.identity.as_str(),
|
|
"UPDATE_HELPER_IDENTITY"
|
|
);
|
|
let meta = metadata(db)?;
|
|
let lease = self
|
|
.lease
|
|
.as_ref()
|
|
.context("UPDATE_HELPER_READY_REQUIRED")?;
|
|
ensure!(
|
|
handoff.helper_journal_id == meta.journal_id
|
|
&& handoff.helper_checkpoint == meta.checkpoint
|
|
&& Some(handoff.agent_journal_id) == meta.agent_journal
|
|
&& Some(handoff.previous_decision_id) == meta.decision
|
|
&& handoff.previous_decision_id == lease.decision_id
|
|
&& handoff.connection_generation == generation
|
|
&& handoff.boot_id == boot.current_boot()?,
|
|
"UPDATE_HANDOFF_SYNC_BINDING"
|
|
);
|
|
let public_key = guard.health_public_key()?;
|
|
ensure!(
|
|
rmm_domain::agent_update::job_health_public_key(&job).map_err(anyhow::Error::msg)?
|
|
== public_key,
|
|
"UPDATE_HEALTH_PIN_BINDING"
|
|
);
|
|
crate::update_handoff::consume(db, guard, &permit, prepared, public_key, &|tx| {
|
|
let fresh_now = clock.now()?;
|
|
ensure!(
|
|
self.can_start(tx, guard, generation, clock, boot)
|
|
&& fresh_now >= recorded_at
|
|
&& fresh_now - recorded_at <= chrono::Duration::seconds(1)
|
|
&& boot.current_boot()? == handoff.boot_id,
|
|
"UPDATE_ADMISSION_CHANGED_BEFORE_CONSUMPTION"
|
|
);
|
|
let (_, fresh) = self
|
|
.binding
|
|
.verify_with(request, fresh_now, &self.verifier)?;
|
|
ensure!(fresh == permit, "UPDATE_ORIGINAL_GRANT_CHANGED");
|
|
Ok(())
|
|
})?;
|
|
self.close();
|
|
Ok(())
|
|
}
|
|
/// Deny-only: the grant may be expired, but no historical timestamp is ever
|
|
/// passed to an execute verifier. Existing consumption is immutable. A new
|
|
/// c/start/no-effect outcome is committed atomically and remains locked for
|
|
/// the ordinary independently signed Sync decision.
|
|
pub fn abandon_no_start(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
guard: &impl NoStartGuard,
|
|
anchor: &NoStartAnchor,
|
|
request: &Request,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<Outcome> {
|
|
self.close();
|
|
guard.check(db)?;
|
|
let original = self.binding.verify_historical_for_denial_with(
|
|
request,
|
|
self.identity,
|
|
&self.verifier,
|
|
)?;
|
|
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
|
if let Some(outcome) = crate::ledger::existing_no_start(&tx, &original)? {
|
|
// No changed row and no classification of a missing original start.
|
|
tx.commit()?;
|
|
return Ok(outcome);
|
|
}
|
|
self.verify_no_start_anchor(&tx, guard, anchor, clock, boot)?;
|
|
let observation = guard.verify_no_dispatch(request, anchor)?;
|
|
let now = self.verify_no_start_anchor(&tx, guard, anchor, clock, boot)?;
|
|
ensure!(
|
|
!observation.intent_id.is_nil()
|
|
&& digest(&observation.intent_sha256)
|
|
&& observation.original_boot_id == anchor.boot_id
|
|
&& observation.intent_recorded_at >= anchor.captured_at
|
|
&& observation.intent_recorded_at <= now,
|
|
"DENIAL_INTENT_BINDING"
|
|
);
|
|
let outcome =
|
|
crate::ledger::record_no_start_in(&tx, &original, anchor.boot_id, now, &observation)?;
|
|
guard.check(&tx)?;
|
|
ensure!(
|
|
guard.live_instance_sha256()? == anchor.helper_instance_sha256,
|
|
"DENIAL_HELPER_INSTANCE_CHANGED"
|
|
);
|
|
ensure!(
|
|
guard.verify_no_dispatch(request, anchor)? == observation,
|
|
"DENIAL_INTENT_CHANGED_DURING_COMMIT"
|
|
);
|
|
tx.commit()?;
|
|
Ok(outcome)
|
|
}
|
|
fn verify_no_start_anchor(
|
|
&self,
|
|
db: &Connection,
|
|
guard: &impl NoStartGuard,
|
|
anchor: &NoStartAnchor,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<DateTime<Utc>> {
|
|
guard.check(db)?;
|
|
let now = clock.now()?;
|
|
ensure!(
|
|
anchor.helper_born_at == self.born
|
|
&& now >= self.born
|
|
&& boot.current_boot()? == anchor.boot_id
|
|
&& !anchor.boot_id.is_nil()
|
|
&& anchor.execution_identity == self.identity.as_str()
|
|
&& anchor.binding_sha256 == hash(&serde_json::to_vec(&self.binding)?)
|
|
&& digest(&anchor.helper_instance_sha256)
|
|
&& guard.live_instance_sha256()? == anchor.helper_instance_sha256,
|
|
"DENIAL_HELPER_CONTEXT_CHANGED"
|
|
);
|
|
let meta = metadata(db)?;
|
|
ensure!(
|
|
meta.fault.is_none()
|
|
&& meta.journal_id == anchor.journal_id
|
|
&& meta.checkpoint == anchor.checkpoint
|
|
&& meta.decision == Some(anchor.decision_id)
|
|
&& meta.generation == anchor.connection_generation
|
|
&& ledger_fingerprint(db)? == anchor.ledger_fingerprint,
|
|
"DENIAL_HELPER_LINEAGE_CHANGED"
|
|
);
|
|
// Exact equality is deliberate: a first implementation refuses even
|
|
// legitimate intervening progress rather than inventing lineage links.
|
|
verify_checkpoint_repair_history_with(
|
|
db,
|
|
&self.binding,
|
|
None,
|
|
self.identity,
|
|
&self.verifier,
|
|
)?;
|
|
let (response, raw): (Vec<u8>, Vec<u8>) = db.query_row(
|
|
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
|
|
[anchor.decision_id.to_string()],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)?;
|
|
ensure!(
|
|
hash(&response) == anchor.signed_response_sha256
|
|
&& hash(&raw) == anchor.sync_request_sha256,
|
|
"DENIAL_READY_BYTES_CHANGED"
|
|
);
|
|
let signed: SignedResponse = serde_json::from_slice(&response)?;
|
|
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
|
|
// Historical Sync evidence only, never a lease. Its original signature
|
|
// and immutable stored request remain mandatory after expiry.
|
|
let decision = verify(&self.verifier, &self.binding, &signed, parsed.issued_at)?;
|
|
let capture: Vec<u8> = db.query_row(
|
|
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
|
|
[anchor.checkpoint],
|
|
|r| r.get(0),
|
|
)?;
|
|
let capture: Snapshot = serde_json::from_slice(&capture)?;
|
|
ensure!(
|
|
decision.ready
|
|
&& decision.helper_ready
|
|
&& decision.decision_id == anchor.decision_id
|
|
&& decision.boot_id == anchor.boot_id
|
|
&& decision.helper_journal_id == Some(anchor.journal_id)
|
|
&& decision.helper_checkpoint == anchor.checkpoint
|
|
&& decision.connection_generation == anchor.connection_generation
|
|
&& capture.captured_at == anchor.captured_at
|
|
&& capture.boot_id == anchor.boot_id
|
|
&& capture.captured_at <= now,
|
|
"DENIAL_CONFIRMED_READY_REQUIRED"
|
|
);
|
|
Ok(now)
|
|
}
|
|
/// Every call creates a NEW durable checkpoint challenge and closes the old
|
|
/// lease. Use apply's returned snapshot for its ACK, not another snapshot().
|
|
pub fn snapshot(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
guard: &impl ExecutorLedgerGuard,
|
|
generation: u64,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<Snapshot> {
|
|
self.close();
|
|
let result = self.snapshot_inner(db, guard, generation, clock, boot);
|
|
if let Err(error) = &result {
|
|
persist_fault(db, error);
|
|
}
|
|
result
|
|
}
|
|
fn snapshot_inner(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
guard: &impl ExecutorLedgerGuard,
|
|
generation: u64,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<Snapshot> {
|
|
guard.check(db)?;
|
|
let now = clock.now()?;
|
|
ensure!(
|
|
now >= self.born && generation > 0,
|
|
"CLOCK_OR_CHANNEL_REGRESSION"
|
|
);
|
|
let boot_id = boot.current_boot()?;
|
|
ensure!(!boot_id.is_nil(), "BOOT_ID_REQUIRED");
|
|
initialize(db, &self.binding)?;
|
|
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
|
let meta = metadata(&tx)?;
|
|
ensure!(meta.fault.is_none(), "HELPER_MANUAL_RECOVERY_REQUIRED");
|
|
let highest: u64 = tx.query_row(
|
|
"SELECT COALESCE(MAX(checkpoint),0) FROM helper_sync_captures",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(meta.checkpoint == highest, "HELPER_CHECKPOINT_ROLLBACK");
|
|
ensure!(generation >= meta.generation, "STALE_HELPER_CONNECTION");
|
|
validate_history(&tx, &self.binding, &meta, generation, &self.verifier)?;
|
|
let attempts = load_rows(
|
|
&tx,
|
|
&self.binding,
|
|
true,
|
|
self.identity,
|
|
&self.verifier,
|
|
Some(LiveUpdateContext {
|
|
guard,
|
|
now,
|
|
boot: boot_id,
|
|
}),
|
|
)?;
|
|
ensure!(attempts.len() <= 256, "HELPER_SYNC_CAPACITY");
|
|
let fingerprint = ledger_fingerprint(&tx)?;
|
|
// Two independently gated profiles share one signed helper checkpoint.
|
|
// Join only a still-unapplied root-owned capture created after THIS
|
|
// process was born, on the same fenced channel and unchanged ledger.
|
|
if meta.checkpoint > 0 && meta.generation == generation {
|
|
let applied: bool = tx.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM helper_sync_decisions WHERE checkpoint=?1)",
|
|
[meta.checkpoint],
|
|
|r| r.get(0),
|
|
)?;
|
|
let old: Option<(Vec<u8>, String)> = tx
|
|
.query_row(
|
|
"SELECT snapshot,fingerprint FROM helper_sync_captures WHERE checkpoint=?1",
|
|
[meta.checkpoint],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)
|
|
.optional()?;
|
|
if !applied && let Some((bytes, old_fingerprint)) = old {
|
|
let shared: Snapshot = serde_json::from_slice(&bytes)?;
|
|
if shared.captured_at >= self.born
|
|
&& shared.captured_at <= now
|
|
&& now - shared.captured_at < chrono::Duration::seconds(30)
|
|
&& shared.boot_id == boot_id
|
|
&& shared.previous_decision_id == meta.decision
|
|
&& old_fingerprint == fingerprint
|
|
&& serde_json::to_vec(&shared.attempts)? == serde_json::to_vec(&attempts)?
|
|
{
|
|
tx.commit()?;
|
|
self.capture = Some(Capture {
|
|
snapshot: shared.clone(),
|
|
fingerprint,
|
|
generation,
|
|
});
|
|
return Ok(shared);
|
|
}
|
|
}
|
|
}
|
|
let checkpoint = meta
|
|
.checkpoint
|
|
.checked_add(1)
|
|
.context("CHECKPOINT_OVERFLOW")?;
|
|
i64::try_from(checkpoint)?;
|
|
let snapshot = Snapshot {
|
|
journal_id: meta.journal_id,
|
|
checkpoint,
|
|
previous_decision_id: meta.decision,
|
|
boot_id,
|
|
captured_at: now,
|
|
attempts,
|
|
};
|
|
let bytes = serde_json::to_vec(&snapshot)?;
|
|
ensure!(bytes.len() <= 2 * 1024 * 1024, "HELPER_SNAPSHOT_SIZE");
|
|
tx.execute("INSERT INTO helper_sync_captures(checkpoint,snapshot,fingerprint,captured_at) VALUES(?1,?2,?3,?4)",params![checkpoint,bytes,fingerprint,now.to_rfc3339()])?;
|
|
tx.execute(
|
|
"UPDATE helper_sync_meta SET checkpoint=?1,generation=?2 WHERE id=1",
|
|
params![checkpoint, generation],
|
|
)?;
|
|
tx.commit()?;
|
|
self.capture = Some(Capture {
|
|
snapshot: snapshot.clone(),
|
|
fingerprint,
|
|
generation,
|
|
});
|
|
Ok(snapshot)
|
|
}
|
|
/// raw_request is SyncRequest::encode_to_vec(), NOT the outer Envelope or a
|
|
/// JSON view. Verify signature before decoding fields. No unsigned unlocks.
|
|
pub fn apply(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
guard: &impl ExecutorLedgerGuard,
|
|
response: &SignedResponse,
|
|
raw_request: &[u8],
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<Applied> {
|
|
self.lease = None;
|
|
let result = self.apply_inner(db, guard, response, raw_request, clock, boot);
|
|
if let Err(error) = &result {
|
|
self.lease = None;
|
|
persist_fault(db, error);
|
|
}
|
|
result
|
|
}
|
|
fn apply_inner(
|
|
&mut self,
|
|
db: &mut Connection,
|
|
guard: &impl ExecutorLedgerGuard,
|
|
response: &SignedResponse,
|
|
raw_request: &[u8],
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> Result<Applied> {
|
|
guard.check(db)?;
|
|
let now = clock.now()?;
|
|
let capture = self
|
|
.capture
|
|
.as_ref()
|
|
.context("FRESH_PROCESS_SNAPSHOT_REQUIRED")?;
|
|
let decision = verify(&self.verifier, &self.binding, response, now)?;
|
|
ensure!(
|
|
!decision
|
|
.attempts
|
|
.iter()
|
|
.any(|d| d.update_inflight.is_some())
|
|
|| (!decision.ready && !decision.helper_ready),
|
|
"UPDATE_INFLIGHT_CANNOT_READY"
|
|
);
|
|
let request = RequestView::decode(raw_request)?;
|
|
let current_boot = boot.current_boot()?;
|
|
ensure!(
|
|
now >= self.born
|
|
&& decision.issued_at >= self.born
|
|
&& decision.issued_at >= capture.snapshot.captured_at
|
|
&& current_boot == capture.snapshot.boot_id
|
|
&& decision.boot_id == current_boot,
|
|
"OLD_PROCESS_DECISION_OR_BOOT"
|
|
);
|
|
ensure!(
|
|
decision.connection_generation == capture.generation
|
|
&& decision.helper_journal_id == Some(capture.snapshot.journal_id)
|
|
&& decision.helper_checkpoint == capture.snapshot.checkpoint
|
|
&& request.helper_journal_id == capture.snapshot.journal_id.to_string()
|
|
&& request.helper_checkpoint == capture.snapshot.checkpoint
|
|
&& request.helper_previous_decision_id
|
|
== capture
|
|
.snapshot
|
|
.previous_decision_id
|
|
.map(|id| id.to_string())
|
|
.unwrap_or_default()
|
|
&& decision.request_id.to_string() == request.request_id
|
|
&& decision.request_hash == hash(raw_request)
|
|
&& decision.journal_id.to_string() == request.journal_id
|
|
&& decision.checkpoint == request.checkpoint
|
|
&& request.assignment_version == self.binding.assignment_version
|
|
&& request.journal_complete,
|
|
"HELPER_SYNC_REQUEST_BINDING"
|
|
);
|
|
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
|
let meta = metadata(&tx)?;
|
|
ensure!(
|
|
meta.fault.is_none()
|
|
&& meta.journal_id == capture.snapshot.journal_id
|
|
&& meta.checkpoint >= capture.snapshot.checkpoint
|
|
&& meta.generation >= capture.generation,
|
|
"HELPER_CHECKPOINT_ROLLBACK"
|
|
);
|
|
ensure!(
|
|
meta.checkpoint == capture.snapshot.checkpoint && meta.generation == capture.generation,
|
|
"STALE_HELPER_CAPTURE"
|
|
);
|
|
ensure!(
|
|
meta.agent_journal
|
|
.is_none_or(|id| id == decision.journal_id)
|
|
&& decision.checkpoint >= meta.agent_checkpoint,
|
|
"AGENT_JOURNAL_ROLLBACK_OR_REPLACEMENT"
|
|
);
|
|
validate_history(
|
|
&tx,
|
|
&self.binding,
|
|
&meta,
|
|
capture.generation,
|
|
&self.verifier,
|
|
)?;
|
|
if let Some(old)=tx.query_row("SELECT response,request,fingerprint FROM helper_sync_decisions WHERE decision_id=?1",[decision.decision_id.to_string()],|r|Ok((r.get::<_,Vec<u8>>(0)?,r.get::<_,Vec<u8>>(1)?,r.get::<_,String>(2)?))).optional()? {
|
|
ensure!(old.0==serde_json::to_vec(response)? && old.1==raw_request && meta.decision==Some(decision.decision_id)
|
|
&& ledger_fingerprint(&tx)?==old.2,"HELPER_REPLAY_OR_LEDGER_CHANGED");
|
|
let rows=load_rows(&tx,&self.binding,false, self.identity, &self.verifier,Some(LiveUpdateContext {guard,now,boot:current_boot}))?;
|
|
let ready=decision.ready && decision.helper_ready && rows.iter().all(|row|definitive(row)
|
|
&& decision.attempts.iter().any(|d|d.job_id==row.job_id && d.attempt_id==row.attempt_id
|
|
&& d.outcome_state==row.state && d.outcome_sha256.as_deref()==Some(row.outcome_sha256.as_str())))
|
|
&& tx.query_row("SELECT count(*) FROM consumed WHERE result IS NULL",[],|r|r.get::<_,i64>(0))?==0;
|
|
let snapshot=Snapshot {journal_id:meta.journal_id,checkpoint:meta.checkpoint,previous_decision_id:meta.decision,
|
|
boot_id:current_boot,captured_at:capture.snapshot.captured_at,attempts:rows};
|
|
tx.commit()?;
|
|
if ready {self.lease=Some(Lease {checkpoint:meta.checkpoint,decision_id:decision.decision_id,boot_id:current_boot,
|
|
generation:capture.generation,until:decision.valid_until,fingerprint:old.2});}
|
|
return Ok(Applied {snapshot,unlocked_grants:vec![],ready});
|
|
}
|
|
ensure!(
|
|
meta.decision == capture.snapshot.previous_decision_id,
|
|
"HELPER_PREVIOUS_DECISION_CHANGED"
|
|
);
|
|
ensure!(
|
|
ledger_fingerprint(&tx)? == capture.fingerprint,
|
|
"LOCAL_LEDGER_CHANGED_DURING_SYNC"
|
|
);
|
|
let rows = load_rows(
|
|
&tx,
|
|
&self.binding,
|
|
false,
|
|
self.identity,
|
|
&self.verifier,
|
|
Some(LiveUpdateContext {
|
|
guard,
|
|
now,
|
|
boot: current_boot,
|
|
}),
|
|
)?;
|
|
ensure!(
|
|
serde_json::to_vec(&rows)? == serde_json::to_vec(&capture.snapshot.attempts)?,
|
|
"LOCAL_SNAPSHOT_CHANGED_DURING_SYNC"
|
|
);
|
|
let summaries = unique_summaries(&request.attempts)?;
|
|
let mut decisions = BTreeMap::new();
|
|
for d in &decision.attempts {
|
|
ensure!(
|
|
decisions.insert((d.job_id, d.attempt_id), d).is_none(),
|
|
"DUPLICATE_SIGNED_ATTEMPT"
|
|
);
|
|
}
|
|
let known: BTreeSet<_> = rows.iter().map(|r| (r.job_id, r.attempt_id)).collect();
|
|
for d in &decision.attempts {
|
|
ensure!(
|
|
d.execution_identity != self.identity.as_str()
|
|
|| d.grant_id.is_none()
|
|
|| known.contains(&(d.job_id, d.attempt_id)),
|
|
"SIGNED_ROOT_ATTEMPT_MISSING_LOCALLY"
|
|
);
|
|
}
|
|
let mut unlocked = Vec::new();
|
|
let mut safe = true;
|
|
for row in &rows {
|
|
let id = (row.job_id, row.attempt_id);
|
|
let local = summaries
|
|
.get(&id)
|
|
.context("ROOT_ATTEMPT_OMITTED_FROM_REQUEST")?;
|
|
let d = decisions
|
|
.get(&id)
|
|
.context("ROOT_ATTEMPT_OMITTED_FROM_DECISION")?;
|
|
validate_summary(row, local)?;
|
|
validate_decision(row, local, d)?;
|
|
if d.unlock {
|
|
let bytes = unlock_outcome(row, d, current_boot)?;
|
|
ensure!(
|
|
tx.execute(
|
|
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
|
|
params![row.grant_id.to_string(), bytes]
|
|
)? == 1,
|
|
"ROOT_CONSUMPTION_MISSING"
|
|
);
|
|
tx.execute(
|
|
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
|
|
params![row.grant_id.to_string(), bytes],
|
|
)?;
|
|
unlocked.push(row.grant_id);
|
|
} else {
|
|
safe &= definitive(row)
|
|
&& d.outcome_state == row.state
|
|
&& d.outcome_sha256.as_deref() == Some(row.outcome_sha256.as_str());
|
|
// A definitive local state with an unresolved consumed lock is
|
|
// not globally ready merely because the server says ready.
|
|
let locked: bool = tx.query_row(
|
|
"SELECT result IS NULL FROM consumed WHERE grant_id=?1",
|
|
[row.grant_id.to_string()],
|
|
|r| r.get(0),
|
|
)?;
|
|
safe &= !locked;
|
|
}
|
|
}
|
|
let fault = recovery_reason(&decision.reason).then(|| decision.reason.clone());
|
|
ensure!(
|
|
!decision.attempts.iter().any(|d| recovery_reason(&d.reason)),
|
|
"SIGNED_LEDGER_RECOVERY_REQUIRED"
|
|
);
|
|
let after = load_rows(
|
|
&tx,
|
|
&self.binding,
|
|
true,
|
|
self.identity,
|
|
&self.verifier,
|
|
Some(LiveUpdateContext {
|
|
guard,
|
|
now,
|
|
boot: current_boot,
|
|
}),
|
|
)?;
|
|
if decision
|
|
.attempts
|
|
.iter()
|
|
.any(|d| d.update_inflight.is_some())
|
|
{
|
|
use prost::Message;
|
|
let applied_at = clock.now()?;
|
|
ensure!(
|
|
applied_at >= now && applied_at < decision.valid_until,
|
|
"UPDATE_SYNC_EXPIRED_BEFORE_APPLY"
|
|
);
|
|
let response_raw = rmm_protocol::SyncResponse {
|
|
decision_bytes: response.decision_bytes.clone(),
|
|
signature: response.signature.clone(),
|
|
}
|
|
.encode_to_vec();
|
|
for decision_attempt in decision
|
|
.attempts
|
|
.iter()
|
|
.filter(|d| d.update_inflight.is_some())
|
|
{
|
|
let row = after
|
|
.iter()
|
|
.find(|r| {
|
|
r.job_id == decision_attempt.job_id
|
|
&& r.attempt_id == decision_attempt.attempt_id
|
|
&& !r.update_handoff_bytes.is_empty()
|
|
})
|
|
.context("UPDATE_APPLIED_ORIGINAL_ROW_REQUIRED")?;
|
|
crate::update_handoff::record_applied_sync(
|
|
&tx,
|
|
&crate::update_handoff::AppliedUpdateSync {
|
|
schema_version: 1,
|
|
grant_id: row.grant_id,
|
|
decision_id: decision.decision_id,
|
|
request_sha256: hash(raw_request),
|
|
response_protobuf_sha256: hash(&response_raw),
|
|
applied_at,
|
|
},
|
|
)?;
|
|
}
|
|
}
|
|
let fingerprint = ledger_fingerprint(&tx)?;
|
|
tx.execute("INSERT INTO helper_sync_decisions(decision_id,checkpoint,response,request,fingerprint) VALUES(?1,?2,?3,?4,?5)",params![decision.decision_id.to_string(),decision.helper_checkpoint,serde_json::to_vec(response)?,raw_request,fingerprint])?;
|
|
tx.execute("UPDATE helper_sync_meta SET last_decision=?1,fault=COALESCE(fault,?2),agent_journal_id=?3,agent_checkpoint=?4 WHERE id=1",params![decision.decision_id.to_string(),fault,decision.journal_id.to_string(),decision.checkpoint])?;
|
|
tx.commit()?;
|
|
let ready = safe
|
|
&& fault.is_none()
|
|
&& decision.ready
|
|
&& decision.helper_ready
|
|
&& after.iter().all(definitive);
|
|
if ready {
|
|
self.lease = Some(Lease {
|
|
checkpoint: decision.helper_checkpoint,
|
|
decision_id: decision.decision_id,
|
|
boot_id: current_boot,
|
|
generation: capture.generation,
|
|
until: decision.valid_until,
|
|
fingerprint,
|
|
});
|
|
}
|
|
Ok(Applied {
|
|
snapshot: Snapshot {
|
|
journal_id: meta.journal_id,
|
|
checkpoint: meta.checkpoint,
|
|
previous_decision_id: Some(decision.decision_id),
|
|
boot_id: current_boot,
|
|
captured_at: capture.snapshot.captured_at,
|
|
attempts: after,
|
|
},
|
|
unlocked_grants: unlocked,
|
|
ready,
|
|
})
|
|
}
|
|
/// Call under the same executor guard immediately BEFORE fresh grant
|
|
/// consumption. Once that operation changes the ledger, resync is required.
|
|
pub fn can_start(
|
|
&self,
|
|
db: &Connection,
|
|
guard: &impl ExecutorLedgerGuard,
|
|
generation: u64,
|
|
clock: &impl Clock,
|
|
boot: &impl BootSource,
|
|
) -> bool {
|
|
let result = (|| -> Result<bool> {
|
|
guard.check(db)?;
|
|
let reading = clock.reading()?;
|
|
let now = reading.own_wall;
|
|
let Some(lease) = &self.lease else {
|
|
return Ok(false);
|
|
};
|
|
let meta = metadata(db)?;
|
|
if meta.fault.is_some()
|
|
|| meta.checkpoint != lease.checkpoint
|
|
|| meta.decision != Some(lease.decision_id)
|
|
|| generation != lease.generation
|
|
{
|
|
return Ok(false);
|
|
}
|
|
validate_history(db, &self.binding, &meta, generation, &self.verifier)?;
|
|
Ok(meta.fault.is_none()
|
|
&& meta.checkpoint == lease.checkpoint
|
|
&& meta.decision == Some(lease.decision_id)
|
|
&& generation == lease.generation
|
|
&& now < lease.until
|
|
&& reading.upper < lease.until
|
|
&& now >= self.born
|
|
&& boot.current_boot()? == lease.boot_id
|
|
&& ledger_fingerprint(db)? == lease.fingerprint
|
|
&& db.query_row(
|
|
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
|
[],
|
|
|r| r.get::<_, i64>(0),
|
|
)? == 0)
|
|
})();
|
|
if let Err(error) = &result {
|
|
persist_fault(db, error);
|
|
}
|
|
result.unwrap_or(false)
|
|
}
|
|
}
|
|
|
|
fn digest(value: &str) -> bool {
|
|
value.len() == 64
|
|
&& value
|
|
.bytes()
|
|
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
|
}
|
|
fn pinned_key(binding: &Binding, verifier: &impl Ed25519Verifier) -> Result<Vec<u8>> {
|
|
let raw = hex::decode(&binding.public_key_hex)?;
|
|
ensure!(raw.len() == 32, "PINNED_HELPER_KEY_SIZE");
|
|
ensure!(verifier.validate_key(&raw)?, "PINNED_HELPER_KEY_INVALID");
|
|
Ok(raw)
|
|
}
|
|
fn verify(
|
|
verifier: &impl Ed25519Verifier,
|
|
binding: &Binding,
|
|
response: &SignedResponse,
|
|
now: DateTime<Utc>,
|
|
) -> Result<ReconcileDecisionPayload> {
|
|
ensure!(
|
|
response.signature.len() == 64 && response.decision_bytes.len() <= 512 * 1024,
|
|
"SYNC_SIGNATURE_SIZE"
|
|
);
|
|
let raw = pinned_key(binding, verifier)?;
|
|
ensure!(
|
|
verifier.verify(&raw, &response.decision_bytes, &response.signature)?,
|
|
"INVALID_SYNC_SIGNATURE"
|
|
);
|
|
let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
|
ensure!(
|
|
d.request_hash.len() == 64
|
|
&& d.request_hash
|
|
.bytes()
|
|
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
|
|
"SIGNED_REQUEST_HASH_MISSING_OR_INVALID"
|
|
);
|
|
i64::try_from(d.checkpoint)?;
|
|
i64::try_from(d.helper_checkpoint)?;
|
|
i64::try_from(d.connection_generation)?;
|
|
for attempt in &d.attempts {
|
|
i64::try_from(attempt.revision)?;
|
|
}
|
|
ensure!(
|
|
d.valid_profile(now)
|
|
&& d.valid_until - d.issued_at <= chrono::Duration::seconds(60)
|
|
&& d.key_id == hash(&raw)
|
|
&& d.tenant_id == binding.tenant_id
|
|
&& d.asset_id == binding.asset_id
|
|
&& d.agent_id == binding.agent_id
|
|
&& d.enrollment_generation == binding.enrollment_generation
|
|
&& d.assignment_version == binding.assignment_version,
|
|
"HELPER_DECISION_PROFILE_OR_IDENTITY"
|
|
);
|
|
Ok(d)
|
|
}
|
|
struct Meta {
|
|
journal_id: Uuid,
|
|
checkpoint: u64,
|
|
generation: u64,
|
|
decision: Option<Uuid>,
|
|
fault: Option<String>,
|
|
agent_journal: Option<Uuid>,
|
|
agent_checkpoint: u64,
|
|
}
|
|
fn initialize(db: &Connection, binding: &Binding) -> Result<()> {
|
|
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
|
ensure!(check == "ok", "ROOT_LEDGER_INTEGRITY");
|
|
db.execute_batch("PRAGMA synchronous=FULL;
|
|
CREATE TABLE IF NOT EXISTS helper_sync_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,binding_sha256 TEXT NOT NULL,last_decision TEXT,fault TEXT,agent_journal_id TEXT,agent_checkpoint INTEGER NOT NULL DEFAULT 0);
|
|
CREATE TABLE IF NOT EXISTS helper_sync_rows(grant_id TEXT PRIMARY KEY,revision INTEGER NOT NULL,fingerprint TEXT NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS helper_sync_captures(checkpoint INTEGER PRIMARY KEY,snapshot BLOB NOT NULL,fingerprint TEXT NOT NULL,captured_at TEXT NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS helper_sync_decisions(decision_id TEXT PRIMARY KEY,checkpoint INTEGER NOT NULL,response BLOB NOT NULL,request BLOB NOT NULL,fingerprint TEXT NOT NULL);")?;
|
|
let binding_hash = hash(&serde_json::to_vec(binding)?);
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO helper_sync_meta(id,journal_id,binding_sha256) VALUES(1,?1,?2)",
|
|
params![Uuid::new_v4().to_string(), binding_hash],
|
|
)?;
|
|
let old: String = db.query_row(
|
|
"SELECT binding_sha256 FROM helper_sync_meta WHERE id=1",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
old == binding_hash,
|
|
"INDEPENDENT_HELPER_BINDING_OR_KEY_CHANGED"
|
|
);
|
|
Ok(())
|
|
}
|
|
fn metadata(db: &Connection) -> Result<Meta> {
|
|
type MetaRow = (
|
|
String,
|
|
u64,
|
|
u64,
|
|
Option<String>,
|
|
Option<String>,
|
|
Option<String>,
|
|
u64,
|
|
);
|
|
let (id,checkpoint,generation,decision,fault,agent,agent_checkpoint):MetaRow=db.query_row("SELECT journal_id,checkpoint,generation,last_decision,fault,agent_journal_id,agent_checkpoint FROM helper_sync_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?)))?;
|
|
Ok(Meta {
|
|
journal_id: Uuid::parse_str(&id)?,
|
|
checkpoint,
|
|
generation,
|
|
decision: decision.map(|s| Uuid::parse_str(&s)).transpose()?,
|
|
fault,
|
|
agent_journal: agent.map(|s| Uuid::parse_str(&s)).transpose()?,
|
|
agent_checkpoint,
|
|
})
|
|
}
|
|
fn validate_history(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
meta: &Meta,
|
|
generation: u64,
|
|
verifier: &impl Ed25519Verifier,
|
|
) -> Result<()> {
|
|
let highest: u64 = db.query_row(
|
|
"SELECT COALESCE(MAX(checkpoint),0) FROM helper_sync_captures",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
highest == meta.checkpoint,
|
|
"HELPER_CAPTURE_HISTORY_ROLLBACK"
|
|
);
|
|
let decision_count: u64 =
|
|
db.query_row("SELECT count(*) FROM helper_sync_decisions", [], |r| {
|
|
r.get(0)
|
|
})?;
|
|
if let Some(id) = meta.decision {
|
|
let stored: Option<Vec<u8>> = db
|
|
.query_row(
|
|
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
|
|
[id.to_string()],
|
|
|r| r.get(0),
|
|
)
|
|
.optional()?;
|
|
let response: SignedResponse =
|
|
serde_json::from_slice(&stored.context("HELPER_DECISION_HISTORY_MISSING_LOCALLY")?)?;
|
|
// Historical already-applied evidence is checked at its ORIGINAL issue
|
|
// time; it never creates a lease or bypasses the fresh-process capture.
|
|
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
|
let previous = verify(verifier, binding, &response, parsed.issued_at)?;
|
|
let highest_decision: u64 = db.query_row(
|
|
"SELECT COALESCE(MAX(checkpoint),0) FROM helper_sync_decisions",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(
|
|
previous.decision_id == id
|
|
&& previous.helper_journal_id == Some(meta.journal_id)
|
|
&& previous.helper_checkpoint == highest_decision
|
|
&& previous.helper_checkpoint <= meta.checkpoint
|
|
&& generation >= previous.connection_generation
|
|
&& meta.agent_journal == Some(previous.journal_id)
|
|
&& meta.agent_checkpoint >= previous.checkpoint,
|
|
"HELPER_DECISION_HISTORY_ROLLBACK"
|
|
);
|
|
} else {
|
|
ensure!(
|
|
decision_count == 0 && meta.agent_journal.is_none() && meta.agent_checkpoint == 0,
|
|
"HELPER_METADATA_HISTORY_ROLLBACK"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
/// Bootstrap-only historical verification; this never creates a lease.
|
|
pub fn verify_checkpoint_repair_history(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
links: Option<(
|
|
&rmm_domain::checkpoint_repair::Receipt,
|
|
&rmm_domain::checkpoint_repair::Evidence,
|
|
)>,
|
|
) -> Result<()> {
|
|
verify_checkpoint_repair_history_with(
|
|
db,
|
|
binding,
|
|
links,
|
|
HelperIdentity::Root,
|
|
&DefaultVerifier::default(),
|
|
)
|
|
}
|
|
pub fn verify_checkpoint_repair_history_with(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
links: Option<(
|
|
&rmm_domain::checkpoint_repair::Receipt,
|
|
&rmm_domain::checkpoint_repair::Evidence,
|
|
)>,
|
|
identity: HelperIdentity,
|
|
verifier: &impl Ed25519Verifier,
|
|
) -> Result<()> {
|
|
let meta = metadata(db)?;
|
|
validate_history(db, binding, &meta, meta.generation, verifier)?;
|
|
let mut statement = db.prepare(
|
|
"SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint",
|
|
)?;
|
|
let entries = statement
|
|
.query_map([], |r| {
|
|
Ok((
|
|
r.get::<_, Vec<u8>>(0)?,
|
|
r.get::<_, Vec<u8>>(1)?,
|
|
r.get::<_, u64>(2)?,
|
|
))
|
|
})?
|
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
|
ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT");
|
|
let mut previous = None;
|
|
let mut previous_checkpoint = 0;
|
|
let mut previous_agent_checkpoint = 0;
|
|
for (signed, raw, checkpoint) in entries {
|
|
let response: SignedResponse = serde_json::from_slice(&signed)?;
|
|
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
|
let decision = verify(verifier, binding, &response, parsed.issued_at)?;
|
|
let request = RequestView::decode(&raw)?;
|
|
let captured: Vec<u8> = db.query_row(
|
|
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
|
|
[checkpoint],
|
|
|r| r.get(0),
|
|
)?;
|
|
let capture: Snapshot = serde_json::from_slice(&captured)?;
|
|
ensure!(
|
|
decision.helper_journal_id == Some(meta.journal_id)
|
|
&& decision.helper_checkpoint == checkpoint
|
|
&& request.helper_journal_id == meta.journal_id.to_string()
|
|
&& request.helper_checkpoint == checkpoint
|
|
&& request.assignment_version == binding.assignment_version
|
|
&& request.journal_complete
|
|
&& decision.request_hash == hash(&raw)
|
|
&& decision.request_id.to_string() == request.request_id
|
|
&& decision.journal_id.to_string() == request.journal_id
|
|
&& decision.checkpoint == request.checkpoint
|
|
&& capture.journal_id == meta.journal_id
|
|
&& capture.checkpoint == checkpoint
|
|
&& capture.boot_id == decision.boot_id
|
|
&& capture.captured_at <= decision.issued_at
|
|
&& capture.previous_decision_id == previous
|
|
&& checkpoint > previous_checkpoint
|
|
&& decision.checkpoint >= previous_agent_checkpoint
|
|
&& decision.connection_generation <= meta.generation
|
|
&& request.helper_previous_decision_id
|
|
== capture
|
|
.previous_decision_id
|
|
.map(|v| v.to_string())
|
|
.unwrap_or_default(),
|
|
"REPAIR_HELPER_HISTORY_BINDING"
|
|
);
|
|
previous = Some(decision.decision_id);
|
|
previous_checkpoint = checkpoint;
|
|
previous_agent_checkpoint = decision.checkpoint;
|
|
}
|
|
let _ = load_rows(db, binding, false, identity, verifier, None)?;
|
|
if let Some((r, e)) = links {
|
|
verify_checkpoint_repair_links(db, binding, r, e, verifier)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
fn verify_checkpoint_repair_links(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
r: &rmm_domain::checkpoint_repair::Receipt,
|
|
e: &rmm_domain::checkpoint_repair::Evidence,
|
|
verifier: &impl Ed25519Verifier,
|
|
) -> Result<()> {
|
|
let x = RequestView::decode(&e.uncommitted_request)?;
|
|
let y = RequestView::decode(&e.rejected_request)?;
|
|
let a: ReconcileDecisionPayload =
|
|
serde_json::from_slice(&e.previous_valid_response.decision_bytes)?;
|
|
let a = verify(
|
|
verifier,
|
|
binding,
|
|
&SignedResponse {
|
|
decision_bytes: e.previous_valid_response.decision_bytes.clone(),
|
|
signature: e.previous_valid_response.signature.clone(),
|
|
},
|
|
a.issued_at,
|
|
)?;
|
|
let old: Vec<u8> = db.query_row(
|
|
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
|
|
[a.decision_id.to_string()],
|
|
|v| v.get(0),
|
|
)?;
|
|
let old: SignedResponse = serde_json::from_slice(&old)?;
|
|
ensure!(
|
|
old.decision_bytes == e.previous_valid_response.decision_bytes
|
|
&& old.signature == e.previous_valid_response.signature
|
|
&& a.decision_id == r.previous_valid_decision_id
|
|
&& hash(&old.decision_bytes) == r.previous_valid_decision_sha256
|
|
&& a.reason == "ATTEMPT_RECONCILIATION_REQUIRED"
|
|
&& !a.ready
|
|
&& !a.helper_ready
|
|
&& x.request_id == r.uncommitted_request_id.to_string()
|
|
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
|
|
&& y.request_id == r.rejected_request_id.to_string()
|
|
&& hash(&e.rejected_request) == r.rejected_request_sha256
|
|
&& x.previous_decision_id == a.decision_id.to_string()
|
|
&& x.previous_request_id.is_empty()
|
|
&& x.helper_previous_decision_id == a.decision_id.to_string()
|
|
&& y.helper_previous_decision_id == a.decision_id.to_string()
|
|
&& y.previous_decision_id.is_empty()
|
|
&& y.previous_request_id == x.request_id
|
|
&& x.journal_id == r.agent_journal_id.to_string()
|
|
&& y.journal_id == x.journal_id
|
|
&& a.journal_id == r.agent_journal_id
|
|
&& x.helper_journal_id == r.helper_journal_id.to_string()
|
|
&& y.helper_journal_id == x.helper_journal_id
|
|
&& a.helper_journal_id == Some(r.helper_journal_id)
|
|
&& a.checkpoint < x.checkpoint
|
|
&& x.checkpoint < y.checkpoint
|
|
&& y.checkpoint == r.agent_checkpoint
|
|
&& a.helper_checkpoint < x.helper_checkpoint
|
|
&& x.helper_checkpoint < y.helper_checkpoint
|
|
&& y.helper_checkpoint == r.helper_checkpoint
|
|
&& x.assignment_version == binding.assignment_version
|
|
&& y.assignment_version == binding.assignment_version
|
|
&& x.journal_complete
|
|
&& y.journal_complete,
|
|
"REPAIR_EXACT_UNSENT_CHAIN_REQUIRED"
|
|
);
|
|
Ok(())
|
|
}
|
|
fn persist_fault(db: &Connection, error: &anyhow::Error) {
|
|
let text = error.to_string();
|
|
if [
|
|
"OMITTED",
|
|
"ROLLBACK",
|
|
"LEDGER_CHANGED",
|
|
"SNAPSHOT_CHANGED",
|
|
"MISSING_LOCALLY",
|
|
"ROOT_CONSUMPTION_MISSING",
|
|
"UNKNOWN_LOCAL_STATE",
|
|
"RECOVERY_REQUIRED",
|
|
"BINDING_OR_KEY_CHANGED",
|
|
"LOCAL_START_MISSING",
|
|
"LOCAL_REQUEST_BINDING",
|
|
"LOCAL_OUTCOME_CONFLICT",
|
|
"LOCAL_SUCCESS_PROOF_MISSING",
|
|
"SIGNED_REQUEST_HASH_MISSING_OR_INVALID",
|
|
"ROOT_SIGNED_ATTEMPT_BINDING",
|
|
]
|
|
.iter()
|
|
.any(|s| text.contains(s))
|
|
{
|
|
let _ = db.execute(
|
|
"UPDATE helper_sync_meta SET fault=COALESCE(fault,?1) WHERE id=1",
|
|
[text],
|
|
);
|
|
}
|
|
}
|
|
fn recovery_reason(reason: &str) -> bool {
|
|
reason.contains("ROLLBACK")
|
|
|| reason.contains("RECOVERY_REQUIRED")
|
|
|| matches!(
|
|
reason,
|
|
"MISSING_SERVER_JOB"
|
|
| "ATTEMPT_BINDING_MISMATCH"
|
|
| "ASSIGNMENT_MISMATCH"
|
|
| "JOURNAL_SEQUENCE_REGRESSION"
|
|
| "SERVER_CHECKPOINT_MISSING"
|
|
| "CHECKPOINT_CHAIN_MISMATCH"
|
|
| "JOURNAL_HISTORY_MISSING"
|
|
| "LOCAL_ATTEMPT_MISSING"
|
|
)
|
|
}
|
|
type LedgerRow = (
|
|
String,
|
|
String,
|
|
u64,
|
|
String,
|
|
String,
|
|
Option<Vec<u8>>,
|
|
Option<Vec<u8>>,
|
|
Option<Vec<u8>>,
|
|
);
|
|
fn ledger_rows(db: &Connection) -> Result<Vec<LedgerRow>> {
|
|
let mut statement=db.prepare("SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e ON e.grant_id=c.grant_id ORDER BY c.grant_id")?;
|
|
Ok(statement
|
|
.query_map([], |r| {
|
|
Ok((
|
|
r.get(0)?,
|
|
r.get(1)?,
|
|
r.get(2)?,
|
|
r.get(3)?,
|
|
r.get(4)?,
|
|
r.get(5)?,
|
|
r.get(6)?,
|
|
r.get(7)?,
|
|
))
|
|
})?
|
|
.collect::<rusqlite::Result<_>>()?)
|
|
}
|
|
pub(crate) fn ledger_fingerprint(db: &Connection) -> Result<String> {
|
|
let rows = ledger_rows(db)?;
|
|
let exists:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='agent_update_handoffs')",[],|r|r.get(0))?;
|
|
if exists {
|
|
let mut statement=db.prepare("SELECT h.grant_id,h.record_sha256,h.head_sha256,h.event_count,h.phase,d.witness_sha256 FROM agent_update_handoffs h JOIN agent_update_delegations d USING(grant_id) ORDER BY h.grant_id")?;
|
|
type UpdateRow = (String, String, String, u32, String, String);
|
|
let updates = statement
|
|
.query_map([], |r| {
|
|
Ok((
|
|
r.get(0)?,
|
|
r.get(1)?,
|
|
r.get(2)?,
|
|
r.get(3)?,
|
|
r.get(4)?,
|
|
r.get(5)?,
|
|
))
|
|
})?
|
|
.collect::<rusqlite::Result<Vec<UpdateRow>>>()?;
|
|
if !updates.is_empty() {
|
|
let mut claims=db.prepare("SELECT grant_id,effect_key,claim,claim_sha256 FROM agent_update_native_claims ORDER BY grant_id,effect_key")?;
|
|
type NativeClaimRow = (String, String, Vec<u8>, String);
|
|
let claims = claims
|
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?
|
|
.collect::<rusqlite::Result<Vec<NativeClaimRow>>>()?;
|
|
let mut observations=db.prepare("SELECT grant_id,effect_key,claim_sha256,observation_sha256,observed_at FROM agent_update_native_observations ORDER BY grant_id,effect_key")?;
|
|
type NativeObservationRow = (String, String, String, String, String);
|
|
let observations = observations
|
|
.query_map([], |r| {
|
|
Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?))
|
|
})?
|
|
.collect::<rusqlite::Result<Vec<NativeObservationRow>>>()?;
|
|
let mut applied=db.prepare("SELECT grant_id,decision_id,witness_sha256 FROM agent_update_sync_applied ORDER BY grant_id,decision_id")?;
|
|
let applied = applied
|
|
.query_map([], |r| {
|
|
Ok((
|
|
r.get::<_, String>(0)?,
|
|
r.get::<_, String>(1)?,
|
|
r.get::<_, String>(2)?,
|
|
))
|
|
})?
|
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
|
return Ok(hash(&serde_json::to_vec(&(
|
|
rows,
|
|
updates,
|
|
claims,
|
|
observations,
|
|
applied,
|
|
))?));
|
|
}
|
|
}
|
|
Ok(hash(&serde_json::to_vec(&rows)?))
|
|
}
|
|
fn canonical(bytes: &[u8]) -> Result<Vec<u8>> {
|
|
let o: ReconciledOutcome = serde_json::from_slice(bytes)?;
|
|
Ok(canonical_outcome_bytes(&o)?)
|
|
}
|
|
struct LiveUpdateContext<'a> {
|
|
guard: &'a dyn ExecutorLedgerGuard,
|
|
now: DateTime<Utc>,
|
|
boot: Uuid,
|
|
}
|
|
fn load_rows(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
refresh: bool,
|
|
identity: HelperIdentity,
|
|
verifier: &impl Ed25519Verifier,
|
|
live: Option<LiveUpdateContext<'_>>,
|
|
) -> Result<Vec<LocalAttempt>> {
|
|
let mut result = Vec::new();
|
|
for (grant, agent, generation, job_id, attempt_id, finished, start, last) in ledger_rows(db)? {
|
|
let start = start.context("LOCAL_START_MISSING_POSSIBLE_EFFECT")?;
|
|
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
|
let original_boot_id = Uuid::parse_str(
|
|
start["previous_boot_id"]
|
|
.as_str()
|
|
.context("LOCAL_ORIGINAL_BOOT_MISSING")?,
|
|
)?;
|
|
let request: Request = serde_json::from_value(start["request"].clone())?;
|
|
let permit: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?;
|
|
// Historical verification checks the original valid grant window, NOT
|
|
// authority to start now. Root consumption remains permanently recorded.
|
|
let (job, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
|
|
ensure!(
|
|
verified == permit
|
|
&& job.execution_identity == identity.as_str()
|
|
&& grant == permit.grant_id.to_string()
|
|
&& agent == binding.agent_id.to_string()
|
|
&& generation == binding.enrollment_generation
|
|
&& job_id == job.job_id.to_string()
|
|
&& attempt_id == job.attempt_id.to_string(),
|
|
"LOCAL_REQUEST_BINDING"
|
|
);
|
|
if let (Some(a), Some(b)) = (&finished, &last) {
|
|
ensure!(canonical(a)? == canonical(b)?, "LOCAL_OUTCOME_CONFLICT");
|
|
}
|
|
let mut outcome_bytes = last
|
|
.as_deref()
|
|
.or(finished.as_deref())
|
|
.map(canonical)
|
|
.transpose()?
|
|
.unwrap_or_default();
|
|
ensure!(outcome_bytes.len() <= 512 * 1024, "LOCAL_OUTCOME_SIZE");
|
|
let mut update_proof =
|
|
if outcome_bytes.is_empty() && matches!(job.action, Action::UpdateAgent { .. }) {
|
|
crate::update_handoff::proof_locked(db, permit.grant_id)?
|
|
} else {
|
|
None
|
|
};
|
|
if let Some(proof) = &update_proof {
|
|
let key = rmm_domain::agent_update::job_health_public_key(&job)
|
|
.map_err(anyhow::Error::msg)?;
|
|
let (handoff, journal) = crate::update_handoff::verify_inflight_proof(proof, key)?;
|
|
if let Some(live) = &live {
|
|
ensure!(
|
|
live.guard.is_update_owner_authority(),
|
|
"UPDATE_OWNER_AUTHORITY_REQUIRED"
|
|
);
|
|
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
|
let events = decoded
|
|
.events
|
|
.iter()
|
|
.map(|raw| serde_json::from_slice::<rmm_updater_handoff::Event>(raw))
|
|
.collect::<std::result::Result<Vec<_>, _>>()?;
|
|
let reason = if live.boot != handoff.boot_id {
|
|
Some(rmm_updater_handoff::UnknownReason::BootChanged)
|
|
} else if u64::try_from(live.now.timestamp())? >= handoff.finish_before {
|
|
Some(rmm_updater_handoff::UnknownReason::DeadlineExpired)
|
|
} else if live
|
|
.guard
|
|
.check_update_owner(&handoff, &proof.delegation, journal.phase(), &events)
|
|
.is_err()
|
|
{
|
|
Some(rmm_updater_handoff::UnknownReason::OwnerLost)
|
|
} else {
|
|
None
|
|
};
|
|
if let Some(reason) = reason {
|
|
ensure!(refresh, "UPDATE_OWNER_LOST_FRESH_CAPTURE_REQUIRED");
|
|
outcome_bytes = crate::update_handoff::mark_unknown_locked(
|
|
db,
|
|
&permit,
|
|
u64::try_from(live.now.timestamp())?,
|
|
reason,
|
|
)?;
|
|
update_proof = None;
|
|
}
|
|
}
|
|
}
|
|
let state = if outcome_bytes.is_empty() {
|
|
if update_proof.is_some() {
|
|
"running"
|
|
} else {
|
|
"unknown"
|
|
}
|
|
.to_string()
|
|
} else {
|
|
let o: ReconciledOutcome = serde_json::from_slice(&outcome_bytes)?;
|
|
o.state
|
|
};
|
|
ensure!(
|
|
matches!(
|
|
state.as_str(),
|
|
"unknown"
|
|
| "awaiting_reboot"
|
|
| "succeeded"
|
|
| "failed"
|
|
| "cancelled"
|
|
| "expired"
|
|
| "running"
|
|
),
|
|
"UNKNOWN_LOCAL_STATE"
|
|
);
|
|
ensure!(!original_boot_id.is_nil(), "LOCAL_ORIGINAL_BOOT_MISSING");
|
|
if state == "succeeded" {
|
|
let o: ReconciledOutcome = serde_json::from_slice(&outcome_bytes)?;
|
|
ensure!(
|
|
o.evidence["postconditions_passed"] == true,
|
|
"LOCAL_SUCCESS_PROOF_MISSING"
|
|
);
|
|
if let Action::RunScript {
|
|
expected_stdout, ..
|
|
} = &job.action
|
|
{
|
|
ensure!(
|
|
o.exit_code == Some(0)
|
|
&& expected_stdout
|
|
.as_ref()
|
|
.is_none_or(|s| !o.output_truncated && o.stdout == s.as_bytes()),
|
|
"LOCAL_SUCCESS_PROOF_MISSING"
|
|
);
|
|
}
|
|
}
|
|
let mut row = LocalAttempt {
|
|
update_handoff_bytes: update_proof
|
|
.map(|p| p.bytes().map_err(anyhow::Error::msg))
|
|
.transpose()?
|
|
.unwrap_or_default(),
|
|
job_id: job.job_id,
|
|
attempt_id: job.attempt_id,
|
|
job_payload_hash: hash(&request.job_bytes),
|
|
grant_id: permit.grant_id,
|
|
authorization_id: permit.authorization_id,
|
|
plan_hash: permit.plan_hash,
|
|
action: job.action.name().into(),
|
|
execution_identity: job.execution_identity,
|
|
revision: 0,
|
|
original_boot_id,
|
|
consumption: "consumed".into(),
|
|
state,
|
|
outcome_sha256: if outcome_bytes.is_empty() {
|
|
String::new()
|
|
} else {
|
|
hash(&outcome_bytes)
|
|
},
|
|
outcome_bytes,
|
|
original_request_sha256: hash(&serde_json::to_vec(&request)?),
|
|
};
|
|
let fingerprint = hash(&serde_json::to_vec(&row)?);
|
|
let old: Option<(u64, String)> = db
|
|
.query_row(
|
|
"SELECT revision,fingerprint FROM helper_sync_rows WHERE grant_id=?1",
|
|
[&grant],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)
|
|
.optional()?;
|
|
row.revision = match old {
|
|
Some((revision, old)) => {
|
|
if old == fingerprint {
|
|
revision
|
|
} else {
|
|
ensure!(refresh, "ROOT_ROW_CHANGED_WITHOUT_SNAPSHOT");
|
|
revision.checked_add(1).context("REVISION_OVERFLOW")?
|
|
}
|
|
}
|
|
None => {
|
|
ensure!(refresh, "ROOT_ROW_MISSING_WITHOUT_SNAPSHOT");
|
|
1
|
|
}
|
|
};
|
|
i64::try_from(row.revision)?;
|
|
if refresh {
|
|
db.execute("INSERT INTO helper_sync_rows(grant_id,revision,fingerprint) VALUES(?1,?2,?3) ON CONFLICT(grant_id) DO UPDATE SET revision=excluded.revision,fingerprint=excluded.fingerprint",params![grant,row.revision,fingerprint])?;
|
|
}
|
|
result.push(row);
|
|
}
|
|
let retained: u64 = db.query_row("SELECT count(*) FROM helper_sync_rows", [], |r| r.get(0))?;
|
|
ensure!(
|
|
retained == result.len() as u64,
|
|
"ROOT_CONSUMPTION_MISSING_LOCALLY"
|
|
);
|
|
Ok(result)
|
|
}
|
|
fn definitive(row: &LocalAttempt) -> bool {
|
|
matches!(
|
|
row.state.as_str(),
|
|
"succeeded" | "failed" | "cancelled" | "expired"
|
|
) && !row.outcome_bytes.is_empty()
|
|
}
|
|
fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<()> {
|
|
ensure!(
|
|
a.job_payload_hash == row.job_payload_hash
|
|
&& a.grant_id == row.grant_id.to_string()
|
|
&& a.consumption == "consumed"
|
|
&& a.state == row.state
|
|
&& a.revision >= row.revision
|
|
&& a.original_boot_id == row.original_boot_id.to_string()
|
|
&& a.outcome_sha256 == row.outcome_sha256
|
|
&& a.update_handoff_bytes == row.update_handoff_bytes
|
|
&& (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes),
|
|
"ROOT_SNAPSHOT_REQUEST_MISMATCH"
|
|
);
|
|
Ok(())
|
|
}
|
|
fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -> Result<()> {
|
|
if !row.update_handoff_bytes.is_empty() {
|
|
let proof =
|
|
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&row.update_handoff_bytes)
|
|
.map_err(anyhow::Error::msg)?;
|
|
let raw = proof.decode().map_err(anyhow::Error::msg)?;
|
|
let h: rmm_updater_handoff::Handoff = serde_json::from_slice(&raw.handoff_record)?;
|
|
let expected = rmm_domain::agent_update::UpdateInflightDecision {
|
|
handoff_sha256: proof.delegation.handoff_sha256.clone(),
|
|
delegation_sha256: hash(&serde_json::to_vec(&proof.delegation)?),
|
|
head_sha256: proof.head_sha256,
|
|
owner_nonce: h.owner_nonce,
|
|
finish_before: h.finish_before,
|
|
};
|
|
ensure!(
|
|
d.update_inflight.as_ref() == Some(&expected)
|
|
&& d.outcome_state == "running"
|
|
&& !d.unlock
|
|
&& d.outcome_bytes.is_empty()
|
|
&& d.outcome_sha256.is_none(),
|
|
"UPDATE_SIGNED_INFLIGHT_BINDING"
|
|
);
|
|
} else {
|
|
ensure!(d.update_inflight.is_none(), "UPDATE_INFLIGHT_NOT_LOCAL");
|
|
}
|
|
ensure!(
|
|
d.job_payload_hash == row.job_payload_hash
|
|
&& d.grant_id == Some(row.grant_id)
|
|
&& d.authorization_id == Some(row.authorization_id)
|
|
&& d.plan_hash.as_deref() == Some(row.plan_hash.as_str())
|
|
&& d.action == row.action
|
|
&& d.execution_identity == row.execution_identity
|
|
&& d.original_boot_id == Some(row.original_boot_id)
|
|
&& (d.revision >= a.revision || (!d.unlock && d.outcome_state == "unknown"))
|
|
&& d.previous_outcome_sha256.as_deref()
|
|
== (!row.outcome_sha256.is_empty()).then_some(row.outcome_sha256.as_str()),
|
|
"ROOT_SIGNED_ATTEMPT_BINDING"
|
|
);
|
|
ensure!(
|
|
!d.unlock
|
|
|| matches!(
|
|
d.outcome_state.as_str(),
|
|
"succeeded" | "failed" | "cancelled" | "expired"
|
|
),
|
|
"UNKNOWN_CANNOT_UNLOCK"
|
|
);
|
|
Ok(())
|
|
}
|
|
fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -> Result<Vec<u8>> {
|
|
let bytes = if d.outcome_bytes.is_empty() {
|
|
row.outcome_bytes.clone()
|
|
} else {
|
|
d.outcome_bytes.clone()
|
|
};
|
|
let o: ReconciledOutcome = serde_json::from_slice(&bytes)?;
|
|
ensure!(
|
|
canonical_outcome_bytes(&o)? == bytes
|
|
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
|
|
&& o.state == d.outcome_state,
|
|
"ROOT_CANONICAL_UNLOCK_OUTCOME"
|
|
);
|
|
if bytes != row.outcome_bytes {
|
|
ensure!(
|
|
row.action == "reboot"
|
|
&& matches!(row.state.as_str(), "unknown" | "awaiting_reboot")
|
|
&& o.state == "succeeded"
|
|
&& row.original_boot_id != current_boot
|
|
&& o.evidence["postconditions_passed"] == true
|
|
&& o.evidence["reconciled_after_reboot"] == true
|
|
&& o.evidence["new_boot_id"].as_str() == Some(current_boot.to_string().as_str())
|
|
&& o.evidence["inventory_message_id"]
|
|
.as_str()
|
|
.is_some_and(|s| Uuid::parse_str(s).is_ok_and(|id| !id.is_nil())),
|
|
"ROOT_REBOOT_NATIVE_BOOT_PROOF_REQUIRED"
|
|
);
|
|
} else {
|
|
ensure!(definitive(row), "UNKNOWN_LOCAL_EFFECT_CANNOT_UNLOCK");
|
|
if o.state == "succeeded" {
|
|
ensure!(
|
|
o.evidence["postconditions_passed"] == true,
|
|
"ROOT_POSTCONDITION_MISSING"
|
|
);
|
|
}
|
|
if matches!(o.state.as_str(), "cancelled" | "expired") {
|
|
ensure!(
|
|
o.evidence["not_started"] == true
|
|
|| (o.state == "cancelled" && o.evidence["abort_confirmed"] == true),
|
|
"ROOT_ABORT_NOT_PROVEN"
|
|
);
|
|
}
|
|
if o.state == "failed" {
|
|
ensure!(!o.error_code.is_empty(), "ROOT_FAILURE_NOT_PROVEN");
|
|
}
|
|
}
|
|
Ok(bytes)
|
|
}
|
|
|
|
// The independent helper verification deliberately retains this small
|
|
// bounded parser reads the EXISTING SyncRequest tags (1..12) and SyncAttempt
|
|
// tags (1..13), not a parallel public protocol. Glue passes the actual canonical
|
|
// prost bytes. Unknown/duplicate fields, wrong wire types and overlong varints
|
|
// fail closed. Nested JobResult/Envelope are not trusted as root outcome proof.
|
|
#[derive(Default)]
|
|
struct RequestView {
|
|
request_id: String,
|
|
journal_id: String,
|
|
checkpoint: u64,
|
|
assignment_version: u64,
|
|
attempts: Vec<AttemptView>,
|
|
journal_complete: bool,
|
|
helper_journal_id: String,
|
|
helper_checkpoint: u64,
|
|
helper_previous_decision_id: String,
|
|
previous_decision_id: String,
|
|
previous_request_id: String,
|
|
}
|
|
#[derive(Default)]
|
|
struct AttemptView {
|
|
job_id: String,
|
|
attempt_id: String,
|
|
job_payload_hash: String,
|
|
grant_id: String,
|
|
consumption: String,
|
|
state: String,
|
|
revision: u64,
|
|
original_boot_id: String,
|
|
outcome_sha256: String,
|
|
outcome_bytes: Vec<u8>,
|
|
update_handoff_bytes: Vec<u8>,
|
|
}
|
|
enum Field<'a> {
|
|
Number(u64),
|
|
Bytes(&'a [u8]),
|
|
}
|
|
fn varint(bytes: &[u8], offset: &mut usize) -> Result<u64> {
|
|
let mut n = 0u64;
|
|
for i in 0..10 {
|
|
let b = *bytes.get(*offset).context("PROTOBUF_TRUNCATED")?;
|
|
*offset += 1;
|
|
ensure!(i != 9 || b <= 1, "PROTOBUF_VARINT_OVERFLOW");
|
|
n |= u64::from(b & 127) << (7 * i);
|
|
if b & 128 == 0 {
|
|
ensure!(i == 0 || b != 0, "PROTOBUF_NONCANONICAL_VARINT");
|
|
return Ok(n);
|
|
}
|
|
}
|
|
anyhow::bail!("PROTOBUF_VARINT_OVERFLOW")
|
|
}
|
|
fn fields(bytes: &[u8]) -> Result<Vec<(u32, Field<'_>)>> {
|
|
let mut offset = 0;
|
|
let mut result = Vec::new();
|
|
while offset < bytes.len() {
|
|
ensure!(result.len() < 2048, "PROTOBUF_FIELD_LIMIT");
|
|
let tag = varint(bytes, &mut offset)?;
|
|
let number = u32::try_from(tag >> 3)?;
|
|
ensure!(number > 0, "PROTOBUF_ZERO_TAG");
|
|
let value = match tag & 7 {
|
|
0 => Field::Number(varint(bytes, &mut offset)?),
|
|
2 => {
|
|
let len = usize::try_from(varint(bytes, &mut offset)?)?;
|
|
let end = offset
|
|
.checked_add(len)
|
|
.context("PROTOBUF_LENGTH_OVERFLOW")?;
|
|
let value = bytes.get(offset..end).context("PROTOBUF_TRUNCATED")?;
|
|
offset = end;
|
|
Field::Bytes(value)
|
|
}
|
|
_ => anyhow::bail!("PROTOBUF_WIRE_TYPE"),
|
|
};
|
|
result.push((number, value));
|
|
}
|
|
Ok(result)
|
|
}
|
|
fn text(field: Field<'_>) -> Result<String> {
|
|
match field {
|
|
Field::Bytes(b) => {
|
|
ensure!(b.len() <= 1024, "PROTOBUF_STRING_SIZE");
|
|
Ok(std::str::from_utf8(b)?.to_string())
|
|
}
|
|
_ => anyhow::bail!("PROTOBUF_EXPECTED_STRING"),
|
|
}
|
|
}
|
|
fn number(field: Field<'_>) -> Result<u64> {
|
|
match field {
|
|
Field::Number(n) => Ok(n),
|
|
_ => anyhow::bail!("PROTOBUF_EXPECTED_NUMBER"),
|
|
}
|
|
}
|
|
fn bytes(field: Field<'_>) -> Result<Vec<u8>> {
|
|
match field {
|
|
Field::Bytes(b) => Ok(b.to_vec()),
|
|
_ => anyhow::bail!("PROTOBUF_EXPECTED_BYTES"),
|
|
}
|
|
}
|
|
impl RequestView {
|
|
fn decode(raw: &[u8]) -> Result<Self> {
|
|
ensure!(raw.len() <= 1024 * 1024, "SYNC_REQUEST_SIZE");
|
|
let mut r = Self::default();
|
|
let mut seen = BTreeSet::new();
|
|
let mut receipts = 0usize;
|
|
for (tag, value) in fields(raw)? {
|
|
ensure!(
|
|
(1..=12).contains(&tag) && (matches!(tag, 6 | 7) || seen.insert(tag)),
|
|
"PROTOBUF_UNKNOWN_OR_DUPLICATE_FIELD"
|
|
);
|
|
match tag {
|
|
1 => r.request_id = text(value)?,
|
|
2 => r.journal_id = text(value)?,
|
|
3 => r.checkpoint = number(value)?,
|
|
4 | 12 => {
|
|
let s = text(value)?;
|
|
if !s.is_empty() {
|
|
Uuid::parse_str(&s)?;
|
|
}
|
|
if tag == 4 {
|
|
r.previous_decision_id = s;
|
|
} else {
|
|
r.previous_request_id = s;
|
|
}
|
|
}
|
|
5 => r.assignment_version = number(value)?,
|
|
6 => {
|
|
ensure!(r.attempts.len() < 256, "SYNC_ATTEMPT_LIMIT");
|
|
r.attempts.push(AttemptView::decode(&bytes(value)?)?);
|
|
}
|
|
7 => {
|
|
receipts += 1;
|
|
ensure!(receipts <= 256, "SYNC_RECEIPT_LIMIT");
|
|
let s = text(value)?;
|
|
ensure!(!Uuid::parse_str(&s)?.is_nil(), "SYNC_RECEIPT_ID");
|
|
}
|
|
8 => {
|
|
let n = number(value)?;
|
|
ensure!(n <= 1, "PROTOBUF_BOOL");
|
|
r.journal_complete = n == 1;
|
|
}
|
|
9 => r.helper_journal_id = text(value)?,
|
|
10 => r.helper_checkpoint = number(value)?,
|
|
11 => r.helper_previous_decision_id = text(value)?,
|
|
_ => unreachable!(),
|
|
}
|
|
}
|
|
ensure!(
|
|
!Uuid::parse_str(&r.request_id)?.is_nil() && !Uuid::parse_str(&r.journal_id)?.is_nil(),
|
|
"SYNC_REQUEST_IDENTITIES"
|
|
);
|
|
i64::try_from(r.checkpoint)?;
|
|
i64::try_from(r.helper_checkpoint)?;
|
|
Ok(r)
|
|
}
|
|
}
|
|
impl AttemptView {
|
|
fn decode(raw: &[u8]) -> Result<Self> {
|
|
let mut r = Self::default();
|
|
let mut seen = BTreeSet::new();
|
|
for (tag, value) in fields(raw)? {
|
|
ensure!(
|
|
(1..=14).contains(&tag) && seen.insert(tag),
|
|
"PROTOBUF_ATTEMPT_FIELD"
|
|
);
|
|
match tag {
|
|
1 => r.job_id = text(value)?,
|
|
2 => r.attempt_id = text(value)?,
|
|
3 => r.job_payload_hash = text(value)?,
|
|
4 => r.grant_id = text(value)?,
|
|
5 => r.consumption = text(value)?,
|
|
6 => r.state = text(value)?,
|
|
7 => r.revision = number(value)?,
|
|
8 => r.original_boot_id = text(value)?,
|
|
9 | 13 => {
|
|
ensure!(bytes(value)?.len() <= 1024 * 1024, "SYNC_RESULT_SIZE");
|
|
}
|
|
10 => {
|
|
let _ = text(value)?;
|
|
}
|
|
11 => r.outcome_sha256 = text(value)?,
|
|
12 => {
|
|
r.outcome_bytes = bytes(value)?;
|
|
ensure!(r.outcome_bytes.len() <= 512 * 1024, "SYNC_OUTCOME_SIZE");
|
|
}
|
|
14 => {
|
|
r.update_handoff_bytes = bytes(value)?;
|
|
rmm_domain::agent_update::UpdateInflightProof::from_bytes(
|
|
&r.update_handoff_bytes,
|
|
)
|
|
.map_err(anyhow::Error::msg)?;
|
|
}
|
|
_ => unreachable!(),
|
|
}
|
|
}
|
|
i64::try_from(r.revision)?;
|
|
Ok(r)
|
|
}
|
|
}
|
|
fn unique_summaries(rows: &[AttemptView]) -> Result<BTreeMap<(Uuid, Uuid), &AttemptView>> {
|
|
let mut map = BTreeMap::new();
|
|
for row in rows {
|
|
let id = (
|
|
Uuid::parse_str(&row.job_id)?,
|
|
Uuid::parse_str(&row.attempt_id)?,
|
|
);
|
|
ensure!(
|
|
!id.0.is_nil() && !id.1.is_nil() && map.insert(id, row).is_none(),
|
|
"DUPLICATE_REQUEST_ATTEMPT"
|
|
);
|
|
}
|
|
Ok(map)
|
|
}
|