Add gated agent update planning, preserved handoff and Linux worker ports
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-14 21:48:21 +01:00
parent dc9cb1b096
commit 1e89c4577f
82 changed files with 25052 additions and 69 deletions
+280 -10
View File
@@ -26,6 +26,23 @@ pub trait BootSource {
/// The guard remains held through snapshot/apply/admission/consumption/effect.
pub trait ExecutorLedgerGuard {
fn check(&self, db: &Connection) -> Result<()>;
/// Only the specifically provisioned update authorizer may judge a live
/// owner. General readers stay closed without changing the original attempt.
fn is_update_owner_authority(&self) -> bool {
false
}
/// Default-deny live owner attestation for the one typed update handoff.
/// Implement only after qualifying fixed service/PID/birth/image and the
/// original immutable delegation under this SAME actual executor guard.
fn check_update_owner(
&self,
_handoff: &rmm_updater_handoff::Handoff,
_delegation: &rmm_domain::agent_update::UpdateDelegation,
_phase: rmm_updater_handoff::Phase,
_events: &[rmm_updater_handoff::Event],
) -> Result<()> {
anyhow::bail!("UPDATE_OWNER_NOT_QUALIFIED")
}
}
/// OS adapter contract, not an IPC assertion. The implementation must pin the
/// live helper PID/creation and ledger/lock file identities, and independently
@@ -120,6 +137,8 @@ pub struct LocalAttempt {
pub outcome_bytes: Vec<u8>,
/// Hash of the exact root-owned Request containing original job/permit bytes.
pub original_request_sha256: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub update_handoff_bytes: Vec<u8>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@@ -238,6 +257,78 @@ impl<V: Ed25519Verifier> HelperSync<V> {
self.verify_no_start_anchor(db, guard, &anchor, clock, boot)?;
Ok(anchor)
}
/// Consume exactly the approved update attempt and delegate its ORIGINAL
/// effect once. This is the only entry into update handoff persistence;
/// the independent live helper gate and all existing checks remain intact.
pub fn begin_update_handoff(
&mut self,
db: &mut Connection,
guard: &impl crate::update_handoff::UpdateAdmissionGuard,
generation: u64,
prepared: &crate::update_handoff::PreparedHandoff<'_>,
clock: &impl Clock,
boot: &impl BootSource,
) -> Result<()> {
let crate::update_handoff::PreparedHandoff {
request,
handoff,
recorded_at,
..
} = *prepared;
ensure!(
self.can_start(db, guard, generation, clock, boot),
"UPDATE_HELPER_READY_REQUIRED"
);
let now = clock.now()?;
ensure!(
recorded_at <= now && now - recorded_at <= chrono::Duration::seconds(1),
"UPDATE_STALE_PREPARED_HANDOFF"
);
let (job, permit) = self.binding.verify_with(request, now, &self.verifier)?;
ensure!(
matches!(job.action, Action::UpdateAgent { .. })
&& job.execution_identity == self.identity.as_str(),
"UPDATE_HELPER_IDENTITY"
);
let meta = metadata(db)?;
let lease = self
.lease
.as_ref()
.context("UPDATE_HELPER_READY_REQUIRED")?;
ensure!(
handoff.helper_journal_id == meta.journal_id
&& handoff.helper_checkpoint == meta.checkpoint
&& Some(handoff.agent_journal_id) == meta.agent_journal
&& Some(handoff.previous_decision_id) == meta.decision
&& handoff.previous_decision_id == lease.decision_id
&& handoff.connection_generation == generation
&& handoff.boot_id == boot.current_boot()?,
"UPDATE_HANDOFF_SYNC_BINDING"
);
let public_key = guard.health_public_key()?;
ensure!(
rmm_domain::agent_update::job_health_public_key(&job).map_err(anyhow::Error::msg)?
== public_key,
"UPDATE_HEALTH_PIN_BINDING"
);
crate::update_handoff::consume(db, guard, &permit, prepared, public_key, &|tx| {
let fresh_now = clock.now()?;
ensure!(
self.can_start(tx, guard, generation, clock, boot)
&& fresh_now >= recorded_at
&& fresh_now - recorded_at <= chrono::Duration::seconds(1)
&& boot.current_boot()? == handoff.boot_id,
"UPDATE_ADMISSION_CHANGED_BEFORE_CONSUMPTION"
);
let (_, fresh) = self
.binding
.verify_with(request, fresh_now, &self.verifier)?;
ensure!(fresh == permit, "UPDATE_ORIGINAL_GRANT_CHANGED");
Ok(())
})?;
self.close();
Ok(())
}
/// Deny-only: the grant may be expired, but no historical timestamp is ever
/// passed to an execute verifier. Existing consumption is immutable. A new
/// c/start/no-effect outcome is committed atomically and remains locked for
@@ -410,7 +501,18 @@ impl<V: Ed25519Verifier> HelperSync<V> {
ensure!(meta.checkpoint == highest, "HELPER_CHECKPOINT_ROLLBACK");
ensure!(generation >= meta.generation, "STALE_HELPER_CONNECTION");
validate_history(&tx, &self.binding, &meta, generation, &self.verifier)?;
let attempts = load_rows(&tx, &self.binding, true, self.identity, &self.verifier)?;
let attempts = load_rows(
&tx,
&self.binding,
true,
self.identity,
&self.verifier,
Some(LiveUpdateContext {
guard,
now,
boot: boot_id,
}),
)?;
ensure!(attempts.len() <= 256, "HELPER_SYNC_CAPACITY");
let fingerprint = ledger_fingerprint(&tx)?;
// Two independently gated profiles share one signed helper checkpoint.
@@ -512,6 +614,14 @@ impl<V: Ed25519Verifier> HelperSync<V> {
.as_ref()
.context("FRESH_PROCESS_SNAPSHOT_REQUIRED")?;
let decision = verify(&self.verifier, &self.binding, response, now)?;
ensure!(
!decision
.attempts
.iter()
.any(|d| d.update_inflight.is_some())
|| (!decision.ready && !decision.helper_ready),
"UPDATE_INFLIGHT_CANNOT_READY"
);
let request = RequestView::decode(raw_request)?;
let current_boot = boot.current_boot()?;
ensure!(
@@ -571,7 +681,7 @@ impl<V: Ed25519Verifier> HelperSync<V> {
if let Some(old)=tx.query_row("SELECT response,request,fingerprint FROM helper_sync_decisions WHERE decision_id=?1",[decision.decision_id.to_string()],|r|Ok((r.get::<_,Vec<u8>>(0)?,r.get::<_,Vec<u8>>(1)?,r.get::<_,String>(2)?))).optional()? {
ensure!(old.0==serde_json::to_vec(response)? && old.1==raw_request && meta.decision==Some(decision.decision_id)
&& ledger_fingerprint(&tx)?==old.2,"HELPER_REPLAY_OR_LEDGER_CHANGED");
let rows=load_rows(&tx,&self.binding,false, self.identity, &self.verifier)?;
let rows=load_rows(&tx,&self.binding,false, self.identity, &self.verifier,Some(LiveUpdateContext {guard,now,boot:current_boot}))?;
let ready=decision.ready && decision.helper_ready && rows.iter().all(|row|definitive(row)
&& decision.attempts.iter().any(|d|d.job_id==row.job_id && d.attempt_id==row.attempt_id
&& d.outcome_state==row.state && d.outcome_sha256.as_deref()==Some(row.outcome_sha256.as_str())))
@@ -591,7 +701,18 @@ impl<V: Ed25519Verifier> HelperSync<V> {
ledger_fingerprint(&tx)? == capture.fingerprint,
"LOCAL_LEDGER_CHANGED_DURING_SYNC"
);
let rows = load_rows(&tx, &self.binding, false, self.identity, &self.verifier)?;
let rows = load_rows(
&tx,
&self.binding,
false,
self.identity,
&self.verifier,
Some(LiveUpdateContext {
guard,
now,
boot: current_boot,
}),
)?;
ensure!(
serde_json::to_vec(&rows)? == serde_json::to_vec(&capture.snapshot.attempts)?,
"LOCAL_SNAPSHOT_CHANGED_DURING_SYNC"
@@ -658,7 +779,18 @@ impl<V: Ed25519Verifier> HelperSync<V> {
!decision.attempts.iter().any(|d| recovery_reason(&d.reason)),
"SIGNED_LEDGER_RECOVERY_REQUIRED"
);
let after = load_rows(&tx, &self.binding, true, self.identity, &self.verifier)?;
let after = load_rows(
&tx,
&self.binding,
true,
self.identity,
&self.verifier,
Some(LiveUpdateContext {
guard,
now,
boot: current_boot,
}),
)?;
let fingerprint = ledger_fingerprint(&tx)?;
tx.execute("INSERT INTO helper_sync_decisions(decision_id,checkpoint,response,request,fingerprint) VALUES(?1,?2,?3,?4,?5)",params![decision.decision_id.to_string(),decision.helper_checkpoint,serde_json::to_vec(response)?,raw_request,fingerprint])?;
tx.execute("UPDATE helper_sync_meta SET last_decision=?1,fault=COALESCE(fault,?2),agent_journal_id=?3,agent_checkpoint=?4 WHERE id=1",params![decision.decision_id.to_string(),fault,decision.journal_id.to_string(),decision.checkpoint])?;
@@ -988,7 +1120,7 @@ pub fn verify_checkpoint_repair_history_with(
previous_checkpoint = checkpoint;
previous_agent_checkpoint = decision.checkpoint;
}
let _ = load_rows(db, binding, false, identity, verifier)?;
let _ = load_rows(db, binding, false, identity, verifier, None)?;
if let Some((r, e)) = links {
verify_checkpoint_repair_links(db, binding, r, e, verifier)?;
}
@@ -1129,18 +1261,62 @@ fn ledger_rows(db: &Connection) -> Result<Vec<LedgerRow>> {
.collect::<rusqlite::Result<_>>()?)
}
fn ledger_fingerprint(db: &Connection) -> Result<String> {
Ok(hash(&serde_json::to_vec(&ledger_rows(db)?)?))
let rows = ledger_rows(db)?;
let exists:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='agent_update_handoffs')",[],|r|r.get(0))?;
if exists {
let mut statement=db.prepare("SELECT h.grant_id,h.record_sha256,h.head_sha256,h.event_count,h.phase,d.witness_sha256 FROM agent_update_handoffs h JOIN agent_update_delegations d USING(grant_id) ORDER BY h.grant_id")?;
type UpdateRow = (String, String, String, u32, String, String);
let updates = statement
.query_map([], |r| {
Ok((
r.get(0)?,
r.get(1)?,
r.get(2)?,
r.get(3)?,
r.get(4)?,
r.get(5)?,
))
})?
.collect::<rusqlite::Result<Vec<UpdateRow>>>()?;
if !updates.is_empty() {
let mut claims=db.prepare("SELECT grant_id,effect_key,claim,claim_sha256 FROM agent_update_native_claims ORDER BY grant_id,effect_key")?;
type NativeClaimRow = (String, String, Vec<u8>, String);
let claims = claims
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?
.collect::<rusqlite::Result<Vec<NativeClaimRow>>>()?;
let mut observations=db.prepare("SELECT grant_id,effect_key,claim_sha256,observation_sha256,observed_at FROM agent_update_native_observations ORDER BY grant_id,effect_key")?;
type NativeObservationRow = (String, String, String, String, String);
let observations = observations
.query_map([], |r| {
Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?))
})?
.collect::<rusqlite::Result<Vec<NativeObservationRow>>>()?;
return Ok(hash(&serde_json::to_vec(&(
rows,
updates,
claims,
observations,
))?));
}
}
Ok(hash(&serde_json::to_vec(&rows)?))
}
fn canonical(bytes: &[u8]) -> Result<Vec<u8>> {
let o: ReconciledOutcome = serde_json::from_slice(bytes)?;
Ok(canonical_outcome_bytes(&o)?)
}
struct LiveUpdateContext<'a> {
guard: &'a dyn ExecutorLedgerGuard,
now: DateTime<Utc>,
boot: Uuid,
}
fn load_rows(
db: &Connection,
binding: &Binding,
refresh: bool,
identity: HelperIdentity,
verifier: &impl Ed25519Verifier,
live: Option<LiveUpdateContext<'_>>,
) -> Result<Vec<LocalAttempt>> {
let mut result = Vec::new();
for (grant, agent, generation, job_id, attempt_id, finished, start, last) in ledger_rows(db)? {
@@ -1169,15 +1345,66 @@ fn load_rows(
if let (Some(a), Some(b)) = (&finished, &last) {
ensure!(canonical(a)? == canonical(b)?, "LOCAL_OUTCOME_CONFLICT");
}
let outcome_bytes = last
let mut outcome_bytes = last
.as_deref()
.or(finished.as_deref())
.map(canonical)
.transpose()?
.unwrap_or_default();
ensure!(outcome_bytes.len() <= 512 * 1024, "LOCAL_OUTCOME_SIZE");
let mut update_proof =
if outcome_bytes.is_empty() && matches!(job.action, Action::UpdateAgent { .. }) {
crate::update_handoff::proof_locked(db, permit.grant_id)?
} else {
None
};
if let Some(proof) = &update_proof {
let key = rmm_domain::agent_update::job_health_public_key(&job)
.map_err(anyhow::Error::msg)?;
let (handoff, journal) = crate::update_handoff::verify_inflight_proof(proof, key)?;
if let Some(live) = &live {
ensure!(
live.guard.is_update_owner_authority(),
"UPDATE_OWNER_AUTHORITY_REQUIRED"
);
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
let events = decoded
.events
.iter()
.map(|raw| serde_json::from_slice::<rmm_updater_handoff::Event>(raw))
.collect::<std::result::Result<Vec<_>, _>>()?;
let reason = if live.boot != handoff.boot_id {
Some(rmm_updater_handoff::UnknownReason::BootChanged)
} else if u64::try_from(live.now.timestamp())? >= handoff.finish_before {
Some(rmm_updater_handoff::UnknownReason::DeadlineExpired)
} else if live
.guard
.check_update_owner(&handoff, &proof.delegation, journal.phase(), &events)
.is_err()
{
Some(rmm_updater_handoff::UnknownReason::OwnerLost)
} else {
None
};
if let Some(reason) = reason {
ensure!(refresh, "UPDATE_OWNER_LOST_FRESH_CAPTURE_REQUIRED");
outcome_bytes = crate::update_handoff::mark_unknown_locked(
db,
&permit,
u64::try_from(live.now.timestamp())?,
reason,
)?;
update_proof = None;
}
}
}
let state = if outcome_bytes.is_empty() {
"unknown".to_string()
if update_proof.is_some() {
"running"
} else {
"unknown"
}
.to_string()
} else {
let o: ReconciledOutcome = serde_json::from_slice(&outcome_bytes)?;
o.state
@@ -1185,7 +1412,13 @@ fn load_rows(
ensure!(
matches!(
state.as_str(),
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
"unknown"
| "awaiting_reboot"
| "succeeded"
| "failed"
| "cancelled"
| "expired"
| "running"
),
"UNKNOWN_LOCAL_STATE"
);
@@ -1210,6 +1443,10 @@ fn load_rows(
}
}
let mut row = LocalAttempt {
update_handoff_bytes: update_proof
.map(|p| p.bytes().map_err(anyhow::Error::msg))
.transpose()?
.unwrap_or_default(),
job_id: job.job_id,
attempt_id: job.attempt_id,
job_payload_hash: hash(&request.job_bytes),
@@ -1280,12 +1517,37 @@ fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<()> {
&& a.revision >= row.revision
&& a.original_boot_id == row.original_boot_id.to_string()
&& a.outcome_sha256 == row.outcome_sha256
&& a.update_handoff_bytes == row.update_handoff_bytes
&& (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes),
"ROOT_SNAPSHOT_REQUEST_MISMATCH"
);
Ok(())
}
fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -> Result<()> {
if !row.update_handoff_bytes.is_empty() {
let proof =
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&row.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let raw = proof.decode().map_err(anyhow::Error::msg)?;
let h: rmm_updater_handoff::Handoff = serde_json::from_slice(&raw.handoff_record)?;
let expected = rmm_domain::agent_update::UpdateInflightDecision {
handoff_sha256: proof.delegation.handoff_sha256.clone(),
delegation_sha256: hash(&serde_json::to_vec(&proof.delegation)?),
head_sha256: proof.head_sha256,
owner_nonce: h.owner_nonce,
finish_before: h.finish_before,
};
ensure!(
d.update_inflight.as_ref() == Some(&expected)
&& d.outcome_state == "running"
&& !d.unlock
&& d.outcome_bytes.is_empty()
&& d.outcome_sha256.is_none(),
"UPDATE_SIGNED_INFLIGHT_BINDING"
);
} else {
ensure!(d.update_inflight.is_none(), "UPDATE_INFLIGHT_NOT_LOCAL");
}
ensure!(
d.job_payload_hash == row.job_payload_hash
&& d.grant_id == Some(row.grant_id)
@@ -1389,6 +1651,7 @@ struct AttemptView {
original_boot_id: String,
outcome_sha256: String,
outcome_bytes: Vec<u8>,
update_handoff_bytes: Vec<u8>,
}
enum Field<'a> {
Number(u64),
@@ -1517,7 +1780,7 @@ impl AttemptView {
let mut seen = BTreeSet::new();
for (tag, value) in fields(raw)? {
ensure!(
(1..=13).contains(&tag) && seen.insert(tag),
(1..=14).contains(&tag) && seen.insert(tag),
"PROTOBUF_ATTEMPT_FIELD"
);
match tag {
@@ -1540,6 +1803,13 @@ impl AttemptView {
r.outcome_bytes = bytes(value)?;
ensure!(r.outcome_bytes.len() <= 512 * 1024, "SYNC_OUTCOME_SIZE");
}
14 => {
r.update_handoff_bytes = bytes(value)?;
rmm_domain::agent_update::UpdateInflightProof::from_bytes(
&r.update_handoff_bytes,
)
.map_err(anyhow::Error::msg)?;
}
_ => unreachable!(),
}
}