Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s

This commit is contained in:
2026-11-16 15:25:46 +01:00
parent 6497c79a91
commit 96a2b6e5ae
44 changed files with 6986 additions and 1024 deletions
+175 -18
View File
@@ -6,7 +6,7 @@ use crate::{
bridge::{BridgeContext, FullBridge, Generation},
bundle, census,
clock::NativeClock,
fence::LedgerFence,
fence::{LedgerFence, RollbackFence},
ledger::{OwnedInputs, OwnedLedger},
original::Original,
participants::{StableParticipant, verify_original_helper},
@@ -31,7 +31,7 @@ use rmm_updater_linux::{
Budget, DurableEffectFence, Effect,
files::{Directory, HeldFile, LedgerGuard, Role},
ipc::{self, Request, Response, State},
services::{Service, ServicePolicy, ServiceProof, Systemd},
services::{Service, ServicePolicy, ServiceProof, Startup, Systemd},
};
use std::{ffi::OsStr, path::Path, time::Duration};
use uuid::Uuid;
@@ -97,6 +97,12 @@ struct Queued {
budget: Budget,
}
impl Runtime {
pub fn failed(&self) -> bool {
self.failed
}
pub fn terminal(&self) -> bool {
self.terminal
}
pub fn new(binding: Binding) -> Result<Self> {
let clock = NativeClock::new()?;
let budget = observation_budget()?;
@@ -155,7 +161,7 @@ impl Runtime {
let services = Systemd::new(self.old_policies.clone())?;
let budget = observation_budget()?;
if let Err(error) = listener.serve_one(&services, &budget, self) {
if self.queued.take().is_some() {
if self.queued.take().is_some() || self.accepted {
self.failed = true;
}
return Err(error);
@@ -226,6 +232,7 @@ impl Runtime {
.verify_delegation(&header.delegation, &budget)?;
let reply = {
let mut bridge = FullBridge::original(BridgeContext {
binding: &self.binding,
handoff: &header.handoff,
delegation: &header.delegation,
services: &services,
@@ -235,7 +242,19 @@ impl Runtime {
health_public_key: self.health_key,
generation: Generation::Original,
})?;
bridge.call(NativeControlOperation::ReadOriginal)?
loop {
let reply = bridge.call(NativeControlOperation::ReadOriginal)?;
if reply.pre_stop_sync.is_some() {
break reply;
}
// Passive read only, under the ORIGINAL boot/deadline/owner.
// Do not stop a reader before the server has retained and the
// actual Full has applied this exact signed inflight context.
budget.check()?;
self.updater
.verify_delegation(&header.delegation, &budget)?;
std::thread::sleep(Duration::from_millis(50));
}
};
let proof = reply.proof.context("UPDATE_ENGINE_ORIGINAL_PROOF")?;
let original = Original::verify(
@@ -281,6 +300,7 @@ impl Runtime {
census::scan_admission(&targets, &known, self.registry.host_context(), &budget)?;
let mut bridge = FullBridge::new(
BridgeContext {
binding: &self.binding,
handoff: original.handoff(),
delegation: original.delegation(),
services: &services,
@@ -381,9 +401,10 @@ impl Runtime {
ledger_identity_sha256: witnesses.ledger.sha256()?,
})?;
// Do not hold a second connection before all readers really stopped.
let swap = (|| -> Result<()> {
let swap = (|| -> Result<Vec<rmm_updater_linux::files::Installed>> {
let stages = witnesses.files.reopen_stages(original.handoff(), &budget)?;
let mut fence = LedgerFence::new(&mut ledger)?;
let mut installed = Vec::new();
for (stage, role) in
stages
.into_iter()
@@ -396,23 +417,27 @@ impl Runtime {
0,
false,
)?;
stage.install(&directory, &os, &budget, &mut fence)?;
installed.push(stage.install(&directory, &os, &budget, &mut fence)?);
}
fence.finish()?;
ledger.append_observed(Evidence::PairVerified {
pair: original.handoff().target.clone(),
retained_state_sha256: original.handoff().retained_state_sha256.clone(),
})
})?;
Ok(installed)
})();
if let Err(error) = swap {
// Exactly one best-effort ORIGINAL Unknown CAS while we still own
// the real guard. Failure is preserved; no restoration or retry.
let _ = ledger.mark_unknown(
u64::try_from(self.clock.now()?.timestamp())?,
rmm_updater_handoff::UnknownReason::ReplaceUncertain,
);
return Err(error);
}
let installed = match swap {
Ok(installed) => installed,
Err(error) => {
// Exactly one best-effort ORIGINAL Unknown CAS while we still own
// the real guard. Failure is preserved; no restoration or retry.
let _ = ledger.mark_unknown(
u64::try_from(self.clock.now()?.timestamp())?,
rmm_updater_handoff::UnknownReason::ReplaceUncertain,
);
return Err(error);
}
};
let prepared = ledger.prepare_return_and_close()?;
// The original DB is actually closed. End the continuous census/lock
// borrow before the same concrete guard can be consumed by Return.
@@ -432,6 +457,7 @@ impl Runtime {
let observer = Systemd::new(target_policies)?;
let mut bridge = FullBridge::new(
BridgeContext {
binding: &self.binding,
handoff: original.handoff(),
delegation: original.delegation(),
services: &observer,
@@ -447,8 +473,139 @@ impl Runtime {
agent_startup.verify()?;
os.release(&budget, &mut fence)?;
agent_startup.return_after_root_release(&budget)?;
let started = stopped.start_once(target, &budget, &mut fence)?;
fence.finish()?;
let started = match stopped.start_once_observed(target, &budget, &mut fence)? {
Startup::Complete { started, .. } => {
fence.finish()?;
started
}
Startup::KnownFailed(failed) => {
let mut conditional = fence.early_full_failure(&failed)?;
let expected_head = conditional.proof().head_sha256.clone();
let (stopped, _, _) = failed.stop_reserved_general(&budget, &mut conditional)?;
conditional.finish()?;
drop(bridge);
let agent_startup =
BoundAgentLock::bind(&witnesses.retained)?.acquire(&stopped, &budget)?;
let os = LedgerGuard::acquire(&stopped, &witnesses.ledger, &budget)?;
self.registry
.verify_known_failure_stopped(&self.policy, &stopped, &budget)?;
let readers = census::scan_stopped(
&targets,
&admission,
&os,
&agent_startup,
&stopped,
self.updater.held_service(),
&budget,
)?;
let readers = readers.bind(
census::ClosureInputs {
targets: &targets,
root: &os,
agent: &agent_startup,
stopped: &stopped,
updater: self.updater.held_service(),
registry: &self.registry,
},
&budget,
)?;
let mut ledger = OwnedLedger::open(OwnedInputs {
os: &os,
readers: &readers,
original: &original,
binding: &self.binding,
clock: &self.clock,
budget: &budget,
updater: &self.updater,
retained: &witnesses.retained,
files: &witnesses.files,
stopped: &stopped,
trust: &trust,
})?;
ledger.begin_rollback_after_early_full_failure(&expected_head)?;
ledger.append_rollback_stopped()?;
{
let mut rollback = RollbackFence::new(&mut ledger)?;
for installed in installed.into_iter().rev() {
let directory = Directory::open(
Path::new(installed.role().target())
.parent()
.context("UPDATE_ROLLBACK_PARENT")?,
0,
false,
)?;
installed.restore(&directory, &os, &budget, &mut rollback)?;
}
rollback.finish()?;
}
ledger.append_observed(Evidence::PredecessorVerified {
pair: original.handoff().predecessor.clone(),
retained_state_sha256: original.handoff().retained_state_sha256.clone(),
predecessor_record_sha256: original.handoff().predecessor_record_sha256.clone(),
})?;
let prepared = ledger.prepare_rollback_return_and_close()?;
drop(readers);
let reservation = prepared.bind(returned::Inputs {
original: &original,
binding: &self.binding,
clock: &self.clock,
budget: &budget,
updater: &self.updater,
retained: &witnesses.retained,
files: &witnesses.files,
trust: &trust,
})?;
let predecessor = Systemd::new(self.old_policies.clone())?;
let observer = Systemd::new(self.old_policies.clone())?;
let mut restored = FullBridge::new(
BridgeContext {
binding: &self.binding,
handoff: original.handoff(),
delegation: original.delegation(),
services: &observer,
updater: &self.updater,
clock: &self.clock,
budget: &budget,
health_public_key: self.health_key,
generation: Generation::Restored,
},
reservation.proof().clone(),
)?;
let mut fence = reservation.start_fence(&observer, &mut restored);
agent_startup.verify()?;
os.release(&budget, &mut fence)?;
agent_startup.return_after_root_release(&budget)?;
let started = stopped.start_once(predecessor, &budget, &mut fence)?;
fence.finish()?;
restored.observe_starts(&started)?;
loop {
original.check_current(&self.binding, &self.clock, &budget)?;
self.updater
.verify_delegation(original.delegation(), &budget)?;
witnesses.retained.verify()?;
for service in &started {
service.verify(&budget)?;
}
let reply = restored.call(NativeControlOperation::ReadOriginal)?;
let proof = reply.proof.context("UPDATE_ROLLBACK_HEALTH_PROOF")?;
self.head = Some(proof.head_sha256.clone());
let (_, journal) =
rmm_domain::agent_update::verify_inflight_proof(&proof, self.health_key)
.map_err(anyhow::Error::msg)?;
match journal.phase() {
Phase::RollbackServicesStarted => {
std::thread::sleep(Duration::from_millis(100))
}
Phase::RollbackHealthVerified => {
restored.call(NativeControlOperation::Finalize)?;
self.head = None;
return Ok(());
}
_ => anyhow::bail!("UPDATE_ROLLBACK_HEALTH_PREFIX_CHANGED"),
}
}
}
};
bridge.observe_starts(&started)?;
loop {
original.check_current(&self.binding, &self.clock, &budget)?;