Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -6,7 +6,7 @@ use crate::{
|
||||
bridge::{BridgeContext, FullBridge, Generation},
|
||||
bundle, census,
|
||||
clock::NativeClock,
|
||||
fence::LedgerFence,
|
||||
fence::{LedgerFence, RollbackFence},
|
||||
ledger::{OwnedInputs, OwnedLedger},
|
||||
original::Original,
|
||||
participants::{StableParticipant, verify_original_helper},
|
||||
@@ -31,7 +31,7 @@ use rmm_updater_linux::{
|
||||
Budget, DurableEffectFence, Effect,
|
||||
files::{Directory, HeldFile, LedgerGuard, Role},
|
||||
ipc::{self, Request, Response, State},
|
||||
services::{Service, ServicePolicy, ServiceProof, Systemd},
|
||||
services::{Service, ServicePolicy, ServiceProof, Startup, Systemd},
|
||||
};
|
||||
use std::{ffi::OsStr, path::Path, time::Duration};
|
||||
use uuid::Uuid;
|
||||
@@ -97,6 +97,12 @@ struct Queued {
|
||||
budget: Budget,
|
||||
}
|
||||
impl Runtime {
|
||||
pub fn failed(&self) -> bool {
|
||||
self.failed
|
||||
}
|
||||
pub fn terminal(&self) -> bool {
|
||||
self.terminal
|
||||
}
|
||||
pub fn new(binding: Binding) -> Result<Self> {
|
||||
let clock = NativeClock::new()?;
|
||||
let budget = observation_budget()?;
|
||||
@@ -155,7 +161,7 @@ impl Runtime {
|
||||
let services = Systemd::new(self.old_policies.clone())?;
|
||||
let budget = observation_budget()?;
|
||||
if let Err(error) = listener.serve_one(&services, &budget, self) {
|
||||
if self.queued.take().is_some() {
|
||||
if self.queued.take().is_some() || self.accepted {
|
||||
self.failed = true;
|
||||
}
|
||||
return Err(error);
|
||||
@@ -226,6 +232,7 @@ impl Runtime {
|
||||
.verify_delegation(&header.delegation, &budget)?;
|
||||
let reply = {
|
||||
let mut bridge = FullBridge::original(BridgeContext {
|
||||
binding: &self.binding,
|
||||
handoff: &header.handoff,
|
||||
delegation: &header.delegation,
|
||||
services: &services,
|
||||
@@ -235,7 +242,19 @@ impl Runtime {
|
||||
health_public_key: self.health_key,
|
||||
generation: Generation::Original,
|
||||
})?;
|
||||
bridge.call(NativeControlOperation::ReadOriginal)?
|
||||
loop {
|
||||
let reply = bridge.call(NativeControlOperation::ReadOriginal)?;
|
||||
if reply.pre_stop_sync.is_some() {
|
||||
break reply;
|
||||
}
|
||||
// Passive read only, under the ORIGINAL boot/deadline/owner.
|
||||
// Do not stop a reader before the server has retained and the
|
||||
// actual Full has applied this exact signed inflight context.
|
||||
budget.check()?;
|
||||
self.updater
|
||||
.verify_delegation(&header.delegation, &budget)?;
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
};
|
||||
let proof = reply.proof.context("UPDATE_ENGINE_ORIGINAL_PROOF")?;
|
||||
let original = Original::verify(
|
||||
@@ -281,6 +300,7 @@ impl Runtime {
|
||||
census::scan_admission(&targets, &known, self.registry.host_context(), &budget)?;
|
||||
let mut bridge = FullBridge::new(
|
||||
BridgeContext {
|
||||
binding: &self.binding,
|
||||
handoff: original.handoff(),
|
||||
delegation: original.delegation(),
|
||||
services: &services,
|
||||
@@ -381,9 +401,10 @@ impl Runtime {
|
||||
ledger_identity_sha256: witnesses.ledger.sha256()?,
|
||||
})?;
|
||||
// Do not hold a second connection before all readers really stopped.
|
||||
let swap = (|| -> Result<()> {
|
||||
let swap = (|| -> Result<Vec<rmm_updater_linux::files::Installed>> {
|
||||
let stages = witnesses.files.reopen_stages(original.handoff(), &budget)?;
|
||||
let mut fence = LedgerFence::new(&mut ledger)?;
|
||||
let mut installed = Vec::new();
|
||||
for (stage, role) in
|
||||
stages
|
||||
.into_iter()
|
||||
@@ -396,23 +417,27 @@ impl Runtime {
|
||||
0,
|
||||
false,
|
||||
)?;
|
||||
stage.install(&directory, &os, &budget, &mut fence)?;
|
||||
installed.push(stage.install(&directory, &os, &budget, &mut fence)?);
|
||||
}
|
||||
fence.finish()?;
|
||||
ledger.append_observed(Evidence::PairVerified {
|
||||
pair: original.handoff().target.clone(),
|
||||
retained_state_sha256: original.handoff().retained_state_sha256.clone(),
|
||||
})
|
||||
})?;
|
||||
Ok(installed)
|
||||
})();
|
||||
if let Err(error) = swap {
|
||||
// Exactly one best-effort ORIGINAL Unknown CAS while we still own
|
||||
// the real guard. Failure is preserved; no restoration or retry.
|
||||
let _ = ledger.mark_unknown(
|
||||
u64::try_from(self.clock.now()?.timestamp())?,
|
||||
rmm_updater_handoff::UnknownReason::ReplaceUncertain,
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
let installed = match swap {
|
||||
Ok(installed) => installed,
|
||||
Err(error) => {
|
||||
// Exactly one best-effort ORIGINAL Unknown CAS while we still own
|
||||
// the real guard. Failure is preserved; no restoration or retry.
|
||||
let _ = ledger.mark_unknown(
|
||||
u64::try_from(self.clock.now()?.timestamp())?,
|
||||
rmm_updater_handoff::UnknownReason::ReplaceUncertain,
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let prepared = ledger.prepare_return_and_close()?;
|
||||
// The original DB is actually closed. End the continuous census/lock
|
||||
// borrow before the same concrete guard can be consumed by Return.
|
||||
@@ -432,6 +457,7 @@ impl Runtime {
|
||||
let observer = Systemd::new(target_policies)?;
|
||||
let mut bridge = FullBridge::new(
|
||||
BridgeContext {
|
||||
binding: &self.binding,
|
||||
handoff: original.handoff(),
|
||||
delegation: original.delegation(),
|
||||
services: &observer,
|
||||
@@ -447,8 +473,139 @@ impl Runtime {
|
||||
agent_startup.verify()?;
|
||||
os.release(&budget, &mut fence)?;
|
||||
agent_startup.return_after_root_release(&budget)?;
|
||||
let started = stopped.start_once(target, &budget, &mut fence)?;
|
||||
fence.finish()?;
|
||||
let started = match stopped.start_once_observed(target, &budget, &mut fence)? {
|
||||
Startup::Complete { started, .. } => {
|
||||
fence.finish()?;
|
||||
started
|
||||
}
|
||||
Startup::KnownFailed(failed) => {
|
||||
let mut conditional = fence.early_full_failure(&failed)?;
|
||||
let expected_head = conditional.proof().head_sha256.clone();
|
||||
let (stopped, _, _) = failed.stop_reserved_general(&budget, &mut conditional)?;
|
||||
conditional.finish()?;
|
||||
drop(bridge);
|
||||
let agent_startup =
|
||||
BoundAgentLock::bind(&witnesses.retained)?.acquire(&stopped, &budget)?;
|
||||
let os = LedgerGuard::acquire(&stopped, &witnesses.ledger, &budget)?;
|
||||
self.registry
|
||||
.verify_known_failure_stopped(&self.policy, &stopped, &budget)?;
|
||||
let readers = census::scan_stopped(
|
||||
&targets,
|
||||
&admission,
|
||||
&os,
|
||||
&agent_startup,
|
||||
&stopped,
|
||||
self.updater.held_service(),
|
||||
&budget,
|
||||
)?;
|
||||
let readers = readers.bind(
|
||||
census::ClosureInputs {
|
||||
targets: &targets,
|
||||
root: &os,
|
||||
agent: &agent_startup,
|
||||
stopped: &stopped,
|
||||
updater: self.updater.held_service(),
|
||||
registry: &self.registry,
|
||||
},
|
||||
&budget,
|
||||
)?;
|
||||
let mut ledger = OwnedLedger::open(OwnedInputs {
|
||||
os: &os,
|
||||
readers: &readers,
|
||||
original: &original,
|
||||
binding: &self.binding,
|
||||
clock: &self.clock,
|
||||
budget: &budget,
|
||||
updater: &self.updater,
|
||||
retained: &witnesses.retained,
|
||||
files: &witnesses.files,
|
||||
stopped: &stopped,
|
||||
trust: &trust,
|
||||
})?;
|
||||
ledger.begin_rollback_after_early_full_failure(&expected_head)?;
|
||||
ledger.append_rollback_stopped()?;
|
||||
{
|
||||
let mut rollback = RollbackFence::new(&mut ledger)?;
|
||||
for installed in installed.into_iter().rev() {
|
||||
let directory = Directory::open(
|
||||
Path::new(installed.role().target())
|
||||
.parent()
|
||||
.context("UPDATE_ROLLBACK_PARENT")?,
|
||||
0,
|
||||
false,
|
||||
)?;
|
||||
installed.restore(&directory, &os, &budget, &mut rollback)?;
|
||||
}
|
||||
rollback.finish()?;
|
||||
}
|
||||
ledger.append_observed(Evidence::PredecessorVerified {
|
||||
pair: original.handoff().predecessor.clone(),
|
||||
retained_state_sha256: original.handoff().retained_state_sha256.clone(),
|
||||
predecessor_record_sha256: original.handoff().predecessor_record_sha256.clone(),
|
||||
})?;
|
||||
let prepared = ledger.prepare_rollback_return_and_close()?;
|
||||
drop(readers);
|
||||
let reservation = prepared.bind(returned::Inputs {
|
||||
original: &original,
|
||||
binding: &self.binding,
|
||||
clock: &self.clock,
|
||||
budget: &budget,
|
||||
updater: &self.updater,
|
||||
retained: &witnesses.retained,
|
||||
files: &witnesses.files,
|
||||
trust: &trust,
|
||||
})?;
|
||||
let predecessor = Systemd::new(self.old_policies.clone())?;
|
||||
let observer = Systemd::new(self.old_policies.clone())?;
|
||||
let mut restored = FullBridge::new(
|
||||
BridgeContext {
|
||||
binding: &self.binding,
|
||||
handoff: original.handoff(),
|
||||
delegation: original.delegation(),
|
||||
services: &observer,
|
||||
updater: &self.updater,
|
||||
clock: &self.clock,
|
||||
budget: &budget,
|
||||
health_public_key: self.health_key,
|
||||
generation: Generation::Restored,
|
||||
},
|
||||
reservation.proof().clone(),
|
||||
)?;
|
||||
let mut fence = reservation.start_fence(&observer, &mut restored);
|
||||
agent_startup.verify()?;
|
||||
os.release(&budget, &mut fence)?;
|
||||
agent_startup.return_after_root_release(&budget)?;
|
||||
let started = stopped.start_once(predecessor, &budget, &mut fence)?;
|
||||
fence.finish()?;
|
||||
restored.observe_starts(&started)?;
|
||||
loop {
|
||||
original.check_current(&self.binding, &self.clock, &budget)?;
|
||||
self.updater
|
||||
.verify_delegation(original.delegation(), &budget)?;
|
||||
witnesses.retained.verify()?;
|
||||
for service in &started {
|
||||
service.verify(&budget)?;
|
||||
}
|
||||
let reply = restored.call(NativeControlOperation::ReadOriginal)?;
|
||||
let proof = reply.proof.context("UPDATE_ROLLBACK_HEALTH_PROOF")?;
|
||||
self.head = Some(proof.head_sha256.clone());
|
||||
let (_, journal) =
|
||||
rmm_domain::agent_update::verify_inflight_proof(&proof, self.health_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
match journal.phase() {
|
||||
Phase::RollbackServicesStarted => {
|
||||
std::thread::sleep(Duration::from_millis(100))
|
||||
}
|
||||
Phase::RollbackHealthVerified => {
|
||||
restored.call(NativeControlOperation::Finalize)?;
|
||||
self.head = None;
|
||||
return Ok(());
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_ROLLBACK_HEALTH_PREFIX_CHANGED"),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
bridge.observe_starts(&started)?;
|
||||
loop {
|
||||
original.check_current(&self.binding, &self.clock, &budget)?;
|
||||
|
||||
Reference in New Issue
Block a user