//! One live original-attempt pass. Activation queues only; the fixed listener //! must finish its reply before this runtime can stop the calling Fullhelper. //! No daemon entrypoint, admission/capability registration or restart recovery. use crate::{ agent_startup::BoundAgentLock, bridge::{BridgeContext, FullBridge, Generation}, bundle, census, clock::NativeClock, fence::{LedgerFence, RollbackFence}, ledger::{OwnedInputs, OwnedLedger}, original::Original, participants::{StableParticipant, verify_original_helper}, reader_registry::QualifiedRegistry, retained::RetainedSet, returned, trust::TrustState, }; use anyhow::{Context, Result, ensure}; use rmm_agent_core::{ auth::Binding, reconcile::SyncClock, update_handoff::{NativeControlOperation, NativeEffect, NativeEffectClaim, NativeService}, }; use rmm_domain::{ agent_update::{ParticipantIdentity, ParticipantRole}, agent_update_provisioning::{HealthKeyPin, InstalledReleaseRecord, LocalUpdatePolicy}, jobs::hash, }; use rmm_updater_handoff::{Evidence, Phase, Process}; use rmm_updater_linux::{ Budget, DurableEffectFence, Effect, files::{Directory, HeldFile, LedgerGuard, Role}, ipc::{self, Request, Response, State}, services::{Service, ServicePolicy, ServiceProof, Startup, Systemd}, }; use std::{ffi::OsStr, path::Path, time::Duration}; use uuid::Uuid; struct Protected { directory: Directory, file: HeldFile, name: String, max: u64, } impl Protected { fn read(path: &str, max: u64) -> Result<(Self, Vec)> { let path = Path::new(path); let directory = Directory::open(path.parent().context("UPDATE_ENGINE_PARENT")?, 0, false)?; let name = path .file_name() .context("UPDATE_ENGINE_NAME")? .to_str() .context("UPDATE_ENGINE_NAME")? .to_owned(); let mut file = directory.inspect(OsStr::new(&name), 0, max)?; let raw = file.bytes(max)?; Ok(( Self { directory, file, name, max, }, raw, )) } fn verify(&self) -> Result<()> { self.directory .file(OsStr::new(&self.name), self.file.proof(), self.max)?; Ok(()) } } /// Construct only in the separately provisioned stable service, with the /// independently protected original job Binding. It offers no approval API. pub struct Runtime { binding: Binding, policy: LocalUpdatePolicy, registry: QualifiedRegistry, policy_sha256: String, installed: InstalledReleaseRecord, installed_sha256: String, health_key: [u8; 32], health_pin_sha256: String, protected: Vec, old_policies: Vec, updater: StableParticipant, clock: NativeClock, queued: Option, accepted: bool, failed: bool, terminal: bool, head: Option, } struct Queued { header: bundle::Header, budget: Budget, } impl Runtime { pub fn failed(&self) -> bool { self.failed } pub fn terminal(&self) -> bool { self.terminal } pub fn new(binding: Binding) -> Result { let clock = NativeClock::new()?; let budget = observation_budget()?; let (policy_file, raw) = Protected::read("/etc/rmm-agent-updater/policy.json", 65536)?; let (policy, policy_sha256) = LocalUpdatePolicy::from_bytes(&raw).map_err(anyhow::Error::msg)?; ensure!( policy.tenant_id == binding.tenant_id && policy.asset_id == binding.asset_id && policy.agent_id == binding.agent_id && policy.enrollment_generation == binding.enrollment_generation && policy.assignment_version == binding.assignment_version, "UPDATE_ENGINE_LOCAL_IDENTITY" ); let (record_file, raw) = Protected::read(&policy.installed_record_path, 32768)?; let installed = InstalledReleaseRecord::from_bytes(&raw).map_err(anyhow::Error::msg)?; let installed_sha256 = hash(&raw); let old_policies = service_policies(&policy, &installed, None)?; let registry = QualifiedRegistry::load(&policy)?; let (health_file, raw) = Protected::read(&policy.health_pin_path, 4096)?; let health: HealthKeyPin = serde_json::from_slice(&raw)?; let health_key = health.public_key().map_err(anyhow::Error::msg)?; let updater = StableParticipant::observe( ServicePolicy { service: Service::Updater, image_sha256: policy.updater_binary_sha256.clone(), unit_sha256: policy.updater_service.unit_sha256.clone(), }, &budget, )?; Ok(Self { binding, policy, registry, policy_sha256, installed, installed_sha256, health_key, health_pin_sha256: hash(&raw), protected: vec![policy_file, record_file, health_file], old_policies, updater, clock, queued: None, accepted: false, failed: false, terminal: false, head: None, }) } /// A lost activation reply disables this live runtime before any stop. /// Drive is deliberately private: it cannot run inside Engine::handle. pub fn serve_one(&mut self, listener: &ipc::Listener) -> Result<()> { self.registry.verify_protected()?; ensure!(!self.failed && !self.terminal, "UPDATE_ENGINE_CLOSED"); let services = Systemd::new(self.old_policies.clone())?; let budget = observation_budget()?; if let Err(error) = listener.serve_one(&services, &budget, self) { if self.queued.take().is_some() || self.accepted { self.failed = true; } return Err(error); } if let Some(queued) = self.queued.take() { // serve_one has written/flushed the bounded reply and dropped its // accepted stream. Only now may the original Full process stop. match self.drive(queued) { Ok(()) => self.terminal = true, Err(error) => { self.failed = true; return Err(error); } } } Ok(()) } fn enqueue( &mut self, peer: &ParticipantIdentity, grant: Uuid, nonce: Uuid, handoff_sha: &str, ) -> Result<()> { ensure!( !self.accepted && self.queued.is_none() && !self.failed && !self.terminal, "UPDATE_ENGINE_NO_RETRY_OR_RESUME" ); // Fuse before any fallible original admission. Even an uncertain reply // cannot dispatch this handoff for a second time in this process. self.accepted = true; for file in &self.protected { file.verify()?; } let header = bundle::read_header(grant, nonce, handoff_sha)?; ensure!( peer.role == ParticipantRole::LinuxFullHelper && *peer == header.delegation.original_helper && header.handoff.updater_sha256 == self.policy.updater_binary_sha256 && header.handoff.updater_policy_sha256 == self.policy_sha256 && header.handoff.predecessor_record_sha256 == self.installed_sha256, "UPDATE_ENGINE_ORIGINAL_PARTICIPANTS_OR_POLICY" ); let finish = chrono::DateTime::from_timestamp(i64::try_from(header.handoff.finish_before)?, 0) .context("UPDATE_ENGINE_FINISH")?; let budget = Budget::new( header.handoff.boot_id, (finish - self.clock.now()?).to_std()?, )?; let services = Systemd::new(self.old_policies.clone())?; verify_original_helper(&services, &header.delegation, &budget)?; self.updater .verify_delegation(&header.delegation, &budget)?; // The original Full is still synchronously waiting for this reply. // Do not recursively query its socket or ledger in the Activate handler. self.queued = Some(Queued { header, budget }); Ok(()) } fn drive(&mut self, queued: Queued) -> Result<()> { let Queued { header, budget } = queued; for file in &self.protected { file.verify()?; } let services = Systemd::new(self.old_policies.clone())?; verify_original_helper(&services, &header.delegation, &budget)?; self.updater .verify_delegation(&header.delegation, &budget)?; let reply = { let mut bridge = FullBridge::original(BridgeContext { binding: &self.binding, handoff: &header.handoff, delegation: &header.delegation, services: &services, updater: &self.updater, clock: &self.clock, budget: &budget, health_public_key: self.health_key, generation: Generation::Original, })?; loop { let reply = bridge.call(NativeControlOperation::ReadOriginal)?; if reply.pre_stop_sync.is_some() { break reply; } // Passive read only, under the ORIGINAL boot/deadline/owner. // Do not stop a reader before the server has retained and the // actual Full has applied this exact signed inflight context. budget.check()?; self.updater .verify_delegation(&header.delegation, &budget)?; std::thread::sleep(Duration::from_millis(50)); } }; let proof = reply.proof.context("UPDATE_ENGINE_ORIGINAL_PROOF")?; let original = Original::verify( &self.binding, &reply .original_start .context("UPDATE_ENGINE_ORIGINAL_START")?, &header.handoff.bytes()?, header.delegation, &self.clock, &budget, )?; let (_, journal) = rmm_domain::agent_update::verify_inflight_proof(&proof, self.health_key) .map_err(anyhow::Error::msg)?; ensure!( journal.phase() == Phase::StopIntent, "UPDATE_ENGINE_NOT_A_FRESH_HANDOFF" ); let trust = TrustState::load(&original, &self.binding, &self.clock)?; ensure!( trust.health_public_key() == self.health_key && original.update().updater.health_key_pin_sha256 == self.health_pin_sha256 && serde_json::to_vec(trust.policy())? == serde_json::to_vec(&self.policy)?, "UPDATE_ENGINE_TRUST_CHANGED" ); let witnesses = bundle::read_for_handoff(original.handoff())?; witnesses.files.reopen_stages(original.handoff(), &budget)?; witnesses.retained.verify()?; let agent_startup = BoundAgentLock::bind(&witnesses.retained)?; self.head = Some(proof.head_sha256.clone()); original.check_current(&self.binding, &self.clock, &budget)?; trust.recheck(&original, &self.clock)?; self.registry.verify_admission(&self.policy, &budget)?; let held = [Service::Agent, Service::General, Service::Full] .into_iter() .map(|s| services.observe(s, &budget)) .collect::>>()?; let known = vec![&held[0], &held[1], &held[2], self.updater.held_service()]; let old = held.iter().map(|h| h.proof().clone()).collect::>(); let targets = census::Targets::bind(&witnesses.retained, &witnesses.ledger, &agent_startup)?; let admission = census::scan_admission(&targets, &known, self.registry.host_context(), &budget)?; let mut bridge = FullBridge::new( BridgeContext { binding: &self.binding, handoff: original.handoff(), delegation: original.delegation(), services: &services, updater: &self.updater, clock: &self.clock, budget: &budget, health_public_key: self.health_key, generation: Generation::Original, }, proof, )?; let mut claims = Vec::new(); for service in [ NativeService::Agent, NativeService::General, NativeService::Full, ] { let expected_head_sha256 = bridge.head_sha256()?.to_owned(); claims.push( bridge .call(NativeControlOperation::ClaimStop { expected_head_sha256, service, })? .claim .context("UPDATE_ENGINE_STOP_CLAIM")?, ); } drop(bridge); let mut stop_fence = StopFence { original: &original, binding: &self.binding, clock: &self.clock, budget: &budget, updater: &self.updater, trust: &trust, retained: &witnesses.retained, registry: &self.registry, targets: &targets, admission: &admission, known: &known, claims, old, next: 0, pending: false, poisoned: false, }; let stopped = services.stop_all(&budget, &mut stop_fence)?; let claims = stop_fence.finish()?; let agent_startup = agent_startup.acquire(&stopped, &budget)?; let os = LedgerGuard::acquire(&stopped, &witnesses.ledger, &budget)?; self.registry.verify_stopped(&self.policy, &budget)?; let readers = census::scan_stopped( &targets, &admission, &os, &agent_startup, &stopped, self.updater.held_service(), &budget, )?; let readers = readers.bind( census::ClosureInputs { targets: &targets, root: &os, agent: &agent_startup, stopped: &stopped, updater: self.updater.held_service(), registry: &self.registry, }, &budget, )?; let mut ledger = OwnedLedger::open(OwnedInputs { os: &os, readers: &readers, original: &original, binding: &self.binding, clock: &self.clock, budget: &budget, updater: &self.updater, retained: &witnesses.retained, files: &witnesses.files, stopped: &stopped, trust: &trust, })?; for (claim, old) in claims.iter().zip(stopped.proofs()) { ledger.observe_native(claim, &hash(&serde_json::to_vec(&old)?))?; } ledger.append_observed(Evidence::AllStopped { observation_sha256: hash(&serde_json::to_vec(&stopped.proofs())?), })?; ledger.append_observed(Evidence::UpdaterOwnsLedger { process: Process { pid: self.updater.identity().pid, birth: self.updater.identity().birth, image_sha256: self.updater.identity().image_sha256.clone(), }, ledger_identity_sha256: witnesses.ledger.sha256()?, })?; // Do not hold a second connection before all readers really stopped. let swap = (|| -> Result> { let stages = witnesses.files.reopen_stages(original.handoff(), &budget)?; let mut fence = LedgerFence::new(&mut ledger)?; let mut installed = Vec::new(); for (stage, role) in stages .into_iter() .zip([Role::Agent, Role::Helper, Role::FullConfig, Role::FullPin]) { let directory = Directory::open( Path::new(role.target()) .parent() .context("UPDATE_ENGINE_TARGET_PARENT")?, 0, false, )?; installed.push(stage.install(&directory, &os, &budget, &mut fence)?); } fence.finish()?; ledger.append_observed(Evidence::PairVerified { pair: original.handoff().target.clone(), retained_state_sha256: original.handoff().retained_state_sha256.clone(), })?; Ok(installed) })(); let installed = match swap { Ok(installed) => installed, Err(error) => { // Exactly one best-effort ORIGINAL Unknown CAS while we still own // the real guard. Failure is preserved; no restoration or retry. let _ = ledger.mark_unknown( u64::try_from(self.clock.now()?.timestamp())?, rmm_updater_handoff::UnknownReason::ReplaceUncertain, ); return Err(error); } }; let prepared = ledger.prepare_return_and_close()?; // The original DB is actually closed. End the continuous census/lock // borrow before the same concrete guard can be consumed by Return. drop(readers); let reservation = prepared.bind(returned::Inputs { original: &original, binding: &self.binding, clock: &self.clock, budget: &budget, updater: &self.updater, retained: &witnesses.retained, files: &witnesses.files, trust: &trust, })?; let target_policies = service_policies(&self.policy, &self.installed, Some(&original))?; let target = Systemd::new(target_policies.clone())?; let observer = Systemd::new(target_policies)?; let mut bridge = FullBridge::new( BridgeContext { binding: &self.binding, handoff: original.handoff(), delegation: original.delegation(), services: &observer, updater: &self.updater, clock: &self.clock, budget: &budget, health_public_key: self.health_key, generation: Generation::Replacement, }, reservation.proof().clone(), )?; let mut fence = reservation.start_fence(&observer, &mut bridge); agent_startup.verify()?; os.release(&budget, &mut fence)?; agent_startup.return_after_root_release(&budget)?; let started = match stopped.start_once_observed(target, &budget, &mut fence)? { Startup::Complete { started, .. } => { fence.finish()?; started } Startup::KnownFailed(failed) => { let mut conditional = fence.early_full_failure(&failed)?; let expected_head = conditional.proof().head_sha256.clone(); let (stopped, _, _) = failed.stop_reserved_general(&budget, &mut conditional)?; conditional.finish()?; drop(bridge); let agent_startup = BoundAgentLock::bind(&witnesses.retained)?.acquire(&stopped, &budget)?; let os = LedgerGuard::acquire(&stopped, &witnesses.ledger, &budget)?; self.registry .verify_known_failure_stopped(&self.policy, &stopped, &budget)?; let readers = census::scan_stopped( &targets, &admission, &os, &agent_startup, &stopped, self.updater.held_service(), &budget, )?; let readers = readers.bind( census::ClosureInputs { targets: &targets, root: &os, agent: &agent_startup, stopped: &stopped, updater: self.updater.held_service(), registry: &self.registry, }, &budget, )?; let mut ledger = OwnedLedger::open(OwnedInputs { os: &os, readers: &readers, original: &original, binding: &self.binding, clock: &self.clock, budget: &budget, updater: &self.updater, retained: &witnesses.retained, files: &witnesses.files, stopped: &stopped, trust: &trust, })?; ledger.begin_rollback_after_early_full_failure(&expected_head)?; ledger.append_rollback_stopped()?; { let mut rollback = RollbackFence::new(&mut ledger)?; for installed in installed.into_iter().rev() { let directory = Directory::open( Path::new(installed.role().target()) .parent() .context("UPDATE_ROLLBACK_PARENT")?, 0, false, )?; installed.restore(&directory, &os, &budget, &mut rollback)?; } rollback.finish()?; } ledger.append_observed(Evidence::PredecessorVerified { pair: original.handoff().predecessor.clone(), retained_state_sha256: original.handoff().retained_state_sha256.clone(), predecessor_record_sha256: original.handoff().predecessor_record_sha256.clone(), })?; let prepared = ledger.prepare_rollback_return_and_close()?; drop(readers); let reservation = prepared.bind(returned::Inputs { original: &original, binding: &self.binding, clock: &self.clock, budget: &budget, updater: &self.updater, retained: &witnesses.retained, files: &witnesses.files, trust: &trust, })?; let predecessor = Systemd::new(self.old_policies.clone())?; let observer = Systemd::new(self.old_policies.clone())?; let mut restored = FullBridge::new( BridgeContext { binding: &self.binding, handoff: original.handoff(), delegation: original.delegation(), services: &observer, updater: &self.updater, clock: &self.clock, budget: &budget, health_public_key: self.health_key, generation: Generation::Restored, }, reservation.proof().clone(), )?; let mut fence = reservation.start_fence(&observer, &mut restored); agent_startup.verify()?; os.release(&budget, &mut fence)?; agent_startup.return_after_root_release(&budget)?; let started = stopped.start_once(predecessor, &budget, &mut fence)?; fence.finish()?; restored.observe_starts(&started)?; loop { original.check_current(&self.binding, &self.clock, &budget)?; self.updater .verify_delegation(original.delegation(), &budget)?; witnesses.retained.verify()?; for service in &started { service.verify(&budget)?; } let reply = restored.call(NativeControlOperation::ReadOriginal)?; let proof = reply.proof.context("UPDATE_ROLLBACK_HEALTH_PROOF")?; self.head = Some(proof.head_sha256.clone()); let (_, journal) = rmm_domain::agent_update::verify_inflight_proof(&proof, self.health_key) .map_err(anyhow::Error::msg)?; match journal.phase() { Phase::RollbackServicesStarted => { std::thread::sleep(Duration::from_millis(100)) } Phase::RollbackHealthVerified => { restored.call(NativeControlOperation::Finalize)?; self.head = None; return Ok(()); } _ => anyhow::bail!("UPDATE_ROLLBACK_HEALTH_PREFIX_CHANGED"), } } } }; bridge.observe_starts(&started)?; loop { original.check_current(&self.binding, &self.clock, &budget)?; self.updater .verify_delegation(original.delegation(), &budget)?; witnesses.retained.verify()?; for service in &started { service.verify(&budget)?; } let reply = bridge.call(NativeControlOperation::ReadOriginal)?; let proof = reply.proof.context("UPDATE_ENGINE_HEALTH_PROOF")?; self.head = Some(proof.head_sha256.clone()); let (_, journal) = rmm_domain::agent_update::verify_inflight_proof(&proof, self.health_key) .map_err(anyhow::Error::msg)?; match journal.phase() { Phase::ServicesStarted => std::thread::sleep(Duration::from_millis(100)), Phase::HealthVerified => { bridge.call(NativeControlOperation::Finalize)?; self.head = None; return Ok(()); } _ => anyhow::bail!("UPDATE_ENGINE_HEALTH_PREFIX_CHANGED"), } } } } impl ipc::Engine for Runtime { fn handle(&mut self, peer: &ParticipantIdentity, request: Request) -> Result { ensure!(!self.failed && !self.terminal, "UPDATE_ENGINE_CLOSED"); self.updater .identity() .validate() .then_some(()) .context("UPDATE_ENGINE_IDENTITY")?; let state = match request { Request::Describe {} => State::Described, Request::Activate { grant_id, owner_nonce, handoff_sha256, } => { self.enqueue(peer, grant_id, owner_nonce, &handoff_sha256)?; State::Inflight } // Health travels through the existing independently validating // new Fullhelper path. This daemon accepts no duplicate authority. _ => anyhow::bail!("UPDATE_ENGINE_UNSUPPORTED_CONTROL"), }; Ok(Response { schema_version: 1, state, ready: false, updater: self.updater.identity().clone(), head_sha256: self.head.clone(), }) } } struct StopFence<'a> { original: &'a Original, binding: &'a Binding, clock: &'a NativeClock, budget: &'a Budget, updater: &'a StableParticipant, trust: &'a TrustState, retained: &'a RetainedSet, registry: &'a QualifiedRegistry, targets: &'a census::Targets, admission: &'a census::Admission, known: &'a [&'a rmm_updater_linux::services::HeldService], claims: Vec, old: Vec, next: usize, pending: bool, poisoned: bool, } impl StopFence<'_> { fn current(&self) -> Result<()> { self.original .check_current(self.binding, self.clock, self.budget)?; self.updater .verify_delegation(self.original.delegation(), self.budget)?; self.trust.recheck(self.original, self.clock)?; self.registry.verify_protected()?; self.retained.verify() } fn finish(self) -> Result> { ensure!( !self.poisoned && !self.pending && self.next == 3, "UPDATE_ENGINE_STOP_INCOMPLETE" ); self.current()?; Ok(self.claims) } } impl DurableEffectFence for StopFence<'_> { fn before_effect(&mut self, effect: &Effect) -> Result<()> { ensure!( !self.poisoned && !self.pending && self.next < 3, "UPDATE_ENGINE_STOP_NO_RETRY" ); self.poisoned = true; self.current()?; if self.next == 0 { self.admission .verify_fresh(self.targets, self.known, self.budget)?; } let service = [Service::Agent, Service::General, Service::Full][self.next]; let native = [ NativeService::Agent, NativeService::General, NativeService::Full, ][self.next]; let claim = &self.claims[self.next]; ensure!( *effect == Effect::StopService { service } && claim.effect() == &NativeEffect::StopService { service: native } && claim.grant_id() == self.original.permit().grant_id && claim.owner_nonce() == self.original.handoff().owner_nonce, "UPDATE_ENGINE_STOP_RESERVATION_CHANGED" ); self.pending = true; Ok(()) } fn after_effect(&mut self, effect: &Effect, observation: &str) -> Result<()> { ensure!( self.poisoned && self.pending && self.next < 3 && *effect == Effect::StopService { service: self.old[self.next].service } && observation == hash(&serde_json::to_vec(&self.old[self.next])?), "UPDATE_ENGINE_STOP_OBSERVATION_CHANGED" ); self.current()?; self.next += 1; self.pending = false; self.poisoned = false; Ok(()) } } fn observation_budget() -> Result { Budget::new( std::fs::read_to_string("/proc/sys/kernel/random/boot_id")? .trim() .parse()?, Duration::from_secs(5), ) } fn service_policies( policy: &LocalUpdatePolicy, installed: &InstalledReleaseRecord, original: Option<&Original>, ) -> Result> { let release = installed .release_envelope .decode() .map_err(anyhow::Error::msg)?; policy .services .iter() .zip([ Service::Agent, Service::General, Service::Full, Service::Apt, ]) .map(|(pin, service)| { ensure!(pin.unit == service.unit(), "UPDATE_ENGINE_SERVICE_SET"); let image_sha256 = match (service, original) { (Service::Agent, Some(o)) => o.handoff().target.agent_sha256.clone(), (Service::General | Service::Full | Service::Apt, Some(o)) => { o.handoff().target.helper_sha256.clone() } (Service::Agent, None) => release.manifest.artifacts[0].sha256.clone(), (_, None) => release.manifest.artifacts[1].sha256.clone(), _ => anyhow::bail!("UPDATE_ENGINE_SERVICE_ROLE"), }; Ok(ServicePolicy { service, image_sha256, unit_sha256: pin.unit_sha256.clone(), }) }) .collect() }