//! PRE-consumption Full admission. Protected local provisioning and exact //! approved artifact bytes are verified before the ordinary HelperSync gate //! atomically writes the original consumption/start/handoff/delegation. use crate::{ Binding, Request, reconcile::{self, ExecutorGuard}, }; use anyhow::{Context, Result, ensure}; use chrono::{DateTime, Utc}; use rmm_agent_core::{ helper_reconcile::ExecutorLedgerGuard, reconcile::SyncClock, update_handoff::{PreparedHandoff, UpdateAdmissionGuard}, }; use rmm_domain::{ agent_update::{ ParticipantIdentity, ParticipantRole, PeerPinTransition, ResolvedAgentUpdate, UpdateDelegation, }, jobs::{Action, JobPayload, hash}, }; use rmm_updater_handoff::{ AttemptBinding, CodePair, FileRole, FileTransition, Handoff, PinTransition, }; use rmm_updater_linux::{ Budget, files::{Directory, HeldFile, Identity, LedgerIdentity, Role, StageDescriptor, Staged}, ipc, services::{HeldService, Service, ServicePolicy, Systemd}, }; use rmm_updater_worker::{ bundle, clock::NativeClock, files::FileWitnesses, full::FullFiles, reader_registry::QualifiedRegistry, retained::{RetainedFileSpec, RetainedSet, Retention}, trust::TrustState, }; use rusqlite::Connection; use serde::{Deserialize, Serialize}; use std::{ cell::RefCell, ffi::OsStr, fs::{File, OpenOptions}, os::unix::{ fs::{MetadataExt, OpenOptionsExt}, net::UnixStream, }, path::{Path, PathBuf}, }; use uuid::Uuid; /// Nonterminal acknowledgement only. Agent must keep the original job running /// without storing an Outcome; the worker may now stop the Agent and Full. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Accepted { pub schema_version: u32, pub operation: String, pub grant_id: Uuid, pub owner_nonce: Uuid, pub handoff_sha256: String, pub original_request_sha256: String, pub ready: bool, pub terminal: bool, } impl Accepted { pub fn matches(&self, request: &Request, grant: Uuid) -> Result<()> { ensure!( self.schema_version == 1 && self.operation == "agent_update_handoff_accepted_v1" && self.grant_id == grant && !self.owner_nonce.is_nil() && rmm_domain::agent_update::digest(&self.handoff_sha256) && self.original_request_sha256 == hash(&serde_json::to_vec(request)?) && !self.ready && !self.terminal, "UPDATE_TYPED_HANDOFF_ACK" ); Ok(()) } } struct Protected { directory: Directory, file: HeldFile, path: PathBuf, maximum: u64, } impl Protected { fn read(path: &Path, maximum: u64) -> Result<(Self, Vec)> { let directory = Directory::open(path.parent().context("UPDATE_ADMISSION_PARENT")?, 0, false)?; let mut file = directory.inspect( path.file_name().context("UPDATE_ADMISSION_NAME")?, 0, maximum, )?; let bytes = file.bytes(maximum)?; Ok(( Self { directory, file, path: path.into(), maximum, }, bytes, )) } fn verify(&self) -> Result<()> { self.directory.file( self.path.file_name().context("UPDATE_ADMISSION_NAME")?, self.file.proof(), self.maximum, )?; Ok(()) } } fn identity(m: &std::fs::Metadata) -> Identity { Identity { device: m.dev(), inode: m.ino(), uid: m.uid(), gid: m.gid(), mode: m.mode(), links: m.nlink(), } } fn participant(service: &HeldService) -> Result { let p = service.proof(); ensure!(p.process.uid == 0, "UPDATE_ADMISSION_ROOT_PARTICIPANT"); if p.service == Service::Updater { ensure!(p.process.gid == 0, "UPDATE_STABLE_UPDATER_GROUP"); } let role = match p.service { Service::Full => ParticipantRole::LinuxFullHelper, Service::Updater => ParticipantRole::LinuxUpdater, _ => anyhow::bail!("UPDATE_ADMISSION_PARTICIPANT_ROLE"), }; Ok(ParticipantIdentity { role, pid: p.process.pid, birth: p.process.birth, image_sha256: p.process.image_sha256.clone(), security_principal: "uid:0".into(), service_instance_sha256: hash(&serde_json::to_vec(p)?), }) } pub struct Local<'a> { pub executor: &'a ExecutorGuard, pub binding: &'a Binding, pub clock: &'a NativeClock, pub job_key_path: &'a Path, pub job_key_raw_sha256: &'a str, } pub struct Prepared<'a> { local: Local<'a>, request: Request, update: ResolvedAgentUpdate, header: Handoff, delegation: UpdateDelegation, trust: TrustState, registry: QualifiedRegistry, files: FileWitnesses, retained: RetainedSet, ledger: LedgerIdentity, ledger_file: File, ledger_directory: Directory, sqlite_handle: usize, full: HeldService, updater: HeldService, full_files: RefCell, _stages: [Staged; 4], protected: Vec, budget: Budget, } impl<'a> Prepared<'a> { pub fn new(db: &Connection, local: Local<'a>, request: &Request) -> Result { local.executor.check(db)?; let (job, permit) = local.binding.verify(request, local.clock.now()?)?; let Action::UpdateAgent { update } = &job.action else { anyhow::bail!("UPDATE_ACTION_REQUIRED") }; ensure!( job.execution_identity == "root" && update.validate() && job.max_runtime_seconds == 600, "UPDATE_ADMISSION_ACTION" ); let trust = TrustState::load_prepared(update, local.binding, local.clock)?; trust.qualify_history(db, local.executor, local.binding)?; let boot: Uuid = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")? .trim() .parse()?; // This is preparation time within the existing start window, not a new // 600s execution lease. The immutable 600s handoff is stamped last. let budget = Budget::new(boot, (permit.start_before - local.clock.now()?).to_std()?)?; let registry = QualifiedRegistry::load(trust.policy())?; registry.verify_admission(trust.policy(), &budget)?; let policies = trust .policy() .services .iter() .zip([ Service::Agent, Service::General, Service::Full, Service::Apt, ]) .map(|(pin, service)| ServicePolicy { service, unit_sha256: pin.unit_sha256.clone(), image_sha256: if service == Service::Agent { update.predecessor.agent_sha256.clone() } else { update.predecessor.helper_sha256.clone() }, }) .collect(); let services = Systemd::new(policies)?; let full = services.observe(Service::Full, &budget)?; ensure!( full.proof().process.pid == std::process::id(), "UPDATE_ACTUAL_FULL_PROCESS_REQUIRED" ); let updater = Systemd::observe_updater( ServicePolicy { service: Service::Updater, image_sha256: update.updater.binary_sha256.clone(), unit_sha256: trust.policy().updater_service.unit_sha256.clone(), }, &budget, )?; let description = ipc::exchange( UnixStream::connect("/run/rmm-agent-updater/control.sock")?, &updater, &budget, &ipc::Request::Describe {}, )?; ensure!( matches!(description.state, ipc::State::Described) && description.updater == participant(&updater)? && description.head_sha256.is_none(), "UPDATE_ADMISSION_LIVE_UPDATER" ); let nonce = Uuid::new_v4(); let manifest = update.manifest().map_err(anyhow::Error::msg)?; let agent = stage_download( &job, Role::Agent, &manifest.artifacts[0].sha256, manifest.artifacts[0].bytes, nonce, &budget, )?; let helper = stage_download( &job, Role::Helper, &manifest.artifacts[1].sha256, manifest.artifacts[1].bytes, nonce, &budget, )?; let mut full_files = FullFiles::open( update, local.job_key_path, local.job_key_raw_sha256, &budget, )?; let [full_config, full_pin] = full_files.stage(nonce, &budget)?; let stages = [agent, helper, full_config, full_pin]; let files = FileWitnesses::capture(&stages, nonce, &budget)?; let (retained_config, raw) = Protected::read( Path::new("/etc/rmm-agent-updater/retained-files.json"), 65536, )?; let mut specs: Vec = serde_json::from_slice(&raw)?; for (path, retention) in [ ( "/var/lib/rmm-agent-remote/journal.sqlite", Retention::MutableDatabase, ), ( "/var/lib/rmm-agent-remote/consumption.sqlite", Retention::MutableDatabase, ), ( "/var/lib/rmm-agent-remote/agent.lock", Retention::ImmutableMetadata, ), ( "/var/lib/rmm-helper/consumption.sqlite", Retention::MutableDatabase, ), ( "/var/lib/rmm-helper/executor.lock", Retention::ImmutableMetadata, ), ( "/var/lib/rmm-agent-remote/identity.json", Retention::ImmutableMetadata, ), ( "/var/lib/rmm-agent-remote/device.key", Retention::ImmutableMetadata, ), ( "/var/lib/rmm-agent-remote/device.pem", Retention::ImmutableMetadata, ), ] { ensure!( specs .iter() .any(|s| s.path == Path::new(path) && s.retention == retention), "UPDATE_MANDATORY_RETAINED_STATE" ); } ensure!( specs.iter().any( |s| s.path == local.job_key_path && s.retention == Retention::ImmutableMetadata ), "UPDATE_RETAINED_JOB_KEY_REQUIRED" ); retain_active_tls(&mut specs, local.binding)?; let retained = RetainedSet::capture(specs)?; let ledger_directory = Directory::open(Path::new("/var/lib/rmm-helper"), 0, true)?; let ledger_file = OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK | libc::O_CLOEXEC) .open("/var/lib/rmm-helper/consumption.sqlite")?; let ledger = LedgerIdentity { ledger: identity(&ledger_file.metadata()?), lock: identity(&local.executor._file.metadata()?), }; ensure!( ledger.ledger == identity(&std::fs::symlink_metadata( "/var/lib/rmm-helper/consumption.sqlite" )?) && ledger.lock == identity(&std::fs::symlink_metadata( "/var/lib/rmm-helper/executor.lock" )?), "UPDATE_ADMISSION_LEDGER_IDENTITIES" ); let (general, raw) = Protected::read(Path::new("/etc/rmm-remote/general-helper-proof.json"), 8192)?; let PeerPinTransition::LinuxGeneralFullV1 { general_pin_sha256, full: transition, } = &update.peer_pin_transition else { anyhow::bail!("UPDATE_FULL_TRANSITION_REQUIRED") }; ensure!( hash(&raw) == *general_pin_sha256, "UPDATE_GENERAL_PIN_CHANGED" ); let (helper_journal,checkpoint,decision,agent_journal,generation):(String,u64,String,String,u64)=db.query_row( "SELECT journal_id,checkpoint,last_decision,agent_journal_id,generation FROM helper_sync_meta WHERE id=1 AND fault IS NULL",[], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?; let mut stop_set = vec![ rmm_updater_handoff::Service::LinuxAgent, rmm_updater_handoff::Service::LinuxGeneralHelper, rmm_updater_handoff::Service::LinuxFullHelper, ]; if trust .policy() .services .iter() .any(|pin| pin.unit == Service::Apt.unit()) { stop_set.push(rmm_updater_handoff::Service::LinuxAptHelper); } let now = local.clock.now()?; let consumed_at = u64::try_from(now.timestamp())?; let start = rmm_agent_core::ledger::start_witness_bytes(&permit, request, boot, now)?; let header = Handoff { schema_version: 2, attempt: AttemptBinding { tenant_id: permit.tenant_id, asset_id: permit.asset_id, agent_id: permit.agent_id, enrollment_generation: permit.enrollment_generation, assignment_version: permit.assignment_version, plan_id: permit .agent_update_plan_id .context("UPDATE_PLAN_ID_REQUIRED")?, plan_sha256: permit.plan_hash.clone(), job_id: permit.job_id, attempt_id: permit.attempt_id, job_payload_sha256: permit.job_payload_hash.clone(), grant_id: permit.grant_id, authorization_id: permit.authorization_id, original_request_sha256: hash(&serde_json::to_vec(request)?), original_start_witness_sha256: hash(&start), }, agent_journal_id: agent_journal.parse()?, helper_journal_id: helper_journal.parse()?, helper_checkpoint: checkpoint, previous_decision_id: decision.parse()?, boot_id: boot, connection_generation: generation, owner_nonce: nonce, updater_sha256: update.updater.binary_sha256.clone(), updater_policy_sha256: update.updater.policy_sha256.clone(), predecessor_record_sha256: update.predecessor.installed_record_sha256.clone(), retained_state_sha256: retained.sha256().into(), ledger_identity_sha256: ledger.sha256()?, staged_files_sha256: files.sha256().into(), predecessor: CodePair { manifest_sha256: update.predecessor.manifest_sha256.clone(), agent_sha256: update.predecessor.agent_sha256.clone(), helper_sha256: update.predecessor.helper_sha256.clone(), pin_sha256: transition.old_pin_sha256.clone(), }, target: CodePair { manifest_sha256: update.manifest_sha256.clone(), agent_sha256: manifest.artifacts[0].sha256.clone(), helper_sha256: manifest.artifacts[1].sha256.clone(), pin_sha256: transition.new_pin_sha256.clone(), }, pin_transition: PinTransition::LinuxGeneralFullV1, pin_transition_sha256: hash(&serde_json::to_vec(&update.peer_pin_transition)?), files: vec![ FileTransition { role: FileRole::AgentBinary, old_sha256: update.predecessor.agent_sha256.clone(), new_sha256: manifest.artifacts[0].sha256.clone(), }, FileTransition { role: FileRole::HelperBinary, old_sha256: update.predecessor.helper_sha256.clone(), new_sha256: manifest.artifacts[1].sha256.clone(), }, FileTransition { role: FileRole::LinuxFullConfig, old_sha256: transition.old_config_sha256.clone(), new_sha256: transition.new_config_sha256.clone(), }, FileTransition { role: FileRole::LinuxFullPin, old_sha256: transition.old_pin_sha256.clone(), new_sha256: transition.new_pin_sha256.clone(), }, ], stop_set, consumed_at, finish_before: consumed_at .checked_add(600) .context("UPDATE_FINISH_OVERFLOW")?, }; let delegation = UpdateDelegation { schema_version: 1, boot_id: boot, owner_nonce: nonce, handoff_sha256: hash(&header.bytes()?), original_helper: participant(&full)?, updater: participant(&updater)?, prepared_at: now, }; bundle::write_before_handoff( &header, &files, &retained, &ledger, bundle::Participants { delegation: &delegation, original_full: &full, updater: &updater, budget: &budget, }, )?; let prepared = Self { local, request: request.clone(), update: update.as_ref().clone(), header, delegation, trust, registry, files, retained, ledger, ledger_file, ledger_directory, sqlite_handle: unsafe { db.handle() } as usize, full, updater, full_files: RefCell::new(full_files), _stages: stages, protected: vec![retained_config, general], budget, }; prepared.check(db)?; Ok(prepared) } pub fn consume( &self, db: &mut Connection, sync: &mut reconcile::HelperSync, clock: &reconcile::NativeClock, ) -> Result { let prepared = PreparedHandoff { request: &self.request, handoff: &self.header, delegation: &self.delegation, recorded_at: self.delegation.prepared_at, }; sync.begin_update_handoff( db, self, self.header.connection_generation, &prepared, clock, &reconcile::NativeBoot, )?; Ok(Accepted { schema_version: 1, operation: "agent_update_handoff_accepted_v1".into(), grant_id: self.header.attempt.grant_id, owner_nonce: self.header.owner_nonce, handoff_sha256: hash(&self.header.bytes()?), original_request_sha256: self.header.attempt.original_request_sha256.clone(), ready: false, terminal: false, }) } /// Exactly once, AFTER the caller flushed Accepted and before returning to /// Full's listener. A transport error is not retried and creates no Outcome. pub fn dispatch_after_reply(self) -> Result<()> { let response = ipc::exchange( UnixStream::connect("/run/rmm-agent-updater/control.sock")?, &self.updater, &self.budget, &ipc::Request::Activate { grant_id: self.header.attempt.grant_id, owner_nonce: self.header.owner_nonce, handoff_sha256: hash(&self.header.bytes()?), }, )?; ensure!( matches!(response.state, ipc::State::Inflight), "UPDATE_ACTIVATION_ACK_REQUIRED" ); Ok(()) } } impl ExecutorLedgerGuard for Prepared<'_> { fn check(&self, db: &Connection) -> Result<()> { self.local.executor.check(db)?; ensure!( unsafe { db.handle() } as usize == self.sqlite_handle, "UPDATE_PREPARED_SAME_CONNECTION" ); self.ledger_directory.verify()?; ensure!( identity(&self.ledger_file.metadata()?) == self.ledger.ledger && identity(&std::fs::symlink_metadata( "/var/lib/rmm-helper/consumption.sqlite" )?) == self.ledger.ledger && identity(&self.local.executor._file.metadata()?) == self.ledger.lock && identity(&std::fs::symlink_metadata( "/var/lib/rmm-helper/executor.lock" )?) == self.ledger.lock, "UPDATE_PREPARED_SAME_LEDGER_AND_LOCK" ); self.budget.check() } } impl UpdateAdmissionGuard for Prepared<'_> { fn health_public_key(&self) -> Result<[u8; 32]> { Ok(self.trust.health_public_key()) } fn verify_prepared_update( &self, request: &Request, handoff: &Handoff, delegation: &UpdateDelegation, now: DateTime, ) -> Result<()> { ensure!( serde_json::to_vec(request)? == serde_json::to_vec(&self.request)? && handoff.bytes()? == self.header.bytes()? && serde_json::to_vec(delegation)? == serde_json::to_vec(&self.delegation)? && now == self.delegation.prepared_at, "UPDATE_PREPARED_ORIGINAL_BYTES_CHANGED" ); self.budget.check()?; self.local .binding .verify(request, self.local.clock.now()?)?; self.trust .recheck_prepared(&self.update, self.local.clock)?; self.registry.verify_protected()?; self.full.verify(&self.budget)?; self.updater.verify(&self.budget)?; ensure!( participant(&self.full)? == delegation.original_helper && participant(&self.updater)? == delegation.updater, "UPDATE_PREPARED_PARTICIPANTS_CHANGED" ); self.full_files.borrow_mut().verify_original(&self.budget)?; self.files.reopen_stages(&self.header, &self.budget)?; self.retained.verify()?; for protected in &self.protected { protected.verify()?; } Ok(()) } } /// Metadata only for the currently selected TLS pair. Never read or export key /// bytes, and never require a bootstrap rewrite after normal TLS rotation. fn retain_active_tls(specs: &mut Vec, binding: &Binding) -> Result<()> { let base = Path::new("/var/lib/rmm-agent-remote/certificates"); let dir = Directory::open_agent_state(base)?; let mut selector = dir.inspect(OsStr::new("current.json"), 993, 4096)?; let raw = selector.bytes(4096)?; let value: serde_json::Value = serde_json::from_slice(&raw)?; ensure!( value["schema_version"] == 1 && value["agent_id"] == binding.agent_id.to_string() && value["enrollment_generation"] == binding.enrollment_generation, "UPDATE_TLS_SELECTOR_IDENTITY" ); let version: Uuid = value["version"] .as_str() .context("UPDATE_TLS_SELECTOR_VERSION")? .parse()?; for path in [ base.join("current.json"), base.join("versions") .join(version.to_string()) .join("device.key"), base.join("versions") .join(version.to_string()) .join("device.pem"), ] { let parent = path.parent().context("UPDATE_TLS_PARENT")?; Directory::open_agent_state(parent)?.verify()?; let m = std::fs::symlink_metadata(&path)?; ensure!( m.is_file() && !m.file_type().is_symlink() && m.uid() == 993 && m.mode() & 0o777 == 0o600 && m.nlink() == 1, "UPDATE_TLS_RETAINED_METADATA" ); let spec = RetainedFileSpec { path, parent_uid: 993, identity: identity(&m), retention: Retention::ImmutableMetadata, }; if let Some(existing) = specs.iter().find(|s| s.path == spec.path) { ensure!(existing == &spec, "UPDATE_TLS_RETAINED_SELECTOR_CHANGED"); } else { specs.push(spec); } } dir.file(OsStr::new("current.json"), selector.proof(), 4096)?; Ok(()) } fn stage_download( job: &JobPayload, role: Role, expected: &str, length: u64, nonce: Uuid, budget: &Budget, ) -> Result { let label = match role { Role::Agent => "agent.bin", Role::Helper => "helper.bin", _ => anyhow::bail!("UPDATE_DOWNLOAD_ROLE"), }; let parent = Path::new("/var/lib/rmm-agent-remote/update-downloads") .join(job.job_id.to_string()) .join(job.attempt_id.to_string()); let source_directory = Directory::open_agent_state(&parent)?; let mut source = source_directory.inspect(OsStr::new(label), 993, 256 * 1024 * 1024)?; ensure!( source.proof().bytes == length && source.proof().sha256 == expected && source.proof().identity.links == 1 && source.proof().identity.mode & 0o777 == 0o600, "UPDATE_PREFETCH_BYTES_REQUIRED" ); let target = Path::new(role.target()); let directory = Directory::open(target.parent().context("UPDATE_TARGET_PARENT")?, 0, false)?; let original = directory.inspect( target.file_name().context("UPDATE_TARGET_NAME")?, 0, 256 * 1024 * 1024, )?; let Action::UpdateAgent { update } = &job.action else { anyhow::bail!("UPDATE_ACTION_REQUIRED") }; ensure!( original.proof().sha256.as_str() == match role { Role::Agent => update.predecessor.agent_sha256.as_str(), Role::Helper => update.predecessor.helper_sha256.as_str(), _ => unreachable!(), }, "UPDATE_PREDECESSOR_IMAGE_CHANGED" ); let staged = Staged::write( &directory, StageDescriptor { role, nonce, original: original.proof(), expected_sha: expected, length, }, source.reader()?, budget, )?; source_directory.file(OsStr::new(label), source.proof(), 256 * 1024 * 1024)?; Ok(staged) } // Passive helper RPC client; it never creates a handoff. pub use crate::update_observer::{observe, observe_record}; #[cfg(test)] mod tests { use super::*; #[test] fn typed_handoff_ack_is_not_an_outcome_or_a_new_start_lease() { let request = Request { job_bytes: b"bound original job".to_vec(), permit_bytes: b"bound original permit".to_vec(), signature: vec![0; 64], plan_hash: "a".repeat(64), authorization_id: Uuid::new_v4(), }; let grant = Uuid::new_v4(); let accepted = Accepted { schema_version: 1, operation: "agent_update_handoff_accepted_v1".into(), grant_id: grant, owner_nonce: Uuid::new_v4(), handoff_sha256: "b".repeat(64), original_request_sha256: hash(&serde_json::to_vec(&request).unwrap()), ready: false, terminal: false, }; let raw = serde_json::to_vec(&accepted).unwrap(); serde_json::from_slice::(&raw) .unwrap() .matches(&request, grant) .unwrap(); assert!(serde_json::from_slice::(&raw).is_err()); let mut next = request.clone(); next.job_bytes.push(0); assert!(accepted.matches(&next, grant).is_err()); assert!(accepted.matches(&request, Uuid::new_v4()).is_err()); let mut false_result = accepted.clone(); false_result.terminal = true; assert!(false_result.matches(&request, grant).is_err()); false_result.terminal = false; false_result.ready = true; assert!(false_result.matches(&request, grant).is_err()); } }