mod agent_update_catalog; mod agent_update_health; mod apt; mod bootstrap; mod certificate_rotation; mod helper_sync; mod https_channel; mod jobs; mod reconcile; #[cfg(test)] mod runtime_tests; mod storage; mod transport; mod wire_channel; use anyhow::{Context, Result, ensure}; use chrono::Utc; use openssl::{ ec::{EcGroup, EcKey}, hash::MessageDigest, nid::Nid, pkey::PKey, x509::{X509NameBuilder, X509Req}, }; use prost::Message; use rmm_protocol::*; use sha2::{Digest, Sha256}; use std::{ env, fs, path::{Path, PathBuf}, sync::{ Arc, atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}, }, time::{Duration, Instant}, }; use uuid::Uuid; struct Agent { address: std::net::SocketAddr, ca: Vec, id: String, boot: String, instance: String, db: rusqlite::Connection, started: Instant, clock: rmm_privileged_helper::TrustedClock, binding: Option, ledger: rusqlite::Connection, completed_tx: tokio::sync::mpsc::Sender, completed_rx: tokio::sync::mpsc::Receiver, inventory_tx: tokio::sync::mpsc::Sender, i64)>>, inventory_rx: tokio::sync::mpsc::Receiver, i64)>>, inventory_inflight: bool, running_cancels: std::collections::HashMap<(String, String), Arc>, sync: Option, sync_clock: C, helpers: Option, prepared: Option, fence: Arc, mode: wire_channel::Mode, certificates: certificate_rotation::RotationStore, rotation_policy: certificate_rotation::RotationPolicy, catalog: agent_update_catalog::Checker, } #[derive(Default)] struct EffectFence { running_generation: AtomicU64, generation: AtomicU64, deadline: AtomicI64, root: AtomicBool, } fn report_closed_gate(error: &anyhow::Error) { let text = error.to_string(); let code = if text.len() <= 96 && text .bytes() .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_') { text.as_str() } else { "SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE" }; eprintln!("agent effect gate closed: {code}"); } impl EffectFence { fn close(&self) { self.running_generation.store(0, Ordering::SeqCst); self.close_starts(); } // A signed Sync/start lease authorizes NEW starts. Its natural expiry // cannot retroactively stop an approved bounded process (spec22.3). // Running effects instead observe their exact cancellation and current // connection generation; transport/signed refusal closes both fences. fn close_starts(&self) { self.generation.store(0, Ordering::SeqCst); self.root.store(false, Ordering::SeqCst); } fn allows(&self, generation: u64, root: bool) -> bool { self.generation.load(Ordering::SeqCst) == generation && generation > 0 && Utc::now().timestamp_millis() < self.deadline.load(Ordering::SeqCst) && (!root || self.root.load(Ordering::SeqCst)) } } struct PendingUpdateHealth(tokio::task::JoinHandle>); impl Drop for PendingUpdateHealth { fn drop(&mut self) { self.0.abort(); } } type Completion = ( JobOffer, rmm_domain::jobs::ExecutionPermitPayload, rmm_privileged_helper::Outcome, ); impl Agent { fn close_gate(&mut self) { self.fence.close(); if let Some(sync) = &mut self.sync { sync.close(); } self.prepared = None; } fn channel(&self, generation: u64) -> Result { Ok(reconcile::Channel { connection_generation: generation, boot_id: Uuid::parse_str(&self.boot)?, }) } fn ready(&self, generation: u64, root: bool) -> bool { self.channel(generation).is_ok_and(|c| { self.sync .as_ref() .is_some_and(|s| s.can_start(&c, root, &self.sync_clock)) }) } async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> { self.fence.close_starts(); let helpers = self.helpers.clone(); let snapshot = tokio::task::spawn_blocking(move || match helpers { Some(h) => h.snapshot(generation), None => Ok(None), }) .await??; let channel = self.channel(generation)?; let Some(sync) = &mut self.sync else { return Ok(()); }; let prepared = sync.prepare( &mut self.db, &self.ledger, channel, snapshot.as_ref(), &self.sync_clock, )?; wire.send(&prepared.envelope).await?; self.prepared = Some(prepared); Ok(()) } async fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> { let prepared = self.prepared.take().context("UNSOLICITED_SYNC_RESPONSE")?; let sync = self.sync.as_mut().context("SYNC_BINDING_REQUIRED")?; let applied = sync.apply(&mut self.db, &response, &self.sync_clock)?; sync.apply_consumption(&mut self.ledger, &self.sync_clock)?; let reason = if applied .reason .bytes() .all(|b| b.is_ascii_uppercase() || b == b'_') { applied.reason.as_str() } else { "RECONCILIATION_REQUIRED" }; eprintln!( "agent reconciliation committed: decision={}, receipts={}, reason={}", applied.decision_id, applied.acknowledged_message_ids.len(), reason ); if let Some(instruction) = applied.helper_instruction { let helpers = self .helpers .clone() .context("HELPER_SYNC_CLIENT_REQUIRED")?; let request = prepared.request.clone(); let snapshot = tokio::task::spawn_blocking(move || helpers.apply(&instruction.response, &request)) .await?? .context("HELPER_SYNC_ACK_REQUIRED")?; if self.ready(generation, false) { self.sync .as_mut() .context("SYNC_REQUIRED")? .confirm_helper(&snapshot)?; } else if snapshot .attempts .iter() .any(|a| !a.update_handoff_bytes.is_empty()) { self.sync .as_mut() .context("SYNC_REQUIRED")? .confirm_update_helper(&snapshot)?; } } if self.ready(generation, false) { let deadline = self .sync .as_ref() .and_then(|s| s.validity_deadline()) .context("SIGNED_LEASE_REQUIRED")?; self.fence .running_generation .store(generation, Ordering::SeqCst); self.fence .deadline .store(deadline.timestamp_millis(), Ordering::SeqCst); self.fence .root .store(self.ready(generation, true), Ordering::SeqCst); self.fence.generation.store(generation, Ordering::SeqCst); } else { // A valid signed refusal is a fence too, not just parse/I/O errors. self.fence.close(); } Ok(()) } fn update_proof(&self) -> Result> { self.binding .as_ref() .map(|binding| { rmm_agent_core::update_health_inventory::current_proof(&self.db, binding) }) .transpose() .map(Option::flatten) } fn start_update_health( &self, proof: &rmm_domain::agent_update::UpdateInflightProof, ) -> Result> { if !self.clock.trusted() { return Ok(None); } let binding = self.binding.as_ref().context("UPDATE_BINDING_REQUIRED")?; let Some(context) = agent_update_health::context(&self.db, binding, proof, self.sync_clock.now()?)? else { return Ok(None); }; let header = context.handoff().clone(); let key = context.health_public_key(); let identity = self.certificates.current(Utc::now().timestamp())?; let client = https_channel::mtls_client( self.address, &self.ca, &identity.certificate_pem, &identity.private_key_pem, )?; let helpers = self .helpers .clone() .context("UPDATE_FULL_HELPER_REQUIRED")?; let address = self.address; Ok(Some(PendingUpdateHealth(tokio::spawn(async move { let receipt = agent_update_health::request(&client, address, &context).await?; tokio::task::spawn_blocking(move || { helpers.submit_update_health(&header, key, &receipt) }) .await??; Ok(()) })))) } fn record( &mut self, generation: u64, offer: &JobOffer, grant: &str, outcome: rmm_privileged_helper::Outcome, ) -> Result<()> { let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?; let tx = self.db.transaction()?; let revision = jobs::transition(&tx, offer, &outcome.state, grant)?; envelope.payload = Some(Payload::JobResult(jobs::result( offer, revision, grant, outcome, ))); jobs::event(&tx, &envelope)?; tx.commit()?; Ok(()) } /// New process only. Persist honest uncertainty before any helper snapshot /// can block; the ordinary first-channel replay publishes it before Sync. fn recover_interrupted(&mut self) -> Result<()> { let Some(binding) = self.binding.clone() else { return Ok(()); }; let interrupted = jobs::interrupted(&self.db, &self.ledger, &binding)?; for (offer, grant) in interrupted { if grant.is_empty() { jobs::interrupted_request(&self.db, &offer)?; } else { let saved = jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?; if matches!( binding.job(&offer.payload_bytes)?.action, rmm_domain::jobs::Action::UpdateAgent { .. } ) && saved.is_none() { // This original may have intentionally handed off and // restarted us. Gates stay closed until a genuine helper // capture proves inflight; no Unknown is ever undone. continue; } self.record( 0, &offer, &grant, saved.unwrap_or_else(|| { rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED") }), )?; } } // No finish/unlock here. Only normal bound reconciliation can release // a consumed attempt once independent definitive evidence is available. Ok(()) } async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> { // A large replay backlog must not monopolize heartbeat/cancel handling. // On backpressure timeout, discard this channel rather than continuing // to use a websocket whose partially sent frame was cancelled. let events = jobs::events(&self.db)?; tokio::time::timeout(Duration::from_secs(2), async { for mut event in events.into_iter().take(8) { event.connection_generation = generation; ws.send(&event).await?; tokio::time::sleep(Duration::from_millis(100)).await; } Ok::<_, anyhow::Error>(()) }) .await??; Ok(()) } fn permit( &mut self, generation: u64, permit: ExecutionPermit, safe_session: bool, ) -> Result<()> { let Some(offer) = jobs::requesting(&self.db)? else { anyhow::bail!("UNSOLICITED_PERMIT") }; if !permit.denial_code.is_empty() { // A denial is not proof that an earlier reserved grant never existed. self.record( generation, &offer, "", rmm_privileged_helper::Outcome::unknown("PERMIT_DENIED_RECONCILIATION_REQUIRED"), )?; return Ok(()); } ensure!( safe_session && self.clock.trusted(), "CLOCK_OR_SYNC_UNTRUSTED" ); let binding = self .binding .as_ref() .context("job trust not provisioned")? .clone(); let request = jobs::permit_request(&offer, &permit)?; let (job, grant) = binding.verify(&request, Utc::now())?; ensure!( self.ready(generation, job.execution_identity == "root"), "FENCED_SYNC_REQUIRED" ); ensure!( jobs::supported_with_binding(&job, Some(&binding)), "UNSUPPORTED_CAPABILITY" ); rmm_privileged_helper::begin_execution(&mut self.ledger, &grant, &request)?; let cancelled: bool = self.db.query_row( "SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2", rusqlite::params![offer.job_id, offer.attempt_id], |r| r.get(0), )?; if cancelled { let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT"); rmm_privileged_helper::persist_outcome(&self.ledger, &grant, &outcome)?; self.record( generation, &offer, &grant.grant_id.to_string(), outcome.clone(), )?; rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?; return Ok(()); } let mut envelope = self.envelope( generation, Payload::JobStateChanged(JobStateChanged::default()), )?; let tx = self.db.transaction()?; let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?; envelope.payload = Some(Payload::JobStateChanged(JobStateChanged { job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(), revision, state: "running".into(), grant_id: grant.grant_id.to_string(), })); jobs::event(&tx, &envelope)?; tx.commit()?; let cancellation = Arc::new(AtomicBool::new(false)); self.running_cancels.insert( (offer.job_id.clone(), offer.attempt_id.clone()), cancellation.clone(), ); let completed = self.completed_tx.clone(); let clock = self.clock.clone(); let fence = self.fence.clone(); tokio::task::spawn_blocking(move || { // Thread-pool scheduling and durable fsync may have consumed the // remaining start window. Recheck at the actual effect boundary. let outcome = if !fence.allows(generation, job.execution_identity == "root") || !clock.trusted() || binding.verify(&request, Utc::now()).is_err() { jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT") } else if job.execution_identity == "root" { apt::invoke_with_control(&binding, &job, &request, || { ensure!( !cancellation.load(Ordering::SeqCst) && fence.running_generation.load(Ordering::SeqCst) == generation && clock.trusted(), "SCRIPT_RUNNING_CANCEL_OR_FENCE" ); Ok(()) }) .unwrap_or_else(|_| { rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN") }) } else { jobs::execute(&job) }; let _ = completed.blocking_send((offer, grant, outcome)); }); Ok(()) } fn envelope(&mut self, generation: u64, payload: Payload) -> Result { Ok(Envelope { protocol_major: 1, protocol_minor: 0, message_id: Uuid::new_v4().to_string(), agent_id: self.id.clone(), enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation), connection_generation: generation, boot_id: self.boot.clone(), sequence: storage::next_sequence(&mut self.db)?, observed_at: Utc::now().timestamp_millis(), payload: Some(payload), }) } fn capabilities(&self) -> Vec { let mut capabilities = vec![ "linux.base/1".into(), "heartbeat/1".into(), "agent.update/1".into(), ]; if self.binding.is_some() { capabilities.push("rmm.sync.v1".into()); } if let Some(binding) = &self.binding && unsafe { libc::geteuid() } != 0 { if rmm_agent_linux::actions::Interpreter::PosixSh.available() { capabilities.push("run_script".into()); } capabilities.push("refresh_inventory".into()); if jobs::logs_available() { capabilities.push("query_logs".into()); } capabilities.extend(apt::capabilities(binding)); capabilities.extend(helper_sync::general_capabilities(binding)); } capabilities.sort(); capabilities.dedup(); capabilities } fn queue_inventory(&mut self) { if self.inventory_inflight || !storage::passive_collection_ready(&self.db).unwrap_or(false) { return; } self.inventory_inflight = true; let sender = self.inventory_tx.clone(); let extended = env::var("RMM_EXTENDED_INVENTORY").unwrap_or_else(|_| "1".into()) == "1"; let binding = self.binding.clone(); let catalog = self.catalog.cached(Utc::now()); tokio::task::spawn_blocking(move || { let collected_at = Utc::now(); let result = rmm_agent_linux::wire::snapshot(extended).and_then(|mut inventory| { if extended { inventory["observations"] .as_array_mut() .context("INVENTORY_OBSERVATIONS")? .push(apt::planning_observation(binding.as_ref())); inventory["collection_completed_at"] = serde_json::json!(Utc::now()); } if let Some(report) = catalog { if inventory.get("observations").is_none() { inventory["observations"] = serde_json::json!([]); inventory["collection_started_at"] = serde_json::json!(collected_at); } inventory["observations"] .as_array_mut() .context("INVENTORY_OBSERVATIONS")? .push(agent_update_catalog::observation(report, Utc::now())); inventory["collection_completed_at"] = serde_json::json!(Utc::now()); } let bytes = serde_json::to_vec(&inventory)?; ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT"); Ok((bytes, Utc::now().timestamp_millis())) }); let _ = sender.blocking_send(result); }); } fn queue_update_observation(&mut self, generation: u64) -> Result<()> { if !storage::passive_collection_ready(&self.db)? { return Ok(()); } // Until the separately qualified protected updater/helper observer is // provisioned, report its real absence; never infer apply readiness from // this schema support or the agent's own code/build version. let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured( rmm_domain::agent_update::Platform::LinuxX64, Utc::now(), ); let address = self.address; let ca = &self.ca; let certificates = &self.certificates; self.catalog .poll(self.binding.as_ref(), self.clock.clone(), address, || { let identity = certificates.current(Utc::now().timestamp())?; https_channel::mtls_client( address, ca, &identity.certificate_pem, &identity.private_key_pem, ) }); let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?; let envelope = self.envelope( generation, Payload::InventorySnapshot(rmm_protocol::InventorySnapshot { snapshot_id: Uuid::new_v4().to_string(), scope: "agent.update".into(), schema_version: 1, chunk_index: 0, chunk_count: 1, sha256: Sha256::digest(&data).to_vec(), data, completeness: "partial".into(), }), )?; jobs::event(&self.db, &envelope) } fn inventory_envelope( &mut self, generation: u64, data: Vec, observed_at: i64, ) -> Result { let mut envelope = self.envelope( generation, Payload::InventorySnapshot(InventorySnapshot { snapshot_id: Uuid::new_v4().to_string(), scope: "linux.base".into(), schema_version: 1, chunk_index: 0, chunk_count: 1, sha256: Sha256::digest(&data).to_vec(), data, completeness: "partial".into(), }), )?; envelope.observed_at = observed_at; Ok(envelope) } async fn session(&mut self) -> Result<()> { let result = self.session_inner().await; self.close_gate(); result } async fn session_inner(&mut self) -> Result<()> { self.close_gate(); self.boot = rmm_agent_linux::boot_id()?; if self.helpers.is_some() { self.helpers = self .binding .as_ref() .map(helper_sync::Helpers::configured) .transpose()?; } let advertised = self.capabilities(); let hello = self.envelope( 0, Payload::ClientHello(ClientHello { agent_version: env!("CARGO_PKG_VERSION").into(), capabilities: advertised.clone(), }), )?; let identity = self.certificates.current(Utc::now().timestamp())?; let mut ws = wire_channel::connect(self.mode, self.address, &self.ca, &identity, hello).await?; let reply = tokio::time::timeout(Duration::from_secs(20), ws.receive()).await??; ensure!( reply.protocol_major == 1 && reply.protocol_minor == 0 && reply.agent_id == self.id && reply.enrollment_generation == self.binding.as_ref().map_or(1, |b| b.enrollment_generation), "invalid server hello" ); let Some(Payload::ServerHello(server)) = reply.payload else { anyhow::bail!("server hello required") }; let generation = server.connection_generation; ensure!( generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32, "unsupported limits" ); if let Some(mut pending) = storage::pending(&self.db)? { // Generation is transport fencing, not part of durable message identity. pending.connection_generation = generation; jobs::event(&self.db, &pending)?; storage::clear_pending(&self.db)?; } self.queue_inventory(); self.queue_update_observation(generation)?; self.replay(&ws, generation).await?; let mut needs_sync = false; if self.sync.is_some() && let Err(error) = self.start_sync(&ws, generation).await { report_closed_gate(&error); self.close_gate(); needs_sync = true; } eprintln!("authenticated channel established; inventory queued for durable acknowledgment"); let mut inventory_at = Instant::now(); let mut update_at = Instant::now(); let mut heartbeat_at = Instant::now(); let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000)); let mut tick = tokio::time::interval(Duration::from_secs(5)); let mut sync_at = Instant::now(); let mut rotation_at = Instant::now(); let mut capability_at = Instant::now(); let mut first_heartbeat = true; let mut health_task: Option = None; let mut health_inventory_prefix: Option = None; loop { tokio::select! { Some(result) = self.inventory_rx.recv() => { self.inventory_inflight = false; if let Ok((data,observed_at)) = result { let inventory = self.inventory_envelope(generation,data,observed_at)?; jobs::event(&self.db,&inventory)?; if let Some(binding)=&self.binding && let Err(error)=rmm_agent_core::update_health_inventory::retain_current(&self.db,&inventory,binding) { report_closed_gate(&error);self.close_gate();needs_sync=true; } } else { eprintln!("inventory collection failed; previous observations remain historical"); } } Some(result)=async { match health_task.as_mut() {Some(task)=>Some((&mut task.0).await),None=>std::future::pending().await} } => { health_task.take(); if let Err(error)=result.map_err(anyhow::Error::from).and_then(std::convert::identity) {report_closed_gate(&error);} self.close_gate();needs_sync=true; }, message = ws.receive() => { let message = message?; ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE"); match message.payload { Some(Payload::Ack(ack)) => { use rusqlite::OptionalExtension; let pending:Option>=self.db.query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?; if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;} jobs::ack(&mut self.db, &ack.message_id)?; }, Some(Payload::SyncResponse(response))=>{if let Err(error)=self.apply_sync(generation,response).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}sync_at=Instant::now();}, Some(Payload::JobOffer(offer)) => { let binding = self.binding.as_ref().context("job trust not provisioned")?.clone(); let accepted = self.envelope(generation, Payload::JobAccepted(JobAccepted { job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(), job_payload_hash: offer.job_payload_hash.clone(), journal_proof: rmm_domain::jobs::hash(&offer.encode_to_vec()), }))?; // Clone before borrowing the transaction; acceptance and the // durable ACK event commit together, before any wire ACK. let tx = self.db.transaction()?; let fresh=jobs::accept(&tx, &binding, &offer)?; jobs::event(&tx, &accepted)?; tx.commit()?; needs_sync|=fresh;self.replay(&ws, generation).await?; } Some(Payload::ExecutionPermit(permit)) => self.permit(generation, permit, self.ready(generation,false))?, Some(Payload::CancelRequest(cancel)) => { let row: Option<(Vec, String, String)> = { use rusqlite::OptionalExtension; self.db.query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()? }; if let Some((bytes,state,grant)) = row { if let Some(flag) = self.running_cancels.get(&(cancel.job_id.clone(),cancel.attempt_id.clone())) { flag.store(true,Ordering::SeqCst); } self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?; if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; } // Running script cancellation is not confirmed by an // inbound request. Its bounded executor reports reality. } let ack = self.envelope(generation, Payload::Ack(Ack { message_id: message.message_id }))?; ws.send(&ack).await?; } _ => anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE"), } } Some((offer, grant, outcome)) = self.completed_rx.recv() => { self.running_cancels.remove(&(offer.job_id.clone(),offer.attempt_id.clone())); // First preserve the original completion without unlocking. // A startup can replay it if the journal transaction is lost. let state = outcome.state.clone(); let refresh_capabilities=state=="succeeded" && serde_json::from_slice::(&offer.payload_bytes)?.action.name()=="apt_native_acceptance"; rmm_privileged_helper::persist_outcome(&self.ledger,&grant,&outcome)?; let serialized = serde_json::to_vec(&outcome)?; self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?; let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?; rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?; if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); } needs_sync=true;self.replay(&ws, generation).await?; if refresh_capabilities {return Ok(());} } _ = tick.tick() => { self.replay(&ws, generation).await?; let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?; let update=self.update_proof()?; let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false); let own_update=update.is_some() || pending_update; if let Some(proof)=&update { let decoded=proof.decode().map_err(anyhow::Error::msg)?; let event:Option=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?; if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) { if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) { // Fixed update health includes one genuinely new // normal base sample, never a fresh timestamp on old data. self.queue_inventory();health_inventory_prefix=Some(proof.head_sha256.clone()); } if health_task.is_none() && self.prepared.is_none() { match self.start_update_health(proof) {Ok(task)=>health_task=task,Err(error)=>{report_closed_gate(&error);self.close_gate();needs_sync=true;}} } } } if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}} if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;} if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) { if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now(); } if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some() && self.clock.trusted() && let Some(offer) = jobs::next(&self.db)? { let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?; if Utc::now() >= job.start_before { self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?; } else if !self.ready(generation,job.execution_identity=="root") { needs_sync=true; } else if Utc::now() >= job.not_before { // Reserve before send. A lost response must never cause // a second permit request for this attempt. jobs::reserve_request(&self.db, &offer)?; let request = self.envelope(generation, Payload::ExecutionPermitRequest(ExecutionPermitRequest { job_id: offer.job_id, attempt_id: offer.attempt_id, job_payload_hash: offer.job_payload_hash, assignment_version: job.assignment_version, }))?; ws.send(&request).await?; } } } } if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after { first_heartbeat = false; let heartbeat = self.envelope( generation, Payload::Heartbeat(Heartbeat { instance_id: self.instance.clone(), uptime_seconds: self.started.elapsed().as_secs(), }), )?; ws.send(&heartbeat).await?; if ws.uses_https() { let poll = self.envelope( generation, Payload::Heartbeat(Heartbeat { instance_id: self.instance.clone(), uptime_seconds: self.started.elapsed().as_secs(), }), )?; ws.poll(poll).await?; } heartbeat_at = Instant::now(); heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000)); } // Planning pins must be refreshed before the 90-second stored-data // planning limit. One inflight blocking collector bounds pressure; // heartbeats never perform this metadata work. if update_at.elapsed() >= Duration::from_secs(60) { self.queue_update_observation(generation)?; update_at = Instant::now(); } if inventory_at.elapsed() >= Duration::from_secs(60) { self.queue_inventory(); inventory_at = Instant::now(); } if rotation_at.elapsed() >= Duration::from_secs(30) { rotation_at = Instant::now(); if self .certificates .due(self.rotation_policy, Utc::now().timestamp())? && self.update_proof()?.is_none() { self.close_gate(); drop(ws); let rotated = self .certificates .renew(self.address, Utc::now().timestamp()) .await?; ensure!( rotated.needs_reconnect && rotated.enrollment_generation == self.binding.as_ref().map_or(1, |b| b.enrollment_generation), "ROTATION_IDENTITY_CHANGED" ); return Ok(()); } } } } } async fn enrollment(dir: &Path, address: std::net::SocketAddr, ca: &[u8]) -> Result { let identity = dir.join("identity.json"); if identity.exists() { let response: EnrollmentResponse = serde_json::from_slice(&fs::read(identity)?)?; Uuid::parse_str(&response.agent_id)?; return Ok(response.agent_id); } let key_path = dir.join("device.key"); let key = if key_path.exists() { PKey::private_key_from_pem(&fs::read(&key_path)?)? } else { let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?; let key = PKey::from_ec_key(EcKey::generate(&group)?)?; storage::create_secret(&key_path, &key.private_key_to_pem_pkcs8()?)?; key }; let mut csr = X509Req::builder()?; let mut name = X509NameBuilder::new()?; name.append_entry_by_text("CN", "new-linux-device")?; csr.set_subject_name(&name.build())?; csr.set_pubkey(&key)?; csr.sign(&key, MessageDigest::sha256())?; let token_path = env::var("RMM_ENROLLMENT_TOKEN_FILE") .context("enrollment token file required for first registration")?; let token = fs::read_to_string(token_path)?.trim().to_owned(); ensure!(token.len() == 64, "invalid enrollment token format"); let client = reqwest::Client::builder() .https_only(true) .tls_built_in_root_certs(false) .add_root_certificate(reqwest::Certificate::from_pem(ca)?) .resolve(&https_channel::server_name()?, address) .no_proxy() .redirect(reqwest::redirect::Policy::none()) .timeout(Duration::from_secs(15)) .build()?; let result = client .post(format!( "https://{}:{}/agent/v1/enroll", https_channel::server_name()?, address.port() )) .json(&EnrollmentRequest { token, csr_pem: String::from_utf8(csr.build().to_pem()?)?, }) .send() .await?; ensure!( result.status().is_success(), "enrollment rejected (response details suppressed)" ); let response: EnrollmentResponse = result.json().await?; let certificate = openssl::x509::X509::from_pem(response.certificate_pem.as_bytes())?; ensure!( certificate.public_key()?.public_eq(&key), "enrollment certificate key mismatch" ); Uuid::parse_str(&response.agent_id)?; storage::create_secret(&dir.join("device.pem"), response.certificate_pem.as_bytes())?; storage::create_secret(&identity, &serde_json::to_vec(&response)?)?; Ok(response.agent_id) } #[tokio::main] async fn main() -> Result<()> { let args: Vec<_> = env::args_os().skip(1).collect(); if args == [std::ffi::OsString::from("--help")] { println!( "rmm-agent [--help]\nEnvironment: RMM_AGENT_STATE, RMM_CA_FILE, RMM_CONNECT_ADDR (TCP only; TLS name rmm-dev.internal), RMM_TRANSPORT=auto|wss|https, RMM_ENROLLMENT_TOKEN_FILE (first enrollment), RMM_JOB_BINDING_FILE, RMM_JOB_SIGNING_KEY_BOOTSTRAP=1, RMM_HELPER_SOCKET, RMM_HELPER_POLICY_FILE, RMM_APT_HELPER_SOCKET, RMM_APT_HELPER_PROOF_FILE, RMM_LOG_UNITS_FILE, RMM_EXTENDED_INVENTORY=0|1 (default 1).\nDevelopment-only rotation test: RMM_AGENT_PROFILE=development and RMM_ROTATION_TEST_AFTER_SECONDS.\nNo argument or environment variable approves jobs or disables TLS/grant verification." ); return Ok(()); } ensure!(args.is_empty(), "UNKNOWN_AGENT_ARGUMENT"); let dir = PathBuf::from(env::var("RMM_AGENT_STATE").unwrap_or_else(|_| "devstate/agent".into())); storage::prepare_dir(&dir)?; let _lock = storage::lock(&dir)?; let address = env::var("RMM_CONNECT_ADDR") .unwrap_or_else(|_| "127.0.0.1:18443".into()) .parse()?; let ca = fs::read(env::var("RMM_CA_FILE").unwrap_or_else(|_| "devstate/pki/ca.pem".into()))?; let id=enrollment(&dir,address,&ca).await.map_err(|_| anyhow::anyhow!("enrollment/identity initialization failed; check trusted CA, token validity and local state"))?; let mut binding = env::var("RMM_JOB_BINDING_FILE") .ok() .map(|path| { let binding = rmm_privileged_helper::read_binding(Path::new(&path), unsafe { libc::geteuid() })?; let identity: EnrollmentResponse = serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?; ensure!( binding.agent_id.to_string() == id && binding.asset_id.to_string() == identity.asset_id, "BOOTSTRAP_IDENTITY_MISMATCH" ); Ok::<_, anyhow::Error>(binding) }) .transpose()?; let certificate_dir = dir.join("certificates"); let certificates = certificate_rotation::RotationStore::open( &certificate_dir, &ca, Uuid::parse_str(&id)?, binding.as_ref().map_or(1, |b| b.enrollment_generation), )?; if !certificate_dir.join("current.json").try_exists()? { certificates.initialize( &fs::read(dir.join("device.pem"))?, &fs::read(dir.join("device.key"))?, Utc::now().timestamp(), )?; } if env::var("RMM_JOB_SIGNING_KEY_BOOTSTRAP").as_deref() == Ok("1") { bootstrap::provision_current( &dir, address, &ca, binding .as_mut() .context("bootstrap requires provisioned device binding")?, &certificates.current(Utc::now().timestamp())?, ) .await?; } let (completed_tx, completed_rx) = tokio::sync::mpsc::channel(1); let (inventory_tx, inventory_rx) = tokio::sync::mpsc::channel(1); let sync = binding .as_ref() .map(|b| reconcile::ClientSync::new(b.clone())); let helpers = binding .as_ref() .map(helper_sync::Helpers::configured) .transpose()?; let mut rotation_policy = certificate_rotation::RotationPolicy::default(); if let Ok(after) = env::var("RMM_ROTATION_TEST_AFTER_SECONDS") { ensure!( env::var("RMM_AGENT_PROFILE").as_deref() == Ok("development"), "ROTATION_TEST_REQUIRES_DEVELOPMENT_PROFILE" ); rotation_policy.begin_after_seconds = after.parse()?; ensure!( rotation_policy.begin_after_seconds > 0, "ROTATION_TEST_INTERVAL" ); } let mode = wire_channel::Mode::parse(&env::var("RMM_TRANSPORT").unwrap_or_else(|_| "auto".into()))?; let mut agent = Agent { address, ca, id, boot: rmm_agent_linux::boot_id()?, instance: Uuid::new_v4().to_string(), db: storage::open(&dir)?, started: Instant::now(), clock: rmm_privileged_helper::TrustedClock::new(), binding, ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() })?, completed_tx, completed_rx, inventory_tx, inventory_rx, inventory_inflight: false, running_cancels: std::collections::HashMap::new(), sync, sync_clock: reconcile::NativeClock::default(), helpers, prepared: None, fence: Arc::new(EffectFence::default()), mode, certificates, rotation_policy, catalog: agent_update_catalog::Checker::default(), }; agent.recover_interrupted()?; let mut backoff = 1_u64; loop { let started = Instant::now(); if let Err(error) = agent.session().await { report_closed_gate(&error); } agent.close_gate(); eprintln!("channel unavailable; retrying with jitter (no jobs can execute)"); if started.elapsed() > Duration::from_secs(60) { backoff = 1; } tokio::time::sleep(https_channel::retry_delay(backoff as u32)).await; backoff = (backoff + 1).min(6); } }