Files
rust-discovery/crates/agent/src/main.rs
T

1091 lines
50 KiB
Rust

mod agent_update_apply;
mod agent_update_catalog;
mod agent_update_health;
mod apt;
mod bootstrap;
mod certificate_rotation;
mod helper_sync;
mod https_channel;
mod jobs;
mod reconcile;
#[cfg(test)]
mod runtime_tests;
mod storage;
mod transport;
mod wire_channel;
use anyhow::{Context, Result, ensure};
use chrono::Utc;
use openssl::{
ec::{EcGroup, EcKey},
hash::MessageDigest,
nid::Nid,
pkey::PKey,
x509::{X509NameBuilder, X509Req},
};
use prost::Message;
use rmm_protocol::*;
use sha2::{Digest, Sha256};
use std::{
env, fs,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use uuid::Uuid;
struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
address: std::net::SocketAddr,
ca: Vec<u8>,
id: String,
boot: String,
instance: String,
db: rusqlite::Connection,
started: Instant,
clock: rmm_privileged_helper::TrustedClock,
binding: Option<rmm_privileged_helper::Binding>,
ledger: rusqlite::Connection,
completed_tx: tokio::sync::mpsc::Sender<Completion>,
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
inventory_tx: tokio::sync::mpsc::Sender<Result<(Vec<u8>, i64)>>,
inventory_rx: tokio::sync::mpsc::Receiver<Result<(Vec<u8>, i64)>>,
inventory_inflight: bool,
running_cancels: std::collections::HashMap<(String, String), Arc<AtomicBool>>,
sync: Option<reconcile::ClientSync>,
sync_clock: C,
helpers: Option<helper_sync::Helpers>,
prepared: Option<reconcile::PreparedSync>,
fence: Arc<EffectFence>,
mode: wire_channel::Mode,
certificates: certificate_rotation::RotationStore,
rotation_policy: certificate_rotation::RotationPolicy,
catalog: agent_update_catalog::Checker,
update_observer: agent_update_apply::Observer,
}
#[derive(Default)]
struct EffectFence {
running_generation: AtomicU64,
generation: AtomicU64,
deadline: AtomicI64,
root: AtomicBool,
}
fn report_closed_gate(error: &anyhow::Error) {
let text = error.to_string();
let code = if text.len() <= 96
&& text
.bytes()
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
{
text.as_str()
} else {
"SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE"
};
eprintln!("agent effect gate closed: {code}");
}
impl EffectFence {
fn close(&self) {
self.running_generation.store(0, Ordering::SeqCst);
self.close_starts();
}
// A signed Sync/start lease authorizes NEW starts. Its natural expiry
// cannot retroactively stop an approved bounded process (spec22.3).
// Running effects instead observe their exact cancellation and current
// connection generation; transport/signed refusal closes both fences.
fn close_starts(&self) {
self.generation.store(0, Ordering::SeqCst);
self.root.store(false, Ordering::SeqCst);
}
fn allows(&self, generation: u64, root: bool) -> bool {
self.generation.load(Ordering::SeqCst) == generation
&& generation > 0
&& Utc::now().timestamp_millis() < self.deadline.load(Ordering::SeqCst)
&& (!root || self.root.load(Ordering::SeqCst))
}
}
struct PendingUpdateHealth(tokio::task::JoinHandle<Result<()>>);
impl Drop for PendingUpdateHealth {
fn drop(&mut self) {
self.0.abort();
}
}
type Completion = (
JobOffer,
rmm_domain::jobs::ExecutionPermitPayload,
rmm_privileged_helper::Outcome,
);
impl<C: reconcile::SyncClock> Agent<C> {
fn close_gate(&mut self) {
self.fence.close();
if let Some(sync) = &mut self.sync {
sync.close();
}
self.prepared = None;
}
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
Ok(reconcile::Channel {
connection_generation: generation,
boot_id: Uuid::parse_str(&self.boot)?,
})
}
fn ready(&self, generation: u64, root: bool) -> bool {
self.channel(generation).is_ok_and(|c| {
self.sync
.as_ref()
.is_some_and(|s| s.can_start(&c, root, &self.sync_clock))
})
}
async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> {
self.fence.close_starts();
let helpers = self.helpers.clone();
let snapshot = tokio::task::spawn_blocking(move || match helpers {
Some(h) => h.snapshot(generation),
None => Ok(None),
})
.await??;
let channel = self.channel(generation)?;
let Some(sync) = &mut self.sync else {
return Ok(());
};
let prepared = sync.prepare(
&mut self.db,
&self.ledger,
channel,
snapshot.as_ref(),
&self.sync_clock,
)?;
wire.send(&prepared.envelope).await?;
self.prepared = Some(prepared);
Ok(())
}
async fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
let prepared = self.prepared.take().context("UNSOLICITED_SYNC_RESPONSE")?;
let sync = self.sync.as_mut().context("SYNC_BINDING_REQUIRED")?;
let applied = sync.apply(&mut self.db, &response, &self.sync_clock)?;
sync.apply_consumption(&mut self.ledger, &self.sync_clock)?;
let reason = if applied
.reason
.bytes()
.all(|b| b.is_ascii_uppercase() || b == b'_')
{
applied.reason.as_str()
} else {
"RECONCILIATION_REQUIRED"
};
eprintln!(
"agent reconciliation committed: decision={}, receipts={}, reason={}",
applied.decision_id,
applied.acknowledged_message_ids.len(),
reason
);
if let Some(instruction) = applied.helper_instruction {
let helpers = self
.helpers
.clone()
.context("HELPER_SYNC_CLIENT_REQUIRED")?;
let request = prepared.request.clone();
let snapshot =
tokio::task::spawn_blocking(move || helpers.apply(&instruction.response, &request))
.await??
.context("HELPER_SYNC_ACK_REQUIRED")?;
if self.ready(generation, false) {
self.sync
.as_mut()
.context("SYNC_REQUIRED")?
.confirm_helper(&snapshot)?;
} else if snapshot
.attempts
.iter()
.any(|a| !a.update_handoff_bytes.is_empty())
{
self.sync
.as_mut()
.context("SYNC_REQUIRED")?
.confirm_update_helper(&snapshot)?;
}
}
if self.ready(generation, false) {
let deadline = self
.sync
.as_ref()
.and_then(|s| s.validity_deadline())
.context("SIGNED_LEASE_REQUIRED")?;
self.fence
.running_generation
.store(generation, Ordering::SeqCst);
self.fence
.deadline
.store(deadline.timestamp_millis(), Ordering::SeqCst);
self.fence
.root
.store(self.ready(generation, true), Ordering::SeqCst);
self.fence.generation.store(generation, Ordering::SeqCst);
} else {
// A valid signed refusal is a fence too, not just parse/I/O errors.
self.fence.close();
}
Ok(())
}
fn update_proof(&self) -> Result<Option<rmm_domain::agent_update::UpdateInflightProof>> {
self.binding
.as_ref()
.map(|binding| {
rmm_agent_core::update_health_inventory::current_proof(&self.db, binding)
})
.transpose()
.map(Option::flatten)
}
fn start_update_health(
&self,
proof: &rmm_domain::agent_update::UpdateInflightProof,
) -> Result<Option<PendingUpdateHealth>> {
if !self.clock.trusted() {
return Ok(None);
}
let binding = self.binding.as_ref().context("UPDATE_BINDING_REQUIRED")?;
let Some(context) =
agent_update_health::context(&self.db, binding, proof, self.sync_clock.now()?)?
else {
return Ok(None);
};
let header = context.handoff().clone();
let key = context.health_public_key();
let identity = self.certificates.current(Utc::now().timestamp())?;
let client = https_channel::mtls_client(
self.address,
&self.ca,
&identity.certificate_pem,
&identity.private_key_pem,
)?;
let helpers = self
.helpers
.clone()
.context("UPDATE_FULL_HELPER_REQUIRED")?;
let address = self.address;
Ok(Some(PendingUpdateHealth(tokio::spawn(async move {
let receipt = agent_update_health::request(&client, address, &context).await?;
tokio::task::spawn_blocking(move || {
helpers.submit_update_health(&header, key, &receipt)
})
.await??;
Ok(())
}))))
}
fn record(
&mut self,
generation: u64,
offer: &JobOffer,
grant: &str,
outcome: rmm_privileged_helper::Outcome,
) -> Result<()> {
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
let tx = self.db.transaction()?;
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
envelope.payload = Some(Payload::JobResult(jobs::result(
offer, revision, grant, outcome,
)));
jobs::event(&tx, &envelope)?;
tx.commit()?;
Ok(())
}
/// New process only. Persist honest uncertainty before any helper snapshot
/// can block; the ordinary first-channel replay publishes it before Sync.
fn recover_interrupted(&mut self) -> Result<()> {
let Some(binding) = self.binding.clone() else {
return Ok(());
};
let interrupted = jobs::interrupted(&self.db, &self.ledger, &binding)?;
for (offer, grant) in interrupted {
if grant.is_empty() {
jobs::interrupted_request(&self.db, &offer)?;
} else {
let saved =
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?;
if matches!(
binding.job(&offer.payload_bytes)?.action,
rmm_domain::jobs::Action::UpdateAgent { .. }
) && saved.is_none()
{
// This original may have intentionally handed off and
// restarted us. Gates stay closed until a genuine helper
// capture proves inflight; no Unknown is ever undone.
continue;
}
self.record(
0,
&offer,
&grant,
saved.unwrap_or_else(|| {
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
}),
)?;
}
}
// No finish/unlock here. Only normal bound reconciliation can release
// a consumed attempt once independent definitive evidence is available.
Ok(())
}
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
// A large replay backlog must not monopolize heartbeat/cancel handling.
// On backpressure timeout, discard this channel rather than continuing
// to use a websocket whose partially sent frame was cancelled.
let events = jobs::events(&self.db)?;
tokio::time::timeout(Duration::from_secs(2), async {
for mut event in events.into_iter().take(8) {
event.connection_generation = generation;
ws.send(&event).await?;
tokio::time::sleep(Duration::from_millis(100)).await;
}
Ok::<_, anyhow::Error>(())
})
.await??;
Ok(())
}
fn permit(
&mut self,
generation: u64,
permit: ExecutionPermit,
safe_session: bool,
) -> Result<()> {
let Some(offer) = jobs::requesting(&self.db)? else {
anyhow::bail!("UNSOLICITED_PERMIT")
};
if !permit.denial_code.is_empty() {
jobs::denied_request(&self.db, &offer, &permit)?;
self.close_gate();
return Ok(());
}
ensure!(
safe_session && self.clock.trusted(),
"CLOCK_OR_SYNC_UNTRUSTED"
);
let binding = self
.binding
.as_ref()
.context("job trust not provisioned")?
.clone();
let request = jobs::permit_request(&offer, &permit)?;
let (job, grant) = binding.verify(&request, Utc::now())?;
ensure!(
self.ready(generation, job.execution_identity == "root"),
"FENCED_SYNC_REQUIRED"
);
ensure!(
jobs::supported_with_binding(&job, Some(&binding)),
"UNSUPPORTED_CAPABILITY"
);
rmm_privileged_helper::begin_execution(&mut self.ledger, &grant, &request)?;
let cancelled: bool = self.db.query_row(
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
rusqlite::params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
if cancelled {
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
rmm_privileged_helper::persist_outcome(&self.ledger, &grant, &outcome)?;
self.record(
generation,
&offer,
&grant.grant_id.to_string(),
outcome.clone(),
)?;
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
return Ok(());
}
let mut envelope = self.envelope(
generation,
Payload::JobStateChanged(JobStateChanged::default()),
)?;
let tx = self.db.transaction()?;
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
revision,
state: "running".into(),
grant_id: grant.grant_id.to_string(),
}));
jobs::event(&tx, &envelope)?;
tx.commit()?;
let cancellation = Arc::new(AtomicBool::new(false));
self.running_cancels.insert(
(offer.job_id.clone(), offer.attempt_id.clone()),
cancellation.clone(),
);
let completed = self.completed_tx.clone();
let clock = self.clock.clone();
let fence = self.fence.clone();
tokio::task::spawn_blocking(move || {
// Thread-pool scheduling and durable fsync may have consumed the
// remaining start window. Recheck at the actual effect boundary.
let outcome = if !fence.allows(generation, job.execution_identity == "root")
|| !clock.trusted()
|| binding.verify(&request, Utc::now()).is_err()
{
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
} else if matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) {
match agent_update_apply::handoff(&binding, &request, grant.grant_id) {
// Full now owns this original. The stable updater may
// stop us. Reconciliation alone supplies its final result.
Ok(_) => return,
Err(_) => rmm_privileged_helper::Outcome::unknown("UPDATE_HANDOFF_UNCONFIRMED"),
}
} else if job.execution_identity == "root" {
apt::invoke_with_control(&binding, &job, &request, || {
ensure!(
!cancellation.load(Ordering::SeqCst)
&& fence.running_generation.load(Ordering::SeqCst) == generation
&& clock.trusted(),
"SCRIPT_RUNNING_CANCEL_OR_FENCE"
);
Ok(())
})
.unwrap_or_else(|_| {
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
})
} else {
jobs::execute(&job)
};
let _ = completed.blocking_send((offer, grant, outcome));
});
Ok(())
}
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
Ok(Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: self.id.clone(),
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
connection_generation: generation,
boot_id: self.boot.clone(),
sequence: storage::next_sequence(&mut self.db)?,
observed_at: Utc::now().timestamp_millis(),
payload: Some(payload),
})
}
fn capabilities(&self) -> Vec<String> {
let mut capabilities = vec![
"linux.base/1".into(),
"heartbeat/1".into(),
"agent.update/1".into(),
];
if self.binding.is_some() {
capabilities.push("rmm.sync.v1".into());
}
if let Some(binding) = &self.binding
&& unsafe { libc::geteuid() } != 0
{
if rmm_agent_linux::actions::Interpreter::PosixSh.available() {
capabilities.push("run_script".into());
}
capabilities.push("refresh_inventory".into());
if jobs::logs_available() {
capabilities.push("query_logs".into());
}
capabilities.extend(apt::capabilities(binding));
capabilities.extend(helper_sync::general_capabilities(binding));
if self.clock.trusted() {
capabilities.extend(self.update_observer.capabilities());
}
}
capabilities.sort();
capabilities.dedup();
capabilities
}
fn queue_inventory(&mut self) {
if self.inventory_inflight || !storage::passive_collection_ready(&self.db).unwrap_or(false)
{
return;
}
self.inventory_inflight = true;
let sender = self.inventory_tx.clone();
let extended = env::var("RMM_EXTENDED_INVENTORY").unwrap_or_else(|_| "1".into()) == "1";
let binding = self.binding.clone();
let catalog = self.catalog.cached(Utc::now());
tokio::task::spawn_blocking(move || {
let collected_at = Utc::now();
let result = rmm_agent_linux::wire::snapshot(extended).and_then(|mut inventory| {
if extended {
inventory["observations"]
.as_array_mut()
.context("INVENTORY_OBSERVATIONS")?
.push(apt::planning_observation(binding.as_ref()));
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
}
if let Some(report) = catalog {
if inventory.get("observations").is_none() {
inventory["observations"] = serde_json::json!([]);
inventory["collection_started_at"] = serde_json::json!(collected_at);
}
inventory["observations"]
.as_array_mut()
.context("INVENTORY_OBSERVATIONS")?
.push(agent_update_catalog::observation(report, Utc::now()));
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
}
let bytes = serde_json::to_vec(&inventory)?;
ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT");
Ok((bytes, Utc::now().timestamp_millis()))
});
let _ = sender.blocking_send(result);
});
}
fn queue_update_observation(&mut self, generation: u64) -> Result<()> {
if !storage::passive_collection_ready(&self.db)? {
return Ok(());
}
let address = self.address;
let ca = &self.ca;
let certificates = &self.certificates;
self.catalog.poll(
self.binding.as_ref(),
self.clock.clone(),
address,
self.update_observer.catalog_record(),
|| {
let identity = certificates.current(Utc::now().timestamp())?;
https_channel::mtls_client(
address,
ca,
&identity.certificate_pem,
&identity.private_key_pem,
)
},
);
let observation = self.update_observer.poll(
self.binding.as_ref(),
self.catalog.candidate_envelope(Utc::now()),
);
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
let envelope = self.envelope(
generation,
Payload::InventorySnapshot(rmm_protocol::InventorySnapshot {
snapshot_id: Uuid::new_v4().to_string(),
scope: "agent.update".into(),
schema_version: 1,
chunk_index: 0,
chunk_count: 1,
sha256: Sha256::digest(&data).to_vec(),
data,
completeness: "partial".into(),
}),
)?;
jobs::event(&self.db, &envelope)
}
fn inventory_envelope(
&mut self,
generation: u64,
data: Vec<u8>,
observed_at: i64,
) -> Result<Envelope> {
let mut envelope = self.envelope(
generation,
Payload::InventorySnapshot(InventorySnapshot {
snapshot_id: Uuid::new_v4().to_string(),
scope: "linux.base".into(),
schema_version: 1,
chunk_index: 0,
chunk_count: 1,
sha256: Sha256::digest(&data).to_vec(),
data,
completeness: "partial".into(),
}),
)?;
envelope.observed_at = observed_at;
Ok(envelope)
}
async fn session(&mut self) -> Result<()> {
let result = self.session_inner().await;
self.close_gate();
result
}
async fn session_inner(&mut self) -> Result<()> {
self.close_gate();
self.boot = rmm_agent_linux::boot_id()?;
if self.helpers.is_some() {
self.helpers = self
.binding
.as_ref()
.map(helper_sync::Helpers::configured)
.transpose()?;
}
let advertised = self.capabilities();
let hello = self.envelope(
0,
Payload::ClientHello(ClientHello {
agent_version: env::var("RMM_INSTALLED_RELEASE_VERSION").unwrap_or_else(|_| env!("CARGO_PKG_VERSION").into()),
capabilities: advertised.clone(),
}),
)?;
let identity = self.certificates.current(Utc::now().timestamp())?;
let mut ws =
wire_channel::connect(self.mode, self.address, &self.ca, &identity, hello).await?;
let reply = tokio::time::timeout(Duration::from_secs(20), ws.receive()).await??;
ensure!(
reply.protocol_major == 1
&& reply.protocol_minor == 0
&& reply.agent_id == self.id
&& reply.enrollment_generation
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
"invalid server hello"
);
let Some(Payload::ServerHello(server)) = reply.payload else {
anyhow::bail!("server hello required")
};
let generation = server.connection_generation;
ensure!(
generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32,
"unsupported limits"
);
if let Some(mut pending) = storage::pending(&self.db)? {
// Generation is transport fencing, not part of durable message identity.
pending.connection_generation = generation;
jobs::event(&self.db, &pending)?;
storage::clear_pending(&self.db)?;
}
self.queue_inventory();
self.queue_update_observation(generation)?;
self.replay(&ws, generation).await?;
let mut needs_sync = false;
let mut permit_after = Instant::now();
if self.sync.is_some()
&& let Err(error) = self.start_sync(&ws, generation).await
{
report_closed_gate(&error);
self.close_gate();
needs_sync = true;
}
eprintln!("authenticated channel established; inventory queued for durable acknowledgment");
let mut inventory_at = Instant::now();
let mut update_at = Instant::now();
let mut heartbeat_at = Instant::now();
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
let mut tick = tokio::time::interval(Duration::from_secs(5));
let mut sync_at = Instant::now();
let mut rotation_at = Instant::now();
let mut capability_at = Instant::now();
let mut first_heartbeat = true;
let mut health_task: Option<PendingUpdateHealth> = None;
let mut health_inventory_prefix: Option<String> = None;
let mut update_prefetch: Option<(JobOffer, agent_update_apply::Prefetch)> = None;
let mut prefetched = std::collections::HashSet::new();
loop {
tokio::select! {
Some(result) = self.inventory_rx.recv() => {
self.inventory_inflight = false;
if let Ok((data,observed_at)) = result {
let inventory = self.inventory_envelope(generation,data,observed_at)?;
jobs::event(&self.db,&inventory)?;
if let Some(binding)=&self.binding
&& let Err(error)=rmm_agent_core::update_health_inventory::retain_current(&self.db,&inventory,binding) {
report_closed_gate(&error);self.close_gate();needs_sync=true;
}
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
}
Some(result)=async {
match health_task.as_mut() {Some(task)=>Some((&mut task.0).await),None=>std::future::pending().await}
} => {
health_task.take();
if let Err(error)=result.map_err(anyhow::Error::from).and_then(std::convert::identity) {report_closed_gate(&error);}
self.close_gate();needs_sync=true;
},
message = ws.receive() => {
let message = message?;
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
match message.payload {
Some(Payload::Ack(ack)) => {
use rusqlite::OptionalExtension;
let pending:Option<Vec<u8>>=self.db.query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
jobs::ack(&mut self.db, &ack.message_id)?;
},
Some(Payload::SyncResponse(response))=>{if let Err(error)=self.apply_sync(generation,response).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}sync_at=Instant::now();},
Some(Payload::JobOffer(offer)) => {
let binding = self.binding.as_ref().context("job trust not provisioned")?.clone();
let accepted = self.envelope(generation, Payload::JobAccepted(JobAccepted {
job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(),
job_payload_hash: offer.job_payload_hash.clone(), journal_proof: rmm_domain::jobs::hash(&offer.encode_to_vec()),
}))?;
// Clone before borrowing the transaction; acceptance and the
// durable ACK event commit together, before any wire ACK.
let tx = self.db.transaction()?;
let fresh=jobs::accept(&tx, &binding, &offer)?;
jobs::event(&tx, &accepted)?;
tx.commit()?;
needs_sync|=fresh;self.replay(&ws, generation).await?;
}
Some(Payload::ExecutionPermit(permit)) => {
if !permit.denial_code.is_empty() {
needs_sync = true;
permit_after = Instant::now() + Duration::from_secs(30);
}
self.permit(generation, permit, self.ready(generation,false))?;
},
Some(Payload::CancelRequest(cancel)) => {
let row: Option<(Vec<u8>, String, String)> = {
use rusqlite::OptionalExtension;
self.db.query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?
};
if let Some((bytes,state,grant)) = row {
if let Some(flag) = self.running_cancels.get(&(cancel.job_id.clone(),cancel.attempt_id.clone())) { flag.store(true,Ordering::SeqCst); }
self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?;
if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; }
// Running script cancellation is not confirmed by an
// inbound request. Its bounded executor reports reality.
}
let ack = self.envelope(generation, Payload::Ack(Ack { message_id: message.message_id }))?;
ws.send(&ack).await?;
}
_ => anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE"),
}
}
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
self.running_cancels.remove(&(offer.job_id.clone(),offer.attempt_id.clone()));
// First preserve the original completion without unlocking.
// A startup can replay it if the journal transaction is lost.
let state = outcome.state.clone();
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
rmm_privileged_helper::persist_outcome(&self.ledger,&grant,&outcome)?;
let serialized = serde_json::to_vec(&outcome)?;
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?;
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); }
needs_sync=true;self.replay(&ws, generation).await?;
if refresh_capabilities {return Ok(());}
}
_ = tick.tick() => {
self.replay(&ws, generation).await?;
if update_prefetch.as_ref().is_some_and(|(_,task)|task.0.is_finished()) {
let (offer, mut task)=update_prefetch.take().context("UPDATE_PREFETCH_REQUIRED")?;
match (&mut task.0).await {
Ok(Ok(())) => {prefetched.insert((offer.job_id.clone(),offer.attempt_id.clone()));},
_ => {self.record(generation,&offer,"",jobs::no_start("failed","UPDATE_ARTIFACT_PREFETCH_FAILED"))?;needs_sync=true;}
}
}
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
let update=self.update_proof()?;
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
let own_update=update.is_some() || pending_update;
if let Some(proof)=&update {
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..}|rmm_updater_handoff::Evidence::RollbackServicesStarted{..})) {
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
// Fixed update health includes one genuinely new
// normal base sample, never a fresh timestamp on old data.
self.queue_inventory();health_inventory_prefix=Some(proof.head_sha256.clone());
}
if health_task.is_none() && self.prepared.is_none() {
match self.start_update_health(proof) {Ok(task)=>health_task=task,Err(error)=>{report_closed_gate(&error);self.close_gate();needs_sync=true;}}
}
}
}
// Planning pins must be refreshed before the 90-second stored-data
// planning limit. One inflight blocking collector bounds pressure;
// heartbeats never perform this metadata work.
if update_at.elapsed() >= Duration::from_secs(60) {
self.queue_update_observation(generation)?;
update_at = Instant::now();
}
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
}
if Instant::now() >= permit_after && !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
&& self.clock.trusted() && let Some(offer) = jobs::next(&self.db)? {
let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?;
if Utc::now() >= job.start_before {
self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?;
} else if !self.ready(generation,job.execution_identity=="root") {
needs_sync=true;
} else if Utc::now() >= job.not_before {
if matches!(job.action,rmm_domain::jobs::Action::UpdateAgent{..}) && !prefetched.contains(&(offer.job_id.clone(),offer.attempt_id.clone())) {
if update_prefetch.is_none() {
let identity=self.certificates.current(Utc::now().timestamp())?;
let client=https_channel::mtls_client(self.address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?;
let base=format!("https://{}:{}",https_channel::server_name()?,self.address.port());
update_prefetch=Some((offer.clone(),agent_update_apply::Prefetch(tokio::spawn(agent_update_apply::prefetch(client,base,job)))));
}
continue;
}
// Reserve before send. A lost response must never cause
// a second permit request for this attempt.
// The root updater holds this lock exclusively while it checks
// the durable journal and switches a release. Shared ownership
// covers reservation so a grant cannot race an idle check.
let _release_gate = if let Some(path) = env::var_os("RMM_RELEASE_LOCK") {
let file = fs::File::open(path)?;
match fs2::FileExt::try_lock_shared(&file) {
Ok(()) => Some(file),
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => continue,
Err(e) => return Err(e.into()),
}
} else { None };
jobs::reserve_request(&self.db, &offer)?;
let request = self.envelope(generation, Payload::ExecutionPermitRequest(ExecutionPermitRequest {
job_id: offer.job_id, attempt_id: offer.attempt_id,
job_payload_hash: offer.job_payload_hash, assignment_version: job.assignment_version,
}))?;
ws.send(&request).await?;
}
}
}
}
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
first_heartbeat = false;
let heartbeat = self.envelope(
generation,
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
}),
)?;
ws.send(&heartbeat).await?;
if ws.uses_https() {
let poll = self.envelope(
generation,
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
}),
)?;
ws.poll(poll).await?;
}
heartbeat_at = Instant::now();
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
}
if inventory_at.elapsed() >= Duration::from_secs(60) {
self.queue_inventory();
inventory_at = Instant::now();
}
if rotation_at.elapsed() >= Duration::from_secs(30) {
rotation_at = Instant::now();
if self
.certificates
.due(self.rotation_policy, Utc::now().timestamp())?
&& self.update_proof()?.is_none()
{
self.close_gate();
drop(ws);
let rotated = self
.certificates
.renew(self.address, Utc::now().timestamp())
.await?;
ensure!(
rotated.needs_reconnect
&& rotated.enrollment_generation
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
"ROTATION_IDENTITY_CHANGED"
);
return Ok(());
}
}
}
}
}
async fn enrollment(dir: &Path, address: std::net::SocketAddr, ca: &[u8]) -> Result<String> {
let identity = dir.join("identity.json");
if identity.exists() {
let response: EnrollmentResponse = serde_json::from_slice(&fs::read(identity)?)?;
Uuid::parse_str(&response.agent_id)?;
return Ok(response.agent_id);
}
let key_path = dir.join("device.key");
let key = if key_path.exists() {
PKey::private_key_from_pem(&fs::read(&key_path)?)?
} else {
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
storage::create_secret(&key_path, &key.private_key_to_pem_pkcs8()?)?;
key
};
let mut csr = X509Req::builder()?;
let mut name = X509NameBuilder::new()?;
name.append_entry_by_text("CN", "new-linux-device")?;
csr.set_subject_name(&name.build())?;
csr.set_pubkey(&key)?;
csr.sign(&key, MessageDigest::sha256())?;
let token_path = env::var("RMM_ENROLLMENT_TOKEN_FILE")
.context("enrollment token file required for first registration")?;
let token = fs::read_to_string(token_path)?.trim().to_owned();
ensure!(token.len() == 64, "invalid enrollment token format");
let client = reqwest::Client::builder()
.https_only(true)
.tls_built_in_root_certs(false)
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
.resolve(&https_channel::server_name()?, address)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.timeout(Duration::from_secs(15))
.build()?;
let result = client
.post(format!(
"https://{}:{}/agent/v1/enroll",
https_channel::server_name()?,
address.port()
))
.json(&EnrollmentRequest {
token,
csr_pem: String::from_utf8(csr.build().to_pem()?)?,
})
.send()
.await?;
ensure!(
result.status().is_success(),
"enrollment rejected (response details suppressed)"
);
let response: EnrollmentResponse = result.json().await?;
let certificate = openssl::x509::X509::from_pem(response.certificate_pem.as_bytes())?;
ensure!(
certificate.public_key()?.public_eq(&key),
"enrollment certificate key mismatch"
);
Uuid::parse_str(&response.agent_id)?;
storage::create_secret(&dir.join("device.pem"), response.certificate_pem.as_bytes())?;
storage::create_secret(&identity, &serde_json::to_vec(&response)?)?;
Ok(response.agent_id)
}
#[tokio::main]
async fn main() -> Result<()> {
let args: Vec<_> = env::args_os().skip(1).collect();
if args == [std::ffi::OsString::from("--help")] {
println!(
"rmm-agent [--help]\nEnvironment: RMM_AGENT_STATE, RMM_CA_FILE, RMM_CONNECT_ADDR (TCP only; TLS name rmm-dev.internal), RMM_TRANSPORT=auto|wss|https, RMM_ENROLLMENT_TOKEN_FILE (first enrollment), RMM_JOB_BINDING_FILE, RMM_JOB_SIGNING_KEY_BOOTSTRAP=1, RMM_HELPER_SOCKET, RMM_HELPER_POLICY_FILE, RMM_APT_HELPER_SOCKET, RMM_APT_HELPER_PROOF_FILE, RMM_LOG_UNITS_FILE, RMM_EXTENDED_INVENTORY=0|1 (default 1).\nDevelopment-only rotation test: RMM_AGENT_PROFILE=development and RMM_ROTATION_TEST_AFTER_SECONDS.\nNo argument or environment variable approves jobs or disables TLS/grant verification."
);
return Ok(());
}
ensure!(args.is_empty(), "UNKNOWN_AGENT_ARGUMENT");
let dir =
PathBuf::from(env::var("RMM_AGENT_STATE").unwrap_or_else(|_| "devstate/agent".into()));
storage::prepare_dir(&dir)?;
let _lock = storage::lock(&dir)?;
let address = env::var("RMM_CONNECT_ADDR")
.unwrap_or_else(|_| "127.0.0.1:18443".into())
.parse()?;
let ca = fs::read(env::var("RMM_CA_FILE").unwrap_or_else(|_| "devstate/pki/ca.pem".into()))?;
let id=enrollment(&dir,address,&ca).await.map_err(|_| anyhow::anyhow!("enrollment/identity initialization failed; check trusted CA, token validity and local state"))?;
let mut binding = env::var("RMM_JOB_BINDING_FILE")
.ok()
.map(|path| {
let binding =
rmm_privileged_helper::read_binding(Path::new(&path), unsafe { libc::geteuid() })?;
let identity: EnrollmentResponse =
serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?;
ensure!(
binding.agent_id.to_string() == id
&& binding.asset_id.to_string() == identity.asset_id,
"BOOTSTRAP_IDENTITY_MISMATCH"
);
Ok::<_, anyhow::Error>(binding)
})
.transpose()?;
if binding.is_none() && let Some(tenant) = env::var_os("RMM_ENROLLMENT_TENANT") {
let identity: EnrollmentResponse = serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?;
binding = Some(rmm_privileged_helper::Binding {
tenant_id: Uuid::parse_str(tenant.to_str().context("invalid enrollment tenant")?)?,
agent_id: Uuid::parse_str(&identity.agent_id)?, asset_id: Uuid::parse_str(&identity.asset_id)?,
enrollment_generation: 1, assignment_version: 1, public_key_hex: String::new(),
});
}
let certificate_dir = dir.join("certificates");
let certificates = certificate_rotation::RotationStore::open(
&certificate_dir,
&ca,
Uuid::parse_str(&id)?,
binding.as_ref().map_or(1, |b| b.enrollment_generation),
)?;
if !certificate_dir.join("current.json").try_exists()? {
certificates.initialize(
&fs::read(dir.join("device.pem"))?,
&fs::read(dir.join("device.key"))?,
Utc::now().timestamp(),
)?;
}
if env::var("RMM_JOB_SIGNING_KEY_BOOTSTRAP").as_deref() == Ok("1") {
bootstrap::provision_current(
&dir,
address,
&ca,
binding
.as_mut()
.context("bootstrap requires provisioned device binding")?,
&certificates.current(Utc::now().timestamp())?,
)
.await?;
}
let (completed_tx, completed_rx) = tokio::sync::mpsc::channel(1);
let (inventory_tx, inventory_rx) = tokio::sync::mpsc::channel(1);
let sync = binding
.as_ref()
.map(|b| reconcile::ClientSync::new(b.clone()));
let helpers = binding
.as_ref()
.map(helper_sync::Helpers::configured)
.transpose()?;
let mut rotation_policy = certificate_rotation::RotationPolicy::default();
if let Ok(after) = env::var("RMM_ROTATION_TEST_AFTER_SECONDS") {
ensure!(
env::var("RMM_AGENT_PROFILE").as_deref() == Ok("development"),
"ROTATION_TEST_REQUIRES_DEVELOPMENT_PROFILE"
);
rotation_policy.begin_after_seconds = after.parse()?;
ensure!(
rotation_policy.begin_after_seconds > 0,
"ROTATION_TEST_INTERVAL"
);
}
let mode =
wire_channel::Mode::parse(&env::var("RMM_TRANSPORT").unwrap_or_else(|_| "https".into()))?;
let mut agent = Agent {
address,
ca,
id,
boot: rmm_agent_linux::boot_id()?,
instance: Uuid::new_v4().to_string(),
db: storage::open(&dir)?,
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding,
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() })?,
completed_tx,
completed_rx,
inventory_tx,
inventory_rx,
inventory_inflight: false,
running_cancels: std::collections::HashMap::new(),
sync,
sync_clock: reconcile::NativeClock::default(),
helpers,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode,
certificates,
rotation_policy,
catalog: agent_update_catalog::Checker::default(),
update_observer: agent_update_apply::Observer::default(),
};
agent.recover_interrupted()?;
let mut backoff = 1_u64;
loop {
let started = Instant::now();
if let Err(error) = agent.session().await {
report_closed_gate(&error);
}
agent.close_gate();
eprintln!("channel unavailable; retrying with jitter (no jobs can execute)");
if started.elapsed() > Duration::from_secs(60) {
backoff = 1;
}
tokio::time::sleep(https_channel::retry_delay(backoff as u32)).await;
backoff = (backoff + 1).min(6);
}
}