feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,805 @@
|
||||
mod apt;
|
||||
mod bootstrap;
|
||||
mod certificate_rotation;
|
||||
mod helper_sync;
|
||||
mod https_channel;
|
||||
mod jobs;
|
||||
mod reconcile;
|
||||
#[cfg(test)]
|
||||
mod runtime_tests;
|
||||
mod storage;
|
||||
mod transport;
|
||||
mod wire_channel;
|
||||
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use openssl::{
|
||||
ec::{EcGroup, EcKey},
|
||||
hash::MessageDigest,
|
||||
nid::Nid,
|
||||
pkey::PKey,
|
||||
x509::{X509NameBuilder, X509Req},
|
||||
};
|
||||
use prost::Message;
|
||||
use rmm_protocol::*;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
||||
address: std::net::SocketAddr,
|
||||
ca: Vec<u8>,
|
||||
id: String,
|
||||
boot: String,
|
||||
instance: String,
|
||||
db: rusqlite::Connection,
|
||||
started: Instant,
|
||||
clock: rmm_privileged_helper::TrustedClock,
|
||||
binding: Option<rmm_privileged_helper::Binding>,
|
||||
ledger: rusqlite::Connection,
|
||||
completed_tx: tokio::sync::mpsc::Sender<Completion>,
|
||||
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
||||
inventory_tx: tokio::sync::mpsc::Sender<Result<(Vec<u8>, i64)>>,
|
||||
inventory_rx: tokio::sync::mpsc::Receiver<Result<(Vec<u8>, i64)>>,
|
||||
inventory_inflight: bool,
|
||||
sync: Option<reconcile::ClientSync>,
|
||||
sync_clock: C,
|
||||
helpers: Option<helper_sync::Helpers>,
|
||||
prepared: Option<reconcile::PreparedSync>,
|
||||
fence: Arc<EffectFence>,
|
||||
mode: wire_channel::Mode,
|
||||
certificates: certificate_rotation::RotationStore,
|
||||
rotation_policy: certificate_rotation::RotationPolicy,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
generation: AtomicU64,
|
||||
deadline: AtomicI64,
|
||||
root: AtomicBool,
|
||||
}
|
||||
fn report_closed_gate(error: &anyhow::Error) {
|
||||
let text = error.to_string();
|
||||
let code = if text.len() <= 96
|
||||
&& text
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
{
|
||||
text.as_str()
|
||||
} else {
|
||||
"SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE"
|
||||
};
|
||||
eprintln!("agent effect gate closed: {code}");
|
||||
}
|
||||
impl EffectFence {
|
||||
fn close(&self) {
|
||||
self.generation.store(0, Ordering::SeqCst);
|
||||
self.root.store(false, Ordering::SeqCst);
|
||||
}
|
||||
fn allows(&self, generation: u64, root: bool) -> bool {
|
||||
self.generation.load(Ordering::SeqCst) == generation
|
||||
&& generation > 0
|
||||
&& Utc::now().timestamp_millis() < self.deadline.load(Ordering::SeqCst)
|
||||
&& (!root || self.root.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
type Completion = (
|
||||
JobOffer,
|
||||
rmm_domain::jobs::ExecutionPermitPayload,
|
||||
rmm_privileged_helper::Outcome,
|
||||
);
|
||||
impl<C: reconcile::SyncClock> Agent<C> {
|
||||
fn close_gate(&mut self) {
|
||||
self.fence.close();
|
||||
if let Some(sync) = &mut self.sync {
|
||||
sync.close();
|
||||
}
|
||||
self.prepared = None;
|
||||
}
|
||||
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
|
||||
Ok(reconcile::Channel {
|
||||
connection_generation: generation,
|
||||
boot_id: Uuid::parse_str(&self.boot)?,
|
||||
})
|
||||
}
|
||||
fn ready(&self, generation: u64, root: bool) -> bool {
|
||||
self.channel(generation).is_ok_and(|c| {
|
||||
self.sync
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.can_start(&c, root, &self.sync_clock))
|
||||
})
|
||||
}
|
||||
async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
self.fence.close();
|
||||
let helpers = self.helpers.clone();
|
||||
let snapshot = tokio::task::spawn_blocking(move || match helpers {
|
||||
Some(h) => h.snapshot(generation),
|
||||
None => Ok(None),
|
||||
})
|
||||
.await??;
|
||||
let channel = self.channel(generation)?;
|
||||
let Some(sync) = &mut self.sync else {
|
||||
return Ok(());
|
||||
};
|
||||
let prepared = sync.prepare(
|
||||
&mut self.db,
|
||||
&self.ledger,
|
||||
channel,
|
||||
snapshot.as_ref(),
|
||||
&self.sync_clock,
|
||||
)?;
|
||||
wire.send(&prepared.envelope).await?;
|
||||
self.prepared = Some(prepared);
|
||||
Ok(())
|
||||
}
|
||||
async fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
||||
let prepared = self.prepared.take().context("UNSOLICITED_SYNC_RESPONSE")?;
|
||||
let sync = self.sync.as_mut().context("SYNC_BINDING_REQUIRED")?;
|
||||
let applied = sync.apply(&mut self.db, &response, &self.sync_clock)?;
|
||||
sync.apply_consumption(&mut self.ledger, &self.sync_clock)?;
|
||||
let reason = if applied
|
||||
.reason
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b == b'_')
|
||||
{
|
||||
applied.reason.as_str()
|
||||
} else {
|
||||
"RECONCILIATION_REQUIRED"
|
||||
};
|
||||
eprintln!(
|
||||
"agent reconciliation committed: decision={}, receipts={}, reason={}",
|
||||
applied.decision_id,
|
||||
applied.acknowledged_message_ids.len(),
|
||||
reason
|
||||
);
|
||||
if let Some(instruction) = applied.helper_instruction {
|
||||
let helpers = self
|
||||
.helpers
|
||||
.clone()
|
||||
.context("HELPER_SYNC_CLIENT_REQUIRED")?;
|
||||
let request = prepared.request.clone();
|
||||
let snapshot =
|
||||
tokio::task::spawn_blocking(move || helpers.apply(&instruction.response, &request))
|
||||
.await??
|
||||
.context("HELPER_SYNC_ACK_REQUIRED")?;
|
||||
if self.ready(generation, false) {
|
||||
self.sync
|
||||
.as_mut()
|
||||
.context("SYNC_REQUIRED")?
|
||||
.confirm_helper(&snapshot)?;
|
||||
}
|
||||
}
|
||||
if self.ready(generation, false) {
|
||||
let deadline = self
|
||||
.sync
|
||||
.as_ref()
|
||||
.and_then(|s| s.validity_deadline())
|
||||
.context("SIGNED_LEASE_REQUIRED")?;
|
||||
self.fence
|
||||
.deadline
|
||||
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
||||
self.fence
|
||||
.root
|
||||
.store(self.ready(generation, true), Ordering::SeqCst);
|
||||
self.fence.generation.store(generation, Ordering::SeqCst);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn record(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
outcome: rmm_privileged_helper::Outcome,
|
||||
) -> Result<()> {
|
||||
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
|
||||
let tx = self.db.transaction()?;
|
||||
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
||||
envelope.payload = Some(Payload::JobResult(jobs::result(
|
||||
offer, revision, grant, outcome,
|
||||
)));
|
||||
jobs::event(&tx, &envelope)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
// A large replay backlog must not monopolize heartbeat/cancel handling.
|
||||
// On backpressure timeout, discard this channel rather than continuing
|
||||
// to use a websocket whose partially sent frame was cancelled.
|
||||
let events = jobs::events(&self.db)?;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
for mut event in events.into_iter().take(8) {
|
||||
event.connection_generation = generation;
|
||||
ws.send(&event).await?;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
fn permit(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
permit: ExecutionPermit,
|
||||
safe_session: bool,
|
||||
) -> Result<()> {
|
||||
let Some(offer) = jobs::requesting(&self.db)? else {
|
||||
anyhow::bail!("UNSOLICITED_PERMIT")
|
||||
};
|
||||
if !permit.denial_code.is_empty() {
|
||||
// A denial is not proof that an earlier reserved grant never existed.
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
"",
|
||||
rmm_privileged_helper::Outcome::unknown("PERMIT_DENIED_RECONCILIATION_REQUIRED"),
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
safe_session && self.clock.trusted(),
|
||||
"CLOCK_OR_SYNC_UNTRUSTED"
|
||||
);
|
||||
let binding = self
|
||||
.binding
|
||||
.as_ref()
|
||||
.context("job trust not provisioned")?
|
||||
.clone();
|
||||
let request = jobs::permit_request(&offer, &permit)?;
|
||||
let (job, grant) = binding.verify(&request, Utc::now())?;
|
||||
ensure!(
|
||||
self.ready(generation, job.execution_identity == "root"),
|
||||
"FENCED_SYNC_REQUIRED"
|
||||
);
|
||||
ensure!(
|
||||
jobs::supported_with_binding(&job, Some(&binding)),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
);
|
||||
rmm_privileged_helper::consume(&mut self.ledger, &grant)?;
|
||||
rmm_privileged_helper::save_start(&self.ledger, &grant, &request)?;
|
||||
let cancelled: bool = self.db.query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if cancelled {
|
||||
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
&grant.grant_id.to_string(),
|
||||
outcome.clone(),
|
||||
)?;
|
||||
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut envelope = self.envelope(
|
||||
generation,
|
||||
Payload::JobStateChanged(JobStateChanged::default()),
|
||||
)?;
|
||||
let tx = self.db.transaction()?;
|
||||
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
|
||||
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
||||
job_id: offer.job_id.clone(),
|
||||
attempt_id: offer.attempt_id.clone(),
|
||||
revision,
|
||||
state: "running".into(),
|
||||
grant_id: grant.grant_id.to_string(),
|
||||
}));
|
||||
jobs::event(&tx, &envelope)?;
|
||||
tx.commit()?;
|
||||
let completed = self.completed_tx.clone();
|
||||
let clock = self.clock.clone();
|
||||
let fence = self.fence.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Thread-pool scheduling and durable fsync may have consumed the
|
||||
// remaining start window. Recheck at the actual effect boundary.
|
||||
let outcome = if !fence.allows(generation, job.execution_identity == "root")
|
||||
|| !clock.trusted()
|
||||
|| binding.verify(&request, Utc::now()).is_err()
|
||||
{
|
||||
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
|
||||
} else if job.execution_identity == "root" {
|
||||
apt::invoke(&binding, &job, &request).unwrap_or_else(|_| {
|
||||
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
|
||||
})
|
||||
} else {
|
||||
jobs::execute(&job)
|
||||
};
|
||||
let _ = completed.blocking_send((offer, grant, outcome));
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
|
||||
Ok(Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: self.id.clone(),
|
||||
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
connection_generation: generation,
|
||||
boot_id: self.boot.clone(),
|
||||
sequence: storage::next_sequence(&mut self.db)?,
|
||||
observed_at: Utc::now().timestamp_millis(),
|
||||
payload: Some(payload),
|
||||
})
|
||||
}
|
||||
fn capabilities(&self) -> Vec<String> {
|
||||
let mut capabilities = vec!["linux.base/1".into(), "heartbeat/1".into()];
|
||||
if self.binding.is_some() {
|
||||
capabilities.push("rmm.sync.v1".into());
|
||||
}
|
||||
if let Some(binding) = &self.binding
|
||||
&& unsafe { libc::geteuid() } != 0
|
||||
{
|
||||
if rmm_agent_linux::actions::Interpreter::PosixSh.available() {
|
||||
capabilities.push("run_script".into());
|
||||
}
|
||||
capabilities.push("refresh_inventory".into());
|
||||
if jobs::logs_available() {
|
||||
capabilities.push("query_logs".into());
|
||||
}
|
||||
capabilities.extend(apt::capabilities(binding));
|
||||
capabilities.extend(helper_sync::general_capabilities(binding));
|
||||
}
|
||||
capabilities.sort();
|
||||
capabilities.dedup();
|
||||
capabilities
|
||||
}
|
||||
fn queue_inventory(&mut self) {
|
||||
if self.inventory_inflight || !storage::passive_collection_ready(&self.db).unwrap_or(false)
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.inventory_inflight = true;
|
||||
let sender = self.inventory_tx.clone();
|
||||
let extended = env::var("RMM_EXTENDED_INVENTORY").unwrap_or_else(|_| "1".into()) == "1";
|
||||
let binding = self.binding.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = rmm_agent_linux::wire::snapshot(extended).and_then(|mut inventory| {
|
||||
if extended {
|
||||
inventory["observations"]
|
||||
.as_array_mut()
|
||||
.context("INVENTORY_OBSERVATIONS")?
|
||||
.push(apt::planning_observation(binding.as_ref()));
|
||||
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
|
||||
}
|
||||
let bytes = serde_json::to_vec(&inventory)?;
|
||||
ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT");
|
||||
Ok((bytes, Utc::now().timestamp_millis()))
|
||||
});
|
||||
let _ = sender.blocking_send(result);
|
||||
});
|
||||
}
|
||||
fn inventory_envelope(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
data: Vec<u8>,
|
||||
observed_at: i64,
|
||||
) -> Result<Envelope> {
|
||||
let mut envelope = self.envelope(
|
||||
generation,
|
||||
Payload::InventorySnapshot(InventorySnapshot {
|
||||
snapshot_id: Uuid::new_v4().to_string(),
|
||||
scope: "linux.base".into(),
|
||||
schema_version: 1,
|
||||
chunk_index: 0,
|
||||
chunk_count: 1,
|
||||
sha256: Sha256::digest(&data).to_vec(),
|
||||
data,
|
||||
completeness: "partial".into(),
|
||||
}),
|
||||
)?;
|
||||
envelope.observed_at = observed_at;
|
||||
Ok(envelope)
|
||||
}
|
||||
async fn session(&mut self) -> Result<()> {
|
||||
let result = self.session_inner().await;
|
||||
self.close_gate();
|
||||
result
|
||||
}
|
||||
async fn session_inner(&mut self) -> Result<()> {
|
||||
self.close_gate();
|
||||
self.boot = rmm_agent_linux::boot_id()?;
|
||||
if self.helpers.is_some() {
|
||||
self.helpers = self
|
||||
.binding
|
||||
.as_ref()
|
||||
.map(helper_sync::Helpers::configured)
|
||||
.transpose()?;
|
||||
}
|
||||
let advertised = self.capabilities();
|
||||
let hello = self.envelope(
|
||||
0,
|
||||
Payload::ClientHello(ClientHello {
|
||||
agent_version: env!("CARGO_PKG_VERSION").into(),
|
||||
capabilities: advertised.clone(),
|
||||
}),
|
||||
)?;
|
||||
let identity = self.certificates.current(Utc::now().timestamp())?;
|
||||
let mut ws =
|
||||
wire_channel::connect(self.mode, self.address, &self.ca, &identity, hello).await?;
|
||||
let reply = tokio::time::timeout(Duration::from_secs(20), ws.receive()).await??;
|
||||
ensure!(
|
||||
reply.protocol_major == 1
|
||||
&& reply.protocol_minor == 0
|
||||
&& reply.agent_id == self.id
|
||||
&& reply.enrollment_generation
|
||||
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
"invalid server hello"
|
||||
);
|
||||
let Some(Payload::ServerHello(server)) = reply.payload else {
|
||||
anyhow::bail!("server hello required")
|
||||
};
|
||||
let generation = server.connection_generation;
|
||||
ensure!(
|
||||
generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32,
|
||||
"unsupported limits"
|
||||
);
|
||||
if let Some(mut pending) = storage::pending(&self.db)? {
|
||||
// Generation is transport fencing, not part of durable message identity.
|
||||
pending.connection_generation = generation;
|
||||
jobs::event(&self.db, &pending)?;
|
||||
storage::clear_pending(&self.db)?;
|
||||
}
|
||||
self.queue_inventory();
|
||||
self.replay(&ws, generation).await?;
|
||||
let mut needs_sync = false;
|
||||
if self.sync.is_some()
|
||||
&& let Err(error) = self.start_sync(&ws, generation).await
|
||||
{
|
||||
report_closed_gate(&error);
|
||||
self.close_gate();
|
||||
needs_sync = true;
|
||||
}
|
||||
eprintln!("authenticated channel established; inventory queued for durable acknowledgment");
|
||||
let mut inventory_at = Instant::now();
|
||||
let mut heartbeat_at = Instant::now();
|
||||
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
||||
let mut sync_at = Instant::now();
|
||||
let mut rotation_at = Instant::now();
|
||||
let mut capability_at = Instant::now();
|
||||
let mut first_heartbeat = true;
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(result) = self.inventory_rx.recv() => {
|
||||
self.inventory_inflight = false;
|
||||
if let Ok((data,observed_at)) = result {
|
||||
let inventory = self.inventory_envelope(generation,data,observed_at)?;
|
||||
jobs::event(&self.db,&inventory)?;
|
||||
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
|
||||
}
|
||||
message = ws.receive() => {
|
||||
let message = message?;
|
||||
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
|
||||
match message.payload {
|
||||
Some(Payload::Ack(ack)) => {
|
||||
use rusqlite::OptionalExtension;
|
||||
let pending:Option<Vec<u8>>=self.db.query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
|
||||
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
|
||||
jobs::ack(&mut self.db, &ack.message_id)?;
|
||||
},
|
||||
Some(Payload::SyncResponse(response))=>{if let Err(error)=self.apply_sync(generation,response).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}sync_at=Instant::now();},
|
||||
Some(Payload::JobOffer(offer)) => {
|
||||
let binding = self.binding.as_ref().context("job trust not provisioned")?.clone();
|
||||
let accepted = self.envelope(generation, Payload::JobAccepted(JobAccepted {
|
||||
job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(),
|
||||
job_payload_hash: offer.job_payload_hash.clone(), journal_proof: rmm_domain::jobs::hash(&offer.encode_to_vec()),
|
||||
}))?;
|
||||
// Clone before borrowing the transaction; acceptance and the
|
||||
// durable ACK event commit together, before any wire ACK.
|
||||
let tx = self.db.transaction()?;
|
||||
let fresh=jobs::accept(&tx, &binding, &offer)?;
|
||||
jobs::event(&tx, &accepted)?;
|
||||
tx.commit()?;
|
||||
needs_sync|=fresh;self.replay(&ws, generation).await?;
|
||||
}
|
||||
Some(Payload::ExecutionPermit(permit)) => self.permit(generation, permit, self.ready(generation,false))?,
|
||||
Some(Payload::CancelRequest(cancel)) => {
|
||||
let row: Option<(Vec<u8>, String, String)> = {
|
||||
use rusqlite::OptionalExtension;
|
||||
self.db.query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?
|
||||
};
|
||||
if let Some((bytes,state,grant)) = row {
|
||||
self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
||||
if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; }
|
||||
// Running script cancellation is not confirmed by an
|
||||
// inbound request. Its bounded executor reports reality.
|
||||
}
|
||||
let ack = self.envelope(generation, Payload::Ack(Ack { message_id: message.message_id }))?;
|
||||
ws.send(&ack).await?;
|
||||
}
|
||||
_ => anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE"),
|
||||
}
|
||||
}
|
||||
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
|
||||
// Persist the wire result before releasing a consumption lock.
|
||||
// A crash in between is fail-closed, not a second execution.
|
||||
let state = outcome.state.clone();
|
||||
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
|
||||
let serialized = serde_json::to_vec(&outcome)?;
|
||||
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
|
||||
let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?;
|
||||
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
|
||||
if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); }
|
||||
needs_sync=true;self.replay(&ws, generation).await?;
|
||||
if refresh_capabilities {return Ok(());}
|
||||
}
|
||||
_ = tick.tick() => {
|
||||
self.replay(&ws, generation).await?;
|
||||
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
||||
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
||||
if self.prepared.is_some() && !busy && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && !busy && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
|
||||
&& self.clock.trusted() && let Some(offer) = jobs::next(&self.db)? {
|
||||
let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?;
|
||||
if Utc::now() >= job.start_before {
|
||||
self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?;
|
||||
} else if !self.ready(generation,job.execution_identity=="root") {
|
||||
needs_sync=true;
|
||||
} else if Utc::now() >= job.not_before {
|
||||
// Reserve before send. A lost response must never cause
|
||||
// a second permit request for this attempt.
|
||||
jobs::reserve_request(&self.db, &offer)?;
|
||||
let request = self.envelope(generation, Payload::ExecutionPermitRequest(ExecutionPermitRequest {
|
||||
job_id: offer.job_id, attempt_id: offer.attempt_id,
|
||||
job_payload_hash: offer.job_payload_hash, assignment_version: job.assignment_version,
|
||||
}))?;
|
||||
ws.send(&request).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
|
||||
first_heartbeat = false;
|
||||
let heartbeat = self.envelope(
|
||||
generation,
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
}),
|
||||
)?;
|
||||
ws.send(&heartbeat).await?;
|
||||
if ws.uses_https() {
|
||||
let poll = self.envelope(
|
||||
generation,
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
}),
|
||||
)?;
|
||||
ws.poll(poll).await?;
|
||||
}
|
||||
heartbeat_at = Instant::now();
|
||||
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
}
|
||||
// Planning pins must be refreshed before the 90-second stored-data
|
||||
// planning limit. One inflight blocking collector bounds pressure;
|
||||
// heartbeats never perform this metadata work.
|
||||
if inventory_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_inventory();
|
||||
inventory_at = Instant::now();
|
||||
}
|
||||
if rotation_at.elapsed() >= Duration::from_secs(30) {
|
||||
rotation_at = Instant::now();
|
||||
if self
|
||||
.certificates
|
||||
.due(self.rotation_policy, Utc::now().timestamp())?
|
||||
{
|
||||
self.close_gate();
|
||||
drop(ws);
|
||||
let rotated = self
|
||||
.certificates
|
||||
.renew(self.address, Utc::now().timestamp())
|
||||
.await?;
|
||||
ensure!(
|
||||
rotated.needs_reconnect
|
||||
&& rotated.enrollment_generation
|
||||
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
"ROTATION_IDENTITY_CHANGED"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enrollment(dir: &Path, address: std::net::SocketAddr, ca: &[u8]) -> Result<String> {
|
||||
let identity = dir.join("identity.json");
|
||||
if identity.exists() {
|
||||
let response: EnrollmentResponse = serde_json::from_slice(&fs::read(identity)?)?;
|
||||
Uuid::parse_str(&response.agent_id)?;
|
||||
return Ok(response.agent_id);
|
||||
}
|
||||
let key_path = dir.join("device.key");
|
||||
let key = if key_path.exists() {
|
||||
PKey::private_key_from_pem(&fs::read(&key_path)?)?
|
||||
} else {
|
||||
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
|
||||
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
|
||||
storage::create_secret(&key_path, &key.private_key_to_pem_pkcs8()?)?;
|
||||
key
|
||||
};
|
||||
let mut csr = X509Req::builder()?;
|
||||
let mut name = X509NameBuilder::new()?;
|
||||
name.append_entry_by_text("CN", "new-linux-device")?;
|
||||
csr.set_subject_name(&name.build())?;
|
||||
csr.set_pubkey(&key)?;
|
||||
csr.sign(&key, MessageDigest::sha256())?;
|
||||
let token_path = env::var("RMM_ENROLLMENT_TOKEN_FILE")
|
||||
.context("enrollment token file required for first registration")?;
|
||||
let token = fs::read_to_string(token_path)?.trim().to_owned();
|
||||
ensure!(token.len() == 64, "invalid enrollment token format");
|
||||
let client = reqwest::Client::builder()
|
||||
.https_only(true)
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
||||
.resolve("rmm-dev.internal", address)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(Duration::from_secs(15))
|
||||
.build()?;
|
||||
let result = client
|
||||
.post(format!(
|
||||
"https://rmm-dev.internal:{}/agent/v1/enroll",
|
||||
address.port()
|
||||
))
|
||||
.json(&EnrollmentRequest {
|
||||
token,
|
||||
csr_pem: String::from_utf8(csr.build().to_pem()?)?,
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
ensure!(
|
||||
result.status().is_success(),
|
||||
"enrollment rejected (response details suppressed)"
|
||||
);
|
||||
let response: EnrollmentResponse = result.json().await?;
|
||||
let certificate = openssl::x509::X509::from_pem(response.certificate_pem.as_bytes())?;
|
||||
ensure!(
|
||||
certificate.public_key()?.public_eq(&key),
|
||||
"enrollment certificate key mismatch"
|
||||
);
|
||||
Uuid::parse_str(&response.agent_id)?;
|
||||
storage::create_secret(&dir.join("device.pem"), response.certificate_pem.as_bytes())?;
|
||||
storage::create_secret(&identity, &serde_json::to_vec(&response)?)?;
|
||||
Ok(response.agent_id)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args: Vec<_> = env::args_os().skip(1).collect();
|
||||
if args == [std::ffi::OsString::from("--help")] {
|
||||
println!(
|
||||
"rmm-agent [--help]\nEnvironment: RMM_AGENT_STATE, RMM_CA_FILE, RMM_CONNECT_ADDR (TCP only; TLS name rmm-dev.internal), RMM_TRANSPORT=auto|wss|https, RMM_ENROLLMENT_TOKEN_FILE (first enrollment), RMM_JOB_BINDING_FILE, RMM_JOB_SIGNING_KEY_BOOTSTRAP=1, RMM_HELPER_SOCKET, RMM_HELPER_POLICY_FILE, RMM_APT_HELPER_SOCKET, RMM_APT_HELPER_PROOF_FILE, RMM_LOG_UNITS_FILE, RMM_EXTENDED_INVENTORY=0|1 (default 1).\nDevelopment-only rotation test: RMM_AGENT_PROFILE=development and RMM_ROTATION_TEST_AFTER_SECONDS.\nNo argument or environment variable approves jobs or disables TLS/grant verification."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(args.is_empty(), "UNKNOWN_AGENT_ARGUMENT");
|
||||
let dir =
|
||||
PathBuf::from(env::var("RMM_AGENT_STATE").unwrap_or_else(|_| "devstate/agent".into()));
|
||||
storage::prepare_dir(&dir)?;
|
||||
let _lock = storage::lock(&dir)?;
|
||||
let address = env::var("RMM_CONNECT_ADDR")
|
||||
.unwrap_or_else(|_| "127.0.0.1:18443".into())
|
||||
.parse()?;
|
||||
let ca = fs::read(env::var("RMM_CA_FILE").unwrap_or_else(|_| "devstate/pki/ca.pem".into()))?;
|
||||
let id=enrollment(&dir,address,&ca).await.map_err(|_| anyhow::anyhow!("enrollment/identity initialization failed; check trusted CA, token validity and local state"))?;
|
||||
let mut binding = env::var("RMM_JOB_BINDING_FILE")
|
||||
.ok()
|
||||
.map(|path| {
|
||||
let binding =
|
||||
rmm_privileged_helper::read_binding(Path::new(&path), unsafe { libc::geteuid() })?;
|
||||
let identity: EnrollmentResponse =
|
||||
serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?;
|
||||
ensure!(
|
||||
binding.agent_id.to_string() == id
|
||||
&& binding.asset_id.to_string() == identity.asset_id,
|
||||
"BOOTSTRAP_IDENTITY_MISMATCH"
|
||||
);
|
||||
Ok::<_, anyhow::Error>(binding)
|
||||
})
|
||||
.transpose()?;
|
||||
let certificate_dir = dir.join("certificates");
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&certificate_dir,
|
||||
&ca,
|
||||
Uuid::parse_str(&id)?,
|
||||
binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
)?;
|
||||
if !certificate_dir.join("current.json").try_exists()? {
|
||||
certificates.initialize(
|
||||
&fs::read(dir.join("device.pem"))?,
|
||||
&fs::read(dir.join("device.key"))?,
|
||||
Utc::now().timestamp(),
|
||||
)?;
|
||||
}
|
||||
if env::var("RMM_JOB_SIGNING_KEY_BOOTSTRAP").as_deref() == Ok("1") {
|
||||
bootstrap::provision_current(
|
||||
&dir,
|
||||
address,
|
||||
&ca,
|
||||
binding
|
||||
.as_mut()
|
||||
.context("bootstrap requires provisioned device binding")?,
|
||||
&certificates.current(Utc::now().timestamp())?,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let (completed_tx, completed_rx) = tokio::sync::mpsc::channel(1);
|
||||
let (inventory_tx, inventory_rx) = tokio::sync::mpsc::channel(1);
|
||||
let sync = binding
|
||||
.as_ref()
|
||||
.map(|b| reconcile::ClientSync::new(b.clone()));
|
||||
let helpers = binding
|
||||
.as_ref()
|
||||
.map(helper_sync::Helpers::configured)
|
||||
.transpose()?;
|
||||
let mut rotation_policy = certificate_rotation::RotationPolicy::default();
|
||||
if let Ok(after) = env::var("RMM_ROTATION_TEST_AFTER_SECONDS") {
|
||||
ensure!(
|
||||
env::var("RMM_AGENT_PROFILE").as_deref() == Ok("development"),
|
||||
"ROTATION_TEST_REQUIRES_DEVELOPMENT_PROFILE"
|
||||
);
|
||||
rotation_policy.begin_after_seconds = after.parse()?;
|
||||
ensure!(
|
||||
rotation_policy.begin_after_seconds > 0,
|
||||
"ROTATION_TEST_INTERVAL"
|
||||
);
|
||||
}
|
||||
let mode =
|
||||
wire_channel::Mode::parse(&env::var("RMM_TRANSPORT").unwrap_or_else(|_| "auto".into()))?;
|
||||
let mut agent = Agent {
|
||||
address,
|
||||
ca,
|
||||
id,
|
||||
boot: rmm_agent_linux::boot_id()?,
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db: storage::open(&dir)?,
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding,
|
||||
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() })?,
|
||||
completed_tx,
|
||||
completed_rx,
|
||||
inventory_tx,
|
||||
inventory_rx,
|
||||
inventory_inflight: false,
|
||||
sync,
|
||||
sync_clock: reconcile::NativeClock::default(),
|
||||
helpers,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode,
|
||||
certificates,
|
||||
rotation_policy,
|
||||
};
|
||||
let mut backoff = 1_u64;
|
||||
loop {
|
||||
let started = Instant::now();
|
||||
if let Err(error) = agent.session().await {
|
||||
report_closed_gate(&error);
|
||||
}
|
||||
agent.close_gate();
|
||||
eprintln!("channel unavailable; retrying with jitter (no jobs can execute)");
|
||||
if started.elapsed() > Duration::from_secs(60) {
|
||||
backoff = 1;
|
||||
}
|
||||
tokio::time::sleep(https_channel::retry_delay(backoff as u32)).await;
|
||||
backoff = (backoff + 1).min(6);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user