feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s

This commit is contained in:
BuTzZ
2026-11-12 22:06:25 +01:00
commit 6c3203d725
194 changed files with 49234 additions and 0 deletions
+563
View File
@@ -0,0 +1,563 @@
use anyhow::{Result, ensure};
use prost::Message;
use rmm_domain::jobs::{Action, JobPayload, hash};
use rmm_privileged_helper::{Binding, Outcome, Request};
use rmm_protocol::*;
use rusqlite::{Connection, OptionalExtension, params};
use uuid::Uuid;
pub fn supported(job: &JobPayload) -> bool {
if rmm_privileged_helper::apt::is_apt_action(job) {
// A configured environment path alone is never a capability proof.
// Caller binding is independently provisioned, not taken from the job.
return false;
}
// The network process must never execute a script with accidental root.
supported_as(
job,
unsafe { libc::geteuid() },
std::env::var_os("RMM_HELPER_SOCKET").is_some(),
)
}
pub fn supported_with_binding(job: &JobPayload, binding: Option<&Binding>) -> bool {
if rmm_privileged_helper::apt::is_apt_action(job) {
// SAFETY: geteuid has no preconditions.
return unsafe { libc::geteuid() } != 0
&& binding.is_some_and(|b| crate::apt::supported(b, job));
}
if job.execution_identity == "root" {
return unsafe { libc::geteuid() } != 0
&& binding.is_some_and(|b| crate::helper_sync::general_supported(b, job));
}
supported(job)
}
fn supported_as(job: &JobPayload, uid: u32, helper_configured: bool) -> bool {
let unprivileged = uid != 0;
if job.execution_identity == "root" && unprivileged {
return helper_configured
&& matches!(
job.action,
Action::RunScript { .. } | Action::SetServiceState { .. } | Action::Reboot { .. }
);
}
job.execution_identity == "agent_service"
&& unprivileged
&& match &job.action {
Action::RunScript { interpreter, .. } => match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
},
Action::RefreshInventory { scope } => scope == "linux.base",
Action::QueryLogs { .. } => log_request(job).is_ok(),
_ => false,
}
}
pub fn accept(db: &Connection, binding: &Binding, offer: &JobOffer) -> Result<bool> {
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"OFFER_BINDING"
);
ensure!(
supported_with_binding(&job, Some(binding)),
"UNSUPPORTED_CAPABILITY"
);
let bytes = offer.encode_to_vec();
let existing: Option<Vec<u8>> = db
.query_row(
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)
.optional()?;
if let Some(old) = existing {
ensure!(old == bytes, "OFFER_CONFLICT");
return Ok(false);
}
let count: i64 = db.query_row("SELECT count(*) FROM job_attempts", [], |r| r.get(0))?;
ensure!(count < 10000, "JOURNAL_CAPACITY_NEW_EFFECTS_BLOCKED");
db.execute(
"INSERT INTO job_attempts(job,attempt,offer,state) VALUES(?1,?2,?3,'accepted')",
params![offer.job_id, offer.attempt_id, bytes],
)?;
Ok(true)
}
fn hex_hash(s: &str) -> bool {
s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
db.execute(
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
params![envelope.message_id, envelope.encode_to_vec()],
)?;
Ok(())
}
pub fn events(db: &Connection) -> Result<Vec<Envelope>> {
let mut statement = db.prepare("SELECT payload FROM job_events ORDER BY rowid LIMIT 100")?;
let bytes = statement
.query_map([], |r| r.get::<_, Vec<u8>>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
bytes
.into_iter()
.map(|b| Ok(Envelope::decode(b.as_slice())?))
.collect()
}
pub fn ack(db: &mut Connection, id: &str) -> Result<()> {
let tx = db.transaction()?;
crate::reconcile::archive_events(&tx)?;
tx.execute("DELETE FROM job_events WHERE id=?1", [id])?;
tx.commit()?;
Ok(())
}
pub fn next(db: &Connection) -> Result<Option<JobOffer>> {
// One device-wide effect at a time; unknown/reserved attempts retain locks.
let busy: i64 = db.query_row("SELECT count(*) FROM job_attempts WHERE state IN ('requesting','running','unknown','awaiting_reboot')", [], |r| r.get(0))?;
if busy != 0 {
return Ok(None);
}
let mut statement = db.prepare("SELECT offer FROM job_attempts WHERE state='accepted' AND cancel_requested=0 ORDER BY rowid")?;
let rows = statement.query_map([], |r| r.get::<_, Vec<u8>>(0))?;
let now = chrono::Utc::now();
for bytes in rows {
let offer = JobOffer::decode(bytes?.as_slice())?;
let job: JobPayload = serde_json::from_slice(&offer.payload_bytes)?;
if now >= job.not_before || now >= job.start_before {
return Ok(Some(offer));
}
}
Ok(None)
}
pub fn requesting(db: &Connection) -> Result<Option<JobOffer>> {
let bytes: Option<Vec<u8>> = db
.query_row(
"SELECT offer FROM job_attempts WHERE state='requesting' LIMIT 1",
[],
|r| r.get(0),
)
.optional()?;
bytes
.map(|b| Ok(JobOffer::decode(b.as_slice())?))
.transpose()
}
pub fn reserve_request(db: &Connection, offer: &JobOffer) -> Result<()> {
ensure!(db.execute("UPDATE job_attempts SET state='requesting' WHERE job=?1 AND attempt=?2 AND state='accepted' AND cancel_requested=0",
params![offer.job_id,offer.attempt_id])? == 1, "ATTEMPT_NOT_STARTABLE");
Ok(())
}
pub fn transition(db: &Connection, offer: &JobOffer, state: &str, grant: &str) -> Result<u64> {
let previous: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
let allowed = match previous.as_str() {
"accepted" | "requesting" => matches!(
state,
"running" | "failed" | "cancelled" | "expired" | "unknown"
),
"running" | "verifying" => matches!(
state,
"verifying" | "succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
),
// Only a definitive completion of the already-running executor calls
// this reconciliation path. No new effect is started from unknown.
"unknown" | "awaiting_reboot" => matches!(
state,
"succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
),
_ => false,
};
ensure!(allowed, "INVALID_LOCAL_STATE_TRANSITION");
let rev: i64 = db.query_row("UPDATE job_attempts SET state=?3,grant_id=?4,revision=revision+1 WHERE job=?1 AND attempt=?2 RETURNING revision",
params![offer.job_id,offer.attempt_id,state,grant], |r| r.get(0))?;
Ok(u64::try_from(rev)?)
}
pub fn result(offer: &JobOffer, revision: u64, grant: &str, outcome: Outcome) -> JobResult {
JobResult {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
revision,
state: outcome.state,
grant_id: grant.into(),
stdout: outcome.stdout,
stderr: outcome.stderr,
output_truncated: outcome.output_truncated,
reboot_required: outcome.reboot_required,
exit_code: outcome.exit_code,
evidence_json: outcome.evidence.to_string(),
error_code: outcome.error_code,
}
}
pub fn no_start(state: &str, code: &str) -> Outcome {
Outcome {
state: state.into(),
evidence: serde_json::json!({"not_started":true,"postconditions_passed":false}),
..Outcome::unknown(code)
}
}
pub fn execute(job: &JobPayload) -> Outcome {
let result = match &job.action {
Action::RunScript { .. } => rmm_privileged_helper::script(job),
Action::RefreshInventory { .. } => rmm_agent_linux::collect().and_then(|i| {
let bytes = serde_json::to_vec(&i)?;
ensure!(bytes.len() <= job.max_output_bytes as usize, "INVENTORY_OUTPUT_LIMIT");
Ok(Outcome { state: "succeeded".into(), stdout: bytes, stderr: vec![], exit_code: None,
output_truncated: false, reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":true,"source_id":"linux.base","observed_at":chrono::Utc::now(),"completeness":"partial"}), error_code: String::new() })
}),
Action::QueryLogs { .. } => query_logs(job),
_ => Err(anyhow::anyhow!("unsupported action")),
};
result.unwrap_or_else(|_| Outcome::unknown("EXECUTOR_EFFECT_UNKNOWN"))
}
fn allowed_log_units() -> Result<Vec<rmm_agent_linux::actions::ServiceUnit>> {
let path = std::env::var_os("RMM_LOG_UNITS_FILE")
.ok_or_else(|| anyhow::anyhow!("LOG_ALLOWLIST_NOT_CONFIGURED"))?;
let path = std::path::Path::new(&path);
rmm_privileged_helper::private_path(path, unsafe { libc::geteuid() }, false)?;
ensure!(
std::fs::metadata(path)?.len() <= 32768,
"LOG_ALLOWLIST_SIZE"
);
let units: Vec<String> = serde_json::from_slice(&std::fs::read(path)?)?;
ensure!(
!units.is_empty() && units.len() <= 128,
"LOG_ALLOWLIST_SIZE"
);
units
.iter()
.map(|u| rmm_agent_linux::actions::ServiceUnit::new(u))
.collect()
}
pub fn logs_available() -> bool {
allowed_log_units().is_ok()
&& rmm_agent_linux::inventory::capabilities()
.get("logs.journald")
.is_some_and(|o| o.data == Some(true))
}
fn log_request(job: &JobPayload) -> Result<rmm_agent_linux::inventory::JournalRequest> {
log_request_for(job, &allowed_log_units()?)
}
fn log_request_for(
job: &JobPayload,
allowed: &[rmm_agent_linux::actions::ServiceUnit],
) -> Result<rmm_agent_linux::inventory::JournalRequest> {
let Action::QueryLogs {
unit,
since,
until,
max_lines,
} = &job.action
else {
anyhow::bail!("NOT_LOG_QUERY")
};
ensure!(job.max_runtime_seconds >= 20, "LOG_RUNTIME_TOO_SHORT");
rmm_agent_linux::inventory::JournalRequest::new(
rmm_agent_linux::actions::ServiceUnit::new(unit)?,
allowed,
u64::try_from(since.timestamp())?,
u64::try_from(until.timestamp())?,
*max_lines as usize,
(job.max_output_bytes as usize).min(256 * 1024),
)?
.precise_window(*since, *until)
}
fn query_logs(job: &JobPayload) -> Result<Outcome> {
let observation = rmm_agent_linux::inventory::query_journal(&log_request(job)?);
let valid = observation.quality == rmm_agent_linux::inventory::Quality::Valid;
let error = observation.error_code.clone().unwrap_or_default();
let mut value = serde_json::to_value(&observation)?;
while serde_json::to_vec(&value)?.len() > job.max_output_bytes as usize {
let Some(lines) = value["data"]["json_lines"]
.as_array_mut()
.filter(|v| !v.is_empty())
else {
return Ok(Outcome {
state: "failed".into(),
..Outcome::unknown("LOG_RESULT_METADATA_EXCEEDS_OUTPUT_LIMIT")
});
};
lines.remove(0);
value["truncated"] = serde_json::json!(true);
value["completeness"] = serde_json::json!("partial");
value["error_code"] = serde_json::json!("LOG_SERIALIZED_OUTPUT_LIMIT");
}
Ok(Outcome {
state: if valid { "succeeded" } else { "failed" }.into(),
stdout: serde_json::to_vec(&value)?,
stderr: vec![],
exit_code: None,
output_truncated: value["truncated"] == true,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":valid,"source_id":observation.source_id,
"observed_at":value["observed_at"],"quality":value["quality"],"completeness":value["completeness"],
"cursor_status":"not_advanced_no_ack_contract","provider_error_code":error}),
error_code: if valid {
String::new()
} else if error.is_empty() {
"LOG_PROVIDER_UNAVAILABLE".into()
} else {
error
},
})
}
pub fn permit_request(offer: &JobOffer, permit: &ExecutionPermit) -> Result<Request> {
Ok(Request {
job_bytes: offer.payload_bytes.clone(),
permit_bytes: permit.payload_bytes.clone(),
signature: permit.signature.clone(),
plan_hash: offer.plan_hash.clone(),
authorization_id: Uuid::parse_str(&offer.authorization_id)?,
})
}
/// With no backend SyncResponse contract, a reconnect/restart cannot prove an
/// earlier reservation never started. Telemetry and evidence replay continue;
/// all new effects stop. Do not paper over this with a new start request.
#[cfg(test)]
pub fn begin_session(db: &Connection) -> Result<bool> {
let inserted = db.execute(
"INSERT OR IGNORE INTO safety(key,value) VALUES('connected','1')",
[],
)?;
Ok(inserted == 1)
}
#[cfg(test)]
pub fn uncertain(db: &Connection) -> Result<Vec<(JobOffer, String)>> {
let mut statement = db.prepare(
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running')",
)?;
let bytes = statement
.query_map([], |r| {
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
bytes
.into_iter()
.map(|(bytes, grant)| Ok((JobOffer::decode(bytes.as_slice())?, grant)))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage;
fn fixture() -> (std::path::PathBuf, Binding, JobOffer) {
let path = std::env::temp_dir().join(format!("rmm-agent-jobs-{}", Uuid::new_v4()));
storage::prepare_dir(&path).unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 2,
public_key_hex: "00".repeat(32),
};
let now = chrono::Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 2,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RefreshInventory {
scope: "linux.base".into(),
},
execution_identity: "agent_service".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(10),
max_runtime_seconds: 10,
max_output_bytes: 4096,
};
let bytes = rmm_domain::jobs::encode(&job).unwrap();
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: Uuid::new_v4().to_string(),
plan_hash: hash(b"plan"),
job_payload_hash: hash(&bytes),
payload_bytes: bytes,
};
(path, binding, offer)
}
#[test]
fn sqlite_reopen_offer_dedup_and_no_new_permit_after_ambiguity() {
let (path, binding, offer) = fixture();
let mut db = storage::open(&path).unwrap();
let job = binding.job(&offer.payload_bytes).unwrap();
if unsafe { libc::geteuid() } == 0 {
assert!(!supported(&job));
assert!(accept(&db, &binding, &offer).is_err());
} else {
assert!(accept(&db, &binding, &offer).unwrap());
assert!(!accept(&db, &binding, &offer).unwrap());
let mut changed = offer.clone();
changed.plan_hash = hash(b"different plan");
assert!(accept(&db, &binding, &changed).is_err());
assert!(begin_session(&db).unwrap());
let envelope = Envelope {
message_id: Uuid::new_v4().to_string(),
sequence: storage::next_sequence(&mut db).unwrap(),
payload: Some(Payload::JobAccepted(JobAccepted {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
job_payload_hash: offer.job_payload_hash.clone(),
journal_proof: "durable".into(),
})),
..Default::default()
};
event(&db, &envelope).unwrap();
reserve_request(&db, &offer).unwrap();
assert!(reserve_request(&db, &offer).is_err());
assert!(next(&db).unwrap().is_none());
drop(db);
db = storage::open(&path).unwrap();
assert!(!begin_session(&db).unwrap());
assert_eq!(uncertain(&db).unwrap().len(), 1);
assert_eq!(events(&db).unwrap(), vec![envelope.clone()]);
assert_eq!(transition(&db, &offer, "unknown", "").unwrap(), 1);
assert!(next(&db).unwrap().is_none());
assert!(!accept(&db, &binding, &offer).unwrap());
ack(&mut db, &envelope.message_id).unwrap();
assert!(events(&db).unwrap().is_empty());
assert!(next(&db).unwrap().is_none());
}
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn cancellation_expiry_and_atomic_event_rollback() {
let (path, binding, offer) = fixture();
let mut db = storage::open(&path).unwrap();
if unsafe { libc::geteuid() } != 0 {
accept(&db, &binding, &offer).unwrap();
db.execute("UPDATE job_attempts SET cancel_requested=1", [])
.unwrap();
assert!(reserve_request(&db, &offer).is_err());
let cancelled = no_start("cancelled", "CANCELLED");
assert_eq!(cancelled.evidence["not_started"], true);
let tx = db.transaction().unwrap();
assert_eq!(transition(&tx, &offer, "cancelled", "").unwrap(), 1);
let e = Envelope {
message_id: Uuid::new_v4().to_string(),
payload: Some(Payload::JobResult(result(&offer, 1, "", cancelled))),
..Default::default()
};
event(&tx, &e).unwrap();
tx.rollback().unwrap();
assert!(events(&db).unwrap().is_empty());
assert!(next(&db).unwrap().is_none());
let expired = no_start("expired", "DEADLINE_EXPIRED");
assert_eq!(expired.evidence["not_started"], true);
assert_eq!(transition(&db, &offer, "expired", "").unwrap(), 1);
assert!(next(&db).unwrap().is_none());
} else {
assert!(accept(&db, &binding, &offer).is_err());
}
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn network_process_never_executes_as_accidental_root() {
let (path, binding, offer) = fixture();
let mut job = binding.job(&offer.payload_bytes).unwrap();
assert!(!supported_as(&job, 0, false));
assert!(!supported_as(&job, 0, true));
job.execution_identity = "root".into();
assert!(!supported_as(&job, 1000, false));
assert!(!supported_as(&job, 0, true));
job.action = Action::Reboot {
reason: "explicit root only".into(),
reconnect_timeout_seconds: 60,
dependency_group: "test".into(),
};
assert!(supported_as(&job, 1000, true));
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn scheduled_future_job_does_not_starve_ready_job() {
let (path, binding, offer) = fixture();
let db = storage::open(&path).unwrap();
if unsafe { libc::geteuid() } != 0 {
let mut future = offer.clone();
let mut job: JobPayload = serde_json::from_slice(&future.payload_bytes).unwrap();
job.job_id = Uuid::new_v4();
job.attempt_id = Uuid::new_v4();
job.not_before = chrono::Utc::now() + chrono::Duration::minutes(1);
future.job_id = job.job_id.to_string();
future.attempt_id = job.attempt_id.to_string();
future.payload_bytes = rmm_domain::jobs::encode(&job).unwrap();
future.job_payload_hash = hash(&future.payload_bytes);
accept(&db, &binding, &future).unwrap();
accept(&db, &binding, &offer).unwrap();
assert_eq!(next(&db).unwrap().unwrap().job_id, offer.job_id);
}
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn live_logs_enforce_actual_action_window_allowlist_limits_and_runtime() {
let (path, binding, offer) = fixture();
let mut job = binding.job(&offer.payload_bytes).unwrap();
let unit = rmm_agent_linux::actions::ServiceUnit::new("rmm-test.service").unwrap();
let now = chrono::Utc::now();
job.max_runtime_seconds = 20;
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::minutes(1),
until: now,
max_lines: 10,
};
assert!(log_request_for(&job, &[]).is_err());
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_ok());
job.max_runtime_seconds = 10;
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
job.max_runtime_seconds = 20;
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::hours(25),
until: now,
max_lines: 10,
};
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::minutes(1),
until: now,
max_lines: 1001,
};
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn refresh_is_a_new_measurement_with_real_observation_time() {
let (path, binding, offer) = fixture();
let job = binding.job(&offer.payload_bytes).unwrap();
let before = chrono::Utc::now();
let outcome = execute(&job);
assert_eq!(outcome.state, "succeeded");
let observed =
chrono::DateTime::parse_from_rfc3339(outcome.evidence["observed_at"].as_str().unwrap())
.unwrap();
assert!(observed >= before);
assert!(
serde_json::from_slice::<rmm_domain::Inventory>(&outcome.stdout)
.unwrap()
.validate()
);
std::fs::remove_dir_all(path).unwrap();
}
}