Files
rust-discovery/crates/agent/src/runtime_tests.rs
T
2026-11-16 15:25:46 +01:00

585 lines
23 KiB
Rust

//! Actual dispatcher/transport tests with a local mTLS fixture, no real server
//! approval, root helper, package/service/reboot or target VM effects.
use super::*;
use openssl::{
asn1::Asn1Time,
bn::BigNum,
ssl::{SslAcceptor, SslMethod, SslVerifyMode},
x509::{
X509, X509NameBuilder,
extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName},
},
};
use rmm_domain::reconciliation::ReconcileDecisionPayload;
use std::{
io::{Read, Write},
net::TcpListener,
};
struct Clock(bool);
#[test]
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
use openssl::sign::Signer;
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
use rmm_privileged_helper::{Binding, Request};
use rusqlite::params;
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
storage::prepare_dir(&dir).unwrap();
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf 'fixture only; never executed\\n'".into(),
arguments: vec![],
expected_stdout: Some("fixture only; never executed\n".into()),
windows_execution_profile: None,
privileged_script_profile: None,
},
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(5),
max_runtime_seconds: 120,
max_output_bytes: 4096,
};
let permit = ExecutionPermitPayload {
agent_update_plan_id: None,
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: Uuid::new_v4(),
tenant_id: job.tenant_id,
asset_id: job.asset_id,
agent_id: job.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: job.job_id,
attempt_id: job.attempt_id,
plan_hash: hash(b"fixed test plan; no native approval"),
job_payload_hash: hash(&encode(&job).unwrap()),
action: "run_script".into(),
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::seconds(59),
max_runtime_seconds: 120,
key_id: hash(&public),
};
let mut request = Request {
job_bytes: encode(&job).unwrap(),
permit_bytes: encode(&permit).unwrap(),
signature: vec![],
plan_hash: permit.plan_hash.clone(),
authorization_id: permit.authorization_id,
};
request.signature = Signer::new_without_digest(&key)
.unwrap()
.sign_oneshot_to_vec(&request.permit_bytes)
.unwrap();
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: permit.authorization_id.to_string(),
plan_hash: permit.plan_hash.clone(),
job_payload_hash: permit.job_payload_hash.clone(),
payload_bytes: request.job_bytes.clone(),
};
let db = storage::open(&dir).unwrap();
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
let mut queued_job = job.clone();
queued_job.job_id = Uuid::new_v4();
queued_job.attempt_id = Uuid::new_v4();
let mut queued = offer.clone();
queued.job_id = queued_job.job_id.to_string();
queued.attempt_id = queued_job.attempt_id.to_string();
queued.payload_bytes = encode(&queued_job).unwrap();
queued.job_payload_hash = hash(&queued.payload_bytes);
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
let (tx, rx) = tokio::sync::mpsc::channel(1);
let (itx, irx) = tokio::sync::mpsc::channel(1);
let certificates = certificate_rotation::RotationStore::open(
&dir.join("certificates"),
&[],
binding.agent_id,
1,
)
.unwrap();
let mut agent = Agent {
address: "127.0.0.1:1".parse().unwrap(),
ca: vec![],
id: binding.agent_id.to_string(),
boot: rmm_agent_linux::boot_id().unwrap(),
instance: Uuid::new_v4().to_string(),
db,
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding: Some(binding.clone()),
ledger,
completed_tx: tx,
completed_rx: rx,
inventory_tx: itx,
inventory_rx: irx,
inventory_inflight: false,
running_cancels: std::collections::HashMap::new(),
sync: Some(reconcile::ClientSync::new(binding.clone())),
sync_clock: Clock(true),
helpers: None,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode: wire_channel::Mode::Wss,
certificates,
rotation_policy: certificate_rotation::RotationPolicy::default(),
catalog: agent_update_catalog::Checker::default(),
update_observer: agent_update_apply::Observer::default(),
};
// A forged local consumption/start cannot become a purported grant result.
agent
.ledger
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
.unwrap();
assert!(agent.recover_interrupted().is_err());
assert!(jobs::events(&agent.db).unwrap().is_empty());
agent
.ledger
.execute(
"UPDATE consumed SET agent=?1",
[binding.agent_id.to_string()],
)
.unwrap();
agent.recover_interrupted().unwrap();
let original = jobs::events(&agent.db).unwrap();
assert_eq!(original.len(), 1);
let Some(Payload::JobResult(result)) = &original[0].payload else {
panic!("real durable result required")
};
assert_eq!(result.state, "unknown");
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
assert_eq!(result.grant_id, permit.grant_id.to_string());
assert_eq!(result.revision, 2);
assert!(result.stdout.is_empty() && result.exit_code.is_none());
assert!(!agent.fence.allows(1, true));
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
assert_eq!(jobs::events(&reopened).unwrap(), original);
let requesting: (String, u64) = reopened
.query_row(
"SELECT state,revision FROM job_attempts WHERE job=?1",
[&queued.job_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(requesting, ("unknown".into(), 1));
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert_eq!(retained, (1, 1, 1));
// The same process/restart cannot invent another revision or message ID.
agent.recover_interrupted().unwrap();
assert_eq!(jobs::events(&agent.db).unwrap(), original);
drop(reopened);
drop(agent);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
let gate = EffectFence::default();
assert!(!gate.allows(1, false));
gate.deadline
.store(Utc::now().timestamp_millis() + 10000, Ordering::SeqCst);
gate.generation.store(1, Ordering::SeqCst);
assert!(gate.allows(1, false));
assert!(!gate.allows(2, false));
assert!(!gate.allows(1, true));
gate.root.store(true, Ordering::SeqCst);
assert!(gate.allows(1, true));
gate.deadline.store(0, Ordering::SeqCst);
assert!(!gate.allows(1, false));
gate.close();
assert!(!gate.allows(1, true));
}
impl reconcile::SyncClock for Clock {
fn now(&self) -> Result<chrono::DateTime<Utc>> {
ensure!(self.0, "CLOCK_UNTRUSTED");
Ok(Utc::now())
}
}
pub(super) fn certificate(
name: &str,
issuer: Option<(&X509, &PKey<openssl::pkey::Private>)>,
server: bool,
) -> (X509, PKey<openssl::pkey::Private>) {
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap();
let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap();
let mut subject = X509NameBuilder::new().unwrap();
subject.append_entry_by_text("CN", name).unwrap();
let subject = subject.build();
let mut b = X509::builder().unwrap();
b.set_version(2).unwrap();
b.set_serial_number(
&BigNum::from_u32(rand::random_range(1..u32::MAX))
.unwrap()
.to_asn1_integer()
.unwrap(),
)
.unwrap();
b.set_subject_name(&subject).unwrap();
b.set_issuer_name(issuer.map_or(subject.as_ref(), |(ca, _)| ca.subject_name()))
.unwrap();
b.set_pubkey(&key).unwrap();
b.set_not_before(&Asn1Time::days_from_now(0).unwrap())
.unwrap();
b.set_not_after(&Asn1Time::days_from_now(1).unwrap())
.unwrap();
if issuer.is_none() {
b.append_extension(BasicConstraints::new().critical().ca().build().unwrap())
.unwrap();
b.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap())
.unwrap();
} else {
b.append_extension(BasicConstraints::new().build().unwrap())
.unwrap();
let mut u = ExtendedKeyUsage::new();
if server {
u.server_auth();
} else {
u.client_auth();
}
b.append_extension(u.build().unwrap()).unwrap();
if server {
let context = b.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None);
let san = SubjectAlternativeName::new()
.dns(name)
.build(&context)
.unwrap();
b.append_extension(san).unwrap();
}
}
b.sign(issuer.map_or(&key, |(_, k)| k), MessageDigest::sha256())
.unwrap();
(b.build(), key)
}
fn reply(input: &Envelope, generation: u64, payload: Payload) -> Envelope {
Envelope {
message_id: Uuid::new_v4().to_string(),
connection_generation: generation,
payload: Some(payload),
..input.clone()
}
}
struct Server {
binding: rmm_privileged_helper::Binding,
key: PKey<openssl::pkey::Private>,
generation: u64,
previous: Option<Uuid>,
requests: usize,
heartbeats: usize,
forged: bool,
}
impl Server {
fn response(&mut self, e: &Envelope) -> Envelope {
match e.payload.as_ref().unwrap() {
Payload::ClientHello(h) => {
assert!(h.capabilities.iter().any(|c| c == "rmm.sync.v1"));
reply(
e,
self.generation,
Payload::ServerHello(ServerHello {
connection_generation: self.generation,
max_message_bytes: rmm_domain::MAX_MESSAGE as u32,
}),
)
}
Payload::SyncRequest(r) => {
assert!(r.journal_complete && r.attempts.is_empty());
if let Some(previous) = self.previous {
assert_eq!(r.previous_decision_id, previous.to_string());
}
let now = Utc::now();
let d = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::parse_str(&r.request_id).unwrap(),
request_hash: rmm_domain::jobs::hash(&r.encode_to_vec()),
tenant_id: self.binding.tenant_id,
asset_id: self.binding.asset_id,
agent_id: self.binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
connection_generation: self.generation,
boot_id: Uuid::parse_str(&e.boot_id).unwrap(),
journal_id: Uuid::parse_str(&r.journal_id).unwrap(),
checkpoint: r.checkpoint,
helper_journal_id: None,
helper_checkpoint: 0,
issued_at: now,
valid_until: now + chrono::Duration::seconds(60),
ready: true,
helper_ready: false,
reason: "CONSISTENT".into(),
attempts: vec![],
acknowledged_message_ids: vec![],
key_id: rmm_domain::jobs::hash(&self.key.raw_public_key().unwrap()),
};
let bytes = rmm_domain::jobs::encode(&d).unwrap();
let mut signature = openssl::sign::Signer::new_without_digest(&self.key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap();
if self.forged {
signature[0] ^= 1;
} else {
self.previous = Some(d.decision_id);
}
self.requests += 1;
reply(
e,
self.generation,
Payload::SyncResponse(SyncResponse {
decision_bytes: bytes,
signature,
}),
)
}
Payload::Heartbeat(_) => {
self.heartbeats += 1;
reply(
e,
self.generation,
Payload::Ack(Ack {
message_id: e.message_id.clone(),
}),
)
}
Payload::InventorySnapshot(_) => reply(
e,
self.generation,
Payload::Ack(Ack {
message_id: e.message_id.clone(),
}),
),
_ => panic!("fixture received unexpected execution authority"),
}
}
}
fn http_body(stream: &mut impl Read) -> Vec<u8> {
let mut head = vec![];
while !head.ends_with(b"\r\n\r\n") {
let mut b = [0; 1];
stream.read_exact(&mut b).unwrap();
head.push(b[0]);
assert!(head.len() < 8192);
}
let text = String::from_utf8(head).unwrap();
let length: usize = text
.lines()
.filter_map(|line| line.split_once(':'))
.find(|(k, _)| k.eq_ignore_ascii_case("content-length"))
.unwrap()
.1
.trim()
.parse()
.unwrap();
assert!(length <= rmm_domain::MAX_MESSAGE);
let mut bytes = vec![0; length];
stream.read_exact(&mut bytes).unwrap();
bytes
}
async fn run(mode: wire_channel::Mode, trusted: bool, forged: bool, rounds: u64) {
let dir = PathBuf::from("/tmp/opencode").join(format!("runtime-{}", Uuid::new_v4()));
storage::prepare_dir(&dir).unwrap();
let id = Uuid::new_v4();
let (ca, ca_key) = certificate("runtime CA", None, false);
let (server, server_key) = certificate("rmm-dev.internal", Some((&ca, &ca_key)), true);
let (device, device_key) = certificate(&id.to_string(), Some((&ca, &ca_key)), false);
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = rmm_privileged_helper::Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap();
acceptor.set_certificate(&server).unwrap();
acceptor.set_private_key(&server_key).unwrap();
acceptor.cert_store_mut().add_cert(ca.clone()).unwrap();
acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT);
acceptor.set_session_id_context(b"rmm-runtime").unwrap();
let acceptor = acceptor.build();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server_binding = binding.clone();
let worker = std::thread::spawn(move || {
let mut state = Server {
binding: server_binding,
key,
generation: 1,
previous: None,
requests: 0,
heartbeats: 0,
forged,
};
for generation in 1..=rounds {
state.generation = generation;
loop {
let (socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(8)))
.unwrap();
let mut tls = acceptor.accept(socket).unwrap();
assert!(tls.ssl().peer_certificate().is_some());
if mode == wire_channel::Mode::Wss {
let mut ws = tokio_tungstenite::tungstenite::accept(tls).unwrap();
loop {
let message = ws.read().unwrap();
let tokio_tungstenite::tungstenite::Message::Binary(bytes) = message else {
continue;
};
let e = Envelope::decode(bytes).unwrap();
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
state.heartbeats += 1;
let _ = ws.close(None);
break;
}
if matches!(e.payload, Some(Payload::ClientHello(_))) {
ws.send(tokio_tungstenite::tungstenite::Message::Ping(
vec![1].into(),
))
.unwrap();
}
ws.send(tokio_tungstenite::tungstenite::Message::Binary(
state.response(&e).encode_to_vec().into(),
))
.unwrap();
}
break;
} else {
let batch = MessageBatch::decode(http_body(&mut tls).as_slice()).unwrap();
let e = &batch.messages[0];
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
state.heartbeats += 1;
tls.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
break;
}
let response = MessageBatch {
protocol_major: 1,
messages: vec![state.response(e)],
}
.encode_to_vec();
let header = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
response.len()
);
tls.write_all(header.as_bytes()).unwrap();
tls.write_all(&response).unwrap();
}
}
}
(state.requests, state.heartbeats)
});
let certificates = certificate_rotation::RotationStore::open(
&dir.join("certificates"),
&ca.to_pem().unwrap(),
id,
1,
)
.unwrap();
certificates
.initialize(
&device.to_pem().unwrap(),
&device_key.private_key_to_pem_pkcs8().unwrap(),
Utc::now().timestamp(),
)
.unwrap();
let (tx, rx) = tokio::sync::mpsc::channel(1);
let (itx, irx) = tokio::sync::mpsc::channel(1);
let mut agent = Agent {
address,
ca: ca.to_pem().unwrap(),
id: id.to_string(),
boot: rmm_agent_linux::boot_id().unwrap(),
instance: Uuid::new_v4().to_string(),
db: storage::open(&dir).unwrap(),
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding: Some(binding.clone()),
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(),
completed_tx: tx,
completed_rx: rx,
inventory_tx: itx,
inventory_rx: irx,
inventory_inflight: false,
running_cancels: std::collections::HashMap::new(),
sync: Some(reconcile::ClientSync::new(binding)),
sync_clock: Clock(trusted),
helpers: None,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode,
certificates,
rotation_policy: certificate_rotation::RotationPolicy::default(),
catalog: agent_update_catalog::Checker::default(),
update_observer: agent_update_apply::Observer::default(),
};
for _ in 0..rounds {
assert!(
tokio::time::timeout(Duration::from_secs(10), agent.session())
.await
.unwrap()
.is_err()
);
assert!(!agent.fence.allows(1, false));
}
let (requests, heartbeats) = worker.join().unwrap();
assert_eq!(heartbeats, rounds as usize);
if trusted {
assert_eq!(requests, rounds as usize);
let decided: Option<String> = agent
.db
.query_row("SELECT last_decision FROM reconcile_meta", [], |r| r.get(0))
.unwrap();
assert_eq!(decided.is_some(), !forged);
} else {
assert_eq!(requests, 0);
}
drop(agent);
fs::remove_dir_all(dir).unwrap();
}
#[tokio::test]
async fn actual_https_dispatcher_commits_signed_sync_and_clean_reconnect() {
run(wire_channel::Mode::Https, true, false, 2).await;
}
#[tokio::test]
async fn actual_wss_dispatcher_handles_ping_and_clean_reconnect() {
run(wire_channel::Mode::Wss, true, false, 2).await;
}
#[tokio::test]
async fn forged_sync_never_opens_dispatcher_gate() {
run(wire_channel::Mode::Https, true, true, 1).await;
}
#[tokio::test]
async fn untrusted_clock_keeps_telemetry_transport_but_never_requests_start() {
run(wire_channel::Mode::Https, false, false, 1).await;
}