//! Actual dispatcher/transport tests with a local mTLS fixture, no real server //! approval, root helper, package/service/reboot or target VM effects. use super::*; use openssl::{ asn1::Asn1Time, bn::BigNum, ssl::{SslAcceptor, SslMethod, SslVerifyMode}, x509::{ X509, X509NameBuilder, extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName}, }, }; use rmm_domain::reconciliation::ReconcileDecisionPayload; use std::{ io::{Read, Write}, net::TcpListener, }; struct Clock(bool); #[test] fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() { use openssl::sign::Signer; use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash}; use rmm_privileged_helper::{Binding, Request}; use rusqlite::params; let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4())); storage::prepare_dir(&dir).unwrap(); let key = PKey::generate_ed25519().unwrap(); let public = key.raw_public_key().unwrap(); let binding = Binding { tenant_id: Uuid::new_v4(), asset_id: Uuid::new_v4(), agent_id: Uuid::new_v4(), enrollment_generation: 1, assignment_version: 1, public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(), }; let now = Utc::now(); let job = JobPayload { schema_version: 1, tenant_id: binding.tenant_id, asset_id: binding.asset_id, agent_id: binding.agent_id, enrollment_generation: 1, assignment_version: 1, job_id: Uuid::new_v4(), attempt_id: Uuid::new_v4(), action: Action::RunScript { interpreter: "posix_sh".into(), script_utf8: "printf 'fixture only; never executed\\n'".into(), arguments: vec![], expected_stdout: Some("fixture only; never executed\n".into()), windows_execution_profile: None, privileged_script_profile: None, }, execution_identity: "root".into(), not_before: now, start_before: now + chrono::Duration::minutes(5), max_runtime_seconds: 120, max_output_bytes: 4096, }; let permit = ExecutionPermitPayload { agent_update_plan_id: None, domain: "rmm.execution-permit".into(), schema_version: 1, grant_id: Uuid::new_v4(), authorization_id: Uuid::new_v4(), tenant_id: job.tenant_id, asset_id: job.asset_id, agent_id: job.agent_id, enrollment_generation: 1, assignment_version: 1, job_id: job.job_id, attempt_id: job.attempt_id, plan_hash: hash(b"fixed test plan; no native approval"), job_payload_hash: hash(&encode(&job).unwrap()), action: "run_script".into(), execution_identity: "root".into(), not_before: now, start_before: now + chrono::Duration::seconds(59), max_runtime_seconds: 120, key_id: hash(&public), }; let mut request = Request { job_bytes: encode(&job).unwrap(), permit_bytes: encode(&permit).unwrap(), signature: vec![], plan_hash: permit.plan_hash.clone(), authorization_id: permit.authorization_id, }; request.signature = Signer::new_without_digest(&key) .unwrap() .sign_oneshot_to_vec(&request.permit_bytes) .unwrap(); let offer = JobOffer { job_id: job.job_id.to_string(), attempt_id: job.attempt_id.to_string(), authorization_id: permit.authorization_id.to_string(), plan_hash: permit.plan_hash.clone(), job_payload_hash: permit.job_payload_hash.clone(), payload_bytes: request.job_bytes.clone(), }; let db = storage::open(&dir).unwrap(); let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(); rmm_privileged_helper::consume(&mut ledger, &permit).unwrap(); rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap(); db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)", params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap(); let mut queued_job = job.clone(); queued_job.job_id = Uuid::new_v4(); queued_job.attempt_id = Uuid::new_v4(); let mut queued = offer.clone(); queued.job_id = queued_job.job_id.to_string(); queued.attempt_id = queued_job.attempt_id.to_string(); queued.payload_bytes = encode(&queued_job).unwrap(); queued.job_payload_hash = hash(&queued.payload_bytes); db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')", params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap(); let (tx, rx) = tokio::sync::mpsc::channel(1); let (itx, irx) = tokio::sync::mpsc::channel(1); let certificates = certificate_rotation::RotationStore::open( &dir.join("certificates"), &[], binding.agent_id, 1, ) .unwrap(); let mut agent = Agent { address: "127.0.0.1:1".parse().unwrap(), ca: vec![], id: binding.agent_id.to_string(), boot: rmm_agent_linux::boot_id().unwrap(), instance: Uuid::new_v4().to_string(), db, started: Instant::now(), clock: rmm_privileged_helper::TrustedClock::new(), binding: Some(binding.clone()), ledger, completed_tx: tx, completed_rx: rx, inventory_tx: itx, inventory_rx: irx, inventory_inflight: false, running_cancels: std::collections::HashMap::new(), sync: Some(reconcile::ClientSync::new(binding.clone())), sync_clock: Clock(true), helpers: None, prepared: None, fence: Arc::new(EffectFence::default()), mode: wire_channel::Mode::Wss, certificates, rotation_policy: certificate_rotation::RotationPolicy::default(), }; // A forged local consumption/start cannot become a purported grant result. agent .ledger .execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()]) .unwrap(); assert!(agent.recover_interrupted().is_err()); assert!(jobs::events(&agent.db).unwrap().is_empty()); agent .ledger .execute( "UPDATE consumed SET agent=?1", [binding.agent_id.to_string()], ) .unwrap(); agent.recover_interrupted().unwrap(); let original = jobs::events(&agent.db).unwrap(); assert_eq!(original.len(), 1); let Some(Payload::JobResult(result)) = &original[0].payload else { panic!("real durable result required") }; assert_eq!(result.state, "unknown"); assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED"); assert_eq!(result.grant_id, permit.grant_id.to_string()); assert_eq!(result.revision, 2); assert!(result.stdout.is_empty() && result.exit_code.is_none()); assert!(!agent.fence.allows(1, true)); let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap(); assert_eq!(jobs::events(&reopened).unwrap(), original); let requesting: (String, u64) = reopened .query_row( "SELECT state,revision FROM job_attempts WHERE job=?1", [&queued.job_id], |r| Ok((r.get(0)?, r.get(1)?)), ) .unwrap(); assert_eq!(requesting, ("unknown".into(), 1)); let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); assert_eq!(retained, (1, 1, 1)); // The same process/restart cannot invent another revision or message ID. agent.recover_interrupted().unwrap(); assert_eq!(jobs::events(&agent.db).unwrap(), original); drop(reopened); drop(agent); fs::remove_dir_all(dir).unwrap(); } #[test] fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() { let gate = EffectFence::default(); assert!(!gate.allows(1, false)); gate.deadline .store(Utc::now().timestamp_millis() + 10000, Ordering::SeqCst); gate.generation.store(1, Ordering::SeqCst); assert!(gate.allows(1, false)); assert!(!gate.allows(2, false)); assert!(!gate.allows(1, true)); gate.root.store(true, Ordering::SeqCst); assert!(gate.allows(1, true)); gate.deadline.store(0, Ordering::SeqCst); assert!(!gate.allows(1, false)); gate.close(); assert!(!gate.allows(1, true)); } impl reconcile::SyncClock for Clock { fn now(&self) -> Result> { ensure!(self.0, "CLOCK_UNTRUSTED"); Ok(Utc::now()) } } fn certificate( name: &str, issuer: Option<(&X509, &PKey)>, server: bool, ) -> (X509, PKey) { let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap(); let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap(); let mut subject = X509NameBuilder::new().unwrap(); subject.append_entry_by_text("CN", name).unwrap(); let subject = subject.build(); let mut b = X509::builder().unwrap(); b.set_version(2).unwrap(); b.set_serial_number( &BigNum::from_u32(rand::random_range(1..u32::MAX)) .unwrap() .to_asn1_integer() .unwrap(), ) .unwrap(); b.set_subject_name(&subject).unwrap(); b.set_issuer_name(issuer.map_or(subject.as_ref(), |(ca, _)| ca.subject_name())) .unwrap(); b.set_pubkey(&key).unwrap(); b.set_not_before(&Asn1Time::days_from_now(0).unwrap()) .unwrap(); b.set_not_after(&Asn1Time::days_from_now(1).unwrap()) .unwrap(); if issuer.is_none() { b.append_extension(BasicConstraints::new().critical().ca().build().unwrap()) .unwrap(); b.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap()) .unwrap(); } else { b.append_extension(BasicConstraints::new().build().unwrap()) .unwrap(); let mut u = ExtendedKeyUsage::new(); if server { u.server_auth(); } else { u.client_auth(); } b.append_extension(u.build().unwrap()).unwrap(); if server { let context = b.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None); let san = SubjectAlternativeName::new() .dns(name) .build(&context) .unwrap(); b.append_extension(san).unwrap(); } } b.sign(issuer.map_or(&key, |(_, k)| k), MessageDigest::sha256()) .unwrap(); (b.build(), key) } fn reply(input: &Envelope, generation: u64, payload: Payload) -> Envelope { Envelope { message_id: Uuid::new_v4().to_string(), connection_generation: generation, payload: Some(payload), ..input.clone() } } struct Server { binding: rmm_privileged_helper::Binding, key: PKey, generation: u64, previous: Option, requests: usize, heartbeats: usize, forged: bool, } impl Server { fn response(&mut self, e: &Envelope) -> Envelope { match e.payload.as_ref().unwrap() { Payload::ClientHello(h) => { assert!(h.capabilities.iter().any(|c| c == "rmm.sync.v1")); reply( e, self.generation, Payload::ServerHello(ServerHello { connection_generation: self.generation, max_message_bytes: rmm_domain::MAX_MESSAGE as u32, }), ) } Payload::SyncRequest(r) => { assert!(r.journal_complete && r.attempts.is_empty()); if let Some(previous) = self.previous { assert_eq!(r.previous_decision_id, previous.to_string()); } let now = Utc::now(); let d = ReconcileDecisionPayload { domain: "rmm.reconciliation-decision".into(), schema_version: 1, decision_id: Uuid::new_v4(), request_id: Uuid::parse_str(&r.request_id).unwrap(), request_hash: rmm_domain::jobs::hash(&r.encode_to_vec()), tenant_id: self.binding.tenant_id, asset_id: self.binding.asset_id, agent_id: self.binding.agent_id, enrollment_generation: 1, assignment_version: 1, connection_generation: self.generation, boot_id: Uuid::parse_str(&e.boot_id).unwrap(), journal_id: Uuid::parse_str(&r.journal_id).unwrap(), checkpoint: r.checkpoint, helper_journal_id: None, helper_checkpoint: 0, issued_at: now, valid_until: now + chrono::Duration::seconds(60), ready: true, helper_ready: false, reason: "CONSISTENT".into(), attempts: vec![], acknowledged_message_ids: vec![], key_id: rmm_domain::jobs::hash(&self.key.raw_public_key().unwrap()), }; let bytes = rmm_domain::jobs::encode(&d).unwrap(); let mut signature = openssl::sign::Signer::new_without_digest(&self.key) .unwrap() .sign_oneshot_to_vec(&bytes) .unwrap(); if self.forged { signature[0] ^= 1; } else { self.previous = Some(d.decision_id); } self.requests += 1; reply( e, self.generation, Payload::SyncResponse(SyncResponse { decision_bytes: bytes, signature, }), ) } Payload::Heartbeat(_) => { self.heartbeats += 1; reply( e, self.generation, Payload::Ack(Ack { message_id: e.message_id.clone(), }), ) } Payload::InventorySnapshot(_) => reply( e, self.generation, Payload::Ack(Ack { message_id: e.message_id.clone(), }), ), _ => panic!("fixture received unexpected execution authority"), } } } fn http_body(stream: &mut impl Read) -> Vec { let mut head = vec![]; while !head.ends_with(b"\r\n\r\n") { let mut b = [0; 1]; stream.read_exact(&mut b).unwrap(); head.push(b[0]); assert!(head.len() < 8192); } let text = String::from_utf8(head).unwrap(); let length: usize = text .lines() .filter_map(|line| line.split_once(':')) .find(|(k, _)| k.eq_ignore_ascii_case("content-length")) .unwrap() .1 .trim() .parse() .unwrap(); assert!(length <= rmm_domain::MAX_MESSAGE); let mut bytes = vec![0; length]; stream.read_exact(&mut bytes).unwrap(); bytes } async fn run(mode: wire_channel::Mode, trusted: bool, forged: bool, rounds: u64) { let dir = PathBuf::from("/tmp/opencode").join(format!("runtime-{}", Uuid::new_v4())); storage::prepare_dir(&dir).unwrap(); let id = Uuid::new_v4(); let (ca, ca_key) = certificate("runtime CA", None, false); let (server, server_key) = certificate("rmm-dev.internal", Some((&ca, &ca_key)), true); let (device, device_key) = certificate(&id.to_string(), Some((&ca, &ca_key)), false); let key = PKey::generate_ed25519().unwrap(); let public = key.raw_public_key().unwrap(); let binding = rmm_privileged_helper::Binding { tenant_id: Uuid::new_v4(), asset_id: Uuid::new_v4(), agent_id: id, enrollment_generation: 1, assignment_version: 1, public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(), }; let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap(); acceptor.set_certificate(&server).unwrap(); acceptor.set_private_key(&server_key).unwrap(); acceptor.cert_store_mut().add_cert(ca.clone()).unwrap(); acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT); acceptor.set_session_id_context(b"rmm-runtime").unwrap(); let acceptor = acceptor.build(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server_binding = binding.clone(); let worker = std::thread::spawn(move || { let mut state = Server { binding: server_binding, key, generation: 1, previous: None, requests: 0, heartbeats: 0, forged, }; for generation in 1..=rounds { state.generation = generation; loop { let (socket, _) = listener.accept().unwrap(); socket .set_read_timeout(Some(Duration::from_secs(8))) .unwrap(); let mut tls = acceptor.accept(socket).unwrap(); assert!(tls.ssl().peer_certificate().is_some()); if mode == wire_channel::Mode::Wss { let mut ws = tokio_tungstenite::tungstenite::accept(tls).unwrap(); loop { let message = ws.read().unwrap(); let tokio_tungstenite::tungstenite::Message::Binary(bytes) = message else { continue; }; let e = Envelope::decode(bytes).unwrap(); if matches!(e.payload, Some(Payload::Heartbeat(_))) { state.heartbeats += 1; let _ = ws.close(None); break; } if matches!(e.payload, Some(Payload::ClientHello(_))) { ws.send(tokio_tungstenite::tungstenite::Message::Ping( vec![1].into(), )) .unwrap(); } ws.send(tokio_tungstenite::tungstenite::Message::Binary( state.response(&e).encode_to_vec().into(), )) .unwrap(); } break; } else { let batch = MessageBatch::decode(http_body(&mut tls).as_slice()).unwrap(); let e = &batch.messages[0]; if matches!(e.payload, Some(Payload::Heartbeat(_))) { state.heartbeats += 1; tls.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap(); break; } let response = MessageBatch { protocol_major: 1, messages: vec![state.response(e)], } .encode_to_vec(); let header = format!( "HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", response.len() ); tls.write_all(header.as_bytes()).unwrap(); tls.write_all(&response).unwrap(); } } } (state.requests, state.heartbeats) }); let certificates = certificate_rotation::RotationStore::open( &dir.join("certificates"), &ca.to_pem().unwrap(), id, 1, ) .unwrap(); certificates .initialize( &device.to_pem().unwrap(), &device_key.private_key_to_pem_pkcs8().unwrap(), Utc::now().timestamp(), ) .unwrap(); let (tx, rx) = tokio::sync::mpsc::channel(1); let (itx, irx) = tokio::sync::mpsc::channel(1); let mut agent = Agent { address, ca: ca.to_pem().unwrap(), id: id.to_string(), boot: rmm_agent_linux::boot_id().unwrap(), instance: Uuid::new_v4().to_string(), db: storage::open(&dir).unwrap(), started: Instant::now(), clock: rmm_privileged_helper::TrustedClock::new(), binding: Some(binding.clone()), ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(), completed_tx: tx, completed_rx: rx, inventory_tx: itx, inventory_rx: irx, inventory_inflight: false, running_cancels: std::collections::HashMap::new(), sync: Some(reconcile::ClientSync::new(binding)), sync_clock: Clock(trusted), helpers: None, prepared: None, fence: Arc::new(EffectFence::default()), mode, certificates, rotation_policy: certificate_rotation::RotationPolicy::default(), }; for _ in 0..rounds { assert!( tokio::time::timeout(Duration::from_secs(10), agent.session()) .await .unwrap() .is_err() ); assert!(!agent.fence.allows(1, false)); } let (requests, heartbeats) = worker.join().unwrap(); assert_eq!(heartbeats, rounds as usize); if trusted { assert_eq!(requests, rounds as usize); let decided: Option = agent .db .query_row("SELECT last_decision FROM reconcile_meta", [], |r| r.get(0)) .unwrap(); assert_eq!(decided.is_some(), !forged); } else { assert_eq!(requests, 0); } drop(agent); fs::remove_dir_all(dir).unwrap(); } #[tokio::test] async fn actual_https_dispatcher_commits_signed_sync_and_clean_reconnect() { run(wire_channel::Mode::Https, true, false, 2).await; } #[tokio::test] async fn actual_wss_dispatcher_handles_ping_and_clean_reconnect() { run(wire_channel::Mode::Wss, true, false, 2).await; } #[tokio::test] async fn forged_sync_never_opens_dispatcher_gate() { run(wire_channel::Mode::Https, true, true, 1).await; } #[tokio::test] async fn untrusted_clock_keeps_telemetry_transport_but_never_requests_start() { run(wire_channel::Mode::Https, false, false, 1).await; }