feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,400 @@
|
||||
//! Actual dispatcher/transport tests with a local mTLS fixture, no real server
|
||||
//! approval, root helper, package/service/reboot or target VM effects.
|
||||
use super::*;
|
||||
use openssl::{
|
||||
asn1::Asn1Time,
|
||||
bn::BigNum,
|
||||
ssl::{SslAcceptor, SslMethod, SslVerifyMode},
|
||||
x509::{
|
||||
X509, X509NameBuilder,
|
||||
extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName},
|
||||
},
|
||||
};
|
||||
use rmm_domain::reconciliation::ReconcileDecisionPayload;
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
net::TcpListener,
|
||||
};
|
||||
|
||||
struct Clock(bool);
|
||||
#[test]
|
||||
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
|
||||
let gate = EffectFence::default();
|
||||
assert!(!gate.allows(1, false));
|
||||
gate.deadline
|
||||
.store(Utc::now().timestamp_millis() + 10000, Ordering::SeqCst);
|
||||
gate.generation.store(1, Ordering::SeqCst);
|
||||
assert!(gate.allows(1, false));
|
||||
assert!(!gate.allows(2, false));
|
||||
assert!(!gate.allows(1, true));
|
||||
gate.root.store(true, Ordering::SeqCst);
|
||||
assert!(gate.allows(1, true));
|
||||
gate.deadline.store(0, Ordering::SeqCst);
|
||||
assert!(!gate.allows(1, false));
|
||||
gate.close();
|
||||
assert!(!gate.allows(1, true));
|
||||
}
|
||||
impl reconcile::SyncClock for Clock {
|
||||
fn now(&self) -> Result<chrono::DateTime<Utc>> {
|
||||
ensure!(self.0, "CLOCK_UNTRUSTED");
|
||||
Ok(Utc::now())
|
||||
}
|
||||
}
|
||||
fn certificate(
|
||||
name: &str,
|
||||
issuer: Option<(&X509, &PKey<openssl::pkey::Private>)>,
|
||||
server: bool,
|
||||
) -> (X509, PKey<openssl::pkey::Private>) {
|
||||
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap();
|
||||
let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap();
|
||||
let mut subject = X509NameBuilder::new().unwrap();
|
||||
subject.append_entry_by_text("CN", name).unwrap();
|
||||
let subject = subject.build();
|
||||
let mut b = X509::builder().unwrap();
|
||||
b.set_version(2).unwrap();
|
||||
b.set_serial_number(
|
||||
&BigNum::from_u32(rand::random_range(1..u32::MAX))
|
||||
.unwrap()
|
||||
.to_asn1_integer()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
b.set_subject_name(&subject).unwrap();
|
||||
b.set_issuer_name(issuer.map_or(subject.as_ref(), |(ca, _)| ca.subject_name()))
|
||||
.unwrap();
|
||||
b.set_pubkey(&key).unwrap();
|
||||
b.set_not_before(&Asn1Time::days_from_now(0).unwrap())
|
||||
.unwrap();
|
||||
b.set_not_after(&Asn1Time::days_from_now(1).unwrap())
|
||||
.unwrap();
|
||||
if issuer.is_none() {
|
||||
b.append_extension(BasicConstraints::new().critical().ca().build().unwrap())
|
||||
.unwrap();
|
||||
b.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap())
|
||||
.unwrap();
|
||||
} else {
|
||||
b.append_extension(BasicConstraints::new().build().unwrap())
|
||||
.unwrap();
|
||||
let mut u = ExtendedKeyUsage::new();
|
||||
if server {
|
||||
u.server_auth();
|
||||
} else {
|
||||
u.client_auth();
|
||||
}
|
||||
b.append_extension(u.build().unwrap()).unwrap();
|
||||
if server {
|
||||
let context = b.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None);
|
||||
let san = SubjectAlternativeName::new()
|
||||
.dns(name)
|
||||
.build(&context)
|
||||
.unwrap();
|
||||
b.append_extension(san).unwrap();
|
||||
}
|
||||
}
|
||||
b.sign(issuer.map_or(&key, |(_, k)| k), MessageDigest::sha256())
|
||||
.unwrap();
|
||||
(b.build(), key)
|
||||
}
|
||||
fn reply(input: &Envelope, generation: u64, payload: Payload) -> Envelope {
|
||||
Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
connection_generation: generation,
|
||||
payload: Some(payload),
|
||||
..input.clone()
|
||||
}
|
||||
}
|
||||
struct Server {
|
||||
binding: rmm_privileged_helper::Binding,
|
||||
key: PKey<openssl::pkey::Private>,
|
||||
generation: u64,
|
||||
previous: Option<Uuid>,
|
||||
requests: usize,
|
||||
heartbeats: usize,
|
||||
forged: bool,
|
||||
}
|
||||
impl Server {
|
||||
fn response(&mut self, e: &Envelope) -> Envelope {
|
||||
match e.payload.as_ref().unwrap() {
|
||||
Payload::ClientHello(h) => {
|
||||
assert!(h.capabilities.iter().any(|c| c == "rmm.sync.v1"));
|
||||
reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::ServerHello(ServerHello {
|
||||
connection_generation: self.generation,
|
||||
max_message_bytes: rmm_domain::MAX_MESSAGE as u32,
|
||||
}),
|
||||
)
|
||||
}
|
||||
Payload::SyncRequest(r) => {
|
||||
assert!(r.journal_complete && r.attempts.is_empty());
|
||||
if let Some(previous) = self.previous {
|
||||
assert_eq!(r.previous_decision_id, previous.to_string());
|
||||
}
|
||||
let now = Utc::now();
|
||||
let d = ReconcileDecisionPayload {
|
||||
domain: "rmm.reconciliation-decision".into(),
|
||||
schema_version: 1,
|
||||
decision_id: Uuid::new_v4(),
|
||||
request_id: Uuid::parse_str(&r.request_id).unwrap(),
|
||||
request_hash: rmm_domain::jobs::hash(&r.encode_to_vec()),
|
||||
tenant_id: self.binding.tenant_id,
|
||||
asset_id: self.binding.asset_id,
|
||||
agent_id: self.binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
connection_generation: self.generation,
|
||||
boot_id: Uuid::parse_str(&e.boot_id).unwrap(),
|
||||
journal_id: Uuid::parse_str(&r.journal_id).unwrap(),
|
||||
checkpoint: r.checkpoint,
|
||||
helper_journal_id: None,
|
||||
helper_checkpoint: 0,
|
||||
issued_at: now,
|
||||
valid_until: now + chrono::Duration::seconds(60),
|
||||
ready: true,
|
||||
helper_ready: false,
|
||||
reason: "CONSISTENT".into(),
|
||||
attempts: vec![],
|
||||
acknowledged_message_ids: vec![],
|
||||
key_id: rmm_domain::jobs::hash(&self.key.raw_public_key().unwrap()),
|
||||
};
|
||||
let bytes = rmm_domain::jobs::encode(&d).unwrap();
|
||||
let mut signature = openssl::sign::Signer::new_without_digest(&self.key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&bytes)
|
||||
.unwrap();
|
||||
if self.forged {
|
||||
signature[0] ^= 1;
|
||||
} else {
|
||||
self.previous = Some(d.decision_id);
|
||||
}
|
||||
self.requests += 1;
|
||||
reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::SyncResponse(SyncResponse {
|
||||
decision_bytes: bytes,
|
||||
signature,
|
||||
}),
|
||||
)
|
||||
}
|
||||
Payload::Heartbeat(_) => {
|
||||
self.heartbeats += 1;
|
||||
reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::Ack(Ack {
|
||||
message_id: e.message_id.clone(),
|
||||
}),
|
||||
)
|
||||
}
|
||||
Payload::InventorySnapshot(_) => reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::Ack(Ack {
|
||||
message_id: e.message_id.clone(),
|
||||
}),
|
||||
),
|
||||
_ => panic!("fixture received unexpected execution authority"),
|
||||
}
|
||||
}
|
||||
}
|
||||
fn http_body(stream: &mut impl Read) -> Vec<u8> {
|
||||
let mut head = vec![];
|
||||
while !head.ends_with(b"\r\n\r\n") {
|
||||
let mut b = [0; 1];
|
||||
stream.read_exact(&mut b).unwrap();
|
||||
head.push(b[0]);
|
||||
assert!(head.len() < 8192);
|
||||
}
|
||||
let text = String::from_utf8(head).unwrap();
|
||||
let length: usize = text
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case("content-length"))
|
||||
.unwrap()
|
||||
.1
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert!(length <= rmm_domain::MAX_MESSAGE);
|
||||
let mut bytes = vec![0; length];
|
||||
stream.read_exact(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
async fn run(mode: wire_channel::Mode, trusted: bool, forged: bool, rounds: u64) {
|
||||
let dir = PathBuf::from("/tmp/opencode").join(format!("runtime-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&dir).unwrap();
|
||||
let id = Uuid::new_v4();
|
||||
let (ca, ca_key) = certificate("runtime CA", None, false);
|
||||
let (server, server_key) = certificate("rmm-dev.internal", Some((&ca, &ca_key)), true);
|
||||
let (device, device_key) = certificate(&id.to_string(), Some((&ca, &ca_key)), false);
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let public = key.raw_public_key().unwrap();
|
||||
let binding = rmm_privileged_helper::Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
};
|
||||
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap();
|
||||
acceptor.set_certificate(&server).unwrap();
|
||||
acceptor.set_private_key(&server_key).unwrap();
|
||||
acceptor.cert_store_mut().add_cert(ca.clone()).unwrap();
|
||||
acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT);
|
||||
acceptor.set_session_id_context(b"rmm-runtime").unwrap();
|
||||
let acceptor = acceptor.build();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server_binding = binding.clone();
|
||||
let worker = std::thread::spawn(move || {
|
||||
let mut state = Server {
|
||||
binding: server_binding,
|
||||
key,
|
||||
generation: 1,
|
||||
previous: None,
|
||||
requests: 0,
|
||||
heartbeats: 0,
|
||||
forged,
|
||||
};
|
||||
for generation in 1..=rounds {
|
||||
state.generation = generation;
|
||||
loop {
|
||||
let (socket, _) = listener.accept().unwrap();
|
||||
socket
|
||||
.set_read_timeout(Some(Duration::from_secs(8)))
|
||||
.unwrap();
|
||||
let mut tls = acceptor.accept(socket).unwrap();
|
||||
assert!(tls.ssl().peer_certificate().is_some());
|
||||
if mode == wire_channel::Mode::Wss {
|
||||
let mut ws = tokio_tungstenite::tungstenite::accept(tls).unwrap();
|
||||
loop {
|
||||
let message = ws.read().unwrap();
|
||||
let tokio_tungstenite::tungstenite::Message::Binary(bytes) = message else {
|
||||
continue;
|
||||
};
|
||||
let e = Envelope::decode(bytes).unwrap();
|
||||
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
|
||||
state.heartbeats += 1;
|
||||
let _ = ws.close(None);
|
||||
break;
|
||||
}
|
||||
if matches!(e.payload, Some(Payload::ClientHello(_))) {
|
||||
ws.send(tokio_tungstenite::tungstenite::Message::Ping(
|
||||
vec![1].into(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
ws.send(tokio_tungstenite::tungstenite::Message::Binary(
|
||||
state.response(&e).encode_to_vec().into(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
let batch = MessageBatch::decode(http_body(&mut tls).as_slice()).unwrap();
|
||||
let e = &batch.messages[0];
|
||||
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
|
||||
state.heartbeats += 1;
|
||||
tls.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
|
||||
break;
|
||||
}
|
||||
let response = MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: vec![state.response(e)],
|
||||
}
|
||||
.encode_to_vec();
|
||||
let header = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
response.len()
|
||||
);
|
||||
tls.write_all(header.as_bytes()).unwrap();
|
||||
tls.write_all(&response).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
(state.requests, state.heartbeats)
|
||||
});
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&dir.join("certificates"),
|
||||
&ca.to_pem().unwrap(),
|
||||
id,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
certificates
|
||||
.initialize(
|
||||
&device.to_pem().unwrap(),
|
||||
&device_key.private_key_to_pem_pkcs8().unwrap(),
|
||||
Utc::now().timestamp(),
|
||||
)
|
||||
.unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
let (itx, irx) = tokio::sync::mpsc::channel(1);
|
||||
let mut agent = Agent {
|
||||
address,
|
||||
ca: ca.to_pem().unwrap(),
|
||||
id: id.to_string(),
|
||||
boot: rmm_agent_linux::boot_id().unwrap(),
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db: storage::open(&dir).unwrap(),
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding: Some(binding.clone()),
|
||||
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(),
|
||||
completed_tx: tx,
|
||||
completed_rx: rx,
|
||||
inventory_tx: itx,
|
||||
inventory_rx: irx,
|
||||
inventory_inflight: false,
|
||||
sync: Some(reconcile::ClientSync::new(binding)),
|
||||
sync_clock: Clock(trusted),
|
||||
helpers: None,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode,
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
};
|
||||
for _ in 0..rounds {
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(10), agent.session())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_err()
|
||||
);
|
||||
assert!(!agent.fence.allows(1, false));
|
||||
}
|
||||
let (requests, heartbeats) = worker.join().unwrap();
|
||||
assert_eq!(heartbeats, rounds as usize);
|
||||
if trusted {
|
||||
assert_eq!(requests, rounds as usize);
|
||||
let decided: Option<String> = agent
|
||||
.db
|
||||
.query_row("SELECT last_decision FROM reconcile_meta", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(decided.is_some(), !forged);
|
||||
} else {
|
||||
assert_eq!(requests, 0);
|
||||
}
|
||||
drop(agent);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn actual_https_dispatcher_commits_signed_sync_and_clean_reconnect() {
|
||||
run(wire_channel::Mode::Https, true, false, 2).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn actual_wss_dispatcher_handles_ping_and_clean_reconnect() {
|
||||
run(wire_channel::Mode::Wss, true, false, 2).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn forged_sync_never_opens_dispatcher_gate() {
|
||||
run(wire_channel::Mode::Https, true, true, 1).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn untrusted_clock_keeps_telemetry_transport_but_never_requests_start() {
|
||||
run(wire_channel::Mode::Https, false, false, 1).await;
|
||||
}
|
||||
Reference in New Issue
Block a user