feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s

This commit is contained in:
BuTzZ
2026-11-12 22:06:25 +01:00
commit 6c3203d725
194 changed files with 49234 additions and 0 deletions
+326
View File
@@ -0,0 +1,326 @@
//! HTTPS alternative to WSS, using the *same* durable envelopes.
//!
//! The caller owns the journal, sequence allocation and retry policy. In particular,
//! HTTP success never removes a journal entry: only a returned concrete `Ack` may
//! be passed to the journal consumer. There is deliberately no offline send queue.
//! PollRequest has no cursor field in wire profile 1: its fresh heartbeat carries
//! the boot/sequence/connection context. Do not invent a JSON cursor or ACK range.
use anyhow::{Context, Result, ensure};
use prost::Message;
use reqwest::{Client, StatusCode};
use rmm_protocol::{Envelope, MessageBatch, Payload, PollRequest, ServerHello};
use std::{collections::VecDeque, net::SocketAddr, time::Duration};
pub const MAX_BODY: usize = 1024 * 1024;
pub const MAX_BATCH: usize = 16;
const MAX_INCOMING: usize = 32;
/// No environment proxy trust, redirects, alternate names or system CA fallback.
/// `address` is a parsed RMM_CONNECT_ADDR; it overrides TCP only, never SNI.
pub fn mtls_client(address: SocketAddr, ca: &[u8], cert: &[u8], key: &[u8]) -> Result<Client> {
ensure!(
address.port() != 0 && !address.ip().is_unspecified() && !address.ip().is_multicast(),
"INVALID_CONNECT_ADDRESS"
);
let mut identity = Vec::with_capacity(cert.len() + key.len() + 1);
identity.extend_from_slice(cert);
identity.push(b'\n');
identity.extend_from_slice(key);
let client = Client::builder()
.https_only(true)
.tls_built_in_root_certs(false)
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
.identity(reqwest::Identity::from_pem(&identity)?)
.resolve("rmm-dev.internal", address)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(35))
.pool_max_idle_per_host(2)
.build()?;
Ok(client)
}
/// Bounded response reading also applies to chunked bodies, before prost parsing.
pub async fn bounded_body(
mut response: reqwest::Response,
limit: usize,
media: &str,
) -> Result<Vec<u8>> {
ensure!(
response.status() == StatusCode::OK,
"HTTPS_STATUS_{}",
response.status().as_u16()
);
ensure!(
response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
== Some(media),
"HTTPS_CONTENT_TYPE"
);
ensure!(
!response
.headers()
.contains_key(reqwest::header::CONTENT_ENCODING),
"HTTPS_COMPRESSION_UNSUPPORTED"
);
ensure!(
response.content_length().is_none_or(|n| n <= limit as u64),
"HTTPS_BODY_LIMIT"
);
let mut bytes = Vec::new();
while let Some(chunk) = response.chunk().await? {
ensure!(
chunk.len() <= limit.saturating_sub(bytes.len()),
"HTTPS_BODY_LIMIT"
);
bytes.extend_from_slice(&chunk);
}
Ok(bytes)
}
/// Only an explicitly classified WebSocket upgrade refusal may select fallback.
/// TLS, authentication and ambiguous network failures must remain blocked/retried
/// with normal verification. Never infer this classification from an error string.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum WssFailure {
UpgradeBlocked,
TlsOrIdentity,
Authentication,
Other,
}
pub fn allow_https_fallback(failure: WssFailure, attempts: u32) -> bool {
failure == WssFailure::UpgradeBlocked && (2..=3).contains(&attempts)
}
pub fn retry_delay(attempt: u32) -> Duration {
let ceiling = (1_u64 << attempt.min(6)).min(60) * 1000;
Duration::from_millis(rand::random_range(0..=ceiling))
}
pub struct HttpsChannel {
client: Client,
origin: String,
agent: String,
enrollment: u64,
generation: u64,
incoming: VecDeque<Envelope>,
next_request: tokio::time::Instant,
fenced: bool,
}
impl HttpsChannel {
/// Performs ClientHello first. ServerHello remains in the receive queue so
/// the scheduler can close its effect gate and apply reconciliation normally.
pub async fn connect(client: Client, address: SocketAddr, hello: Envelope) -> Result<Self> {
ensure!(
hello.connection_generation == 0
&& matches!(hello.payload, Some(Payload::ClientHello(_))),
"CLIENT_HELLO_REQUIRED"
);
ensure!(
rmm_protocol::valid_envelope(&hello, &hello.agent_id, 0),
"INVALID_CLIENT_HELLO"
);
let mut channel = Self {
client,
origin: format!("https://rmm-dev.internal:{}", address.port()),
agent: hello.agent_id.clone(),
enrollment: hello.enrollment_generation,
generation: 0,
incoming: VecDeque::new(),
next_request: tokio::time::Instant::now(),
fenced: false,
};
let batch = MessageBatch {
protocol_major: 1,
messages: vec![hello],
};
let reply = channel
.request("messages", batch.encode_to_vec(), Duration::from_secs(15))
.await?;
let first = reply.messages.first().context("SERVER_HELLO_REQUIRED")?;
let Some(Payload::ServerHello(ServerHello {
connection_generation,
max_message_bytes,
})) = &first.payload
else {
anyhow::bail!("SERVER_HELLO_REQUIRED")
};
ensure!(
*connection_generation > 0 && *max_message_bytes as usize == MAX_BODY,
"UNSUPPORTED_SERVER_LIMITS"
);
channel.generation = *connection_generation;
channel.accept(reply)?;
Ok(channel)
}
pub fn generation(&self) -> u64 {
self.generation
}
pub fn receive(&mut self) -> Option<Envelope> {
self.incoming.pop_front()
}
pub fn fence(&mut self) {
self.fenced = true;
}
fn validate_outgoing(&self, envelope: &Envelope) -> Result<()> {
ensure!(
rmm_protocol::valid_envelope(envelope, &self.agent, self.generation)
&& envelope.enrollment_generation == self.enrollment,
"INVALID_OUTGOING_ENVELOPE"
);
ensure!(envelope.encoded_len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
Ok(())
}
/// Caller retries durable messages unchanged after uncertainty. Fresh permit
/// requests must instead enter reconciliation on a lost response, not retry.
pub async fn send(&mut self, envelope: &Envelope) -> Result<()> {
self.send_batch(std::slice::from_ref(envelope)).await
}
pub async fn send_batch(&mut self, messages: &[Envelope]) -> Result<()> {
ensure!(
!messages.is_empty() && messages.len() <= MAX_BATCH,
"HTTPS_BATCH_LIMIT"
);
ensure!(
self.incoming.len() + messages.len() < MAX_INCOMING,
"HTTPS_BACKPRESSURE"
);
for e in messages {
self.validate_outgoing(e)?;
ensure!(
!matches!(e.payload, Some(Payload::ClientHello(_))),
"HELLO_REQUIRES_NEW_CHANNEL"
);
if self.fenced || messages.len() > 1 {
ensure!(
matches!(
e.payload,
Some(
Payload::InventorySnapshot(_)
| Payload::JobAccepted(_)
| Payload::JobStateChanged(_)
| Payload::JobResult(_)
)
),
"FENCED_OR_NON_DURABLE_BATCH"
);
}
}
// Check before allocating/cloning the full encoded batch.
let encoded_size = messages
.iter()
.try_fold(2_usize, |n, e| n.checked_add(e.encoded_len() + 5))
.context("HTTPS_BODY_LIMIT")?;
ensure!(encoded_size <= MAX_BODY, "HTTPS_BODY_LIMIT");
let reply = self
.request(
"messages",
MessageBatch {
protocol_major: 1,
messages: messages.to_vec(),
}
.encode_to_vec(),
Duration::from_secs(15),
)
.await?;
self.accept(reply)
}
/// At most one poll per channel (`&mut self`); cancelling a dropped poll is
/// safe, but its heartbeat ID must not be reused for a different envelope.
pub async fn poll(&mut self, heartbeat: Envelope, wait_seconds: u32) -> Result<()> {
ensure!(!self.fenced, "HTTPS_CHANNEL_FENCED");
ensure!(
self.incoming.len() + 2 <= MAX_INCOMING,
"HTTPS_BACKPRESSURE"
);
self.validate_outgoing(&heartbeat)?;
ensure!(
wait_seconds <= 25 && matches!(heartbeat.payload, Some(Payload::Heartbeat(_))),
"INVALID_POLL"
);
let request = PollRequest {
envelope: Some(heartbeat),
wait_seconds,
};
let reply = self
.request(
"poll",
request.encode_to_vec(),
Duration::from_secs(u64::from(wait_seconds) + 10),
)
.await?;
self.accept(reply)
}
async fn request(
&mut self,
route: &str,
body: Vec<u8>,
timeout: Duration,
) -> Result<MessageBatch> {
ensure!(body.len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
tokio::time::sleep_until(self.next_request).await;
// <=10 HTTP requests/sec, with jitter, even with zero-wait polls.
self.next_request =
tokio::time::Instant::now() + Duration::from_millis(rand::random_range(100..=150));
let response = self
.client
.post(format!("{}/agent/v1/{route}", self.origin))
.header(reqwest::header::CONTENT_TYPE, "application/x-protobuf")
.header(reqwest::header::ACCEPT, "application/x-protobuf")
.timeout(timeout)
.body(body)
.send()
.await?;
if response.status() == StatusCode::CONFLICT
|| response.status() == StatusCode::UNAUTHORIZED
|| response.status() == StatusCode::FORBIDDEN
{
self.fence();
}
let bytes = bounded_body(response, MAX_BODY, "application/x-protobuf").await?;
let batch = MessageBatch::decode(bytes.as_slice())?;
ensure!(
batch.protocol_major == 1 && batch.messages.len() <= MAX_BATCH,
"INVALID_SERVER_BATCH"
);
Ok(batch)
}
fn accept(&mut self, reply: MessageBatch) -> Result<()> {
ensure!(
self.incoming.len() + reply.messages.len() <= MAX_INCOMING,
"HTTPS_BACKPRESSURE"
);
// Validate the entire batch before exposing any control message.
for e in &reply.messages {
ensure!(
e.protocol_major == 1
&& e.protocol_minor == 0
&& e.agent_id == self.agent
&& e.enrollment_generation == self.enrollment
&& e.connection_generation == self.generation
&& matches!(
e.payload,
Some(
Payload::ServerHello(_)
| Payload::Ack(_)
| Payload::JobOffer(_)
| Payload::ExecutionPermit(_)
| Payload::CancelRequest(_)
| Payload::SyncResponse(_)
)
),
"INVALID_SERVER_ENVELOPE"
);
if self.fenced {
ensure!(
matches!(e.payload, Some(Payload::Ack(_))),
"HTTPS_CHANNEL_FENCED"
);
}
}
self.incoming.extend(reply.messages);
Ok(())
}
}