feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,326 @@
|
||||
//! HTTPS alternative to WSS, using the *same* durable envelopes.
|
||||
//!
|
||||
//! The caller owns the journal, sequence allocation and retry policy. In particular,
|
||||
//! HTTP success never removes a journal entry: only a returned concrete `Ack` may
|
||||
//! be passed to the journal consumer. There is deliberately no offline send queue.
|
||||
//! PollRequest has no cursor field in wire profile 1: its fresh heartbeat carries
|
||||
//! the boot/sequence/connection context. Do not invent a JSON cursor or ACK range.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use prost::Message;
|
||||
use reqwest::{Client, StatusCode};
|
||||
use rmm_protocol::{Envelope, MessageBatch, Payload, PollRequest, ServerHello};
|
||||
use std::{collections::VecDeque, net::SocketAddr, time::Duration};
|
||||
|
||||
pub const MAX_BODY: usize = 1024 * 1024;
|
||||
pub const MAX_BATCH: usize = 16;
|
||||
const MAX_INCOMING: usize = 32;
|
||||
|
||||
/// No environment proxy trust, redirects, alternate names or system CA fallback.
|
||||
/// `address` is a parsed RMM_CONNECT_ADDR; it overrides TCP only, never SNI.
|
||||
pub fn mtls_client(address: SocketAddr, ca: &[u8], cert: &[u8], key: &[u8]) -> Result<Client> {
|
||||
ensure!(
|
||||
address.port() != 0 && !address.ip().is_unspecified() && !address.ip().is_multicast(),
|
||||
"INVALID_CONNECT_ADDRESS"
|
||||
);
|
||||
let mut identity = Vec::with_capacity(cert.len() + key.len() + 1);
|
||||
identity.extend_from_slice(cert);
|
||||
identity.push(b'\n');
|
||||
identity.extend_from_slice(key);
|
||||
let client = Client::builder()
|
||||
.https_only(true)
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
||||
.identity(reqwest::Identity::from_pem(&identity)?)
|
||||
.resolve("rmm-dev.internal", address)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(35))
|
||||
.pool_max_idle_per_host(2)
|
||||
.build()?;
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Bounded response reading also applies to chunked bodies, before prost parsing.
|
||||
pub async fn bounded_body(
|
||||
mut response: reqwest::Response,
|
||||
limit: usize,
|
||||
media: &str,
|
||||
) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
response.status() == StatusCode::OK,
|
||||
"HTTPS_STATUS_{}",
|
||||
response.status().as_u16()
|
||||
);
|
||||
ensure!(
|
||||
response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
== Some(media),
|
||||
"HTTPS_CONTENT_TYPE"
|
||||
);
|
||||
ensure!(
|
||||
!response
|
||||
.headers()
|
||||
.contains_key(reqwest::header::CONTENT_ENCODING),
|
||||
"HTTPS_COMPRESSION_UNSUPPORTED"
|
||||
);
|
||||
ensure!(
|
||||
response.content_length().is_none_or(|n| n <= limit as u64),
|
||||
"HTTPS_BODY_LIMIT"
|
||||
);
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
ensure!(
|
||||
chunk.len() <= limit.saturating_sub(bytes.len()),
|
||||
"HTTPS_BODY_LIMIT"
|
||||
);
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Only an explicitly classified WebSocket upgrade refusal may select fallback.
|
||||
/// TLS, authentication and ambiguous network failures must remain blocked/retried
|
||||
/// with normal verification. Never infer this classification from an error string.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum WssFailure {
|
||||
UpgradeBlocked,
|
||||
TlsOrIdentity,
|
||||
Authentication,
|
||||
Other,
|
||||
}
|
||||
pub fn allow_https_fallback(failure: WssFailure, attempts: u32) -> bool {
|
||||
failure == WssFailure::UpgradeBlocked && (2..=3).contains(&attempts)
|
||||
}
|
||||
pub fn retry_delay(attempt: u32) -> Duration {
|
||||
let ceiling = (1_u64 << attempt.min(6)).min(60) * 1000;
|
||||
Duration::from_millis(rand::random_range(0..=ceiling))
|
||||
}
|
||||
|
||||
pub struct HttpsChannel {
|
||||
client: Client,
|
||||
origin: String,
|
||||
agent: String,
|
||||
enrollment: u64,
|
||||
generation: u64,
|
||||
incoming: VecDeque<Envelope>,
|
||||
next_request: tokio::time::Instant,
|
||||
fenced: bool,
|
||||
}
|
||||
impl HttpsChannel {
|
||||
/// Performs ClientHello first. ServerHello remains in the receive queue so
|
||||
/// the scheduler can close its effect gate and apply reconciliation normally.
|
||||
pub async fn connect(client: Client, address: SocketAddr, hello: Envelope) -> Result<Self> {
|
||||
ensure!(
|
||||
hello.connection_generation == 0
|
||||
&& matches!(hello.payload, Some(Payload::ClientHello(_))),
|
||||
"CLIENT_HELLO_REQUIRED"
|
||||
);
|
||||
ensure!(
|
||||
rmm_protocol::valid_envelope(&hello, &hello.agent_id, 0),
|
||||
"INVALID_CLIENT_HELLO"
|
||||
);
|
||||
let mut channel = Self {
|
||||
client,
|
||||
origin: format!("https://rmm-dev.internal:{}", address.port()),
|
||||
agent: hello.agent_id.clone(),
|
||||
enrollment: hello.enrollment_generation,
|
||||
generation: 0,
|
||||
incoming: VecDeque::new(),
|
||||
next_request: tokio::time::Instant::now(),
|
||||
fenced: false,
|
||||
};
|
||||
let batch = MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: vec![hello],
|
||||
};
|
||||
let reply = channel
|
||||
.request("messages", batch.encode_to_vec(), Duration::from_secs(15))
|
||||
.await?;
|
||||
let first = reply.messages.first().context("SERVER_HELLO_REQUIRED")?;
|
||||
let Some(Payload::ServerHello(ServerHello {
|
||||
connection_generation,
|
||||
max_message_bytes,
|
||||
})) = &first.payload
|
||||
else {
|
||||
anyhow::bail!("SERVER_HELLO_REQUIRED")
|
||||
};
|
||||
ensure!(
|
||||
*connection_generation > 0 && *max_message_bytes as usize == MAX_BODY,
|
||||
"UNSUPPORTED_SERVER_LIMITS"
|
||||
);
|
||||
channel.generation = *connection_generation;
|
||||
channel.accept(reply)?;
|
||||
Ok(channel)
|
||||
}
|
||||
pub fn generation(&self) -> u64 {
|
||||
self.generation
|
||||
}
|
||||
pub fn receive(&mut self) -> Option<Envelope> {
|
||||
self.incoming.pop_front()
|
||||
}
|
||||
pub fn fence(&mut self) {
|
||||
self.fenced = true;
|
||||
}
|
||||
fn validate_outgoing(&self, envelope: &Envelope) -> Result<()> {
|
||||
ensure!(
|
||||
rmm_protocol::valid_envelope(envelope, &self.agent, self.generation)
|
||||
&& envelope.enrollment_generation == self.enrollment,
|
||||
"INVALID_OUTGOING_ENVELOPE"
|
||||
);
|
||||
ensure!(envelope.encoded_len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
|
||||
Ok(())
|
||||
}
|
||||
/// Caller retries durable messages unchanged after uncertainty. Fresh permit
|
||||
/// requests must instead enter reconciliation on a lost response, not retry.
|
||||
pub async fn send(&mut self, envelope: &Envelope) -> Result<()> {
|
||||
self.send_batch(std::slice::from_ref(envelope)).await
|
||||
}
|
||||
pub async fn send_batch(&mut self, messages: &[Envelope]) -> Result<()> {
|
||||
ensure!(
|
||||
!messages.is_empty() && messages.len() <= MAX_BATCH,
|
||||
"HTTPS_BATCH_LIMIT"
|
||||
);
|
||||
ensure!(
|
||||
self.incoming.len() + messages.len() < MAX_INCOMING,
|
||||
"HTTPS_BACKPRESSURE"
|
||||
);
|
||||
for e in messages {
|
||||
self.validate_outgoing(e)?;
|
||||
ensure!(
|
||||
!matches!(e.payload, Some(Payload::ClientHello(_))),
|
||||
"HELLO_REQUIRES_NEW_CHANNEL"
|
||||
);
|
||||
if self.fenced || messages.len() > 1 {
|
||||
ensure!(
|
||||
matches!(
|
||||
e.payload,
|
||||
Some(
|
||||
Payload::InventorySnapshot(_)
|
||||
| Payload::JobAccepted(_)
|
||||
| Payload::JobStateChanged(_)
|
||||
| Payload::JobResult(_)
|
||||
)
|
||||
),
|
||||
"FENCED_OR_NON_DURABLE_BATCH"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Check before allocating/cloning the full encoded batch.
|
||||
let encoded_size = messages
|
||||
.iter()
|
||||
.try_fold(2_usize, |n, e| n.checked_add(e.encoded_len() + 5))
|
||||
.context("HTTPS_BODY_LIMIT")?;
|
||||
ensure!(encoded_size <= MAX_BODY, "HTTPS_BODY_LIMIT");
|
||||
let reply = self
|
||||
.request(
|
||||
"messages",
|
||||
MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: messages.to_vec(),
|
||||
}
|
||||
.encode_to_vec(),
|
||||
Duration::from_secs(15),
|
||||
)
|
||||
.await?;
|
||||
self.accept(reply)
|
||||
}
|
||||
/// At most one poll per channel (`&mut self`); cancelling a dropped poll is
|
||||
/// safe, but its heartbeat ID must not be reused for a different envelope.
|
||||
pub async fn poll(&mut self, heartbeat: Envelope, wait_seconds: u32) -> Result<()> {
|
||||
ensure!(!self.fenced, "HTTPS_CHANNEL_FENCED");
|
||||
ensure!(
|
||||
self.incoming.len() + 2 <= MAX_INCOMING,
|
||||
"HTTPS_BACKPRESSURE"
|
||||
);
|
||||
self.validate_outgoing(&heartbeat)?;
|
||||
ensure!(
|
||||
wait_seconds <= 25 && matches!(heartbeat.payload, Some(Payload::Heartbeat(_))),
|
||||
"INVALID_POLL"
|
||||
);
|
||||
let request = PollRequest {
|
||||
envelope: Some(heartbeat),
|
||||
wait_seconds,
|
||||
};
|
||||
let reply = self
|
||||
.request(
|
||||
"poll",
|
||||
request.encode_to_vec(),
|
||||
Duration::from_secs(u64::from(wait_seconds) + 10),
|
||||
)
|
||||
.await?;
|
||||
self.accept(reply)
|
||||
}
|
||||
async fn request(
|
||||
&mut self,
|
||||
route: &str,
|
||||
body: Vec<u8>,
|
||||
timeout: Duration,
|
||||
) -> Result<MessageBatch> {
|
||||
ensure!(body.len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
|
||||
tokio::time::sleep_until(self.next_request).await;
|
||||
// <=10 HTTP requests/sec, with jitter, even with zero-wait polls.
|
||||
self.next_request =
|
||||
tokio::time::Instant::now() + Duration::from_millis(rand::random_range(100..=150));
|
||||
let response = self
|
||||
.client
|
||||
.post(format!("{}/agent/v1/{route}", self.origin))
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/x-protobuf")
|
||||
.header(reqwest::header::ACCEPT, "application/x-protobuf")
|
||||
.timeout(timeout)
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
if response.status() == StatusCode::CONFLICT
|
||||
|| response.status() == StatusCode::UNAUTHORIZED
|
||||
|| response.status() == StatusCode::FORBIDDEN
|
||||
{
|
||||
self.fence();
|
||||
}
|
||||
let bytes = bounded_body(response, MAX_BODY, "application/x-protobuf").await?;
|
||||
let batch = MessageBatch::decode(bytes.as_slice())?;
|
||||
ensure!(
|
||||
batch.protocol_major == 1 && batch.messages.len() <= MAX_BATCH,
|
||||
"INVALID_SERVER_BATCH"
|
||||
);
|
||||
Ok(batch)
|
||||
}
|
||||
fn accept(&mut self, reply: MessageBatch) -> Result<()> {
|
||||
ensure!(
|
||||
self.incoming.len() + reply.messages.len() <= MAX_INCOMING,
|
||||
"HTTPS_BACKPRESSURE"
|
||||
);
|
||||
// Validate the entire batch before exposing any control message.
|
||||
for e in &reply.messages {
|
||||
ensure!(
|
||||
e.protocol_major == 1
|
||||
&& e.protocol_minor == 0
|
||||
&& e.agent_id == self.agent
|
||||
&& e.enrollment_generation == self.enrollment
|
||||
&& e.connection_generation == self.generation
|
||||
&& matches!(
|
||||
e.payload,
|
||||
Some(
|
||||
Payload::ServerHello(_)
|
||||
| Payload::Ack(_)
|
||||
| Payload::JobOffer(_)
|
||||
| Payload::ExecutionPermit(_)
|
||||
| Payload::CancelRequest(_)
|
||||
| Payload::SyncResponse(_)
|
||||
)
|
||||
),
|
||||
"INVALID_SERVER_ENVELOPE"
|
||||
);
|
||||
if self.fenced {
|
||||
ensure!(
|
||||
matches!(e.payload, Some(Payload::Ack(_))),
|
||||
"HTTPS_CHANNEL_FENCED"
|
||||
);
|
||||
}
|
||||
}
|
||||
self.incoming.extend(reply.messages);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user