264 lines
12 KiB
Rust
264 lines
12 KiB
Rust
//! One durable ingestion service for WSS and HTTPS; transport only frames replies.
|
|
use anyhow::{Context, Result, ensure};
|
|
use chrono::{DateTime, Utc};
|
|
use prost::Message;
|
|
use rmm_domain::MAX_MESSAGE;
|
|
use rmm_protocol::*;
|
|
use serde_json::{Value, json};
|
|
use sha2::{Digest, Sha256};
|
|
use sqlx::Row;
|
|
use uuid::Uuid;
|
|
|
|
pub fn reply(e: &Envelope, payload: Payload, generation: u64) -> Envelope {
|
|
Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: e.agent_id.clone(),
|
|
enrollment_generation: 1,
|
|
connection_generation: generation,
|
|
boot_id: e.boot_id.clone(),
|
|
sequence: e.sequence,
|
|
observed_at: Utc::now().timestamp_millis(),
|
|
payload: Some(payload),
|
|
}
|
|
}
|
|
pub async fn binding(state: &crate::State, fingerprint: &str) -> Result<(Uuid, Uuid)> {
|
|
sqlx::query_as("SELECT tenant_id,agent_id FROM device_binding($1)")
|
|
.bind(fingerprint)
|
|
.fetch_optional(&state.pool)
|
|
.await?
|
|
.context("device not authorized")
|
|
}
|
|
pub async fn control_lock(
|
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
) -> Result<()> {
|
|
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
|
.bind(format!("agent-control:{tenant}:{agent}"))
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
pub async fn open(
|
|
state: &crate::State,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
transport: &str,
|
|
) -> Result<Envelope> {
|
|
ensure!(valid_envelope(e, &agent.to_string(), 0), "invalid hello");
|
|
let Some(Payload::ClientHello(client)) = &e.payload else {
|
|
anyhow::bail!("hello required")
|
|
};
|
|
let observed =
|
|
DateTime::from_timestamp_millis(e.observed_at).context("invalid hello timestamp")?;
|
|
ensure!(
|
|
(Utc::now() - observed).num_seconds().unsigned_abs() <= 45,
|
|
"hello clock outside window"
|
|
);
|
|
ensure!(
|
|
client.capabilities.len() <= 64 && client.capabilities.iter().all(|c| c.len() <= 128),
|
|
"capability bound"
|
|
);
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
control_lock(&mut tx, tenant, agent).await?;
|
|
anyhow::ensure!(
|
|
client.agent_version.len() <= 64
|
|
&& client.agent_version.bytes().all(|b| b.is_ascii_graphic()),
|
|
"invalid agent version"
|
|
);
|
|
let generation:i64=sqlx::query_scalar("UPDATE agent_installations SET connection_generation=connection_generation+1,capabilities=$2,transport=$3,connection_boot_id=$4,restore_blocked=restore_blocked OR $5<=last_sequence,sequence_regressed=sequence_regressed OR $5<=last_sequence,sync_required=$6,sync_ready=NOT $6,helper_sync_ready=false,last_sequence=$5,reported_agent_version=NULLIF($7,'') WHERE agent_id=$1 AND revoked_at IS NULL RETURNING connection_generation").bind(agent).bind(serde_json::to_value(&client.capabilities)?).bind(transport).bind(Uuid::parse_str(&e.boot_id)?).bind(i64::try_from(e.sequence)?).bind(client.capabilities.iter().any(|c|c=="rmm.sync.v1")).bind(&client.agent_version).fetch_one(&mut *tx).await?;
|
|
tx.commit().await?;
|
|
Ok(reply(
|
|
e,
|
|
Payload::ServerHello(ServerHello {
|
|
connection_generation: u64::try_from(generation)?,
|
|
max_message_bytes: MAX_MESSAGE as u32,
|
|
}),
|
|
u64::try_from(generation)?,
|
|
))
|
|
}
|
|
pub async fn ingest(
|
|
state: &crate::State,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
transport: &str,
|
|
) -> Result<Envelope> {
|
|
ensure!(
|
|
valid_envelope(e, &agent.to_string(), e.connection_generation),
|
|
"invalid envelope"
|
|
);
|
|
let job = matches!(
|
|
e.payload,
|
|
Some(Payload::JobAccepted(_) | Payload::JobStateChanged(_) | Payload::JobResult(_))
|
|
);
|
|
let durable = job || matches!(e.payload, Some(Payload::InventorySnapshot(_)));
|
|
let observed =
|
|
DateTime::from_timestamp_millis(e.observed_at).context("invalid observation time")?;
|
|
let age = (Utc::now() - observed).num_seconds();
|
|
ensure!(
|
|
age >= -45 && (job || age <= if durable { 72 * 3600 } else { 45 }),
|
|
"stale/future envelope"
|
|
);
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
control_lock(&mut tx, tenant, agent).await?;
|
|
let row=sqlx::query("SELECT revoked_at,connection_generation,transport,connection_boot_id,last_sequence,capabilities,customer_transfer_pending FROM agent_installations WHERE agent_id=$1 FOR UPDATE").bind(agent).fetch_one(&mut *tx).await?;
|
|
let active = row.get::<Option<DateTime<Utc>>, _>("revoked_at").is_none()
|
|
&& row.get::<i64, _>("connection_generation") == i64::try_from(e.connection_generation)?
|
|
&& row.get::<String, _>("transport") == transport;
|
|
ensure!(active || job, "revoked/fenced channel");
|
|
let rate:i32=sqlx::query_scalar("UPDATE agent_installations SET ingress_count=CASE WHEN ingress_window<now()-interval '1 second' THEN 1 ELSE ingress_count+1 END,ingress_window=CASE WHEN ingress_window<now()-interval '1 second' THEN now() ELSE ingress_window END WHERE agent_id=$1 RETURNING ingress_count").bind(agent).fetch_one(&mut *tx).await?;
|
|
ensure!(rate <= 20, "device ingress rate limit");
|
|
if !durable {
|
|
ensure!(
|
|
active
|
|
&& row.get::<Option<Uuid>, _>("connection_boot_id")
|
|
== Some(Uuid::parse_str(&e.boot_id)?)
|
|
&& i64::try_from(e.sequence)? > row.get::<i64, _>("last_sequence"),
|
|
"fresh message replay"
|
|
);
|
|
}
|
|
if active {
|
|
sqlx::query("UPDATE agent_installations SET last_sequence=GREATEST(last_sequence,$2) WHERE agent_id=$1").bind(agent).bind(i64::try_from(e.sequence)?).execute(&mut *tx).await?;
|
|
}
|
|
if let Some(Payload::SyncRequest(request)) = &e.payload {
|
|
let response =
|
|
crate::synchronization::reconcile(state, &mut tx, tenant, agent, e, request).await?;
|
|
tx.commit().await?;
|
|
return Ok(reply(
|
|
e,
|
|
Payload::SyncResponse(response),
|
|
e.connection_generation,
|
|
));
|
|
}
|
|
if let Some(Payload::ExecutionPermitRequest(request)) = &e.payload {
|
|
tx.commit().await?;
|
|
return Ok(reply(
|
|
e,
|
|
Payload::ExecutionPermit(
|
|
crate::job_gateway::permit(state, tenant, agent, e, request).await?,
|
|
),
|
|
e.connection_generation,
|
|
));
|
|
}
|
|
match &e.payload {
|
|
Some(Payload::Heartbeat(h)) => {
|
|
Uuid::parse_str(&h.instance_id)?;
|
|
let usage = h.system_usage.as_ref().and_then(|sample| system_usage(sample, observed));
|
|
sqlx::query("UPDATE agent_installations SET last_contact_at=now(),
|
|
system_usage=CASE WHEN $2::jsonb IS NULL THEN system_usage
|
|
ELSE COALESCE(system_usage,'{}'::jsonb) || $2 END,
|
|
system_usage_observed_at=CASE WHEN $2::jsonb ? 'ram_available_bytes'
|
|
THEN $3 ELSE system_usage_observed_at END
|
|
WHERE agent_id=$1 AND revoked_at IS NULL")
|
|
.bind(agent).bind(usage).bind(observed).execute(&mut *tx).await?;
|
|
}
|
|
Some(Payload::InventorySnapshot(s)) => {
|
|
ensure!(
|
|
matches!(
|
|
s.scope.as_str(),
|
|
"linux.base" | "windows.base" | "opnsense.base" | "agent.update"
|
|
) && s.schema_version == 1
|
|
&& s.chunk_count == 1
|
|
&& s.chunk_index == 0
|
|
&& s.completeness == "partial"
|
|
&& s.data.len() <= 512 * 1024,
|
|
"unsupported snapshot"
|
|
);
|
|
ensure!(
|
|
s.sha256 == Sha256::digest(&s.data).as_slice(),
|
|
"snapshot hash mismatch"
|
|
);
|
|
let capabilities: Vec<String> = serde_json::from_value(row.get("capabilities"))?;
|
|
ensure!(
|
|
capabilities.contains(&format!("{}/1", s.scope)),
|
|
"snapshot capability mismatch"
|
|
);
|
|
let inventory =
|
|
rmm_domain::inventory_document::validate_for_scope(&s.data, Utc::now(), &s.scope)
|
|
.map_err(anyhow::Error::msg)?;
|
|
let mut identity = e.clone();
|
|
identity.connection_generation = 0;
|
|
let hash = hex::encode(Sha256::digest(identity.encode_to_vec()));
|
|
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$10,'valid','partial',$7,$8,$9) ON CONFLICT(tenant_id,message_id) DO NOTHING").bind(tenant).bind(agent).bind(Uuid::parse_str(&e.message_id)?).bind(Uuid::parse_str(&e.boot_id)?).bind(i64::try_from(e.sequence)?).bind(observed).bind(Uuid::parse_str(&s.snapshot_id)?).bind(serde_json::to_value(inventory)?).bind(&hash).bind(&s.scope).execute(&mut *tx).await?;
|
|
let stored: String =
|
|
sqlx::query_scalar("SELECT payload_sha256 FROM observations WHERE message_id=$1")
|
|
.bind(Uuid::parse_str(&e.message_id)?)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
ensure!(stored == hash, "message identity conflict");
|
|
}
|
|
Some(Payload::JobAccepted(_) | Payload::JobStateChanged(_) | Payload::JobResult(_)) => {
|
|
crate::job_gateway::event(&mut tx, tenant, agent, e).await?
|
|
}
|
|
_ => anyhow::bail!("unsupported capability"),
|
|
}
|
|
if row.get::<bool, _>("customer_transfer_pending")
|
|
&& (matches!(&e.payload, Some(Payload::Heartbeat(_)))
|
|
|| matches!(&e.payload, Some(Payload::InventorySnapshot(s)) if s.scope != "agent.update"))
|
|
{
|
|
sqlx::query("SELECT finish_device_customer_transfer($1,$2)")
|
|
.bind(tenant)
|
|
.bind(agent)
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
}
|
|
tx.commit().await?;
|
|
Ok(reply(
|
|
e,
|
|
Payload::Ack(Ack {
|
|
message_id: e.message_id.clone(),
|
|
}),
|
|
e.connection_generation,
|
|
))
|
|
}
|
|
|
|
fn system_usage(sample: &SystemUsage, observed: DateTime<Utc>) -> Option<Value> {
|
|
let cpu = sample
|
|
.cpu_usage_basis_points
|
|
.filter(|&value| value <= 10_000);
|
|
let memory = sample
|
|
.ram_available_bytes
|
|
.zip(sample.ram_total_bytes)
|
|
.filter(|&(available, total)| total > 0 && available <= total);
|
|
let mut usage = serde_json::Map::new();
|
|
if let Some(cpu) = cpu {
|
|
usage.insert("cpu_usage_percent".into(), json!(f64::from(cpu) / 100.0));
|
|
usage.insert("cpu_usage_observed_at".into(), json!(observed));
|
|
}
|
|
if let Some((available, total)) = memory {
|
|
usage.insert("ram_available_bytes".into(), json!(available));
|
|
usage.insert("ram_total_bytes".into(), json!(total));
|
|
}
|
|
(!usage.is_empty()).then_some(Value::Object(usage))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn accepts_zero_usage_and_ignores_invalid_metrics_without_losing_the_heartbeat() {
|
|
let mut sample = SystemUsage {
|
|
cpu_usage_basis_points: Some(0),
|
|
ram_available_bytes: Some(0),
|
|
ram_total_bytes: Some(1),
|
|
};
|
|
let now = Utc::now();
|
|
let measured = system_usage(&sample, now).unwrap();
|
|
assert_eq!(measured["cpu_usage_percent"], json!(0.0));
|
|
assert_eq!(measured["cpu_usage_observed_at"], json!(now));
|
|
assert_eq!(measured["ram_available_bytes"], json!(0));
|
|
assert_eq!(measured["ram_total_bytes"], json!(1));
|
|
sample.cpu_usage_basis_points = Some(10_001);
|
|
let partial = system_usage(&sample, now).unwrap();
|
|
assert!(partial.get("cpu_usage_percent").is_none());
|
|
assert!(partial.get("cpu_usage_observed_at").is_none());
|
|
sample.ram_available_bytes = Some(2);
|
|
assert_eq!(system_usage(&sample, now), None);
|
|
}
|
|
}
|