Validate Proxmox observations, evaluate scheduled backups and link VMs to their Proxmox host
Rust development slice / linux (push) Failing after 15s

linux.proxmox gets a strict validator and a 128 KiB scope budget. Device
details expose guests, backup jobs with ok/failed/missed/running status and
linked devices. Base snapshots reconcile unique guest MACs into
assets.hypervisor_asset_id with hypervisor_source='proxmox'; manual choices
are kept and can be handed back to automatic detection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-20 12:10:35 +01:00
co-authored by Claude Opus 5.5
parent 1d2d52847c
commit 70977df577
12 changed files with 597 additions and 10 deletions
@@ -526,6 +526,14 @@ mod tests {
assert_eq!(data.backup_runs[2].guests, [GuestResult { vmid: 101, result: "ok" }]); assert_eq!(data.backup_runs[2].guests, [GuestResult { vmid: 101, result: "ok" }]);
} }
#[test]
fn agent_document_passes_the_server_validator() {
let observation = block(collect(&fixture(), NOW));
let data = serde_json::to_value(observation.data.unwrap()).unwrap();
rmm_domain::inventory_document::proxmox::validate(&data).unwrap();
assert!(rmm_domain::inventory_document::proxmox::backup_status(&data).is_some());
}
#[test] #[test]
fn missing_local_node_or_api_is_an_error_observation() { fn missing_local_node_or_api_is_an_error_observation() {
let observation = block(collect(&Fixture(HashMap::new()), NOW)); let observation = block(collect(&Fixture(HashMap::new()), NOW));
+5
View File
@@ -1,5 +1,6 @@
//! Additive bounded passive metadata; the strict original Inventory stays intact. //! Additive bounded passive metadata; the strict original Inventory stays intact.
mod opnsense; mod opnsense;
pub mod proxmox;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
@@ -174,6 +175,7 @@ pub fn validate_for_scope(
for item in observations { for item in observations {
let max_bytes = match (scope, item["scope"].as_str()) { let max_bytes = match (scope, item["scope"].as_str()) {
("linux.base", Some("linux.software")) => LINUX_SOFTWARE_OBSERVATION_MAX_BYTES, ("linux.base", Some("linux.software")) => LINUX_SOFTWARE_OBSERVATION_MAX_BYTES,
("linux.base", Some("linux.proxmox")) => PROXMOX_OBSERVATION_MAX_BYTES,
("windows.base", Some("windows.software")) => SOFTWARE_OBSERVATION_MAX_BYTES, ("windows.base", Some("windows.software")) => SOFTWARE_OBSERVATION_MAX_BYTES,
_ => 32768, _ => 32768,
}; };
@@ -215,6 +217,9 @@ pub fn validate_for_scope(
return Err("INVALID_AGENT_UPDATE_CATALOG_REPORT"); return Err("INVALID_AGENT_UPDATE_CATALOG_REPORT");
} }
} }
if o.scope == "linux.proxmox" && o.quality == "valid" {
proxmox::validate(&o.data)?;
}
if !o.scope.starts_with(prefix) if !o.scope.starts_with(prefix)
|| o.scope.len() > 64 || o.scope.len() > 64
|| !(o || !(o
@@ -0,0 +1,257 @@
//! Strict `linux.proxmox` wire document and the deterministic backup evaluation.
//! The agent reports raw scheduler data; whether every scheduled backup ran is
//! decided here so the server, API and UI share one rule.
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet};
/// Runs that start shortly before the computed due time still count (clock skew).
const START_TOLERANCE_SECONDS: i64 = 120;
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Proxmox {
pub node: String,
pub cluster: Option<String>,
pub version: Option<String>,
pub guests: Vec<Guest>,
pub backup_jobs: Vec<BackupJob>,
pub backup_runs: Vec<BackupRun>,
pub not_backed_up: Vec<u32>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Guest {
pub vmid: u32,
pub name: String,
pub kind: String,
pub status: String,
pub template: bool,
pub cpus: Option<u32>,
pub memory_bytes: Option<u64>,
pub disk_bytes: Option<u64>,
pub uptime_seconds: Option<u64>,
pub pool: Option<String>,
pub macs: Vec<String>,
pub smbios_uuid: Option<String>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BackupJob {
pub id: String,
pub enabled: bool,
pub schedule: Option<String>,
pub storage: Option<String>,
pub mode: Option<String>,
pub selection: String,
pub covered_vmids: Vec<u32>,
pub next_run: Option<i64>,
pub last_expected_run: Option<i64>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BackupRun {
pub upid: String,
pub started_at: i64,
pub ended_at: Option<i64>,
pub status: String,
pub message: Option<String>,
pub guests: Vec<GuestResult>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct GuestResult {
pub vmid: u32,
pub result: String,
}
fn clean(s: &str, limit: usize) -> bool {
!s.trim().is_empty() && s.len() <= limit && !s.chars().any(char::is_control)
}
fn opt(s: &Option<String>, limit: usize) -> bool {
s.as_deref().is_none_or(|s| clean(s, limit))
}
pub fn is_mac(s: &str) -> bool {
s.len() == 17 && s.split(':').count() == 6
&& s.split(':').all(|p| p.len() == 2 && p.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
}
fn vmid(id: u32) -> bool {
(100..1_000_000_000).contains(&id)
}
/// Validates the `data` member of a valid `linux.proxmox` observation.
pub fn validate(data: &Value) -> Result<(), &'static str> {
let err = "INVALID_PROXMOX_OBSERVATION";
let p: Proxmox = serde_json::from_value(data.clone()).map_err(|_| err)?;
let guests_ok = p.guests.len() <= 512 && p.guests.iter().all(|g| {
vmid(g.vmid) && clean(&g.name, 128) && matches!(g.kind.as_str(), "qemu" | "lxc")
&& clean(&g.status, 32) && opt(&g.pool, 64) && g.macs.len() <= 32
&& g.macs.iter().all(|m| is_mac(m))
&& g.smbios_uuid.as_deref().is_none_or(|u| uuid::Uuid::parse_str(u).is_ok_and(|x| x.hyphenated().to_string() == u))
}) && p.guests.windows(2).all(|w| w[0].vmid < w[1].vmid);
let jobs_ok = p.backup_jobs.len() <= 64 && p.backup_jobs.iter().all(|j| {
clean(&j.id, 64) && opt(&j.schedule, 128) && opt(&j.storage, 64) && opt(&j.mode, 16)
&& matches!(j.selection.as_str(), "all" | "pool" | "vmid")
&& j.covered_vmids.len() <= 512 && j.covered_vmids.iter().all(|id| vmid(*id))
});
let runs_ok = p.backup_runs.len() <= 64 && p.backup_runs.iter().all(|r| {
clean(&r.upid, 160) && r.upid.starts_with("UPID:") && r.started_at > 0
&& r.ended_at.is_none_or(|e| e >= r.started_at)
&& matches!(r.status.as_str(), "ok" | "warning" | "error" | "running")
&& (r.status == "running") == r.ended_at.is_none()
&& opt(&r.message, 256) && r.guests.len() <= 512
&& r.guests.iter().all(|g| vmid(g.vmid) && matches!(g.result.as_str(), "ok" | "error"))
});
if !clean(&p.node, 64) || !opt(&p.cluster, 64) || !opt(&p.version, 32)
|| !guests_ok || !jobs_ok || !runs_ok
|| p.not_backed_up.len() > 512 || !p.not_backed_up.iter().all(|id| vmid(*id))
{
return Err(err);
}
Ok(())
}
/// Status per job and guest plus the overall answer "did every scheduled backup run?".
/// job: ok | failed | missed | running | pending (not yet due) | disabled | unknown
pub fn backup_status(data: &Value) -> Option<Value> {
let p: Proxmox = serde_json::from_value(data.clone()).ok()?;
let mut last_success: BTreeMap<u32, i64> = BTreeMap::new();
for run in &p.backup_runs {
for g in run.guests.iter().filter(|g| g.result == "ok") {
let at = run.ended_at.unwrap_or(run.started_at);
last_success.entry(g.vmid).and_modify(|t| *t = (*t).max(at)).or_insert(at);
}
}
let mut jobs = Vec::new();
let mut all_ran = true;
let mut problems = 0usize;
for job in &p.backup_jobs {
let mut guests = Vec::new();
let status = if !job.enabled {
"disabled"
} else if job.schedule.is_none() {
"unknown"
} else if let Some(due) = job.last_expected_run {
let since = due - START_TOLERANCE_SECONDS;
let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect();
let running = window.iter().any(|r| r.status == "running");
let mut states = BTreeSet::new();
for id in &job.covered_vmids {
// Newest statement about this guest inside the due window wins.
let result = window.iter()
.filter_map(|r| r.guests.iter().find(|g| g.vmid == *id).map(|g| (r.started_at, g.result.as_str())))
.max_by_key(|(at, _)| *at).map(|(_, r)| r);
let state = match (result, running) {
(Some("ok"), _) => "ok",
(Some(_), _) => "failed",
(None, true) => "running",
(None, false) => "missed",
};
states.insert(state);
guests.push(json!({"vmid":id,"status":state,"last_success_at":last_success.get(id)}));
}
if states.contains("failed") { "failed" }
else if states.contains("missed") { "missed" }
else if states.contains("running") { "running" }
else { "ok" }
} else {
"pending"
};
if matches!(status, "failed" | "missed" | "unknown") {
all_ran = false;
problems += 1;
}
jobs.push(json!({"id":job.id,"status":status,"last_expected_run":job.last_expected_run,
"next_run":job.next_run,"guests":guests}));
}
let enabled = p.backup_jobs.iter().filter(|j| j.enabled).count();
Some(json!({
"all_scheduled_backups_ran": if enabled == 0 { Value::Null } else { Value::Bool(all_ran) },
"enabled_jobs": enabled,
"problem_jobs": problems,
"jobs": jobs,
"last_success_at": last_success,
}))
}
#[cfg(test)]
mod tests {
use super::*;
fn sample() -> Value {
json!({"node":"pve","cluster":null,"version":"9.2.2",
"guests":[
{"vmid":100,"name":"debian","kind":"qemu","status":"running","template":false,"cpus":1,
"memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,
"macs":["bc:24:11:88:35:97"],"smbios_uuid":"787f4ae9-446f-4fac-b3d8-4074170bf823"},
{"vmid":101,"name":"alpine","kind":"lxc","status":"running","template":false,"cpus":1,
"memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,"macs":[],"smbios_uuid":null}],
"backup_jobs":[
{"id":"a","enabled":true,"schedule":"*/2:00","storage":"local","mode":"snapshot",
"selection":"vmid","covered_vmids":[100],"next_run":2000,"last_expected_run":1000},
{"id":"b","enabled":true,"schedule":"daily","storage":"local","mode":"snapshot",
"selection":"all","covered_vmids":[100,101],"next_run":5000,"last_expected_run":900},
{"id":"c","enabled":false,"schedule":"daily","storage":null,"mode":null,
"selection":"all","covered_vmids":[101],"next_run":null,"last_expected_run":null},
{"id":"d","enabled":true,"schedule":"sat 03:00","storage":null,"mode":null,
"selection":"vmid","covered_vmids":[101],"next_run":9000,"last_expected_run":null}],
"backup_runs":[
{"upid":"UPID:pve:1:2:3:vzdump::root@pam:","started_at":1001,"ended_at":1050,"status":"error",
"message":"job errors","guests":[{"vmid":100,"result":"ok"},{"vmid":101,"result":"error"}]},
{"upid":"UPID:pve:1:2:4:vzdump:100:root@pam:","started_at":100,"ended_at":120,"status":"ok",
"message":null,"guests":[{"vmid":100,"result":"ok"}]}],
"not_backed_up":[]})
}
#[test]
fn accepts_agent_shape_and_rejects_unsafe_values() {
assert!(validate(&sample()).is_ok());
for (pointer, bad) in [
("/guests/0/macs/0", json!("BC:24:11:88:35:97")),
("/guests/0/kind", json!("docker")),
("/guests/0/vmid", json!(5)),
("/guests/0/smbios_uuid", json!("not-a-uuid")),
("/backup_runs/0/status", json!("running")),
("/backup_jobs/0/selection", json!("tag")),
("/node", json!("pve\nX")),
] {
let mut doc = sample();
*doc.pointer_mut(pointer).unwrap() = bad;
assert!(validate(&doc).is_err(), "{pointer}");
}
let mut doc = sample();
doc["extra"] = json!(1);
assert!(validate(&doc).is_err());
}
#[test]
fn evaluates_due_runs_per_job_and_guest() {
let status = backup_status(&sample()).unwrap();
let jobs: BTreeMap<String, Value> = status["jobs"].as_array().unwrap().iter()
.map(|j| (j["id"].as_str().unwrap().to_owned(), j.clone())).collect();
assert_eq!(jobs["a"]["status"], "ok");
assert_eq!(jobs["b"]["status"], "failed");
assert_eq!(jobs["b"]["guests"][1]["status"], "failed");
assert_eq!(jobs["c"]["status"], "disabled");
assert_eq!(jobs["d"]["status"], "pending");
assert_eq!(status["all_scheduled_backups_ran"], false);
assert_eq!(status["last_success_at"]["100"], 1050);
}
#[test]
fn missing_and_running_runs_are_distinguished() {
let mut doc = sample();
doc["backup_runs"] = json!([]);
let status = backup_status(&doc).unwrap();
assert_eq!(status["jobs"][0]["status"], "missed");
doc["backup_runs"] = json!([{"upid":"UPID:pve:9:9:9:vzdump::root@pam:","started_at":1000,
"ended_at":null,"status":"running","message":null,"guests":[]}]);
assert_eq!(backup_status(&doc).unwrap()["jobs"][0]["status"], "running");
doc["backup_jobs"] = json!([]);
assert!(backup_status(&doc).unwrap()["all_scheduled_backups_ran"].is_null());
}
}
+3
View File
@@ -190,6 +190,9 @@ pub async fn ingest(
.fetch_one(&mut *tx) .fetch_one(&mut *tx)
.await?; .await?;
ensure!(stored == hash, "message identity conflict"); ensure!(stored == hash, "message identity conflict");
if matches!(s.scope.as_str(), "linux.base" | "windows.base") {
crate::proxmox_links::reconcile(&mut tx, tenant).await;
}
} }
Some(Payload::JobAccepted(_) | Payload::JobStateChanged(_) | Payload::JobResult(_)) => { Some(Payload::JobAccepted(_) | Payload::JobStateChanged(_) | Payload::JobResult(_)) => {
crate::job_gateway::event(&mut tx, tenant, agent, e).await? crate::job_gateway::event(&mut tx, tenant, agent, e).await?
+23 -9
View File
@@ -208,6 +208,9 @@ struct HypervisorAssignment {
// null clears the relation. A missing field is a malformed update. // null clears the relation. A missing field is a malformed update.
#[serde(deserialize_with = "required_hypervisor_value")] #[serde(deserialize_with = "required_hypervisor_value")]
hypervisor_asset_id: Option<Uuid>, hypervisor_asset_id: Option<Uuid>,
/// true hands the decision back to automatic Proxmox detection (requires null id).
#[serde(default)]
automatic: bool,
} }
fn required_hypervisor_value<'de, D: serde::Deserializer<'de>>(value: D) -> Result<Option<Uuid>, D::Error> { fn required_hypervisor_value<'de, D: serde::Deserializer<'de>>(value: D) -> Result<Option<Uuid>, D::Error> {
@@ -227,24 +230,35 @@ async fn assign_hypervisor(
// Match the asset lock order for simultaneous edits of the same two devices. // Match the asset lock order for simultaneous edits of the same two devices.
let ids = [asset, input.hypervisor_asset_id.unwrap_or(asset)]; let ids = [asset, input.hypervisor_asset_id.unwrap_or(asset)];
let rows: Vec<(Uuid, Option<Uuid>)> = sqlx::query_as( if input.automatic && input.hypervisor_asset_id.is_some() {
"SELECT asset_id,hypervisor_asset_id FROM assets return Err(Error(StatusCode::UNPROCESSABLE_ENTITY, "AUTOMATIC_REQUIRES_NULL_HYPERVISOR"));
}
let rows: Vec<(Uuid, Option<Uuid>, Option<String>)> = sqlx::query_as(
"SELECT asset_id,hypervisor_asset_id,hypervisor_source FROM assets
WHERE tenant_id=$1 AND asset_id=ANY($2) ORDER BY asset_id FOR UPDATE", WHERE tenant_id=$1 AND asset_id=ANY($2) ORDER BY asset_id FOR UPDATE",
) )
.bind(tenant).bind(ids.as_slice()).fetch_all(&mut *tx).await?; .bind(tenant).bind(ids.as_slice()).fetch_all(&mut *tx).await?;
let previous = rows.into_iter().find(|(id, _)| *id == asset) let (_, previous, previous_source) = rows.into_iter().find(|(id, _, _)| *id == asset)
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?.1; .ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
crate::devices::validate_vm_host_assignment( crate::devices::validate_vm_host_assignment(
&mut tx, tenant, asset, pair.agent_id, input.hypervisor_asset_id, &mut tx, tenant, asset, pair.agent_id, input.hypervisor_asset_id,
).await?; ).await?;
if previous != input.hypervisor_asset_id { let source = (!input.automatic).then_some("manual");
sqlx::query("UPDATE assets SET hypervisor_asset_id=$3 WHERE tenant_id=$1 AND asset_id=$2") if previous != input.hypervisor_asset_id || previous_source.as_deref() != source {
.bind(tenant).bind(asset).bind(input.hypervisor_asset_id).execute(&mut *tx).await?; sqlx::query("UPDATE assets SET hypervisor_asset_id=$3,hypervisor_source=$4 WHERE tenant_id=$1 AND asset_id=$2")
.bind(tenant).bind(asset).bind(input.hypervisor_asset_id).bind(source).execute(&mut *tx).await?;
audit(&mut tx, &p, EVENT, json!({"asset_id":asset,"agent_id":pair.agent_id, audit(&mut tx, &p, EVENT, json!({"asset_id":asset,"agent_id":pair.agent_id,
"previous_hypervisor_asset_id":previous,"hypervisor_asset_id":input.hypervisor_asset_id})).await?; "previous_hypervisor_asset_id":previous,"hypervisor_asset_id":input.hypervisor_asset_id,
"previous_source":previous_source,"source":source.unwrap_or("automatic")})).await?;
} }
if input.automatic {
crate::proxmox_links::reconcile(&mut tx, tenant).await;
}
let (current, current_source): (Option<Uuid>, Option<String>) = sqlx::query_as(
"SELECT hypervisor_asset_id,hypervisor_source FROM assets WHERE tenant_id=$1 AND asset_id=$2")
.bind(tenant).bind(asset).fetch_one(&mut *tx).await?;
tx.commit().await?; tx.commit().await?;
Ok(Json(json!({"asset_id":asset,"hypervisor_asset_id":input.hypervisor_asset_id}))) Ok(Json(json!({"asset_id":asset,"hypervisor_asset_id":current,"hypervisor_source":current_source})))
} }
async fn remove_device( async fn remove_device(
+1 -1
View File
@@ -3,7 +3,7 @@ WITH classified AS (
a.last_authenticated_at,a.last_authenticated_certificate_sha256,a.certificate_sha256, a.last_authenticated_at,a.last_authenticated_certificate_sha256,a.certificate_sha256,
a.enrollment_generation,a.assignment_version,a.capabilities,a.reported_agent_version, a.enrollment_generation,a.assignment_version,a.capabilities,a.reported_agent_version,
a.system_usage,a.system_usage_observed_at, a.system_usage,a.system_usage_observed_at,
asset.archived_at,asset.hypervisor_asset_id,asset.purpose,t.name AS customer_name, asset.archived_at,asset.hypervisor_asset_id,asset.hypervisor_source,asset.purpose,t.name AS customer_name,
CASE CASE
WHEN a.revoked_at IS NOT NULL THEN 'revoked' WHEN a.revoked_at IS NOT NULL THEN 'revoked'
WHEN asset.archived_at IS NOT NULL THEN 'archived' WHEN asset.archived_at IS NOT NULL THEN 'archived'
+134
View File
@@ -261,6 +261,12 @@ pub async fn detail(
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?; .ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
let hosts = assigned_hosts(&mut tx, tenant, now, std::slice::from_ref(&row)).await?; let hosts = assigned_hosts(&mut tx, tenant, now, std::slice::from_ref(&row)).await?;
let mut item = project(&row, now, true, &hosts); let mut item = project(&row, now, true, &hosts);
if item["proxmox"]["data"].is_object() {
let rows: Vec<(Uuid, Uuid, Option<String>, Option<String>, Value)> =
sqlx::query_as(include_str!("proxmox_guest_devices.sql"))
.bind(tenant).bind(asset).fetch_all(&mut *tx).await?;
item["proxmox"]["guest_devices"] = proxmox_guest_devices(&item["proxmox"]["data"], &rows);
}
item["recent_jobs"] = if p.permits_scope("operations:read") item["recent_jobs"] = if p.permits_scope("operations:read")
&& crate::permissions::allowed(&mut tx, &p.name, "operations:read").await? && crate::permissions::allowed(&mut tx, &p.name, "operations:read").await?
{ {
@@ -282,6 +288,25 @@ pub async fn detail(
Ok(Json(item)) Ok(Json(item))
} }
/// Linked devices per reported guest, matched by the MACs both sides report.
fn proxmox_guest_devices(
data: &Value,
rows: &[(Uuid, Uuid, Option<String>, Option<String>, Value)],
) -> Value {
let mut result = Vec::new();
for guest in data["guests"].as_array().into_iter().flatten() {
let macs: Vec<&str> = guest["macs"].as_array().into_iter().flatten()
.filter_map(Value::as_str).collect();
let linked = rows.iter().find(|(_, _, _, _, device)| device.as_array().into_iter().flatten()
.filter_map(Value::as_str).any(|m| macs.contains(&m)));
if let (Some(vmid), Some((asset_id, agent_id, source, hostname, _))) = (guest["vmid"].as_u64(), linked) {
result.push(json!({"vmid":vmid,"asset_id":asset_id,"agent_id":agent_id,
"hostname":hostname,"source":source}));
}
}
Value::Array(result)
}
fn observed<'a>(data: &'a Value, scope: &str) -> &'a Value { fn observed<'a>(data: &'a Value, scope: &str) -> &'a Value {
data["observations"] data["observations"]
.as_array() .as_array()
@@ -569,12 +594,33 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool, hosts: &HashMap<Uuid, Ho
} else { Value::Null }; } else { Value::Null };
item["disks_truncated"] = json!(scope["truncated"] == true); item["disks_truncated"] = json!(scope["truncated"] == true);
} }
if let Some(scope) = data["observations"].as_array().and_then(|scopes|
scopes.iter().find(|s| s["scope"] == "linux.proxmox")) {
let status = (scope["quality"] == "valid")
.then(|| rmm_domain::inventory_document::proxmox::backup_status(&scope["data"]))
.flatten();
item["proxmox_backup"] = status.as_ref().map_or(Value::Null, |s| json!({
"all_scheduled_backups_ran":s["all_scheduled_backups_ran"],
"problem_jobs":s["problem_jobs"],"enabled_jobs":s["enabled_jobs"]}));
if full {
item["proxmox"] = json!({
"quality":scope["quality"],"error_code":scope["error_code"],
"observed_at":scope["observed_at"],"truncated":scope["truncated"],
"data":if scope["quality"] == "valid" { scope["data"].clone() } else { Value::Null },
"backup_status":status,"guest_devices":[]});
}
}
if device_type == Some(DeviceType::Virtual) && row.get::<String, _>("lifecycle") == "current" { if device_type == Some(DeviceType::Virtual) && row.get::<String, _>("lifecycle") == "current" {
let assigned_id: Option<Uuid> = row.get("hypervisor_asset_id"); let assigned_id: Option<Uuid> = row.get("hypervisor_asset_id");
let assigned = assigned_id.and_then(|id| hosts.get(&id)); let assigned = assigned_id.and_then(|id| hosts.get(&id));
item["hypervisor"] = hypervisor_display(&data, assigned); item["hypervisor"] = hypervisor_display(&data, assigned);
let source = row.get::<Option<String>, _>("hypervisor_source");
if item["hypervisor"]["kind"] == "device" {
item["hypervisor"]["source"] = json!(source);
}
if full { if full {
item["assigned_hypervisor_asset_id"] = json!(assigned_id); item["assigned_hypervisor_asset_id"] = json!(assigned_id);
item["hypervisor_source"] = json!(source);
} }
} }
if full { if full {
@@ -668,6 +714,94 @@ mod tests {
assert!(hypervisor_display(&data, None).is_null()); assert!(hypervisor_display(&data, None).is_null());
} }
#[tokio::test]
#[ignore = "requires the disposable RLS database from tests/local_access_database.py"]
async fn proxmox_guest_macs_link_vms_automatically_but_never_override_people() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await.unwrap();
let (tenant, other) = (Uuid::new_v4(), Uuid::new_v4());
let (host, vm, manual_vm, foreign_vm, twin_a, twin_b) =
(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
for customer in [tenant, other] {
sqlx::query("INSERT INTO tenants(tenant_id,name) VALUES($1,'Proxmox link test')")
.bind(customer).execute(&admin).await.unwrap();
}
let guests = |macs: &[&str]| json!({"node":"pve","guests":macs.iter().enumerate().map(|(i, m)|
json!({"vmid":100 + i,"macs":[m]})).collect::<Vec<_>>()});
let device = |kind: &str, mac: &str, proxmox: Option<Value>| {
let mut scopes = vec![
json!({"scope":"linux.hardware","quality":"valid","data":{"device_type":kind}}),
json!({"scope":"linux.interfaces","quality":"valid","data":[{"name":"eth0","mac":mac}]}),
];
if let Some(data) = proxmox {
scopes.push(json!({"scope":"linux.proxmox","quality":"valid","data":data}));
}
json!({"hostname":"h","os":"Debian","root_available_bytes":1,"root_total_bytes":2,"observations":scopes})
};
let insert = |customer: Uuid, id: Uuid, data: Value, sequence: i64| {
let admin = admin.clone();
async move {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),'linux.base','valid','partial',$6,$7,$8)")
.bind(customer).bind(id).bind(Uuid::new_v4()).bind(Uuid::new_v4()).bind(sequence)
.bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).execute(&admin).await.unwrap();
}
};
let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04"];
for (customer, id, data) in [
(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all)))),
(tenant, vm, device("virtual", "BC:24:11:00:00:01", None)),
(tenant, manual_vm, device("virtual", "bc:24:11:00:00:02", None)),
(other, foreign_vm, device("virtual", "bc:24:11:00:00:03", None)),
(tenant, twin_a, device("virtual", "bc:24:11:00:00:04", None)),
(tenant, twin_b, device("virtual", "bc:24:11:00:00:04", None)),
] {
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
.bind(customer).bind(id).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,asset_id,agent_id,certificate_sha256,last_contact_at,last_authenticated_at,last_authenticated_certificate_sha256) VALUES($1,$2,$2,$3,now(),now(),$3)")
.bind(customer).bind(id).bind(rmm_domain::jobs::hash(format!("{customer}:{id}").as_bytes()))
.execute(&admin).await.unwrap();
insert(customer, id, data, 1).await;
}
sqlx::query("UPDATE assets SET hypervisor_source='manual' WHERE asset_id=$1")
.bind(manual_vm).execute(&admin).await.unwrap();
let link = |id: Uuid| {
let admin = admin.clone();
async move {
sqlx::query_as::<_, (Option<Uuid>, Option<String>)>(
"SELECT hypervisor_asset_id,hypervisor_source FROM assets WHERE asset_id=$1")
.bind(id).fetch_one(&admin).await.unwrap()
}
};
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
crate::proxmox_links::reconcile(&mut tx, tenant).await;
tx.commit().await.unwrap();
assert_eq!(link(vm).await, (Some(host), Some("proxmox".into())));
assert_eq!(link(manual_vm).await, (None, Some("manual".into())));
assert_eq!(link(foreign_vm).await, (None, None));
assert_eq!(link(twin_a).await, (None, None), "duplicate MACs are ambiguous");
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='device_hypervisor_detected'")
.bind(tenant).fetch_one(&admin).await.unwrap();
assert_eq!(audits, 1);
// Idempotent: a second pass changes nothing.
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
crate::proxmox_links::reconcile(&mut tx, tenant).await;
tx.commit().await.unwrap();
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
.bind(tenant).fetch_one(&admin).await.unwrap();
assert_eq!(audits, 1);
// The guest left the host: the automatic link is released.
insert(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all[1..]))), 2).await;
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
crate::proxmox_links::reconcile(&mut tx, tenant).await;
tx.commit().await.unwrap();
assert_eq!(link(vm).await, (None, None));
}
#[tokio::test] #[tokio::test]
#[ignore = "requires the disposable RLS database from tests/local_access_database.py"] #[ignore = "requires the disposable RLS database from tests/local_access_database.py"]
async fn host_links_resolve_outside_the_page_and_stay_inside_the_customer() { async fn host_links_resolve_outside_the_page_and_stay_inside_the_customer() {
+1
View File
@@ -27,6 +27,7 @@ mod monitoring;
mod oidc; mod oidc;
mod pagination; mod pagination;
mod permissions; mod permissions;
mod proxmox_links;
mod root_policy; mod root_policy;
mod managed_scripts; mod managed_scripts;
mod scheduler; mod scheduler;
@@ -0,0 +1,28 @@
-- Current devices of customer $1 linked to host $2, with the MACs they report themselves.
SELECT asset.asset_id, inst.agent_id, asset.hypervisor_source,
inventory.data->>'hostname' AS hostname,
COALESCE((
SELECT jsonb_agg(DISTINCT lower(i->>'mac'))
FROM jsonb_array_elements(CASE WHEN jsonb_typeof(inventory.data->'observations')='array'
THEN inventory.data->'observations' ELSE '[]'::jsonb END) o,
jsonb_array_elements(CASE
WHEN jsonb_typeof(o->'data')='array' THEN o->'data'
WHEN jsonb_typeof(o#>'{data,entries}')='array' THEN o#>'{data,entries}'
ELSE '[]'::jsonb END) i
WHERE o->>'scope' IN ('linux.interfaces','windows.interfaces') AND o->>'quality'='valid'
AND i->>'mac' IS NOT NULL
), '[]'::jsonb) AS macs
FROM assets asset
JOIN LATERAL (
SELECT a.agent_id, a.revoked_at FROM agent_installations a
WHERE a.tenant_id=asset.tenant_id AND a.asset_id=asset.asset_id AND NOT a.customer_transfer_pending
ORDER BY a.enrollment_generation DESC, a.created_at DESC, a.agent_id LIMIT 1
) inst ON inst.revoked_at IS NULL
LEFT JOIN LATERAL (
SELECT data FROM observations
WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base')
ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1
) inventory ON true
WHERE asset.tenant_id=$1 AND asset.hypervisor_asset_id=$2 AND asset.archived_at IS NULL
ORDER BY asset.asset_id
LIMIT 1024
+24
View File
@@ -0,0 +1,24 @@
//! Automatic VM -> Proxmox host mapping from the hosts' `linux.proxmox` guest lists.
//! Runs inside the ingesting customer transaction behind a savepoint: a failed
//! reconciliation never blocks acknowledging the inventory snapshot itself.
use sqlx::{Acquire, Postgres, Transaction};
use uuid::Uuid;
pub async fn reconcile(tx: &mut Transaction<'_, Postgres>, tenant: Uuid) {
let result = async {
let mut savepoint = tx.begin().await?;
let events: Vec<Uuid> = sqlx::query_scalar(include_str!("proxmox_links.sql"))
.bind(tenant)
.fetch_all(&mut *savepoint)
.await?;
savepoint.commit().await?;
Ok::<_, sqlx::Error>(events.len())
}
.await;
match result {
Ok(0) => (),
Ok(changed) => eprintln!("proxmox guest links updated: {changed}"),
// Database detail stays out of the log line; the snapshot is still acknowledged.
Err(_) => eprintln!("proxmox guest link reconciliation skipped"),
}
}
+103
View File
@@ -0,0 +1,103 @@
-- Automatic VM -> Proxmox host links for one customer ($1). A link is set only for a
-- unique MAC match between a host's current guest list and a current virtual device of
-- the same customer. Manual decisions (hypervisor_source='manual') are never touched.
WITH current_installation AS (
SELECT asset.asset_id, asset.hypervisor_asset_id, asset.hypervisor_source, inst.agent_id
FROM assets asset
JOIN LATERAL (
SELECT a.agent_id, a.revoked_at
FROM agent_installations a
WHERE a.tenant_id=asset.tenant_id AND a.asset_id=asset.asset_id
AND NOT a.customer_transfer_pending
AND a.last_authenticated_at IS NOT NULL
ORDER BY a.enrollment_generation DESC, a.created_at DESC, a.agent_id
LIMIT 1
) inst ON inst.revoked_at IS NULL
WHERE asset.tenant_id=$1 AND asset.archived_at IS NULL
),
latest AS (
SELECT c.*, inventory.data
FROM current_installation c
JOIN LATERAL (
SELECT data FROM observations
WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base')
ORDER BY observed_at DESC, received_at DESC, sequence DESC
LIMIT 1
) inventory ON true
),
scopes AS (
SELECT l.asset_id, o
FROM latest l,
jsonb_array_elements(CASE WHEN jsonb_typeof(l.data->'observations')='array'
THEN l.data->'observations' ELSE '[]'::jsonb END) o
WHERE o->>'quality'='valid'
),
proxmox_hosts AS (
SELECT asset_id, o->'data' AS data FROM scopes WHERE o->>'scope'='linux.proxmox'
),
guest_macs AS (
SELECT DISTINCT h.asset_id AS host_asset_id, lower(mac) AS mac
FROM proxmox_hosts h,
jsonb_array_elements(h.data->'guests') g,
jsonb_array_elements_text(g->'macs') mac
),
virtual_devices AS (
SELECT asset_id FROM scopes
WHERE o->>'scope' IN ('linux.hardware','windows.hardware') AND o#>>'{data,device_type}'='virtual'
),
device_macs AS (
SELECT DISTINCT s.asset_id, lower(i->>'mac') AS mac
FROM scopes s
JOIN virtual_devices v ON v.asset_id=s.asset_id,
jsonb_array_elements(CASE
WHEN jsonb_typeof(s.o->'data')='array' THEN s.o->'data'
WHEN jsonb_typeof(s.o#>'{data,entries}')='array' THEN s.o#>'{data,entries}'
ELSE '[]'::jsonb END) i
WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces')
AND i->>'mac' ~* '^[0-9a-f]{2}(:[0-9a-f]{2}){5}$'
AND lower(i->>'mac') <> '00:00:00:00:00:00'
),
-- A MAC that more than one device or more than one host claims proves nothing.
unique_macs AS (
SELECT d.asset_id, g.host_asset_id
FROM device_macs d JOIN guest_macs g ON g.mac=d.mac
WHERE d.asset_id <> g.host_asset_id
AND (SELECT count(*) FROM device_macs d2 WHERE d2.mac=d.mac)=1
AND (SELECT count(*) FROM guest_macs g2 WHERE g2.mac=d.mac)=1
),
matches AS (
SELECT asset_id, min(host_asset_id::text)::uuid AS host_asset_id
FROM unique_macs GROUP BY asset_id HAVING count(DISTINCT host_asset_id)=1
),
desired AS (
-- New or changed automatic links.
SELECT l.asset_id, l.hypervisor_asset_id AS previous, m.host_asset_id AS next
FROM latest l JOIN matches m ON m.asset_id=l.asset_id
WHERE l.hypervisor_source IS DISTINCT FROM 'manual'
AND (l.hypervisor_asset_id IS DISTINCT FROM m.host_asset_id OR l.hypervisor_source IS NULL)
UNION ALL
-- The assigned host reports a valid guest list that no longer contains this device.
SELECT l.asset_id, l.hypervisor_asset_id, NULL
FROM latest l
WHERE l.hypervisor_source='proxmox'
AND NOT EXISTS (SELECT FROM matches m WHERE m.asset_id=l.asset_id)
AND EXISTS (SELECT FROM proxmox_hosts h WHERE h.asset_id=l.hypervisor_asset_id)
AND EXISTS (SELECT FROM device_macs d WHERE d.asset_id=l.asset_id)
),
changed AS (
UPDATE assets a
SET hypervisor_asset_id=d.next,
hypervisor_source=CASE WHEN d.next IS NULL THEN NULL ELSE 'proxmox' END
FROM desired d
WHERE a.tenant_id=$1 AND a.asset_id=d.asset_id
RETURNING a.asset_id, d.previous, d.next
)
INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data,created_at)
SELECT $1, gen_random_uuid(), 'system:proxmox-guest-detection',
CASE WHEN next IS NULL THEN 'device_hypervisor_released' ELSE 'device_hypervisor_detected' END,
jsonb_build_object('asset_id',asset_id,'previous_hypervisor_asset_id',previous,
'hypervisor_asset_id',next,'match','mac'),
now()
FROM changed
WHERE previous IS DISTINCT FROM next
RETURNING event_id
+10
View File
@@ -0,0 +1,10 @@
SET LOCAL lock_timeout = '3s';
SET LOCAL statement_timeout = '30s';
-- NULL: no automatic decision yet, 'proxmox': set from a Proxmox host's guest list,
-- 'manual': chosen by a person and never overwritten by automatic detection.
ALTER TABLE assets
ADD COLUMN hypervisor_source text,
ADD CONSTRAINT assets_hypervisor_source_check
CHECK (hypervisor_source IN ('manual','proxmox'));
UPDATE assets SET hypervisor_source='manual' WHERE hypervisor_asset_id IS NOT NULL;
GRANT UPDATE (hypervisor_source) ON assets TO rmm_app;