Validate Proxmox observations, evaluate scheduled backups and link VMs to their Proxmox host
Rust development slice / linux (push) Failing after 15s

linux.proxmox gets a strict validator and a 128 KiB scope budget. Device
details expose guests, backup jobs with ok/failed/missed/running status and
linked devices. Base snapshots reconcile unique guest MACs into
assets.hypervisor_asset_id with hypervisor_source='proxmox'; manual choices
are kept and can be handed back to automatic detection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-20 12:10:35 +01:00
co-authored by Claude Opus 5.5
parent 1d2d52847c
commit 70977df577
12 changed files with 597 additions and 10 deletions
+3
View File
@@ -190,6 +190,9 @@ pub async fn ingest(
.fetch_one(&mut *tx)
.await?;
ensure!(stored == hash, "message identity conflict");
if matches!(s.scope.as_str(), "linux.base" | "windows.base") {
crate::proxmox_links::reconcile(&mut tx, tenant).await;
}
}
Some(Payload::JobAccepted(_) | Payload::JobStateChanged(_) | Payload::JobResult(_)) => {
crate::job_gateway::event(&mut tx, tenant, agent, e).await?
+23 -9
View File
@@ -208,6 +208,9 @@ struct HypervisorAssignment {
// null clears the relation. A missing field is a malformed update.
#[serde(deserialize_with = "required_hypervisor_value")]
hypervisor_asset_id: Option<Uuid>,
/// true hands the decision back to automatic Proxmox detection (requires null id).
#[serde(default)]
automatic: bool,
}
fn required_hypervisor_value<'de, D: serde::Deserializer<'de>>(value: D) -> Result<Option<Uuid>, D::Error> {
@@ -227,24 +230,35 @@ async fn assign_hypervisor(
// Match the asset lock order for simultaneous edits of the same two devices.
let ids = [asset, input.hypervisor_asset_id.unwrap_or(asset)];
let rows: Vec<(Uuid, Option<Uuid>)> = sqlx::query_as(
"SELECT asset_id,hypervisor_asset_id FROM assets
if input.automatic && input.hypervisor_asset_id.is_some() {
return Err(Error(StatusCode::UNPROCESSABLE_ENTITY, "AUTOMATIC_REQUIRES_NULL_HYPERVISOR"));
}
let rows: Vec<(Uuid, Option<Uuid>, Option<String>)> = sqlx::query_as(
"SELECT asset_id,hypervisor_asset_id,hypervisor_source FROM assets
WHERE tenant_id=$1 AND asset_id=ANY($2) ORDER BY asset_id FOR UPDATE",
)
.bind(tenant).bind(ids.as_slice()).fetch_all(&mut *tx).await?;
let previous = rows.into_iter().find(|(id, _)| *id == asset)
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?.1;
let (_, previous, previous_source) = rows.into_iter().find(|(id, _, _)| *id == asset)
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
crate::devices::validate_vm_host_assignment(
&mut tx, tenant, asset, pair.agent_id, input.hypervisor_asset_id,
).await?;
if previous != input.hypervisor_asset_id {
sqlx::query("UPDATE assets SET hypervisor_asset_id=$3 WHERE tenant_id=$1 AND asset_id=$2")
.bind(tenant).bind(asset).bind(input.hypervisor_asset_id).execute(&mut *tx).await?;
let source = (!input.automatic).then_some("manual");
if previous != input.hypervisor_asset_id || previous_source.as_deref() != source {
sqlx::query("UPDATE assets SET hypervisor_asset_id=$3,hypervisor_source=$4 WHERE tenant_id=$1 AND asset_id=$2")
.bind(tenant).bind(asset).bind(input.hypervisor_asset_id).bind(source).execute(&mut *tx).await?;
audit(&mut tx, &p, EVENT, json!({"asset_id":asset,"agent_id":pair.agent_id,
"previous_hypervisor_asset_id":previous,"hypervisor_asset_id":input.hypervisor_asset_id})).await?;
"previous_hypervisor_asset_id":previous,"hypervisor_asset_id":input.hypervisor_asset_id,
"previous_source":previous_source,"source":source.unwrap_or("automatic")})).await?;
}
if input.automatic {
crate::proxmox_links::reconcile(&mut tx, tenant).await;
}
let (current, current_source): (Option<Uuid>, Option<String>) = sqlx::query_as(
"SELECT hypervisor_asset_id,hypervisor_source FROM assets WHERE tenant_id=$1 AND asset_id=$2")
.bind(tenant).bind(asset).fetch_one(&mut *tx).await?;
tx.commit().await?;
Ok(Json(json!({"asset_id":asset,"hypervisor_asset_id":input.hypervisor_asset_id})))
Ok(Json(json!({"asset_id":asset,"hypervisor_asset_id":current,"hypervisor_source":current_source})))
}
async fn remove_device(
+1 -1
View File
@@ -3,7 +3,7 @@ WITH classified AS (
a.last_authenticated_at,a.last_authenticated_certificate_sha256,a.certificate_sha256,
a.enrollment_generation,a.assignment_version,a.capabilities,a.reported_agent_version,
a.system_usage,a.system_usage_observed_at,
asset.archived_at,asset.hypervisor_asset_id,asset.purpose,t.name AS customer_name,
asset.archived_at,asset.hypervisor_asset_id,asset.hypervisor_source,asset.purpose,t.name AS customer_name,
CASE
WHEN a.revoked_at IS NOT NULL THEN 'revoked'
WHEN asset.archived_at IS NOT NULL THEN 'archived'
+134
View File
@@ -261,6 +261,12 @@ pub async fn detail(
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
let hosts = assigned_hosts(&mut tx, tenant, now, std::slice::from_ref(&row)).await?;
let mut item = project(&row, now, true, &hosts);
if item["proxmox"]["data"].is_object() {
let rows: Vec<(Uuid, Uuid, Option<String>, Option<String>, Value)> =
sqlx::query_as(include_str!("proxmox_guest_devices.sql"))
.bind(tenant).bind(asset).fetch_all(&mut *tx).await?;
item["proxmox"]["guest_devices"] = proxmox_guest_devices(&item["proxmox"]["data"], &rows);
}
item["recent_jobs"] = if p.permits_scope("operations:read")
&& crate::permissions::allowed(&mut tx, &p.name, "operations:read").await?
{
@@ -282,6 +288,25 @@ pub async fn detail(
Ok(Json(item))
}
/// Linked devices per reported guest, matched by the MACs both sides report.
fn proxmox_guest_devices(
data: &Value,
rows: &[(Uuid, Uuid, Option<String>, Option<String>, Value)],
) -> Value {
let mut result = Vec::new();
for guest in data["guests"].as_array().into_iter().flatten() {
let macs: Vec<&str> = guest["macs"].as_array().into_iter().flatten()
.filter_map(Value::as_str).collect();
let linked = rows.iter().find(|(_, _, _, _, device)| device.as_array().into_iter().flatten()
.filter_map(Value::as_str).any(|m| macs.contains(&m)));
if let (Some(vmid), Some((asset_id, agent_id, source, hostname, _))) = (guest["vmid"].as_u64(), linked) {
result.push(json!({"vmid":vmid,"asset_id":asset_id,"agent_id":agent_id,
"hostname":hostname,"source":source}));
}
}
Value::Array(result)
}
fn observed<'a>(data: &'a Value, scope: &str) -> &'a Value {
data["observations"]
.as_array()
@@ -569,12 +594,33 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool, hosts: &HashMap<Uuid, Ho
} else { Value::Null };
item["disks_truncated"] = json!(scope["truncated"] == true);
}
if let Some(scope) = data["observations"].as_array().and_then(|scopes|
scopes.iter().find(|s| s["scope"] == "linux.proxmox")) {
let status = (scope["quality"] == "valid")
.then(|| rmm_domain::inventory_document::proxmox::backup_status(&scope["data"]))
.flatten();
item["proxmox_backup"] = status.as_ref().map_or(Value::Null, |s| json!({
"all_scheduled_backups_ran":s["all_scheduled_backups_ran"],
"problem_jobs":s["problem_jobs"],"enabled_jobs":s["enabled_jobs"]}));
if full {
item["proxmox"] = json!({
"quality":scope["quality"],"error_code":scope["error_code"],
"observed_at":scope["observed_at"],"truncated":scope["truncated"],
"data":if scope["quality"] == "valid" { scope["data"].clone() } else { Value::Null },
"backup_status":status,"guest_devices":[]});
}
}
if device_type == Some(DeviceType::Virtual) && row.get::<String, _>("lifecycle") == "current" {
let assigned_id: Option<Uuid> = row.get("hypervisor_asset_id");
let assigned = assigned_id.and_then(|id| hosts.get(&id));
item["hypervisor"] = hypervisor_display(&data, assigned);
let source = row.get::<Option<String>, _>("hypervisor_source");
if item["hypervisor"]["kind"] == "device" {
item["hypervisor"]["source"] = json!(source);
}
if full {
item["assigned_hypervisor_asset_id"] = json!(assigned_id);
item["hypervisor_source"] = json!(source);
}
}
if full {
@@ -668,6 +714,94 @@ mod tests {
assert!(hypervisor_display(&data, None).is_null());
}
#[tokio::test]
#[ignore = "requires the disposable RLS database from tests/local_access_database.py"]
async fn proxmox_guest_macs_link_vms_automatically_but_never_override_people() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await.unwrap();
let (tenant, other) = (Uuid::new_v4(), Uuid::new_v4());
let (host, vm, manual_vm, foreign_vm, twin_a, twin_b) =
(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
for customer in [tenant, other] {
sqlx::query("INSERT INTO tenants(tenant_id,name) VALUES($1,'Proxmox link test')")
.bind(customer).execute(&admin).await.unwrap();
}
let guests = |macs: &[&str]| json!({"node":"pve","guests":macs.iter().enumerate().map(|(i, m)|
json!({"vmid":100 + i,"macs":[m]})).collect::<Vec<_>>()});
let device = |kind: &str, mac: &str, proxmox: Option<Value>| {
let mut scopes = vec![
json!({"scope":"linux.hardware","quality":"valid","data":{"device_type":kind}}),
json!({"scope":"linux.interfaces","quality":"valid","data":[{"name":"eth0","mac":mac}]}),
];
if let Some(data) = proxmox {
scopes.push(json!({"scope":"linux.proxmox","quality":"valid","data":data}));
}
json!({"hostname":"h","os":"Debian","root_available_bytes":1,"root_total_bytes":2,"observations":scopes})
};
let insert = |customer: Uuid, id: Uuid, data: Value, sequence: i64| {
let admin = admin.clone();
async move {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),'linux.base','valid','partial',$6,$7,$8)")
.bind(customer).bind(id).bind(Uuid::new_v4()).bind(Uuid::new_v4()).bind(sequence)
.bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).execute(&admin).await.unwrap();
}
};
let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04"];
for (customer, id, data) in [
(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all)))),
(tenant, vm, device("virtual", "BC:24:11:00:00:01", None)),
(tenant, manual_vm, device("virtual", "bc:24:11:00:00:02", None)),
(other, foreign_vm, device("virtual", "bc:24:11:00:00:03", None)),
(tenant, twin_a, device("virtual", "bc:24:11:00:00:04", None)),
(tenant, twin_b, device("virtual", "bc:24:11:00:00:04", None)),
] {
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
.bind(customer).bind(id).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,asset_id,agent_id,certificate_sha256,last_contact_at,last_authenticated_at,last_authenticated_certificate_sha256) VALUES($1,$2,$2,$3,now(),now(),$3)")
.bind(customer).bind(id).bind(rmm_domain::jobs::hash(format!("{customer}:{id}").as_bytes()))
.execute(&admin).await.unwrap();
insert(customer, id, data, 1).await;
}
sqlx::query("UPDATE assets SET hypervisor_source='manual' WHERE asset_id=$1")
.bind(manual_vm).execute(&admin).await.unwrap();
let link = |id: Uuid| {
let admin = admin.clone();
async move {
sqlx::query_as::<_, (Option<Uuid>, Option<String>)>(
"SELECT hypervisor_asset_id,hypervisor_source FROM assets WHERE asset_id=$1")
.bind(id).fetch_one(&admin).await.unwrap()
}
};
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
crate::proxmox_links::reconcile(&mut tx, tenant).await;
tx.commit().await.unwrap();
assert_eq!(link(vm).await, (Some(host), Some("proxmox".into())));
assert_eq!(link(manual_vm).await, (None, Some("manual".into())));
assert_eq!(link(foreign_vm).await, (None, None));
assert_eq!(link(twin_a).await, (None, None), "duplicate MACs are ambiguous");
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='device_hypervisor_detected'")
.bind(tenant).fetch_one(&admin).await.unwrap();
assert_eq!(audits, 1);
// Idempotent: a second pass changes nothing.
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
crate::proxmox_links::reconcile(&mut tx, tenant).await;
tx.commit().await.unwrap();
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
.bind(tenant).fetch_one(&admin).await.unwrap();
assert_eq!(audits, 1);
// The guest left the host: the automatic link is released.
insert(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all[1..]))), 2).await;
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
crate::proxmox_links::reconcile(&mut tx, tenant).await;
tx.commit().await.unwrap();
assert_eq!(link(vm).await, (None, None));
}
#[tokio::test]
#[ignore = "requires the disposable RLS database from tests/local_access_database.py"]
async fn host_links_resolve_outside_the_page_and_stay_inside_the_customer() {
+1
View File
@@ -27,6 +27,7 @@ mod monitoring;
mod oidc;
mod pagination;
mod permissions;
mod proxmox_links;
mod root_policy;
mod managed_scripts;
mod scheduler;
@@ -0,0 +1,28 @@
-- Current devices of customer $1 linked to host $2, with the MACs they report themselves.
SELECT asset.asset_id, inst.agent_id, asset.hypervisor_source,
inventory.data->>'hostname' AS hostname,
COALESCE((
SELECT jsonb_agg(DISTINCT lower(i->>'mac'))
FROM jsonb_array_elements(CASE WHEN jsonb_typeof(inventory.data->'observations')='array'
THEN inventory.data->'observations' ELSE '[]'::jsonb END) o,
jsonb_array_elements(CASE
WHEN jsonb_typeof(o->'data')='array' THEN o->'data'
WHEN jsonb_typeof(o#>'{data,entries}')='array' THEN o#>'{data,entries}'
ELSE '[]'::jsonb END) i
WHERE o->>'scope' IN ('linux.interfaces','windows.interfaces') AND o->>'quality'='valid'
AND i->>'mac' IS NOT NULL
), '[]'::jsonb) AS macs
FROM assets asset
JOIN LATERAL (
SELECT a.agent_id, a.revoked_at FROM agent_installations a
WHERE a.tenant_id=asset.tenant_id AND a.asset_id=asset.asset_id AND NOT a.customer_transfer_pending
ORDER BY a.enrollment_generation DESC, a.created_at DESC, a.agent_id LIMIT 1
) inst ON inst.revoked_at IS NULL
LEFT JOIN LATERAL (
SELECT data FROM observations
WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base')
ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1
) inventory ON true
WHERE asset.tenant_id=$1 AND asset.hypervisor_asset_id=$2 AND asset.archived_at IS NULL
ORDER BY asset.asset_id
LIMIT 1024
+24
View File
@@ -0,0 +1,24 @@
//! Automatic VM -> Proxmox host mapping from the hosts' `linux.proxmox` guest lists.
//! Runs inside the ingesting customer transaction behind a savepoint: a failed
//! reconciliation never blocks acknowledging the inventory snapshot itself.
use sqlx::{Acquire, Postgres, Transaction};
use uuid::Uuid;
pub async fn reconcile(tx: &mut Transaction<'_, Postgres>, tenant: Uuid) {
let result = async {
let mut savepoint = tx.begin().await?;
let events: Vec<Uuid> = sqlx::query_scalar(include_str!("proxmox_links.sql"))
.bind(tenant)
.fetch_all(&mut *savepoint)
.await?;
savepoint.commit().await?;
Ok::<_, sqlx::Error>(events.len())
}
.await;
match result {
Ok(0) => (),
Ok(changed) => eprintln!("proxmox guest links updated: {changed}"),
// Database detail stays out of the log line; the snapshot is still acknowledged.
Err(_) => eprintln!("proxmox guest link reconciliation skipped"),
}
}
+103
View File
@@ -0,0 +1,103 @@
-- Automatic VM -> Proxmox host links for one customer ($1). A link is set only for a
-- unique MAC match between a host's current guest list and a current virtual device of
-- the same customer. Manual decisions (hypervisor_source='manual') are never touched.
WITH current_installation AS (
SELECT asset.asset_id, asset.hypervisor_asset_id, asset.hypervisor_source, inst.agent_id
FROM assets asset
JOIN LATERAL (
SELECT a.agent_id, a.revoked_at
FROM agent_installations a
WHERE a.tenant_id=asset.tenant_id AND a.asset_id=asset.asset_id
AND NOT a.customer_transfer_pending
AND a.last_authenticated_at IS NOT NULL
ORDER BY a.enrollment_generation DESC, a.created_at DESC, a.agent_id
LIMIT 1
) inst ON inst.revoked_at IS NULL
WHERE asset.tenant_id=$1 AND asset.archived_at IS NULL
),
latest AS (
SELECT c.*, inventory.data
FROM current_installation c
JOIN LATERAL (
SELECT data FROM observations
WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base')
ORDER BY observed_at DESC, received_at DESC, sequence DESC
LIMIT 1
) inventory ON true
),
scopes AS (
SELECT l.asset_id, o
FROM latest l,
jsonb_array_elements(CASE WHEN jsonb_typeof(l.data->'observations')='array'
THEN l.data->'observations' ELSE '[]'::jsonb END) o
WHERE o->>'quality'='valid'
),
proxmox_hosts AS (
SELECT asset_id, o->'data' AS data FROM scopes WHERE o->>'scope'='linux.proxmox'
),
guest_macs AS (
SELECT DISTINCT h.asset_id AS host_asset_id, lower(mac) AS mac
FROM proxmox_hosts h,
jsonb_array_elements(h.data->'guests') g,
jsonb_array_elements_text(g->'macs') mac
),
virtual_devices AS (
SELECT asset_id FROM scopes
WHERE o->>'scope' IN ('linux.hardware','windows.hardware') AND o#>>'{data,device_type}'='virtual'
),
device_macs AS (
SELECT DISTINCT s.asset_id, lower(i->>'mac') AS mac
FROM scopes s
JOIN virtual_devices v ON v.asset_id=s.asset_id,
jsonb_array_elements(CASE
WHEN jsonb_typeof(s.o->'data')='array' THEN s.o->'data'
WHEN jsonb_typeof(s.o#>'{data,entries}')='array' THEN s.o#>'{data,entries}'
ELSE '[]'::jsonb END) i
WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces')
AND i->>'mac' ~* '^[0-9a-f]{2}(:[0-9a-f]{2}){5}$'
AND lower(i->>'mac') <> '00:00:00:00:00:00'
),
-- A MAC that more than one device or more than one host claims proves nothing.
unique_macs AS (
SELECT d.asset_id, g.host_asset_id
FROM device_macs d JOIN guest_macs g ON g.mac=d.mac
WHERE d.asset_id <> g.host_asset_id
AND (SELECT count(*) FROM device_macs d2 WHERE d2.mac=d.mac)=1
AND (SELECT count(*) FROM guest_macs g2 WHERE g2.mac=d.mac)=1
),
matches AS (
SELECT asset_id, min(host_asset_id::text)::uuid AS host_asset_id
FROM unique_macs GROUP BY asset_id HAVING count(DISTINCT host_asset_id)=1
),
desired AS (
-- New or changed automatic links.
SELECT l.asset_id, l.hypervisor_asset_id AS previous, m.host_asset_id AS next
FROM latest l JOIN matches m ON m.asset_id=l.asset_id
WHERE l.hypervisor_source IS DISTINCT FROM 'manual'
AND (l.hypervisor_asset_id IS DISTINCT FROM m.host_asset_id OR l.hypervisor_source IS NULL)
UNION ALL
-- The assigned host reports a valid guest list that no longer contains this device.
SELECT l.asset_id, l.hypervisor_asset_id, NULL
FROM latest l
WHERE l.hypervisor_source='proxmox'
AND NOT EXISTS (SELECT FROM matches m WHERE m.asset_id=l.asset_id)
AND EXISTS (SELECT FROM proxmox_hosts h WHERE h.asset_id=l.hypervisor_asset_id)
AND EXISTS (SELECT FROM device_macs d WHERE d.asset_id=l.asset_id)
),
changed AS (
UPDATE assets a
SET hypervisor_asset_id=d.next,
hypervisor_source=CASE WHEN d.next IS NULL THEN NULL ELSE 'proxmox' END
FROM desired d
WHERE a.tenant_id=$1 AND a.asset_id=d.asset_id
RETURNING a.asset_id, d.previous, d.next
)
INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data,created_at)
SELECT $1, gen_random_uuid(), 'system:proxmox-guest-detection',
CASE WHEN next IS NULL THEN 'device_hypervisor_released' ELSE 'device_hypervisor_detected' END,
jsonb_build_object('asset_id',asset_id,'previous_hypervisor_asset_id',previous,
'hypervisor_asset_id',next,'match','mac'),
now()
FROM changed
WHERE previous IS DISTINCT FROM next
RETURNING event_id