From f3a471c1c51c0e0f4120b4365d29b8d086fcae37 Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Sat, 21 Nov 2026 13:05:36 +0100 Subject: [PATCH] Link virtual OPNsense firewalls to their Proxmox host automatically The MAC matching only read linux.base and windows.base snapshots and ran only after those. OPNsense sends a flat opnsense.base snapshot (hardware.device_type, interfaces.entries); it is now normalised into the same scopes, triggers the reconciliation and appears in the host's guest list. Manual decisions still win. Co-Authored-By: Claude Opus 5.5 --- crates/server/src/agent_service.rs | 2 +- crates/server/src/devices.rs | 24 ++++++++++++++++----- crates/server/src/proxmox_guest_devices.sql | 9 +++++++- crates/server/src/proxmox_links.rs | 1 + crates/server/src/proxmox_links.sql | 20 ++++++++++++----- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/crates/server/src/agent_service.rs b/crates/server/src/agent_service.rs index 14ad1f9..88643ee 100644 --- a/crates/server/src/agent_service.rs +++ b/crates/server/src/agent_service.rs @@ -190,7 +190,7 @@ pub async fn ingest( .fetch_one(&mut *tx) .await?; ensure!(stored == hash, "message identity conflict"); - if matches!(s.scope.as_str(), "linux.base" | "windows.base") { + if matches!(s.scope.as_str(), "linux.base" | "windows.base" | "opnsense.base") { crate::proxmox_links::reconcile(&mut tx, tenant).await; } } diff --git a/crates/server/src/devices.rs b/crates/server/src/devices.rs index a4bd665..7c8e85a 100644 --- a/crates/server/src/devices.rs +++ b/crates/server/src/devices.rs @@ -724,6 +724,7 @@ mod tests { let (tenant, other) = (Uuid::new_v4(), Uuid::new_v4()); let (host, vm, manual_vm, foreign_vm, twin_a, twin_b) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + let (fw, foreign_fw, physical_fw) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); for customer in [tenant, other] { sqlx::query("INSERT INTO tenants(tenant_id,name) VALUES($1,'Proxmox link test')") .bind(customer).execute(&admin).await.unwrap(); @@ -740,15 +741,22 @@ mod tests { } json!({"hostname":"h","os":"Debian","root_available_bytes":1,"root_total_bytes":2,"observations":scopes}) }; + // OPNsense snapshots are flat; MACs come from `interfaces.entries`. + let firewall = |kind: &str, macs: &[&str]| json!({"hostname":"fw","hardware":{"device_type":kind}, + "interfaces":{"status":"ok","entries":macs.iter().enumerate().map(|(i, m)| + json!({"name":format!("vtnet{i}"),"mac":m})).chain([json!({"name":"lo0","mac":null})]) + .collect::>()}}); let insert = |customer: Uuid, id: Uuid, data: Value, sequence: i64| { let admin = admin.clone(); async move { - sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),'linux.base','valid','partial',$6,$7,$8)") + let source = if data.get("interfaces").is_some() { "opnsense.base" } else { "linux.base" }; + sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),$9,'valid','partial',$6,$7,$8)") .bind(customer).bind(id).bind(Uuid::new_v4()).bind(Uuid::new_v4()).bind(sequence) - .bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).execute(&admin).await.unwrap(); + .bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).bind(source).execute(&admin).await.unwrap(); } }; - let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04"]; + let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04", + "bc:24:11:00:00:05", "bc:24:11:00:00:06", "bc:24:11:00:00:07"]; for (customer, id, data) in [ (tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all)))), (tenant, vm, device("virtual", "BC:24:11:00:00:01", None)), @@ -756,6 +764,9 @@ mod tests { (other, foreign_vm, device("virtual", "bc:24:11:00:00:03", None)), (tenant, twin_a, device("virtual", "bc:24:11:00:00:04", None)), (tenant, twin_b, device("virtual", "bc:24:11:00:00:04", None)), + (tenant, fw, firewall("virtual", &["BC:24:11:00:00:05", "02:00:00:00:00:01"])), + (other, foreign_fw, firewall("virtual", &["bc:24:11:00:00:06"])), + (tenant, physical_fw, firewall("physical", &["bc:24:11:00:00:07"])), ] { sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)") .bind(customer).bind(id).execute(&admin).await.unwrap(); @@ -782,9 +793,12 @@ mod tests { assert_eq!(link(manual_vm).await, (None, Some("manual".into()))); assert_eq!(link(foreign_vm).await, (None, None)); assert_eq!(link(twin_a).await, (None, None), "duplicate MACs are ambiguous"); + assert_eq!(link(fw).await, (Some(host), Some("proxmox".into())), "virtual OPNsense guest"); + assert_eq!(link(foreign_fw).await, (None, None)); + assert_eq!(link(physical_fw).await, (None, None), "only virtual devices are guests"); let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='device_hypervisor_detected'") .bind(tenant).fetch_one(&admin).await.unwrap(); - assert_eq!(audits, 1); + assert_eq!(audits, 2); // Idempotent: a second pass changes nothing. let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); @@ -792,7 +806,7 @@ mod tests { tx.commit().await.unwrap(); let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1") .bind(tenant).fetch_one(&admin).await.unwrap(); - assert_eq!(audits, 1); + assert_eq!(audits, 2); // The guest left the host: the automatic link is released. insert(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all[1..]))), 2).await; diff --git a/crates/server/src/proxmox_guest_devices.sql b/crates/server/src/proxmox_guest_devices.sql index 3f285b0..5b2971e 100644 --- a/crates/server/src/proxmox_guest_devices.sql +++ b/crates/server/src/proxmox_guest_devices.sql @@ -11,6 +11,13 @@ SELECT asset.asset_id, inst.agent_id, asset.hypervisor_source, ELSE '[]'::jsonb END) i WHERE o->>'scope' IN ('linux.interfaces','windows.interfaces') AND o->>'quality'='valid' AND i->>'mac' IS NOT NULL + ), ( + -- OPNsense snapshots are flat. + SELECT jsonb_agg(DISTINCT lower(i->>'mac')) + FROM jsonb_array_elements(CASE WHEN inventory.data#>>'{interfaces,status}'='ok' + AND jsonb_typeof(inventory.data#>'{interfaces,entries}')='array' + THEN inventory.data#>'{interfaces,entries}' ELSE '[]'::jsonb END) i + WHERE i->>'mac' IS NOT NULL ), '[]'::jsonb) AS macs FROM assets asset JOIN LATERAL ( @@ -20,7 +27,7 @@ JOIN LATERAL ( ) inst ON inst.revoked_at IS NULL LEFT JOIN LATERAL ( SELECT data FROM observations - WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base') + WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base','opnsense.base') ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1 ) inventory ON true WHERE asset.tenant_id=$1 AND asset.hypervisor_asset_id=$2 AND asset.archived_at IS NULL diff --git a/crates/server/src/proxmox_links.rs b/crates/server/src/proxmox_links.rs index 8ace19c..15effa7 100644 --- a/crates/server/src/proxmox_links.rs +++ b/crates/server/src/proxmox_links.rs @@ -1,4 +1,5 @@ //! Automatic VM -> Proxmox host mapping from the hosts' `linux.proxmox` guest lists. +//! Guests are virtual Linux, Windows and OPNsense devices, matched by interface MAC. //! Runs inside the ingesting customer transaction behind a savepoint: a failed //! reconciliation never blocks acknowledging the inventory snapshot itself. use sqlx::{Acquire, Postgres, Transaction}; diff --git a/crates/server/src/proxmox_links.sql b/crates/server/src/proxmox_links.sql index dc18790..0a10111 100644 --- a/crates/server/src/proxmox_links.sql +++ b/crates/server/src/proxmox_links.sql @@ -16,11 +16,11 @@ WITH current_installation AS ( WHERE asset.tenant_id=$1 AND asset.archived_at IS NULL ), latest AS ( - SELECT c.*, inventory.data + SELECT c.*, inventory.source_id, inventory.data FROM current_installation c JOIN LATERAL ( - SELECT data FROM observations - WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base') + SELECT source_id, data FROM observations + WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base','opnsense.base') ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1 ) inventory ON true @@ -31,6 +31,15 @@ scopes AS ( jsonb_array_elements(CASE WHEN jsonb_typeof(l.data->'observations')='array' THEN l.data->'observations' ELSE '[]'::jsonb END) o WHERE o->>'quality'='valid' + UNION ALL + -- An OPNsense snapshot is flat: hardware and interfaces sit at the top level. + SELECT l.asset_id, jsonb_build_object('scope','opnsense.hardware','quality','valid','data',l.data->'hardware') + FROM latest l + WHERE l.source_id='opnsense.base' AND jsonb_typeof(l.data->'hardware')='object' + UNION ALL + SELECT l.asset_id, jsonb_build_object('scope','opnsense.interfaces','quality','valid','data',l.data->'interfaces') + FROM latest l + WHERE l.source_id='opnsense.base' AND l.data#>>'{interfaces,status}'='ok' ), proxmox_hosts AS ( SELECT asset_id, o->'data' AS data FROM scopes WHERE o->>'scope'='linux.proxmox' @@ -43,7 +52,8 @@ guest_macs AS ( ), virtual_devices AS ( SELECT asset_id FROM scopes - WHERE o->>'scope' IN ('linux.hardware','windows.hardware') AND o#>>'{data,device_type}'='virtual' + WHERE o->>'scope' IN ('linux.hardware','windows.hardware','opnsense.hardware') + AND o#>>'{data,device_type}'='virtual' ), device_macs AS ( SELECT DISTINCT s.asset_id, lower(i->>'mac') AS mac @@ -53,7 +63,7 @@ device_macs AS ( WHEN jsonb_typeof(s.o->'data')='array' THEN s.o->'data' WHEN jsonb_typeof(s.o#>'{data,entries}')='array' THEN s.o#>'{data,entries}' ELSE '[]'::jsonb END) i - WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces') + WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces','opnsense.interfaces') AND i->>'mac' ~* '^[0-9a-f]{2}(:[0-9a-f]{2}){5}$' AND lower(i->>'mac') <> '00:00:00:00:00:00' ),