Link virtual OPNsense firewalls to their Proxmox host automatically
The MAC matching only read linux.base and windows.base snapshots and ran only after those. OPNsense sends a flat opnsense.base snapshot (hardware.device_type, interfaces.entries); it is now normalised into the same scopes, triggers the reconciliation and appears in the host's guest list. Manual decisions still win. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -190,7 +190,7 @@ pub async fn ingest(
|
|||||||
.fetch_one(&mut *tx)
|
.fetch_one(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
ensure!(stored == hash, "message identity conflict");
|
ensure!(stored == hash, "message identity conflict");
|
||||||
if matches!(s.scope.as_str(), "linux.base" | "windows.base") {
|
if matches!(s.scope.as_str(), "linux.base" | "windows.base" | "opnsense.base") {
|
||||||
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -724,6 +724,7 @@ mod tests {
|
|||||||
let (tenant, other) = (Uuid::new_v4(), Uuid::new_v4());
|
let (tenant, other) = (Uuid::new_v4(), Uuid::new_v4());
|
||||||
let (host, vm, manual_vm, foreign_vm, twin_a, twin_b) =
|
let (host, vm, manual_vm, foreign_vm, twin_a, twin_b) =
|
||||||
(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
|
(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
|
||||||
|
let (fw, foreign_fw, physical_fw) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
|
||||||
for customer in [tenant, other] {
|
for customer in [tenant, other] {
|
||||||
sqlx::query("INSERT INTO tenants(tenant_id,name) VALUES($1,'Proxmox link test')")
|
sqlx::query("INSERT INTO tenants(tenant_id,name) VALUES($1,'Proxmox link test')")
|
||||||
.bind(customer).execute(&admin).await.unwrap();
|
.bind(customer).execute(&admin).await.unwrap();
|
||||||
@@ -740,15 +741,22 @@ mod tests {
|
|||||||
}
|
}
|
||||||
json!({"hostname":"h","os":"Debian","root_available_bytes":1,"root_total_bytes":2,"observations":scopes})
|
json!({"hostname":"h","os":"Debian","root_available_bytes":1,"root_total_bytes":2,"observations":scopes})
|
||||||
};
|
};
|
||||||
|
// OPNsense snapshots are flat; MACs come from `interfaces.entries`.
|
||||||
|
let firewall = |kind: &str, macs: &[&str]| json!({"hostname":"fw","hardware":{"device_type":kind},
|
||||||
|
"interfaces":{"status":"ok","entries":macs.iter().enumerate().map(|(i, m)|
|
||||||
|
json!({"name":format!("vtnet{i}"),"mac":m})).chain([json!({"name":"lo0","mac":null})])
|
||||||
|
.collect::<Vec<_>>()}});
|
||||||
let insert = |customer: Uuid, id: Uuid, data: Value, sequence: i64| {
|
let insert = |customer: Uuid, id: Uuid, data: Value, sequence: i64| {
|
||||||
let admin = admin.clone();
|
let admin = admin.clone();
|
||||||
async move {
|
async move {
|
||||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),'linux.base','valid','partial',$6,$7,$8)")
|
let source = if data.get("interfaces").is_some() { "opnsense.base" } else { "linux.base" };
|
||||||
|
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),$9,'valid','partial',$6,$7,$8)")
|
||||||
.bind(customer).bind(id).bind(Uuid::new_v4()).bind(Uuid::new_v4()).bind(sequence)
|
.bind(customer).bind(id).bind(Uuid::new_v4()).bind(Uuid::new_v4()).bind(sequence)
|
||||||
.bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).execute(&admin).await.unwrap();
|
.bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).bind(source).execute(&admin).await.unwrap();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04"];
|
let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04",
|
||||||
|
"bc:24:11:00:00:05", "bc:24:11:00:00:06", "bc:24:11:00:00:07"];
|
||||||
for (customer, id, data) in [
|
for (customer, id, data) in [
|
||||||
(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all)))),
|
(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all)))),
|
||||||
(tenant, vm, device("virtual", "BC:24:11:00:00:01", None)),
|
(tenant, vm, device("virtual", "BC:24:11:00:00:01", None)),
|
||||||
@@ -756,6 +764,9 @@ mod tests {
|
|||||||
(other, foreign_vm, device("virtual", "bc:24:11:00:00:03", None)),
|
(other, foreign_vm, device("virtual", "bc:24:11:00:00:03", None)),
|
||||||
(tenant, twin_a, device("virtual", "bc:24:11:00:00:04", None)),
|
(tenant, twin_a, device("virtual", "bc:24:11:00:00:04", None)),
|
||||||
(tenant, twin_b, device("virtual", "bc:24:11:00:00:04", None)),
|
(tenant, twin_b, device("virtual", "bc:24:11:00:00:04", None)),
|
||||||
|
(tenant, fw, firewall("virtual", &["BC:24:11:00:00:05", "02:00:00:00:00:01"])),
|
||||||
|
(other, foreign_fw, firewall("virtual", &["bc:24:11:00:00:06"])),
|
||||||
|
(tenant, physical_fw, firewall("physical", &["bc:24:11:00:00:07"])),
|
||||||
] {
|
] {
|
||||||
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
|
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
|
||||||
.bind(customer).bind(id).execute(&admin).await.unwrap();
|
.bind(customer).bind(id).execute(&admin).await.unwrap();
|
||||||
@@ -782,9 +793,12 @@ mod tests {
|
|||||||
assert_eq!(link(manual_vm).await, (None, Some("manual".into())));
|
assert_eq!(link(manual_vm).await, (None, Some("manual".into())));
|
||||||
assert_eq!(link(foreign_vm).await, (None, None));
|
assert_eq!(link(foreign_vm).await, (None, None));
|
||||||
assert_eq!(link(twin_a).await, (None, None), "duplicate MACs are ambiguous");
|
assert_eq!(link(twin_a).await, (None, None), "duplicate MACs are ambiguous");
|
||||||
|
assert_eq!(link(fw).await, (Some(host), Some("proxmox".into())), "virtual OPNsense guest");
|
||||||
|
assert_eq!(link(foreign_fw).await, (None, None));
|
||||||
|
assert_eq!(link(physical_fw).await, (None, None), "only virtual devices are guests");
|
||||||
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='device_hypervisor_detected'")
|
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='device_hypervisor_detected'")
|
||||||
.bind(tenant).fetch_one(&admin).await.unwrap();
|
.bind(tenant).fetch_one(&admin).await.unwrap();
|
||||||
assert_eq!(audits, 1);
|
assert_eq!(audits, 2);
|
||||||
|
|
||||||
// Idempotent: a second pass changes nothing.
|
// Idempotent: a second pass changes nothing.
|
||||||
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
||||||
@@ -792,7 +806,7 @@ mod tests {
|
|||||||
tx.commit().await.unwrap();
|
tx.commit().await.unwrap();
|
||||||
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
|
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
|
||||||
.bind(tenant).fetch_one(&admin).await.unwrap();
|
.bind(tenant).fetch_one(&admin).await.unwrap();
|
||||||
assert_eq!(audits, 1);
|
assert_eq!(audits, 2);
|
||||||
|
|
||||||
// The guest left the host: the automatic link is released.
|
// The guest left the host: the automatic link is released.
|
||||||
insert(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all[1..]))), 2).await;
|
insert(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all[1..]))), 2).await;
|
||||||
|
|||||||
@@ -11,6 +11,13 @@ SELECT asset.asset_id, inst.agent_id, asset.hypervisor_source,
|
|||||||
ELSE '[]'::jsonb END) i
|
ELSE '[]'::jsonb END) i
|
||||||
WHERE o->>'scope' IN ('linux.interfaces','windows.interfaces') AND o->>'quality'='valid'
|
WHERE o->>'scope' IN ('linux.interfaces','windows.interfaces') AND o->>'quality'='valid'
|
||||||
AND i->>'mac' IS NOT NULL
|
AND i->>'mac' IS NOT NULL
|
||||||
|
), (
|
||||||
|
-- OPNsense snapshots are flat.
|
||||||
|
SELECT jsonb_agg(DISTINCT lower(i->>'mac'))
|
||||||
|
FROM jsonb_array_elements(CASE WHEN inventory.data#>>'{interfaces,status}'='ok'
|
||||||
|
AND jsonb_typeof(inventory.data#>'{interfaces,entries}')='array'
|
||||||
|
THEN inventory.data#>'{interfaces,entries}' ELSE '[]'::jsonb END) i
|
||||||
|
WHERE i->>'mac' IS NOT NULL
|
||||||
), '[]'::jsonb) AS macs
|
), '[]'::jsonb) AS macs
|
||||||
FROM assets asset
|
FROM assets asset
|
||||||
JOIN LATERAL (
|
JOIN LATERAL (
|
||||||
@@ -20,7 +27,7 @@ JOIN LATERAL (
|
|||||||
) inst ON inst.revoked_at IS NULL
|
) inst ON inst.revoked_at IS NULL
|
||||||
LEFT JOIN LATERAL (
|
LEFT JOIN LATERAL (
|
||||||
SELECT data FROM observations
|
SELECT data FROM observations
|
||||||
WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base')
|
WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base','opnsense.base')
|
||||||
ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1
|
ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1
|
||||||
) inventory ON true
|
) inventory ON true
|
||||||
WHERE asset.tenant_id=$1 AND asset.hypervisor_asset_id=$2 AND asset.archived_at IS NULL
|
WHERE asset.tenant_id=$1 AND asset.hypervisor_asset_id=$2 AND asset.archived_at IS NULL
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
//! Automatic VM -> Proxmox host mapping from the hosts' `linux.proxmox` guest lists.
|
//! Automatic VM -> Proxmox host mapping from the hosts' `linux.proxmox` guest lists.
|
||||||
|
//! Guests are virtual Linux, Windows and OPNsense devices, matched by interface MAC.
|
||||||
//! Runs inside the ingesting customer transaction behind a savepoint: a failed
|
//! Runs inside the ingesting customer transaction behind a savepoint: a failed
|
||||||
//! reconciliation never blocks acknowledging the inventory snapshot itself.
|
//! reconciliation never blocks acknowledging the inventory snapshot itself.
|
||||||
use sqlx::{Acquire, Postgres, Transaction};
|
use sqlx::{Acquire, Postgres, Transaction};
|
||||||
|
|||||||
@@ -16,11 +16,11 @@ WITH current_installation AS (
|
|||||||
WHERE asset.tenant_id=$1 AND asset.archived_at IS NULL
|
WHERE asset.tenant_id=$1 AND asset.archived_at IS NULL
|
||||||
),
|
),
|
||||||
latest AS (
|
latest AS (
|
||||||
SELECT c.*, inventory.data
|
SELECT c.*, inventory.source_id, inventory.data
|
||||||
FROM current_installation c
|
FROM current_installation c
|
||||||
JOIN LATERAL (
|
JOIN LATERAL (
|
||||||
SELECT data FROM observations
|
SELECT source_id, data FROM observations
|
||||||
WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base')
|
WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base','opnsense.base')
|
||||||
ORDER BY observed_at DESC, received_at DESC, sequence DESC
|
ORDER BY observed_at DESC, received_at DESC, sequence DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
) inventory ON true
|
) inventory ON true
|
||||||
@@ -31,6 +31,15 @@ scopes AS (
|
|||||||
jsonb_array_elements(CASE WHEN jsonb_typeof(l.data->'observations')='array'
|
jsonb_array_elements(CASE WHEN jsonb_typeof(l.data->'observations')='array'
|
||||||
THEN l.data->'observations' ELSE '[]'::jsonb END) o
|
THEN l.data->'observations' ELSE '[]'::jsonb END) o
|
||||||
WHERE o->>'quality'='valid'
|
WHERE o->>'quality'='valid'
|
||||||
|
UNION ALL
|
||||||
|
-- An OPNsense snapshot is flat: hardware and interfaces sit at the top level.
|
||||||
|
SELECT l.asset_id, jsonb_build_object('scope','opnsense.hardware','quality','valid','data',l.data->'hardware')
|
||||||
|
FROM latest l
|
||||||
|
WHERE l.source_id='opnsense.base' AND jsonb_typeof(l.data->'hardware')='object'
|
||||||
|
UNION ALL
|
||||||
|
SELECT l.asset_id, jsonb_build_object('scope','opnsense.interfaces','quality','valid','data',l.data->'interfaces')
|
||||||
|
FROM latest l
|
||||||
|
WHERE l.source_id='opnsense.base' AND l.data#>>'{interfaces,status}'='ok'
|
||||||
),
|
),
|
||||||
proxmox_hosts AS (
|
proxmox_hosts AS (
|
||||||
SELECT asset_id, o->'data' AS data FROM scopes WHERE o->>'scope'='linux.proxmox'
|
SELECT asset_id, o->'data' AS data FROM scopes WHERE o->>'scope'='linux.proxmox'
|
||||||
@@ -43,7 +52,8 @@ guest_macs AS (
|
|||||||
),
|
),
|
||||||
virtual_devices AS (
|
virtual_devices AS (
|
||||||
SELECT asset_id FROM scopes
|
SELECT asset_id FROM scopes
|
||||||
WHERE o->>'scope' IN ('linux.hardware','windows.hardware') AND o#>>'{data,device_type}'='virtual'
|
WHERE o->>'scope' IN ('linux.hardware','windows.hardware','opnsense.hardware')
|
||||||
|
AND o#>>'{data,device_type}'='virtual'
|
||||||
),
|
),
|
||||||
device_macs AS (
|
device_macs AS (
|
||||||
SELECT DISTINCT s.asset_id, lower(i->>'mac') AS mac
|
SELECT DISTINCT s.asset_id, lower(i->>'mac') AS mac
|
||||||
@@ -53,7 +63,7 @@ device_macs AS (
|
|||||||
WHEN jsonb_typeof(s.o->'data')='array' THEN s.o->'data'
|
WHEN jsonb_typeof(s.o->'data')='array' THEN s.o->'data'
|
||||||
WHEN jsonb_typeof(s.o#>'{data,entries}')='array' THEN s.o#>'{data,entries}'
|
WHEN jsonb_typeof(s.o#>'{data,entries}')='array' THEN s.o#>'{data,entries}'
|
||||||
ELSE '[]'::jsonb END) i
|
ELSE '[]'::jsonb END) i
|
||||||
WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces')
|
WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces','opnsense.interfaces')
|
||||||
AND i->>'mac' ~* '^[0-9a-f]{2}(:[0-9a-f]{2}){5}$'
|
AND i->>'mac' ~* '^[0-9a-f]{2}(:[0-9a-f]{2}){5}$'
|
||||||
AND lower(i->>'mac') <> '00:00:00:00:00:00'
|
AND lower(i->>'mac') <> '00:00:00:00:00:00'
|
||||||
),
|
),
|
||||||
|
|||||||
Reference in New Issue
Block a user