Validate Proxmox observations, evaluate scheduled backups and link VMs to their Proxmox host
Rust development slice / linux (push) Failing after 15s
Rust development slice / linux (push) Failing after 15s
linux.proxmox gets a strict validator and a 128 KiB scope budget. Device details expose guests, backup jobs with ok/failed/missed/running status and linked devices. Base snapshots reconcile unique guest MACs into assets.hypervisor_asset_id with hypervisor_source='proxmox'; manual choices are kept and can be handed back to automatic detection. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -526,6 +526,14 @@ mod tests {
|
||||
assert_eq!(data.backup_runs[2].guests, [GuestResult { vmid: 101, result: "ok" }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_document_passes_the_server_validator() {
|
||||
let observation = block(collect(&fixture(), NOW));
|
||||
let data = serde_json::to_value(observation.data.unwrap()).unwrap();
|
||||
rmm_domain::inventory_document::proxmox::validate(&data).unwrap();
|
||||
assert!(rmm_domain::inventory_document::proxmox::backup_status(&data).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_local_node_or_api_is_an_error_observation() {
|
||||
let observation = block(collect(&Fixture(HashMap::new()), NOW));
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Additive bounded passive metadata; the strict original Inventory stays intact.
|
||||
mod opnsense;
|
||||
pub mod proxmox;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -174,6 +175,7 @@ pub fn validate_for_scope(
|
||||
for item in observations {
|
||||
let max_bytes = match (scope, item["scope"].as_str()) {
|
||||
("linux.base", Some("linux.software")) => LINUX_SOFTWARE_OBSERVATION_MAX_BYTES,
|
||||
("linux.base", Some("linux.proxmox")) => PROXMOX_OBSERVATION_MAX_BYTES,
|
||||
("windows.base", Some("windows.software")) => SOFTWARE_OBSERVATION_MAX_BYTES,
|
||||
_ => 32768,
|
||||
};
|
||||
@@ -215,6 +217,9 @@ pub fn validate_for_scope(
|
||||
return Err("INVALID_AGENT_UPDATE_CATALOG_REPORT");
|
||||
}
|
||||
}
|
||||
if o.scope == "linux.proxmox" && o.quality == "valid" {
|
||||
proxmox::validate(&o.data)?;
|
||||
}
|
||||
if !o.scope.starts_with(prefix)
|
||||
|| o.scope.len() > 64
|
||||
|| !(o
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
//! Strict `linux.proxmox` wire document and the deterministic backup evaluation.
|
||||
//! The agent reports raw scheduler data; whether every scheduled backup ran is
|
||||
//! decided here so the server, API and UI share one rule.
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
/// Runs that start shortly before the computed due time still count (clock skew).
|
||||
const START_TOLERANCE_SECONDS: i64 = 120;
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Proxmox {
|
||||
pub node: String,
|
||||
pub cluster: Option<String>,
|
||||
pub version: Option<String>,
|
||||
pub guests: Vec<Guest>,
|
||||
pub backup_jobs: Vec<BackupJob>,
|
||||
pub backup_runs: Vec<BackupRun>,
|
||||
pub not_backed_up: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Guest {
|
||||
pub vmid: u32,
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub status: String,
|
||||
pub template: bool,
|
||||
pub cpus: Option<u32>,
|
||||
pub memory_bytes: Option<u64>,
|
||||
pub disk_bytes: Option<u64>,
|
||||
pub uptime_seconds: Option<u64>,
|
||||
pub pool: Option<String>,
|
||||
pub macs: Vec<String>,
|
||||
pub smbios_uuid: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackupJob {
|
||||
pub id: String,
|
||||
pub enabled: bool,
|
||||
pub schedule: Option<String>,
|
||||
pub storage: Option<String>,
|
||||
pub mode: Option<String>,
|
||||
pub selection: String,
|
||||
pub covered_vmids: Vec<u32>,
|
||||
pub next_run: Option<i64>,
|
||||
pub last_expected_run: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackupRun {
|
||||
pub upid: String,
|
||||
pub started_at: i64,
|
||||
pub ended_at: Option<i64>,
|
||||
pub status: String,
|
||||
pub message: Option<String>,
|
||||
pub guests: Vec<GuestResult>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GuestResult {
|
||||
pub vmid: u32,
|
||||
pub result: String,
|
||||
}
|
||||
|
||||
fn clean(s: &str, limit: usize) -> bool {
|
||||
!s.trim().is_empty() && s.len() <= limit && !s.chars().any(char::is_control)
|
||||
}
|
||||
fn opt(s: &Option<String>, limit: usize) -> bool {
|
||||
s.as_deref().is_none_or(|s| clean(s, limit))
|
||||
}
|
||||
pub fn is_mac(s: &str) -> bool {
|
||||
s.len() == 17 && s.split(':').count() == 6
|
||||
&& s.split(':').all(|p| p.len() == 2 && p.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
|
||||
}
|
||||
fn vmid(id: u32) -> bool {
|
||||
(100..1_000_000_000).contains(&id)
|
||||
}
|
||||
|
||||
/// Validates the `data` member of a valid `linux.proxmox` observation.
|
||||
pub fn validate(data: &Value) -> Result<(), &'static str> {
|
||||
let err = "INVALID_PROXMOX_OBSERVATION";
|
||||
let p: Proxmox = serde_json::from_value(data.clone()).map_err(|_| err)?;
|
||||
let guests_ok = p.guests.len() <= 512 && p.guests.iter().all(|g| {
|
||||
vmid(g.vmid) && clean(&g.name, 128) && matches!(g.kind.as_str(), "qemu" | "lxc")
|
||||
&& clean(&g.status, 32) && opt(&g.pool, 64) && g.macs.len() <= 32
|
||||
&& g.macs.iter().all(|m| is_mac(m))
|
||||
&& g.smbios_uuid.as_deref().is_none_or(|u| uuid::Uuid::parse_str(u).is_ok_and(|x| x.hyphenated().to_string() == u))
|
||||
}) && p.guests.windows(2).all(|w| w[0].vmid < w[1].vmid);
|
||||
let jobs_ok = p.backup_jobs.len() <= 64 && p.backup_jobs.iter().all(|j| {
|
||||
clean(&j.id, 64) && opt(&j.schedule, 128) && opt(&j.storage, 64) && opt(&j.mode, 16)
|
||||
&& matches!(j.selection.as_str(), "all" | "pool" | "vmid")
|
||||
&& j.covered_vmids.len() <= 512 && j.covered_vmids.iter().all(|id| vmid(*id))
|
||||
});
|
||||
let runs_ok = p.backup_runs.len() <= 64 && p.backup_runs.iter().all(|r| {
|
||||
clean(&r.upid, 160) && r.upid.starts_with("UPID:") && r.started_at > 0
|
||||
&& r.ended_at.is_none_or(|e| e >= r.started_at)
|
||||
&& matches!(r.status.as_str(), "ok" | "warning" | "error" | "running")
|
||||
&& (r.status == "running") == r.ended_at.is_none()
|
||||
&& opt(&r.message, 256) && r.guests.len() <= 512
|
||||
&& r.guests.iter().all(|g| vmid(g.vmid) && matches!(g.result.as_str(), "ok" | "error"))
|
||||
});
|
||||
if !clean(&p.node, 64) || !opt(&p.cluster, 64) || !opt(&p.version, 32)
|
||||
|| !guests_ok || !jobs_ok || !runs_ok
|
||||
|| p.not_backed_up.len() > 512 || !p.not_backed_up.iter().all(|id| vmid(*id))
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Status per job and guest plus the overall answer "did every scheduled backup run?".
|
||||
/// job: ok | failed | missed | running | pending (not yet due) | disabled | unknown
|
||||
pub fn backup_status(data: &Value) -> Option<Value> {
|
||||
let p: Proxmox = serde_json::from_value(data.clone()).ok()?;
|
||||
let mut last_success: BTreeMap<u32, i64> = BTreeMap::new();
|
||||
for run in &p.backup_runs {
|
||||
for g in run.guests.iter().filter(|g| g.result == "ok") {
|
||||
let at = run.ended_at.unwrap_or(run.started_at);
|
||||
last_success.entry(g.vmid).and_modify(|t| *t = (*t).max(at)).or_insert(at);
|
||||
}
|
||||
}
|
||||
let mut jobs = Vec::new();
|
||||
let mut all_ran = true;
|
||||
let mut problems = 0usize;
|
||||
for job in &p.backup_jobs {
|
||||
let mut guests = Vec::new();
|
||||
let status = if !job.enabled {
|
||||
"disabled"
|
||||
} else if job.schedule.is_none() {
|
||||
"unknown"
|
||||
} else if let Some(due) = job.last_expected_run {
|
||||
let since = due - START_TOLERANCE_SECONDS;
|
||||
let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect();
|
||||
let running = window.iter().any(|r| r.status == "running");
|
||||
let mut states = BTreeSet::new();
|
||||
for id in &job.covered_vmids {
|
||||
// Newest statement about this guest inside the due window wins.
|
||||
let result = window.iter()
|
||||
.filter_map(|r| r.guests.iter().find(|g| g.vmid == *id).map(|g| (r.started_at, g.result.as_str())))
|
||||
.max_by_key(|(at, _)| *at).map(|(_, r)| r);
|
||||
let state = match (result, running) {
|
||||
(Some("ok"), _) => "ok",
|
||||
(Some(_), _) => "failed",
|
||||
(None, true) => "running",
|
||||
(None, false) => "missed",
|
||||
};
|
||||
states.insert(state);
|
||||
guests.push(json!({"vmid":id,"status":state,"last_success_at":last_success.get(id)}));
|
||||
}
|
||||
if states.contains("failed") { "failed" }
|
||||
else if states.contains("missed") { "missed" }
|
||||
else if states.contains("running") { "running" }
|
||||
else { "ok" }
|
||||
} else {
|
||||
"pending"
|
||||
};
|
||||
if matches!(status, "failed" | "missed" | "unknown") {
|
||||
all_ran = false;
|
||||
problems += 1;
|
||||
}
|
||||
jobs.push(json!({"id":job.id,"status":status,"last_expected_run":job.last_expected_run,
|
||||
"next_run":job.next_run,"guests":guests}));
|
||||
}
|
||||
let enabled = p.backup_jobs.iter().filter(|j| j.enabled).count();
|
||||
Some(json!({
|
||||
"all_scheduled_backups_ran": if enabled == 0 { Value::Null } else { Value::Bool(all_ran) },
|
||||
"enabled_jobs": enabled,
|
||||
"problem_jobs": problems,
|
||||
"jobs": jobs,
|
||||
"last_success_at": last_success,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample() -> Value {
|
||||
json!({"node":"pve","cluster":null,"version":"9.2.2",
|
||||
"guests":[
|
||||
{"vmid":100,"name":"debian","kind":"qemu","status":"running","template":false,"cpus":1,
|
||||
"memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,
|
||||
"macs":["bc:24:11:88:35:97"],"smbios_uuid":"787f4ae9-446f-4fac-b3d8-4074170bf823"},
|
||||
{"vmid":101,"name":"alpine","kind":"lxc","status":"running","template":false,"cpus":1,
|
||||
"memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,"macs":[],"smbios_uuid":null}],
|
||||
"backup_jobs":[
|
||||
{"id":"a","enabled":true,"schedule":"*/2:00","storage":"local","mode":"snapshot",
|
||||
"selection":"vmid","covered_vmids":[100],"next_run":2000,"last_expected_run":1000},
|
||||
{"id":"b","enabled":true,"schedule":"daily","storage":"local","mode":"snapshot",
|
||||
"selection":"all","covered_vmids":[100,101],"next_run":5000,"last_expected_run":900},
|
||||
{"id":"c","enabled":false,"schedule":"daily","storage":null,"mode":null,
|
||||
"selection":"all","covered_vmids":[101],"next_run":null,"last_expected_run":null},
|
||||
{"id":"d","enabled":true,"schedule":"sat 03:00","storage":null,"mode":null,
|
||||
"selection":"vmid","covered_vmids":[101],"next_run":9000,"last_expected_run":null}],
|
||||
"backup_runs":[
|
||||
{"upid":"UPID:pve:1:2:3:vzdump::root@pam:","started_at":1001,"ended_at":1050,"status":"error",
|
||||
"message":"job errors","guests":[{"vmid":100,"result":"ok"},{"vmid":101,"result":"error"}]},
|
||||
{"upid":"UPID:pve:1:2:4:vzdump:100:root@pam:","started_at":100,"ended_at":120,"status":"ok",
|
||||
"message":null,"guests":[{"vmid":100,"result":"ok"}]}],
|
||||
"not_backed_up":[]})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_agent_shape_and_rejects_unsafe_values() {
|
||||
assert!(validate(&sample()).is_ok());
|
||||
for (pointer, bad) in [
|
||||
("/guests/0/macs/0", json!("BC:24:11:88:35:97")),
|
||||
("/guests/0/kind", json!("docker")),
|
||||
("/guests/0/vmid", json!(5)),
|
||||
("/guests/0/smbios_uuid", json!("not-a-uuid")),
|
||||
("/backup_runs/0/status", json!("running")),
|
||||
("/backup_jobs/0/selection", json!("tag")),
|
||||
("/node", json!("pve\nX")),
|
||||
] {
|
||||
let mut doc = sample();
|
||||
*doc.pointer_mut(pointer).unwrap() = bad;
|
||||
assert!(validate(&doc).is_err(), "{pointer}");
|
||||
}
|
||||
let mut doc = sample();
|
||||
doc["extra"] = json!(1);
|
||||
assert!(validate(&doc).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evaluates_due_runs_per_job_and_guest() {
|
||||
let status = backup_status(&sample()).unwrap();
|
||||
let jobs: BTreeMap<String, Value> = status["jobs"].as_array().unwrap().iter()
|
||||
.map(|j| (j["id"].as_str().unwrap().to_owned(), j.clone())).collect();
|
||||
assert_eq!(jobs["a"]["status"], "ok");
|
||||
assert_eq!(jobs["b"]["status"], "failed");
|
||||
assert_eq!(jobs["b"]["guests"][1]["status"], "failed");
|
||||
assert_eq!(jobs["c"]["status"], "disabled");
|
||||
assert_eq!(jobs["d"]["status"], "pending");
|
||||
assert_eq!(status["all_scheduled_backups_ran"], false);
|
||||
assert_eq!(status["last_success_at"]["100"], 1050);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_and_running_runs_are_distinguished() {
|
||||
let mut doc = sample();
|
||||
doc["backup_runs"] = json!([]);
|
||||
let status = backup_status(&doc).unwrap();
|
||||
assert_eq!(status["jobs"][0]["status"], "missed");
|
||||
doc["backup_runs"] = json!([{"upid":"UPID:pve:9:9:9:vzdump::root@pam:","started_at":1000,
|
||||
"ended_at":null,"status":"running","message":null,"guests":[]}]);
|
||||
assert_eq!(backup_status(&doc).unwrap()["jobs"][0]["status"], "running");
|
||||
doc["backup_jobs"] = json!([]);
|
||||
assert!(backup_status(&doc).unwrap()["all_scheduled_backups_ran"].is_null());
|
||||
}
|
||||
}
|
||||
@@ -190,6 +190,9 @@ pub async fn ingest(
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
ensure!(stored == hash, "message identity conflict");
|
||||
if matches!(s.scope.as_str(), "linux.base" | "windows.base") {
|
||||
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
||||
}
|
||||
}
|
||||
Some(Payload::JobAccepted(_) | Payload::JobStateChanged(_) | Payload::JobResult(_)) => {
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, e).await?
|
||||
|
||||
@@ -208,6 +208,9 @@ struct HypervisorAssignment {
|
||||
// null clears the relation. A missing field is a malformed update.
|
||||
#[serde(deserialize_with = "required_hypervisor_value")]
|
||||
hypervisor_asset_id: Option<Uuid>,
|
||||
/// true hands the decision back to automatic Proxmox detection (requires null id).
|
||||
#[serde(default)]
|
||||
automatic: bool,
|
||||
}
|
||||
|
||||
fn required_hypervisor_value<'de, D: serde::Deserializer<'de>>(value: D) -> Result<Option<Uuid>, D::Error> {
|
||||
@@ -227,24 +230,35 @@ async fn assign_hypervisor(
|
||||
|
||||
// Match the asset lock order for simultaneous edits of the same two devices.
|
||||
let ids = [asset, input.hypervisor_asset_id.unwrap_or(asset)];
|
||||
let rows: Vec<(Uuid, Option<Uuid>)> = sqlx::query_as(
|
||||
"SELECT asset_id,hypervisor_asset_id FROM assets
|
||||
if input.automatic && input.hypervisor_asset_id.is_some() {
|
||||
return Err(Error(StatusCode::UNPROCESSABLE_ENTITY, "AUTOMATIC_REQUIRES_NULL_HYPERVISOR"));
|
||||
}
|
||||
let rows: Vec<(Uuid, Option<Uuid>, Option<String>)> = sqlx::query_as(
|
||||
"SELECT asset_id,hypervisor_asset_id,hypervisor_source FROM assets
|
||||
WHERE tenant_id=$1 AND asset_id=ANY($2) ORDER BY asset_id FOR UPDATE",
|
||||
)
|
||||
.bind(tenant).bind(ids.as_slice()).fetch_all(&mut *tx).await?;
|
||||
let previous = rows.into_iter().find(|(id, _)| *id == asset)
|
||||
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?.1;
|
||||
let (_, previous, previous_source) = rows.into_iter().find(|(id, _, _)| *id == asset)
|
||||
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
|
||||
crate::devices::validate_vm_host_assignment(
|
||||
&mut tx, tenant, asset, pair.agent_id, input.hypervisor_asset_id,
|
||||
).await?;
|
||||
if previous != input.hypervisor_asset_id {
|
||||
sqlx::query("UPDATE assets SET hypervisor_asset_id=$3 WHERE tenant_id=$1 AND asset_id=$2")
|
||||
.bind(tenant).bind(asset).bind(input.hypervisor_asset_id).execute(&mut *tx).await?;
|
||||
let source = (!input.automatic).then_some("manual");
|
||||
if previous != input.hypervisor_asset_id || previous_source.as_deref() != source {
|
||||
sqlx::query("UPDATE assets SET hypervisor_asset_id=$3,hypervisor_source=$4 WHERE tenant_id=$1 AND asset_id=$2")
|
||||
.bind(tenant).bind(asset).bind(input.hypervisor_asset_id).bind(source).execute(&mut *tx).await?;
|
||||
audit(&mut tx, &p, EVENT, json!({"asset_id":asset,"agent_id":pair.agent_id,
|
||||
"previous_hypervisor_asset_id":previous,"hypervisor_asset_id":input.hypervisor_asset_id})).await?;
|
||||
"previous_hypervisor_asset_id":previous,"hypervisor_asset_id":input.hypervisor_asset_id,
|
||||
"previous_source":previous_source,"source":source.unwrap_or("automatic")})).await?;
|
||||
}
|
||||
if input.automatic {
|
||||
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
||||
}
|
||||
let (current, current_source): (Option<Uuid>, Option<String>) = sqlx::query_as(
|
||||
"SELECT hypervisor_asset_id,hypervisor_source FROM assets WHERE tenant_id=$1 AND asset_id=$2")
|
||||
.bind(tenant).bind(asset).fetch_one(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
Ok(Json(json!({"asset_id":asset,"hypervisor_asset_id":input.hypervisor_asset_id})))
|
||||
Ok(Json(json!({"asset_id":asset,"hypervisor_asset_id":current,"hypervisor_source":current_source})))
|
||||
}
|
||||
|
||||
async fn remove_device(
|
||||
|
||||
@@ -3,7 +3,7 @@ WITH classified AS (
|
||||
a.last_authenticated_at,a.last_authenticated_certificate_sha256,a.certificate_sha256,
|
||||
a.enrollment_generation,a.assignment_version,a.capabilities,a.reported_agent_version,
|
||||
a.system_usage,a.system_usage_observed_at,
|
||||
asset.archived_at,asset.hypervisor_asset_id,asset.purpose,t.name AS customer_name,
|
||||
asset.archived_at,asset.hypervisor_asset_id,asset.hypervisor_source,asset.purpose,t.name AS customer_name,
|
||||
CASE
|
||||
WHEN a.revoked_at IS NOT NULL THEN 'revoked'
|
||||
WHEN asset.archived_at IS NOT NULL THEN 'archived'
|
||||
|
||||
@@ -261,6 +261,12 @@ pub async fn detail(
|
||||
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
|
||||
let hosts = assigned_hosts(&mut tx, tenant, now, std::slice::from_ref(&row)).await?;
|
||||
let mut item = project(&row, now, true, &hosts);
|
||||
if item["proxmox"]["data"].is_object() {
|
||||
let rows: Vec<(Uuid, Uuid, Option<String>, Option<String>, Value)> =
|
||||
sqlx::query_as(include_str!("proxmox_guest_devices.sql"))
|
||||
.bind(tenant).bind(asset).fetch_all(&mut *tx).await?;
|
||||
item["proxmox"]["guest_devices"] = proxmox_guest_devices(&item["proxmox"]["data"], &rows);
|
||||
}
|
||||
item["recent_jobs"] = if p.permits_scope("operations:read")
|
||||
&& crate::permissions::allowed(&mut tx, &p.name, "operations:read").await?
|
||||
{
|
||||
@@ -282,6 +288,25 @@ pub async fn detail(
|
||||
Ok(Json(item))
|
||||
}
|
||||
|
||||
/// Linked devices per reported guest, matched by the MACs both sides report.
|
||||
fn proxmox_guest_devices(
|
||||
data: &Value,
|
||||
rows: &[(Uuid, Uuid, Option<String>, Option<String>, Value)],
|
||||
) -> Value {
|
||||
let mut result = Vec::new();
|
||||
for guest in data["guests"].as_array().into_iter().flatten() {
|
||||
let macs: Vec<&str> = guest["macs"].as_array().into_iter().flatten()
|
||||
.filter_map(Value::as_str).collect();
|
||||
let linked = rows.iter().find(|(_, _, _, _, device)| device.as_array().into_iter().flatten()
|
||||
.filter_map(Value::as_str).any(|m| macs.contains(&m)));
|
||||
if let (Some(vmid), Some((asset_id, agent_id, source, hostname, _))) = (guest["vmid"].as_u64(), linked) {
|
||||
result.push(json!({"vmid":vmid,"asset_id":asset_id,"agent_id":agent_id,
|
||||
"hostname":hostname,"source":source}));
|
||||
}
|
||||
}
|
||||
Value::Array(result)
|
||||
}
|
||||
|
||||
fn observed<'a>(data: &'a Value, scope: &str) -> &'a Value {
|
||||
data["observations"]
|
||||
.as_array()
|
||||
@@ -569,12 +594,33 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool, hosts: &HashMap<Uuid, Ho
|
||||
} else { Value::Null };
|
||||
item["disks_truncated"] = json!(scope["truncated"] == true);
|
||||
}
|
||||
if let Some(scope) = data["observations"].as_array().and_then(|scopes|
|
||||
scopes.iter().find(|s| s["scope"] == "linux.proxmox")) {
|
||||
let status = (scope["quality"] == "valid")
|
||||
.then(|| rmm_domain::inventory_document::proxmox::backup_status(&scope["data"]))
|
||||
.flatten();
|
||||
item["proxmox_backup"] = status.as_ref().map_or(Value::Null, |s| json!({
|
||||
"all_scheduled_backups_ran":s["all_scheduled_backups_ran"],
|
||||
"problem_jobs":s["problem_jobs"],"enabled_jobs":s["enabled_jobs"]}));
|
||||
if full {
|
||||
item["proxmox"] = json!({
|
||||
"quality":scope["quality"],"error_code":scope["error_code"],
|
||||
"observed_at":scope["observed_at"],"truncated":scope["truncated"],
|
||||
"data":if scope["quality"] == "valid" { scope["data"].clone() } else { Value::Null },
|
||||
"backup_status":status,"guest_devices":[]});
|
||||
}
|
||||
}
|
||||
if device_type == Some(DeviceType::Virtual) && row.get::<String, _>("lifecycle") == "current" {
|
||||
let assigned_id: Option<Uuid> = row.get("hypervisor_asset_id");
|
||||
let assigned = assigned_id.and_then(|id| hosts.get(&id));
|
||||
item["hypervisor"] = hypervisor_display(&data, assigned);
|
||||
let source = row.get::<Option<String>, _>("hypervisor_source");
|
||||
if item["hypervisor"]["kind"] == "device" {
|
||||
item["hypervisor"]["source"] = json!(source);
|
||||
}
|
||||
if full {
|
||||
item["assigned_hypervisor_asset_id"] = json!(assigned_id);
|
||||
item["hypervisor_source"] = json!(source);
|
||||
}
|
||||
}
|
||||
if full {
|
||||
@@ -668,6 +714,94 @@ mod tests {
|
||||
assert!(hypervisor_display(&data, None).is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the disposable RLS database from tests/local_access_database.py"]
|
||||
async fn proxmox_guest_macs_link_vms_automatically_but_never_override_people() {
|
||||
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
|
||||
.await.unwrap();
|
||||
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
|
||||
.await.unwrap();
|
||||
let (tenant, other) = (Uuid::new_v4(), Uuid::new_v4());
|
||||
let (host, vm, manual_vm, foreign_vm, twin_a, twin_b) =
|
||||
(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
|
||||
for customer in [tenant, other] {
|
||||
sqlx::query("INSERT INTO tenants(tenant_id,name) VALUES($1,'Proxmox link test')")
|
||||
.bind(customer).execute(&admin).await.unwrap();
|
||||
}
|
||||
let guests = |macs: &[&str]| json!({"node":"pve","guests":macs.iter().enumerate().map(|(i, m)|
|
||||
json!({"vmid":100 + i,"macs":[m]})).collect::<Vec<_>>()});
|
||||
let device = |kind: &str, mac: &str, proxmox: Option<Value>| {
|
||||
let mut scopes = vec![
|
||||
json!({"scope":"linux.hardware","quality":"valid","data":{"device_type":kind}}),
|
||||
json!({"scope":"linux.interfaces","quality":"valid","data":[{"name":"eth0","mac":mac}]}),
|
||||
];
|
||||
if let Some(data) = proxmox {
|
||||
scopes.push(json!({"scope":"linux.proxmox","quality":"valid","data":data}));
|
||||
}
|
||||
json!({"hostname":"h","os":"Debian","root_available_bytes":1,"root_total_bytes":2,"observations":scopes})
|
||||
};
|
||||
let insert = |customer: Uuid, id: Uuid, data: Value, sequence: i64| {
|
||||
let admin = admin.clone();
|
||||
async move {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,now(),'linux.base','valid','partial',$6,$7,$8)")
|
||||
.bind(customer).bind(id).bind(Uuid::new_v4()).bind(Uuid::new_v4()).bind(sequence)
|
||||
.bind(Uuid::new_v4()).bind(data).bind("0".repeat(64)).execute(&admin).await.unwrap();
|
||||
}
|
||||
};
|
||||
let all = ["bc:24:11:00:00:01", "bc:24:11:00:00:02", "bc:24:11:00:00:03", "bc:24:11:00:00:04"];
|
||||
for (customer, id, data) in [
|
||||
(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all)))),
|
||||
(tenant, vm, device("virtual", "BC:24:11:00:00:01", None)),
|
||||
(tenant, manual_vm, device("virtual", "bc:24:11:00:00:02", None)),
|
||||
(other, foreign_vm, device("virtual", "bc:24:11:00:00:03", None)),
|
||||
(tenant, twin_a, device("virtual", "bc:24:11:00:00:04", None)),
|
||||
(tenant, twin_b, device("virtual", "bc:24:11:00:00:04", None)),
|
||||
] {
|
||||
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
|
||||
.bind(customer).bind(id).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,asset_id,agent_id,certificate_sha256,last_contact_at,last_authenticated_at,last_authenticated_certificate_sha256) VALUES($1,$2,$2,$3,now(),now(),$3)")
|
||||
.bind(customer).bind(id).bind(rmm_domain::jobs::hash(format!("{customer}:{id}").as_bytes()))
|
||||
.execute(&admin).await.unwrap();
|
||||
insert(customer, id, data, 1).await;
|
||||
}
|
||||
sqlx::query("UPDATE assets SET hypervisor_source='manual' WHERE asset_id=$1")
|
||||
.bind(manual_vm).execute(&admin).await.unwrap();
|
||||
let link = |id: Uuid| {
|
||||
let admin = admin.clone();
|
||||
async move {
|
||||
sqlx::query_as::<_, (Option<Uuid>, Option<String>)>(
|
||||
"SELECT hypervisor_asset_id,hypervisor_source FROM assets WHERE asset_id=$1")
|
||||
.bind(id).fetch_one(&admin).await.unwrap()
|
||||
}
|
||||
};
|
||||
|
||||
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
||||
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
||||
tx.commit().await.unwrap();
|
||||
assert_eq!(link(vm).await, (Some(host), Some("proxmox".into())));
|
||||
assert_eq!(link(manual_vm).await, (None, Some("manual".into())));
|
||||
assert_eq!(link(foreign_vm).await, (None, None));
|
||||
assert_eq!(link(twin_a).await, (None, None), "duplicate MACs are ambiguous");
|
||||
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='device_hypervisor_detected'")
|
||||
.bind(tenant).fetch_one(&admin).await.unwrap();
|
||||
assert_eq!(audits, 1);
|
||||
|
||||
// Idempotent: a second pass changes nothing.
|
||||
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
||||
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
||||
tx.commit().await.unwrap();
|
||||
let audits: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
|
||||
.bind(tenant).fetch_one(&admin).await.unwrap();
|
||||
assert_eq!(audits, 1);
|
||||
|
||||
// The guest left the host: the automatic link is released.
|
||||
insert(tenant, host, device("hypervisor", "52:54:00:00:00:99", Some(guests(&all[1..]))), 2).await;
|
||||
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
||||
crate::proxmox_links::reconcile(&mut tx, tenant).await;
|
||||
tx.commit().await.unwrap();
|
||||
assert_eq!(link(vm).await, (None, None));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the disposable RLS database from tests/local_access_database.py"]
|
||||
async fn host_links_resolve_outside_the_page_and_stay_inside_the_customer() {
|
||||
|
||||
@@ -27,6 +27,7 @@ mod monitoring;
|
||||
mod oidc;
|
||||
mod pagination;
|
||||
mod permissions;
|
||||
mod proxmox_links;
|
||||
mod root_policy;
|
||||
mod managed_scripts;
|
||||
mod scheduler;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
-- Current devices of customer $1 linked to host $2, with the MACs they report themselves.
|
||||
SELECT asset.asset_id, inst.agent_id, asset.hypervisor_source,
|
||||
inventory.data->>'hostname' AS hostname,
|
||||
COALESCE((
|
||||
SELECT jsonb_agg(DISTINCT lower(i->>'mac'))
|
||||
FROM jsonb_array_elements(CASE WHEN jsonb_typeof(inventory.data->'observations')='array'
|
||||
THEN inventory.data->'observations' ELSE '[]'::jsonb END) o,
|
||||
jsonb_array_elements(CASE
|
||||
WHEN jsonb_typeof(o->'data')='array' THEN o->'data'
|
||||
WHEN jsonb_typeof(o#>'{data,entries}')='array' THEN o#>'{data,entries}'
|
||||
ELSE '[]'::jsonb END) i
|
||||
WHERE o->>'scope' IN ('linux.interfaces','windows.interfaces') AND o->>'quality'='valid'
|
||||
AND i->>'mac' IS NOT NULL
|
||||
), '[]'::jsonb) AS macs
|
||||
FROM assets asset
|
||||
JOIN LATERAL (
|
||||
SELECT a.agent_id, a.revoked_at FROM agent_installations a
|
||||
WHERE a.tenant_id=asset.tenant_id AND a.asset_id=asset.asset_id AND NOT a.customer_transfer_pending
|
||||
ORDER BY a.enrollment_generation DESC, a.created_at DESC, a.agent_id LIMIT 1
|
||||
) inst ON inst.revoked_at IS NULL
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT data FROM observations
|
||||
WHERE tenant_id=asset.tenant_id AND agent_id=inst.agent_id AND source_id IN ('linux.base','windows.base')
|
||||
ORDER BY observed_at DESC, received_at DESC, sequence DESC LIMIT 1
|
||||
) inventory ON true
|
||||
WHERE asset.tenant_id=$1 AND asset.hypervisor_asset_id=$2 AND asset.archived_at IS NULL
|
||||
ORDER BY asset.asset_id
|
||||
LIMIT 1024
|
||||
@@ -0,0 +1,24 @@
|
||||
//! Automatic VM -> Proxmox host mapping from the hosts' `linux.proxmox` guest lists.
|
||||
//! Runs inside the ingesting customer transaction behind a savepoint: a failed
|
||||
//! reconciliation never blocks acknowledging the inventory snapshot itself.
|
||||
use sqlx::{Acquire, Postgres, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub async fn reconcile(tx: &mut Transaction<'_, Postgres>, tenant: Uuid) {
|
||||
let result = async {
|
||||
let mut savepoint = tx.begin().await?;
|
||||
let events: Vec<Uuid> = sqlx::query_scalar(include_str!("proxmox_links.sql"))
|
||||
.bind(tenant)
|
||||
.fetch_all(&mut *savepoint)
|
||||
.await?;
|
||||
savepoint.commit().await?;
|
||||
Ok::<_, sqlx::Error>(events.len())
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(0) => (),
|
||||
Ok(changed) => eprintln!("proxmox guest links updated: {changed}"),
|
||||
// Database detail stays out of the log line; the snapshot is still acknowledged.
|
||||
Err(_) => eprintln!("proxmox guest link reconciliation skipped"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
-- Automatic VM -> Proxmox host links for one customer ($1). A link is set only for a
|
||||
-- unique MAC match between a host's current guest list and a current virtual device of
|
||||
-- the same customer. Manual decisions (hypervisor_source='manual') are never touched.
|
||||
WITH current_installation AS (
|
||||
SELECT asset.asset_id, asset.hypervisor_asset_id, asset.hypervisor_source, inst.agent_id
|
||||
FROM assets asset
|
||||
JOIN LATERAL (
|
||||
SELECT a.agent_id, a.revoked_at
|
||||
FROM agent_installations a
|
||||
WHERE a.tenant_id=asset.tenant_id AND a.asset_id=asset.asset_id
|
||||
AND NOT a.customer_transfer_pending
|
||||
AND a.last_authenticated_at IS NOT NULL
|
||||
ORDER BY a.enrollment_generation DESC, a.created_at DESC, a.agent_id
|
||||
LIMIT 1
|
||||
) inst ON inst.revoked_at IS NULL
|
||||
WHERE asset.tenant_id=$1 AND asset.archived_at IS NULL
|
||||
),
|
||||
latest AS (
|
||||
SELECT c.*, inventory.data
|
||||
FROM current_installation c
|
||||
JOIN LATERAL (
|
||||
SELECT data FROM observations
|
||||
WHERE tenant_id=$1 AND agent_id=c.agent_id AND source_id IN ('linux.base','windows.base')
|
||||
ORDER BY observed_at DESC, received_at DESC, sequence DESC
|
||||
LIMIT 1
|
||||
) inventory ON true
|
||||
),
|
||||
scopes AS (
|
||||
SELECT l.asset_id, o
|
||||
FROM latest l,
|
||||
jsonb_array_elements(CASE WHEN jsonb_typeof(l.data->'observations')='array'
|
||||
THEN l.data->'observations' ELSE '[]'::jsonb END) o
|
||||
WHERE o->>'quality'='valid'
|
||||
),
|
||||
proxmox_hosts AS (
|
||||
SELECT asset_id, o->'data' AS data FROM scopes WHERE o->>'scope'='linux.proxmox'
|
||||
),
|
||||
guest_macs AS (
|
||||
SELECT DISTINCT h.asset_id AS host_asset_id, lower(mac) AS mac
|
||||
FROM proxmox_hosts h,
|
||||
jsonb_array_elements(h.data->'guests') g,
|
||||
jsonb_array_elements_text(g->'macs') mac
|
||||
),
|
||||
virtual_devices AS (
|
||||
SELECT asset_id FROM scopes
|
||||
WHERE o->>'scope' IN ('linux.hardware','windows.hardware') AND o#>>'{data,device_type}'='virtual'
|
||||
),
|
||||
device_macs AS (
|
||||
SELECT DISTINCT s.asset_id, lower(i->>'mac') AS mac
|
||||
FROM scopes s
|
||||
JOIN virtual_devices v ON v.asset_id=s.asset_id,
|
||||
jsonb_array_elements(CASE
|
||||
WHEN jsonb_typeof(s.o->'data')='array' THEN s.o->'data'
|
||||
WHEN jsonb_typeof(s.o#>'{data,entries}')='array' THEN s.o#>'{data,entries}'
|
||||
ELSE '[]'::jsonb END) i
|
||||
WHERE s.o->>'scope' IN ('linux.interfaces','windows.interfaces')
|
||||
AND i->>'mac' ~* '^[0-9a-f]{2}(:[0-9a-f]{2}){5}$'
|
||||
AND lower(i->>'mac') <> '00:00:00:00:00:00'
|
||||
),
|
||||
-- A MAC that more than one device or more than one host claims proves nothing.
|
||||
unique_macs AS (
|
||||
SELECT d.asset_id, g.host_asset_id
|
||||
FROM device_macs d JOIN guest_macs g ON g.mac=d.mac
|
||||
WHERE d.asset_id <> g.host_asset_id
|
||||
AND (SELECT count(*) FROM device_macs d2 WHERE d2.mac=d.mac)=1
|
||||
AND (SELECT count(*) FROM guest_macs g2 WHERE g2.mac=d.mac)=1
|
||||
),
|
||||
matches AS (
|
||||
SELECT asset_id, min(host_asset_id::text)::uuid AS host_asset_id
|
||||
FROM unique_macs GROUP BY asset_id HAVING count(DISTINCT host_asset_id)=1
|
||||
),
|
||||
desired AS (
|
||||
-- New or changed automatic links.
|
||||
SELECT l.asset_id, l.hypervisor_asset_id AS previous, m.host_asset_id AS next
|
||||
FROM latest l JOIN matches m ON m.asset_id=l.asset_id
|
||||
WHERE l.hypervisor_source IS DISTINCT FROM 'manual'
|
||||
AND (l.hypervisor_asset_id IS DISTINCT FROM m.host_asset_id OR l.hypervisor_source IS NULL)
|
||||
UNION ALL
|
||||
-- The assigned host reports a valid guest list that no longer contains this device.
|
||||
SELECT l.asset_id, l.hypervisor_asset_id, NULL
|
||||
FROM latest l
|
||||
WHERE l.hypervisor_source='proxmox'
|
||||
AND NOT EXISTS (SELECT FROM matches m WHERE m.asset_id=l.asset_id)
|
||||
AND EXISTS (SELECT FROM proxmox_hosts h WHERE h.asset_id=l.hypervisor_asset_id)
|
||||
AND EXISTS (SELECT FROM device_macs d WHERE d.asset_id=l.asset_id)
|
||||
),
|
||||
changed AS (
|
||||
UPDATE assets a
|
||||
SET hypervisor_asset_id=d.next,
|
||||
hypervisor_source=CASE WHEN d.next IS NULL THEN NULL ELSE 'proxmox' END
|
||||
FROM desired d
|
||||
WHERE a.tenant_id=$1 AND a.asset_id=d.asset_id
|
||||
RETURNING a.asset_id, d.previous, d.next
|
||||
)
|
||||
INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data,created_at)
|
||||
SELECT $1, gen_random_uuid(), 'system:proxmox-guest-detection',
|
||||
CASE WHEN next IS NULL THEN 'device_hypervisor_released' ELSE 'device_hypervisor_detected' END,
|
||||
jsonb_build_object('asset_id',asset_id,'previous_hypervisor_asset_id',previous,
|
||||
'hypervisor_asset_id',next,'match','mac'),
|
||||
now()
|
||||
FROM changed
|
||||
WHERE previous IS DISTINCT FROM next
|
||||
RETURNING event_id
|
||||
@@ -0,0 +1,10 @@
|
||||
SET LOCAL lock_timeout = '3s';
|
||||
SET LOCAL statement_timeout = '30s';
|
||||
-- NULL: no automatic decision yet, 'proxmox': set from a Proxmox host's guest list,
|
||||
-- 'manual': chosen by a person and never overwritten by automatic detection.
|
||||
ALTER TABLE assets
|
||||
ADD COLUMN hypervisor_source text,
|
||||
ADD CONSTRAINT assets_hypervisor_source_check
|
||||
CHECK (hypervisor_source IN ('manual','proxmox'));
|
||||
UPDATE assets SET hypervisor_source='manual' WHERE hypervisor_asset_id IS NOT NULL;
|
||||
GRANT UPDATE (hypervisor_source) ON assets TO rmm_app;
|
||||
Reference in New Issue
Block a user