//! Read-only Proxmox VE observation: hosted guests, scheduled vzdump jobs and recent //! backup runs. Uses a dedicated PVEAuditor API token supplied by the installer as a //! systemd credential. No writes, no root, no shell; the transport is injected so the //! orchestration and parsing stay testable without a Proxmox host. use super::{Observation, Quality}; use serde::Serialize; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; pub const SOURCE: &str = "proxmox.api:/api2/json"; pub const OBSERVATION_MAX_BYTES: usize = rmm_domain::inventory_document::PROXMOX_OBSERVATION_MAX_BYTES; pub const MAX_GUESTS: usize = 512; pub const MAX_JOBS: usize = 64; pub const MAX_RUNS: usize = 64; const MAX_LOG_LINES: usize = 4000; /// Weekly schedules must still show their previous due time. const SCHEDULE_LOOKBACK_SECONDS: i64 = 8 * 24 * 3600; const SCHEDULE_ITERATIONS: usize = 100; #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Proxmox { pub node: String, pub cluster: Option, pub version: Option, pub guests: Vec, pub backup_jobs: Vec, pub backup_runs: Vec, /// Local guests that no backup job selects (Proxmox's own analysis). pub not_backed_up: Vec, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Guest { pub vmid: u32, pub name: String, pub kind: &'static str, pub status: String, pub template: bool, pub cpus: Option, pub memory_bytes: Option, pub disk_bytes: Option, pub uptime_seconds: Option, pub pool: Option, pub macs: Vec, pub smbios_uuid: Option, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BackupJob { pub id: String, pub enabled: bool, pub schedule: Option, pub storage: Option, pub mode: Option, pub selection: &'static str, /// Local guests this job backs up, resolved from all/vmid/pool minus exclude. pub covered_vmids: Vec, pub next_run: Option, /// Most recent scheduled time at or before the observation time. pub last_expected_run: Option, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BackupRun { pub upid: String, pub started_at: i64, pub ended_at: Option, /// ok | warning | error | running pub status: &'static str, pub message: Option, pub guests: Vec, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct GuestResult { pub vmid: u32, /// ok | error pub result: &'static str, } /// Bounded GET of one `/api2/json` path, returning the `data` member. pub trait Api { fn get(&self, path: &str) -> impl std::future::Future> + Send; } pub fn not_configured() -> Observation { Observation::unavailable(SOURCE, Quality::Unsupported, "PROXMOX_NOT_CONFIGURED") } pub fn failed(code: &str) -> Observation { Observation::unavailable(SOURCE, Quality::Error, code) } /// The token secret never appears in errors, logs or observations. pub fn token_header(credential: &str) -> Option { let token = credential.trim(); let (id, secret) = token.split_once('=')?; let (user, name) = id.split_once('!')?; let ok = |s: &str| !s.is_empty() && s.len() <= 128 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"-_.@".contains(&b)); (ok(user) && user.contains('@') && ok(name) && ok(secret)) .then(|| format!("PVEAPIToken={token}")) } pub async fn collect(api: &impl Api, now: i64) -> Observation { match collect_data(api, now).await { Ok((data, truncated)) => { let mut observation = Observation::valid(SOURCE, data); if truncated { observation = observation.partial("PROXMOX_LIMIT"); observation.truncated = true; } fit(observation) } Err(error) => { let code = error.to_string(); failed(if code.starts_with("PROXMOX_") && code.len() <= 64 { &code } else { "PROXMOX_API_ERROR" }) } } } async fn collect_data(api: &impl Api, now: i64) -> anyhow::Result<(Proxmox, bool)> { let status = api.get("/cluster/status").await?; let rows = status.as_array().ok_or_else(|| anyhow::anyhow!("PROXMOX_API_SHAPE"))?; let node = rows.iter() .find(|r| r["type"] == "node" && flag(&r["local"]) == Some(true)) .and_then(|r| r["name"].as_str()) .filter(|n| clean(n, 64)) .ok_or_else(|| anyhow::anyhow!("PROXMOX_LOCAL_NODE_UNKNOWN"))? .to_owned(); let cluster = rows.iter().find(|r| r["type"] == "cluster") .and_then(|r| r["name"].as_str()).filter(|n| clean(n, 64)).map(str::to_owned); let version = api.get("/version").await.ok() .and_then(|v| v["version"].as_str().filter(|s| clean(s, 32)).map(str::to_owned)); let mut truncated = false; let resources = api.get("/cluster/resources?type=vm").await?; let mut guests = Vec::new(); for row in resources.as_array().into_iter().flatten() { if row["node"].as_str() != Some(node.as_str()) { continue; } let Some(mut guest) = guest(row) else { continue }; if guests.len() == MAX_GUESTS { truncated = true; break; } let path = format!("/nodes/{node}/{}/{}/config", guest.kind, guest.vmid); if let Ok(config) = api.get(&path).await { guest.macs = macs(&config); guest.smbios_uuid = config["smbios1"].as_str().and_then(smbios_uuid); } else { truncated = true; } guests.push(guest); } guests.sort_by_key(|g| g.vmid); let jobs = api.get("/cluster/backup").await?; let mut backup_jobs = Vec::new(); for row in jobs.as_array().into_iter().flatten() { if row["type"].as_str().is_some_and(|t| t != "vzdump") { continue; } if backup_jobs.len() == MAX_JOBS { truncated = true; break; } let Some(mut job) = backup_job(row, &node, &guests) else { continue }; if job.enabled && let Some(schedule) = &job.schedule { job.last_expected_run = last_expected_run(api, schedule, now).await; } backup_jobs.push(job); } let local: BTreeSet = guests.iter().map(|g| g.vmid).collect(); let not_backed_up = api.get("/cluster/backup-info/not-backed-up").await.ok() .and_then(|v| v.as_array().cloned()).unwrap_or_default().iter() .filter_map(|r| vmid(&r["vmid"])).filter(|id| local.contains(id)) .collect::>().into_iter().collect(); let since = now - SCHEDULE_LOOKBACK_SECONDS; let tasks = api.get(&format!( "/nodes/{node}/tasks?typefilter=vzdump&source=all&since={since}&limit={MAX_RUNS}" )).await?; let mut backup_runs = Vec::new(); for row in tasks.as_array().into_iter().flatten() { let Some(mut run) = backup_run(row) else { continue }; if backup_runs.len() == MAX_RUNS { truncated = true; break; } let encoded = percent_encode(&run.upid); match api.get(&format!("/nodes/{node}/tasks/{encoded}/log?start=0&limit={MAX_LOG_LINES}")).await { Ok(log) => { let lines: Vec<&str> = log.as_array().into_iter().flatten() .filter_map(|l| l["t"].as_str()).collect(); run.guests = guest_results(&lines); if run.guests.is_empty() { // A single-guest task names its guest in the UPID. if let (Some(id), "ok" | "warning") = (upid_vmid(&run.upid), run.status) { run.guests.push(GuestResult { vmid: id, result: "ok" }); } } if lines.len() >= MAX_LOG_LINES { truncated = true; } } Err(_) => truncated = true, } backup_runs.push(run); } backup_runs.sort_by(|a, b| b.started_at.cmp(&a.started_at).then_with(|| a.upid.cmp(&b.upid))); Ok((Proxmox { node, cluster, version, guests, backup_jobs, backup_runs, not_backed_up }, truncated)) } /// The scheduler's own calendar evaluation, so no local schedule parser can disagree. /// The API returns at most 100 occurrences, so frequent schedules use shorter windows. async fn last_expected_run(api: &impl Api, schedule: &str, now: i64) -> Option { for lookback in [SCHEDULE_LOOKBACK_SECONDS, 86_400, 7_200, 600] { let path = format!( "/cluster/jobs/schedule-analyze?schedule={}&starttime={}&iterations={SCHEDULE_ITERATIONS}", percent_encode(schedule), now - lookback ); let runs: Vec = api.get(&path).await.ok()?.as_array()?.iter() .filter_map(|r| r["timestamp"].as_i64()).collect(); if runs.len() < SCHEDULE_ITERATIONS || runs.iter().any(|t| *t > now) { return runs.into_iter().filter(|t| *t <= now).max(); } } None } /// Keep the newest information when the scope exceeds its wire budget. fn fit(mut observation: Observation) -> Observation { let size = |o: &Observation| serde_json::to_vec(o).map(|b| b.len()).unwrap_or(usize::MAX); while size(&observation) > OBSERVATION_MAX_BYTES - 1024 { let Some(data) = observation.data.as_mut() else { break }; if data.backup_runs.len() > 8 { data.backup_runs.pop(); } else if !data.guests.is_empty() { data.guests.pop(); } else { return failed("WIRE_SCOPE_LIMIT"); } observation.truncated = true; observation = observation.partial("PROXMOX_LIMIT"); } observation } fn guest(row: &Value) -> Option { let kind = match row["type"].as_str()? { "qemu" => "qemu", "lxc" => "lxc", _ => return None, }; let vmid = vmid(&row["vmid"])?; let name = row["name"].as_str().filter(|n| clean(n, 128)).unwrap_or("").to_owned(); Some(Guest { vmid, name: if name.is_empty() { format!("{kind}-{vmid}") } else { name }, kind, status: row["status"].as_str().filter(|s| clean(s, 32)).unwrap_or("unknown").to_owned(), template: flag(&row["template"]) == Some(true), cpus: row["maxcpu"].as_f64().filter(|c| *c >= 0.0 && *c < 65536.0).map(|c| c.ceil() as u32), memory_bytes: row["maxmem"].as_u64(), disk_bytes: row["maxdisk"].as_u64(), uptime_seconds: row["uptime"].as_u64(), pool: row["pool"].as_str().filter(|p| clean(p, 64)).map(str::to_owned), macs: Vec::new(), smbios_uuid: None, }) } fn backup_job(row: &Value, node: &str, guests: &[Guest]) -> Option { let id = row["id"].as_str().filter(|s| clean(s, 64))?.to_owned(); let text = |key: &str, limit| row[key].as_str().filter(|s| clean(s, limit)).map(str::to_owned); let list = |key: &str| -> BTreeSet { row[key].as_str().unwrap_or("").split([',', ' ', ';']) .filter_map(|s| s.trim().parse().ok()).collect() }; let exclude = list("exclude"); let pool = text("pool", 64); let vmids = list("vmid"); let all = flag(&row["all"]) == Some(true); let selection = if all { "all" } else if pool.is_some() { "pool" } else { "vmid" }; let node_matches = row["node"].as_str().is_none_or(|n| n == node); let covered_vmids = guests.iter() .filter(|g| node_matches && !g.template) .filter(|g| match selection { "all" => !exclude.contains(&g.vmid), "pool" => g.pool == pool, _ => vmids.contains(&g.vmid), }) .map(|g| g.vmid).collect(); Some(BackupJob { id, enabled: flag(&row["enabled"]).unwrap_or(true), schedule: text("schedule", 128).or_else(|| { // Pre-7.1 format: days of week plus start time. let time = text("starttime", 16)?; Some(text("dow", 64).map_or(time.clone(), |dow| format!("{dow} {time}"))) }), storage: text("storage", 64), mode: text("mode", 16), selection, covered_vmids, next_run: row["next-run"].as_i64(), last_expected_run: None, }) } fn backup_run(row: &Value) -> Option { let upid = row["upid"].as_str().filter(|s| clean(s, 160))?.to_owned(); let started_at = row["starttime"].as_i64()?; let ended_at = row["endtime"].as_i64(); let raw = row["status"].as_str(); let (status, message) = match (ended_at, raw) { (None, _) | (_, None) => ("running", None), (Some(_), Some("OK")) => ("ok", None), (Some(_), Some(s)) if s.starts_with("WARNINGS") => ("warning", Some(s)), (Some(_), Some(s)) => ("error", Some(s)), }; Some(BackupRun { upid, started_at, ended_at, status, message: message.map(|m| truncate(m, 256)), guests: Vec::new(), }) } /// vzdump task log lines; the last statement per guest wins. pub fn guest_results(lines: &[&str]) -> Vec { let mut results = BTreeMap::new(); for line in lines { let line = line.trim(); if let Some(rest) = line.strip_prefix("INFO: Finished Backup of VM ") && let Some(id) = leading_number(rest) { results.insert(id, "ok"); } else if let Some(rest) = line.strip_prefix("ERROR: Backup of VM ") && let Some(id) = leading_number(rest) { results.insert(id, "error"); } } results.into_iter().map(|(vmid, result)| GuestResult { vmid, result }).collect() } /// `netN` values: qemu `virtio=AA:..,bridge=..`; lxc `name=eth0,hwaddr=AA:..`. pub fn macs(config: &Value) -> Vec { let mut found = BTreeSet::new(); if let Some(map) = config.as_object() { for (key, value) in map { let is_net = key.strip_prefix("net").is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())); let Some(value) = value.as_str().filter(|_| is_net) else { continue }; for part in value.split(',') { if let Some((_, candidate)) = part.split_once('=') && let Some(mac) = normalize_mac(candidate) { found.insert(mac); } } } } found.into_iter().take(32).collect() } pub fn smbios_uuid(value: &str) -> Option { value.split(',').find_map(|part| part.strip_prefix("uuid=")) .and_then(|u| uuid::Uuid::parse_str(u.trim()).ok()) .filter(|u| !u.is_nil()) .map(|u| u.hyphenated().to_string()) } pub fn normalize_mac(value: &str) -> Option { let parts: Vec<&str> = value.trim().split([':', '-']).collect(); (parts.len() == 6 && parts.iter().all(|p| p.len() == 2 && p.bytes().all(|b| b.is_ascii_hexdigit()))) .then(|| parts.join(":").to_ascii_lowercase()) .filter(|m| m != "00:00:00:00:00:00") } /// UPID:node:pid:pstart:starttime:type:id:user: — `id` is the vmid for single-guest runs. fn upid_vmid(upid: &str) -> Option { upid.split(':').nth(6).and_then(|id| id.parse().ok()) } fn vmid(value: &Value) -> Option { value.as_u64().or_else(|| value.as_str()?.parse().ok()) .filter(|id| (100..1_000_000_000).contains(id)).map(|id| id as u32) } fn flag(value: &Value) -> Option { match value { Value::Bool(b) => Some(*b), Value::Number(n) => n.as_u64().map(|n| n != 0), Value::String(s) => match s.as_str() { "1" | "true" => Some(true), "0" | "false" => Some(false), _ => None }, _ => None, } } fn leading_number(text: &str) -> Option { let digits: String = text.chars().take_while(char::is_ascii_digit).collect(); digits.parse().ok() } fn clean(s: &str, limit: usize) -> bool { !s.trim().is_empty() && s.len() <= limit && !s.chars().any(char::is_control) } fn truncate(s: &str, limit: usize) -> String { let s: String = s.chars().filter(|c| !c.is_control()).collect(); if s.len() <= limit { return s; } let mut end = limit; while !s.is_char_boundary(end) { end -= 1; } s[..end].to_owned() } pub fn percent_encode(value: &str) -> String { value.bytes().map(|b| if b.is_ascii_alphanumeric() || b"-_.~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") }).collect() } #[cfg(test)] mod tests { use super::*; use serde_json::json; use std::collections::HashMap; struct Fixture(HashMap); impl Api for Fixture { async fn get(&self, path: &str) -> anyhow::Result { let key = path.split('?').next().unwrap(); self.0.get(path).or_else(|| self.0.get(key)).cloned() .ok_or_else(|| anyhow::anyhow!("PROXMOX_HTTP_404")) } } fn block(future: F) -> F::Output { tokio::runtime::Builder::new_current_thread().build().unwrap().block_on(future) } const NOW: i64 = 1_763_640_000; const UPID_JOB: &str = "UPID:pve:00001234:00ABCDEF:691EE8A0:vzdump::root@pam:"; const UPID_ONE: &str = "UPID:pve:00001235:00ABCDF0:691EE000:vzdump:101:root@pam:"; fn fixture() -> Fixture { let mut m = HashMap::new(); m.insert("/cluster/status".into(), json!([ {"type":"cluster","name":"lab"}, {"type":"node","name":"pve","local":1}, {"type":"node","name":"pve2","local":0}])); m.insert("/version".into(), json!({"version":"9.2.4"})); m.insert("/cluster/resources".into(), json!([ {"type":"qemu","vmid":100,"name":"debian","node":"pve","status":"running","maxcpu":2,"maxmem":1073741824u64,"maxdisk":10737418240u64,"uptime":42,"template":0}, {"type":"lxc","vmid":101,"name":"alpine","node":"pve","status":"stopped","maxcpu":1,"maxmem":134217728,"pool":"lab"}, {"type":"qemu","vmid":9000,"name":"tpl","node":"pve","status":"stopped","template":1}, {"type":"qemu","vmid":200,"name":"elsewhere","node":"pve2","status":"running"}])); m.insert("/nodes/pve/qemu/100/config".into(), json!({ "net0":"virtio=BC:24:11:AA:BB:01,bridge=vmbr0,firewall=1", "net1":"e1000=bc-24-11-aa-bb-02,bridge=vmbr1", "smbios1":"uuid=6E1A2B3C-0000-4000-8000-0000000000AA", "netmask":"virtio=BC:24:11:FF:FF:FF"})); m.insert("/nodes/pve/lxc/101/config".into(), json!({ "net0":"name=eth0,bridge=vmbr0,hwaddr=BC:24:11:AA:BB:03,ip=dhcp,type=veth"})); m.insert("/nodes/pve/qemu/9000/config".into(), json!({})); m.insert("/cluster/backup".into(), json!([ {"id":"backup-a","type":"vzdump","schedule":"*/2:00","storage":"local","mode":"snapshot","vmid":"100,200","enabled":1,"next-run":NOW+3600}, {"id":"backup-all","type":"vzdump","schedule":"sat 03:00","all":1,"exclude":"100","enabled":0}, {"id":"backup-pool","type":"vzdump","dow":"mon,tue","starttime":"02:30","pool":"lab"}, {"id":"backup-other-node","type":"vzdump","schedule":"daily","all":1,"node":"pve2"}])); m.insert("/cluster/jobs/schedule-analyze".into(), json!([ {"timestamp":NOW-7200},{"timestamp":NOW-1},{"timestamp":NOW+7199}])); m.insert("/cluster/backup-info/not-backed-up".into(), json!([{"vmid":101},{"vmid":200}])); m.insert("/nodes/pve/tasks".into(), json!([ {"upid":UPID_JOB,"starttime":NOW-600,"endtime":NOW-500,"status":"job errors"}, {"upid":UPID_ONE,"starttime":NOW-9000,"endtime":NOW-8900,"status":"OK"}, {"upid":"UPID:pve:1:2:3:vzdump::root@pam:","starttime":NOW-10}])); m.insert(format!("/nodes/pve/tasks/{}/log", percent_encode(UPID_JOB)), json!([ {"n":1,"t":"INFO: starting new backup job: vzdump 100 101 --storage local"}, {"n":2,"t":"INFO: Starting Backup of VM 100 (qemu)"}, {"n":3,"t":"INFO: Finished Backup of VM 100 (00:00:12)"}, {"n":4,"t":"ERROR: Backup of VM 101 failed - storage full"}])); m.insert(format!("/nodes/pve/tasks/{}/log", percent_encode(UPID_ONE)), json!([ {"n":1,"t":"INFO: Backup job finished successfully"}])); m.insert("/nodes/pve/tasks/UPID%3Apve%3A1%3A2%3A3%3Avzdump%3A%3Aroot%40pam%3A/log".into(), json!([])); Fixture(m) } #[test] fn collects_local_guests_jobs_and_runs() { let observation = block(collect(&fixture(), NOW)); assert_eq!(observation.quality, Quality::Valid); let data = observation.data.unwrap(); assert_eq!(data.node, "pve"); assert_eq!(data.cluster.as_deref(), Some("lab")); assert_eq!(data.guests.iter().map(|g| g.vmid).collect::>(), [100, 101, 9000]); let vm = &data.guests[0]; assert_eq!(vm.macs, ["bc:24:11:aa:bb:01", "bc:24:11:aa:bb:02"]); assert_eq!(vm.smbios_uuid.as_deref(), Some("6e1a2b3c-0000-4000-8000-0000000000aa")); assert_eq!(data.guests[1].macs, ["bc:24:11:aa:bb:03"]); let jobs: BTreeMap<_, _> = data.backup_jobs.iter().map(|j| (j.id.as_str(), j)).collect(); assert_eq!(jobs["backup-a"].covered_vmids, [100]); assert_eq!(jobs["backup-a"].last_expected_run, Some(NOW - 1)); assert!(!jobs["backup-all"].enabled); assert_eq!(jobs["backup-all"].covered_vmids, [101]); assert_eq!(jobs["backup-all"].last_expected_run, None); assert_eq!(jobs["backup-pool"].schedule.as_deref(), Some("mon,tue 02:30")); assert_eq!(jobs["backup-pool"].covered_vmids, [101]); assert!(jobs["backup-other-node"].covered_vmids.is_empty()); assert_eq!(data.not_backed_up, [101]); assert_eq!(data.backup_runs[0].status, "running"); assert_eq!(data.backup_runs[1].status, "error"); assert_eq!(data.backup_runs[1].guests, [GuestResult { vmid: 100, result: "ok" }, GuestResult { vmid: 101, result: "error" }]); assert_eq!(data.backup_runs[2].guests, [GuestResult { vmid: 101, result: "ok" }]); } #[test] fn agent_document_passes_the_server_validator() { let observation = block(collect(&fixture(), NOW)); let data = serde_json::to_value(observation.data.unwrap()).unwrap(); rmm_domain::inventory_document::proxmox::validate(&data).unwrap(); assert!(rmm_domain::inventory_document::proxmox::backup_status(&data).is_some()); } #[test] fn missing_local_node_or_api_is_an_error_observation() { let observation = block(collect(&Fixture(HashMap::new()), NOW)); assert_eq!(observation.quality, Quality::Error); assert_eq!(observation.error_code.as_deref(), Some("PROXMOX_HTTP_404")); assert!(observation.data.is_none()); assert_eq!(not_configured().error_code.as_deref(), Some("PROXMOX_NOT_CONFIGURED")); } #[test] fn oversized_observation_keeps_newest_runs_and_marks_truncation() { let mut data = block(collect(&fixture(), NOW)).data.unwrap(); let guest = data.guests[0].clone(); data.guests = (0..512).map(|i| Guest { vmid: 100 + i, name: "x".repeat(120), ..guest.clone() }).collect(); let observation = fit(Observation::valid(SOURCE, data)); assert!(observation.truncated); assert!(serde_json::to_vec(&observation).unwrap().len() <= OBSERVATION_MAX_BYTES); assert_eq!(observation.data.unwrap().backup_runs.len(), 3); } #[test] fn token_header_rejects_malformed_credentials() { assert_eq!(token_header("rmm-agent@pve!inventory=0f3c-11\n").as_deref(), Some("PVEAPIToken=rmm-agent@pve!inventory=0f3c-11")); for bad in ["", "rmm-agent@pve=abc", "rmm-agent!x=abc", "a@pve!x=a b", "a@pve!x=a\r\nX: y"] { assert!(token_header(bad).is_none(), "{bad}"); } } #[test] fn mac_and_uuid_parsing_is_strict() { assert_eq!(normalize_mac("BC:24:11:0a:0B:0c").as_deref(), Some("bc:24:11:0a:0b:0c")); assert!(normalize_mac("bc:24:11:0a:0b").is_none()); assert!(normalize_mac("00:00:00:00:00:00").is_none()); assert!(smbios_uuid("manufacturer=QEMU").is_none()); assert!(smbios_uuid("uuid=00000000-0000-0000-0000-000000000000").is_none()); } }