//! Strict `linux.proxmox` wire document and the deterministic backup evaluation. //! The agent reports raw scheduler data; whether every scheduled backup ran is //! decided here so the server, API and UI share one rule. use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::collections::{BTreeMap, BTreeSet}; /// Runs that start shortly before the computed due time still count (clock skew). const START_TOLERANCE_SECONDS: i64 = 120; #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Proxmox { pub node: String, pub cluster: Option, pub version: Option, pub guests: Vec, pub backup_jobs: Vec, pub backup_runs: Vec, pub not_backed_up: Vec, } #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Guest { pub vmid: u32, pub name: String, pub kind: String, pub status: String, pub template: bool, pub cpus: Option, pub memory_bytes: Option, pub disk_bytes: Option, pub uptime_seconds: Option, pub pool: Option, pub macs: Vec, pub smbios_uuid: Option, } #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct BackupJob { pub id: String, pub enabled: bool, pub schedule: Option, pub storage: Option, pub mode: Option, pub selection: String, pub covered_vmids: Vec, pub next_run: Option, pub last_expected_run: Option, } #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct BackupRun { pub upid: String, pub started_at: i64, pub ended_at: Option, pub status: String, pub message: Option, pub guests: Vec, } #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct GuestResult { pub vmid: u32, pub result: String, } fn clean(s: &str, limit: usize) -> bool { !s.trim().is_empty() && s.len() <= limit && !s.chars().any(char::is_control) } fn opt(s: &Option, limit: usize) -> bool { s.as_deref().is_none_or(|s| clean(s, limit)) } pub fn is_mac(s: &str) -> bool { s.len() == 17 && s.split(':').count() == 6 && s.split(':').all(|p| p.len() == 2 && p.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) } fn vmid(id: u32) -> bool { (100..1_000_000_000).contains(&id) } /// Validates the `data` member of a valid `linux.proxmox` observation. pub fn validate(data: &Value) -> Result<(), &'static str> { let err = "INVALID_PROXMOX_OBSERVATION"; let p: Proxmox = serde_json::from_value(data.clone()).map_err(|_| err)?; let guests_ok = p.guests.len() <= 512 && p.guests.iter().all(|g| { vmid(g.vmid) && clean(&g.name, 128) && matches!(g.kind.as_str(), "qemu" | "lxc") && clean(&g.status, 32) && opt(&g.pool, 64) && g.macs.len() <= 32 && g.macs.iter().all(|m| is_mac(m)) && g.smbios_uuid.as_deref().is_none_or(|u| uuid::Uuid::parse_str(u).is_ok_and(|x| x.hyphenated().to_string() == u)) }) && p.guests.windows(2).all(|w| w[0].vmid < w[1].vmid); let jobs_ok = p.backup_jobs.len() <= 64 && p.backup_jobs.iter().all(|j| { clean(&j.id, 64) && opt(&j.schedule, 128) && opt(&j.storage, 64) && opt(&j.mode, 16) && matches!(j.selection.as_str(), "all" | "pool" | "vmid") && j.covered_vmids.len() <= 512 && j.covered_vmids.iter().all(|id| vmid(*id)) }); let runs_ok = p.backup_runs.len() <= 64 && p.backup_runs.iter().all(|r| { clean(&r.upid, 160) && r.upid.starts_with("UPID:") && r.started_at > 0 && r.ended_at.is_none_or(|e| e >= r.started_at) && matches!(r.status.as_str(), "ok" | "warning" | "error" | "running") && (r.status == "running") == r.ended_at.is_none() && opt(&r.message, 256) && r.guests.len() <= 512 && r.guests.iter().all(|g| vmid(g.vmid) && matches!(g.result.as_str(), "ok" | "error")) }); if !clean(&p.node, 64) || !opt(&p.cluster, 64) || !opt(&p.version, 32) || !guests_ok || !jobs_ok || !runs_ok || p.not_backed_up.len() > 512 || !p.not_backed_up.iter().all(|id| vmid(*id)) { return Err(err); } Ok(()) } /// Status per job and guest plus the overall answer "did every scheduled backup run?". /// job: ok | failed | missed | running | pending (not yet due) | disabled | unknown pub fn backup_status(data: &Value) -> Option { let p: Proxmox = serde_json::from_value(data.clone()).ok()?; let mut last_success: BTreeMap = BTreeMap::new(); for run in &p.backup_runs { for g in run.guests.iter().filter(|g| g.result == "ok") { let at = run.ended_at.unwrap_or(run.started_at); last_success.entry(g.vmid).and_modify(|t| *t = (*t).max(at)).or_insert(at); } } let mut jobs = Vec::new(); let mut all_ran = true; let mut problems = 0usize; for job in &p.backup_jobs { let mut guests = Vec::new(); let status = if !job.enabled { "disabled" } else if job.schedule.is_none() { "unknown" } else if let Some(due) = job.last_expected_run { let since = due - START_TOLERANCE_SECONDS; let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect(); let running = window.iter().any(|r| r.status == "running"); // vzdump can fail before its first guest (e.g. storage unavailable): the log // then names no guest. Such a run counts as a failure, never as "missed". let failed_before = |id: u32| window.iter().any(|r| r.status == "error" && r.guests.is_empty() && upid_vmid(&r.upid).is_none_or(|v| v == id)); let mut states = BTreeSet::new(); for id in &job.covered_vmids { // Newest statement about this guest inside the due window wins. let result = window.iter() .filter_map(|r| r.guests.iter().find(|g| g.vmid == *id).map(|g| (r.started_at, g.result.as_str()))) .max_by_key(|(at, _)| *at).map(|(_, r)| r); let state = match (result, running) { (Some("ok"), _) => "ok", (Some(_), _) => "failed", (None, true) => "running", (None, false) if failed_before(*id) => "failed", (None, false) => "missed", }; states.insert(state); guests.push(json!({"vmid":id,"status":state,"last_success_at":last_success.get(id)})); } if states.contains("failed") { "failed" } else if states.contains("missed") { "missed" } else if states.contains("running") { "running" } else { "ok" } } else { "pending" }; if matches!(status, "failed" | "missed" | "unknown") { all_ran = false; problems += 1; } jobs.push(json!({"id":job.id,"status":status,"last_expected_run":job.last_expected_run, "next_run":job.next_run,"guests":guests})); } let enabled = p.backup_jobs.iter().filter(|j| j.enabled).count(); Some(json!({ "all_scheduled_backups_ran": if enabled == 0 { Value::Null } else { Value::Bool(all_ran) }, "enabled_jobs": enabled, "problem_jobs": problems, "jobs": jobs, "last_success_at": last_success, })) } /// UPID:node:pid:pstart:starttime:type:id:user: with `id` set for single-guest tasks. fn upid_vmid(upid: &str) -> Option { upid.split(':').nth(6).filter(|id| !id.is_empty()).and_then(|id| id.parse().ok()) } #[cfg(test)] mod tests { use super::*; fn sample() -> Value { json!({"node":"pve","cluster":null,"version":"9.2.2", "guests":[ {"vmid":100,"name":"debian","kind":"qemu","status":"running","template":false,"cpus":1, "memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null, "macs":["bc:24:11:88:35:97"],"smbios_uuid":"787f4ae9-446f-4fac-b3d8-4074170bf823"}, {"vmid":101,"name":"alpine","kind":"lxc","status":"running","template":false,"cpus":1, "memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,"macs":[],"smbios_uuid":null}], "backup_jobs":[ {"id":"a","enabled":true,"schedule":"*/2:00","storage":"local","mode":"snapshot", "selection":"vmid","covered_vmids":[100],"next_run":2000,"last_expected_run":1000}, {"id":"b","enabled":true,"schedule":"daily","storage":"local","mode":"snapshot", "selection":"all","covered_vmids":[100,101],"next_run":5000,"last_expected_run":900}, {"id":"c","enabled":false,"schedule":"daily","storage":null,"mode":null, "selection":"all","covered_vmids":[101],"next_run":null,"last_expected_run":null}, {"id":"d","enabled":true,"schedule":"sat 03:00","storage":null,"mode":null, "selection":"vmid","covered_vmids":[101],"next_run":9000,"last_expected_run":null}], "backup_runs":[ {"upid":"UPID:pve:1:2:3:vzdump::root@pam:","started_at":1001,"ended_at":1050,"status":"error", "message":"job errors","guests":[{"vmid":100,"result":"ok"},{"vmid":101,"result":"error"}]}, {"upid":"UPID:pve:1:2:4:vzdump:100:root@pam:","started_at":100,"ended_at":120,"status":"ok", "message":null,"guests":[{"vmid":100,"result":"ok"}]}], "not_backed_up":[]}) } #[test] fn accepts_agent_shape_and_rejects_unsafe_values() { assert!(validate(&sample()).is_ok()); for (pointer, bad) in [ ("/guests/0/macs/0", json!("BC:24:11:88:35:97")), ("/guests/0/kind", json!("docker")), ("/guests/0/vmid", json!(5)), ("/guests/0/smbios_uuid", json!("not-a-uuid")), ("/backup_runs/0/status", json!("running")), ("/backup_jobs/0/selection", json!("tag")), ("/node", json!("pve\nX")), ] { let mut doc = sample(); *doc.pointer_mut(pointer).unwrap() = bad; assert!(validate(&doc).is_err(), "{pointer}"); } let mut doc = sample(); doc["extra"] = json!(1); assert!(validate(&doc).is_err()); } #[test] fn evaluates_due_runs_per_job_and_guest() { let status = backup_status(&sample()).unwrap(); let jobs: BTreeMap = status["jobs"].as_array().unwrap().iter() .map(|j| (j["id"].as_str().unwrap().to_owned(), j.clone())).collect(); assert_eq!(jobs["a"]["status"], "ok"); assert_eq!(jobs["b"]["status"], "failed"); assert_eq!(jobs["b"]["guests"][1]["status"], "failed"); assert_eq!(jobs["c"]["status"], "disabled"); assert_eq!(jobs["d"]["status"], "pending"); assert_eq!(status["all_scheduled_backups_ran"], false); assert_eq!(status["last_success_at"]["100"], 1050); } /// Observation reported by the Proxmox VE 9.2 test host on 2026-11-20. #[test] fn real_proxmox_92_report_validates_and_counts_the_due_run() { let data: Value = serde_json::from_str(include_str!("proxmox_pve92_fixture.json")).unwrap(); validate(&data).unwrap(); let status = backup_status(&data).unwrap(); assert_eq!(status["jobs"][0]["id"], "backup-debian"); assert_eq!(status["jobs"][0]["status"], "ok"); assert_eq!(status["all_scheduled_backups_ran"], true); assert_eq!(data["not_backed_up"], json!([101])); } /// Same host at 14:00: the scheduled run failed because its storage was disabled. #[test] fn a_run_failing_before_any_guest_is_failed_not_missed() { let data: Value = serde_json::from_str(include_str!("proxmox_pve92_failed_fixture.json")).unwrap(); validate(&data).unwrap(); let status = backup_status(&data).unwrap(); assert_eq!(status["jobs"][0]["status"], "failed"); assert_eq!(status["jobs"][0]["guests"][0]["status"], "failed"); assert_eq!(status["all_scheduled_backups_ran"], false); // The earlier successful backup stays visible as the last success. assert!(status["last_success_at"]["100"].is_number()); // A failed single-guest task of another guest does not fail this job. let mut other = data.clone(); other["backup_runs"][0]["upid"] = json!("UPID:pve:00006CA0:000B1402:6B004453:vzdump:101:root@pam:"); assert_eq!(backup_status(&other).unwrap()["jobs"][0]["status"], "missed"); } #[test] fn missing_and_running_runs_are_distinguished() { let mut doc = sample(); doc["backup_runs"] = json!([]); let status = backup_status(&doc).unwrap(); assert_eq!(status["jobs"][0]["status"], "missed"); doc["backup_runs"] = json!([{"upid":"UPID:pve:9:9:9:vzdump::root@pam:","started_at":1000, "ended_at":null,"status":"running","message":null,"guests":[]}]); assert_eq!(backup_status(&doc).unwrap()["jobs"][0]["status"], "running"); doc["backup_jobs"] = json!([]); assert!(backup_status(&doc).unwrap()["all_scheduled_backups_ran"].is_null()); } }