From a1b22070de4311d68349419d5268570879e798ee Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Fri, 20 Nov 2026 14:01:29 +0100 Subject: [PATCH] Count a vzdump run that fails before its first guest as a failed backup Found on the Proxmox test host: with the target storage disabled the scheduled run ends with an error and its log names no guest, which was reported as 'missed'. Such a job-level or same-guest run now marks the covered guests failed. Co-Authored-By: Claude Opus 5.5 --- .../domain/src/inventory_document/proxmox.rs | 27 +++++++ .../proxmox_pve92_failed_fixture.json | 80 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 crates/domain/src/inventory_document/proxmox_pve92_failed_fixture.json diff --git a/crates/domain/src/inventory_document/proxmox.rs b/crates/domain/src/inventory_document/proxmox.rs index 84ae7da..64110c9 100644 --- a/crates/domain/src/inventory_document/proxmox.rs +++ b/crates/domain/src/inventory_document/proxmox.rs @@ -139,6 +139,10 @@ pub fn backup_status(data: &Value) -> Option { let since = due - START_TOLERANCE_SECONDS; let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect(); let running = window.iter().any(|r| r.status == "running"); + // vzdump can fail before its first guest (e.g. storage unavailable): the log + // then names no guest. Such a run counts as a failure, never as "missed". + let failed_before = |id: u32| window.iter().any(|r| r.status == "error" && r.guests.is_empty() + && upid_vmid(&r.upid).is_none_or(|v| v == id)); let mut states = BTreeSet::new(); for id in &job.covered_vmids { // Newest statement about this guest inside the due window wins. @@ -149,6 +153,7 @@ pub fn backup_status(data: &Value) -> Option { (Some("ok"), _) => "ok", (Some(_), _) => "failed", (None, true) => "running", + (None, false) if failed_before(*id) => "failed", (None, false) => "missed", }; states.insert(state); @@ -178,6 +183,11 @@ pub fn backup_status(data: &Value) -> Option { })) } +/// UPID:node:pid:pstart:starttime:type:id:user: with `id` set for single-guest tasks. +fn upid_vmid(upid: &str) -> Option { + upid.split(':').nth(6).filter(|id| !id.is_empty()).and_then(|id| id.parse().ok()) +} + #[cfg(test)] mod tests { use super::*; @@ -254,6 +264,23 @@ mod tests { assert_eq!(data["not_backed_up"], json!([101])); } + /// Same host at 14:00: the scheduled run failed because its storage was disabled. + #[test] + fn a_run_failing_before_any_guest_is_failed_not_missed() { + let data: Value = serde_json::from_str(include_str!("proxmox_pve92_failed_fixture.json")).unwrap(); + validate(&data).unwrap(); + let status = backup_status(&data).unwrap(); + assert_eq!(status["jobs"][0]["status"], "failed"); + assert_eq!(status["jobs"][0]["guests"][0]["status"], "failed"); + assert_eq!(status["all_scheduled_backups_ran"], false); + // The earlier successful backup stays visible as the last success. + assert!(status["last_success_at"]["100"].is_number()); + // A failed single-guest task of another guest does not fail this job. + let mut other = data.clone(); + other["backup_runs"][0]["upid"] = json!("UPID:pve:00006CA0:000B1402:6B004453:vzdump:101:root@pam:"); + assert_eq!(backup_status(&other).unwrap()["jobs"][0]["status"], "missed"); + } + #[test] fn missing_and_running_runs_are_distinguished() { let mut doc = sample(); diff --git a/crates/domain/src/inventory_document/proxmox_pve92_failed_fixture.json b/crates/domain/src/inventory_document/proxmox_pve92_failed_fixture.json new file mode 100644 index 0000000..8a2e8e9 --- /dev/null +++ b/crates/domain/src/inventory_document/proxmox_pve92_failed_fixture.json @@ -0,0 +1,80 @@ +{ + "backup_jobs": [ + { + "covered_vmids": [ + 100 + ], + "enabled": true, + "id": "backup-debian", + "last_expected_run": 1795179600, + "mode": "snapshot", + "next_run": 1795186800, + "schedule": "*/2:00", + "selection": "vmid", + "storage": "local" + } + ], + "backup_runs": [ + { + "ended_at": 1795179610, + "guests": [], + "message": "could not activate storage 'local': storage 'local' is disabled", + "started_at": 1795179603, + "status": "error", + "upid": "UPID:pve:00006CA0:000B1402:6B004453:vzdump:100:root@pam:" + }, + { + "ended_at": 1795172488, + "guests": [ + { + "result": "ok", + "vmid": 100 + } + ], + "message": null, + "started_at": 1795172468, + "status": "ok", + "upid": "UPID:pve:00000C75:000030ED:6B002874:vzdump:100:root@pam:" + } + ], + "cluster": null, + "guests": [ + { + "cpus": 1, + "disk_bytes": 8589934592, + "kind": "qemu", + "macs": [ + "bc:24:11:88:35:97" + ], + "memory_bytes": 1073741824, + "name": "debian-guest", + "pool": null, + "smbios_uuid": "787f4ae9-446f-4fac-b3d8-4074170bf823", + "status": "running", + "template": false, + "uptime_seconds": 7166, + "vmid": 100 + }, + { + "cpus": 1, + "disk_bytes": 1020702720, + "kind": "lxc", + "macs": [ + "bc:24:11:b7:4c:99" + ], + "memory_bytes": 134217728, + "name": "alpine-ct", + "pool": null, + "smbios_uuid": null, + "status": "running", + "template": false, + "uptime_seconds": 7164, + "vmid": 101 + } + ], + "node": "pve", + "not_backed_up": [ + 101 + ], + "version": "9.2.2" +} \ No newline at end of file