Count a vzdump run that fails before its first guest as a failed backup
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
Found on the Proxmox test host: with the target storage disabled the scheduled run ends with an error and its log names no guest, which was reported as 'missed'. Such a job-level or same-guest run now marks the covered guests failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -139,6 +139,10 @@ pub fn backup_status(data: &Value) -> Option<Value> {
|
||||
let since = due - START_TOLERANCE_SECONDS;
|
||||
let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect();
|
||||
let running = window.iter().any(|r| r.status == "running");
|
||||
// vzdump can fail before its first guest (e.g. storage unavailable): the log
|
||||
// then names no guest. Such a run counts as a failure, never as "missed".
|
||||
let failed_before = |id: u32| window.iter().any(|r| r.status == "error" && r.guests.is_empty()
|
||||
&& upid_vmid(&r.upid).is_none_or(|v| v == id));
|
||||
let mut states = BTreeSet::new();
|
||||
for id in &job.covered_vmids {
|
||||
// Newest statement about this guest inside the due window wins.
|
||||
@@ -149,6 +153,7 @@ pub fn backup_status(data: &Value) -> Option<Value> {
|
||||
(Some("ok"), _) => "ok",
|
||||
(Some(_), _) => "failed",
|
||||
(None, true) => "running",
|
||||
(None, false) if failed_before(*id) => "failed",
|
||||
(None, false) => "missed",
|
||||
};
|
||||
states.insert(state);
|
||||
@@ -178,6 +183,11 @@ pub fn backup_status(data: &Value) -> Option<Value> {
|
||||
}))
|
||||
}
|
||||
|
||||
/// UPID:node:pid:pstart:starttime:type:id:user: with `id` set for single-guest tasks.
|
||||
fn upid_vmid(upid: &str) -> Option<u32> {
|
||||
upid.split(':').nth(6).filter(|id| !id.is_empty()).and_then(|id| id.parse().ok())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -254,6 +264,23 @@ mod tests {
|
||||
assert_eq!(data["not_backed_up"], json!([101]));
|
||||
}
|
||||
|
||||
/// Same host at 14:00: the scheduled run failed because its storage was disabled.
|
||||
#[test]
|
||||
fn a_run_failing_before_any_guest_is_failed_not_missed() {
|
||||
let data: Value = serde_json::from_str(include_str!("proxmox_pve92_failed_fixture.json")).unwrap();
|
||||
validate(&data).unwrap();
|
||||
let status = backup_status(&data).unwrap();
|
||||
assert_eq!(status["jobs"][0]["status"], "failed");
|
||||
assert_eq!(status["jobs"][0]["guests"][0]["status"], "failed");
|
||||
assert_eq!(status["all_scheduled_backups_ran"], false);
|
||||
// The earlier successful backup stays visible as the last success.
|
||||
assert!(status["last_success_at"]["100"].is_number());
|
||||
// A failed single-guest task of another guest does not fail this job.
|
||||
let mut other = data.clone();
|
||||
other["backup_runs"][0]["upid"] = json!("UPID:pve:00006CA0:000B1402:6B004453:vzdump:101:root@pam:");
|
||||
assert_eq!(backup_status(&other).unwrap()["jobs"][0]["status"], "missed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_and_running_runs_are_distinguished() {
|
||||
let mut doc = sample();
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"backup_jobs": [
|
||||
{
|
||||
"covered_vmids": [
|
||||
100
|
||||
],
|
||||
"enabled": true,
|
||||
"id": "backup-debian",
|
||||
"last_expected_run": 1795179600,
|
||||
"mode": "snapshot",
|
||||
"next_run": 1795186800,
|
||||
"schedule": "*/2:00",
|
||||
"selection": "vmid",
|
||||
"storage": "local"
|
||||
}
|
||||
],
|
||||
"backup_runs": [
|
||||
{
|
||||
"ended_at": 1795179610,
|
||||
"guests": [],
|
||||
"message": "could not activate storage 'local': storage 'local' is disabled",
|
||||
"started_at": 1795179603,
|
||||
"status": "error",
|
||||
"upid": "UPID:pve:00006CA0:000B1402:6B004453:vzdump:100:root@pam:"
|
||||
},
|
||||
{
|
||||
"ended_at": 1795172488,
|
||||
"guests": [
|
||||
{
|
||||
"result": "ok",
|
||||
"vmid": 100
|
||||
}
|
||||
],
|
||||
"message": null,
|
||||
"started_at": 1795172468,
|
||||
"status": "ok",
|
||||
"upid": "UPID:pve:00000C75:000030ED:6B002874:vzdump:100:root@pam:"
|
||||
}
|
||||
],
|
||||
"cluster": null,
|
||||
"guests": [
|
||||
{
|
||||
"cpus": 1,
|
||||
"disk_bytes": 8589934592,
|
||||
"kind": "qemu",
|
||||
"macs": [
|
||||
"bc:24:11:88:35:97"
|
||||
],
|
||||
"memory_bytes": 1073741824,
|
||||
"name": "debian-guest",
|
||||
"pool": null,
|
||||
"smbios_uuid": "787f4ae9-446f-4fac-b3d8-4074170bf823",
|
||||
"status": "running",
|
||||
"template": false,
|
||||
"uptime_seconds": 7166,
|
||||
"vmid": 100
|
||||
},
|
||||
{
|
||||
"cpus": 1,
|
||||
"disk_bytes": 1020702720,
|
||||
"kind": "lxc",
|
||||
"macs": [
|
||||
"bc:24:11:b7:4c:99"
|
||||
],
|
||||
"memory_bytes": 134217728,
|
||||
"name": "alpine-ct",
|
||||
"pool": null,
|
||||
"smbios_uuid": null,
|
||||
"status": "running",
|
||||
"template": false,
|
||||
"uptime_seconds": 7164,
|
||||
"vmid": 101
|
||||
}
|
||||
],
|
||||
"node": "pve",
|
||||
"not_backed_up": [
|
||||
101
|
||||
],
|
||||
"version": "9.2.2"
|
||||
}
|
||||
Reference in New Issue
Block a user