Count a vzdump run that fails before its first guest as a failed backup
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
Found on the Proxmox test host: with the target storage disabled the scheduled run ends with an error and its log names no guest, which was reported as 'missed'. Such a job-level or same-guest run now marks the covered guests failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -139,6 +139,10 @@ pub fn backup_status(data: &Value) -> Option<Value> {
|
|||||||
let since = due - START_TOLERANCE_SECONDS;
|
let since = due - START_TOLERANCE_SECONDS;
|
||||||
let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect();
|
let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect();
|
||||||
let running = window.iter().any(|r| r.status == "running");
|
let running = window.iter().any(|r| r.status == "running");
|
||||||
|
// vzdump can fail before its first guest (e.g. storage unavailable): the log
|
||||||
|
// then names no guest. Such a run counts as a failure, never as "missed".
|
||||||
|
let failed_before = |id: u32| window.iter().any(|r| r.status == "error" && r.guests.is_empty()
|
||||||
|
&& upid_vmid(&r.upid).is_none_or(|v| v == id));
|
||||||
let mut states = BTreeSet::new();
|
let mut states = BTreeSet::new();
|
||||||
for id in &job.covered_vmids {
|
for id in &job.covered_vmids {
|
||||||
// Newest statement about this guest inside the due window wins.
|
// Newest statement about this guest inside the due window wins.
|
||||||
@@ -149,6 +153,7 @@ pub fn backup_status(data: &Value) -> Option<Value> {
|
|||||||
(Some("ok"), _) => "ok",
|
(Some("ok"), _) => "ok",
|
||||||
(Some(_), _) => "failed",
|
(Some(_), _) => "failed",
|
||||||
(None, true) => "running",
|
(None, true) => "running",
|
||||||
|
(None, false) if failed_before(*id) => "failed",
|
||||||
(None, false) => "missed",
|
(None, false) => "missed",
|
||||||
};
|
};
|
||||||
states.insert(state);
|
states.insert(state);
|
||||||
@@ -178,6 +183,11 @@ pub fn backup_status(data: &Value) -> Option<Value> {
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// UPID:node:pid:pstart:starttime:type:id:user: with `id` set for single-guest tasks.
|
||||||
|
fn upid_vmid(upid: &str) -> Option<u32> {
|
||||||
|
upid.split(':').nth(6).filter(|id| !id.is_empty()).and_then(|id| id.parse().ok())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -254,6 +264,23 @@ mod tests {
|
|||||||
assert_eq!(data["not_backed_up"], json!([101]));
|
assert_eq!(data["not_backed_up"], json!([101]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Same host at 14:00: the scheduled run failed because its storage was disabled.
|
||||||
|
#[test]
|
||||||
|
fn a_run_failing_before_any_guest_is_failed_not_missed() {
|
||||||
|
let data: Value = serde_json::from_str(include_str!("proxmox_pve92_failed_fixture.json")).unwrap();
|
||||||
|
validate(&data).unwrap();
|
||||||
|
let status = backup_status(&data).unwrap();
|
||||||
|
assert_eq!(status["jobs"][0]["status"], "failed");
|
||||||
|
assert_eq!(status["jobs"][0]["guests"][0]["status"], "failed");
|
||||||
|
assert_eq!(status["all_scheduled_backups_ran"], false);
|
||||||
|
// The earlier successful backup stays visible as the last success.
|
||||||
|
assert!(status["last_success_at"]["100"].is_number());
|
||||||
|
// A failed single-guest task of another guest does not fail this job.
|
||||||
|
let mut other = data.clone();
|
||||||
|
other["backup_runs"][0]["upid"] = json!("UPID:pve:00006CA0:000B1402:6B004453:vzdump:101:root@pam:");
|
||||||
|
assert_eq!(backup_status(&other).unwrap()["jobs"][0]["status"], "missed");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn missing_and_running_runs_are_distinguished() {
|
fn missing_and_running_runs_are_distinguished() {
|
||||||
let mut doc = sample();
|
let mut doc = sample();
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
{
|
||||||
|
"backup_jobs": [
|
||||||
|
{
|
||||||
|
"covered_vmids": [
|
||||||
|
100
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"id": "backup-debian",
|
||||||
|
"last_expected_run": 1795179600,
|
||||||
|
"mode": "snapshot",
|
||||||
|
"next_run": 1795186800,
|
||||||
|
"schedule": "*/2:00",
|
||||||
|
"selection": "vmid",
|
||||||
|
"storage": "local"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"backup_runs": [
|
||||||
|
{
|
||||||
|
"ended_at": 1795179610,
|
||||||
|
"guests": [],
|
||||||
|
"message": "could not activate storage 'local': storage 'local' is disabled",
|
||||||
|
"started_at": 1795179603,
|
||||||
|
"status": "error",
|
||||||
|
"upid": "UPID:pve:00006CA0:000B1402:6B004453:vzdump:100:root@pam:"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ended_at": 1795172488,
|
||||||
|
"guests": [
|
||||||
|
{
|
||||||
|
"result": "ok",
|
||||||
|
"vmid": 100
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"message": null,
|
||||||
|
"started_at": 1795172468,
|
||||||
|
"status": "ok",
|
||||||
|
"upid": "UPID:pve:00000C75:000030ED:6B002874:vzdump:100:root@pam:"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"cluster": null,
|
||||||
|
"guests": [
|
||||||
|
{
|
||||||
|
"cpus": 1,
|
||||||
|
"disk_bytes": 8589934592,
|
||||||
|
"kind": "qemu",
|
||||||
|
"macs": [
|
||||||
|
"bc:24:11:88:35:97"
|
||||||
|
],
|
||||||
|
"memory_bytes": 1073741824,
|
||||||
|
"name": "debian-guest",
|
||||||
|
"pool": null,
|
||||||
|
"smbios_uuid": "787f4ae9-446f-4fac-b3d8-4074170bf823",
|
||||||
|
"status": "running",
|
||||||
|
"template": false,
|
||||||
|
"uptime_seconds": 7166,
|
||||||
|
"vmid": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cpus": 1,
|
||||||
|
"disk_bytes": 1020702720,
|
||||||
|
"kind": "lxc",
|
||||||
|
"macs": [
|
||||||
|
"bc:24:11:b7:4c:99"
|
||||||
|
],
|
||||||
|
"memory_bytes": 134217728,
|
||||||
|
"name": "alpine-ct",
|
||||||
|
"pool": null,
|
||||||
|
"smbios_uuid": null,
|
||||||
|
"status": "running",
|
||||||
|
"template": false,
|
||||||
|
"uptime_seconds": 7164,
|
||||||
|
"vmid": 101
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"node": "pve",
|
||||||
|
"not_backed_up": [
|
||||||
|
101
|
||||||
|
],
|
||||||
|
"version": "9.2.2"
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user