Rust development slice / linux (push) Failing after 7s
Found on the Proxmox test host: with the target storage disabled the scheduled run ends with an error and its log names no guest, which was reported as 'missed'. Such a job-level or same-guest run now marks the covered guests failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
297 lines
13 KiB
Rust
297 lines
13 KiB
Rust
//! Strict `linux.proxmox` wire document and the deterministic backup evaluation.
|
|
//! The agent reports raw scheduler data; whether every scheduled backup ran is
|
|
//! decided here so the server, API and UI share one rule.
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::{Value, json};
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
/// Runs that start shortly before the computed due time still count (clock skew).
|
|
const START_TOLERANCE_SECONDS: i64 = 120;
|
|
|
|
#[derive(Deserialize, Serialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct Proxmox {
|
|
pub node: String,
|
|
pub cluster: Option<String>,
|
|
pub version: Option<String>,
|
|
pub guests: Vec<Guest>,
|
|
pub backup_jobs: Vec<BackupJob>,
|
|
pub backup_runs: Vec<BackupRun>,
|
|
pub not_backed_up: Vec<u32>,
|
|
}
|
|
|
|
#[derive(Deserialize, Serialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct Guest {
|
|
pub vmid: u32,
|
|
pub name: String,
|
|
pub kind: String,
|
|
pub status: String,
|
|
pub template: bool,
|
|
pub cpus: Option<u32>,
|
|
pub memory_bytes: Option<u64>,
|
|
pub disk_bytes: Option<u64>,
|
|
pub uptime_seconds: Option<u64>,
|
|
pub pool: Option<String>,
|
|
pub macs: Vec<String>,
|
|
pub smbios_uuid: Option<String>,
|
|
}
|
|
|
|
#[derive(Deserialize, Serialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BackupJob {
|
|
pub id: String,
|
|
pub enabled: bool,
|
|
pub schedule: Option<String>,
|
|
pub storage: Option<String>,
|
|
pub mode: Option<String>,
|
|
pub selection: String,
|
|
pub covered_vmids: Vec<u32>,
|
|
pub next_run: Option<i64>,
|
|
pub last_expected_run: Option<i64>,
|
|
}
|
|
|
|
#[derive(Deserialize, Serialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BackupRun {
|
|
pub upid: String,
|
|
pub started_at: i64,
|
|
pub ended_at: Option<i64>,
|
|
pub status: String,
|
|
pub message: Option<String>,
|
|
pub guests: Vec<GuestResult>,
|
|
}
|
|
|
|
#[derive(Deserialize, Serialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct GuestResult {
|
|
pub vmid: u32,
|
|
pub result: String,
|
|
}
|
|
|
|
fn clean(s: &str, limit: usize) -> bool {
|
|
!s.trim().is_empty() && s.len() <= limit && !s.chars().any(char::is_control)
|
|
}
|
|
fn opt(s: &Option<String>, limit: usize) -> bool {
|
|
s.as_deref().is_none_or(|s| clean(s, limit))
|
|
}
|
|
pub fn is_mac(s: &str) -> bool {
|
|
s.len() == 17 && s.split(':').count() == 6
|
|
&& s.split(':').all(|p| p.len() == 2 && p.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
|
|
}
|
|
fn vmid(id: u32) -> bool {
|
|
(100..1_000_000_000).contains(&id)
|
|
}
|
|
|
|
/// Validates the `data` member of a valid `linux.proxmox` observation.
|
|
pub fn validate(data: &Value) -> Result<(), &'static str> {
|
|
let err = "INVALID_PROXMOX_OBSERVATION";
|
|
let p: Proxmox = serde_json::from_value(data.clone()).map_err(|_| err)?;
|
|
let guests_ok = p.guests.len() <= 512 && p.guests.iter().all(|g| {
|
|
vmid(g.vmid) && clean(&g.name, 128) && matches!(g.kind.as_str(), "qemu" | "lxc")
|
|
&& clean(&g.status, 32) && opt(&g.pool, 64) && g.macs.len() <= 32
|
|
&& g.macs.iter().all(|m| is_mac(m))
|
|
&& g.smbios_uuid.as_deref().is_none_or(|u| uuid::Uuid::parse_str(u).is_ok_and(|x| x.hyphenated().to_string() == u))
|
|
}) && p.guests.windows(2).all(|w| w[0].vmid < w[1].vmid);
|
|
let jobs_ok = p.backup_jobs.len() <= 64 && p.backup_jobs.iter().all(|j| {
|
|
clean(&j.id, 64) && opt(&j.schedule, 128) && opt(&j.storage, 64) && opt(&j.mode, 16)
|
|
&& matches!(j.selection.as_str(), "all" | "pool" | "vmid")
|
|
&& j.covered_vmids.len() <= 512 && j.covered_vmids.iter().all(|id| vmid(*id))
|
|
});
|
|
let runs_ok = p.backup_runs.len() <= 64 && p.backup_runs.iter().all(|r| {
|
|
clean(&r.upid, 160) && r.upid.starts_with("UPID:") && r.started_at > 0
|
|
&& r.ended_at.is_none_or(|e| e >= r.started_at)
|
|
&& matches!(r.status.as_str(), "ok" | "warning" | "error" | "running")
|
|
&& (r.status == "running") == r.ended_at.is_none()
|
|
&& opt(&r.message, 256) && r.guests.len() <= 512
|
|
&& r.guests.iter().all(|g| vmid(g.vmid) && matches!(g.result.as_str(), "ok" | "error"))
|
|
});
|
|
if !clean(&p.node, 64) || !opt(&p.cluster, 64) || !opt(&p.version, 32)
|
|
|| !guests_ok || !jobs_ok || !runs_ok
|
|
|| p.not_backed_up.len() > 512 || !p.not_backed_up.iter().all(|id| vmid(*id))
|
|
{
|
|
return Err(err);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Status per job and guest plus the overall answer "did every scheduled backup run?".
|
|
/// job: ok | failed | missed | running | pending (not yet due) | disabled | unknown
|
|
pub fn backup_status(data: &Value) -> Option<Value> {
|
|
let p: Proxmox = serde_json::from_value(data.clone()).ok()?;
|
|
let mut last_success: BTreeMap<u32, i64> = BTreeMap::new();
|
|
for run in &p.backup_runs {
|
|
for g in run.guests.iter().filter(|g| g.result == "ok") {
|
|
let at = run.ended_at.unwrap_or(run.started_at);
|
|
last_success.entry(g.vmid).and_modify(|t| *t = (*t).max(at)).or_insert(at);
|
|
}
|
|
}
|
|
let mut jobs = Vec::new();
|
|
let mut all_ran = true;
|
|
let mut problems = 0usize;
|
|
for job in &p.backup_jobs {
|
|
let mut guests = Vec::new();
|
|
let status = if !job.enabled {
|
|
"disabled"
|
|
} else if job.schedule.is_none() {
|
|
"unknown"
|
|
} else if let Some(due) = job.last_expected_run {
|
|
let since = due - START_TOLERANCE_SECONDS;
|
|
let window: Vec<&BackupRun> = p.backup_runs.iter().filter(|r| r.started_at >= since).collect();
|
|
let running = window.iter().any(|r| r.status == "running");
|
|
// vzdump can fail before its first guest (e.g. storage unavailable): the log
|
|
// then names no guest. Such a run counts as a failure, never as "missed".
|
|
let failed_before = |id: u32| window.iter().any(|r| r.status == "error" && r.guests.is_empty()
|
|
&& upid_vmid(&r.upid).is_none_or(|v| v == id));
|
|
let mut states = BTreeSet::new();
|
|
for id in &job.covered_vmids {
|
|
// Newest statement about this guest inside the due window wins.
|
|
let result = window.iter()
|
|
.filter_map(|r| r.guests.iter().find(|g| g.vmid == *id).map(|g| (r.started_at, g.result.as_str())))
|
|
.max_by_key(|(at, _)| *at).map(|(_, r)| r);
|
|
let state = match (result, running) {
|
|
(Some("ok"), _) => "ok",
|
|
(Some(_), _) => "failed",
|
|
(None, true) => "running",
|
|
(None, false) if failed_before(*id) => "failed",
|
|
(None, false) => "missed",
|
|
};
|
|
states.insert(state);
|
|
guests.push(json!({"vmid":id,"status":state,"last_success_at":last_success.get(id)}));
|
|
}
|
|
if states.contains("failed") { "failed" }
|
|
else if states.contains("missed") { "missed" }
|
|
else if states.contains("running") { "running" }
|
|
else { "ok" }
|
|
} else {
|
|
"pending"
|
|
};
|
|
if matches!(status, "failed" | "missed" | "unknown") {
|
|
all_ran = false;
|
|
problems += 1;
|
|
}
|
|
jobs.push(json!({"id":job.id,"status":status,"last_expected_run":job.last_expected_run,
|
|
"next_run":job.next_run,"guests":guests}));
|
|
}
|
|
let enabled = p.backup_jobs.iter().filter(|j| j.enabled).count();
|
|
Some(json!({
|
|
"all_scheduled_backups_ran": if enabled == 0 { Value::Null } else { Value::Bool(all_ran) },
|
|
"enabled_jobs": enabled,
|
|
"problem_jobs": problems,
|
|
"jobs": jobs,
|
|
"last_success_at": last_success,
|
|
}))
|
|
}
|
|
|
|
/// UPID:node:pid:pstart:starttime:type:id:user: with `id` set for single-guest tasks.
|
|
fn upid_vmid(upid: &str) -> Option<u32> {
|
|
upid.split(':').nth(6).filter(|id| !id.is_empty()).and_then(|id| id.parse().ok())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn sample() -> Value {
|
|
json!({"node":"pve","cluster":null,"version":"9.2.2",
|
|
"guests":[
|
|
{"vmid":100,"name":"debian","kind":"qemu","status":"running","template":false,"cpus":1,
|
|
"memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,
|
|
"macs":["bc:24:11:88:35:97"],"smbios_uuid":"787f4ae9-446f-4fac-b3d8-4074170bf823"},
|
|
{"vmid":101,"name":"alpine","kind":"lxc","status":"running","template":false,"cpus":1,
|
|
"memory_bytes":1,"disk_bytes":1,"uptime_seconds":5,"pool":null,"macs":[],"smbios_uuid":null}],
|
|
"backup_jobs":[
|
|
{"id":"a","enabled":true,"schedule":"*/2:00","storage":"local","mode":"snapshot",
|
|
"selection":"vmid","covered_vmids":[100],"next_run":2000,"last_expected_run":1000},
|
|
{"id":"b","enabled":true,"schedule":"daily","storage":"local","mode":"snapshot",
|
|
"selection":"all","covered_vmids":[100,101],"next_run":5000,"last_expected_run":900},
|
|
{"id":"c","enabled":false,"schedule":"daily","storage":null,"mode":null,
|
|
"selection":"all","covered_vmids":[101],"next_run":null,"last_expected_run":null},
|
|
{"id":"d","enabled":true,"schedule":"sat 03:00","storage":null,"mode":null,
|
|
"selection":"vmid","covered_vmids":[101],"next_run":9000,"last_expected_run":null}],
|
|
"backup_runs":[
|
|
{"upid":"UPID:pve:1:2:3:vzdump::root@pam:","started_at":1001,"ended_at":1050,"status":"error",
|
|
"message":"job errors","guests":[{"vmid":100,"result":"ok"},{"vmid":101,"result":"error"}]},
|
|
{"upid":"UPID:pve:1:2:4:vzdump:100:root@pam:","started_at":100,"ended_at":120,"status":"ok",
|
|
"message":null,"guests":[{"vmid":100,"result":"ok"}]}],
|
|
"not_backed_up":[]})
|
|
}
|
|
|
|
#[test]
|
|
fn accepts_agent_shape_and_rejects_unsafe_values() {
|
|
assert!(validate(&sample()).is_ok());
|
|
for (pointer, bad) in [
|
|
("/guests/0/macs/0", json!("BC:24:11:88:35:97")),
|
|
("/guests/0/kind", json!("docker")),
|
|
("/guests/0/vmid", json!(5)),
|
|
("/guests/0/smbios_uuid", json!("not-a-uuid")),
|
|
("/backup_runs/0/status", json!("running")),
|
|
("/backup_jobs/0/selection", json!("tag")),
|
|
("/node", json!("pve\nX")),
|
|
] {
|
|
let mut doc = sample();
|
|
*doc.pointer_mut(pointer).unwrap() = bad;
|
|
assert!(validate(&doc).is_err(), "{pointer}");
|
|
}
|
|
let mut doc = sample();
|
|
doc["extra"] = json!(1);
|
|
assert!(validate(&doc).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn evaluates_due_runs_per_job_and_guest() {
|
|
let status = backup_status(&sample()).unwrap();
|
|
let jobs: BTreeMap<String, Value> = status["jobs"].as_array().unwrap().iter()
|
|
.map(|j| (j["id"].as_str().unwrap().to_owned(), j.clone())).collect();
|
|
assert_eq!(jobs["a"]["status"], "ok");
|
|
assert_eq!(jobs["b"]["status"], "failed");
|
|
assert_eq!(jobs["b"]["guests"][1]["status"], "failed");
|
|
assert_eq!(jobs["c"]["status"], "disabled");
|
|
assert_eq!(jobs["d"]["status"], "pending");
|
|
assert_eq!(status["all_scheduled_backups_ran"], false);
|
|
assert_eq!(status["last_success_at"]["100"], 1050);
|
|
}
|
|
|
|
/// Observation reported by the Proxmox VE 9.2 test host on 2026-11-20.
|
|
#[test]
|
|
fn real_proxmox_92_report_validates_and_counts_the_due_run() {
|
|
let data: Value = serde_json::from_str(include_str!("proxmox_pve92_fixture.json")).unwrap();
|
|
validate(&data).unwrap();
|
|
let status = backup_status(&data).unwrap();
|
|
assert_eq!(status["jobs"][0]["id"], "backup-debian");
|
|
assert_eq!(status["jobs"][0]["status"], "ok");
|
|
assert_eq!(status["all_scheduled_backups_ran"], true);
|
|
assert_eq!(data["not_backed_up"], json!([101]));
|
|
}
|
|
|
|
/// Same host at 14:00: the scheduled run failed because its storage was disabled.
|
|
#[test]
|
|
fn a_run_failing_before_any_guest_is_failed_not_missed() {
|
|
let data: Value = serde_json::from_str(include_str!("proxmox_pve92_failed_fixture.json")).unwrap();
|
|
validate(&data).unwrap();
|
|
let status = backup_status(&data).unwrap();
|
|
assert_eq!(status["jobs"][0]["status"], "failed");
|
|
assert_eq!(status["jobs"][0]["guests"][0]["status"], "failed");
|
|
assert_eq!(status["all_scheduled_backups_ran"], false);
|
|
// The earlier successful backup stays visible as the last success.
|
|
assert!(status["last_success_at"]["100"].is_number());
|
|
// A failed single-guest task of another guest does not fail this job.
|
|
let mut other = data.clone();
|
|
other["backup_runs"][0]["upid"] = json!("UPID:pve:00006CA0:000B1402:6B004453:vzdump:101:root@pam:");
|
|
assert_eq!(backup_status(&other).unwrap()["jobs"][0]["status"], "missed");
|
|
}
|
|
|
|
#[test]
|
|
fn missing_and_running_runs_are_distinguished() {
|
|
let mut doc = sample();
|
|
doc["backup_runs"] = json!([]);
|
|
let status = backup_status(&doc).unwrap();
|
|
assert_eq!(status["jobs"][0]["status"], "missed");
|
|
doc["backup_runs"] = json!([{"upid":"UPID:pve:9:9:9:vzdump::root@pam:","started_at":1000,
|
|
"ended_at":null,"status":"running","message":null,"guests":[]}]);
|
|
assert_eq!(backup_status(&doc).unwrap()["jobs"][0]["status"], "running");
|
|
doc["backup_jobs"] = json!([]);
|
|
assert!(backup_status(&doc).unwrap()["all_scheduled_backups_ran"].is_null());
|
|
}
|
|
}
|