Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 11:22:53 +01:00
parent 056665be9e
commit c4100d80cd
6 changed files with 412 additions and 8 deletions
+156
View File
@@ -0,0 +1,156 @@
//! Bounded passive runtime diagnostics. These observations are never input to
//! job admission, clock qualification, signed Sync or recovery.
#[cfg(windows)]
use anyhow::{Result, ensure};
use serde::Serialize;
#[cfg(windows)]
use std::{io::Write, path::Path};
#[derive(Clone, Debug, Serialize)]
pub struct Counts {
pub accepted: u64,
pub requesting: u64,
pub running: u64,
pub verifying: u64,
pub unknown: u64,
pub awaiting_reboot: u64,
pub events: u64,
pub legacy_events: u64,
}
#[derive(Clone, Debug, Serialize)]
pub struct Observation {
pub schema_version: u32,
pub observed_at: chrono::DateTime<chrono::Utc>,
pub agent_id: String,
pub boot_id: String,
pub connection_generation: u64,
pub ready: bool,
pub helper_ready: bool,
pub clock_qualified: bool,
pub clock_failure: &'static str,
pub needs_sync: bool,
pub prepared: bool,
pub helper_control: &'static str,
pub helper_apply_unconfirmed: bool,
pub signed_gate_present: bool,
pub signed_ready: bool,
pub local_applied: bool,
pub helper_ack_pending: bool,
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
pub counts: Counts,
}
pub fn clock_code(error: &anyhow::Error) -> &'static str {
const CODES: &[&str] = &[
"TIME_STALE_RESPONSE",
"TIME_NATIVE_NO_EVIDENCE",
"TIME_RUNTIME_NO_EVIDENCE",
"TIME_RUNTIME_NOT_CONFIGURED",
"TIME_NATIVE_EVIDENCE_INVALIDATED",
"TIME_NATIVE_CONNECTION_CHANGED",
"TIME_RUNTIME_CONNECTION_CHANGED",
"TIME_NATIVE_CONFIG_CHANGED",
"TIME_RUNTIME_NO_CONNECTION",
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
"TIME_SOURCE_AGE_AT_CHECK",
];
error
.chain()
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
.unwrap_or("CLOCK_CHECK_FAILED")
}
/// Refresh is control work, not permission. A temporary loss of the100ms clock
/// evidence never discards an otherwise retained signed decision or extends it.
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
pub fn refresh_due(
needs_sync: bool,
elapsed: std::time::Duration,
valid_until: Option<chrono::DateTime<chrono::Utc>>,
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
) -> bool {
needs_sync
|| elapsed >= std::time::Duration::from_secs(40)
|| valid_until.is_none()
|| valid_until
.zip(clock_upper)
.is_some_and(|(until, upper)| upper >= until)
}
#[cfg(windows)]
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
use crate::native;
native::private_path(state, true)?;
let path = state.join("job-readiness.json");
if path.try_exists()? {
native::private_path(&path, false)?;
}
let temporary = state.join("job-readiness.pending");
let raw = serde_json::to_vec(value)?;
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
// A stale pending file is neither overwritten nor treated as current.
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
use windows_sys::Win32::Storage::FileSystem::*;
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
ensure!(
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
&& info.dwFileAttributes
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
== 0
&& info.nNumberOfLinks == 1,
"JOB_READINESS_FILE_ID"
);
Ok([
info.dwVolumeSerialNumber,
info.nFileIndexHigh,
info.nFileIndexLow,
])
}
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)?;
let created = identity(&f)?;
let result = (|| -> Result<()> {
native::private_path(&temporary, false)?;
f.write_all(&raw)?;
f.sync_all()?;
Ok(())
})();
drop(f);
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
if result.is_err() {
// Remove only the exact file this call created, never a replacement,
// reparse point or unknown pending file from a previous invocation.
if let Ok(check) = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)
&& identity(&check).is_ok_and(|value| value == created)
{
let _ = std::fs::remove_file(&temporary);
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = now + chrono::Duration::seconds(30);
let one = std::time::Duration::from_secs(1);
assert!(!refresh_due(false, one, Some(until), None));
assert!(!refresh_due(false, one, Some(until), Some(now)));
assert!(refresh_due(false, one, Some(until), Some(until)));
assert!(refresh_due(false, one, None, None));
assert!(refresh_due(true, one, Some(until), None));
assert!(refresh_due(
false,
std::time::Duration::from_secs(40),
Some(until),
None
));
}
}