Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
//! Bounded passive runtime diagnostics. These observations are never input to
|
||||
//! job admission, clock qualification, signed Sync or recovery.
|
||||
#[cfg(windows)]
|
||||
use anyhow::{Result, ensure};
|
||||
use serde::Serialize;
|
||||
#[cfg(windows)]
|
||||
use std::{io::Write, path::Path};
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Counts {
|
||||
pub accepted: u64,
|
||||
pub requesting: u64,
|
||||
pub running: u64,
|
||||
pub verifying: u64,
|
||||
pub unknown: u64,
|
||||
pub awaiting_reboot: u64,
|
||||
pub events: u64,
|
||||
pub legacy_events: u64,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Observation {
|
||||
pub schema_version: u32,
|
||||
pub observed_at: chrono::DateTime<chrono::Utc>,
|
||||
pub agent_id: String,
|
||||
pub boot_id: String,
|
||||
pub connection_generation: u64,
|
||||
pub ready: bool,
|
||||
pub helper_ready: bool,
|
||||
pub clock_qualified: bool,
|
||||
pub clock_failure: &'static str,
|
||||
pub needs_sync: bool,
|
||||
pub prepared: bool,
|
||||
pub helper_control: &'static str,
|
||||
pub helper_apply_unconfirmed: bool,
|
||||
pub signed_gate_present: bool,
|
||||
pub signed_ready: bool,
|
||||
pub local_applied: bool,
|
||||
pub helper_ack_pending: bool,
|
||||
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub counts: Counts,
|
||||
}
|
||||
pub fn clock_code(error: &anyhow::Error) -> &'static str {
|
||||
const CODES: &[&str] = &[
|
||||
"TIME_STALE_RESPONSE",
|
||||
"TIME_NATIVE_NO_EVIDENCE",
|
||||
"TIME_RUNTIME_NO_EVIDENCE",
|
||||
"TIME_RUNTIME_NOT_CONFIGURED",
|
||||
"TIME_NATIVE_EVIDENCE_INVALIDATED",
|
||||
"TIME_NATIVE_CONNECTION_CHANGED",
|
||||
"TIME_RUNTIME_CONNECTION_CHANGED",
|
||||
"TIME_NATIVE_CONFIG_CHANGED",
|
||||
"TIME_RUNTIME_NO_CONNECTION",
|
||||
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
|
||||
"TIME_SOURCE_AGE_AT_CHECK",
|
||||
];
|
||||
error
|
||||
.chain()
|
||||
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
|
||||
.unwrap_or("CLOCK_CHECK_FAILED")
|
||||
}
|
||||
/// Refresh is control work, not permission. A temporary loss of the100ms clock
|
||||
/// evidence never discards an otherwise retained signed decision or extends it.
|
||||
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
|
||||
pub fn refresh_due(
|
||||
needs_sync: bool,
|
||||
elapsed: std::time::Duration,
|
||||
valid_until: Option<chrono::DateTime<chrono::Utc>>,
|
||||
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
|
||||
) -> bool {
|
||||
needs_sync
|
||||
|| elapsed >= std::time::Duration::from_secs(40)
|
||||
|| valid_until.is_none()
|
||||
|| valid_until
|
||||
.zip(clock_upper)
|
||||
.is_some_and(|(until, upper)| upper >= until)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
|
||||
use crate::native;
|
||||
native::private_path(state, true)?;
|
||||
let path = state.join("job-readiness.json");
|
||||
if path.try_exists()? {
|
||||
native::private_path(&path, false)?;
|
||||
}
|
||||
let temporary = state.join("job-readiness.pending");
|
||||
let raw = serde_json::to_vec(value)?;
|
||||
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
|
||||
// A stale pending file is neither overwritten nor treated as current.
|
||||
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
|
||||
use windows_sys::Win32::Storage::FileSystem::*;
|
||||
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
|
||||
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
|
||||
ensure!(
|
||||
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
|
||||
&& info.dwFileAttributes
|
||||
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
|
||||
== 0
|
||||
&& info.nNumberOfLinks == 1,
|
||||
"JOB_READINESS_FILE_ID"
|
||||
);
|
||||
Ok([
|
||||
info.dwVolumeSerialNumber,
|
||||
info.nFileIndexHigh,
|
||||
info.nFileIndexLow,
|
||||
])
|
||||
}
|
||||
let mut f = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&temporary)?;
|
||||
let created = identity(&f)?;
|
||||
let result = (|| -> Result<()> {
|
||||
native::private_path(&temporary, false)?;
|
||||
f.write_all(&raw)?;
|
||||
f.sync_all()?;
|
||||
Ok(())
|
||||
})();
|
||||
drop(f);
|
||||
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
|
||||
if result.is_err() {
|
||||
// Remove only the exact file this call created, never a replacement,
|
||||
// reparse point or unknown pending file from a previous invocation.
|
||||
if let Ok(check) = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&temporary)
|
||||
&& identity(&check).is_ok_and(|value| value == created)
|
||||
{
|
||||
let _ = std::fs::remove_file(&temporary);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
|
||||
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let until = now + chrono::Duration::seconds(30);
|
||||
let one = std::time::Duration::from_secs(1);
|
||||
assert!(!refresh_due(false, one, Some(until), None));
|
||||
assert!(!refresh_due(false, one, Some(until), Some(now)));
|
||||
assert!(refresh_due(false, one, Some(until), Some(until)));
|
||||
assert!(refresh_due(false, one, None, None));
|
||||
assert!(refresh_due(true, one, Some(until), None));
|
||||
assert!(refresh_due(
|
||||
false,
|
||||
std::time::Duration::from_secs(40),
|
||||
Some(until),
|
||||
None
|
||||
));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user