Files
rust-discovery/crates/agent-windows/src/job_readiness.rs
T

370 lines
13 KiB
Rust

//! Bounded passive runtime diagnostics. These observations are never input to
//! job admission, clock qualification, signed Sync or recovery.
#[cfg(windows)]
use anyhow::{Result, ensure};
use serde::Serialize;
#[cfg(windows)]
use std::{io::Write, path::Path};
#[derive(Clone, Debug, Serialize)]
pub struct Counts {
pub accepted: u64,
pub requesting: u64,
pub running: u64,
pub verifying: u64,
pub unknown: u64,
pub awaiting_reboot: u64,
pub events: u64,
pub legacy_events: u64,
}
#[derive(Clone, Debug, Serialize)]
pub struct Observation {
pub schema_version: u32,
pub observed_at: chrono::DateTime<chrono::Utc>,
pub agent_id: String,
pub boot_id: String,
pub connection_generation: u64,
pub ready: bool,
pub helper_ready: bool,
pub clock_qualified: bool,
pub clock_failure: &'static str,
pub needs_sync: bool,
pub prepared: bool,
pub deferred_sync: &'static str,
pub deferred_sync_wait_ms: u64,
pub helper_control: &'static str,
pub helper_apply_unconfirmed: bool,
pub signed_gate_present: bool,
pub signed_ready: bool,
pub local_applied: bool,
pub helper_ack_pending: bool,
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
pub counts: Counts,
}
pub fn clock_code(error: &anyhow::Error) -> &'static str {
const CODES: &[&str] = &[
"TIME_STALE_RESPONSE",
"TIME_NATIVE_NO_EVIDENCE",
"TIME_RUNTIME_NO_EVIDENCE",
"TIME_RUNTIME_NOT_CONFIGURED",
"TIME_NATIVE_EVIDENCE_INVALIDATED",
"TIME_NATIVE_CONNECTION_CHANGED",
"TIME_RUNTIME_CONNECTION_CHANGED",
"TIME_NATIVE_CONFIG_CHANGED",
"TIME_RUNTIME_NO_CONNECTION",
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
"TIME_SOURCE_AGE_AT_CHECK",
];
error
.chain()
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
.unwrap_or("CLOCK_CHECK_FAILED")
}
/// Refresh is control work, not permission. A temporary loss of the100ms clock
/// evidence never discards an otherwise retained signed decision or extends it.
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
/// A requested refresh waits at least `MIN_SYNC_INTERVAL` after the previous
/// start, so a persistently not-ready decision cannot become a tight loop.
pub const MIN_SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3);
pub fn refresh_due(
needs_sync: bool,
elapsed: std::time::Duration,
valid_until: Option<chrono::DateTime<chrono::Utc>>,
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
) -> bool {
elapsed >= MIN_SYNC_INTERVAL
&& (needs_sync
|| elapsed >= std::time::Duration::from_secs(40)
|| valid_until.is_none()
|| valid_until
.zip(clock_upper)
.is_some_and(|(until, upper)| upper >= until))
}
#[cfg(windows)]
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
use crate::native;
native::private_path(state, true)?;
let path = state.join("job-readiness.json");
if path.try_exists()? {
native::private_path(&path, false)?;
}
let temporary = state.join("job-readiness.pending");
let raw = serde_json::to_vec(value)?;
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
// A stale pending file is neither overwritten nor treated as current.
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
use windows_sys::Win32::Storage::FileSystem::*;
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
ensure!(
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
&& info.dwFileAttributes
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
== 0
&& info.nNumberOfLinks == 1,
"JOB_READINESS_FILE_ID"
);
Ok([
info.dwVolumeSerialNumber,
info.nFileIndexHigh,
info.nFileIndexLow,
])
}
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)?;
let created = identity(&f)?;
let result = (|| -> Result<()> {
native::private_path(&temporary, false)?;
f.write_all(&raw)?;
f.sync_all()?;
Ok(())
})();
drop(f);
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
if result.is_err() {
// Remove only the exact file this call created, never a replacement,
// reparse point or unknown pending file from a previous invocation.
if let Ok(check) = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)
&& identity(&check).is_ok_and(|value| value == created)
{
let _ = std::fs::remove_file(&temporary);
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = now + chrono::Duration::seconds(30);
let one = std::time::Duration::from_secs(1);
let min = MIN_SYNC_INTERVAL;
assert!(!refresh_due(false, min, Some(until), None));
assert!(!refresh_due(false, min, Some(until), Some(now)));
assert!(refresh_due(false, min, Some(until), Some(until)));
assert!(refresh_due(false, min, None, None));
assert!(refresh_due(true, min, Some(until), None));
// No trigger may restart Sync sooner than the minimum interval.
assert!(!refresh_due(true, one, None, Some(until)));
assert!(refresh_due(
false,
std::time::Duration::from_secs(40),
Some(until),
None
));
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ResponseWindow {
WaitForClock,
WithinWindow,
Expired,
}
/// Only scheduling of an independently authenticated, already bound control
/// reply. It cannot qualify a clock or turn this reply into an authorization.
pub fn response_window(
issued: chrono::DateTime<chrono::Utc>,
until: chrono::DateTime<chrono::Utc>,
held: std::time::Duration,
reading: Option<(chrono::DateTime<chrono::Utc>, chrono::DateTime<chrono::Utc>)>,
) -> ResponseWindow {
if held >= std::time::Duration::from_secs(60) || until <= issued {
return ResponseWindow::Expired;
}
let Some((wall, upper)) = reading else {
return ResponseWindow::WaitForClock;
};
if upper < wall || upper >= until {
return ResponseWindow::Expired;
}
if wall < issued {
return ResponseWindow::WaitForClock;
}
ResponseWindow::WithinWindow
}
#[cfg(test)]
mod response_tests {
use super::*;
#[test]
fn signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = issued + chrono::Duration::seconds(30);
let short = std::time::Duration::from_millis(20);
assert_eq!(
response_window(issued, until, short, None),
ResponseWindow::WaitForClock
);
assert_eq!(
response_window(
issued,
until,
short,
Some((
issued - chrono::Duration::milliseconds(500),
issued + chrono::Duration::milliseconds(100)
))
),
ResponseWindow::WaitForClock
);
assert_eq!(
response_window(
issued,
until,
short,
Some((issued, issued + chrono::Duration::milliseconds(100)))
),
ResponseWindow::WithinWindow
);
assert_eq!(
response_window(
issued,
until,
short,
Some((until - chrono::Duration::milliseconds(1), until))
),
ResponseWindow::Expired
);
assert_eq!(
response_window(issued, until, std::time::Duration::from_secs(60), None),
ResponseWindow::Expired
);
}
}
/// The actual SYSTEM Apply precommit wait, shared with its portable contract
/// tests. It reads only the helper's independent clock, never an agent sample.
/// A successful wait grants no lease and invokes no execution or application.
pub fn wait_for_helper_apply_clock(
clock: &impl rmm_agent_core::reconcile::SyncClock,
issued: chrono::DateTime<chrono::Utc>,
until: chrono::DateTime<chrono::Utc>,
) -> anyhow::Result<()> {
let started = std::time::Instant::now();
loop {
anyhow::ensure!(
started.elapsed() < std::time::Duration::from_secs(3),
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
);
match response_window(
issued,
until,
started.elapsed(),
clock.reading().ok().map(|r| (r.own_wall, r.upper)),
) {
ResponseWindow::WithinWindow => return Ok(()),
ResponseWindow::Expired => {
anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED")
}
ResponseWindow::WaitForClock => {
std::thread::sleep(std::time::Duration::from_millis(20))
}
}
}
}
#[cfg(test)]
mod helper_apply_tests {
use super::*;
use rmm_agent_core::reconcile::{ClockReading, SyncClock};
use std::{
collections::VecDeque,
sync::{
Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
type Sample=Option<(chrono::DateTime<chrono::Utc>,chrono::DateTime<chrono::Utc>)>;
struct IndependentClock {
samples:Mutex<VecDeque<Sample>>,
reads: AtomicUsize,
}
impl IndependentClock {
fn new(
samples: Vec<Sample>,
) -> Self {
Self {
samples: Mutex::new(samples.into()),
reads: AtomicUsize::new(0),
}
}
}
impl SyncClock for IndependentClock {
fn now(&self) -> anyhow::Result<chrono::DateTime<chrono::Utc>> {
Ok(self.reading()?.own_wall)
}
fn reading(&self) -> anyhow::Result<ClockReading> {
self.reads.fetch_add(1, Ordering::SeqCst);
let (own_wall, upper) = self
.samples
.lock()
.unwrap()
.pop_front()
.flatten()
.ok_or_else(|| anyhow::anyhow!("TIME_NATIVE_NO_EVIDENCE"))?;
Ok(ClockReading { own_wall, upper })
}
}
#[test]
fn helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = issued + chrono::Duration::seconds(30);
let unrelated_agent =
IndependentClock::new(vec![Some((issued, issued + chrono::Duration::seconds(1)))]);
let helper = IndependentClock::new(vec![
None,
Some((
issued - chrono::Duration::milliseconds(500),
issued + chrono::Duration::seconds(1),
)),
Some((issued, issued + chrono::Duration::seconds(1))),
]);
let started = Instant::now();
wait_for_helper_apply_clock(&helper, issued, until).unwrap();
assert_eq!(helper.reads.load(Ordering::SeqCst), 3);
assert_eq!(unrelated_agent.reads.load(Ordering::SeqCst), 0);
assert!(started.elapsed() >= Duration::from_millis(40));
assert!(started.elapsed() < Duration::from_secs(3));
}
#[test]
fn helper_wait_refuses_the_original_deadline_without_a_second_sample() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = issued + chrono::Duration::seconds(30);
let helper = IndependentClock::new(vec![
Some((until - chrono::Duration::seconds(1), until)),
Some((issued, issued + chrono::Duration::seconds(1))),
]);
assert_eq!(
wait_for_helper_apply_clock(&helper, issued, until)
.unwrap_err()
.to_string(),
"SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED"
);
assert_eq!(helper.reads.load(Ordering::SeqCst), 1);
}
#[test]
fn helper_wait_finishes_closed_after_three_seconds_without_evidence() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let helper = IndependentClock::new(vec![]);
let started = Instant::now();
assert_eq!(
wait_for_helper_apply_clock(&helper, issued, issued + chrono::Duration::hours(1))
.unwrap_err()
.to_string(),
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
);
assert!(started.elapsed() >= Duration::from_secs(3));
assert!(started.elapsed() < Duration::from_secs(10));
assert!(helper.reads.load(Ordering::SeqCst) > 1);
}
}