//! Bounded passive runtime diagnostics. These observations are never input to //! job admission, clock qualification, signed Sync or recovery. #[cfg(windows)] use anyhow::{Result, ensure}; use serde::Serialize; #[cfg(windows)] use std::{io::Write, path::Path}; #[derive(Clone, Debug, Serialize)] pub struct Counts { pub accepted: u64, pub requesting: u64, pub running: u64, pub verifying: u64, pub unknown: u64, pub awaiting_reboot: u64, pub events: u64, pub legacy_events: u64, } #[derive(Clone, Debug, Serialize)] pub struct Observation { pub schema_version: u32, pub observed_at: chrono::DateTime, pub agent_id: String, pub boot_id: String, pub connection_generation: u64, pub ready: bool, pub helper_ready: bool, pub clock_qualified: bool, pub clock_failure: &'static str, pub needs_sync: bool, pub prepared: bool, pub deferred_sync: &'static str, pub deferred_sync_wait_ms: u64, pub helper_control: &'static str, pub helper_apply_unconfirmed: bool, pub signed_gate_present: bool, pub signed_ready: bool, pub local_applied: bool, pub helper_ack_pending: bool, pub signed_valid_until: Option>, pub counts: Counts, } pub fn clock_code(error: &anyhow::Error) -> &'static str { const CODES: &[&str] = &[ "TIME_STALE_RESPONSE", "TIME_NATIVE_NO_EVIDENCE", "TIME_RUNTIME_NO_EVIDENCE", "TIME_RUNTIME_NOT_CONFIGURED", "TIME_NATIVE_EVIDENCE_INVALIDATED", "TIME_NATIVE_CONNECTION_CHANGED", "TIME_RUNTIME_CONNECTION_CHANGED", "TIME_NATIVE_CONFIG_CHANGED", "TIME_RUNTIME_NO_CONNECTION", "TIME_LOCAL_WALL_OUTSIDE_BOUND", "TIME_SOURCE_AGE_AT_CHECK", "TIME_WINDOWS_LOCAL_CLOCK", ]; error .chain() .find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code)) .unwrap_or("CLOCK_CHECK_FAILED") } /// Refresh is control work, not permission. A temporary loss of the100ms clock /// evidence never discards an otherwise retained signed decision or extends it. /// Fresh clock/readiness checks remain mandatory at each actual job gate. /// A requested refresh waits at least `MIN_SYNC_INTERVAL` after the previous /// start, so a persistently not-ready decision cannot become a tight loop. pub const MIN_SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3); pub fn refresh_due( needs_sync: bool, elapsed: std::time::Duration, valid_until: Option>, clock_upper: Option>, ) -> bool { elapsed >= MIN_SYNC_INTERVAL && (needs_sync || elapsed >= std::time::Duration::from_secs(40) || valid_until.is_none() || valid_until .zip(clock_upper) .is_some_and(|(until, upper)| upper >= until)) } #[cfg(windows)] pub fn persist(state: &Path, value: &Observation) -> Result<()> { use crate::native; native::private_path(state, true)?; let path = state.join("job-readiness.json"); if path.try_exists()? { native::private_path(&path, false)?; } let temporary = state.join("job-readiness.pending"); let raw = serde_json::to_vec(value)?; ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT"); // A stale pending file is neither overwritten nor treated as current. use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle}; use windows_sys::Win32::Storage::FileSystem::*; fn identity(file: &std::fs::File) -> Result<[u32; 3]> { let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; ensure!( unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0 && info.dwFileAttributes & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY) == 0 && info.nNumberOfLinks == 1, "JOB_READINESS_FILE_ID" ); Ok([ info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow, ]) } let mut f = std::fs::OpenOptions::new() .write(true) .create_new(true) .share_mode(FILE_SHARE_READ) .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) .open(&temporary)?; let created = identity(&f)?; let result = (|| -> Result<()> { native::private_path(&temporary, false)?; f.write_all(&raw)?; f.sync_all()?; Ok(()) })(); drop(f); let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into)); if result.is_err() { // Remove only the exact file this call created, never a replacement, // reparse point or unknown pending file from a previous invocation. if let Ok(check) = std::fs::OpenOptions::new() .read(true) .share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE) .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) .open(&temporary) && identity(&check).is_ok_and(|value| value == created) { let _ = std::fs::remove_file(&temporary); } } result } #[cfg(test)] mod tests { use super::*; #[test] fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() { let now = chrono::DateTime::from_timestamp(100, 0).unwrap(); let until = now + chrono::Duration::seconds(30); let one = std::time::Duration::from_secs(1); let min = MIN_SYNC_INTERVAL; assert!(!refresh_due(false, min, Some(until), None)); assert!(!refresh_due(false, min, Some(until), Some(now))); assert!(refresh_due(false, min, Some(until), Some(until))); assert!(refresh_due(false, min, None, None)); assert!(refresh_due(true, min, Some(until), None)); // No trigger may restart Sync sooner than the minimum interval. assert!(!refresh_due(true, one, None, Some(until))); assert!(refresh_due( false, std::time::Duration::from_secs(40), Some(until), None )); } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ResponseWindow { WaitForClock, WithinWindow, Expired, } /// Only scheduling of an independently authenticated, already bound control /// reply. It cannot qualify a clock or turn this reply into an authorization. pub fn response_window( issued: chrono::DateTime, until: chrono::DateTime, held: std::time::Duration, reading: Option<(chrono::DateTime, chrono::DateTime)>, ) -> ResponseWindow { if held >= std::time::Duration::from_secs(60) || until <= issued { return ResponseWindow::Expired; } let Some((wall, upper)) = reading else { return ResponseWindow::WaitForClock; }; if upper < wall || upper >= until { return ResponseWindow::Expired; } if wall < issued { return ResponseWindow::WaitForClock; } ResponseWindow::WithinWindow } #[cfg(test)] mod response_tests { use super::*; #[test] fn signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline() { let issued = chrono::DateTime::from_timestamp(100, 0).unwrap(); let until = issued + chrono::Duration::seconds(30); let short = std::time::Duration::from_millis(20); assert_eq!( response_window(issued, until, short, None), ResponseWindow::WaitForClock ); assert_eq!( response_window( issued, until, short, Some(( issued - chrono::Duration::milliseconds(500), issued + chrono::Duration::milliseconds(100) )) ), ResponseWindow::WaitForClock ); assert_eq!( response_window( issued, until, short, Some((issued, issued + chrono::Duration::milliseconds(100))) ), ResponseWindow::WithinWindow ); assert_eq!( response_window( issued, until, short, Some((until - chrono::Duration::milliseconds(1), until)) ), ResponseWindow::Expired ); assert_eq!( response_window(issued, until, std::time::Duration::from_secs(60), None), ResponseWindow::Expired ); } } /// The actual SYSTEM Apply precommit wait, shared with its portable contract /// tests. It reads only the helper's independent clock, never an agent sample. /// A successful wait grants no lease and invokes no execution or application. pub fn wait_for_helper_apply_clock( clock: &impl rmm_agent_core::reconcile::SyncClock, issued: chrono::DateTime, until: chrono::DateTime, ) -> anyhow::Result<()> { let started = std::time::Instant::now(); loop { anyhow::ensure!( started.elapsed() < std::time::Duration::from_secs(3), "SYSTEM_APPLY_CLOCK_WAIT_EXPIRED" ); match response_window( issued, until, started.elapsed(), clock.reading().ok().map(|r| (r.own_wall, r.upper)), ) { ResponseWindow::WithinWindow => return Ok(()), ResponseWindow::Expired => { anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED") } ResponseWindow::WaitForClock => { std::thread::sleep(std::time::Duration::from_millis(20)) } } } } #[cfg(test)] mod helper_apply_tests { use super::*; use rmm_agent_core::reconcile::{ClockReading, SyncClock}; use std::{ collections::VecDeque, sync::{ Mutex, atomic::{AtomicUsize, Ordering}, }, time::{Duration, Instant}, }; type Sample=Option<(chrono::DateTime,chrono::DateTime)>; struct IndependentClock { samples:Mutex>, reads: AtomicUsize, } impl IndependentClock { fn new( samples: Vec, ) -> Self { Self { samples: Mutex::new(samples.into()), reads: AtomicUsize::new(0), } } } impl SyncClock for IndependentClock { fn now(&self) -> anyhow::Result> { Ok(self.reading()?.own_wall) } fn reading(&self) -> anyhow::Result { self.reads.fetch_add(1, Ordering::SeqCst); let (own_wall, upper) = self .samples .lock() .unwrap() .pop_front() .flatten() .ok_or_else(|| anyhow::anyhow!("TIME_NATIVE_NO_EVIDENCE"))?; Ok(ClockReading { own_wall, upper }) } } #[test] fn helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue() { let issued = chrono::DateTime::from_timestamp(100, 0).unwrap(); let until = issued + chrono::Duration::seconds(30); let unrelated_agent = IndependentClock::new(vec![Some((issued, issued + chrono::Duration::seconds(1)))]); let helper = IndependentClock::new(vec![ None, Some(( issued - chrono::Duration::milliseconds(500), issued + chrono::Duration::seconds(1), )), Some((issued, issued + chrono::Duration::seconds(1))), ]); let started = Instant::now(); wait_for_helper_apply_clock(&helper, issued, until).unwrap(); assert_eq!(helper.reads.load(Ordering::SeqCst), 3); assert_eq!(unrelated_agent.reads.load(Ordering::SeqCst), 0); assert!(started.elapsed() >= Duration::from_millis(40)); assert!(started.elapsed() < Duration::from_secs(3)); } #[test] fn helper_wait_refuses_the_original_deadline_without_a_second_sample() { let issued = chrono::DateTime::from_timestamp(100, 0).unwrap(); let until = issued + chrono::Duration::seconds(30); let helper = IndependentClock::new(vec![ Some((until - chrono::Duration::seconds(1), until)), Some((issued, issued + chrono::Duration::seconds(1))), ]); assert_eq!( wait_for_helper_apply_clock(&helper, issued, until) .unwrap_err() .to_string(), "SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED" ); assert_eq!(helper.reads.load(Ordering::SeqCst), 1); } #[test] fn helper_wait_finishes_closed_after_three_seconds_without_evidence() { let issued = chrono::DateTime::from_timestamp(100, 0).unwrap(); let helper = IndependentClock::new(vec![]); let started = Instant::now(); assert_eq!( wait_for_helper_apply_clock(&helper, issued, issued + chrono::Duration::hours(1)) .unwrap_err() .to_string(), "SYSTEM_APPLY_CLOCK_WAIT_EXPIRED" ); assert!(started.elapsed() >= Duration::from_secs(3)); assert!(started.elapsed() < Duration::from_secs(10)); assert!(helper.reads.load(Ordering::SeqCst) > 1); } }