370 lines
13 KiB
Rust
370 lines
13 KiB
Rust
//! Bounded passive runtime diagnostics. These observations are never input to
|
|
//! job admission, clock qualification, signed Sync or recovery.
|
|
#[cfg(windows)]
|
|
use anyhow::{Result, ensure};
|
|
use serde::Serialize;
|
|
#[cfg(windows)]
|
|
use std::{io::Write, path::Path};
|
|
#[derive(Clone, Debug, Serialize)]
|
|
pub struct Counts {
|
|
pub accepted: u64,
|
|
pub requesting: u64,
|
|
pub running: u64,
|
|
pub verifying: u64,
|
|
pub unknown: u64,
|
|
pub awaiting_reboot: u64,
|
|
pub events: u64,
|
|
pub legacy_events: u64,
|
|
}
|
|
#[derive(Clone, Debug, Serialize)]
|
|
pub struct Observation {
|
|
pub schema_version: u32,
|
|
pub observed_at: chrono::DateTime<chrono::Utc>,
|
|
pub agent_id: String,
|
|
pub boot_id: String,
|
|
pub connection_generation: u64,
|
|
pub ready: bool,
|
|
pub helper_ready: bool,
|
|
pub clock_qualified: bool,
|
|
pub clock_failure: &'static str,
|
|
pub needs_sync: bool,
|
|
pub prepared: bool,
|
|
pub deferred_sync: &'static str,
|
|
pub deferred_sync_wait_ms: u64,
|
|
pub helper_control: &'static str,
|
|
pub helper_apply_unconfirmed: bool,
|
|
pub signed_gate_present: bool,
|
|
pub signed_ready: bool,
|
|
pub local_applied: bool,
|
|
pub helper_ack_pending: bool,
|
|
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
|
|
pub counts: Counts,
|
|
}
|
|
pub fn clock_code(error: &anyhow::Error) -> &'static str {
|
|
const CODES: &[&str] = &[
|
|
"TIME_STALE_RESPONSE",
|
|
"TIME_NATIVE_NO_EVIDENCE",
|
|
"TIME_RUNTIME_NO_EVIDENCE",
|
|
"TIME_RUNTIME_NOT_CONFIGURED",
|
|
"TIME_NATIVE_EVIDENCE_INVALIDATED",
|
|
"TIME_NATIVE_CONNECTION_CHANGED",
|
|
"TIME_RUNTIME_CONNECTION_CHANGED",
|
|
"TIME_NATIVE_CONFIG_CHANGED",
|
|
"TIME_RUNTIME_NO_CONNECTION",
|
|
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
|
|
"TIME_SOURCE_AGE_AT_CHECK",
|
|
];
|
|
error
|
|
.chain()
|
|
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
|
|
.unwrap_or("CLOCK_CHECK_FAILED")
|
|
}
|
|
/// Refresh is control work, not permission. A temporary loss of the100ms clock
|
|
/// evidence never discards an otherwise retained signed decision or extends it.
|
|
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
|
|
/// A requested refresh waits at least `MIN_SYNC_INTERVAL` after the previous
|
|
/// start, so a persistently not-ready decision cannot become a tight loop.
|
|
pub const MIN_SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3);
|
|
pub fn refresh_due(
|
|
needs_sync: bool,
|
|
elapsed: std::time::Duration,
|
|
valid_until: Option<chrono::DateTime<chrono::Utc>>,
|
|
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
|
|
) -> bool {
|
|
elapsed >= MIN_SYNC_INTERVAL
|
|
&& (needs_sync
|
|
|| elapsed >= std::time::Duration::from_secs(40)
|
|
|| valid_until.is_none()
|
|
|| valid_until
|
|
.zip(clock_upper)
|
|
.is_some_and(|(until, upper)| upper >= until))
|
|
}
|
|
#[cfg(windows)]
|
|
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
|
|
use crate::native;
|
|
native::private_path(state, true)?;
|
|
let path = state.join("job-readiness.json");
|
|
if path.try_exists()? {
|
|
native::private_path(&path, false)?;
|
|
}
|
|
let temporary = state.join("job-readiness.pending");
|
|
let raw = serde_json::to_vec(value)?;
|
|
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
|
|
// A stale pending file is neither overwritten nor treated as current.
|
|
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
|
|
use windows_sys::Win32::Storage::FileSystem::*;
|
|
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
|
|
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
|
|
ensure!(
|
|
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
|
|
&& info.dwFileAttributes
|
|
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
|
|
== 0
|
|
&& info.nNumberOfLinks == 1,
|
|
"JOB_READINESS_FILE_ID"
|
|
);
|
|
Ok([
|
|
info.dwVolumeSerialNumber,
|
|
info.nFileIndexHigh,
|
|
info.nFileIndexLow,
|
|
])
|
|
}
|
|
let mut f = std::fs::OpenOptions::new()
|
|
.write(true)
|
|
.create_new(true)
|
|
.share_mode(FILE_SHARE_READ)
|
|
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
|
|
.open(&temporary)?;
|
|
let created = identity(&f)?;
|
|
let result = (|| -> Result<()> {
|
|
native::private_path(&temporary, false)?;
|
|
f.write_all(&raw)?;
|
|
f.sync_all()?;
|
|
Ok(())
|
|
})();
|
|
drop(f);
|
|
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
|
|
if result.is_err() {
|
|
// Remove only the exact file this call created, never a replacement,
|
|
// reparse point or unknown pending file from a previous invocation.
|
|
if let Ok(check) = std::fs::OpenOptions::new()
|
|
.read(true)
|
|
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
|
|
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
|
|
.open(&temporary)
|
|
&& identity(&check).is_ok_and(|value| value == created)
|
|
{
|
|
let _ = std::fs::remove_file(&temporary);
|
|
}
|
|
}
|
|
result
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
#[test]
|
|
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
|
|
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
|
let until = now + chrono::Duration::seconds(30);
|
|
let one = std::time::Duration::from_secs(1);
|
|
let min = MIN_SYNC_INTERVAL;
|
|
assert!(!refresh_due(false, min, Some(until), None));
|
|
assert!(!refresh_due(false, min, Some(until), Some(now)));
|
|
assert!(refresh_due(false, min, Some(until), Some(until)));
|
|
assert!(refresh_due(false, min, None, None));
|
|
assert!(refresh_due(true, min, Some(until), None));
|
|
// No trigger may restart Sync sooner than the minimum interval.
|
|
assert!(!refresh_due(true, one, None, Some(until)));
|
|
assert!(refresh_due(
|
|
false,
|
|
std::time::Duration::from_secs(40),
|
|
Some(until),
|
|
None
|
|
));
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub enum ResponseWindow {
|
|
WaitForClock,
|
|
WithinWindow,
|
|
Expired,
|
|
}
|
|
/// Only scheduling of an independently authenticated, already bound control
|
|
/// reply. It cannot qualify a clock or turn this reply into an authorization.
|
|
pub fn response_window(
|
|
issued: chrono::DateTime<chrono::Utc>,
|
|
until: chrono::DateTime<chrono::Utc>,
|
|
held: std::time::Duration,
|
|
reading: Option<(chrono::DateTime<chrono::Utc>, chrono::DateTime<chrono::Utc>)>,
|
|
) -> ResponseWindow {
|
|
if held >= std::time::Duration::from_secs(60) || until <= issued {
|
|
return ResponseWindow::Expired;
|
|
}
|
|
let Some((wall, upper)) = reading else {
|
|
return ResponseWindow::WaitForClock;
|
|
};
|
|
if upper < wall || upper >= until {
|
|
return ResponseWindow::Expired;
|
|
}
|
|
if wall < issued {
|
|
return ResponseWindow::WaitForClock;
|
|
}
|
|
ResponseWindow::WithinWindow
|
|
}
|
|
#[cfg(test)]
|
|
mod response_tests {
|
|
use super::*;
|
|
#[test]
|
|
fn signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline() {
|
|
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
|
let until = issued + chrono::Duration::seconds(30);
|
|
let short = std::time::Duration::from_millis(20);
|
|
assert_eq!(
|
|
response_window(issued, until, short, None),
|
|
ResponseWindow::WaitForClock
|
|
);
|
|
assert_eq!(
|
|
response_window(
|
|
issued,
|
|
until,
|
|
short,
|
|
Some((
|
|
issued - chrono::Duration::milliseconds(500),
|
|
issued + chrono::Duration::milliseconds(100)
|
|
))
|
|
),
|
|
ResponseWindow::WaitForClock
|
|
);
|
|
assert_eq!(
|
|
response_window(
|
|
issued,
|
|
until,
|
|
short,
|
|
Some((issued, issued + chrono::Duration::milliseconds(100)))
|
|
),
|
|
ResponseWindow::WithinWindow
|
|
);
|
|
assert_eq!(
|
|
response_window(
|
|
issued,
|
|
until,
|
|
short,
|
|
Some((until - chrono::Duration::milliseconds(1), until))
|
|
),
|
|
ResponseWindow::Expired
|
|
);
|
|
assert_eq!(
|
|
response_window(issued, until, std::time::Duration::from_secs(60), None),
|
|
ResponseWindow::Expired
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The actual SYSTEM Apply precommit wait, shared with its portable contract
|
|
/// tests. It reads only the helper's independent clock, never an agent sample.
|
|
/// A successful wait grants no lease and invokes no execution or application.
|
|
pub fn wait_for_helper_apply_clock(
|
|
clock: &impl rmm_agent_core::reconcile::SyncClock,
|
|
issued: chrono::DateTime<chrono::Utc>,
|
|
until: chrono::DateTime<chrono::Utc>,
|
|
) -> anyhow::Result<()> {
|
|
let started = std::time::Instant::now();
|
|
loop {
|
|
anyhow::ensure!(
|
|
started.elapsed() < std::time::Duration::from_secs(3),
|
|
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
|
|
);
|
|
match response_window(
|
|
issued,
|
|
until,
|
|
started.elapsed(),
|
|
clock.reading().ok().map(|r| (r.own_wall, r.upper)),
|
|
) {
|
|
ResponseWindow::WithinWindow => return Ok(()),
|
|
ResponseWindow::Expired => {
|
|
anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED")
|
|
}
|
|
ResponseWindow::WaitForClock => {
|
|
std::thread::sleep(std::time::Duration::from_millis(20))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#[cfg(test)]
|
|
mod helper_apply_tests {
|
|
use super::*;
|
|
use rmm_agent_core::reconcile::{ClockReading, SyncClock};
|
|
use std::{
|
|
collections::VecDeque,
|
|
sync::{
|
|
Mutex,
|
|
atomic::{AtomicUsize, Ordering},
|
|
},
|
|
time::{Duration, Instant},
|
|
};
|
|
type Sample=Option<(chrono::DateTime<chrono::Utc>,chrono::DateTime<chrono::Utc>)>;
|
|
struct IndependentClock {
|
|
samples:Mutex<VecDeque<Sample>>,
|
|
reads: AtomicUsize,
|
|
}
|
|
impl IndependentClock {
|
|
fn new(
|
|
samples: Vec<Sample>,
|
|
) -> Self {
|
|
Self {
|
|
samples: Mutex::new(samples.into()),
|
|
reads: AtomicUsize::new(0),
|
|
}
|
|
}
|
|
}
|
|
impl SyncClock for IndependentClock {
|
|
fn now(&self) -> anyhow::Result<chrono::DateTime<chrono::Utc>> {
|
|
Ok(self.reading()?.own_wall)
|
|
}
|
|
fn reading(&self) -> anyhow::Result<ClockReading> {
|
|
self.reads.fetch_add(1, Ordering::SeqCst);
|
|
let (own_wall, upper) = self
|
|
.samples
|
|
.lock()
|
|
.unwrap()
|
|
.pop_front()
|
|
.flatten()
|
|
.ok_or_else(|| anyhow::anyhow!("TIME_NATIVE_NO_EVIDENCE"))?;
|
|
Ok(ClockReading { own_wall, upper })
|
|
}
|
|
}
|
|
#[test]
|
|
fn helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue() {
|
|
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
|
let until = issued + chrono::Duration::seconds(30);
|
|
let unrelated_agent =
|
|
IndependentClock::new(vec![Some((issued, issued + chrono::Duration::seconds(1)))]);
|
|
let helper = IndependentClock::new(vec![
|
|
None,
|
|
Some((
|
|
issued - chrono::Duration::milliseconds(500),
|
|
issued + chrono::Duration::seconds(1),
|
|
)),
|
|
Some((issued, issued + chrono::Duration::seconds(1))),
|
|
]);
|
|
let started = Instant::now();
|
|
wait_for_helper_apply_clock(&helper, issued, until).unwrap();
|
|
assert_eq!(helper.reads.load(Ordering::SeqCst), 3);
|
|
assert_eq!(unrelated_agent.reads.load(Ordering::SeqCst), 0);
|
|
assert!(started.elapsed() >= Duration::from_millis(40));
|
|
assert!(started.elapsed() < Duration::from_secs(3));
|
|
}
|
|
#[test]
|
|
fn helper_wait_refuses_the_original_deadline_without_a_second_sample() {
|
|
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
|
let until = issued + chrono::Duration::seconds(30);
|
|
let helper = IndependentClock::new(vec![
|
|
Some((until - chrono::Duration::seconds(1), until)),
|
|
Some((issued, issued + chrono::Duration::seconds(1))),
|
|
]);
|
|
assert_eq!(
|
|
wait_for_helper_apply_clock(&helper, issued, until)
|
|
.unwrap_err()
|
|
.to_string(),
|
|
"SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED"
|
|
);
|
|
assert_eq!(helper.reads.load(Ordering::SeqCst), 1);
|
|
}
|
|
#[test]
|
|
fn helper_wait_finishes_closed_after_three_seconds_without_evidence() {
|
|
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
|
let helper = IndependentClock::new(vec![]);
|
|
let started = Instant::now();
|
|
assert_eq!(
|
|
wait_for_helper_apply_clock(&helper, issued, issued + chrono::Duration::hours(1))
|
|
.unwrap_err()
|
|
.to_string(),
|
|
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
|
|
);
|
|
assert!(started.elapsed() >= Duration::from_secs(3));
|
|
assert!(started.elapsed() < Duration::from_secs(10));
|
|
assert!(helper.reads.load(Ordering::SeqCst) > 1);
|
|
}
|
|
}
|