Wait for the SYSTEM helper's own clock before sync apply
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 13:39:57 +01:00
parent 10dce856bf
commit f54872af8e
6 changed files with 332 additions and 1 deletions
+3 -1
View File
@@ -507,7 +507,9 @@ impl Client {
reading.upper < valid_until && reading.own_wall < valid_until,
"HELPER_APPLY_EXPIRED"
);
match self.call(operation.clone()).await {
// The helper waits at most3s for ITS own fresh time evidence in
// this same original call. Longer transport timeout is no lease.
match self.call_bounded(operation.clone(),Duration::from_secs(8)).await {
Ok(reply) => break reply,
Err(error) => {
// Retry only contention before dispatch. An authenticated
+31
View File
@@ -843,6 +843,7 @@ pub struct SystemHelper {
boot: WindowsBoot,
sync: Option<HelperSync<DalekVerifier>>,
last_diagnostic: Option<Instant>,
last_apply_diagnostic: Option<Instant>,
unsettled_msi: Option<UnsettledMsi>,
}
impl SystemHelper {
@@ -867,6 +868,7 @@ impl SystemHelper {
boot,
sync,
last_diagnostic: None,
last_apply_diagnostic: None,
unsettled_msi: None,
})
}
@@ -920,6 +922,30 @@ impl SystemHelper {
);
Ok(())
}
fn wait_apply_clock(&self,issued:chrono::DateTime<chrono::Utc>,until:chrono::DateTime<chrono::Utc>)->Result<()> {
crate::job_readiness::wait_for_helper_apply_clock(&self.clock,issued,until)
}
/// Passive, bounded static denial metadata only. Never feeds a gate.
fn record_apply_error(&mut self,error:&anyhow::Error)->Result<()> {
if self.last_apply_diagnostic.is_some_and(|at|at.elapsed()<Duration::from_secs(5)){return Ok(());}
self.last_apply_diagnostic=Some(Instant::now());self.recheck()?;
const CODES:&[&str]=&["SYSTEM_APPLY_CLOCK_WAIT_EXPIRED","SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED","SYNC_PROFILE_OR_CLOCK","FRESH_PROCESS_SNAPSHOT_REQUIRED","OLD_PROCESS_DECISION_OR_BOOT","ROOT_SNAPSHOT_REQUEST_MISMATCH","TIME_RUNTIME_NO_EVIDENCE","TIME_NATIVE_NO_EVIDENCE","TIME_STALE_RESPONSE","TIME_NATIVE_EVIDENCE_INVALIDATED","TIME_SOURCE_AGE_AT_CHECK","TIME_LOCAL_WALL_OUTSIDE_BOUND","TIME_NATIVE_CONFIG_CHANGED","TIME_RUNTIME_CONNECTION_CHANGED","TIME_RUNTIME_GENERATION"];
let code=error.chain().find_map(|e|CODES.iter().copied().find(|c|*c==e.to_string())).unwrap_or("SYSTEM_APPLY_VALIDATION_OR_STORAGE_FAILURE");
let path=Path::new(STATE_ROOT).join("apply-last-error.json");
let pending=Path::new(STATE_ROOT).join(".apply-last-error.pending");
let writers=[self.settings.helper.service_sid.clone()];
if path.try_exists()?{let _old=PinnedFile::open(&path,false,&[],&writers,false)?;}
let bytes=serde_json::to_vec(&serde_json::json!({"schema_version":1,"observed_at_utc":chrono::Utc::now(),"helper_pid":std::process::id(),"code":code,"ready":false,"diagnostic_only":true}))?;
ensure!(bytes.len()<=1024,"SYSTEM_APPLY_DIAGNOSTIC_LIMIT");
use std::io::Write;
let mut file=OpenOptions::new().write(true).create_new(true).open(&pending)?;
let identity=ipc::file_identity(&file)?;
{let _check=PinnedFile::open_read_shared(&pending,false,&[],&writers)?;}
file.write_all(&bytes)?;file.sync_all()?;drop(file);
std::fs::rename(&pending,&path)?;
let checked=PinnedFile::open(&path,false,&[],&writers,false)?;
ensure!(checked.identity==identity,"SYSTEM_APPLY_DIAGNOSTIC_REPLACED");Ok(())
}
fn recheck(&mut self) -> Result<()> {
ipc::require_self(&self.settings.helper)?;
for file in &mut self.files {
@@ -996,6 +1022,10 @@ impl SystemHelper {
serde_json::from_slice(&response.decision_bytes)?;
self.clock
.require_generation(decision.connection_generation)?;
// Same authenticated original call, BEFORE any application.
// The independent time pipe continues on its own thread.
// No signature/deadline/lease is relaxed and no Execute retry.
self.wait_apply_clock(decision.issued_at,decision.valid_until)?;
self.sync()?;
let value = self.sync.as_mut().unwrap().apply(
&mut self.db,
@@ -1084,6 +1114,7 @@ impl SystemHelper {
let body = match result {
Ok(body) => body,
Err(e) => {
if matches!(call.operation,Operation::Apply{..}) {let _diagnostic=self.record_apply_error(&e);}
if let Some(sync) = &mut self.sync {
sync.close();
}
+127
View File
@@ -233,3 +233,130 @@ mod response_tests {
);
}
}
/// The actual SYSTEM Apply precommit wait, shared with its portable contract
/// tests. It reads only the helper's independent clock, never an agent sample.
/// A successful wait grants no lease and invokes no execution or application.
pub fn wait_for_helper_apply_clock(
clock: &impl rmm_agent_core::reconcile::SyncClock,
issued: chrono::DateTime<chrono::Utc>,
until: chrono::DateTime<chrono::Utc>,
) -> anyhow::Result<()> {
let started = std::time::Instant::now();
loop {
anyhow::ensure!(
started.elapsed() < std::time::Duration::from_secs(3),
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
);
match response_window(
issued,
until,
started.elapsed(),
clock.reading().ok().map(|r| (r.own_wall, r.upper)),
) {
ResponseWindow::WithinWindow => return Ok(()),
ResponseWindow::Expired => {
anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED")
}
ResponseWindow::WaitForClock => {
std::thread::sleep(std::time::Duration::from_millis(20))
}
}
}
}
#[cfg(test)]
mod helper_apply_tests {
use super::*;
use rmm_agent_core::reconcile::{ClockReading, SyncClock};
use std::{
collections::VecDeque,
sync::{
Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
type Sample=Option<(chrono::DateTime<chrono::Utc>,chrono::DateTime<chrono::Utc>)>;
struct IndependentClock {
samples:Mutex<VecDeque<Sample>>,
reads: AtomicUsize,
}
impl IndependentClock {
fn new(
samples: Vec<Sample>,
) -> Self {
Self {
samples: Mutex::new(samples.into()),
reads: AtomicUsize::new(0),
}
}
}
impl SyncClock for IndependentClock {
fn now(&self) -> anyhow::Result<chrono::DateTime<chrono::Utc>> {
Ok(self.reading()?.own_wall)
}
fn reading(&self) -> anyhow::Result<ClockReading> {
self.reads.fetch_add(1, Ordering::SeqCst);
let (own_wall, upper) = self
.samples
.lock()
.unwrap()
.pop_front()
.flatten()
.ok_or_else(|| anyhow::anyhow!("TIME_NATIVE_NO_EVIDENCE"))?;
Ok(ClockReading { own_wall, upper })
}
}
#[test]
fn helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = issued + chrono::Duration::seconds(30);
let unrelated_agent =
IndependentClock::new(vec![Some((issued, issued + chrono::Duration::seconds(1)))]);
let helper = IndependentClock::new(vec![
None,
Some((
issued - chrono::Duration::milliseconds(500),
issued + chrono::Duration::seconds(1),
)),
Some((issued, issued + chrono::Duration::seconds(1))),
]);
let started = Instant::now();
wait_for_helper_apply_clock(&helper, issued, until).unwrap();
assert_eq!(helper.reads.load(Ordering::SeqCst), 3);
assert_eq!(unrelated_agent.reads.load(Ordering::SeqCst), 0);
assert!(started.elapsed() >= Duration::from_millis(40));
assert!(started.elapsed() < Duration::from_secs(3));
}
#[test]
fn helper_wait_refuses_the_original_deadline_without_a_second_sample() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = issued + chrono::Duration::seconds(30);
let helper = IndependentClock::new(vec![
Some((until - chrono::Duration::seconds(1), until)),
Some((issued, issued + chrono::Duration::seconds(1))),
]);
assert_eq!(
wait_for_helper_apply_clock(&helper, issued, until)
.unwrap_err()
.to_string(),
"SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED"
);
assert_eq!(helper.reads.load(Ordering::SeqCst), 1);
}
#[test]
fn helper_wait_finishes_closed_after_three_seconds_without_evidence() {
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
let helper = IndependentClock::new(vec![]);
let started = Instant::now();
assert_eq!(
wait_for_helper_apply_clock(&helper, issued, issued + chrono::Duration::hours(1))
.unwrap_err()
.to_string(),
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
);
assert!(started.elapsed() >= Duration::from_secs(3));
assert!(started.elapsed() < Duration::from_secs(10));
assert!(helper.reads.load(Ordering::SeqCst) > 1);
}
}