Wait for the SYSTEM helper's own clock before sync apply
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -233,3 +233,130 @@ mod response_tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The actual SYSTEM Apply precommit wait, shared with its portable contract
|
||||
/// tests. It reads only the helper's independent clock, never an agent sample.
|
||||
/// A successful wait grants no lease and invokes no execution or application.
|
||||
pub fn wait_for_helper_apply_clock(
|
||||
clock: &impl rmm_agent_core::reconcile::SyncClock,
|
||||
issued: chrono::DateTime<chrono::Utc>,
|
||||
until: chrono::DateTime<chrono::Utc>,
|
||||
) -> anyhow::Result<()> {
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
anyhow::ensure!(
|
||||
started.elapsed() < std::time::Duration::from_secs(3),
|
||||
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
|
||||
);
|
||||
match response_window(
|
||||
issued,
|
||||
until,
|
||||
started.elapsed(),
|
||||
clock.reading().ok().map(|r| (r.own_wall, r.upper)),
|
||||
) {
|
||||
ResponseWindow::WithinWindow => return Ok(()),
|
||||
ResponseWindow::Expired => {
|
||||
anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED")
|
||||
}
|
||||
ResponseWindow::WaitForClock => {
|
||||
std::thread::sleep(std::time::Duration::from_millis(20))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod helper_apply_tests {
|
||||
use super::*;
|
||||
use rmm_agent_core::reconcile::{ClockReading, SyncClock};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{
|
||||
Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
type Sample=Option<(chrono::DateTime<chrono::Utc>,chrono::DateTime<chrono::Utc>)>;
|
||||
struct IndependentClock {
|
||||
samples:Mutex<VecDeque<Sample>>,
|
||||
reads: AtomicUsize,
|
||||
}
|
||||
impl IndependentClock {
|
||||
fn new(
|
||||
samples: Vec<Sample>,
|
||||
) -> Self {
|
||||
Self {
|
||||
samples: Mutex::new(samples.into()),
|
||||
reads: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl SyncClock for IndependentClock {
|
||||
fn now(&self) -> anyhow::Result<chrono::DateTime<chrono::Utc>> {
|
||||
Ok(self.reading()?.own_wall)
|
||||
}
|
||||
fn reading(&self) -> anyhow::Result<ClockReading> {
|
||||
self.reads.fetch_add(1, Ordering::SeqCst);
|
||||
let (own_wall, upper) = self
|
||||
.samples
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.flatten()
|
||||
.ok_or_else(|| anyhow::anyhow!("TIME_NATIVE_NO_EVIDENCE"))?;
|
||||
Ok(ClockReading { own_wall, upper })
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let until = issued + chrono::Duration::seconds(30);
|
||||
let unrelated_agent =
|
||||
IndependentClock::new(vec![Some((issued, issued + chrono::Duration::seconds(1)))]);
|
||||
let helper = IndependentClock::new(vec![
|
||||
None,
|
||||
Some((
|
||||
issued - chrono::Duration::milliseconds(500),
|
||||
issued + chrono::Duration::seconds(1),
|
||||
)),
|
||||
Some((issued, issued + chrono::Duration::seconds(1))),
|
||||
]);
|
||||
let started = Instant::now();
|
||||
wait_for_helper_apply_clock(&helper, issued, until).unwrap();
|
||||
assert_eq!(helper.reads.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(unrelated_agent.reads.load(Ordering::SeqCst), 0);
|
||||
assert!(started.elapsed() >= Duration::from_millis(40));
|
||||
assert!(started.elapsed() < Duration::from_secs(3));
|
||||
}
|
||||
#[test]
|
||||
fn helper_wait_refuses_the_original_deadline_without_a_second_sample() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let until = issued + chrono::Duration::seconds(30);
|
||||
let helper = IndependentClock::new(vec![
|
||||
Some((until - chrono::Duration::seconds(1), until)),
|
||||
Some((issued, issued + chrono::Duration::seconds(1))),
|
||||
]);
|
||||
assert_eq!(
|
||||
wait_for_helper_apply_clock(&helper, issued, until)
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED"
|
||||
);
|
||||
assert_eq!(helper.reads.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
#[test]
|
||||
fn helper_wait_finishes_closed_after_three_seconds_without_evidence() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let helper = IndependentClock::new(vec![]);
|
||||
let started = Instant::now();
|
||||
assert_eq!(
|
||||
wait_for_helper_apply_clock(&helper, issued, issued + chrono::Duration::hours(1))
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
|
||||
);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3));
|
||||
assert!(started.elapsed() < Duration::from_secs(10));
|
||||
assert!(helper.reads.load(Ordering::SeqCst) > 1);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user