Retain Windows signed sync responses across transient clock failures
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -500,7 +500,7 @@ impl Client {
|
||||
"HELPER_APPLY_EXPIRED"
|
||||
);
|
||||
let Ok(reading) = clock.reading() else {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
continue;
|
||||
};
|
||||
ensure!(
|
||||
@@ -519,7 +519,7 @@ impl Client {
|
||||
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
|
||||
return Err(error);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -29,6 +29,8 @@ pub struct Observation {
|
||||
pub clock_failure: &'static str,
|
||||
pub needs_sync: bool,
|
||||
pub prepared: bool,
|
||||
pub deferred_sync: &'static str,
|
||||
pub deferred_sync_wait_ms: u64,
|
||||
pub helper_control: &'static str,
|
||||
pub helper_apply_unconfirmed: bool,
|
||||
pub signed_gate_present: bool,
|
||||
@@ -154,3 +156,80 @@ mod tests {
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ResponseWindow {
|
||||
WaitForClock,
|
||||
WithinWindow,
|
||||
Expired,
|
||||
}
|
||||
/// Only scheduling of an independently authenticated, already bound control
|
||||
/// reply. It cannot qualify a clock or turn this reply into an authorization.
|
||||
pub fn response_window(
|
||||
issued: chrono::DateTime<chrono::Utc>,
|
||||
until: chrono::DateTime<chrono::Utc>,
|
||||
held: std::time::Duration,
|
||||
reading: Option<(chrono::DateTime<chrono::Utc>, chrono::DateTime<chrono::Utc>)>,
|
||||
) -> ResponseWindow {
|
||||
if held >= std::time::Duration::from_secs(60) || until <= issued {
|
||||
return ResponseWindow::Expired;
|
||||
}
|
||||
let Some((wall, upper)) = reading else {
|
||||
return ResponseWindow::WaitForClock;
|
||||
};
|
||||
if upper < wall || upper >= until {
|
||||
return ResponseWindow::Expired;
|
||||
}
|
||||
if wall < issued {
|
||||
return ResponseWindow::WaitForClock;
|
||||
}
|
||||
ResponseWindow::WithinWindow
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod response_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let until = issued + chrono::Duration::seconds(30);
|
||||
let short = std::time::Duration::from_millis(20);
|
||||
assert_eq!(
|
||||
response_window(issued, until, short, None),
|
||||
ResponseWindow::WaitForClock
|
||||
);
|
||||
assert_eq!(
|
||||
response_window(
|
||||
issued,
|
||||
until,
|
||||
short,
|
||||
Some((
|
||||
issued - chrono::Duration::milliseconds(500),
|
||||
issued + chrono::Duration::milliseconds(100)
|
||||
))
|
||||
),
|
||||
ResponseWindow::WaitForClock
|
||||
);
|
||||
assert_eq!(
|
||||
response_window(
|
||||
issued,
|
||||
until,
|
||||
short,
|
||||
Some((issued, issued + chrono::Duration::milliseconds(100)))
|
||||
),
|
||||
ResponseWindow::WithinWindow
|
||||
);
|
||||
assert_eq!(
|
||||
response_window(
|
||||
issued,
|
||||
until,
|
||||
short,
|
||||
Some((until - chrono::Duration::milliseconds(1), until))
|
||||
),
|
||||
ResponseWindow::Expired
|
||||
);
|
||||
assert_eq!(
|
||||
response_window(issued, until, std::time::Duration::from_secs(60), None),
|
||||
ResponseWindow::Expired
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,6 +400,19 @@ fn needs_bound_outcome(
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
struct DeferredSync {
|
||||
generation: u64,
|
||||
response: SyncResponse,
|
||||
issued_at: chrono::DateTime<Utc>,
|
||||
valid_until: chrono::DateTime<Utc>,
|
||||
received_at: Instant,
|
||||
applied: Option<reconcile::Applied>,
|
||||
}
|
||||
fn clock_temporarily_unavailable(error: &anyhow::Error) -> bool {
|
||||
error
|
||||
.chain()
|
||||
.any(|cause| cause.to_string().starts_with("TIME_"))
|
||||
}
|
||||
struct Running {
|
||||
offer: JobOffer,
|
||||
grant: ExecutionPermitPayload,
|
||||
@@ -547,6 +560,7 @@ struct Agent {
|
||||
time_refresh: Option<TimeRefresh>,
|
||||
sync: Option<reconcile::ClientSync>,
|
||||
prepared: bool,
|
||||
deferred_sync: Option<DeferredSync>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
prepared_request: Option<Vec<u8>>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
@@ -841,7 +855,7 @@ impl Agent {
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
async fn drain_helper_apply(&mut self) {
|
||||
async fn drain_helper_apply(&mut self, until: Instant) {
|
||||
// Keep the same bounded original worker and independent time lane
|
||||
// alive until its acknowledgement settles. This never opens a gate,
|
||||
// confirms a new generation, retries Apply or authorizes an effect.
|
||||
@@ -851,7 +865,8 @@ impl Agent {
|
||||
match control {
|
||||
HelperControl::Apply(_, task) => {
|
||||
if let Ok(Ok(Ok(_snapshot))) =
|
||||
tokio::time::timeout(Duration::from_secs(70), task).await
|
||||
tokio::time::timeout(until.saturating_duration_since(Instant::now()), task)
|
||||
.await
|
||||
{
|
||||
self.helper_apply_unconfirmed = false;
|
||||
}
|
||||
@@ -889,6 +904,7 @@ impl Agent {
|
||||
sync.close();
|
||||
}
|
||||
self.prepared = false;
|
||||
self.deferred_sync = None;
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.prepared_request = None;
|
||||
@@ -968,6 +984,19 @@ impl Agent {
|
||||
clock_failure: clock.as_ref().err().map_or("", clock_code),
|
||||
needs_sync,
|
||||
prepared: self.prepared,
|
||||
deferred_sync: self.deferred_sync.as_ref().map_or("none", |pending| {
|
||||
if pending.applied.is_some() {
|
||||
"local_consumption"
|
||||
} else {
|
||||
"fresh_response_clock"
|
||||
}
|
||||
}),
|
||||
deferred_sync_wait_ms: self.deferred_sync.as_ref().map_or(0, |p| {
|
||||
p.received_at
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
}),
|
||||
helper_control,
|
||||
helper_apply_unconfirmed: unconfirmed,
|
||||
signed_gate_present: gate.is_some(),
|
||||
@@ -1147,6 +1176,7 @@ impl Agent {
|
||||
return false;
|
||||
}
|
||||
self.config.enable_jobs
|
||||
&& self.deferred_sync.is_none()
|
||||
&& !self.helper_control_pending()
|
||||
&& self.channel(generation).is_ok_and(|channel| {
|
||||
self.sync
|
||||
@@ -1447,17 +1477,101 @@ impl Agent {
|
||||
}
|
||||
fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
||||
ensure!(self.prepared, "UNSOLICITED_SYNC_RESPONSE");
|
||||
self.prepared = false;
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
||||
#[cfg(feature = "system-helper")]
|
||||
if applied.helper_instruction.is_some() {
|
||||
self.helper_apply_unconfirmed = true;
|
||||
if let Some(pending) = &self.deferred_sync {
|
||||
ensure!(
|
||||
pending.generation == generation && pending.response == response,
|
||||
"DEFERRED_SYNC_RESPONSE_CHANGED"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
sync.apply_consumption(
|
||||
// Authenticate exact bytes/request/channel BEFORE retaining a reply.
|
||||
// Declared issued_at is used only for historical signature/profile
|
||||
// verification here. The regular fresh clock gates below still decide
|
||||
// whether either state-application phase may run now.
|
||||
ensure!(
|
||||
response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64,
|
||||
"SYNC_RESPONSE_LIMIT"
|
||||
);
|
||||
let decision: rmm_domain::reconciliation::ReconcileDecisionPayload =
|
||||
serde_json::from_slice(&response.decision_bytes)?;
|
||||
let raw:Vec<u8>=self.journal.connection().query_row("SELECT request FROM reconcile_requests WHERE id=(SELECT pending_request FROM reconcile_meta WHERE id=1)",[],|r|r.get(0))?;
|
||||
ensure!(raw.len() <= rmm_domain::MAX_MESSAGE, "SYNC_REQUEST_LIMIT");
|
||||
let request = SyncRequest::decode(raw.as_slice())?;
|
||||
ensure!(request.encode_to_vec() == raw, "SYNC_REQUEST_CANONICAL");
|
||||
reconcile::verify_response_with(
|
||||
&rmm_agent_core::crypto::DalekVerifier,
|
||||
self.binding.as_ref().context("JOB_BINDING_REQUIRED")?,
|
||||
&request,
|
||||
&self.channel(generation)?,
|
||||
&response,
|
||||
decision.issued_at,
|
||||
)?;
|
||||
self.deferred_sync = Some(DeferredSync {
|
||||
generation,
|
||||
response,
|
||||
issued_at: decision.issued_at,
|
||||
valid_until: decision.valid_until,
|
||||
received_at: Instant::now(),
|
||||
applied: None,
|
||||
});
|
||||
self.poll_sync_apply(generation, true)
|
||||
}
|
||||
fn poll_sync_apply(&mut self, generation: u64, open_gate: bool) -> Result<()> {
|
||||
let Some(mut pending) = self.deferred_sync.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
ensure!(pending.generation == generation, "DEFERRED_SYNC_GENERATION");
|
||||
use rmm_agent_windows::job_readiness::{ResponseWindow, response_window};
|
||||
match response_window(
|
||||
pending.issued_at,
|
||||
pending.valid_until,
|
||||
pending.received_at.elapsed(),
|
||||
self.clock.reading().ok().map(|r| (r.own_wall, r.upper)),
|
||||
) {
|
||||
ResponseWindow::WaitForClock => {
|
||||
self.deferred_sync = Some(pending);
|
||||
return Ok(());
|
||||
}
|
||||
ResponseWindow::Expired => anyhow::bail!("DEFERRED_SYNC_RESPONSE_EXPIRED"),
|
||||
ResponseWindow::WithinWindow => {}
|
||||
}
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
if pending.applied.is_none() {
|
||||
match sync.apply(
|
||||
self.journal.connection_mut(),
|
||||
&pending.response,
|
||||
&self.clock,
|
||||
) {
|
||||
Ok(applied) => {
|
||||
#[cfg(feature = "system-helper")]
|
||||
if applied.helper_instruction.is_some() {
|
||||
self.helper_apply_unconfirmed = true;
|
||||
}
|
||||
pending.applied = Some(applied);
|
||||
}
|
||||
Err(error) if clock_temporarily_unavailable(&error) => {
|
||||
self.deferred_sync = Some(pending);
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
match sync.apply_consumption(
|
||||
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
||||
&self.clock,
|
||||
)?;
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(error) if clock_temporarily_unavailable(&error) => {
|
||||
self.deferred_sync = Some(pending);
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let applied = pending
|
||||
.applied
|
||||
.take()
|
||||
.context("DEFERRED_SYNC_APPLIED_REQUIRED")?;
|
||||
self.prepared = false;
|
||||
if let Some(instruction) = applied.helper_instruction {
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
@@ -1493,7 +1607,12 @@ impl Agent {
|
||||
{
|
||||
self.prepared_request = None;
|
||||
}
|
||||
self.open_fence(generation)
|
||||
if open_gate {
|
||||
self.open_fence(generation)
|
||||
} else {
|
||||
self.fence.close();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn open_fence(&self, generation: u64) -> Result<()> {
|
||||
if self.ready(generation) {
|
||||
@@ -1955,12 +2074,32 @@ impl Agent {
|
||||
self.helper_apply_unconfirmed = false;
|
||||
}
|
||||
let result = self.session_inner(stop).await;
|
||||
self.close_gate();
|
||||
self.fence.close();
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.drain_helper_apply().await;
|
||||
let until = Instant::now() + Duration::from_secs(70);
|
||||
// Complete the already authenticated original control phases while
|
||||
// their finite deadline and independent time lane still permit it.
|
||||
// No job is running when this control pipeline was prepared.
|
||||
while self.running.is_none() && self.deferred_sync.is_some() && Instant::now() < until {
|
||||
let generation = self
|
||||
.deferred_sync
|
||||
.as_ref()
|
||||
.expect("checked pending control")
|
||||
.generation;
|
||||
if self.poll_sync_apply(generation, false).is_err() {
|
||||
break;
|
||||
}
|
||||
if self.deferred_sync.is_some() {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
self.close_gate();
|
||||
self.drain_helper_apply(until).await;
|
||||
self.stop_time_refresh().await;
|
||||
}
|
||||
#[cfg(not(feature = "system-helper"))]
|
||||
self.close_gate();
|
||||
result
|
||||
}
|
||||
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||
@@ -2056,6 +2195,7 @@ impl Agent {
|
||||
tokio::select! {
|
||||
Some(completion)=self.completed_rx.recv()=>{self.completed(generation,completion)?;needs_sync=true;},
|
||||
_=tick.tick()=>{
|
||||
if !stop.load(Ordering::SeqCst) && self.poll_sync_apply(generation,true).is_err(){self.close_gate();needs_sync=true;}
|
||||
#[cfg(feature = "system-helper")]
|
||||
if !stop.load(Ordering::SeqCst) && self.poll_helper(&channel,generation).await.is_err() {self.close_gate();needs_sync=self.sync.is_some();}
|
||||
if !stop.load(Ordering::SeqCst) && let Some((pending_generation,permit))=self.deferred_permit.take() {
|
||||
@@ -2069,7 +2209,7 @@ impl Agent {
|
||||
let busy=self.running.is_some() || self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?;
|
||||
if !stop.load(Ordering::SeqCst) && !busy {
|
||||
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
||||
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared && self.deferred_sync.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
||||
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
@@ -2365,7 +2505,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
helper_checked:false,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_refused:false,
|
||||
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
||||
prepared:false,deferred_sync:None,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1;
|
||||
|
||||
Reference in New Issue
Block a user