Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 11:22:53 +01:00
parent 056665be9e
commit c4100d80cd
6 changed files with 412 additions and 8 deletions
+164 -8
View File
@@ -518,10 +518,13 @@ impl HelperControl {
}
fn abort(self) -> Option<Self> {
match self {
Self::Snapshot(_, t) | Self::Apply(_, t) => {
Self::Snapshot(_, t) => {
t.abort();
None
}
// Apply may already have committed in the helper. Retain its
// bounded worker until settled; never lose the only reply owner.
apply @ Self::Apply(..) => Some(apply),
mirror @ Self::Mirror(..) => Some(mirror),
}
}
@@ -570,6 +573,7 @@ struct Agent {
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
running: Option<Running>,
deferred_permit: Option<(u64, ExecutionPermit)>,
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
}
impl Agent {
fn rotation_without_effects(&self) -> Result<()> {
@@ -836,6 +840,30 @@ impl Agent {
}
}
}
#[cfg(feature = "system-helper")]
async fn drain_helper_apply(&mut self) {
// Keep the same bounded original worker and independent time lane
// alive until its acknowledgement settles. This never opens a gate,
// confirms a new generation, retries Apply or authorizes an effect.
let Some(control) = self.helper_control.take() else {
return;
};
match control {
HelperControl::Apply(_, task) => {
if let Ok(Ok(Ok(_snapshot))) =
tokio::time::timeout(Duration::from_secs(70), task).await
{
self.helper_apply_unconfirmed = false;
}
}
mirror @ HelperControl::Mirror(..) => {
self.helper_control = Some(mirror);
}
HelperControl::Snapshot(_, task) => {
task.abort();
}
}
}
fn close_gate(&mut self) {
self.fence.close();
#[cfg(feature = "system-helper")]
@@ -885,6 +913,71 @@ impl Agent {
false
}
}
fn helper_control_busy_for_sync(&self) -> bool {
#[cfg(feature = "system-helper")]
{
self.helper_control.is_some()
|| self.helper_probe.is_some()
|| !self.helper_checked
|| self.helper_refused
}
#[cfg(not(feature = "system-helper"))]
{
false
}
}
fn readiness_observation(
&self,
generation: u64,
needs_sync: bool,
) -> Result<rmm_agent_windows::job_readiness::Observation> {
use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code};
let clock = self.clock.reading();
let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?;
let mut counts = counts;
counts.events =
self.journal
.connection()
.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
counts.legacy_events = self.journal.count()?;
let gate = self.sync.as_ref().and_then(|s| s.gate_observation());
#[cfg(feature = "system-helper")]
let (helper_control, unconfirmed) = (
match &self.helper_control {
Some(HelperControl::Snapshot(..)) => "snapshot",
Some(HelperControl::Mirror(..)) => "mirror",
Some(HelperControl::Apply(..)) => "apply",
None if self.helper_probe.is_some() => "probe",
None if self.helper_refused => "refused",
None if !self.helper_checked => "not_checked",
None => "idle",
},
self.helper_apply_unconfirmed,
);
#[cfg(not(feature = "system-helper"))]
let (helper_control, unconfirmed) = ("not_configured", false);
Ok(Observation {
schema_version: 1,
observed_at: Utc::now(),
agent_id: self.identity.agent_id.clone(),
boot_id: self.boot.clone(),
connection_generation: generation,
ready: self.ready(generation),
helper_ready: self.ready_identity(generation, true),
clock_qualified: clock.is_ok(),
clock_failure: clock.as_ref().err().map_or("", clock_code),
needs_sync,
prepared: self.prepared,
helper_control,
helper_apply_unconfirmed: unconfirmed,
signed_gate_present: gate.is_some(),
signed_ready: gate.is_some_and(|g| g.signed_ready),
local_applied: gate.is_some_and(|g| g.local_applied),
helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending),
signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()),
counts,
})
}
#[cfg(feature = "system-helper")]
async fn poll_helper(
&mut self,
@@ -958,8 +1051,14 @@ impl Agent {
.await?;
}
HelperControl::Apply(expected, task) => {
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
let snapshot = task.await??;
if expected != generation {
// A drained old reply is retained evidence only. A new
// generation requires a fresh snapshot and signed Sync.
self.helper_apply_unconfirmed = false;
self.close_gate();
return Ok(());
}
self.helper_anchor = snapshot.no_start_anchor;
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
@@ -1267,7 +1366,10 @@ impl Agent {
generation: u64,
) -> Result<()> {
self.fence.close();
ensure!(!self.helper_control_pending(), "HELPER_CONTROL_NOT_SETTLED");
ensure!(
!self.helper_control_busy_for_sync(),
"HELPER_CONTROL_NOT_SETTLED"
);
#[cfg(feature = "system-helper")]
if let Some(helper) = self.helper.clone() {
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
@@ -1479,6 +1581,10 @@ impl Agent {
)?;
return Ok(());
}
if self.clock.reading().is_err() {
self.deferred_permit = Some((generation, permit));
return Ok(());
}
ensure!(
self.ready(generation) && self.running.is_none(),
"FENCED_SYNC_REQUIRED"
@@ -1489,7 +1595,13 @@ impl Agent {
.context("JOB_BINDING_REQUIRED")?
.clone();
let request = jobs::permit_request(&offer, &permit)?;
let now = self.clock.now()?;
let now = match self.clock.now() {
Ok(now) => now,
Err(_) => {
self.deferred_permit = Some((generation, permit));
return Ok(());
}
};
// Authenticate before interpreting timing. A freshly issued grant may
// need two seconds before the full uncertainty interval is inside it.
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
@@ -1845,7 +1957,10 @@ impl Agent {
let result = self.session_inner(stop).await;
self.close_gate();
#[cfg(feature = "system-helper")]
self.stop_time_refresh().await;
{
self.drain_helper_apply().await;
self.stop_time_refresh().await;
}
result
}
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
@@ -1928,7 +2043,9 @@ impl Agent {
let mut rotation_after = Duration::from_secs(30);
let mut heartbeat = Instant::now() - Duration::from_secs(60);
let mut heartbeat_after = Duration::from_secs(30);
let mut tick = tokio::time::interval(Duration::from_secs(1));
let mut tick = tokio::time::interval(Duration::from_millis(20));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut readiness_at = Instant::now() - Duration::from_secs(5);
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
if stop.load(Ordering::SeqCst) {
self.close_gate();
@@ -1953,7 +2070,7 @@ impl Agent {
if !stop.load(Ordering::SeqCst) && !busy {
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.sync.is_some() && !self.prepared && !self.helper_control_pending() && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) {
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
}
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
@@ -2010,6 +2127,22 @@ impl Agent {
}
}
}
if readiness_at.elapsed() >= Duration::from_secs(5) {
readiness_at = Instant::now();
if self
.readiness_worker
.as_ref()
.is_none_or(|worker| worker.is_finished())
{
self.readiness_worker.take();
if let Ok(observation) = self.readiness_observation(generation, needs_sync) {
let state = self.config.state_directory.clone();
self.readiness_worker = Some(tokio::task::spawn_blocking(move || {
rmm_agent_windows::job_readiness::persist(&state, &observation)
}));
}
}
}
if stop.load(Ordering::SeqCst) {
continue;
}
@@ -2232,7 +2365,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
helper_checked:false,
#[cfg(feature="system-helper")]
helper_refused:false,
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
};
agent.recover_interrupted()?;
let mut backoff = 1;
@@ -2317,6 +2450,29 @@ pub fn main() -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "system-helper")]
#[tokio::test]
async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() {
let (sender, receiver) = tokio::sync::oneshot::channel::<()>();
let task = tokio::spawn(async move {
receiver.await?;
Err(anyhow::anyhow!("original fixture reply"))
});
let kept = HelperControl::Apply(7, task)
.abort()
.expect("Apply must remain owned");
assert!(!kept.finished());
sender.send(()).unwrap();
let HelperControl::Apply(generation, task) = kept else {
panic!("wrong retained operation")
};
assert_eq!(generation, 7);
assert_eq!(
task.await.unwrap().err().unwrap().to_string(),
"original fixture reply"
);
}
#[cfg(feature = "system-helper")]
#[test]
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {