Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -518,10 +518,13 @@ impl HelperControl {
|
||||
}
|
||||
fn abort(self) -> Option<Self> {
|
||||
match self {
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => {
|
||||
Self::Snapshot(_, t) => {
|
||||
t.abort();
|
||||
None
|
||||
}
|
||||
// Apply may already have committed in the helper. Retain its
|
||||
// bounded worker until settled; never lose the only reply owner.
|
||||
apply @ Self::Apply(..) => Some(apply),
|
||||
mirror @ Self::Mirror(..) => Some(mirror),
|
||||
}
|
||||
}
|
||||
@@ -570,6 +573,7 @@ struct Agent {
|
||||
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
||||
running: Option<Running>,
|
||||
deferred_permit: Option<(u64, ExecutionPermit)>,
|
||||
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
|
||||
}
|
||||
impl Agent {
|
||||
fn rotation_without_effects(&self) -> Result<()> {
|
||||
@@ -836,6 +840,30 @@ impl Agent {
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
async fn drain_helper_apply(&mut self) {
|
||||
// Keep the same bounded original worker and independent time lane
|
||||
// alive until its acknowledgement settles. This never opens a gate,
|
||||
// confirms a new generation, retries Apply or authorizes an effect.
|
||||
let Some(control) = self.helper_control.take() else {
|
||||
return;
|
||||
};
|
||||
match control {
|
||||
HelperControl::Apply(_, task) => {
|
||||
if let Ok(Ok(Ok(_snapshot))) =
|
||||
tokio::time::timeout(Duration::from_secs(70), task).await
|
||||
{
|
||||
self.helper_apply_unconfirmed = false;
|
||||
}
|
||||
}
|
||||
mirror @ HelperControl::Mirror(..) => {
|
||||
self.helper_control = Some(mirror);
|
||||
}
|
||||
HelperControl::Snapshot(_, task) => {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
fn close_gate(&mut self) {
|
||||
self.fence.close();
|
||||
#[cfg(feature = "system-helper")]
|
||||
@@ -885,6 +913,71 @@ impl Agent {
|
||||
false
|
||||
}
|
||||
}
|
||||
fn helper_control_busy_for_sync(&self) -> bool {
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.helper_control.is_some()
|
||||
|| self.helper_probe.is_some()
|
||||
|| !self.helper_checked
|
||||
|| self.helper_refused
|
||||
}
|
||||
#[cfg(not(feature = "system-helper"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
fn readiness_observation(
|
||||
&self,
|
||||
generation: u64,
|
||||
needs_sync: bool,
|
||||
) -> Result<rmm_agent_windows::job_readiness::Observation> {
|
||||
use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code};
|
||||
let clock = self.clock.reading();
|
||||
let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?;
|
||||
let mut counts = counts;
|
||||
counts.events =
|
||||
self.journal
|
||||
.connection()
|
||||
.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
||||
counts.legacy_events = self.journal.count()?;
|
||||
let gate = self.sync.as_ref().and_then(|s| s.gate_observation());
|
||||
#[cfg(feature = "system-helper")]
|
||||
let (helper_control, unconfirmed) = (
|
||||
match &self.helper_control {
|
||||
Some(HelperControl::Snapshot(..)) => "snapshot",
|
||||
Some(HelperControl::Mirror(..)) => "mirror",
|
||||
Some(HelperControl::Apply(..)) => "apply",
|
||||
None if self.helper_probe.is_some() => "probe",
|
||||
None if self.helper_refused => "refused",
|
||||
None if !self.helper_checked => "not_checked",
|
||||
None => "idle",
|
||||
},
|
||||
self.helper_apply_unconfirmed,
|
||||
);
|
||||
#[cfg(not(feature = "system-helper"))]
|
||||
let (helper_control, unconfirmed) = ("not_configured", false);
|
||||
Ok(Observation {
|
||||
schema_version: 1,
|
||||
observed_at: Utc::now(),
|
||||
agent_id: self.identity.agent_id.clone(),
|
||||
boot_id: self.boot.clone(),
|
||||
connection_generation: generation,
|
||||
ready: self.ready(generation),
|
||||
helper_ready: self.ready_identity(generation, true),
|
||||
clock_qualified: clock.is_ok(),
|
||||
clock_failure: clock.as_ref().err().map_or("", clock_code),
|
||||
needs_sync,
|
||||
prepared: self.prepared,
|
||||
helper_control,
|
||||
helper_apply_unconfirmed: unconfirmed,
|
||||
signed_gate_present: gate.is_some(),
|
||||
signed_ready: gate.is_some_and(|g| g.signed_ready),
|
||||
local_applied: gate.is_some_and(|g| g.local_applied),
|
||||
helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending),
|
||||
signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()),
|
||||
counts,
|
||||
})
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
async fn poll_helper(
|
||||
&mut self,
|
||||
@@ -958,8 +1051,14 @@ impl Agent {
|
||||
.await?;
|
||||
}
|
||||
HelperControl::Apply(expected, task) => {
|
||||
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
||||
let snapshot = task.await??;
|
||||
if expected != generation {
|
||||
// A drained old reply is retained evidence only. A new
|
||||
// generation requires a fresh snapshot and signed Sync.
|
||||
self.helper_apply_unconfirmed = false;
|
||||
self.close_gate();
|
||||
return Ok(());
|
||||
}
|
||||
self.helper_anchor = snapshot.no_start_anchor;
|
||||
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
@@ -1267,7 +1366,10 @@ impl Agent {
|
||||
generation: u64,
|
||||
) -> Result<()> {
|
||||
self.fence.close();
|
||||
ensure!(!self.helper_control_pending(), "HELPER_CONTROL_NOT_SETTLED");
|
||||
ensure!(
|
||||
!self.helper_control_busy_for_sync(),
|
||||
"HELPER_CONTROL_NOT_SETTLED"
|
||||
);
|
||||
#[cfg(feature = "system-helper")]
|
||||
if let Some(helper) = self.helper.clone() {
|
||||
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
||||
@@ -1479,6 +1581,10 @@ impl Agent {
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
if self.clock.reading().is_err() {
|
||||
self.deferred_permit = Some((generation, permit));
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
self.ready(generation) && self.running.is_none(),
|
||||
"FENCED_SYNC_REQUIRED"
|
||||
@@ -1489,7 +1595,13 @@ impl Agent {
|
||||
.context("JOB_BINDING_REQUIRED")?
|
||||
.clone();
|
||||
let request = jobs::permit_request(&offer, &permit)?;
|
||||
let now = self.clock.now()?;
|
||||
let now = match self.clock.now() {
|
||||
Ok(now) => now,
|
||||
Err(_) => {
|
||||
self.deferred_permit = Some((generation, permit));
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
// Authenticate before interpreting timing. A freshly issued grant may
|
||||
// need two seconds before the full uncertainty interval is inside it.
|
||||
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
|
||||
@@ -1845,7 +1957,10 @@ impl Agent {
|
||||
let result = self.session_inner(stop).await;
|
||||
self.close_gate();
|
||||
#[cfg(feature = "system-helper")]
|
||||
self.stop_time_refresh().await;
|
||||
{
|
||||
self.drain_helper_apply().await;
|
||||
self.stop_time_refresh().await;
|
||||
}
|
||||
result
|
||||
}
|
||||
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||
@@ -1928,7 +2043,9 @@ impl Agent {
|
||||
let mut rotation_after = Duration::from_secs(30);
|
||||
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
||||
let mut heartbeat_after = Duration::from_secs(30);
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(1));
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut readiness_at = Instant::now() - Duration::from_secs(5);
|
||||
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
self.close_gate();
|
||||
@@ -1953,7 +2070,7 @@ impl Agent {
|
||||
if !stop.load(Ordering::SeqCst) && !busy {
|
||||
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
||||
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.sync.is_some() && !self.prepared && !self.helper_control_pending() && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) {
|
||||
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
||||
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
||||
@@ -2010,6 +2127,22 @@ impl Agent {
|
||||
}
|
||||
}
|
||||
}
|
||||
if readiness_at.elapsed() >= Duration::from_secs(5) {
|
||||
readiness_at = Instant::now();
|
||||
if self
|
||||
.readiness_worker
|
||||
.as_ref()
|
||||
.is_none_or(|worker| worker.is_finished())
|
||||
{
|
||||
self.readiness_worker.take();
|
||||
if let Ok(observation) = self.readiness_observation(generation, needs_sync) {
|
||||
let state = self.config.state_directory.clone();
|
||||
self.readiness_worker = Some(tokio::task::spawn_blocking(move || {
|
||||
rmm_agent_windows::job_readiness::persist(&state, &observation)
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
continue;
|
||||
}
|
||||
@@ -2232,7 +2365,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
helper_checked:false,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_refused:false,
|
||||
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,
|
||||
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1;
|
||||
@@ -2317,6 +2450,29 @@ pub fn main() -> Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[cfg(feature = "system-helper")]
|
||||
#[tokio::test]
|
||||
async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() {
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel::<()>();
|
||||
let task = tokio::spawn(async move {
|
||||
receiver.await?;
|
||||
Err(anyhow::anyhow!("original fixture reply"))
|
||||
});
|
||||
let kept = HelperControl::Apply(7, task)
|
||||
.abort()
|
||||
.expect("Apply must remain owned");
|
||||
assert!(!kept.finished());
|
||||
sender.send(()).unwrap();
|
||||
let HelperControl::Apply(generation, task) = kept else {
|
||||
panic!("wrong retained operation")
|
||||
};
|
||||
assert_eq!(generation, 7);
|
||||
assert_eq!(
|
||||
task.await.unwrap().err().unwrap().to_string(),
|
||||
"original fixture reply"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "system-helper")]
|
||||
#[test]
|
||||
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
|
||||
|
||||
Reference in New Issue
Block a user