Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 11:22:53 +01:00
parent 056665be9e
commit c4100d80cd
6 changed files with 412 additions and 8 deletions
+6
View File
@@ -72,6 +72,9 @@ pub struct Applied {
pub helper_instruction: Option<HelperInstruction>,
pub reason: String,
}
/// Diagnostics only; never accepted as an authorization or reusable lease.
#[derive(Clone,Copy)]
pub struct GateObservation {pub signed_ready:bool,pub local_applied:bool,pub helper_ack_pending:bool}
struct Gate {
channel: Channel,
until: DateTime<Utc>,
@@ -119,6 +122,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
self.channel = None;
self.pending = None;
}
pub fn gate_observation(&self)->Option<GateObservation> {
self.gate.as_ref().map(|g|GateObservation{signed_ready:g.ready,local_applied:g.local_applied,helper_ack_pending:g.helper.is_some()})
}
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
self.gate.as_ref().map(|gate| gate.until)
}
+156
View File
@@ -0,0 +1,156 @@
//! Bounded passive runtime diagnostics. These observations are never input to
//! job admission, clock qualification, signed Sync or recovery.
#[cfg(windows)]
use anyhow::{Result, ensure};
use serde::Serialize;
#[cfg(windows)]
use std::{io::Write, path::Path};
#[derive(Clone, Debug, Serialize)]
pub struct Counts {
pub accepted: u64,
pub requesting: u64,
pub running: u64,
pub verifying: u64,
pub unknown: u64,
pub awaiting_reboot: u64,
pub events: u64,
pub legacy_events: u64,
}
#[derive(Clone, Debug, Serialize)]
pub struct Observation {
pub schema_version: u32,
pub observed_at: chrono::DateTime<chrono::Utc>,
pub agent_id: String,
pub boot_id: String,
pub connection_generation: u64,
pub ready: bool,
pub helper_ready: bool,
pub clock_qualified: bool,
pub clock_failure: &'static str,
pub needs_sync: bool,
pub prepared: bool,
pub helper_control: &'static str,
pub helper_apply_unconfirmed: bool,
pub signed_gate_present: bool,
pub signed_ready: bool,
pub local_applied: bool,
pub helper_ack_pending: bool,
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
pub counts: Counts,
}
pub fn clock_code(error: &anyhow::Error) -> &'static str {
const CODES: &[&str] = &[
"TIME_STALE_RESPONSE",
"TIME_NATIVE_NO_EVIDENCE",
"TIME_RUNTIME_NO_EVIDENCE",
"TIME_RUNTIME_NOT_CONFIGURED",
"TIME_NATIVE_EVIDENCE_INVALIDATED",
"TIME_NATIVE_CONNECTION_CHANGED",
"TIME_RUNTIME_CONNECTION_CHANGED",
"TIME_NATIVE_CONFIG_CHANGED",
"TIME_RUNTIME_NO_CONNECTION",
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
"TIME_SOURCE_AGE_AT_CHECK",
];
error
.chain()
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
.unwrap_or("CLOCK_CHECK_FAILED")
}
/// Refresh is control work, not permission. A temporary loss of the100ms clock
/// evidence never discards an otherwise retained signed decision or extends it.
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
pub fn refresh_due(
needs_sync: bool,
elapsed: std::time::Duration,
valid_until: Option<chrono::DateTime<chrono::Utc>>,
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
) -> bool {
needs_sync
|| elapsed >= std::time::Duration::from_secs(40)
|| valid_until.is_none()
|| valid_until
.zip(clock_upper)
.is_some_and(|(until, upper)| upper >= until)
}
#[cfg(windows)]
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
use crate::native;
native::private_path(state, true)?;
let path = state.join("job-readiness.json");
if path.try_exists()? {
native::private_path(&path, false)?;
}
let temporary = state.join("job-readiness.pending");
let raw = serde_json::to_vec(value)?;
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
// A stale pending file is neither overwritten nor treated as current.
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
use windows_sys::Win32::Storage::FileSystem::*;
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
ensure!(
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
&& info.dwFileAttributes
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
== 0
&& info.nNumberOfLinks == 1,
"JOB_READINESS_FILE_ID"
);
Ok([
info.dwVolumeSerialNumber,
info.nFileIndexHigh,
info.nFileIndexLow,
])
}
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)?;
let created = identity(&f)?;
let result = (|| -> Result<()> {
native::private_path(&temporary, false)?;
f.write_all(&raw)?;
f.sync_all()?;
Ok(())
})();
drop(f);
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
if result.is_err() {
// Remove only the exact file this call created, never a replacement,
// reparse point or unknown pending file from a previous invocation.
if let Ok(check) = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)
&& identity(&check).is_ok_and(|value| value == created)
{
let _ = std::fs::remove_file(&temporary);
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = now + chrono::Duration::seconds(30);
let one = std::time::Duration::from_secs(1);
assert!(!refresh_due(false, one, Some(until), None));
assert!(!refresh_due(false, one, Some(until), Some(now)));
assert!(refresh_due(false, one, Some(until), Some(until)));
assert!(refresh_due(false, one, None, None));
assert!(refresh_due(true, one, Some(until), None));
assert!(refresh_due(
false,
std::time::Duration::from_secs(40),
Some(until),
None
));
}
}
+2
View File
@@ -40,3 +40,5 @@ pub mod update_native;
pub const SERVICE_NAME: &str = "RmmDiscoveryWindows";
pub const CAPABILITIES: &[&str] = &["windows.base/1", "heartbeat/1"];
pub mod job_readiness;
+164 -8
View File
@@ -518,10 +518,13 @@ impl HelperControl {
}
fn abort(self) -> Option<Self> {
match self {
Self::Snapshot(_, t) | Self::Apply(_, t) => {
Self::Snapshot(_, t) => {
t.abort();
None
}
// Apply may already have committed in the helper. Retain its
// bounded worker until settled; never lose the only reply owner.
apply @ Self::Apply(..) => Some(apply),
mirror @ Self::Mirror(..) => Some(mirror),
}
}
@@ -570,6 +573,7 @@ struct Agent {
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
running: Option<Running>,
deferred_permit: Option<(u64, ExecutionPermit)>,
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
}
impl Agent {
fn rotation_without_effects(&self) -> Result<()> {
@@ -836,6 +840,30 @@ impl Agent {
}
}
}
#[cfg(feature = "system-helper")]
async fn drain_helper_apply(&mut self) {
// Keep the same bounded original worker and independent time lane
// alive until its acknowledgement settles. This never opens a gate,
// confirms a new generation, retries Apply or authorizes an effect.
let Some(control) = self.helper_control.take() else {
return;
};
match control {
HelperControl::Apply(_, task) => {
if let Ok(Ok(Ok(_snapshot))) =
tokio::time::timeout(Duration::from_secs(70), task).await
{
self.helper_apply_unconfirmed = false;
}
}
mirror @ HelperControl::Mirror(..) => {
self.helper_control = Some(mirror);
}
HelperControl::Snapshot(_, task) => {
task.abort();
}
}
}
fn close_gate(&mut self) {
self.fence.close();
#[cfg(feature = "system-helper")]
@@ -885,6 +913,71 @@ impl Agent {
false
}
}
fn helper_control_busy_for_sync(&self) -> bool {
#[cfg(feature = "system-helper")]
{
self.helper_control.is_some()
|| self.helper_probe.is_some()
|| !self.helper_checked
|| self.helper_refused
}
#[cfg(not(feature = "system-helper"))]
{
false
}
}
fn readiness_observation(
&self,
generation: u64,
needs_sync: bool,
) -> Result<rmm_agent_windows::job_readiness::Observation> {
use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code};
let clock = self.clock.reading();
let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?;
let mut counts = counts;
counts.events =
self.journal
.connection()
.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
counts.legacy_events = self.journal.count()?;
let gate = self.sync.as_ref().and_then(|s| s.gate_observation());
#[cfg(feature = "system-helper")]
let (helper_control, unconfirmed) = (
match &self.helper_control {
Some(HelperControl::Snapshot(..)) => "snapshot",
Some(HelperControl::Mirror(..)) => "mirror",
Some(HelperControl::Apply(..)) => "apply",
None if self.helper_probe.is_some() => "probe",
None if self.helper_refused => "refused",
None if !self.helper_checked => "not_checked",
None => "idle",
},
self.helper_apply_unconfirmed,
);
#[cfg(not(feature = "system-helper"))]
let (helper_control, unconfirmed) = ("not_configured", false);
Ok(Observation {
schema_version: 1,
observed_at: Utc::now(),
agent_id: self.identity.agent_id.clone(),
boot_id: self.boot.clone(),
connection_generation: generation,
ready: self.ready(generation),
helper_ready: self.ready_identity(generation, true),
clock_qualified: clock.is_ok(),
clock_failure: clock.as_ref().err().map_or("", clock_code),
needs_sync,
prepared: self.prepared,
helper_control,
helper_apply_unconfirmed: unconfirmed,
signed_gate_present: gate.is_some(),
signed_ready: gate.is_some_and(|g| g.signed_ready),
local_applied: gate.is_some_and(|g| g.local_applied),
helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending),
signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()),
counts,
})
}
#[cfg(feature = "system-helper")]
async fn poll_helper(
&mut self,
@@ -958,8 +1051,14 @@ impl Agent {
.await?;
}
HelperControl::Apply(expected, task) => {
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
let snapshot = task.await??;
if expected != generation {
// A drained old reply is retained evidence only. A new
// generation requires a fresh snapshot and signed Sync.
self.helper_apply_unconfirmed = false;
self.close_gate();
return Ok(());
}
self.helper_anchor = snapshot.no_start_anchor;
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
@@ -1267,7 +1366,10 @@ impl Agent {
generation: u64,
) -> Result<()> {
self.fence.close();
ensure!(!self.helper_control_pending(), "HELPER_CONTROL_NOT_SETTLED");
ensure!(
!self.helper_control_busy_for_sync(),
"HELPER_CONTROL_NOT_SETTLED"
);
#[cfg(feature = "system-helper")]
if let Some(helper) = self.helper.clone() {
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
@@ -1479,6 +1581,10 @@ impl Agent {
)?;
return Ok(());
}
if self.clock.reading().is_err() {
self.deferred_permit = Some((generation, permit));
return Ok(());
}
ensure!(
self.ready(generation) && self.running.is_none(),
"FENCED_SYNC_REQUIRED"
@@ -1489,7 +1595,13 @@ impl Agent {
.context("JOB_BINDING_REQUIRED")?
.clone();
let request = jobs::permit_request(&offer, &permit)?;
let now = self.clock.now()?;
let now = match self.clock.now() {
Ok(now) => now,
Err(_) => {
self.deferred_permit = Some((generation, permit));
return Ok(());
}
};
// Authenticate before interpreting timing. A freshly issued grant may
// need two seconds before the full uncertainty interval is inside it.
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
@@ -1845,7 +1957,10 @@ impl Agent {
let result = self.session_inner(stop).await;
self.close_gate();
#[cfg(feature = "system-helper")]
self.stop_time_refresh().await;
{
self.drain_helper_apply().await;
self.stop_time_refresh().await;
}
result
}
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
@@ -1928,7 +2043,9 @@ impl Agent {
let mut rotation_after = Duration::from_secs(30);
let mut heartbeat = Instant::now() - Duration::from_secs(60);
let mut heartbeat_after = Duration::from_secs(30);
let mut tick = tokio::time::interval(Duration::from_secs(1));
let mut tick = tokio::time::interval(Duration::from_millis(20));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut readiness_at = Instant::now() - Duration::from_secs(5);
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
if stop.load(Ordering::SeqCst) {
self.close_gate();
@@ -1953,7 +2070,7 @@ impl Agent {
if !stop.load(Ordering::SeqCst) && !busy {
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.sync.is_some() && !self.prepared && !self.helper_control_pending() && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) {
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
}
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
@@ -2010,6 +2127,22 @@ impl Agent {
}
}
}
if readiness_at.elapsed() >= Duration::from_secs(5) {
readiness_at = Instant::now();
if self
.readiness_worker
.as_ref()
.is_none_or(|worker| worker.is_finished())
{
self.readiness_worker.take();
if let Ok(observation) = self.readiness_observation(generation, needs_sync) {
let state = self.config.state_directory.clone();
self.readiness_worker = Some(tokio::task::spawn_blocking(move || {
rmm_agent_windows::job_readiness::persist(&state, &observation)
}));
}
}
}
if stop.load(Ordering::SeqCst) {
continue;
}
@@ -2232,7 +2365,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
helper_checked:false,
#[cfg(feature="system-helper")]
helper_refused:false,
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
};
agent.recover_interrupted()?;
let mut backoff = 1;
@@ -2317,6 +2450,29 @@ pub fn main() -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "system-helper")]
#[tokio::test]
async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() {
let (sender, receiver) = tokio::sync::oneshot::channel::<()>();
let task = tokio::spawn(async move {
receiver.await?;
Err(anyhow::anyhow!("original fixture reply"))
});
let kept = HelperControl::Apply(7, task)
.abort()
.expect("Apply must remain owned");
assert!(!kept.finished());
sender.send(()).unwrap();
let HelperControl::Apply(generation, task) = kept else {
panic!("wrong retained operation")
};
assert_eq!(generation, 7);
assert_eq!(
task.await.unwrap().err().unwrap().to_string(),
"original fixture reply"
);
}
#[cfg(feature = "system-helper")]
#[test]
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {