Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 11:22:53 +01:00
parent 056665be9e
commit c4100d80cd
6 changed files with 412 additions and 8 deletions
+6
View File
@@ -72,6 +72,9 @@ pub struct Applied {
pub helper_instruction: Option<HelperInstruction>,
pub reason: String,
}
/// Diagnostics only; never accepted as an authorization or reusable lease.
#[derive(Clone,Copy)]
pub struct GateObservation {pub signed_ready:bool,pub local_applied:bool,pub helper_ack_pending:bool}
struct Gate {
channel: Channel,
until: DateTime<Utc>,
@@ -119,6 +122,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
self.channel = None;
self.pending = None;
}
pub fn gate_observation(&self)->Option<GateObservation> {
self.gate.as_ref().map(|g|GateObservation{signed_ready:g.ready,local_applied:g.local_applied,helper_ack_pending:g.helper.is_some()})
}
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
self.gate.as_ref().map(|gate| gate.until)
}
+156
View File
@@ -0,0 +1,156 @@
//! Bounded passive runtime diagnostics. These observations are never input to
//! job admission, clock qualification, signed Sync or recovery.
#[cfg(windows)]
use anyhow::{Result, ensure};
use serde::Serialize;
#[cfg(windows)]
use std::{io::Write, path::Path};
#[derive(Clone, Debug, Serialize)]
pub struct Counts {
pub accepted: u64,
pub requesting: u64,
pub running: u64,
pub verifying: u64,
pub unknown: u64,
pub awaiting_reboot: u64,
pub events: u64,
pub legacy_events: u64,
}
#[derive(Clone, Debug, Serialize)]
pub struct Observation {
pub schema_version: u32,
pub observed_at: chrono::DateTime<chrono::Utc>,
pub agent_id: String,
pub boot_id: String,
pub connection_generation: u64,
pub ready: bool,
pub helper_ready: bool,
pub clock_qualified: bool,
pub clock_failure: &'static str,
pub needs_sync: bool,
pub prepared: bool,
pub helper_control: &'static str,
pub helper_apply_unconfirmed: bool,
pub signed_gate_present: bool,
pub signed_ready: bool,
pub local_applied: bool,
pub helper_ack_pending: bool,
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
pub counts: Counts,
}
pub fn clock_code(error: &anyhow::Error) -> &'static str {
const CODES: &[&str] = &[
"TIME_STALE_RESPONSE",
"TIME_NATIVE_NO_EVIDENCE",
"TIME_RUNTIME_NO_EVIDENCE",
"TIME_RUNTIME_NOT_CONFIGURED",
"TIME_NATIVE_EVIDENCE_INVALIDATED",
"TIME_NATIVE_CONNECTION_CHANGED",
"TIME_RUNTIME_CONNECTION_CHANGED",
"TIME_NATIVE_CONFIG_CHANGED",
"TIME_RUNTIME_NO_CONNECTION",
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
"TIME_SOURCE_AGE_AT_CHECK",
];
error
.chain()
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
.unwrap_or("CLOCK_CHECK_FAILED")
}
/// Refresh is control work, not permission. A temporary loss of the100ms clock
/// evidence never discards an otherwise retained signed decision or extends it.
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
pub fn refresh_due(
needs_sync: bool,
elapsed: std::time::Duration,
valid_until: Option<chrono::DateTime<chrono::Utc>>,
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
) -> bool {
needs_sync
|| elapsed >= std::time::Duration::from_secs(40)
|| valid_until.is_none()
|| valid_until
.zip(clock_upper)
.is_some_and(|(until, upper)| upper >= until)
}
#[cfg(windows)]
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
use crate::native;
native::private_path(state, true)?;
let path = state.join("job-readiness.json");
if path.try_exists()? {
native::private_path(&path, false)?;
}
let temporary = state.join("job-readiness.pending");
let raw = serde_json::to_vec(value)?;
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
// A stale pending file is neither overwritten nor treated as current.
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
use windows_sys::Win32::Storage::FileSystem::*;
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
ensure!(
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
&& info.dwFileAttributes
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
== 0
&& info.nNumberOfLinks == 1,
"JOB_READINESS_FILE_ID"
);
Ok([
info.dwVolumeSerialNumber,
info.nFileIndexHigh,
info.nFileIndexLow,
])
}
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)?;
let created = identity(&f)?;
let result = (|| -> Result<()> {
native::private_path(&temporary, false)?;
f.write_all(&raw)?;
f.sync_all()?;
Ok(())
})();
drop(f);
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
if result.is_err() {
// Remove only the exact file this call created, never a replacement,
// reparse point or unknown pending file from a previous invocation.
if let Ok(check) = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(&temporary)
&& identity(&check).is_ok_and(|value| value == created)
{
let _ = std::fs::remove_file(&temporary);
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = now + chrono::Duration::seconds(30);
let one = std::time::Duration::from_secs(1);
assert!(!refresh_due(false, one, Some(until), None));
assert!(!refresh_due(false, one, Some(until), Some(now)));
assert!(refresh_due(false, one, Some(until), Some(until)));
assert!(refresh_due(false, one, None, None));
assert!(refresh_due(true, one, Some(until), None));
assert!(refresh_due(
false,
std::time::Duration::from_secs(40),
Some(until),
None
));
}
}
+2
View File
@@ -40,3 +40,5 @@ pub mod update_native;
pub const SERVICE_NAME: &str = "RmmDiscoveryWindows";
pub const CAPABILITIES: &[&str] = &["windows.base/1", "heartbeat/1"];
pub mod job_readiness;
+164 -8
View File
@@ -518,10 +518,13 @@ impl HelperControl {
}
fn abort(self) -> Option<Self> {
match self {
Self::Snapshot(_, t) | Self::Apply(_, t) => {
Self::Snapshot(_, t) => {
t.abort();
None
}
// Apply may already have committed in the helper. Retain its
// bounded worker until settled; never lose the only reply owner.
apply @ Self::Apply(..) => Some(apply),
mirror @ Self::Mirror(..) => Some(mirror),
}
}
@@ -570,6 +573,7 @@ struct Agent {
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
running: Option<Running>,
deferred_permit: Option<(u64, ExecutionPermit)>,
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
}
impl Agent {
fn rotation_without_effects(&self) -> Result<()> {
@@ -836,6 +840,30 @@ impl Agent {
}
}
}
#[cfg(feature = "system-helper")]
async fn drain_helper_apply(&mut self) {
// Keep the same bounded original worker and independent time lane
// alive until its acknowledgement settles. This never opens a gate,
// confirms a new generation, retries Apply or authorizes an effect.
let Some(control) = self.helper_control.take() else {
return;
};
match control {
HelperControl::Apply(_, task) => {
if let Ok(Ok(Ok(_snapshot))) =
tokio::time::timeout(Duration::from_secs(70), task).await
{
self.helper_apply_unconfirmed = false;
}
}
mirror @ HelperControl::Mirror(..) => {
self.helper_control = Some(mirror);
}
HelperControl::Snapshot(_, task) => {
task.abort();
}
}
}
fn close_gate(&mut self) {
self.fence.close();
#[cfg(feature = "system-helper")]
@@ -885,6 +913,71 @@ impl Agent {
false
}
}
fn helper_control_busy_for_sync(&self) -> bool {
#[cfg(feature = "system-helper")]
{
self.helper_control.is_some()
|| self.helper_probe.is_some()
|| !self.helper_checked
|| self.helper_refused
}
#[cfg(not(feature = "system-helper"))]
{
false
}
}
fn readiness_observation(
&self,
generation: u64,
needs_sync: bool,
) -> Result<rmm_agent_windows::job_readiness::Observation> {
use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code};
let clock = self.clock.reading();
let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?;
let mut counts = counts;
counts.events =
self.journal
.connection()
.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
counts.legacy_events = self.journal.count()?;
let gate = self.sync.as_ref().and_then(|s| s.gate_observation());
#[cfg(feature = "system-helper")]
let (helper_control, unconfirmed) = (
match &self.helper_control {
Some(HelperControl::Snapshot(..)) => "snapshot",
Some(HelperControl::Mirror(..)) => "mirror",
Some(HelperControl::Apply(..)) => "apply",
None if self.helper_probe.is_some() => "probe",
None if self.helper_refused => "refused",
None if !self.helper_checked => "not_checked",
None => "idle",
},
self.helper_apply_unconfirmed,
);
#[cfg(not(feature = "system-helper"))]
let (helper_control, unconfirmed) = ("not_configured", false);
Ok(Observation {
schema_version: 1,
observed_at: Utc::now(),
agent_id: self.identity.agent_id.clone(),
boot_id: self.boot.clone(),
connection_generation: generation,
ready: self.ready(generation),
helper_ready: self.ready_identity(generation, true),
clock_qualified: clock.is_ok(),
clock_failure: clock.as_ref().err().map_or("", clock_code),
needs_sync,
prepared: self.prepared,
helper_control,
helper_apply_unconfirmed: unconfirmed,
signed_gate_present: gate.is_some(),
signed_ready: gate.is_some_and(|g| g.signed_ready),
local_applied: gate.is_some_and(|g| g.local_applied),
helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending),
signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()),
counts,
})
}
#[cfg(feature = "system-helper")]
async fn poll_helper(
&mut self,
@@ -958,8 +1051,14 @@ impl Agent {
.await?;
}
HelperControl::Apply(expected, task) => {
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
let snapshot = task.await??;
if expected != generation {
// A drained old reply is retained evidence only. A new
// generation requires a fresh snapshot and signed Sync.
self.helper_apply_unconfirmed = false;
self.close_gate();
return Ok(());
}
self.helper_anchor = snapshot.no_start_anchor;
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
@@ -1267,7 +1366,10 @@ impl Agent {
generation: u64,
) -> Result<()> {
self.fence.close();
ensure!(!self.helper_control_pending(), "HELPER_CONTROL_NOT_SETTLED");
ensure!(
!self.helper_control_busy_for_sync(),
"HELPER_CONTROL_NOT_SETTLED"
);
#[cfg(feature = "system-helper")]
if let Some(helper) = self.helper.clone() {
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
@@ -1479,6 +1581,10 @@ impl Agent {
)?;
return Ok(());
}
if self.clock.reading().is_err() {
self.deferred_permit = Some((generation, permit));
return Ok(());
}
ensure!(
self.ready(generation) && self.running.is_none(),
"FENCED_SYNC_REQUIRED"
@@ -1489,7 +1595,13 @@ impl Agent {
.context("JOB_BINDING_REQUIRED")?
.clone();
let request = jobs::permit_request(&offer, &permit)?;
let now = self.clock.now()?;
let now = match self.clock.now() {
Ok(now) => now,
Err(_) => {
self.deferred_permit = Some((generation, permit));
return Ok(());
}
};
// Authenticate before interpreting timing. A freshly issued grant may
// need two seconds before the full uncertainty interval is inside it.
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
@@ -1845,7 +1957,10 @@ impl Agent {
let result = self.session_inner(stop).await;
self.close_gate();
#[cfg(feature = "system-helper")]
self.stop_time_refresh().await;
{
self.drain_helper_apply().await;
self.stop_time_refresh().await;
}
result
}
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
@@ -1928,7 +2043,9 @@ impl Agent {
let mut rotation_after = Duration::from_secs(30);
let mut heartbeat = Instant::now() - Duration::from_secs(60);
let mut heartbeat_after = Duration::from_secs(30);
let mut tick = tokio::time::interval(Duration::from_secs(1));
let mut tick = tokio::time::interval(Duration::from_millis(20));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut readiness_at = Instant::now() - Duration::from_secs(5);
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
if stop.load(Ordering::SeqCst) {
self.close_gate();
@@ -1953,7 +2070,7 @@ impl Agent {
if !stop.load(Ordering::SeqCst) && !busy {
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.sync.is_some() && !self.prepared && !self.helper_control_pending() && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) {
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
}
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
@@ -2010,6 +2127,22 @@ impl Agent {
}
}
}
if readiness_at.elapsed() >= Duration::from_secs(5) {
readiness_at = Instant::now();
if self
.readiness_worker
.as_ref()
.is_none_or(|worker| worker.is_finished())
{
self.readiness_worker.take();
if let Ok(observation) = self.readiness_observation(generation, needs_sync) {
let state = self.config.state_directory.clone();
self.readiness_worker = Some(tokio::task::spawn_blocking(move || {
rmm_agent_windows::job_readiness::persist(&state, &observation)
}));
}
}
}
if stop.load(Ordering::SeqCst) {
continue;
}
@@ -2232,7 +2365,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
helper_checked:false,
#[cfg(feature="system-helper")]
helper_refused:false,
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
};
agent.recover_interrupted()?;
let mut backoff = 1;
@@ -2317,6 +2450,29 @@ pub fn main() -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "system-helper")]
#[tokio::test]
async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() {
let (sender, receiver) = tokio::sync::oneshot::channel::<()>();
let task = tokio::spawn(async move {
receiver.await?;
Err(anyhow::anyhow!("original fixture reply"))
});
let kept = HelperControl::Apply(7, task)
.abort()
.expect("Apply must remain owned");
assert!(!kept.finished());
sender.send(()).unwrap();
let HelperControl::Apply(generation, task) = kept else {
panic!("wrong retained operation")
};
assert_eq!(generation, 7);
assert_eq!(
task.await.unwrap().err().unwrap().to_string(),
"original fixture reply"
);
}
#[cfg(feature = "system-helper")]
#[test]
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
@@ -0,0 +1,59 @@
{
"schema_version": 1,
"source_files": [
{
"path": "crates/agent-windows/src/runtime.rs",
"source_sha256": "571eb94025fe0fb1fecbab050ff034d21cad8122193996606579768480e6b1cf",
"target_sha256": "e542569c3e94fd9fdef32523df58c184718a52840b7a549e6bcc2199852507bc"
},
{
"path": "crates/agent-windows/src/job_readiness.rs",
"source_sha256": null,
"target_sha256": "ff0f6814e08d471db2d1dfcf04bfb9dc516a108a1f7e305c4fcec376f16a0b6f"
},
{
"path": "crates/agent-windows/src/lib.rs",
"source_sha256": "609e6bf4cd895b81f0153113b17444fe8453212d6b691b73dcea26a63a920d45",
"target_sha256": "574e55d368e7c6668065c42f9d1898856dc659a46d21dfec5c6bd623a0d98138"
},
{
"path": "crates/agent-core/src/reconcile.rs",
"source_sha256": "d924e3cfd1f782169f3dee4dd7875cd5fba493fbaecb64e13abcb16729be9f74",
"target_sha256": "06180f8897b4a69eec4ed7cb265caa7afaf156e13283b1025fda944d0cc3e5c0"
}
],
"scope": "Native targeted scheduler and original Apply-worker ownership unit checks; no deployed runtime or job success claim",
"results": [
{
"exit_code": 0,
"stdout": "\nrunning 1 test\ntest job_readiness::tests::transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 31 filtered out; finished in 0.00s\n\n",
"stderr": "",
"binary_sha256": "18ac38c319cf6019309738ae7022360b221ff8a4a4aa8ee99393f921e40c0909",
"native_platform": "cloud-linux-x86_64",
"uid": 65534,
"isolated_network": true,
"no_new_privs": true,
"verified": true
},
{
"exit_code": 0,
"product_job": false,
"binary_sha256": "6562c0664bf7238ae090bf1d0ec045db05b2177fb71bc8e180a28c4066da63a6",
"bootstrap_administrator": true,
"stderr": "",
"native_platform": "windows-11-x86_64",
"stdout": "\nrunning 1 test\ntest job_readiness::tests::transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 58 filtered out; finished in 0.00s\n\n",
"verified": true
},
{
"exit_code": 0,
"product_job": false,
"binary_sha256": "a663806b07630d80b818324275ff292b02725bc591975654fe849c2e8ff5578f",
"bootstrap_administrator": true,
"stderr": "",
"native_platform": "windows-11-x86_64",
"stdout": "\nrunning 1 test\ntest runtime::tests::gate_close_retains_the_original_helper_apply_worker_without_replaying_it ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 8 filtered out; finished in 0.00s\n\n",
"verified": true
}
]
}
+25
View File
@@ -213,3 +213,28 @@ Es wurden weder Rechte geändert noch die Prüfung gelockert. Dieser Fehler
bedeutet: **keine erfolgreiche native Updater-OS-Abnahme und keine
Apply-Capability**. Die vier vorherigen portablen Verträge bleiben davon
getrennte, erfolgreiche Prüfergebnisse.
## Schedulerfehler eingegrenzt, Korrektur nativ geprüft
Die bisherige Windows-Schleife startete bei jedem vorübergehend nicht mehr
frischen 100-ms-Zeitbeleg einen vollständigen neuen SignedSync, bevor sie
angenommene Jobs auswählte. Das kann den Jobstart verdrängen. Die Korrektur
erneuert den signierten Abgleich nur bei tatsächlichem Bedarf beziehungsweise
Ablauf und prüft wartende Arbeit alle 20 ms ohne Tick-Rückstau. Der
100-ms-Vertrag und sämtliche frischen Start-/Permitprüfungen bleiben Pflicht;
ein vorhandener SignedSync macht einen veralteten Zeitbeleg nicht gültig.
Zusätzlich ging beim Schließen eines Gates der Besitzer eines bereits
ausgesandten Helper-Apply verloren. Der neue Ablauf behält diesen begrenzten
Originalworker und lässt ihn beim kontrollierten Schließen auslaufen, ohne
Apply zu wiederholen oder aus einem alten Kanal Bereitschaft abzuleiten.
Eine begrenzte passive Statusdatei zeigt künftig die tatsächlichen
Sperrgründe; ihr Inhalt ist niemals Eingabe einer Genehmigung oder Freigabe.
Am 16. November bestand die gezielte Schedulingprüfung zuerst nativ auf der
Cloud-Linux-VM (1/1). Auf Windows bestanden anschließend die Schedulingprüfung
und die Originalworker-Besitzprüfung (je 1/1). Das isolierte Windows-GNU-
Clippy lief für alle Targets mit `-D warnings` erfolgreich.
[Exakte Quellen, native Resultate und Binärhashes](evidence/windows-job-scheduling-20261116.json).
Diese Ergebnisse sind **noch kein erfolgreicher allgemeiner Skriptlauf und
kein Nachweis eines bereits erfolgten Dienstwechsels**.