diff --git a/crates/agent-core/src/reconcile.rs b/crates/agent-core/src/reconcile.rs index c18a9cf..c4a9f2a 100644 --- a/crates/agent-core/src/reconcile.rs +++ b/crates/agent-core/src/reconcile.rs @@ -72,6 +72,9 @@ pub struct Applied { pub helper_instruction: Option, pub reason: String, } +/// Diagnostics only; never accepted as an authorization or reusable lease. +#[derive(Clone,Copy)] +pub struct GateObservation {pub signed_ready:bool,pub local_applied:bool,pub helper_ack_pending:bool} struct Gate { channel: Channel, until: DateTime, @@ -119,6 +122,9 @@ impl ClientSync { self.channel = None; self.pending = None; } + pub fn gate_observation(&self)->Option { + self.gate.as_ref().map(|g|GateObservation{signed_ready:g.ready,local_applied:g.local_applied,helper_ack_pending:g.helper.is_some()}) + } pub fn validity_deadline(&self) -> Option> { self.gate.as_ref().map(|gate| gate.until) } diff --git a/crates/agent-windows/src/job_readiness.rs b/crates/agent-windows/src/job_readiness.rs new file mode 100644 index 0000000..318d32e --- /dev/null +++ b/crates/agent-windows/src/job_readiness.rs @@ -0,0 +1,156 @@ +//! Bounded passive runtime diagnostics. These observations are never input to +//! job admission, clock qualification, signed Sync or recovery. +#[cfg(windows)] +use anyhow::{Result, ensure}; +use serde::Serialize; +#[cfg(windows)] +use std::{io::Write, path::Path}; +#[derive(Clone, Debug, Serialize)] +pub struct Counts { + pub accepted: u64, + pub requesting: u64, + pub running: u64, + pub verifying: u64, + pub unknown: u64, + pub awaiting_reboot: u64, + pub events: u64, + pub legacy_events: u64, +} +#[derive(Clone, Debug, Serialize)] +pub struct Observation { + pub schema_version: u32, + pub observed_at: chrono::DateTime, + pub agent_id: String, + pub boot_id: String, + pub connection_generation: u64, + pub ready: bool, + pub helper_ready: bool, + pub clock_qualified: bool, + pub clock_failure: &'static str, + pub needs_sync: bool, + pub prepared: bool, + pub helper_control: &'static str, + pub helper_apply_unconfirmed: bool, + pub signed_gate_present: bool, + pub signed_ready: bool, + pub local_applied: bool, + pub helper_ack_pending: bool, + pub signed_valid_until: Option>, + pub counts: Counts, +} +pub fn clock_code(error: &anyhow::Error) -> &'static str { + const CODES: &[&str] = &[ + "TIME_STALE_RESPONSE", + "TIME_NATIVE_NO_EVIDENCE", + "TIME_RUNTIME_NO_EVIDENCE", + "TIME_RUNTIME_NOT_CONFIGURED", + "TIME_NATIVE_EVIDENCE_INVALIDATED", + "TIME_NATIVE_CONNECTION_CHANGED", + "TIME_RUNTIME_CONNECTION_CHANGED", + "TIME_NATIVE_CONFIG_CHANGED", + "TIME_RUNTIME_NO_CONNECTION", + "TIME_LOCAL_WALL_OUTSIDE_BOUND", + "TIME_SOURCE_AGE_AT_CHECK", + ]; + error + .chain() + .find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code)) + .unwrap_or("CLOCK_CHECK_FAILED") +} +/// Refresh is control work, not permission. A temporary loss of the100ms clock +/// evidence never discards an otherwise retained signed decision or extends it. +/// Fresh clock/readiness checks remain mandatory at each actual job gate. +pub fn refresh_due( + needs_sync: bool, + elapsed: std::time::Duration, + valid_until: Option>, + clock_upper: Option>, +) -> bool { + needs_sync + || elapsed >= std::time::Duration::from_secs(40) + || valid_until.is_none() + || valid_until + .zip(clock_upper) + .is_some_and(|(until, upper)| upper >= until) +} +#[cfg(windows)] +pub fn persist(state: &Path, value: &Observation) -> Result<()> { + use crate::native; + native::private_path(state, true)?; + let path = state.join("job-readiness.json"); + if path.try_exists()? { + native::private_path(&path, false)?; + } + let temporary = state.join("job-readiness.pending"); + let raw = serde_json::to_vec(value)?; + ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT"); + // A stale pending file is neither overwritten nor treated as current. + use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle}; + use windows_sys::Win32::Storage::FileSystem::*; + fn identity(file: &std::fs::File) -> Result<[u32; 3]> { + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + ensure!( + unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0 + && info.dwFileAttributes + & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY) + == 0 + && info.nNumberOfLinks == 1, + "JOB_READINESS_FILE_ID" + ); + Ok([ + info.dwVolumeSerialNumber, + info.nFileIndexHigh, + info.nFileIndexLow, + ]) + } + let mut f = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .share_mode(FILE_SHARE_READ) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(&temporary)?; + let created = identity(&f)?; + let result = (|| -> Result<()> { + native::private_path(&temporary, false)?; + f.write_all(&raw)?; + f.sync_all()?; + Ok(()) + })(); + drop(f); + let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into)); + if result.is_err() { + // Remove only the exact file this call created, never a replacement, + // reparse point or unknown pending file from a previous invocation. + if let Ok(check) = std::fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(&temporary) + && identity(&check).is_ok_and(|value| value == created) + { + let _ = std::fs::remove_file(&temporary); + } + } + result +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() { + let now = chrono::DateTime::from_timestamp(100, 0).unwrap(); + let until = now + chrono::Duration::seconds(30); + let one = std::time::Duration::from_secs(1); + assert!(!refresh_due(false, one, Some(until), None)); + assert!(!refresh_due(false, one, Some(until), Some(now))); + assert!(refresh_due(false, one, Some(until), Some(until))); + assert!(refresh_due(false, one, None, None)); + assert!(refresh_due(true, one, Some(until), None)); + assert!(refresh_due( + false, + std::time::Duration::from_secs(40), + Some(until), + None + )); + } +} diff --git a/crates/agent-windows/src/lib.rs b/crates/agent-windows/src/lib.rs index f3f652e..7085308 100644 --- a/crates/agent-windows/src/lib.rs +++ b/crates/agent-windows/src/lib.rs @@ -40,3 +40,5 @@ pub mod update_native; pub const SERVICE_NAME: &str = "RmmDiscoveryWindows"; pub const CAPABILITIES: &[&str] = &["windows.base/1", "heartbeat/1"]; + +pub mod job_readiness; diff --git a/crates/agent-windows/src/runtime.rs b/crates/agent-windows/src/runtime.rs index f60d933..83fc8f9 100644 --- a/crates/agent-windows/src/runtime.rs +++ b/crates/agent-windows/src/runtime.rs @@ -518,10 +518,13 @@ impl HelperControl { } fn abort(self) -> Option { match self { - Self::Snapshot(_, t) | Self::Apply(_, t) => { + Self::Snapshot(_, t) => { t.abort(); None } + // Apply may already have committed in the helper. Retain its + // bounded worker until settled; never lose the only reply owner. + apply @ Self::Apply(..) => Some(apply), mirror @ Self::Mirror(..) => Some(mirror), } } @@ -570,6 +573,7 @@ struct Agent { completed_rx: tokio::sync::mpsc::Receiver, running: Option, deferred_permit: Option<(u64, ExecutionPermit)>, + readiness_worker: Option>>, } impl Agent { fn rotation_without_effects(&self) -> Result<()> { @@ -836,6 +840,30 @@ impl Agent { } } } + #[cfg(feature = "system-helper")] + async fn drain_helper_apply(&mut self) { + // Keep the same bounded original worker and independent time lane + // alive until its acknowledgement settles. This never opens a gate, + // confirms a new generation, retries Apply or authorizes an effect. + let Some(control) = self.helper_control.take() else { + return; + }; + match control { + HelperControl::Apply(_, task) => { + if let Ok(Ok(Ok(_snapshot))) = + tokio::time::timeout(Duration::from_secs(70), task).await + { + self.helper_apply_unconfirmed = false; + } + } + mirror @ HelperControl::Mirror(..) => { + self.helper_control = Some(mirror); + } + HelperControl::Snapshot(_, task) => { + task.abort(); + } + } + } fn close_gate(&mut self) { self.fence.close(); #[cfg(feature = "system-helper")] @@ -885,6 +913,71 @@ impl Agent { false } } + fn helper_control_busy_for_sync(&self) -> bool { + #[cfg(feature = "system-helper")] + { + self.helper_control.is_some() + || self.helper_probe.is_some() + || !self.helper_checked + || self.helper_refused + } + #[cfg(not(feature = "system-helper"))] + { + false + } + } + fn readiness_observation( + &self, + generation: u64, + needs_sync: bool, + ) -> Result { + use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code}; + let clock = self.clock.reading(); + let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?; + let mut counts = counts; + counts.events = + self.journal + .connection() + .query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?; + counts.legacy_events = self.journal.count()?; + let gate = self.sync.as_ref().and_then(|s| s.gate_observation()); + #[cfg(feature = "system-helper")] + let (helper_control, unconfirmed) = ( + match &self.helper_control { + Some(HelperControl::Snapshot(..)) => "snapshot", + Some(HelperControl::Mirror(..)) => "mirror", + Some(HelperControl::Apply(..)) => "apply", + None if self.helper_probe.is_some() => "probe", + None if self.helper_refused => "refused", + None if !self.helper_checked => "not_checked", + None => "idle", + }, + self.helper_apply_unconfirmed, + ); + #[cfg(not(feature = "system-helper"))] + let (helper_control, unconfirmed) = ("not_configured", false); + Ok(Observation { + schema_version: 1, + observed_at: Utc::now(), + agent_id: self.identity.agent_id.clone(), + boot_id: self.boot.clone(), + connection_generation: generation, + ready: self.ready(generation), + helper_ready: self.ready_identity(generation, true), + clock_qualified: clock.is_ok(), + clock_failure: clock.as_ref().err().map_or("", clock_code), + needs_sync, + prepared: self.prepared, + helper_control, + helper_apply_unconfirmed: unconfirmed, + signed_gate_present: gate.is_some(), + signed_ready: gate.is_some_and(|g| g.signed_ready), + local_applied: gate.is_some_and(|g| g.local_applied), + helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending), + signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()), + counts, + }) + } #[cfg(feature = "system-helper")] async fn poll_helper( &mut self, @@ -958,8 +1051,14 @@ impl Agent { .await?; } HelperControl::Apply(expected, task) => { - ensure!(expected == generation, "HELPER_CHANNEL_CHANGED"); let snapshot = task.await??; + if expected != generation { + // A drained old reply is retained evidence only. A new + // generation requires a fresh snapshot and signed Sync. + self.helper_apply_unconfirmed = false; + self.close_gate(); + return Ok(()); + } self.helper_anchor = snapshot.no_start_anchor; let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw); let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?; @@ -1267,7 +1366,10 @@ impl Agent { generation: u64, ) -> Result<()> { self.fence.close(); - ensure!(!self.helper_control_pending(), "HELPER_CONTROL_NOT_SETTLED"); + ensure!( + !self.helper_control_busy_for_sync(), + "HELPER_CONTROL_NOT_SETTLED" + ); #[cfg(feature = "system-helper")] if let Some(helper) = self.helper.clone() { ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING"); @@ -1479,6 +1581,10 @@ impl Agent { )?; return Ok(()); } + if self.clock.reading().is_err() { + self.deferred_permit = Some((generation, permit)); + return Ok(()); + } ensure!( self.ready(generation) && self.running.is_none(), "FENCED_SYNC_REQUIRED" @@ -1489,7 +1595,13 @@ impl Agent { .context("JOB_BINDING_REQUIRED")? .clone(); let request = jobs::permit_request(&offer, &permit)?; - let now = self.clock.now()?; + let now = match self.clock.now() { + Ok(now) => now, + Err(_) => { + self.deferred_permit = Some((generation, permit)); + return Ok(()); + } + }; // Authenticate before interpreting timing. A freshly issued grant may // need two seconds before the full uncertainty interval is inside it. let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?; @@ -1845,7 +1957,10 @@ impl Agent { let result = self.session_inner(stop).await; self.close_gate(); #[cfg(feature = "system-helper")] - self.stop_time_refresh().await; + { + self.drain_helper_apply().await; + self.stop_time_refresh().await; + } result } async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> { @@ -1928,7 +2043,9 @@ impl Agent { let mut rotation_after = Duration::from_secs(30); let mut heartbeat = Instant::now() - Duration::from_secs(60); let mut heartbeat_after = Duration::from_secs(30); - let mut tick = tokio::time::interval(Duration::from_secs(1)); + let mut tick = tokio::time::interval(Duration::from_millis(20)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut readiness_at = Instant::now() - Duration::from_secs(5); while !stop.load(Ordering::SeqCst) || self.running.is_some() { if stop.load(Ordering::SeqCst) { self.close_gate(); @@ -1953,7 +2070,7 @@ impl Agent { if !stop.load(Ordering::SeqCst) && !busy { if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}} if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;} - if self.sync.is_some() && !self.prepared && !self.helper_control_pending() && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) { + if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) { if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now(); } if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 { @@ -2010,6 +2127,22 @@ impl Agent { } } } + if readiness_at.elapsed() >= Duration::from_secs(5) { + readiness_at = Instant::now(); + if self + .readiness_worker + .as_ref() + .is_none_or(|worker| worker.is_finished()) + { + self.readiness_worker.take(); + if let Ok(observation) = self.readiness_observation(generation, needs_sync) { + let state = self.config.state_directory.clone(); + self.readiness_worker = Some(tokio::task::spawn_blocking(move || { + rmm_agent_windows::job_readiness::persist(&state, &observation) + })); + } + } + } if stop.load(Ordering::SeqCst) { continue; } @@ -2232,7 +2365,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> { helper_checked:false, #[cfg(feature="system-helper")] helper_refused:false, - prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None, + prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None, }; agent.recover_interrupted()?; let mut backoff = 1; @@ -2317,6 +2450,29 @@ pub fn main() -> Result<()> { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "system-helper")] + #[tokio::test] + async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() { + let (sender, receiver) = tokio::sync::oneshot::channel::<()>(); + let task = tokio::spawn(async move { + receiver.await?; + Err(anyhow::anyhow!("original fixture reply")) + }); + let kept = HelperControl::Apply(7, task) + .abort() + .expect("Apply must remain owned"); + assert!(!kept.finished()); + sender.send(()).unwrap(); + let HelperControl::Apply(generation, task) = kept else { + panic!("wrong retained operation") + }; + assert_eq!(generation, 7); + assert_eq!( + task.await.unwrap().err().unwrap().to_string(), + "original fixture reply" + ); + } + #[cfg(feature = "system-helper")] #[test] fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() { diff --git a/docs/evidence/windows-job-scheduling-20261116.json b/docs/evidence/windows-job-scheduling-20261116.json new file mode 100644 index 0000000..cf4cf6f --- /dev/null +++ b/docs/evidence/windows-job-scheduling-20261116.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "source_files": [ + { + "path": "crates/agent-windows/src/runtime.rs", + "source_sha256": "571eb94025fe0fb1fecbab050ff034d21cad8122193996606579768480e6b1cf", + "target_sha256": "e542569c3e94fd9fdef32523df58c184718a52840b7a549e6bcc2199852507bc" + }, + { + "path": "crates/agent-windows/src/job_readiness.rs", + "source_sha256": null, + "target_sha256": "ff0f6814e08d471db2d1dfcf04bfb9dc516a108a1f7e305c4fcec376f16a0b6f" + }, + { + "path": "crates/agent-windows/src/lib.rs", + "source_sha256": "609e6bf4cd895b81f0153113b17444fe8453212d6b691b73dcea26a63a920d45", + "target_sha256": "574e55d368e7c6668065c42f9d1898856dc659a46d21dfec5c6bd623a0d98138" + }, + { + "path": "crates/agent-core/src/reconcile.rs", + "source_sha256": "d924e3cfd1f782169f3dee4dd7875cd5fba493fbaecb64e13abcb16729be9f74", + "target_sha256": "06180f8897b4a69eec4ed7cb265caa7afaf156e13283b1025fda944d0cc3e5c0" + } + ], + "scope": "Native targeted scheduler and original Apply-worker ownership unit checks; no deployed runtime or job success claim", + "results": [ + { + "exit_code": 0, + "stdout": "\nrunning 1 test\ntest job_readiness::tests::transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 31 filtered out; finished in 0.00s\n\n", + "stderr": "", + "binary_sha256": "18ac38c319cf6019309738ae7022360b221ff8a4a4aa8ee99393f921e40c0909", + "native_platform": "cloud-linux-x86_64", + "uid": 65534, + "isolated_network": true, + "no_new_privs": true, + "verified": true + }, + { + "exit_code": 0, + "product_job": false, + "binary_sha256": "6562c0664bf7238ae090bf1d0ec045db05b2177fb71bc8e180a28c4066da63a6", + "bootstrap_administrator": true, + "stderr": "", + "native_platform": "windows-11-x86_64", + "stdout": "\nrunning 1 test\ntest job_readiness::tests::transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 58 filtered out; finished in 0.00s\n\n", + "verified": true + }, + { + "exit_code": 0, + "product_job": false, + "binary_sha256": "a663806b07630d80b818324275ff292b02725bc591975654fe849c2e8ff5578f", + "bootstrap_administrator": true, + "stderr": "", + "native_platform": "windows-11-x86_64", + "stdout": "\nrunning 1 test\ntest runtime::tests::gate_close_retains_the_original_helper_apply_worker_without_replaying_it ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 8 filtered out; finished in 0.00s\n\n", + "verified": true + } + ] +} diff --git a/docs/windows-mvp-completion.md b/docs/windows-mvp-completion.md index 44f5b64..ae2fff6 100644 --- a/docs/windows-mvp-completion.md +++ b/docs/windows-mvp-completion.md @@ -213,3 +213,28 @@ Es wurden weder Rechte geändert noch die Prüfung gelockert. Dieser Fehler bedeutet: **keine erfolgreiche native Updater-OS-Abnahme und keine Apply-Capability**. Die vier vorherigen portablen Verträge bleiben davon getrennte, erfolgreiche Prüfergebnisse. + +## Schedulerfehler eingegrenzt, Korrektur nativ geprüft + +Die bisherige Windows-Schleife startete bei jedem vorübergehend nicht mehr +frischen 100-ms-Zeitbeleg einen vollständigen neuen SignedSync, bevor sie +angenommene Jobs auswählte. Das kann den Jobstart verdrängen. Die Korrektur +erneuert den signierten Abgleich nur bei tatsächlichem Bedarf beziehungsweise +Ablauf und prüft wartende Arbeit alle 20 ms ohne Tick-Rückstau. Der +100-ms-Vertrag und sämtliche frischen Start-/Permitprüfungen bleiben Pflicht; +ein vorhandener SignedSync macht einen veralteten Zeitbeleg nicht gültig. + +Zusätzlich ging beim Schließen eines Gates der Besitzer eines bereits +ausgesandten Helper-Apply verloren. Der neue Ablauf behält diesen begrenzten +Originalworker und lässt ihn beim kontrollierten Schließen auslaufen, ohne +Apply zu wiederholen oder aus einem alten Kanal Bereitschaft abzuleiten. +Eine begrenzte passive Statusdatei zeigt künftig die tatsächlichen +Sperrgründe; ihr Inhalt ist niemals Eingabe einer Genehmigung oder Freigabe. + +Am 16. November bestand die gezielte Schedulingprüfung zuerst nativ auf der +Cloud-Linux-VM (1/1). Auf Windows bestanden anschließend die Schedulingprüfung +und die Originalworker-Besitzprüfung (je 1/1). Das isolierte Windows-GNU- +Clippy lief für alle Targets mit `-D warnings` erfolgreich. +[Exakte Quellen, native Resultate und Binärhashes](evidence/windows-job-scheduling-20261116.json). +Diese Ergebnisse sind **noch kein erfolgreicher allgemeiner Skriptlauf und +kein Nachweis eines bereits erfolgten Dienstwechsels**.