Fix Windows job scheduling starvation and retain original helper Apply worker
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -72,6 +72,9 @@ pub struct Applied {
|
|||||||
pub helper_instruction: Option<HelperInstruction>,
|
pub helper_instruction: Option<HelperInstruction>,
|
||||||
pub reason: String,
|
pub reason: String,
|
||||||
}
|
}
|
||||||
|
/// Diagnostics only; never accepted as an authorization or reusable lease.
|
||||||
|
#[derive(Clone,Copy)]
|
||||||
|
pub struct GateObservation {pub signed_ready:bool,pub local_applied:bool,pub helper_ack_pending:bool}
|
||||||
struct Gate {
|
struct Gate {
|
||||||
channel: Channel,
|
channel: Channel,
|
||||||
until: DateTime<Utc>,
|
until: DateTime<Utc>,
|
||||||
@@ -119,6 +122,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
|||||||
self.channel = None;
|
self.channel = None;
|
||||||
self.pending = None;
|
self.pending = None;
|
||||||
}
|
}
|
||||||
|
pub fn gate_observation(&self)->Option<GateObservation> {
|
||||||
|
self.gate.as_ref().map(|g|GateObservation{signed_ready:g.ready,local_applied:g.local_applied,helper_ack_pending:g.helper.is_some()})
|
||||||
|
}
|
||||||
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
|
pub fn validity_deadline(&self) -> Option<DateTime<Utc>> {
|
||||||
self.gate.as_ref().map(|gate| gate.until)
|
self.gate.as_ref().map(|gate| gate.until)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
//! Bounded passive runtime diagnostics. These observations are never input to
|
||||||
|
//! job admission, clock qualification, signed Sync or recovery.
|
||||||
|
#[cfg(windows)]
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
|
use serde::Serialize;
|
||||||
|
#[cfg(windows)]
|
||||||
|
use std::{io::Write, path::Path};
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct Counts {
|
||||||
|
pub accepted: u64,
|
||||||
|
pub requesting: u64,
|
||||||
|
pub running: u64,
|
||||||
|
pub verifying: u64,
|
||||||
|
pub unknown: u64,
|
||||||
|
pub awaiting_reboot: u64,
|
||||||
|
pub events: u64,
|
||||||
|
pub legacy_events: u64,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct Observation {
|
||||||
|
pub schema_version: u32,
|
||||||
|
pub observed_at: chrono::DateTime<chrono::Utc>,
|
||||||
|
pub agent_id: String,
|
||||||
|
pub boot_id: String,
|
||||||
|
pub connection_generation: u64,
|
||||||
|
pub ready: bool,
|
||||||
|
pub helper_ready: bool,
|
||||||
|
pub clock_qualified: bool,
|
||||||
|
pub clock_failure: &'static str,
|
||||||
|
pub needs_sync: bool,
|
||||||
|
pub prepared: bool,
|
||||||
|
pub helper_control: &'static str,
|
||||||
|
pub helper_apply_unconfirmed: bool,
|
||||||
|
pub signed_gate_present: bool,
|
||||||
|
pub signed_ready: bool,
|
||||||
|
pub local_applied: bool,
|
||||||
|
pub helper_ack_pending: bool,
|
||||||
|
pub signed_valid_until: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
pub counts: Counts,
|
||||||
|
}
|
||||||
|
pub fn clock_code(error: &anyhow::Error) -> &'static str {
|
||||||
|
const CODES: &[&str] = &[
|
||||||
|
"TIME_STALE_RESPONSE",
|
||||||
|
"TIME_NATIVE_NO_EVIDENCE",
|
||||||
|
"TIME_RUNTIME_NO_EVIDENCE",
|
||||||
|
"TIME_RUNTIME_NOT_CONFIGURED",
|
||||||
|
"TIME_NATIVE_EVIDENCE_INVALIDATED",
|
||||||
|
"TIME_NATIVE_CONNECTION_CHANGED",
|
||||||
|
"TIME_RUNTIME_CONNECTION_CHANGED",
|
||||||
|
"TIME_NATIVE_CONFIG_CHANGED",
|
||||||
|
"TIME_RUNTIME_NO_CONNECTION",
|
||||||
|
"TIME_LOCAL_WALL_OUTSIDE_BOUND",
|
||||||
|
"TIME_SOURCE_AGE_AT_CHECK",
|
||||||
|
];
|
||||||
|
error
|
||||||
|
.chain()
|
||||||
|
.find_map(|e| CODES.iter().copied().find(|code| e.to_string() == *code))
|
||||||
|
.unwrap_or("CLOCK_CHECK_FAILED")
|
||||||
|
}
|
||||||
|
/// Refresh is control work, not permission. A temporary loss of the100ms clock
|
||||||
|
/// evidence never discards an otherwise retained signed decision or extends it.
|
||||||
|
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
|
||||||
|
pub fn refresh_due(
|
||||||
|
needs_sync: bool,
|
||||||
|
elapsed: std::time::Duration,
|
||||||
|
valid_until: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
) -> bool {
|
||||||
|
needs_sync
|
||||||
|
|| elapsed >= std::time::Duration::from_secs(40)
|
||||||
|
|| valid_until.is_none()
|
||||||
|
|| valid_until
|
||||||
|
.zip(clock_upper)
|
||||||
|
.is_some_and(|(until, upper)| upper >= until)
|
||||||
|
}
|
||||||
|
#[cfg(windows)]
|
||||||
|
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
|
||||||
|
use crate::native;
|
||||||
|
native::private_path(state, true)?;
|
||||||
|
let path = state.join("job-readiness.json");
|
||||||
|
if path.try_exists()? {
|
||||||
|
native::private_path(&path, false)?;
|
||||||
|
}
|
||||||
|
let temporary = state.join("job-readiness.pending");
|
||||||
|
let raw = serde_json::to_vec(value)?;
|
||||||
|
ensure!(raw.len() <= 8192, "JOB_READINESS_LIMIT");
|
||||||
|
// A stale pending file is neither overwritten nor treated as current.
|
||||||
|
use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle};
|
||||||
|
use windows_sys::Win32::Storage::FileSystem::*;
|
||||||
|
fn identity(file: &std::fs::File) -> Result<[u32; 3]> {
|
||||||
|
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
|
||||||
|
ensure!(
|
||||||
|
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
|
||||||
|
&& info.dwFileAttributes
|
||||||
|
& (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)
|
||||||
|
== 0
|
||||||
|
&& info.nNumberOfLinks == 1,
|
||||||
|
"JOB_READINESS_FILE_ID"
|
||||||
|
);
|
||||||
|
Ok([
|
||||||
|
info.dwVolumeSerialNumber,
|
||||||
|
info.nFileIndexHigh,
|
||||||
|
info.nFileIndexLow,
|
||||||
|
])
|
||||||
|
}
|
||||||
|
let mut f = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.share_mode(FILE_SHARE_READ)
|
||||||
|
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
|
||||||
|
.open(&temporary)?;
|
||||||
|
let created = identity(&f)?;
|
||||||
|
let result = (|| -> Result<()> {
|
||||||
|
native::private_path(&temporary, false)?;
|
||||||
|
f.write_all(&raw)?;
|
||||||
|
f.sync_all()?;
|
||||||
|
Ok(())
|
||||||
|
})();
|
||||||
|
drop(f);
|
||||||
|
let result = result.and_then(|()| std::fs::rename(&temporary, &path).map_err(Into::into));
|
||||||
|
if result.is_err() {
|
||||||
|
// Remove only the exact file this call created, never a replacement,
|
||||||
|
// reparse point or unknown pending file from a previous invocation.
|
||||||
|
if let Ok(check) = std::fs::OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE)
|
||||||
|
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
|
||||||
|
.open(&temporary)
|
||||||
|
&& identity(&check).is_ok_and(|value| value == created)
|
||||||
|
{
|
||||||
|
let _ = std::fs::remove_file(&temporary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry() {
|
||||||
|
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||||
|
let until = now + chrono::Duration::seconds(30);
|
||||||
|
let one = std::time::Duration::from_secs(1);
|
||||||
|
assert!(!refresh_due(false, one, Some(until), None));
|
||||||
|
assert!(!refresh_due(false, one, Some(until), Some(now)));
|
||||||
|
assert!(refresh_due(false, one, Some(until), Some(until)));
|
||||||
|
assert!(refresh_due(false, one, None, None));
|
||||||
|
assert!(refresh_due(true, one, Some(until), None));
|
||||||
|
assert!(refresh_due(
|
||||||
|
false,
|
||||||
|
std::time::Duration::from_secs(40),
|
||||||
|
Some(until),
|
||||||
|
None
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,3 +40,5 @@ pub mod update_native;
|
|||||||
|
|
||||||
pub const SERVICE_NAME: &str = "RmmDiscoveryWindows";
|
pub const SERVICE_NAME: &str = "RmmDiscoveryWindows";
|
||||||
pub const CAPABILITIES: &[&str] = &["windows.base/1", "heartbeat/1"];
|
pub const CAPABILITIES: &[&str] = &["windows.base/1", "heartbeat/1"];
|
||||||
|
|
||||||
|
pub mod job_readiness;
|
||||||
|
|||||||
@@ -518,10 +518,13 @@ impl HelperControl {
|
|||||||
}
|
}
|
||||||
fn abort(self) -> Option<Self> {
|
fn abort(self) -> Option<Self> {
|
||||||
match self {
|
match self {
|
||||||
Self::Snapshot(_, t) | Self::Apply(_, t) => {
|
Self::Snapshot(_, t) => {
|
||||||
t.abort();
|
t.abort();
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
// Apply may already have committed in the helper. Retain its
|
||||||
|
// bounded worker until settled; never lose the only reply owner.
|
||||||
|
apply @ Self::Apply(..) => Some(apply),
|
||||||
mirror @ Self::Mirror(..) => Some(mirror),
|
mirror @ Self::Mirror(..) => Some(mirror),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -570,6 +573,7 @@ struct Agent {
|
|||||||
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
||||||
running: Option<Running>,
|
running: Option<Running>,
|
||||||
deferred_permit: Option<(u64, ExecutionPermit)>,
|
deferred_permit: Option<(u64, ExecutionPermit)>,
|
||||||
|
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
|
||||||
}
|
}
|
||||||
impl Agent {
|
impl Agent {
|
||||||
fn rotation_without_effects(&self) -> Result<()> {
|
fn rotation_without_effects(&self) -> Result<()> {
|
||||||
@@ -836,6 +840,30 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
|
async fn drain_helper_apply(&mut self) {
|
||||||
|
// Keep the same bounded original worker and independent time lane
|
||||||
|
// alive until its acknowledgement settles. This never opens a gate,
|
||||||
|
// confirms a new generation, retries Apply or authorizes an effect.
|
||||||
|
let Some(control) = self.helper_control.take() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match control {
|
||||||
|
HelperControl::Apply(_, task) => {
|
||||||
|
if let Ok(Ok(Ok(_snapshot))) =
|
||||||
|
tokio::time::timeout(Duration::from_secs(70), task).await
|
||||||
|
{
|
||||||
|
self.helper_apply_unconfirmed = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mirror @ HelperControl::Mirror(..) => {
|
||||||
|
self.helper_control = Some(mirror);
|
||||||
|
}
|
||||||
|
HelperControl::Snapshot(_, task) => {
|
||||||
|
task.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
fn close_gate(&mut self) {
|
fn close_gate(&mut self) {
|
||||||
self.fence.close();
|
self.fence.close();
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
@@ -885,6 +913,71 @@ impl Agent {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
fn helper_control_busy_for_sync(&self) -> bool {
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
|
{
|
||||||
|
self.helper_control.is_some()
|
||||||
|
|| self.helper_probe.is_some()
|
||||||
|
|| !self.helper_checked
|
||||||
|
|| self.helper_refused
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "system-helper"))]
|
||||||
|
{
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn readiness_observation(
|
||||||
|
&self,
|
||||||
|
generation: u64,
|
||||||
|
needs_sync: bool,
|
||||||
|
) -> Result<rmm_agent_windows::job_readiness::Observation> {
|
||||||
|
use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code};
|
||||||
|
let clock = self.clock.reading();
|
||||||
|
let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?;
|
||||||
|
let mut counts = counts;
|
||||||
|
counts.events =
|
||||||
|
self.journal
|
||||||
|
.connection()
|
||||||
|
.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
||||||
|
counts.legacy_events = self.journal.count()?;
|
||||||
|
let gate = self.sync.as_ref().and_then(|s| s.gate_observation());
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
|
let (helper_control, unconfirmed) = (
|
||||||
|
match &self.helper_control {
|
||||||
|
Some(HelperControl::Snapshot(..)) => "snapshot",
|
||||||
|
Some(HelperControl::Mirror(..)) => "mirror",
|
||||||
|
Some(HelperControl::Apply(..)) => "apply",
|
||||||
|
None if self.helper_probe.is_some() => "probe",
|
||||||
|
None if self.helper_refused => "refused",
|
||||||
|
None if !self.helper_checked => "not_checked",
|
||||||
|
None => "idle",
|
||||||
|
},
|
||||||
|
self.helper_apply_unconfirmed,
|
||||||
|
);
|
||||||
|
#[cfg(not(feature = "system-helper"))]
|
||||||
|
let (helper_control, unconfirmed) = ("not_configured", false);
|
||||||
|
Ok(Observation {
|
||||||
|
schema_version: 1,
|
||||||
|
observed_at: Utc::now(),
|
||||||
|
agent_id: self.identity.agent_id.clone(),
|
||||||
|
boot_id: self.boot.clone(),
|
||||||
|
connection_generation: generation,
|
||||||
|
ready: self.ready(generation),
|
||||||
|
helper_ready: self.ready_identity(generation, true),
|
||||||
|
clock_qualified: clock.is_ok(),
|
||||||
|
clock_failure: clock.as_ref().err().map_or("", clock_code),
|
||||||
|
needs_sync,
|
||||||
|
prepared: self.prepared,
|
||||||
|
helper_control,
|
||||||
|
helper_apply_unconfirmed: unconfirmed,
|
||||||
|
signed_gate_present: gate.is_some(),
|
||||||
|
signed_ready: gate.is_some_and(|g| g.signed_ready),
|
||||||
|
local_applied: gate.is_some_and(|g| g.local_applied),
|
||||||
|
helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending),
|
||||||
|
signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()),
|
||||||
|
counts,
|
||||||
|
})
|
||||||
|
}
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
async fn poll_helper(
|
async fn poll_helper(
|
||||||
&mut self,
|
&mut self,
|
||||||
@@ -958,8 +1051,14 @@ impl Agent {
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
HelperControl::Apply(expected, task) => {
|
HelperControl::Apply(expected, task) => {
|
||||||
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
|
||||||
let snapshot = task.await??;
|
let snapshot = task.await??;
|
||||||
|
if expected != generation {
|
||||||
|
// A drained old reply is retained evidence only. A new
|
||||||
|
// generation requires a fresh snapshot and signed Sync.
|
||||||
|
self.helper_apply_unconfirmed = false;
|
||||||
|
self.close_gate();
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
self.helper_anchor = snapshot.no_start_anchor;
|
self.helper_anchor = snapshot.no_start_anchor;
|
||||||
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
|
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
|
||||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||||
@@ -1267,7 +1366,10 @@ impl Agent {
|
|||||||
generation: u64,
|
generation: u64,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.fence.close();
|
self.fence.close();
|
||||||
ensure!(!self.helper_control_pending(), "HELPER_CONTROL_NOT_SETTLED");
|
ensure!(
|
||||||
|
!self.helper_control_busy_for_sync(),
|
||||||
|
"HELPER_CONTROL_NOT_SETTLED"
|
||||||
|
);
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
if let Some(helper) = self.helper.clone() {
|
if let Some(helper) = self.helper.clone() {
|
||||||
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
||||||
@@ -1479,6 +1581,10 @@ impl Agent {
|
|||||||
)?;
|
)?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
if self.clock.reading().is_err() {
|
||||||
|
self.deferred_permit = Some((generation, permit));
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
ensure!(
|
ensure!(
|
||||||
self.ready(generation) && self.running.is_none(),
|
self.ready(generation) && self.running.is_none(),
|
||||||
"FENCED_SYNC_REQUIRED"
|
"FENCED_SYNC_REQUIRED"
|
||||||
@@ -1489,7 +1595,13 @@ impl Agent {
|
|||||||
.context("JOB_BINDING_REQUIRED")?
|
.context("JOB_BINDING_REQUIRED")?
|
||||||
.clone();
|
.clone();
|
||||||
let request = jobs::permit_request(&offer, &permit)?;
|
let request = jobs::permit_request(&offer, &permit)?;
|
||||||
let now = self.clock.now()?;
|
let now = match self.clock.now() {
|
||||||
|
Ok(now) => now,
|
||||||
|
Err(_) => {
|
||||||
|
self.deferred_permit = Some((generation, permit));
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
// Authenticate before interpreting timing. A freshly issued grant may
|
// Authenticate before interpreting timing. A freshly issued grant may
|
||||||
// need two seconds before the full uncertainty interval is inside it.
|
// need two seconds before the full uncertainty interval is inside it.
|
||||||
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
|
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
|
||||||
@@ -1845,7 +1957,10 @@ impl Agent {
|
|||||||
let result = self.session_inner(stop).await;
|
let result = self.session_inner(stop).await;
|
||||||
self.close_gate();
|
self.close_gate();
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
|
{
|
||||||
|
self.drain_helper_apply().await;
|
||||||
self.stop_time_refresh().await;
|
self.stop_time_refresh().await;
|
||||||
|
}
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||||
@@ -1928,7 +2043,9 @@ impl Agent {
|
|||||||
let mut rotation_after = Duration::from_secs(30);
|
let mut rotation_after = Duration::from_secs(30);
|
||||||
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
||||||
let mut heartbeat_after = Duration::from_secs(30);
|
let mut heartbeat_after = Duration::from_secs(30);
|
||||||
let mut tick = tokio::time::interval(Duration::from_secs(1));
|
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
||||||
|
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||||
|
let mut readiness_at = Instant::now() - Duration::from_secs(5);
|
||||||
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
|
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
|
||||||
if stop.load(Ordering::SeqCst) {
|
if stop.load(Ordering::SeqCst) {
|
||||||
self.close_gate();
|
self.close_gate();
|
||||||
@@ -1953,7 +2070,7 @@ impl Agent {
|
|||||||
if !stop.load(Ordering::SeqCst) && !busy {
|
if !stop.load(Ordering::SeqCst) && !busy {
|
||||||
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
||||||
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||||
if self.sync.is_some() && !self.prepared && !self.helper_control_pending() && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) {
|
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
||||||
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||||
}
|
}
|
||||||
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
||||||
@@ -2010,6 +2127,22 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if readiness_at.elapsed() >= Duration::from_secs(5) {
|
||||||
|
readiness_at = Instant::now();
|
||||||
|
if self
|
||||||
|
.readiness_worker
|
||||||
|
.as_ref()
|
||||||
|
.is_none_or(|worker| worker.is_finished())
|
||||||
|
{
|
||||||
|
self.readiness_worker.take();
|
||||||
|
if let Ok(observation) = self.readiness_observation(generation, needs_sync) {
|
||||||
|
let state = self.config.state_directory.clone();
|
||||||
|
self.readiness_worker = Some(tokio::task::spawn_blocking(move || {
|
||||||
|
rmm_agent_windows::job_readiness::persist(&state, &observation)
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if stop.load(Ordering::SeqCst) {
|
if stop.load(Ordering::SeqCst) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -2232,7 +2365,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
|||||||
helper_checked:false,
|
helper_checked:false,
|
||||||
#[cfg(feature="system-helper")]
|
#[cfg(feature="system-helper")]
|
||||||
helper_refused:false,
|
helper_refused:false,
|
||||||
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,
|
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
||||||
};
|
};
|
||||||
agent.recover_interrupted()?;
|
agent.recover_interrupted()?;
|
||||||
let mut backoff = 1;
|
let mut backoff = 1;
|
||||||
@@ -2317,6 +2450,29 @@ pub fn main() -> Result<()> {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() {
|
||||||
|
let (sender, receiver) = tokio::sync::oneshot::channel::<()>();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
receiver.await?;
|
||||||
|
Err(anyhow::anyhow!("original fixture reply"))
|
||||||
|
});
|
||||||
|
let kept = HelperControl::Apply(7, task)
|
||||||
|
.abort()
|
||||||
|
.expect("Apply must remain owned");
|
||||||
|
assert!(!kept.finished());
|
||||||
|
sender.send(()).unwrap();
|
||||||
|
let HelperControl::Apply(generation, task) = kept else {
|
||||||
|
panic!("wrong retained operation")
|
||||||
|
};
|
||||||
|
assert_eq!(generation, 7);
|
||||||
|
assert_eq!(
|
||||||
|
task.await.unwrap().err().unwrap().to_string(),
|
||||||
|
"original fixture reply"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
#[test]
|
#[test]
|
||||||
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
|
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"source_files": [
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/runtime.rs",
|
||||||
|
"source_sha256": "571eb94025fe0fb1fecbab050ff034d21cad8122193996606579768480e6b1cf",
|
||||||
|
"target_sha256": "e542569c3e94fd9fdef32523df58c184718a52840b7a549e6bcc2199852507bc"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/job_readiness.rs",
|
||||||
|
"source_sha256": null,
|
||||||
|
"target_sha256": "ff0f6814e08d471db2d1dfcf04bfb9dc516a108a1f7e305c4fcec376f16a0b6f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/lib.rs",
|
||||||
|
"source_sha256": "609e6bf4cd895b81f0153113b17444fe8453212d6b691b73dcea26a63a920d45",
|
||||||
|
"target_sha256": "574e55d368e7c6668065c42f9d1898856dc659a46d21dfec5c6bd623a0d98138"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-core/src/reconcile.rs",
|
||||||
|
"source_sha256": "d924e3cfd1f782169f3dee4dd7875cd5fba493fbaecb64e13abcb16729be9f74",
|
||||||
|
"target_sha256": "06180f8897b4a69eec4ed7cb265caa7afaf156e13283b1025fda944d0cc3e5c0"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"scope": "Native targeted scheduler and original Apply-worker ownership unit checks; no deployed runtime or job success claim",
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"stdout": "\nrunning 1 test\ntest job_readiness::tests::transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 31 filtered out; finished in 0.00s\n\n",
|
||||||
|
"stderr": "",
|
||||||
|
"binary_sha256": "18ac38c319cf6019309738ae7022360b221ff8a4a4aa8ee99393f921e40c0909",
|
||||||
|
"native_platform": "cloud-linux-x86_64",
|
||||||
|
"uid": 65534,
|
||||||
|
"isolated_network": true,
|
||||||
|
"no_new_privs": true,
|
||||||
|
"verified": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"product_job": false,
|
||||||
|
"binary_sha256": "6562c0664bf7238ae090bf1d0ec045db05b2177fb71bc8e180a28c4066da63a6",
|
||||||
|
"bootstrap_administrator": true,
|
||||||
|
"stderr": "",
|
||||||
|
"native_platform": "windows-11-x86_64",
|
||||||
|
"stdout": "\nrunning 1 test\ntest job_readiness::tests::transient_clock_loss_does_not_restart_a_retained_signed_sync_or_extend_expiry ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 58 filtered out; finished in 0.00s\n\n",
|
||||||
|
"verified": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"product_job": false,
|
||||||
|
"binary_sha256": "a663806b07630d80b818324275ff292b02725bc591975654fe849c2e8ff5578f",
|
||||||
|
"bootstrap_administrator": true,
|
||||||
|
"stderr": "",
|
||||||
|
"native_platform": "windows-11-x86_64",
|
||||||
|
"stdout": "\nrunning 1 test\ntest runtime::tests::gate_close_retains_the_original_helper_apply_worker_without_replaying_it ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 8 filtered out; finished in 0.00s\n\n",
|
||||||
|
"verified": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -213,3 +213,28 @@ Es wurden weder Rechte geändert noch die Prüfung gelockert. Dieser Fehler
|
|||||||
bedeutet: **keine erfolgreiche native Updater-OS-Abnahme und keine
|
bedeutet: **keine erfolgreiche native Updater-OS-Abnahme und keine
|
||||||
Apply-Capability**. Die vier vorherigen portablen Verträge bleiben davon
|
Apply-Capability**. Die vier vorherigen portablen Verträge bleiben davon
|
||||||
getrennte, erfolgreiche Prüfergebnisse.
|
getrennte, erfolgreiche Prüfergebnisse.
|
||||||
|
|
||||||
|
## Schedulerfehler eingegrenzt, Korrektur nativ geprüft
|
||||||
|
|
||||||
|
Die bisherige Windows-Schleife startete bei jedem vorübergehend nicht mehr
|
||||||
|
frischen 100-ms-Zeitbeleg einen vollständigen neuen SignedSync, bevor sie
|
||||||
|
angenommene Jobs auswählte. Das kann den Jobstart verdrängen. Die Korrektur
|
||||||
|
erneuert den signierten Abgleich nur bei tatsächlichem Bedarf beziehungsweise
|
||||||
|
Ablauf und prüft wartende Arbeit alle 20 ms ohne Tick-Rückstau. Der
|
||||||
|
100-ms-Vertrag und sämtliche frischen Start-/Permitprüfungen bleiben Pflicht;
|
||||||
|
ein vorhandener SignedSync macht einen veralteten Zeitbeleg nicht gültig.
|
||||||
|
|
||||||
|
Zusätzlich ging beim Schließen eines Gates der Besitzer eines bereits
|
||||||
|
ausgesandten Helper-Apply verloren. Der neue Ablauf behält diesen begrenzten
|
||||||
|
Originalworker und lässt ihn beim kontrollierten Schließen auslaufen, ohne
|
||||||
|
Apply zu wiederholen oder aus einem alten Kanal Bereitschaft abzuleiten.
|
||||||
|
Eine begrenzte passive Statusdatei zeigt künftig die tatsächlichen
|
||||||
|
Sperrgründe; ihr Inhalt ist niemals Eingabe einer Genehmigung oder Freigabe.
|
||||||
|
|
||||||
|
Am 16. November bestand die gezielte Schedulingprüfung zuerst nativ auf der
|
||||||
|
Cloud-Linux-VM (1/1). Auf Windows bestanden anschließend die Schedulingprüfung
|
||||||
|
und die Originalworker-Besitzprüfung (je 1/1). Das isolierte Windows-GNU-
|
||||||
|
Clippy lief für alle Targets mit `-D warnings` erfolgreich.
|
||||||
|
[Exakte Quellen, native Resultate und Binärhashes](evidence/windows-job-scheduling-20261116.json).
|
||||||
|
Diese Ergebnisse sind **noch kein erfolgreicher allgemeiner Skriptlauf und
|
||||||
|
kein Nachweis eines bereits erfolgten Dienstwechsels**.
|
||||||
|
|||||||
Reference in New Issue
Block a user