Wait for the SYSTEM helper's own clock before sync apply
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -507,7 +507,9 @@ impl Client {
|
||||
reading.upper < valid_until && reading.own_wall < valid_until,
|
||||
"HELPER_APPLY_EXPIRED"
|
||||
);
|
||||
match self.call(operation.clone()).await {
|
||||
// The helper waits at most3s for ITS own fresh time evidence in
|
||||
// this same original call. Longer transport timeout is no lease.
|
||||
match self.call_bounded(operation.clone(),Duration::from_secs(8)).await {
|
||||
Ok(reply) => break reply,
|
||||
Err(error) => {
|
||||
// Retry only contention before dispatch. An authenticated
|
||||
|
||||
@@ -843,6 +843,7 @@ pub struct SystemHelper {
|
||||
boot: WindowsBoot,
|
||||
sync: Option<HelperSync<DalekVerifier>>,
|
||||
last_diagnostic: Option<Instant>,
|
||||
last_apply_diagnostic: Option<Instant>,
|
||||
unsettled_msi: Option<UnsettledMsi>,
|
||||
}
|
||||
impl SystemHelper {
|
||||
@@ -867,6 +868,7 @@ impl SystemHelper {
|
||||
boot,
|
||||
sync,
|
||||
last_diagnostic: None,
|
||||
last_apply_diagnostic: None,
|
||||
unsettled_msi: None,
|
||||
})
|
||||
}
|
||||
@@ -920,6 +922,30 @@ impl SystemHelper {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn wait_apply_clock(&self,issued:chrono::DateTime<chrono::Utc>,until:chrono::DateTime<chrono::Utc>)->Result<()> {
|
||||
crate::job_readiness::wait_for_helper_apply_clock(&self.clock,issued,until)
|
||||
}
|
||||
/// Passive, bounded static denial metadata only. Never feeds a gate.
|
||||
fn record_apply_error(&mut self,error:&anyhow::Error)->Result<()> {
|
||||
if self.last_apply_diagnostic.is_some_and(|at|at.elapsed()<Duration::from_secs(5)){return Ok(());}
|
||||
self.last_apply_diagnostic=Some(Instant::now());self.recheck()?;
|
||||
const CODES:&[&str]=&["SYSTEM_APPLY_CLOCK_WAIT_EXPIRED","SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED","SYNC_PROFILE_OR_CLOCK","FRESH_PROCESS_SNAPSHOT_REQUIRED","OLD_PROCESS_DECISION_OR_BOOT","ROOT_SNAPSHOT_REQUEST_MISMATCH","TIME_RUNTIME_NO_EVIDENCE","TIME_NATIVE_NO_EVIDENCE","TIME_STALE_RESPONSE","TIME_NATIVE_EVIDENCE_INVALIDATED","TIME_SOURCE_AGE_AT_CHECK","TIME_LOCAL_WALL_OUTSIDE_BOUND","TIME_NATIVE_CONFIG_CHANGED","TIME_RUNTIME_CONNECTION_CHANGED","TIME_RUNTIME_GENERATION"];
|
||||
let code=error.chain().find_map(|e|CODES.iter().copied().find(|c|*c==e.to_string())).unwrap_or("SYSTEM_APPLY_VALIDATION_OR_STORAGE_FAILURE");
|
||||
let path=Path::new(STATE_ROOT).join("apply-last-error.json");
|
||||
let pending=Path::new(STATE_ROOT).join(".apply-last-error.pending");
|
||||
let writers=[self.settings.helper.service_sid.clone()];
|
||||
if path.try_exists()?{let _old=PinnedFile::open(&path,false,&[],&writers,false)?;}
|
||||
let bytes=serde_json::to_vec(&serde_json::json!({"schema_version":1,"observed_at_utc":chrono::Utc::now(),"helper_pid":std::process::id(),"code":code,"ready":false,"diagnostic_only":true}))?;
|
||||
ensure!(bytes.len()<=1024,"SYSTEM_APPLY_DIAGNOSTIC_LIMIT");
|
||||
use std::io::Write;
|
||||
let mut file=OpenOptions::new().write(true).create_new(true).open(&pending)?;
|
||||
let identity=ipc::file_identity(&file)?;
|
||||
{let _check=PinnedFile::open_read_shared(&pending,false,&[],&writers)?;}
|
||||
file.write_all(&bytes)?;file.sync_all()?;drop(file);
|
||||
std::fs::rename(&pending,&path)?;
|
||||
let checked=PinnedFile::open(&path,false,&[],&writers,false)?;
|
||||
ensure!(checked.identity==identity,"SYSTEM_APPLY_DIAGNOSTIC_REPLACED");Ok(())
|
||||
}
|
||||
fn recheck(&mut self) -> Result<()> {
|
||||
ipc::require_self(&self.settings.helper)?;
|
||||
for file in &mut self.files {
|
||||
@@ -996,6 +1022,10 @@ impl SystemHelper {
|
||||
serde_json::from_slice(&response.decision_bytes)?;
|
||||
self.clock
|
||||
.require_generation(decision.connection_generation)?;
|
||||
// Same authenticated original call, BEFORE any application.
|
||||
// The independent time pipe continues on its own thread.
|
||||
// No signature/deadline/lease is relaxed and no Execute retry.
|
||||
self.wait_apply_clock(decision.issued_at,decision.valid_until)?;
|
||||
self.sync()?;
|
||||
let value = self.sync.as_mut().unwrap().apply(
|
||||
&mut self.db,
|
||||
@@ -1084,6 +1114,7 @@ impl SystemHelper {
|
||||
let body = match result {
|
||||
Ok(body) => body,
|
||||
Err(e) => {
|
||||
if matches!(call.operation,Operation::Apply{..}) {let _diagnostic=self.record_apply_error(&e);}
|
||||
if let Some(sync) = &mut self.sync {
|
||||
sync.close();
|
||||
}
|
||||
|
||||
@@ -233,3 +233,130 @@ mod response_tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The actual SYSTEM Apply precommit wait, shared with its portable contract
|
||||
/// tests. It reads only the helper's independent clock, never an agent sample.
|
||||
/// A successful wait grants no lease and invokes no execution or application.
|
||||
pub fn wait_for_helper_apply_clock(
|
||||
clock: &impl rmm_agent_core::reconcile::SyncClock,
|
||||
issued: chrono::DateTime<chrono::Utc>,
|
||||
until: chrono::DateTime<chrono::Utc>,
|
||||
) -> anyhow::Result<()> {
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
anyhow::ensure!(
|
||||
started.elapsed() < std::time::Duration::from_secs(3),
|
||||
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
|
||||
);
|
||||
match response_window(
|
||||
issued,
|
||||
until,
|
||||
started.elapsed(),
|
||||
clock.reading().ok().map(|r| (r.own_wall, r.upper)),
|
||||
) {
|
||||
ResponseWindow::WithinWindow => return Ok(()),
|
||||
ResponseWindow::Expired => {
|
||||
anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED")
|
||||
}
|
||||
ResponseWindow::WaitForClock => {
|
||||
std::thread::sleep(std::time::Duration::from_millis(20))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod helper_apply_tests {
|
||||
use super::*;
|
||||
use rmm_agent_core::reconcile::{ClockReading, SyncClock};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{
|
||||
Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
type Sample=Option<(chrono::DateTime<chrono::Utc>,chrono::DateTime<chrono::Utc>)>;
|
||||
struct IndependentClock {
|
||||
samples:Mutex<VecDeque<Sample>>,
|
||||
reads: AtomicUsize,
|
||||
}
|
||||
impl IndependentClock {
|
||||
fn new(
|
||||
samples: Vec<Sample>,
|
||||
) -> Self {
|
||||
Self {
|
||||
samples: Mutex::new(samples.into()),
|
||||
reads: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl SyncClock for IndependentClock {
|
||||
fn now(&self) -> anyhow::Result<chrono::DateTime<chrono::Utc>> {
|
||||
Ok(self.reading()?.own_wall)
|
||||
}
|
||||
fn reading(&self) -> anyhow::Result<ClockReading> {
|
||||
self.reads.fetch_add(1, Ordering::SeqCst);
|
||||
let (own_wall, upper) = self
|
||||
.samples
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.flatten()
|
||||
.ok_or_else(|| anyhow::anyhow!("TIME_NATIVE_NO_EVIDENCE"))?;
|
||||
Ok(ClockReading { own_wall, upper })
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let until = issued + chrono::Duration::seconds(30);
|
||||
let unrelated_agent =
|
||||
IndependentClock::new(vec![Some((issued, issued + chrono::Duration::seconds(1)))]);
|
||||
let helper = IndependentClock::new(vec![
|
||||
None,
|
||||
Some((
|
||||
issued - chrono::Duration::milliseconds(500),
|
||||
issued + chrono::Duration::seconds(1),
|
||||
)),
|
||||
Some((issued, issued + chrono::Duration::seconds(1))),
|
||||
]);
|
||||
let started = Instant::now();
|
||||
wait_for_helper_apply_clock(&helper, issued, until).unwrap();
|
||||
assert_eq!(helper.reads.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(unrelated_agent.reads.load(Ordering::SeqCst), 0);
|
||||
assert!(started.elapsed() >= Duration::from_millis(40));
|
||||
assert!(started.elapsed() < Duration::from_secs(3));
|
||||
}
|
||||
#[test]
|
||||
fn helper_wait_refuses_the_original_deadline_without_a_second_sample() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let until = issued + chrono::Duration::seconds(30);
|
||||
let helper = IndependentClock::new(vec![
|
||||
Some((until - chrono::Duration::seconds(1), until)),
|
||||
Some((issued, issued + chrono::Duration::seconds(1))),
|
||||
]);
|
||||
assert_eq!(
|
||||
wait_for_helper_apply_clock(&helper, issued, until)
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED"
|
||||
);
|
||||
assert_eq!(helper.reads.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
#[test]
|
||||
fn helper_wait_finishes_closed_after_three_seconds_without_evidence() {
|
||||
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||
let helper = IndependentClock::new(vec![]);
|
||||
let started = Instant::now();
|
||||
assert_eq!(
|
||||
wait_for_helper_apply_clock(&helper, issued, issued + chrono::Duration::hours(1))
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
|
||||
);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3));
|
||||
assert!(started.elapsed() < Duration::from_secs(10));
|
||||
assert!(helper.reads.load(Ordering::SeqCst) > 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"profile": "rmm.windows.helper-apply-clock-portable-units.v1",
|
||||
"executed_date": "2026-11-16",
|
||||
"native_platform": "cloud-linux-x86_64",
|
||||
"production_sources_sha256": {
|
||||
"crates/agent-windows/src/job_readiness.rs": "4c187ce66bd37895935ae1cc04e5e6c242536a36e75a86b0c3e96ae077156e75",
|
||||
"crates/agent-windows/src/helper_system.rs": "6dd86f7fc5e7827040d498dd024444c20bde7e123a7d85f7d566d5467f39d00b",
|
||||
"crates/agent-windows/src/helper_runtime.rs": "e2aaf4921d24b3b0f7fa64dc41db2badc0d434cc92cb47cd5ab715dd564799bb"
|
||||
},
|
||||
"verified": true,
|
||||
"passed": 3,
|
||||
"failed": 0,
|
||||
"real_signed_time_verified": false,
|
||||
"rmm_services_started": false,
|
||||
"product_jobs_executed": false,
|
||||
"checks": [
|
||||
{
|
||||
"case": "helper-own-clock-linux",
|
||||
"receipt_sha256": "9d320d58194cb98b82d9ba29f7b54e6e5f8080f3ff1211d8051bcecef692d6f7",
|
||||
"result": {
|
||||
"exit_code": 0,
|
||||
"stdout": "\nrunning 1 test\ntest job_readiness::helper_apply_tests::helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 35 filtered out; finished in 0.04s\n\n",
|
||||
"stderr": "",
|
||||
"binary_sha256": "ef44f9c8ed3664443409c8c4e68d7220418c564057407b0a6f8bff767980808e",
|
||||
"native_platform": "cloud-linux-x86_64",
|
||||
"uid": 65534,
|
||||
"isolated_network": true,
|
||||
"no_new_privs": true,
|
||||
"verified": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "helper-original-deadline-linux",
|
||||
"receipt_sha256": "840e48113e95cdb05f65025de50e4d9a914e7764714afe06d713363aab800540",
|
||||
"result": {
|
||||
"exit_code": 0,
|
||||
"stdout": "\nrunning 1 test\ntest job_readiness::helper_apply_tests::helper_wait_refuses_the_original_deadline_without_a_second_sample ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 35 filtered out; finished in 0.00s\n\n",
|
||||
"stderr": "",
|
||||
"binary_sha256": "ef44f9c8ed3664443409c8c4e68d7220418c564057407b0a6f8bff767980808e",
|
||||
"native_platform": "cloud-linux-x86_64",
|
||||
"uid": 65534,
|
||||
"isolated_network": true,
|
||||
"no_new_privs": true,
|
||||
"verified": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "helper-clock-timeout-linux",
|
||||
"receipt_sha256": "b84543abf26dc627d07b226d27ca2ab4a1efa716c8b88d76ac91b503523fffed",
|
||||
"result": {
|
||||
"exit_code": 0,
|
||||
"stdout": "\nrunning 1 test\ntest job_readiness::helper_apply_tests::helper_wait_finishes_closed_after_three_seconds_without_evidence ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 35 filtered out; finished in 3.01s\n\n",
|
||||
"stderr": "",
|
||||
"binary_sha256": "ef44f9c8ed3664443409c8c4e68d7220418c564057407b0a6f8bff767980808e",
|
||||
"native_platform": "cloud-linux-x86_64",
|
||||
"uid": 65534,
|
||||
"isolated_network": true,
|
||||
"no_new_privs": true,
|
||||
"verified": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"profile": "rmm.windows.helper-apply-clock-portable-units.v1",
|
||||
"executed_date": "2026-11-16",
|
||||
"native_platform": "windows-11-x86_64",
|
||||
"production_sources_sha256": {
|
||||
"crates/agent-windows/src/job_readiness.rs": "4c187ce66bd37895935ae1cc04e5e6c242536a36e75a86b0c3e96ae077156e75",
|
||||
"crates/agent-windows/src/helper_system.rs": "6dd86f7fc5e7827040d498dd024444c20bde7e123a7d85f7d566d5467f39d00b",
|
||||
"crates/agent-windows/src/helper_runtime.rs": "e2aaf4921d24b3b0f7fa64dc41db2badc0d434cc92cb47cd5ab715dd564799bb"
|
||||
},
|
||||
"verified": true,
|
||||
"passed": 3,
|
||||
"failed": 0,
|
||||
"real_signed_time_verified": false,
|
||||
"rmm_services_started": false,
|
||||
"product_jobs_executed": false,
|
||||
"checks": [
|
||||
{
|
||||
"case": "helper-own-clock-windows",
|
||||
"receipt_sha256": "78a2558e6e9c90ada882030d92179daa791b44f66f0f718d23d03d9a3aa25bee",
|
||||
"result": {
|
||||
"exit_code": 0,
|
||||
"product_job": false,
|
||||
"binary_sha256": "262c302e1164a88b8684819211c4d3e11aa6c9c162d54b290678101062c625e2",
|
||||
"bootstrap_administrator": true,
|
||||
"stderr": "",
|
||||
"native_platform": "windows-11-x86_64",
|
||||
"stdout": "\nrunning 1 test\ntest job_readiness::helper_apply_tests::helper_wait_uses_own_fresh_sample_after_unavailable_and_future_issue ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 62 filtered out; finished in 0.04s\n\n",
|
||||
"verified": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "helper-original-deadline-windows",
|
||||
"receipt_sha256": "206c3fa301f3de46caac5b36cd7ee98d0f2ae11d411ea2cf0d9e045b20b660c9",
|
||||
"result": {
|
||||
"exit_code": 0,
|
||||
"product_job": false,
|
||||
"binary_sha256": "262c302e1164a88b8684819211c4d3e11aa6c9c162d54b290678101062c625e2",
|
||||
"bootstrap_administrator": true,
|
||||
"stderr": "",
|
||||
"native_platform": "windows-11-x86_64",
|
||||
"stdout": "\nrunning 1 test\ntest job_readiness::helper_apply_tests::helper_wait_refuses_the_original_deadline_without_a_second_sample ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 62 filtered out; finished in 0.00s\n\n",
|
||||
"verified": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "helper-clock-timeout-windows",
|
||||
"receipt_sha256": "cc0e99a0b850b7b06e20ddc7b2f7c9630f707cb7cc72c03c3d77072bf24db8bd",
|
||||
"result": {
|
||||
"exit_code": 0,
|
||||
"product_job": false,
|
||||
"binary_sha256": "262c302e1164a88b8684819211c4d3e11aa6c9c162d54b290678101062c625e2",
|
||||
"bootstrap_administrator": true,
|
||||
"stderr": "",
|
||||
"native_platform": "windows-11-x86_64",
|
||||
"stdout": "\nrunning 1 test\ntest job_readiness::helper_apply_tests::helper_wait_finishes_closed_after_three_seconds_without_evidence ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 62 filtered out; finished in 3.01s\n\n",
|
||||
"verified": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
# Windows: eigener Zeitbeleg vor Helper-Apply
|
||||
|
||||
Stand 16. November 2026. Implementiert und gezielt nativ geprüft; das neue
|
||||
Produktpaar wurde für diese Abnahme noch nicht installiert oder gestartet.
|
||||
|
||||
Der SYSTEM-Helper wartet innerhalb desselben authentifizierten ursprünglichen
|
||||
Apply-Aufrufs **vor** der Anwendung auf seinen eigenen frischen Zeitbeleg und
|
||||
auf das Erreichen des signierten Ausstellungszeitpunkts. Er wartet höchstens
|
||||
drei Sekunden. Die ursprüngliche Entscheidungsfrist bleibt unverändert; die
|
||||
vollständige reguläre Signatur-/Journal-/Clockprüfung folgt danach weiterhin.
|
||||
Der Client erlaubt für diesen einzelnen Transport acht Sekunden. Eine
|
||||
unbekannte Antwort erlaubt keine erneute Übertragung oder Ausführung.
|
||||
|
||||
Die gemeinsame Produktionsfunktion `wait_for_helper_apply_clock` wurde zuerst
|
||||
auf der echten Linux-Cloud-VM und anschließend mit denselben drei ausgewählten
|
||||
Units auf der Windows-11-VM ausgeführt:
|
||||
|
||||
| Prüfung | Cloud-Linux | Windows |
|
||||
|---|---:|---:|
|
||||
| Eigener Helperbeleg nach fehlendem Beleg und künftigem Ausstellungszeitpunkt | 1 bestanden, 0,04 s | 1 bestanden, 0,04 s |
|
||||
| Originalfrist erreicht: sofort geschlossen, kein zweites Sample | 1 bestanden, 0,00 s | 1 bestanden, 0,00 s |
|
||||
| Kein Beleg: nach begrenzten drei Sekunden geschlossen | 1 bestanden, 3,01 s | 1 bestanden, 3,01 s |
|
||||
|
||||
[Linuxbelege](evidence/windows-helper-clock-20261116-linux.json) und
|
||||
[Windowsbelege](evidence/windows-helper-clock-20261116-windows.json) enthalten
|
||||
Artefakt-/Quellhashes und die tatsächlichen Ausgaben. Linux lief als UID65534
|
||||
mit eigenem Netznamespace und `no_new_privs`. Die Windows-Unitprozesse liefen
|
||||
als Bootstrap; vor und nach der Gruppe waren beide RMM-Dienste tatsächlich
|
||||
`Stopped` (12:27:32 und 12:33:07 UTC). Es gab durch diese Units keine Dienste,
|
||||
Hello-/DB-/TLS-Aufrufe oder Produktjobs. Ein erneuter Upload war zuvor wegen
|
||||
des bereits vorhandenen identischen Artefakts vor jedem Teststart verweigert
|
||||
worden; danach wurde ausschließlich dessen hashgeprüfte vorhandene Datei
|
||||
verwendet.
|
||||
|
||||
Diese Tests verwenden kontrollierte Clockantworten und beweisen die begrenzte
|
||||
Produktwarteschleife, **keinen** realen signierten Zeitvergleich. Tatsächliche
|
||||
Rollen-/Zeitprüfung und regulärer Abgleich des neuen Agent-/Helperpaars stehen
|
||||
nach dessen gesonderter Installation noch aus. Strenges Windows-GNU-Clippy
|
||||
für alle Targets und der isolierte Releasebuild waren erfolgreich.
|
||||
|
||||
Eine normale Apply-Ablehnung hinterlässt künftig höchstens eine begrenzte,
|
||||
statische Datei `apply-last-error.json` im bereits geschützten Helperstate.
|
||||
Sie enthält keine Anfrage-, Skript- oder Schlüsselbytes und fließt nie in
|
||||
Bereitschaft oder Wiederherstellung ein. Der Schutz ersetzt keine allgemeine
|
||||
Heilung divergierender Journalphasen: ein tatsächlicher längerer Zeitverlust
|
||||
kann weiterhin einen geschlossenen administrativen Abschluss erfordern.
|
||||
Reference in New Issue
Block a user