Fence Windows sync until the helper confirms the signed decision
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-15 22:28:49 +01:00
parent 17c53c1602
commit 148969f0c6
5 changed files with 155 additions and 11 deletions
+9
View File
@@ -284,6 +284,15 @@ impl<V: Ed25519Verifier> ClientSync<V> {
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint, old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
"HELPER_JOURNAL_ROLLBACK" "HELPER_JOURNAL_ROLLBACK"
); );
if self.helper_identity == HelperIdentity::System {
// A locally committed Windows decision is not proof of its
// application by SYSTEM. Never advance either checkpoint
// from an older helper head, including after process loss.
ensure!(
meta.decision.is_some() && h.previous_decision_id == meta.decision,
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
);
}
} }
} else { } else {
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED"); ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
+66
View File
@@ -1026,6 +1026,72 @@ fn helper_gate_requires_exact_signed_instruction_and_applied_checkpoint() {
assert!(!c.can_start(&f.channel, true, &f.clock)); assert!(!c.can_start(&f.channel, true, &f.clock));
} }
#[test] #[test]
fn windows_unconfirmed_helper_head_cannot_replace_a_committed_decision() {
let mut f = Fixture::new();
f.helper_identity = HelperIdentity::System;
let mut client = f.client();
let mut helper = HelperSnapshot {
journal_id: Uuid::new_v4(),
checkpoint: 1,
previous_decision_id: None,
attempts: vec![],
};
let first = client
.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock,
)
.unwrap();
let decision = f.decision(&first, true);
f.apply(&mut client, &decision).unwrap();
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
for previous in [None, Some(Uuid::new_v4())] {
let mut reopened = f.client();
helper.previous_decision_id = previous;
assert_eq!(
reopened
.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock
)
.err()
.unwrap()
.to_string(),
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
);
assert!(!reopened.can_start(&f.channel, false, &f.clock));
let count: u64 =
f.db.query_row("SELECT count(*) FROM reconcile_requests", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 1);
}
helper.previous_decision_id = Some(decision.decision_id);
helper.checkpoint += 1;
f.channel.connection_generation += 1;
let mut reopened = f.client();
let next = reopened
.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock,
)
.unwrap();
assert_eq!(
next.request.helper_previous_decision_id,
decision.decision_id.to_string()
);
assert!(!reopened.can_start(&f.channel, false, &f.clock));
}
#[test]
fn independent_helper_checkpoint_regression_is_sticky() { fn independent_helper_checkpoint_regression_is_sticky() {
let mut f = Fixture::new(); let mut f = Fixture::new();
let mut c = f.client(); let mut c = f.client();
+38 -5
View File
@@ -5,7 +5,7 @@ use anyhow::{Context, Result, ensure};
use rmm_agent_core::{ use rmm_agent_core::{
auth::{Binding, Outcome, Request}, auth::{Binding, Outcome, Request},
helper_reconcile::{NoStartAnchor, Snapshot}, helper_reconcile::{NoStartAnchor, Snapshot},
reconcile::HelperSnapshot, reconcile::{HelperSnapshot, SyncClock},
}; };
use rmm_domain::jobs::{Action, JobPayload}; use rmm_domain::jobs::{Action, JobPayload};
use rmm_protocol::{SyncAttempt, SyncResponse}; use rmm_protocol::{SyncAttempt, SyncResponse};
@@ -473,16 +473,49 @@ impl Client {
response: &SyncResponse, response: &SyncResponse,
request_bytes: Vec<u8>, request_bytes: Vec<u8>,
boot: uuid::Uuid, boot: uuid::Uuid,
valid_until: chrono::DateTime<chrono::Utc>,
clock: &impl SyncClock,
) -> Result<ControlSnapshot> { ) -> Result<ControlSnapshot> {
let reply = self let operation = ipc::Operation::Apply {
.call(ipc::Operation::Apply {
response: rmm_agent_core::helper_reconcile::SignedResponse { response: rmm_agent_core::helper_reconcile::SignedResponse {
decision_bytes: response.decision_bytes.clone(), decision_bytes: response.decision_bytes.clone(),
signature: response.signature.clone(), signature: response.signature.clone(),
}, },
request_bytes, request_bytes,
}) };
.await?; let started = std::time::Instant::now();
let reply = loop {
ensure!(
started.elapsed() < Duration::from_secs(60),
"HELPER_APPLY_EXPIRED"
);
let Ok(reading) = clock.reading() else {
tokio::time::sleep(Duration::from_millis(250)).await;
continue;
};
ensure!(
reading.upper < valid_until && reading.own_wall < valid_until,
"HELPER_APPLY_EXPIRED"
);
match self.call(operation.clone()).await {
Ok(reply) => break reply,
Err(error) => {
// Retry only contention before dispatch. An authenticated
// denial or an unknown reply keeps this round closed.
let diagnostic = ipc::diagnostic(&error);
let pipe_busy = diagnostic.stage == "client_open_pipe"
&& diagnostic.win32
== Some(windows_sys::Win32::Foundation::ERROR_PIPE_BUSY);
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
return Err(error);
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
};
if let ipc::ReplyBody::Denied { code } = &reply.body {
return Err(ipc::remote_denial(ipc::Stage::ClientReplyBinding, code));
}
let ipc::ReplyBody::Applied { let ipc::ReplyBody::Applied {
snapshot, snapshot,
no_start_anchor, no_start_anchor,
+22 -1
View File
@@ -498,6 +498,8 @@ struct Agent {
#[cfg(feature = "system-helper")] #[cfg(feature = "system-helper")]
prepared_request: Option<Vec<u8>>, prepared_request: Option<Vec<u8>>,
#[cfg(feature = "system-helper")] #[cfg(feature = "system-helper")]
helper_apply_unconfirmed: bool,
#[cfg(feature = "system-helper")]
helper_lane: Arc<tokio::sync::Semaphore>, helper_lane: Arc<tokio::sync::Semaphore>,
#[cfg(feature = "system-helper")] #[cfg(feature = "system-helper")]
helper: Option<rmm_agent_windows::helper_runtime::Client>, helper: Option<rmm_agent_windows::helper_runtime::Client>,
@@ -693,6 +695,7 @@ impl Agent {
#[cfg(feature = "system-helper")] #[cfg(feature = "system-helper")]
{ {
self.helper_control.is_some() self.helper_control.is_some()
|| self.helper_apply_unconfirmed
|| self.helper_probe.is_some() || self.helper_probe.is_some()
|| !self.helper_checked || !self.helper_checked
|| self.helper_refused || self.helper_refused
@@ -789,6 +792,7 @@ impl Agent {
} else { } else {
sync.confirm_helper(&mapped)?; sync.confirm_helper(&mapped)?;
} }
self.helper_apply_unconfirmed = false;
self.open_fence(generation)?; self.open_fence(generation)?;
} }
} }
@@ -825,6 +829,7 @@ impl Agent {
} }
if self.helper_probe.is_none() if self.helper_probe.is_none()
&& self.helper_control.is_none() && self.helper_control.is_none()
&& !self.helper_apply_unconfirmed
&& !self.prepared && !self.prepared
&& self.running.is_none() && self.running.is_none()
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))? && !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
@@ -1124,6 +1129,10 @@ impl Agent {
self.prepared = false; self.prepared = false;
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?; let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?; let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
#[cfg(feature = "system-helper")]
if applied.helper_instruction.is_some() {
self.helper_apply_unconfirmed = true;
}
sync.apply_consumption( sync.apply_consumption(
self.ledger.as_mut().context("LEDGER_REQUIRED")?, self.ledger.as_mut().context("LEDGER_REQUIRED")?,
&self.clock, &self.clock,
@@ -1140,11 +1149,15 @@ impl Agent {
.prepared_request .prepared_request
.take() .take()
.context("ORIGINAL_SYNC_REQUEST_REQUIRED")?; .context("ORIGINAL_SYNC_REQUEST_REQUIRED")?;
let until = sync.validity_deadline().context("SIGNED_LEASE_REQUIRED")?;
let clock = self.clock.clone();
let boot = self.channel(generation)?.boot_id; let boot = self.channel(generation)?.boot_id;
self.helper_control = Some(HelperControl::Apply( self.helper_control = Some(HelperControl::Apply(
generation, generation,
tokio::spawn(async move { tokio::spawn(async move {
helper.apply(&instruction.response, request, boot).await helper
.apply(&instruction.response, request, boot, until, &clock)
.await
}), }),
)); ));
return Ok(()); return Ok(());
@@ -1594,6 +1607,12 @@ impl Agent {
} }
async fn session(&mut self, stop: &AtomicBool) -> Result<()> { async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
self.close_gate(); self.close_gate();
#[cfg(feature = "system-helper")]
{
// Reconnect discards the old channel. ClientSync::prepare checks
// the authenticated helper head against the durable agent journal.
self.helper_apply_unconfirmed = false;
}
let result = self.session_inner(stop).await; let result = self.session_inner(stop).await;
self.close_gate(); self.close_gate();
#[cfg(feature = "system-helper")] #[cfg(feature = "system-helper")]
@@ -1922,6 +1941,8 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
#[cfg(feature="system-helper")] #[cfg(feature="system-helper")]
prepared_request:None, prepared_request:None,
#[cfg(feature="system-helper")] #[cfg(feature="system-helper")]
helper_apply_unconfirmed:false,
#[cfg(feature="system-helper")]
helper_lane:Arc::new(tokio::sync::Semaphore::new(1)), helper_lane:Arc::new(tokio::sync::Semaphore::new(1)),
#[cfg(feature="system-helper")] #[cfg(feature="system-helper")]
helper:None, helper:None,
+15
View File
@@ -290,3 +290,18 @@ signierte Ausstellungszeitpunkt der ersten Entscheidung lag bei
21:15:39,528352853 UTC. Die Entscheidung folgte damit rund 1,968 Sekunden 21:15:39,528352853 UTC. Die Entscheidung folgte damit rund 1,968 Sekunden
auf die Erfassung. Eine zeitlich vor der Erfassung ausgestellte Antwort auf die Erfassung. Eine zeitlich vor der Erfassung ausgestellte Antwort
scheidet für diesen Fall aus. scheidet für diesen Fall aus.
### Vorbeugende Korrektur im Windows-Abgleich
Nach dem lokalen Speichern einer signierten Entscheidung wartet der
Windows-Agent auf die zugehörige bestätigte Helper-Antwort. Solange diese
aussteht, kann er keinen Folgeabgleich mit leerem Helper-Vorgänger senden.
Nach einem Prozessneustart prüft er den gemeldeten Vorgänger gegen sein
dauerhaftes Journal. Bei belegter Belegung vor dem IPC-Versand kann er
denselben signierten Inhalt innerhalb der bestehenden Frist zustellen.
Die lokale Prüfung bestand mit 28 Reconcile- und 31 Helper-Reconcile-Tests
sowie Windows-Cross-Clippy mit `system-helper` und `-D warnings`.
Der neue Regressionstest reproduziert fehlende und fremde Vorgänger nach
dem ersten lokalen Commit und akzeptiert den später bestätigten richtigen
Vorgänger. Die bestehende Sperre der VM bleibt im Journal archiviert.