Fence Windows sync until the helper confirms the signed decision
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -284,6 +284,15 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
|
||||
"HELPER_JOURNAL_ROLLBACK"
|
||||
);
|
||||
if self.helper_identity == HelperIdentity::System {
|
||||
// A locally committed Windows decision is not proof of its
|
||||
// application by SYSTEM. Never advance either checkpoint
|
||||
// from an older helper head, including after process loss.
|
||||
ensure!(
|
||||
meta.decision.is_some() && h.previous_decision_id == meta.decision,
|
||||
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
|
||||
|
||||
@@ -1026,6 +1026,72 @@ fn helper_gate_requires_exact_signed_instruction_and_applied_checkpoint() {
|
||||
assert!(!c.can_start(&f.channel, true, &f.clock));
|
||||
}
|
||||
#[test]
|
||||
fn windows_unconfirmed_helper_head_cannot_replace_a_committed_decision() {
|
||||
let mut f = Fixture::new();
|
||||
f.helper_identity = HelperIdentity::System;
|
||||
let mut client = f.client();
|
||||
let mut helper = HelperSnapshot {
|
||||
journal_id: Uuid::new_v4(),
|
||||
checkpoint: 1,
|
||||
previous_decision_id: None,
|
||||
attempts: vec![],
|
||||
};
|
||||
let first = client
|
||||
.prepare(
|
||||
&mut f.db,
|
||||
&f.ledger,
|
||||
f.channel.clone(),
|
||||
Some(&helper),
|
||||
&f.clock,
|
||||
)
|
||||
.unwrap();
|
||||
let decision = f.decision(&first, true);
|
||||
f.apply(&mut client, &decision).unwrap();
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
|
||||
for previous in [None, Some(Uuid::new_v4())] {
|
||||
let mut reopened = f.client();
|
||||
helper.previous_decision_id = previous;
|
||||
assert_eq!(
|
||||
reopened
|
||||
.prepare(
|
||||
&mut f.db,
|
||||
&f.ledger,
|
||||
f.channel.clone(),
|
||||
Some(&helper),
|
||||
&f.clock
|
||||
)
|
||||
.err()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
|
||||
);
|
||||
assert!(!reopened.can_start(&f.channel, false, &f.clock));
|
||||
let count: u64 =
|
||||
f.db.query_row("SELECT count(*) FROM reconcile_requests", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
helper.previous_decision_id = Some(decision.decision_id);
|
||||
helper.checkpoint += 1;
|
||||
f.channel.connection_generation += 1;
|
||||
let mut reopened = f.client();
|
||||
let next = reopened
|
||||
.prepare(
|
||||
&mut f.db,
|
||||
&f.ledger,
|
||||
f.channel.clone(),
|
||||
Some(&helper),
|
||||
&f.clock,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
next.request.helper_previous_decision_id,
|
||||
decision.decision_id.to_string()
|
||||
);
|
||||
assert!(!reopened.can_start(&f.channel, false, &f.clock));
|
||||
}
|
||||
#[test]
|
||||
fn independent_helper_checkpoint_regression_is_sticky() {
|
||||
let mut f = Fixture::new();
|
||||
let mut c = f.client();
|
||||
|
||||
@@ -5,7 +5,7 @@ use anyhow::{Context, Result, ensure};
|
||||
use rmm_agent_core::{
|
||||
auth::{Binding, Outcome, Request},
|
||||
helper_reconcile::{NoStartAnchor, Snapshot},
|
||||
reconcile::HelperSnapshot,
|
||||
reconcile::{HelperSnapshot, SyncClock},
|
||||
};
|
||||
use rmm_domain::jobs::{Action, JobPayload};
|
||||
use rmm_protocol::{SyncAttempt, SyncResponse};
|
||||
@@ -473,16 +473,49 @@ impl Client {
|
||||
response: &SyncResponse,
|
||||
request_bytes: Vec<u8>,
|
||||
boot: uuid::Uuid,
|
||||
valid_until: chrono::DateTime<chrono::Utc>,
|
||||
clock: &impl SyncClock,
|
||||
) -> Result<ControlSnapshot> {
|
||||
let reply = self
|
||||
.call(ipc::Operation::Apply {
|
||||
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
},
|
||||
request_bytes,
|
||||
})
|
||||
.await?;
|
||||
let operation = ipc::Operation::Apply {
|
||||
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
},
|
||||
request_bytes,
|
||||
};
|
||||
let started = std::time::Instant::now();
|
||||
let reply = loop {
|
||||
ensure!(
|
||||
started.elapsed() < Duration::from_secs(60),
|
||||
"HELPER_APPLY_EXPIRED"
|
||||
);
|
||||
let Ok(reading) = clock.reading() else {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
continue;
|
||||
};
|
||||
ensure!(
|
||||
reading.upper < valid_until && reading.own_wall < valid_until,
|
||||
"HELPER_APPLY_EXPIRED"
|
||||
);
|
||||
match self.call(operation.clone()).await {
|
||||
Ok(reply) => break reply,
|
||||
Err(error) => {
|
||||
// Retry only contention before dispatch. An authenticated
|
||||
// denial or an unknown reply keeps this round closed.
|
||||
let diagnostic = ipc::diagnostic(&error);
|
||||
let pipe_busy = diagnostic.stage == "client_open_pipe"
|
||||
&& diagnostic.win32
|
||||
== Some(windows_sys::Win32::Foundation::ERROR_PIPE_BUSY);
|
||||
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
|
||||
return Err(error);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
if let ipc::ReplyBody::Denied { code } = &reply.body {
|
||||
return Err(ipc::remote_denial(ipc::Stage::ClientReplyBinding, code));
|
||||
}
|
||||
let ipc::ReplyBody::Applied {
|
||||
snapshot,
|
||||
no_start_anchor,
|
||||
|
||||
@@ -498,6 +498,8 @@ struct Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
prepared_request: Option<Vec<u8>>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_apply_unconfirmed: bool,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_lane: Arc<tokio::sync::Semaphore>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper: Option<rmm_agent_windows::helper_runtime::Client>,
|
||||
@@ -693,6 +695,7 @@ impl Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.helper_control.is_some()
|
||||
|| self.helper_apply_unconfirmed
|
||||
|| self.helper_probe.is_some()
|
||||
|| !self.helper_checked
|
||||
|| self.helper_refused
|
||||
@@ -789,6 +792,7 @@ impl Agent {
|
||||
} else {
|
||||
sync.confirm_helper(&mapped)?;
|
||||
}
|
||||
self.helper_apply_unconfirmed = false;
|
||||
self.open_fence(generation)?;
|
||||
}
|
||||
}
|
||||
@@ -825,6 +829,7 @@ impl Agent {
|
||||
}
|
||||
if self.helper_probe.is_none()
|
||||
&& self.helper_control.is_none()
|
||||
&& !self.helper_apply_unconfirmed
|
||||
&& !self.prepared
|
||||
&& self.running.is_none()
|
||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||
@@ -1124,6 +1129,10 @@ impl Agent {
|
||||
self.prepared = false;
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
||||
#[cfg(feature = "system-helper")]
|
||||
if applied.helper_instruction.is_some() {
|
||||
self.helper_apply_unconfirmed = true;
|
||||
}
|
||||
sync.apply_consumption(
|
||||
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
||||
&self.clock,
|
||||
@@ -1140,11 +1149,15 @@ impl Agent {
|
||||
.prepared_request
|
||||
.take()
|
||||
.context("ORIGINAL_SYNC_REQUEST_REQUIRED")?;
|
||||
let until = sync.validity_deadline().context("SIGNED_LEASE_REQUIRED")?;
|
||||
let clock = self.clock.clone();
|
||||
let boot = self.channel(generation)?.boot_id;
|
||||
self.helper_control = Some(HelperControl::Apply(
|
||||
generation,
|
||||
tokio::spawn(async move {
|
||||
helper.apply(&instruction.response, request, boot).await
|
||||
helper
|
||||
.apply(&instruction.response, request, boot, until, &clock)
|
||||
.await
|
||||
}),
|
||||
));
|
||||
return Ok(());
|
||||
@@ -1594,6 +1607,12 @@ impl Agent {
|
||||
}
|
||||
async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||
self.close_gate();
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
// Reconnect discards the old channel. ClientSync::prepare checks
|
||||
// the authenticated helper head against the durable agent journal.
|
||||
self.helper_apply_unconfirmed = false;
|
||||
}
|
||||
let result = self.session_inner(stop).await;
|
||||
self.close_gate();
|
||||
#[cfg(feature = "system-helper")]
|
||||
@@ -1922,6 +1941,8 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
#[cfg(feature="system-helper")]
|
||||
prepared_request:None,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_apply_unconfirmed:false,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_lane:Arc::new(tokio::sync::Semaphore::new(1)),
|
||||
#[cfg(feature="system-helper")]
|
||||
helper:None,
|
||||
|
||||
@@ -290,3 +290,18 @@ signierte Ausstellungszeitpunkt der ersten Entscheidung lag bei
|
||||
21:15:39,528352853 UTC. Die Entscheidung folgte damit rund 1,968 Sekunden
|
||||
auf die Erfassung. Eine zeitlich vor der Erfassung ausgestellte Antwort
|
||||
scheidet für diesen Fall aus.
|
||||
|
||||
### Vorbeugende Korrektur im Windows-Abgleich
|
||||
|
||||
Nach dem lokalen Speichern einer signierten Entscheidung wartet der
|
||||
Windows-Agent auf die zugehörige bestätigte Helper-Antwort. Solange diese
|
||||
aussteht, kann er keinen Folgeabgleich mit leerem Helper-Vorgänger senden.
|
||||
Nach einem Prozessneustart prüft er den gemeldeten Vorgänger gegen sein
|
||||
dauerhaftes Journal. Bei belegter Belegung vor dem IPC-Versand kann er
|
||||
denselben signierten Inhalt innerhalb der bestehenden Frist zustellen.
|
||||
|
||||
Die lokale Prüfung bestand mit 28 Reconcile- und 31 Helper-Reconcile-Tests
|
||||
sowie Windows-Cross-Clippy mit `system-helper` und `-D warnings`.
|
||||
Der neue Regressionstest reproduziert fehlende und fremde Vorgänger nach
|
||||
dem ersten lokalen Commit und akzeptiert den später bestätigten richtigen
|
||||
Vorgänger. Die bestehende Sperre der VM bleibt im Journal archiviert.
|
||||
|
||||
Reference in New Issue
Block a user