Fence Windows sync until the helper confirms the signed decision
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -284,6 +284,15 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
|||||||
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
|
old == h.journal_id && h.checkpoint >= meta.helper_checkpoint,
|
||||||
"HELPER_JOURNAL_ROLLBACK"
|
"HELPER_JOURNAL_ROLLBACK"
|
||||||
);
|
);
|
||||||
|
if self.helper_identity == HelperIdentity::System {
|
||||||
|
// A locally committed Windows decision is not proof of its
|
||||||
|
// application by SYSTEM. Never advance either checkpoint
|
||||||
|
// from an older helper head, including after process loss.
|
||||||
|
ensure!(
|
||||||
|
meta.decision.is_some() && h.previous_decision_id == meta.decision,
|
||||||
|
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
|
ensure!(meta.helper_id.is_none(), "HELPER_JOURNAL_DISAPPEARED");
|
||||||
|
|||||||
@@ -1026,6 +1026,72 @@ fn helper_gate_requires_exact_signed_instruction_and_applied_checkpoint() {
|
|||||||
assert!(!c.can_start(&f.channel, true, &f.clock));
|
assert!(!c.can_start(&f.channel, true, &f.clock));
|
||||||
}
|
}
|
||||||
#[test]
|
#[test]
|
||||||
|
fn windows_unconfirmed_helper_head_cannot_replace_a_committed_decision() {
|
||||||
|
let mut f = Fixture::new();
|
||||||
|
f.helper_identity = HelperIdentity::System;
|
||||||
|
let mut client = f.client();
|
||||||
|
let mut helper = HelperSnapshot {
|
||||||
|
journal_id: Uuid::new_v4(),
|
||||||
|
checkpoint: 1,
|
||||||
|
previous_decision_id: None,
|
||||||
|
attempts: vec![],
|
||||||
|
};
|
||||||
|
let first = client
|
||||||
|
.prepare(
|
||||||
|
&mut f.db,
|
||||||
|
&f.ledger,
|
||||||
|
f.channel.clone(),
|
||||||
|
Some(&helper),
|
||||||
|
&f.clock,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let decision = f.decision(&first, true);
|
||||||
|
f.apply(&mut client, &decision).unwrap();
|
||||||
|
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||||
|
|
||||||
|
for previous in [None, Some(Uuid::new_v4())] {
|
||||||
|
let mut reopened = f.client();
|
||||||
|
helper.previous_decision_id = previous;
|
||||||
|
assert_eq!(
|
||||||
|
reopened
|
||||||
|
.prepare(
|
||||||
|
&mut f.db,
|
||||||
|
&f.ledger,
|
||||||
|
f.channel.clone(),
|
||||||
|
Some(&helper),
|
||||||
|
&f.clock
|
||||||
|
)
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.to_string(),
|
||||||
|
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
|
||||||
|
);
|
||||||
|
assert!(!reopened.can_start(&f.channel, false, &f.clock));
|
||||||
|
let count: u64 =
|
||||||
|
f.db.query_row("SELECT count(*) FROM reconcile_requests", [], |r| r.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 1);
|
||||||
|
}
|
||||||
|
helper.previous_decision_id = Some(decision.decision_id);
|
||||||
|
helper.checkpoint += 1;
|
||||||
|
f.channel.connection_generation += 1;
|
||||||
|
let mut reopened = f.client();
|
||||||
|
let next = reopened
|
||||||
|
.prepare(
|
||||||
|
&mut f.db,
|
||||||
|
&f.ledger,
|
||||||
|
f.channel.clone(),
|
||||||
|
Some(&helper),
|
||||||
|
&f.clock,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
next.request.helper_previous_decision_id,
|
||||||
|
decision.decision_id.to_string()
|
||||||
|
);
|
||||||
|
assert!(!reopened.can_start(&f.channel, false, &f.clock));
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
fn independent_helper_checkpoint_regression_is_sticky() {
|
fn independent_helper_checkpoint_regression_is_sticky() {
|
||||||
let mut f = Fixture::new();
|
let mut f = Fixture::new();
|
||||||
let mut c = f.client();
|
let mut c = f.client();
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use anyhow::{Context, Result, ensure};
|
|||||||
use rmm_agent_core::{
|
use rmm_agent_core::{
|
||||||
auth::{Binding, Outcome, Request},
|
auth::{Binding, Outcome, Request},
|
||||||
helper_reconcile::{NoStartAnchor, Snapshot},
|
helper_reconcile::{NoStartAnchor, Snapshot},
|
||||||
reconcile::HelperSnapshot,
|
reconcile::{HelperSnapshot, SyncClock},
|
||||||
};
|
};
|
||||||
use rmm_domain::jobs::{Action, JobPayload};
|
use rmm_domain::jobs::{Action, JobPayload};
|
||||||
use rmm_protocol::{SyncAttempt, SyncResponse};
|
use rmm_protocol::{SyncAttempt, SyncResponse};
|
||||||
@@ -473,16 +473,49 @@ impl Client {
|
|||||||
response: &SyncResponse,
|
response: &SyncResponse,
|
||||||
request_bytes: Vec<u8>,
|
request_bytes: Vec<u8>,
|
||||||
boot: uuid::Uuid,
|
boot: uuid::Uuid,
|
||||||
|
valid_until: chrono::DateTime<chrono::Utc>,
|
||||||
|
clock: &impl SyncClock,
|
||||||
) -> Result<ControlSnapshot> {
|
) -> Result<ControlSnapshot> {
|
||||||
let reply = self
|
let operation = ipc::Operation::Apply {
|
||||||
.call(ipc::Operation::Apply {
|
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
||||||
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
decision_bytes: response.decision_bytes.clone(),
|
||||||
decision_bytes: response.decision_bytes.clone(),
|
signature: response.signature.clone(),
|
||||||
signature: response.signature.clone(),
|
},
|
||||||
},
|
request_bytes,
|
||||||
request_bytes,
|
};
|
||||||
})
|
let started = std::time::Instant::now();
|
||||||
.await?;
|
let reply = loop {
|
||||||
|
ensure!(
|
||||||
|
started.elapsed() < Duration::from_secs(60),
|
||||||
|
"HELPER_APPLY_EXPIRED"
|
||||||
|
);
|
||||||
|
let Ok(reading) = clock.reading() else {
|
||||||
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
ensure!(
|
||||||
|
reading.upper < valid_until && reading.own_wall < valid_until,
|
||||||
|
"HELPER_APPLY_EXPIRED"
|
||||||
|
);
|
||||||
|
match self.call(operation.clone()).await {
|
||||||
|
Ok(reply) => break reply,
|
||||||
|
Err(error) => {
|
||||||
|
// Retry only contention before dispatch. An authenticated
|
||||||
|
// denial or an unknown reply keeps this round closed.
|
||||||
|
let diagnostic = ipc::diagnostic(&error);
|
||||||
|
let pipe_busy = diagnostic.stage == "client_open_pipe"
|
||||||
|
&& diagnostic.win32
|
||||||
|
== Some(windows_sys::Win32::Foundation::ERROR_PIPE_BUSY);
|
||||||
|
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let ipc::ReplyBody::Denied { code } = &reply.body {
|
||||||
|
return Err(ipc::remote_denial(ipc::Stage::ClientReplyBinding, code));
|
||||||
|
}
|
||||||
let ipc::ReplyBody::Applied {
|
let ipc::ReplyBody::Applied {
|
||||||
snapshot,
|
snapshot,
|
||||||
no_start_anchor,
|
no_start_anchor,
|
||||||
|
|||||||
@@ -498,6 +498,8 @@ struct Agent {
|
|||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
prepared_request: Option<Vec<u8>>,
|
prepared_request: Option<Vec<u8>>,
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
|
helper_apply_unconfirmed: bool,
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
helper_lane: Arc<tokio::sync::Semaphore>,
|
helper_lane: Arc<tokio::sync::Semaphore>,
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
helper: Option<rmm_agent_windows::helper_runtime::Client>,
|
helper: Option<rmm_agent_windows::helper_runtime::Client>,
|
||||||
@@ -693,6 +695,7 @@ impl Agent {
|
|||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
{
|
{
|
||||||
self.helper_control.is_some()
|
self.helper_control.is_some()
|
||||||
|
|| self.helper_apply_unconfirmed
|
||||||
|| self.helper_probe.is_some()
|
|| self.helper_probe.is_some()
|
||||||
|| !self.helper_checked
|
|| !self.helper_checked
|
||||||
|| self.helper_refused
|
|| self.helper_refused
|
||||||
@@ -789,6 +792,7 @@ impl Agent {
|
|||||||
} else {
|
} else {
|
||||||
sync.confirm_helper(&mapped)?;
|
sync.confirm_helper(&mapped)?;
|
||||||
}
|
}
|
||||||
|
self.helper_apply_unconfirmed = false;
|
||||||
self.open_fence(generation)?;
|
self.open_fence(generation)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -825,6 +829,7 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
if self.helper_probe.is_none()
|
if self.helper_probe.is_none()
|
||||||
&& self.helper_control.is_none()
|
&& self.helper_control.is_none()
|
||||||
|
&& !self.helper_apply_unconfirmed
|
||||||
&& !self.prepared
|
&& !self.prepared
|
||||||
&& self.running.is_none()
|
&& self.running.is_none()
|
||||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||||
@@ -1124,6 +1129,10 @@ impl Agent {
|
|||||||
self.prepared = false;
|
self.prepared = false;
|
||||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||||
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
|
if applied.helper_instruction.is_some() {
|
||||||
|
self.helper_apply_unconfirmed = true;
|
||||||
|
}
|
||||||
sync.apply_consumption(
|
sync.apply_consumption(
|
||||||
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
||||||
&self.clock,
|
&self.clock,
|
||||||
@@ -1140,11 +1149,15 @@ impl Agent {
|
|||||||
.prepared_request
|
.prepared_request
|
||||||
.take()
|
.take()
|
||||||
.context("ORIGINAL_SYNC_REQUEST_REQUIRED")?;
|
.context("ORIGINAL_SYNC_REQUEST_REQUIRED")?;
|
||||||
|
let until = sync.validity_deadline().context("SIGNED_LEASE_REQUIRED")?;
|
||||||
|
let clock = self.clock.clone();
|
||||||
let boot = self.channel(generation)?.boot_id;
|
let boot = self.channel(generation)?.boot_id;
|
||||||
self.helper_control = Some(HelperControl::Apply(
|
self.helper_control = Some(HelperControl::Apply(
|
||||||
generation,
|
generation,
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
helper.apply(&instruction.response, request, boot).await
|
helper
|
||||||
|
.apply(&instruction.response, request, boot, until, &clock)
|
||||||
|
.await
|
||||||
}),
|
}),
|
||||||
));
|
));
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -1594,6 +1607,12 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
|
async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||||
self.close_gate();
|
self.close_gate();
|
||||||
|
#[cfg(feature = "system-helper")]
|
||||||
|
{
|
||||||
|
// Reconnect discards the old channel. ClientSync::prepare checks
|
||||||
|
// the authenticated helper head against the durable agent journal.
|
||||||
|
self.helper_apply_unconfirmed = false;
|
||||||
|
}
|
||||||
let result = self.session_inner(stop).await;
|
let result = self.session_inner(stop).await;
|
||||||
self.close_gate();
|
self.close_gate();
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
@@ -1922,6 +1941,8 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
|||||||
#[cfg(feature="system-helper")]
|
#[cfg(feature="system-helper")]
|
||||||
prepared_request:None,
|
prepared_request:None,
|
||||||
#[cfg(feature="system-helper")]
|
#[cfg(feature="system-helper")]
|
||||||
|
helper_apply_unconfirmed:false,
|
||||||
|
#[cfg(feature="system-helper")]
|
||||||
helper_lane:Arc::new(tokio::sync::Semaphore::new(1)),
|
helper_lane:Arc::new(tokio::sync::Semaphore::new(1)),
|
||||||
#[cfg(feature="system-helper")]
|
#[cfg(feature="system-helper")]
|
||||||
helper:None,
|
helper:None,
|
||||||
|
|||||||
@@ -290,3 +290,18 @@ signierte Ausstellungszeitpunkt der ersten Entscheidung lag bei
|
|||||||
21:15:39,528352853 UTC. Die Entscheidung folgte damit rund 1,968 Sekunden
|
21:15:39,528352853 UTC. Die Entscheidung folgte damit rund 1,968 Sekunden
|
||||||
auf die Erfassung. Eine zeitlich vor der Erfassung ausgestellte Antwort
|
auf die Erfassung. Eine zeitlich vor der Erfassung ausgestellte Antwort
|
||||||
scheidet für diesen Fall aus.
|
scheidet für diesen Fall aus.
|
||||||
|
|
||||||
|
### Vorbeugende Korrektur im Windows-Abgleich
|
||||||
|
|
||||||
|
Nach dem lokalen Speichern einer signierten Entscheidung wartet der
|
||||||
|
Windows-Agent auf die zugehörige bestätigte Helper-Antwort. Solange diese
|
||||||
|
aussteht, kann er keinen Folgeabgleich mit leerem Helper-Vorgänger senden.
|
||||||
|
Nach einem Prozessneustart prüft er den gemeldeten Vorgänger gegen sein
|
||||||
|
dauerhaftes Journal. Bei belegter Belegung vor dem IPC-Versand kann er
|
||||||
|
denselben signierten Inhalt innerhalb der bestehenden Frist zustellen.
|
||||||
|
|
||||||
|
Die lokale Prüfung bestand mit 28 Reconcile- und 31 Helper-Reconcile-Tests
|
||||||
|
sowie Windows-Cross-Clippy mit `system-helper` und `-D warnings`.
|
||||||
|
Der neue Regressionstest reproduziert fehlende und fremde Vorgänger nach
|
||||||
|
dem ersten lokalen Commit und akzeptiert den später bestätigten richtigen
|
||||||
|
Vorgänger. Die bestehende Sperre der VM bleibt im Journal archiviert.
|
||||||
|
|||||||
Reference in New Issue
Block a user