Fence Windows sync until the helper confirms the signed decision
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -5,7 +5,7 @@ use anyhow::{Context, Result, ensure};
|
||||
use rmm_agent_core::{
|
||||
auth::{Binding, Outcome, Request},
|
||||
helper_reconcile::{NoStartAnchor, Snapshot},
|
||||
reconcile::HelperSnapshot,
|
||||
reconcile::{HelperSnapshot, SyncClock},
|
||||
};
|
||||
use rmm_domain::jobs::{Action, JobPayload};
|
||||
use rmm_protocol::{SyncAttempt, SyncResponse};
|
||||
@@ -473,16 +473,49 @@ impl Client {
|
||||
response: &SyncResponse,
|
||||
request_bytes: Vec<u8>,
|
||||
boot: uuid::Uuid,
|
||||
valid_until: chrono::DateTime<chrono::Utc>,
|
||||
clock: &impl SyncClock,
|
||||
) -> Result<ControlSnapshot> {
|
||||
let reply = self
|
||||
.call(ipc::Operation::Apply {
|
||||
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
},
|
||||
request_bytes,
|
||||
})
|
||||
.await?;
|
||||
let operation = ipc::Operation::Apply {
|
||||
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
},
|
||||
request_bytes,
|
||||
};
|
||||
let started = std::time::Instant::now();
|
||||
let reply = loop {
|
||||
ensure!(
|
||||
started.elapsed() < Duration::from_secs(60),
|
||||
"HELPER_APPLY_EXPIRED"
|
||||
);
|
||||
let Ok(reading) = clock.reading() else {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
continue;
|
||||
};
|
||||
ensure!(
|
||||
reading.upper < valid_until && reading.own_wall < valid_until,
|
||||
"HELPER_APPLY_EXPIRED"
|
||||
);
|
||||
match self.call(operation.clone()).await {
|
||||
Ok(reply) => break reply,
|
||||
Err(error) => {
|
||||
// Retry only contention before dispatch. An authenticated
|
||||
// denial or an unknown reply keeps this round closed.
|
||||
let diagnostic = ipc::diagnostic(&error);
|
||||
let pipe_busy = diagnostic.stage == "client_open_pipe"
|
||||
&& diagnostic.win32
|
||||
== Some(windows_sys::Win32::Foundation::ERROR_PIPE_BUSY);
|
||||
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
|
||||
return Err(error);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
if let ipc::ReplyBody::Denied { code } = &reply.body {
|
||||
return Err(ipc::remote_denial(ipc::Stage::ClientReplyBinding, code));
|
||||
}
|
||||
let ipc::ReplyBody::Applied {
|
||||
snapshot,
|
||||
no_start_anchor,
|
||||
|
||||
@@ -498,6 +498,8 @@ struct Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
prepared_request: Option<Vec<u8>>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_apply_unconfirmed: bool,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_lane: Arc<tokio::sync::Semaphore>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper: Option<rmm_agent_windows::helper_runtime::Client>,
|
||||
@@ -693,6 +695,7 @@ impl Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.helper_control.is_some()
|
||||
|| self.helper_apply_unconfirmed
|
||||
|| self.helper_probe.is_some()
|
||||
|| !self.helper_checked
|
||||
|| self.helper_refused
|
||||
@@ -789,6 +792,7 @@ impl Agent {
|
||||
} else {
|
||||
sync.confirm_helper(&mapped)?;
|
||||
}
|
||||
self.helper_apply_unconfirmed = false;
|
||||
self.open_fence(generation)?;
|
||||
}
|
||||
}
|
||||
@@ -825,6 +829,7 @@ impl Agent {
|
||||
}
|
||||
if self.helper_probe.is_none()
|
||||
&& self.helper_control.is_none()
|
||||
&& !self.helper_apply_unconfirmed
|
||||
&& !self.prepared
|
||||
&& self.running.is_none()
|
||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||
@@ -1124,6 +1129,10 @@ impl Agent {
|
||||
self.prepared = false;
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
||||
#[cfg(feature = "system-helper")]
|
||||
if applied.helper_instruction.is_some() {
|
||||
self.helper_apply_unconfirmed = true;
|
||||
}
|
||||
sync.apply_consumption(
|
||||
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
||||
&self.clock,
|
||||
@@ -1140,11 +1149,15 @@ impl Agent {
|
||||
.prepared_request
|
||||
.take()
|
||||
.context("ORIGINAL_SYNC_REQUEST_REQUIRED")?;
|
||||
let until = sync.validity_deadline().context("SIGNED_LEASE_REQUIRED")?;
|
||||
let clock = self.clock.clone();
|
||||
let boot = self.channel(generation)?.boot_id;
|
||||
self.helper_control = Some(HelperControl::Apply(
|
||||
generation,
|
||||
tokio::spawn(async move {
|
||||
helper.apply(&instruction.response, request, boot).await
|
||||
helper
|
||||
.apply(&instruction.response, request, boot, until, &clock)
|
||||
.await
|
||||
}),
|
||||
));
|
||||
return Ok(());
|
||||
@@ -1594,6 +1607,12 @@ impl Agent {
|
||||
}
|
||||
async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||
self.close_gate();
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
// Reconnect discards the old channel. ClientSync::prepare checks
|
||||
// the authenticated helper head against the durable agent journal.
|
||||
self.helper_apply_unconfirmed = false;
|
||||
}
|
||||
let result = self.session_inner(stop).await;
|
||||
self.close_gate();
|
||||
#[cfg(feature = "system-helper")]
|
||||
@@ -1922,6 +1941,8 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
#[cfg(feature="system-helper")]
|
||||
prepared_request:None,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_apply_unconfirmed:false,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_lane:Arc::new(tokio::sync::Semaphore::new(1)),
|
||||
#[cfg(feature="system-helper")]
|
||||
helper:None,
|
||||
|
||||
Reference in New Issue
Block a user