Retain Windows signed sync responses across transient clock failures
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -1577,3 +1577,29 @@ fn original_completion_after_reconnect_unknown_is_recovered_without_reexecuting(
|
|||||||
.is_empty()
|
.is_empty()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transient_clock_failure_keeps_the_same_signed_reply_for_both_control_commit_phases() {
|
||||||
|
let mut f=Fixture::new();
|
||||||
|
let (_offer,_grant,_result)=f.add("accepted",false,None,script());
|
||||||
|
let mut client=f.client();
|
||||||
|
let prepared=f.prepare(&mut client);
|
||||||
|
let decision=f.decision(&prepared,true);
|
||||||
|
let signed=f.sign(&decision);
|
||||||
|
let original=prepared.request.encode_to_vec();
|
||||||
|
f.clock.trusted=false;
|
||||||
|
assert!(client.apply(&mut f.db,&signed,&f.clock).is_err());
|
||||||
|
let (count,pending):(u64,String)=f.db.query_row("SELECT (SELECT count(*) FROM reconcile_requests),pending_request FROM reconcile_meta WHERE id=1",[],|r|Ok((r.get(0)?,r.get(1)?))).unwrap();
|
||||||
|
assert_eq!(count,1);assert_eq!(pending,prepared.request.request_id);
|
||||||
|
f.clock.trusted=true;
|
||||||
|
client.apply(&mut f.db,&signed,&f.clock).unwrap();
|
||||||
|
f.clock.trusted=false;
|
||||||
|
assert!(client.apply_consumption(&mut f.ledger,&f.clock).is_err());
|
||||||
|
assert!(!client.can_start(&f.channel,false,&f.clock));
|
||||||
|
f.clock.trusted=true;
|
||||||
|
client.apply_consumption(&mut f.ledger,&f.clock).unwrap();
|
||||||
|
let (count,raw,saved):(u64,Vec<u8>,Vec<u8>)=f.db.query_row("SELECT (SELECT count(*) FROM reconcile_requests),request,response FROM reconcile_requests WHERE id=?1",[&prepared.request.request_id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||||
|
assert_eq!(count,1);assert_eq!(raw,original);assert_eq!(saved,signed.encode_to_vec());
|
||||||
|
assert_eq!(f.ledger.query_row("SELECT count(*) FROM consumed",[],|r|r.get::<_,u64>(0)).unwrap(),0);
|
||||||
|
assert!(client.can_start(&f.channel,false,&f.clock));
|
||||||
|
}
|
||||||
|
|||||||
@@ -500,7 +500,7 @@ impl Client {
|
|||||||
"HELPER_APPLY_EXPIRED"
|
"HELPER_APPLY_EXPIRED"
|
||||||
);
|
);
|
||||||
let Ok(reading) = clock.reading() else {
|
let Ok(reading) = clock.reading() else {
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
ensure!(
|
ensure!(
|
||||||
@@ -519,7 +519,7 @@ impl Client {
|
|||||||
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
|
if error.to_string() != "HELPER_CONTROL_BUSY" && !pipe_busy {
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ pub struct Observation {
|
|||||||
pub clock_failure: &'static str,
|
pub clock_failure: &'static str,
|
||||||
pub needs_sync: bool,
|
pub needs_sync: bool,
|
||||||
pub prepared: bool,
|
pub prepared: bool,
|
||||||
|
pub deferred_sync: &'static str,
|
||||||
|
pub deferred_sync_wait_ms: u64,
|
||||||
pub helper_control: &'static str,
|
pub helper_control: &'static str,
|
||||||
pub helper_apply_unconfirmed: bool,
|
pub helper_apply_unconfirmed: bool,
|
||||||
pub signed_gate_present: bool,
|
pub signed_gate_present: bool,
|
||||||
@@ -154,3 +156,80 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum ResponseWindow {
|
||||||
|
WaitForClock,
|
||||||
|
WithinWindow,
|
||||||
|
Expired,
|
||||||
|
}
|
||||||
|
/// Only scheduling of an independently authenticated, already bound control
|
||||||
|
/// reply. It cannot qualify a clock or turn this reply into an authorization.
|
||||||
|
pub fn response_window(
|
||||||
|
issued: chrono::DateTime<chrono::Utc>,
|
||||||
|
until: chrono::DateTime<chrono::Utc>,
|
||||||
|
held: std::time::Duration,
|
||||||
|
reading: Option<(chrono::DateTime<chrono::Utc>, chrono::DateTime<chrono::Utc>)>,
|
||||||
|
) -> ResponseWindow {
|
||||||
|
if held >= std::time::Duration::from_secs(60) || until <= issued {
|
||||||
|
return ResponseWindow::Expired;
|
||||||
|
}
|
||||||
|
let Some((wall, upper)) = reading else {
|
||||||
|
return ResponseWindow::WaitForClock;
|
||||||
|
};
|
||||||
|
if upper < wall || upper >= until {
|
||||||
|
return ResponseWindow::Expired;
|
||||||
|
}
|
||||||
|
if wall < issued {
|
||||||
|
return ResponseWindow::WaitForClock;
|
||||||
|
}
|
||||||
|
ResponseWindow::WithinWindow
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
|
mod response_tests {
|
||||||
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline() {
|
||||||
|
let issued = chrono::DateTime::from_timestamp(100, 0).unwrap();
|
||||||
|
let until = issued + chrono::Duration::seconds(30);
|
||||||
|
let short = std::time::Duration::from_millis(20);
|
||||||
|
assert_eq!(
|
||||||
|
response_window(issued, until, short, None),
|
||||||
|
ResponseWindow::WaitForClock
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_window(
|
||||||
|
issued,
|
||||||
|
until,
|
||||||
|
short,
|
||||||
|
Some((
|
||||||
|
issued - chrono::Duration::milliseconds(500),
|
||||||
|
issued + chrono::Duration::milliseconds(100)
|
||||||
|
))
|
||||||
|
),
|
||||||
|
ResponseWindow::WaitForClock
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_window(
|
||||||
|
issued,
|
||||||
|
until,
|
||||||
|
short,
|
||||||
|
Some((issued, issued + chrono::Duration::milliseconds(100)))
|
||||||
|
),
|
||||||
|
ResponseWindow::WithinWindow
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_window(
|
||||||
|
issued,
|
||||||
|
until,
|
||||||
|
short,
|
||||||
|
Some((until - chrono::Duration::milliseconds(1), until))
|
||||||
|
),
|
||||||
|
ResponseWindow::Expired
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response_window(issued, until, std::time::Duration::from_secs(60), None),
|
||||||
|
ResponseWindow::Expired
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -400,6 +400,19 @@ fn needs_bound_outcome(
|
|||||||
}
|
}
|
||||||
Ok(true)
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
struct DeferredSync {
|
||||||
|
generation: u64,
|
||||||
|
response: SyncResponse,
|
||||||
|
issued_at: chrono::DateTime<Utc>,
|
||||||
|
valid_until: chrono::DateTime<Utc>,
|
||||||
|
received_at: Instant,
|
||||||
|
applied: Option<reconcile::Applied>,
|
||||||
|
}
|
||||||
|
fn clock_temporarily_unavailable(error: &anyhow::Error) -> bool {
|
||||||
|
error
|
||||||
|
.chain()
|
||||||
|
.any(|cause| cause.to_string().starts_with("TIME_"))
|
||||||
|
}
|
||||||
struct Running {
|
struct Running {
|
||||||
offer: JobOffer,
|
offer: JobOffer,
|
||||||
grant: ExecutionPermitPayload,
|
grant: ExecutionPermitPayload,
|
||||||
@@ -547,6 +560,7 @@ struct Agent {
|
|||||||
time_refresh: Option<TimeRefresh>,
|
time_refresh: Option<TimeRefresh>,
|
||||||
sync: Option<reconcile::ClientSync>,
|
sync: Option<reconcile::ClientSync>,
|
||||||
prepared: bool,
|
prepared: bool,
|
||||||
|
deferred_sync: Option<DeferredSync>,
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
prepared_request: Option<Vec<u8>>,
|
prepared_request: Option<Vec<u8>>,
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
@@ -841,7 +855,7 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
async fn drain_helper_apply(&mut self) {
|
async fn drain_helper_apply(&mut self, until: Instant) {
|
||||||
// Keep the same bounded original worker and independent time lane
|
// Keep the same bounded original worker and independent time lane
|
||||||
// alive until its acknowledgement settles. This never opens a gate,
|
// alive until its acknowledgement settles. This never opens a gate,
|
||||||
// confirms a new generation, retries Apply or authorizes an effect.
|
// confirms a new generation, retries Apply or authorizes an effect.
|
||||||
@@ -851,7 +865,8 @@ impl Agent {
|
|||||||
match control {
|
match control {
|
||||||
HelperControl::Apply(_, task) => {
|
HelperControl::Apply(_, task) => {
|
||||||
if let Ok(Ok(Ok(_snapshot))) =
|
if let Ok(Ok(Ok(_snapshot))) =
|
||||||
tokio::time::timeout(Duration::from_secs(70), task).await
|
tokio::time::timeout(until.saturating_duration_since(Instant::now()), task)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
self.helper_apply_unconfirmed = false;
|
self.helper_apply_unconfirmed = false;
|
||||||
}
|
}
|
||||||
@@ -889,6 +904,7 @@ impl Agent {
|
|||||||
sync.close();
|
sync.close();
|
||||||
}
|
}
|
||||||
self.prepared = false;
|
self.prepared = false;
|
||||||
|
self.deferred_sync = None;
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
{
|
{
|
||||||
self.prepared_request = None;
|
self.prepared_request = None;
|
||||||
@@ -968,6 +984,19 @@ impl Agent {
|
|||||||
clock_failure: clock.as_ref().err().map_or("", clock_code),
|
clock_failure: clock.as_ref().err().map_or("", clock_code),
|
||||||
needs_sync,
|
needs_sync,
|
||||||
prepared: self.prepared,
|
prepared: self.prepared,
|
||||||
|
deferred_sync: self.deferred_sync.as_ref().map_or("none", |pending| {
|
||||||
|
if pending.applied.is_some() {
|
||||||
|
"local_consumption"
|
||||||
|
} else {
|
||||||
|
"fresh_response_clock"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
deferred_sync_wait_ms: self.deferred_sync.as_ref().map_or(0, |p| {
|
||||||
|
p.received_at
|
||||||
|
.elapsed()
|
||||||
|
.as_millis()
|
||||||
|
.min(u128::from(u64::MAX)) as u64
|
||||||
|
}),
|
||||||
helper_control,
|
helper_control,
|
||||||
helper_apply_unconfirmed: unconfirmed,
|
helper_apply_unconfirmed: unconfirmed,
|
||||||
signed_gate_present: gate.is_some(),
|
signed_gate_present: gate.is_some(),
|
||||||
@@ -1147,6 +1176,7 @@ impl Agent {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
self.config.enable_jobs
|
self.config.enable_jobs
|
||||||
|
&& self.deferred_sync.is_none()
|
||||||
&& !self.helper_control_pending()
|
&& !self.helper_control_pending()
|
||||||
&& self.channel(generation).is_ok_and(|channel| {
|
&& self.channel(generation).is_ok_and(|channel| {
|
||||||
self.sync
|
self.sync
|
||||||
@@ -1447,17 +1477,101 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
||||||
ensure!(self.prepared, "UNSOLICITED_SYNC_RESPONSE");
|
ensure!(self.prepared, "UNSOLICITED_SYNC_RESPONSE");
|
||||||
self.prepared = false;
|
if let Some(pending) = &self.deferred_sync {
|
||||||
|
ensure!(
|
||||||
|
pending.generation == generation && pending.response == response,
|
||||||
|
"DEFERRED_SYNC_RESPONSE_CHANGED"
|
||||||
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// Authenticate exact bytes/request/channel BEFORE retaining a reply.
|
||||||
|
// Declared issued_at is used only for historical signature/profile
|
||||||
|
// verification here. The regular fresh clock gates below still decide
|
||||||
|
// whether either state-application phase may run now.
|
||||||
|
ensure!(
|
||||||
|
response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64,
|
||||||
|
"SYNC_RESPONSE_LIMIT"
|
||||||
|
);
|
||||||
|
let decision: rmm_domain::reconciliation::ReconcileDecisionPayload =
|
||||||
|
serde_json::from_slice(&response.decision_bytes)?;
|
||||||
|
let raw:Vec<u8>=self.journal.connection().query_row("SELECT request FROM reconcile_requests WHERE id=(SELECT pending_request FROM reconcile_meta WHERE id=1)",[],|r|r.get(0))?;
|
||||||
|
ensure!(raw.len() <= rmm_domain::MAX_MESSAGE, "SYNC_REQUEST_LIMIT");
|
||||||
|
let request = SyncRequest::decode(raw.as_slice())?;
|
||||||
|
ensure!(request.encode_to_vec() == raw, "SYNC_REQUEST_CANONICAL");
|
||||||
|
reconcile::verify_response_with(
|
||||||
|
&rmm_agent_core::crypto::DalekVerifier,
|
||||||
|
self.binding.as_ref().context("JOB_BINDING_REQUIRED")?,
|
||||||
|
&request,
|
||||||
|
&self.channel(generation)?,
|
||||||
|
&response,
|
||||||
|
decision.issued_at,
|
||||||
|
)?;
|
||||||
|
self.deferred_sync = Some(DeferredSync {
|
||||||
|
generation,
|
||||||
|
response,
|
||||||
|
issued_at: decision.issued_at,
|
||||||
|
valid_until: decision.valid_until,
|
||||||
|
received_at: Instant::now(),
|
||||||
|
applied: None,
|
||||||
|
});
|
||||||
|
self.poll_sync_apply(generation, true)
|
||||||
|
}
|
||||||
|
fn poll_sync_apply(&mut self, generation: u64, open_gate: bool) -> Result<()> {
|
||||||
|
let Some(mut pending) = self.deferred_sync.take() else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
ensure!(pending.generation == generation, "DEFERRED_SYNC_GENERATION");
|
||||||
|
use rmm_agent_windows::job_readiness::{ResponseWindow, response_window};
|
||||||
|
match response_window(
|
||||||
|
pending.issued_at,
|
||||||
|
pending.valid_until,
|
||||||
|
pending.received_at.elapsed(),
|
||||||
|
self.clock.reading().ok().map(|r| (r.own_wall, r.upper)),
|
||||||
|
) {
|
||||||
|
ResponseWindow::WaitForClock => {
|
||||||
|
self.deferred_sync = Some(pending);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
ResponseWindow::Expired => anyhow::bail!("DEFERRED_SYNC_RESPONSE_EXPIRED"),
|
||||||
|
ResponseWindow::WithinWindow => {}
|
||||||
|
}
|
||||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||||
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
if pending.applied.is_none() {
|
||||||
|
match sync.apply(
|
||||||
|
self.journal.connection_mut(),
|
||||||
|
&pending.response,
|
||||||
|
&self.clock,
|
||||||
|
) {
|
||||||
|
Ok(applied) => {
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
if applied.helper_instruction.is_some() {
|
if applied.helper_instruction.is_some() {
|
||||||
self.helper_apply_unconfirmed = true;
|
self.helper_apply_unconfirmed = true;
|
||||||
}
|
}
|
||||||
sync.apply_consumption(
|
pending.applied = Some(applied);
|
||||||
|
}
|
||||||
|
Err(error) if clock_temporarily_unavailable(&error) => {
|
||||||
|
self.deferred_sync = Some(pending);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match sync.apply_consumption(
|
||||||
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
||||||
&self.clock,
|
&self.clock,
|
||||||
)?;
|
) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(error) if clock_temporarily_unavailable(&error) => {
|
||||||
|
self.deferred_sync = Some(pending);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
}
|
||||||
|
let applied = pending
|
||||||
|
.applied
|
||||||
|
.take()
|
||||||
|
.context("DEFERRED_SYNC_APPLIED_REQUIRED")?;
|
||||||
|
self.prepared = false;
|
||||||
if let Some(instruction) = applied.helper_instruction {
|
if let Some(instruction) = applied.helper_instruction {
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
{
|
{
|
||||||
@@ -1493,7 +1607,12 @@ impl Agent {
|
|||||||
{
|
{
|
||||||
self.prepared_request = None;
|
self.prepared_request = None;
|
||||||
}
|
}
|
||||||
|
if open_gate {
|
||||||
self.open_fence(generation)
|
self.open_fence(generation)
|
||||||
|
} else {
|
||||||
|
self.fence.close();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
fn open_fence(&self, generation: u64) -> Result<()> {
|
fn open_fence(&self, generation: u64) -> Result<()> {
|
||||||
if self.ready(generation) {
|
if self.ready(generation) {
|
||||||
@@ -1955,12 +2074,32 @@ impl Agent {
|
|||||||
self.helper_apply_unconfirmed = false;
|
self.helper_apply_unconfirmed = false;
|
||||||
}
|
}
|
||||||
let result = self.session_inner(stop).await;
|
let result = self.session_inner(stop).await;
|
||||||
self.close_gate();
|
self.fence.close();
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
{
|
{
|
||||||
self.drain_helper_apply().await;
|
let until = Instant::now() + Duration::from_secs(70);
|
||||||
|
// Complete the already authenticated original control phases while
|
||||||
|
// their finite deadline and independent time lane still permit it.
|
||||||
|
// No job is running when this control pipeline was prepared.
|
||||||
|
while self.running.is_none() && self.deferred_sync.is_some() && Instant::now() < until {
|
||||||
|
let generation = self
|
||||||
|
.deferred_sync
|
||||||
|
.as_ref()
|
||||||
|
.expect("checked pending control")
|
||||||
|
.generation;
|
||||||
|
if self.poll_sync_apply(generation, false).is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if self.deferred_sync.is_some() {
|
||||||
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.close_gate();
|
||||||
|
self.drain_helper_apply(until).await;
|
||||||
self.stop_time_refresh().await;
|
self.stop_time_refresh().await;
|
||||||
}
|
}
|
||||||
|
#[cfg(not(feature = "system-helper"))]
|
||||||
|
self.close_gate();
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||||
@@ -2056,6 +2195,7 @@ impl Agent {
|
|||||||
tokio::select! {
|
tokio::select! {
|
||||||
Some(completion)=self.completed_rx.recv()=>{self.completed(generation,completion)?;needs_sync=true;},
|
Some(completion)=self.completed_rx.recv()=>{self.completed(generation,completion)?;needs_sync=true;},
|
||||||
_=tick.tick()=>{
|
_=tick.tick()=>{
|
||||||
|
if !stop.load(Ordering::SeqCst) && self.poll_sync_apply(generation,true).is_err(){self.close_gate();needs_sync=true;}
|
||||||
#[cfg(feature = "system-helper")]
|
#[cfg(feature = "system-helper")]
|
||||||
if !stop.load(Ordering::SeqCst) && self.poll_helper(&channel,generation).await.is_err() {self.close_gate();needs_sync=self.sync.is_some();}
|
if !stop.load(Ordering::SeqCst) && self.poll_helper(&channel,generation).await.is_err() {self.close_gate();needs_sync=self.sync.is_some();}
|
||||||
if !stop.load(Ordering::SeqCst) && let Some((pending_generation,permit))=self.deferred_permit.take() {
|
if !stop.load(Ordering::SeqCst) && let Some((pending_generation,permit))=self.deferred_permit.take() {
|
||||||
@@ -2069,7 +2209,7 @@ impl Agent {
|
|||||||
let busy=self.running.is_some() || self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?;
|
let busy=self.running.is_some() || self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?;
|
||||||
if !stop.load(Ordering::SeqCst) && !busy {
|
if !stop.load(Ordering::SeqCst) && !busy {
|
||||||
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
||||||
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
if self.prepared && self.deferred_sync.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||||
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
||||||
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||||
}
|
}
|
||||||
@@ -2365,7 +2505,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
|||||||
helper_checked:false,
|
helper_checked:false,
|
||||||
#[cfg(feature="system-helper")]
|
#[cfg(feature="system-helper")]
|
||||||
helper_refused:false,
|
helper_refused:false,
|
||||||
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
prepared:false,deferred_sync:None,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
||||||
};
|
};
|
||||||
agent.recover_interrupted()?;
|
agent.recover_interrupted()?;
|
||||||
let mut backoff = 1;
|
let mut backoff = 1;
|
||||||
|
|||||||
@@ -12,10 +12,19 @@ $manifestPath = Join-Path (Join-Path $bootstrap $ManifestSha256) 'successor.json
|
|||||||
if ((Get-FileHash -LiteralPath $manifestPath).Hash.ToLowerInvariant() -cne $ManifestSha256) { throw 'Manifest changed.' }
|
if ((Get-FileHash -LiteralPath $manifestPath).Hash.ToLowerInvariant() -cne $ManifestSha256) { throw 'Manifest changed.' }
|
||||||
$m = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
$m = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||||
$fields = 'agent_build_sha256,case_id,linux_proof_sha256,new_agent_sha256,old_agent_sha256,old_pin_sha256,old_profile_sha256,old_settings_sha256,previous_publication_sha256,windows_proof_sha256'
|
$fields = 'agent_build_sha256,case_id,linux_proof_sha256,new_agent_sha256,old_agent_sha256,old_pin_sha256,old_profile_sha256,old_settings_sha256,previous_publication_sha256,windows_proof_sha256'
|
||||||
|
$previousCaseId = 'c62d698d-d044-4815-acbe-3eeb383d44d5'
|
||||||
|
$requestedPreviousRevision = 4
|
||||||
|
if ($null -ne $m.PSObject.Properties['previous_case_id']) {
|
||||||
|
$fields = 'agent_build_sha256,case_id,linux_proof_sha256,new_agent_sha256,old_agent_sha256,old_pin_sha256,old_profile_sha256,old_settings_sha256,previous_case_id,previous_publication_sha256,previous_revision,windows_proof_sha256'
|
||||||
|
$previousCaseId = [guid]::ParseExact($m.previous_case_id,'D').ToString()
|
||||||
|
if ($previousCaseId -cne $m.previous_case_id -or $m.previous_revision -isnot [int] -or
|
||||||
|
$m.previous_revision -lt 4 -or $m.previous_revision -gt 1000) { throw 'Expected exact previous publication identity.' }
|
||||||
|
$requestedPreviousRevision = $m.previous_revision
|
||||||
|
}
|
||||||
if ((($m.PSObject.Properties.Name | Sort-Object) -join ',') -cne $fields -or
|
if ((($m.PSObject.Properties.Name | Sort-Object) -join ',') -cne $fields -or
|
||||||
[guid]::ParseExact($m.case_id,'D').ToString() -cne $m.case_id) { throw 'Unexpected development manifest schema.' }
|
[guid]::ParseExact($m.case_id,'D').ToString() -cne $m.case_id) { throw 'Unexpected development manifest schema.' }
|
||||||
foreach ($p in $m.PSObject.Properties) {
|
foreach ($p in $m.PSObject.Properties) {
|
||||||
if ($p.Name -cne 'case_id' -and $p.Value -cnotmatch '^[0-9a-f]{64}$') { throw 'Manifest digest format.' }
|
if ($p.Name -notin @('case_id','previous_case_id','previous_revision') -and $p.Value -cnotmatch '^[0-9a-f]{64}$') { throw 'Manifest digest format.' }
|
||||||
}
|
}
|
||||||
if ($m.old_agent_sha256 -ceq $m.new_agent_sha256) { throw 'No successor code.' }
|
if ($m.old_agent_sha256 -ceq $m.new_agent_sha256) { throw 'No successor code.' }
|
||||||
|
|
||||||
@@ -38,25 +47,25 @@ if ($i -lt 0 -or $j -le $i -or $i -ne $text.LastIndexOf($first,[StringComparison
|
|||||||
$EchoPassive = $false
|
$EchoPassive = $false
|
||||||
$Mode = 'Read'
|
$Mode = 'Read'
|
||||||
. ([scriptblock]::Create($text.Substring($i,$j-$i)))
|
. ([scriptblock]::Create($text.Substring($i,$j-$i)))
|
||||||
|
$previousRevision = $requestedPreviousRevision
|
||||||
|
|
||||||
CheckControl $bootstrap $true
|
CheckControl $bootstrap $true
|
||||||
CheckControl (Split-Path $manifestPath) $true
|
CheckControl (Split-Path $manifestPath) $true
|
||||||
CheckControl $manifestPath $false
|
CheckControl $manifestPath $false
|
||||||
$previousCase = Join-Path $bootstrap 'job-passive-pair-c62d698d-d044-4815-acbe-3eeb383d44d5'
|
$previousCase = Join-Path $bootstrap ('job-passive-pair-' + $previousCaseId)
|
||||||
$previousPublication = Join-Path $previousCase 'profile-published.json'
|
$previousPublication = Join-Path $previousCase 'profile-published.json'
|
||||||
CheckControl $previousCase $true
|
CheckControl $previousCase $true
|
||||||
CheckControl $previousPublication $false
|
CheckControl $previousPublication $false
|
||||||
if ((Hash $previousPublication) -cne $m.previous_publication_sha256) { throw 'Previous publication changed.' }
|
if ((Hash $previousPublication) -cne $m.previous_publication_sha256) { throw 'Previous publication changed.' }
|
||||||
$previous = Get-Content -LiteralPath $previousPublication -Raw | ConvertFrom-Json
|
$previous = Get-Content -LiteralPath $previousPublication -Raw | ConvertFrom-Json
|
||||||
if ($previous.successor_revision -ne 4 -or $previous.configuration.sha256 -cne $m.old_profile_sha256 -or
|
if ($previous.successor_revision -ne $previousRevision -or $previous.configuration.sha256 -cne $m.old_profile_sha256 -or
|
||||||
$previous.agent.image_file.sha256 -cne $m.old_agent_sha256) { throw 'Expected actual revision 4 predecessor.' }
|
$previous.agent.image_file.sha256 -cne $m.old_agent_sha256) { throw 'Expected actual predecessor publication.' }
|
||||||
$id = $m.case_id
|
$id = $m.case_id
|
||||||
$work = Join-Path $bootstrap ('job-passive-pair-' + $id)
|
$work = Join-Path $bootstrap ('job-passive-pair-' + $id)
|
||||||
$oldHashes = [ordered]@{agent=$m.old_agent_sha256;settings=$m.old_settings_sha256;pin=$m.old_pin_sha256}
|
$oldHashes = [ordered]@{agent=$m.old_agent_sha256;settings=$m.old_settings_sha256;pin=$m.old_pin_sha256}
|
||||||
$newHashes = [ordered]@{agent=$m.new_agent_sha256}
|
$newHashes = [ordered]@{agent=$m.new_agent_sha256}
|
||||||
$source = Join-Path (Join-Path $bootstrap $newHashes.agent) 'rmm-agent-windows.exe'
|
$source = Join-Path (Join-Path $bootstrap $newHashes.agent) 'rmm-agent-windows.exe'
|
||||||
$profileHash = $m.old_profile_sha256
|
$profileHash = $m.old_profile_sha256
|
||||||
$previousRevision = 4
|
|
||||||
$previousPublicationHash = $m.previous_publication_sha256
|
$previousPublicationHash = $m.previous_publication_sha256
|
||||||
$platformArchive = Join-Path $previousCase 'profile-platform.json'
|
$platformArchive = Join-Path $previousCase 'profile-platform.json'
|
||||||
$roleArchive = Join-Path $previousCase 'profile-role-{0}.json'
|
$roleArchive = Join-Path $previousCase 'profile-role-{0}.json'
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"source_files": [
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/runtime.rs",
|
||||||
|
"source_sha256": "571eb94025fe0fb1fecbab050ff034d21cad8122193996606579768480e6b1cf",
|
||||||
|
"target_sha256": "a9bff2463da165f69d4e70e22bddcb13592e309c7583cc5fa9b8088c91700a85"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/job_readiness.rs",
|
||||||
|
"source_sha256": null,
|
||||||
|
"target_sha256": "d492f292e7deabe59dc7dd88fb02b51c7c9f60ed9b646b284eb314654e63f9f9"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/lib.rs",
|
||||||
|
"source_sha256": "609e6bf4cd895b81f0153113b17444fe8453212d6b691b73dcea26a63a920d45",
|
||||||
|
"target_sha256": "574e55d368e7c6668065c42f9d1898856dc659a46d21dfec5c6bd623a0d98138"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-core/src/reconcile.rs",
|
||||||
|
"source_sha256": "d924e3cfd1f782169f3dee4dd7875cd5fba493fbaecb64e13abcb16729be9f74",
|
||||||
|
"target_sha256": "06180f8897b4a69eec4ed7cb265caa7afaf156e13283b1025fda944d0cc3e5c0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-windows/src/helper_runtime.rs",
|
||||||
|
"source_sha256": "e1a4896331c261f497ee9047cd2f4a40fe9232983b71ff45093c4cd1dc1fab5c",
|
||||||
|
"target_sha256": "d250c0e12f7582b1b1e890493fca8fd93103e41c7e161bb748b8c71af3ac9865"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/agent-core/tests/reconcile.rs",
|
||||||
|
"source_sha256": "e0ba24b22b3259b7b39727ed227612b75deaf6904bc9a563da2ad9709f6d4d79",
|
||||||
|
"target_sha256": "b1a5cc969d95ad376df6446332b2bf963acb73eb7bef6f321b0b02b15cdc6c93"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"scope": "Native clock/deferred signed response contract checks only; no new target job or update acceptance",
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"stdout": "\nrunning 1 test\ntest job_readiness::response_tests::signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 32 filtered out; finished in 0.01s\n\n",
|
||||||
|
"stderr": "",
|
||||||
|
"binary_sha256": "672f6e0523583fd76c6033f5f196329f62930cf05e58aa87cce6dca3d8358f8b",
|
||||||
|
"native_platform": "cloud-linux-x86_64",
|
||||||
|
"uid": 65534,
|
||||||
|
"isolated_network": true,
|
||||||
|
"no_new_privs": true,
|
||||||
|
"verified": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"stdout": "\nrunning 1 test\ntest transient_clock_failure_keeps_the_same_signed_reply_for_both_control_commit_phases ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 28 filtered out; finished in 0.11s\n\n",
|
||||||
|
"stderr": "",
|
||||||
|
"binary_sha256": "a805edf38b1aa8f35c3f30bc596f28f602e62fde0f19d6a4eecee37501dcbfe0",
|
||||||
|
"native_platform": "cloud-linux-x86_64",
|
||||||
|
"uid": 65534,
|
||||||
|
"isolated_network": true,
|
||||||
|
"no_new_privs": true,
|
||||||
|
"verified": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"product_job": false,
|
||||||
|
"binary_sha256": "b9375633933d07ae7d1415077ae655799781b44e2ba98799bb155d8d4554db4d",
|
||||||
|
"bootstrap_administrator": true,
|
||||||
|
"stderr": "",
|
||||||
|
"native_platform": "windows-11-x86_64",
|
||||||
|
"stdout": "\nrunning 1 test\ntest job_readiness::response_tests::signed_response_waits_for_fresh_clock_and_issued_at_without_extending_its_deadline ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 59 filtered out; finished in 0.00s\n\n",
|
||||||
|
"verified": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exit_code": 0,
|
||||||
|
"product_job": false,
|
||||||
|
"binary_sha256": "dba9fe4222770b11ec7e9219f9d93ab5b4641afdd3686f798d0af3b709b91a42",
|
||||||
|
"bootstrap_administrator": true,
|
||||||
|
"stderr": "",
|
||||||
|
"native_platform": "windows-11-x86_64",
|
||||||
|
"stdout": "\nrunning 1 test\ntest transient_clock_failure_keeps_the_same_signed_reply_for_both_control_commit_phases ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 28 filtered out; finished in 0.13s\n\n",
|
||||||
|
"verified": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -273,3 +273,46 @@ Aufnahmefrist konnte kein exakt gemeinsamer signierter Agent-/Helperstand
|
|||||||
belegt werden. Kein Dienst wurde durch diesen Versuch gestoppt, keine
|
belegt werden. Kein Dienst wurde durch diesen Versuch gestoppt, keine
|
||||||
SQL-Datenzeile geändert, kein Job gestartet. Eine gesonderte rein lesende
|
SQL-Datenzeile geändert, kein Job gestartet. Eine gesonderte rein lesende
|
||||||
Diagnose untersucht die abweichende Vorbedingung.
|
Diagnose untersucht die abweichende Vorbedingung.
|
||||||
|
|
||||||
|
## Signierte Antworten behalten; ergänzte native Prüfungen bestanden
|
||||||
|
|
||||||
|
Die lesende Laufzeitdiagnose bestätigte denselben letzten angewandten
|
||||||
|
Entscheidungsbeleg in Agent und Helper, während neue vorbereitete
|
||||||
|
Anfragecheckpoints weiterliefen. Es gab keine neuen Verbräuche, Grants,
|
||||||
|
Helperintents, offenen Ausgaben oder gespeicherten Faults. Die Diagnose
|
||||||
|
erteilte weder Bereitschaft noch eine Reparaturfreigabe.
|
||||||
|
|
||||||
|
Die ergänzte Korrektur authentisiert genau eine Antwort gegen ihren
|
||||||
|
Originalrequest und Kanal, behält sie begrenzt und wartet auf einen frischen
|
||||||
|
100-ms-Zeitbeleg sowie das Erreichen ihres signierten Ausstellungszeitpunkts
|
||||||
|
durch die lokale Uhr. Danach werden beide normalen Commitphasen weiterhin
|
||||||
|
unabhängig geprüft. Ein bloß vorübergehend unbrauchbarer Zeitbeleg verwirft
|
||||||
|
nicht mehr die Antwort oder den bereits erfolgreichen ersten Commit.
|
||||||
|
Keine ursprüngliche Gültigkeitsfrist wird verlängert. Auch der Helper wartet
|
||||||
|
vor seinem ersten Dispatch feiner; eine unklare Antwort löst keine
|
||||||
|
Execute-Wiederholung aus.
|
||||||
|
|
||||||
|
Tatsächlich bestanden zuerst zwei gezielte Tests auf der Cloud-Linux-VM
|
||||||
|
und danach dieselben zwei nativ auf Windows: **je 2/2, keine Fehler**.
|
||||||
|
Die Coreprüfung verwendet beide echten SQLite-Commitphasen mit demselben
|
||||||
|
signierten Reply und vorübergehend ausfallender Uhr. Sie erzeugt keine Jobs,
|
||||||
|
Grants oder Zielwirkung. Der genaue eingefrorene Windows-GNU-Quellsatz
|
||||||
|
bestand zusätzlich alle Targets mit Clippy `-D warnings`.
|
||||||
|
[Native Resultate und exakte Quellen-/Binärhashes](evidence/windows-deferred-sync-20261116.json).
|
||||||
|
|
||||||
|
Der korrigierte Wartungsstopp erfasste am 16. November um **11:55:25 MEZ**
|
||||||
|
den gemeinsamen Originalstand und die exakt erhaltene, noch unbeantwortete
|
||||||
|
reine Syncanfrage unter einer bestehenden SQLite-Schreibreservierung.
|
||||||
|
Dadurch konnten weder Antwortanwendung noch neue Permitreservierung committen.
|
||||||
|
Der Agent wurde genau einmal regulär gestoppt; keine SQL-Zeile, Identität,
|
||||||
|
Signatur, Anfrage oder Ergebnis wurde geändert oder gelöscht. Der Helper
|
||||||
|
blieb erhalten. Der vorherige erfolglose Fall wurde nicht überschrieben.
|
||||||
|
|
||||||
|
Danach wurde ausschließlich die bytegleiche ursprüngliche passive
|
||||||
|
Konfiguration wieder eingesetzt und die alte Version passiv gestartet.
|
||||||
|
Der reguläre geprüfte Nachfolgerwechsel installierte um **12:06:57 MEZ**
|
||||||
|
Agent SHA256 `b24a248e1ebc7d9039c789965413c164b5f83d0f20d34f56d11d8485915905ca`
|
||||||
|
bei unverändertem Helper `59675a3cbb3a5d213fc764f38379d6b81baaf9d47a7124470cc2767d4985763e`.
|
||||||
|
Alle Geräte-/PKI-/Journal-Dateikennungen blieben erhalten. Der zugehörige
|
||||||
|
Fall ist `969d319a-d1a0-41fb-b1af-afc17a26d775`. Zeitprofilveröffentlichung
|
||||||
|
und aktiver neuer Skriptlauf sind weiterhin gesondert nachzuweisen.
|
||||||
|
|||||||
Reference in New Issue
Block a user