Close and reopen the clock on non-transport refresh failures; keep held evidence only across transport loss or busy helper; record per-role refresh diagnostics
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-17 14:28:36 +01:00
parent 6488b6e89d
commit a576c10e8a
+56 -6
View File
@@ -308,6 +308,35 @@ impl VerifiedTimeSample {
} }
} }
#[cfg(feature = "system-helper")] #[cfg(feature = "system-helper")]
const TIME_TRANSPORT: &str = "TIME_TRANSPORT_UNAVAILABLE";
#[cfg(feature = "system-helper")]
/// Transport loss or a busy SYSTEM helper never invalidates held evidence.
fn transient_refresh(error: &anyhow::Error) -> bool {
error
.chain()
.any(|cause| matches!(cause.to_string().as_str(), TIME_TRANSPORT | "HELPER_TIME_BUSY"))
}
#[cfg(feature = "system-helper")]
/// Operator diagnostics only: last refresh failure per role, written on change.
#[derive(Default)]
struct RefreshReport(Option<String>);
#[cfg(feature = "system-helper")]
impl RefreshReport {
fn record(&mut self, state: &Path, role: &str, result: &Result<()>) {
let current = result.as_ref().err().map(|e| {
let mut text = format!("{e:#}");
text.truncate(512);
text
});
if current == self.0 {
return;
}
let value = serde_json::json!({"schema_version":1,"role":role,"observed_at_utc":Utc::now(),"error":current});
let _ = std::fs::write(state.join(format!("time-refresh-{role}.json")), value.to_string());
self.0 = current;
}
}
#[cfg(feature = "system-helper")]
async fn time_response( async fn time_response(
client: &reqwest::Client, client: &reqwest::Client,
address: std::net::SocketAddr, address: std::net::SocketAddr,
@@ -754,6 +783,8 @@ impl Agent {
let own_stop = stop.clone(); let own_stop = stop.clone();
let own_http = client.clone(); let own_http = client.clone();
let address = self.config.connect_address; let address = self.config.connect_address;
let own_state = self.config.state_directory.clone();
let helper_state = self.config.state_directory.clone();
// Only the connection owner publishes observations. Dropping it also // Only the connection owner publishes observations. Dropping it also
// discards unfinished samples, so an old worker cannot overwrite a newer report. // discards unfinished samples, so an old worker cannot overwrite a newer report.
let sample_slot = let sample_slot =
@@ -766,13 +797,16 @@ impl Agent {
.cloned() .cloned()
.collect(); .collect();
tasks.push(tokio::spawn(async move { tasks.push(tokio::spawn(async move {
let mut own_report = RefreshReport::default();
while !own_stop.load(Ordering::SeqCst) { while !own_stop.load(Ordering::SeqCst) {
let result = async { let result = async {
if !own_clock.configured_for(generation) { if !own_clock.configured_for(generation) {
own_clock.open_connection(generation).await?; own_clock.open_connection(generation).await?;
} }
let request = own_clock.begin(generation)?; let request = own_clock.begin(generation)?;
let bytes = time_response(&own_http, address, &request).await?; let bytes = time_response(&own_http, address, &request)
.await
.map_err(|e| e.context(TIME_TRANSPORT))?;
ensure!(!own_stop.load(Ordering::SeqCst), "TIME_CONNECTION_CLOSED"); ensure!(!own_stop.load(Ordering::SeqCst), "TIME_CONNECTION_CLOSED");
own_clock.complete(generation, &bytes)?; own_clock.complete(generation, &bytes)?;
if let Some(slot) = own_receipt.as_ref().filter(|s| s.get().is_none()) { if let Some(slot) = own_receipt.as_ref().filter(|s| s.get().is_none()) {
@@ -782,8 +816,15 @@ impl Agent {
Ok::<(), anyhow::Error>(()) Ok::<(), anyhow::Error>(())
} }
.await; .await;
// A failed refresh keeps the still-valid held evidence; the next // A transport failure keeps the still-valid held evidence and the
// begin supersedes the unanswered challenge (no late completion). // next begin supersedes the unanswered challenge. Any other failure
// closes so the next round reopens a fresh verifier.
if let Err(error) = &result
&& !transient_refresh(error)
{
let _ = own_clock.close_generation(generation);
}
own_report.record(&own_state, "agent", &result);
tokio::time::sleep(mode.interval(result.is_ok())).await; tokio::time::sleep(mode.interval(result.is_ok())).await;
} }
let _ = own_clock.close_generation(generation); let _ = own_clock.close_generation(generation);
@@ -791,10 +832,13 @@ impl Agent {
if let Some(time_helper) = helper.clone() { if let Some(time_helper) = helper.clone() {
let helper_stop = stop.clone(); let helper_stop = stop.clone();
tasks.push(tokio::spawn(async move { tasks.push(tokio::spawn(async move {
let mut helper_report = RefreshReport::default();
while !helper_stop.load(Ordering::SeqCst) { while !helper_stop.load(Ordering::SeqCst) {
let result = async { let result = async {
let request = time_helper.time_begin(generation).await?; let request = time_helper.time_begin(generation).await?;
let bytes = time_response(&client, address, &request).await?; let bytes = time_response(&client, address, &request)
.await
.map_err(|e| e.context(TIME_TRANSPORT))?;
ensure!( ensure!(
!helper_stop.load(Ordering::SeqCst), !helper_stop.load(Ordering::SeqCst),
"TIME_CONNECTION_CLOSED" "TIME_CONNECTION_CLOSED"
@@ -809,8 +853,14 @@ impl Agent {
Ok::<(), anyhow::Error>(()) Ok::<(), anyhow::Error>(())
} }
.await; .await;
// HELPER_TIME_BUSY during a SYSTEM run or a lost reply keeps the // HELPER_TIME_BUSY during a SYSTEM run or a transport failure keeps
// helper's held evidence; generation end still closes it below. // the helper's held evidence; other failures close and reopen.
if let Err(error) = &result
&& !transient_refresh(error)
{
let _ = time_helper.time_close(generation).await;
}
helper_report.record(&helper_state, "system_helper", &result);
tokio::time::sleep(mode.interval(result.is_ok())).await; tokio::time::sleep(mode.interval(result.is_ok())).await;
} }
let _ = time_helper.time_close(generation).await; let _ = time_helper.time_close(generation).await;