Close and reopen the clock on non-transport refresh failures; keep held evidence only across transport loss or busy helper; record per-role refresh diagnostics
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -308,6 +308,35 @@ impl VerifiedTimeSample {
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
const TIME_TRANSPORT: &str = "TIME_TRANSPORT_UNAVAILABLE";
|
||||
#[cfg(feature = "system-helper")]
|
||||
/// Transport loss or a busy SYSTEM helper never invalidates held evidence.
|
||||
fn transient_refresh(error: &anyhow::Error) -> bool {
|
||||
error
|
||||
.chain()
|
||||
.any(|cause| matches!(cause.to_string().as_str(), TIME_TRANSPORT | "HELPER_TIME_BUSY"))
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
/// Operator diagnostics only: last refresh failure per role, written on change.
|
||||
#[derive(Default)]
|
||||
struct RefreshReport(Option<String>);
|
||||
#[cfg(feature = "system-helper")]
|
||||
impl RefreshReport {
|
||||
fn record(&mut self, state: &Path, role: &str, result: &Result<()>) {
|
||||
let current = result.as_ref().err().map(|e| {
|
||||
let mut text = format!("{e:#}");
|
||||
text.truncate(512);
|
||||
text
|
||||
});
|
||||
if current == self.0 {
|
||||
return;
|
||||
}
|
||||
let value = serde_json::json!({"schema_version":1,"role":role,"observed_at_utc":Utc::now(),"error":current});
|
||||
let _ = std::fs::write(state.join(format!("time-refresh-{role}.json")), value.to_string());
|
||||
self.0 = current;
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
async fn time_response(
|
||||
client: &reqwest::Client,
|
||||
address: std::net::SocketAddr,
|
||||
@@ -754,6 +783,8 @@ impl Agent {
|
||||
let own_stop = stop.clone();
|
||||
let own_http = client.clone();
|
||||
let address = self.config.connect_address;
|
||||
let own_state = self.config.state_directory.clone();
|
||||
let helper_state = self.config.state_directory.clone();
|
||||
// Only the connection owner publishes observations. Dropping it also
|
||||
// discards unfinished samples, so an old worker cannot overwrite a newer report.
|
||||
let sample_slot =
|
||||
@@ -766,13 +797,16 @@ impl Agent {
|
||||
.cloned()
|
||||
.collect();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let mut own_report = RefreshReport::default();
|
||||
while !own_stop.load(Ordering::SeqCst) {
|
||||
let result = async {
|
||||
if !own_clock.configured_for(generation) {
|
||||
own_clock.open_connection(generation).await?;
|
||||
}
|
||||
let request = own_clock.begin(generation)?;
|
||||
let bytes = time_response(&own_http, address, &request).await?;
|
||||
let bytes = time_response(&own_http, address, &request)
|
||||
.await
|
||||
.map_err(|e| e.context(TIME_TRANSPORT))?;
|
||||
ensure!(!own_stop.load(Ordering::SeqCst), "TIME_CONNECTION_CLOSED");
|
||||
own_clock.complete(generation, &bytes)?;
|
||||
if let Some(slot) = own_receipt.as_ref().filter(|s| s.get().is_none()) {
|
||||
@@ -782,8 +816,15 @@ impl Agent {
|
||||
Ok::<(), anyhow::Error>(())
|
||||
}
|
||||
.await;
|
||||
// A failed refresh keeps the still-valid held evidence; the next
|
||||
// begin supersedes the unanswered challenge (no late completion).
|
||||
// A transport failure keeps the still-valid held evidence and the
|
||||
// next begin supersedes the unanswered challenge. Any other failure
|
||||
// closes so the next round reopens a fresh verifier.
|
||||
if let Err(error) = &result
|
||||
&& !transient_refresh(error)
|
||||
{
|
||||
let _ = own_clock.close_generation(generation);
|
||||
}
|
||||
own_report.record(&own_state, "agent", &result);
|
||||
tokio::time::sleep(mode.interval(result.is_ok())).await;
|
||||
}
|
||||
let _ = own_clock.close_generation(generation);
|
||||
@@ -791,10 +832,13 @@ impl Agent {
|
||||
if let Some(time_helper) = helper.clone() {
|
||||
let helper_stop = stop.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let mut helper_report = RefreshReport::default();
|
||||
while !helper_stop.load(Ordering::SeqCst) {
|
||||
let result = async {
|
||||
let request = time_helper.time_begin(generation).await?;
|
||||
let bytes = time_response(&client, address, &request).await?;
|
||||
let bytes = time_response(&client, address, &request)
|
||||
.await
|
||||
.map_err(|e| e.context(TIME_TRANSPORT))?;
|
||||
ensure!(
|
||||
!helper_stop.load(Ordering::SeqCst),
|
||||
"TIME_CONNECTION_CLOSED"
|
||||
@@ -809,8 +853,14 @@ impl Agent {
|
||||
Ok::<(), anyhow::Error>(())
|
||||
}
|
||||
.await;
|
||||
// HELPER_TIME_BUSY during a SYSTEM run or a lost reply keeps the
|
||||
// helper's held evidence; generation end still closes it below.
|
||||
// HELPER_TIME_BUSY during a SYSTEM run or a transport failure keeps
|
||||
// the helper's held evidence; other failures close and reopen.
|
||||
if let Err(error) = &result
|
||||
&& !transient_refresh(error)
|
||||
{
|
||||
let _ = time_helper.time_close(generation).await;
|
||||
}
|
||||
helper_report.record(&helper_state, "system_helper", &result);
|
||||
tokio::time::sleep(mode.interval(result.is_ok())).await;
|
||||
}
|
||||
let _ = time_helper.time_close(generation).await;
|
||||
|
||||
Reference in New Issue
Block a user