Wire preserved update control, health evidence and record publication
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-14 23:51:59 +01:00
parent 328e440f73
commit 8e93c9ead5
34 changed files with 7071 additions and 97 deletions
+100 -6
View File
@@ -1,3 +1,4 @@
mod agent_update_health;
mod apt;
mod bootstrap;
mod certificate_rotation;
@@ -100,6 +101,12 @@ impl EffectFence {
&& (!root || self.root.load(Ordering::SeqCst))
}
}
struct PendingUpdateHealth(tokio::task::JoinHandle<Result<()>>);
impl Drop for PendingUpdateHealth {
fn drop(&mut self) {
self.0.abort();
}
}
type Completion = (
JobOffer,
rmm_domain::jobs::ExecutionPermitPayload,
@@ -217,6 +224,51 @@ impl<C: reconcile::SyncClock> Agent<C> {
}
Ok(())
}
fn update_proof(&self) -> Result<Option<rmm_domain::agent_update::UpdateInflightProof>> {
self.binding
.as_ref()
.map(|binding| {
rmm_agent_core::update_health_inventory::current_proof(&self.db, binding)
})
.transpose()
.map(Option::flatten)
}
fn start_update_health(
&self,
proof: &rmm_domain::agent_update::UpdateInflightProof,
) -> Result<Option<PendingUpdateHealth>> {
if !self.clock.trusted() {
return Ok(None);
}
let binding = self.binding.as_ref().context("UPDATE_BINDING_REQUIRED")?;
let Some(context) =
agent_update_health::context(&self.db, binding, proof, self.sync_clock.now()?)?
else {
return Ok(None);
};
let header = context.handoff().clone();
let key = context.health_public_key();
let identity = self.certificates.current(Utc::now().timestamp())?;
let client = https_channel::mtls_client(
self.address,
&self.ca,
&identity.certificate_pem,
&identity.private_key_pem,
)?;
let helpers = self
.helpers
.clone()
.context("UPDATE_FULL_HELPER_REQUIRED")?;
let address = self.address;
Ok(Some(PendingUpdateHealth(tokio::spawn(async move {
let receipt = agent_update_health::request(&client, address, &context).await?;
tokio::task::spawn_blocking(move || {
helpers.submit_update_health(&header, key, &receipt)
})
.await??;
Ok(())
}))))
}
fn record(
&mut self,
generation: u64,
@@ -245,14 +297,25 @@ impl<C: reconcile::SyncClock> Agent<C> {
if grant.is_empty() {
jobs::interrupted_request(&self.db, &offer)?;
} else {
let saved =
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?;
if matches!(
binding.job(&offer.payload_bytes)?.action,
rmm_domain::jobs::Action::UpdateAgent { .. }
) && saved.is_none()
{
// This original may have intentionally handed off and
// restarted us. Gates stay closed until a genuine helper
// capture proves inflight; no Unknown is ever undone.
continue;
}
self.record(
0,
&offer,
&grant,
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?
.unwrap_or_else(|| {
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
}),
saved.unwrap_or_else(|| {
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
}),
)?;
}
}
@@ -567,6 +630,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
let mut rotation_at = Instant::now();
let mut capability_at = Instant::now();
let mut first_heartbeat = true;
let mut health_task: Option<PendingUpdateHealth> = None;
let mut health_inventory_prefix: Option<String> = None;
loop {
tokio::select! {
Some(result) = self.inventory_rx.recv() => {
@@ -574,8 +639,19 @@ impl<C: reconcile::SyncClock> Agent<C> {
if let Ok((data,observed_at)) = result {
let inventory = self.inventory_envelope(generation,data,observed_at)?;
jobs::event(&self.db,&inventory)?;
if let Some(binding)=&self.binding
&& let Err(error)=rmm_agent_core::update_health_inventory::retain_current(&self.db,&inventory,binding) {
report_closed_gate(&error);self.close_gate();needs_sync=true;
}
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
}
Some(result)=async {
match health_task.as_mut() {Some(task)=>Some((&mut task.0).await),None=>std::future::pending().await}
} => {
health_task.take();
if let Err(error)=result.map_err(anyhow::Error::from).and_then(std::convert::identity) {report_closed_gate(&error);}
self.close_gate();needs_sync=true;
},
message = ws.receive() => {
let message = message?;
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
@@ -639,9 +715,26 @@ impl<C: reconcile::SyncClock> Agent<C> {
_ = tick.tick() => {
self.replay(&ws, generation).await?;
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
let update=self.update_proof()?;
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
let own_update=update.is_some() || pending_update;
if let Some(proof)=&update {
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
// Fixed update health includes one genuinely new
// normal base sample, never a fresh timestamp on old data.
self.queue_inventory();health_inventory_prefix=Some(proof.head_sha256.clone());
}
if health_task.is_none() && self.prepared.is_none() {
match self.start_update_health(proof) {Ok(task)=>health_task=task,Err(error)=>{report_closed_gate(&error);self.close_gate();needs_sync=true;}}
}
}
}
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
if self.prepared.is_some() && !busy && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.prepared.is_none() && self.sync.is_some() && !busy && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
}
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
@@ -703,6 +796,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
if self
.certificates
.due(self.rotation_policy, Utc::now().timestamp())?
&& self.update_proof()?.is_none()
{
self.close_gate();
drop(ws);