Wire preserved update control, health evidence and record publication
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
+100
-6
@@ -1,3 +1,4 @@
|
||||
mod agent_update_health;
|
||||
mod apt;
|
||||
mod bootstrap;
|
||||
mod certificate_rotation;
|
||||
@@ -100,6 +101,12 @@ impl EffectFence {
|
||||
&& (!root || self.root.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
struct PendingUpdateHealth(tokio::task::JoinHandle<Result<()>>);
|
||||
impl Drop for PendingUpdateHealth {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
type Completion = (
|
||||
JobOffer,
|
||||
rmm_domain::jobs::ExecutionPermitPayload,
|
||||
@@ -217,6 +224,51 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn update_proof(&self) -> Result<Option<rmm_domain::agent_update::UpdateInflightProof>> {
|
||||
self.binding
|
||||
.as_ref()
|
||||
.map(|binding| {
|
||||
rmm_agent_core::update_health_inventory::current_proof(&self.db, binding)
|
||||
})
|
||||
.transpose()
|
||||
.map(Option::flatten)
|
||||
}
|
||||
fn start_update_health(
|
||||
&self,
|
||||
proof: &rmm_domain::agent_update::UpdateInflightProof,
|
||||
) -> Result<Option<PendingUpdateHealth>> {
|
||||
if !self.clock.trusted() {
|
||||
return Ok(None);
|
||||
}
|
||||
let binding = self.binding.as_ref().context("UPDATE_BINDING_REQUIRED")?;
|
||||
let Some(context) =
|
||||
agent_update_health::context(&self.db, binding, proof, self.sync_clock.now()?)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let header = context.handoff().clone();
|
||||
let key = context.health_public_key();
|
||||
let identity = self.certificates.current(Utc::now().timestamp())?;
|
||||
let client = https_channel::mtls_client(
|
||||
self.address,
|
||||
&self.ca,
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)?;
|
||||
let helpers = self
|
||||
.helpers
|
||||
.clone()
|
||||
.context("UPDATE_FULL_HELPER_REQUIRED")?;
|
||||
let address = self.address;
|
||||
Ok(Some(PendingUpdateHealth(tokio::spawn(async move {
|
||||
let receipt = agent_update_health::request(&client, address, &context).await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
helpers.submit_update_health(&header, key, &receipt)
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}))))
|
||||
}
|
||||
fn record(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
@@ -245,14 +297,25 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(&self.db, &offer)?;
|
||||
} else {
|
||||
let saved =
|
||||
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?;
|
||||
if matches!(
|
||||
binding.job(&offer.payload_bytes)?.action,
|
||||
rmm_domain::jobs::Action::UpdateAgent { .. }
|
||||
) && saved.is_none()
|
||||
{
|
||||
// This original may have intentionally handed off and
|
||||
// restarted us. Gates stay closed until a genuine helper
|
||||
// capture proves inflight; no Unknown is ever undone.
|
||||
continue;
|
||||
}
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&grant,
|
||||
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?
|
||||
.unwrap_or_else(|| {
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
|
||||
}),
|
||||
saved.unwrap_or_else(|| {
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
|
||||
}),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
@@ -567,6 +630,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
let mut rotation_at = Instant::now();
|
||||
let mut capability_at = Instant::now();
|
||||
let mut first_heartbeat = true;
|
||||
let mut health_task: Option<PendingUpdateHealth> = None;
|
||||
let mut health_inventory_prefix: Option<String> = None;
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(result) = self.inventory_rx.recv() => {
|
||||
@@ -574,8 +639,19 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if let Ok((data,observed_at)) = result {
|
||||
let inventory = self.inventory_envelope(generation,data,observed_at)?;
|
||||
jobs::event(&self.db,&inventory)?;
|
||||
if let Some(binding)=&self.binding
|
||||
&& let Err(error)=rmm_agent_core::update_health_inventory::retain_current(&self.db,&inventory,binding) {
|
||||
report_closed_gate(&error);self.close_gate();needs_sync=true;
|
||||
}
|
||||
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
|
||||
}
|
||||
Some(result)=async {
|
||||
match health_task.as_mut() {Some(task)=>Some((&mut task.0).await),None=>std::future::pending().await}
|
||||
} => {
|
||||
health_task.take();
|
||||
if let Err(error)=result.map_err(anyhow::Error::from).and_then(std::convert::identity) {report_closed_gate(&error);}
|
||||
self.close_gate();needs_sync=true;
|
||||
},
|
||||
message = ws.receive() => {
|
||||
let message = message?;
|
||||
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
|
||||
@@ -639,9 +715,26 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
_ = tick.tick() => {
|
||||
self.replay(&ws, generation).await?;
|
||||
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
||||
let update=self.update_proof()?;
|
||||
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
|
||||
let own_update=update.is_some() || pending_update;
|
||||
if let Some(proof)=&update {
|
||||
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
|
||||
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
|
||||
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
|
||||
// Fixed update health includes one genuinely new
|
||||
// normal base sample, never a fresh timestamp on old data.
|
||||
self.queue_inventory();health_inventory_prefix=Some(proof.head_sha256.clone());
|
||||
}
|
||||
if health_task.is_none() && self.prepared.is_none() {
|
||||
match self.start_update_health(proof) {Ok(task)=>health_task=task,Err(error)=>{report_closed_gate(&error);self.close_gate();needs_sync=true;}}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
||||
if self.prepared.is_some() && !busy && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && !busy && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
|
||||
@@ -703,6 +796,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if self
|
||||
.certificates
|
||||
.due(self.rotation_policy, Utc::now().timestamp())?
|
||||
&& self.update_proof()?.is_none()
|
||||
{
|
||||
self.close_gate();
|
||||
drop(ws);
|
||||
|
||||
Reference in New Issue
Block a user