Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s

This commit is contained in:
2026-11-16 15:25:46 +01:00
parent 6497c79a91
commit 96a2b6e5ae
44 changed files with 6986 additions and 1024 deletions
+51 -18
View File
@@ -1,3 +1,4 @@
mod agent_update_apply;
mod agent_update_catalog;
mod agent_update_health;
mod apt;
@@ -62,6 +63,7 @@ struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
certificates: certificate_rotation::RotationStore,
rotation_policy: certificate_rotation::RotationPolicy,
catalog: agent_update_catalog::Checker,
update_observer: agent_update_apply::Observer,
}
#[derive(Default)]
struct EffectFence {
@@ -428,6 +430,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|| binding.verify(&request, Utc::now()).is_err()
{
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
} else if matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) {
match agent_update_apply::handoff(&binding, &request, grant.grant_id) {
// Full now owns this original. The stable updater may
// stop us. Reconciliation alone supplies its final result.
Ok(_) => return,
Err(_) => rmm_privileged_helper::Outcome::unknown("UPDATE_HANDOFF_UNCONFIRMED"),
}
} else if job.execution_identity == "root" {
apt::invoke_with_control(&binding, &job, &request, || {
ensure!(
@@ -483,6 +492,9 @@ impl<C: reconcile::SyncClock> Agent<C> {
}
capabilities.extend(apt::capabilities(binding));
capabilities.extend(helper_sync::general_capabilities(binding));
if self.clock.trusted() {
capabilities.extend(self.update_observer.capabilities());
}
}
capabilities.sort();
capabilities.dedup();
@@ -530,18 +542,15 @@ impl<C: reconcile::SyncClock> Agent<C> {
if !storage::passive_collection_ready(&self.db)? {
return Ok(());
}
// Until the separately qualified protected updater/helper observer is
// provisioned, report its real absence; never infer apply readiness from
// this schema support or the agent's own code/build version.
let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured(
rmm_domain::agent_update::Platform::LinuxX64,
Utc::now(),
);
let address = self.address;
let ca = &self.ca;
let certificates = &self.certificates;
self.catalog
.poll(self.binding.as_ref(), self.clock.clone(), address, || {
self.catalog.poll(
self.binding.as_ref(),
self.clock.clone(),
address,
self.update_observer.catalog_record(),
|| {
let identity = certificates.current(Utc::now().timestamp())?;
https_channel::mtls_client(
address,
@@ -549,7 +558,12 @@ impl<C: reconcile::SyncClock> Agent<C> {
&identity.certificate_pem,
&identity.private_key_pem,
)
});
},
);
let observation = self.update_observer.poll(
self.binding.as_ref(),
self.catalog.candidate_envelope(Utc::now()),
);
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
let envelope = self.envelope(
generation,
@@ -660,6 +674,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
let mut first_heartbeat = true;
let mut health_task: Option<PendingUpdateHealth> = None;
let mut health_inventory_prefix: Option<String> = None;
let mut update_prefetch: Option<(JobOffer, agent_update_apply::Prefetch)> = None;
let mut prefetched = std::collections::HashSet::new();
loop {
tokio::select! {
Some(result) = self.inventory_rx.recv() => {
@@ -742,6 +758,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
}
_ = tick.tick() => {
self.replay(&ws, generation).await?;
if update_prefetch.as_ref().is_some_and(|(_,task)|task.0.is_finished()) {
let (offer, mut task)=update_prefetch.take().context("UPDATE_PREFETCH_REQUIRED")?;
match (&mut task.0).await {
Ok(Ok(())) => {prefetched.insert((offer.job_id.clone(),offer.attempt_id.clone()));},
_ => {self.record(generation,&offer,"",jobs::no_start("failed","UPDATE_ARTIFACT_PREFETCH_FAILED"))?;needs_sync=true;}
}
}
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
let update=self.update_proof()?;
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
@@ -749,7 +772,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
if let Some(proof)=&update {
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..}|rmm_updater_handoff::Evidence::RollbackServicesStarted{..})) {
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
// Fixed update health includes one genuinely new
// normal base sample, never a fresh timestamp on old data.
@@ -760,6 +783,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
}
}
}
// Planning pins must be refreshed before the 90-second stored-data
// planning limit. One inflight blocking collector bounds pressure;
// heartbeats never perform this metadata work.
if update_at.elapsed() >= Duration::from_secs(60) {
self.queue_update_observation(generation)?;
update_at = Instant::now();
}
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
@@ -773,6 +803,15 @@ impl<C: reconcile::SyncClock> Agent<C> {
} else if !self.ready(generation,job.execution_identity=="root") {
needs_sync=true;
} else if Utc::now() >= job.not_before {
if matches!(job.action,rmm_domain::jobs::Action::UpdateAgent{..}) && !prefetched.contains(&(offer.job_id.clone(),offer.attempt_id.clone())) {
if update_prefetch.is_none() {
let identity=self.certificates.current(Utc::now().timestamp())?;
let client=https_channel::mtls_client(self.address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?;
let base=format!("https://{}:{}",https_channel::server_name()?,self.address.port());
update_prefetch=Some((offer.clone(),agent_update_apply::Prefetch(tokio::spawn(agent_update_apply::prefetch(client,base,job)))));
}
continue;
}
// Reserve before send. A lost response must never cause
// a second permit request for this attempt.
jobs::reserve_request(&self.db, &offer)?;
@@ -808,13 +847,6 @@ impl<C: reconcile::SyncClock> Agent<C> {
heartbeat_at = Instant::now();
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
}
// Planning pins must be refreshed before the 90-second stored-data
// planning limit. One inflight blocking collector bounds pressure;
// heartbeats never perform this metadata work.
if update_at.elapsed() >= Duration::from_secs(60) {
self.queue_update_observation(generation)?;
update_at = Instant::now();
}
if inventory_at.elapsed() >= Duration::from_secs(60) {
self.queue_inventory();
inventory_at = Instant::now();
@@ -1017,6 +1049,7 @@ async fn main() -> Result<()> {
certificates,
rotation_policy,
catalog: agent_update_catalog::Checker::default(),
update_observer: agent_update_apply::Observer::default(),
};
agent.recover_interrupted()?;
let mut backoff = 1_u64;