Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
+51
-18
@@ -1,3 +1,4 @@
|
||||
mod agent_update_apply;
|
||||
mod agent_update_catalog;
|
||||
mod agent_update_health;
|
||||
mod apt;
|
||||
@@ -62,6 +63,7 @@ struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
||||
certificates: certificate_rotation::RotationStore,
|
||||
rotation_policy: certificate_rotation::RotationPolicy,
|
||||
catalog: agent_update_catalog::Checker,
|
||||
update_observer: agent_update_apply::Observer,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
@@ -428,6 +430,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
|| binding.verify(&request, Utc::now()).is_err()
|
||||
{
|
||||
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
|
||||
} else if matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) {
|
||||
match agent_update_apply::handoff(&binding, &request, grant.grant_id) {
|
||||
// Full now owns this original. The stable updater may
|
||||
// stop us. Reconciliation alone supplies its final result.
|
||||
Ok(_) => return,
|
||||
Err(_) => rmm_privileged_helper::Outcome::unknown("UPDATE_HANDOFF_UNCONFIRMED"),
|
||||
}
|
||||
} else if job.execution_identity == "root" {
|
||||
apt::invoke_with_control(&binding, &job, &request, || {
|
||||
ensure!(
|
||||
@@ -483,6 +492,9 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
capabilities.extend(apt::capabilities(binding));
|
||||
capabilities.extend(helper_sync::general_capabilities(binding));
|
||||
if self.clock.trusted() {
|
||||
capabilities.extend(self.update_observer.capabilities());
|
||||
}
|
||||
}
|
||||
capabilities.sort();
|
||||
capabilities.dedup();
|
||||
@@ -530,18 +542,15 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if !storage::passive_collection_ready(&self.db)? {
|
||||
return Ok(());
|
||||
}
|
||||
// Until the separately qualified protected updater/helper observer is
|
||||
// provisioned, report its real absence; never infer apply readiness from
|
||||
// this schema support or the agent's own code/build version.
|
||||
let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured(
|
||||
rmm_domain::agent_update::Platform::LinuxX64,
|
||||
Utc::now(),
|
||||
);
|
||||
let address = self.address;
|
||||
let ca = &self.ca;
|
||||
let certificates = &self.certificates;
|
||||
self.catalog
|
||||
.poll(self.binding.as_ref(), self.clock.clone(), address, || {
|
||||
self.catalog.poll(
|
||||
self.binding.as_ref(),
|
||||
self.clock.clone(),
|
||||
address,
|
||||
self.update_observer.catalog_record(),
|
||||
|| {
|
||||
let identity = certificates.current(Utc::now().timestamp())?;
|
||||
https_channel::mtls_client(
|
||||
address,
|
||||
@@ -549,7 +558,12 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)
|
||||
});
|
||||
},
|
||||
);
|
||||
let observation = self.update_observer.poll(
|
||||
self.binding.as_ref(),
|
||||
self.catalog.candidate_envelope(Utc::now()),
|
||||
);
|
||||
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
|
||||
let envelope = self.envelope(
|
||||
generation,
|
||||
@@ -660,6 +674,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
let mut first_heartbeat = true;
|
||||
let mut health_task: Option<PendingUpdateHealth> = None;
|
||||
let mut health_inventory_prefix: Option<String> = None;
|
||||
let mut update_prefetch: Option<(JobOffer, agent_update_apply::Prefetch)> = None;
|
||||
let mut prefetched = std::collections::HashSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(result) = self.inventory_rx.recv() => {
|
||||
@@ -742,6 +758,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
_ = tick.tick() => {
|
||||
self.replay(&ws, generation).await?;
|
||||
if update_prefetch.as_ref().is_some_and(|(_,task)|task.0.is_finished()) {
|
||||
let (offer, mut task)=update_prefetch.take().context("UPDATE_PREFETCH_REQUIRED")?;
|
||||
match (&mut task.0).await {
|
||||
Ok(Ok(())) => {prefetched.insert((offer.job_id.clone(),offer.attempt_id.clone()));},
|
||||
_ => {self.record(generation,&offer,"",jobs::no_start("failed","UPDATE_ARTIFACT_PREFETCH_FAILED"))?;needs_sync=true;}
|
||||
}
|
||||
}
|
||||
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
||||
let update=self.update_proof()?;
|
||||
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
|
||||
@@ -749,7 +772,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if let Some(proof)=&update {
|
||||
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
|
||||
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
|
||||
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..}|rmm_updater_handoff::Evidence::RollbackServicesStarted{..})) {
|
||||
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
|
||||
// Fixed update health includes one genuinely new
|
||||
// normal base sample, never a fresh timestamp on old data.
|
||||
@@ -760,6 +783,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Planning pins must be refreshed before the 90-second stored-data
|
||||
// planning limit. One inflight blocking collector bounds pressure;
|
||||
// heartbeats never perform this metadata work.
|
||||
if update_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_update_observation(generation)?;
|
||||
update_at = Instant::now();
|
||||
}
|
||||
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
||||
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
@@ -773,6 +803,15 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
} else if !self.ready(generation,job.execution_identity=="root") {
|
||||
needs_sync=true;
|
||||
} else if Utc::now() >= job.not_before {
|
||||
if matches!(job.action,rmm_domain::jobs::Action::UpdateAgent{..}) && !prefetched.contains(&(offer.job_id.clone(),offer.attempt_id.clone())) {
|
||||
if update_prefetch.is_none() {
|
||||
let identity=self.certificates.current(Utc::now().timestamp())?;
|
||||
let client=https_channel::mtls_client(self.address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?;
|
||||
let base=format!("https://{}:{}",https_channel::server_name()?,self.address.port());
|
||||
update_prefetch=Some((offer.clone(),agent_update_apply::Prefetch(tokio::spawn(agent_update_apply::prefetch(client,base,job)))));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Reserve before send. A lost response must never cause
|
||||
// a second permit request for this attempt.
|
||||
jobs::reserve_request(&self.db, &offer)?;
|
||||
@@ -808,13 +847,6 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
heartbeat_at = Instant::now();
|
||||
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
}
|
||||
// Planning pins must be refreshed before the 90-second stored-data
|
||||
// planning limit. One inflight blocking collector bounds pressure;
|
||||
// heartbeats never perform this metadata work.
|
||||
if update_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_update_observation(generation)?;
|
||||
update_at = Instant::now();
|
||||
}
|
||||
if inventory_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_inventory();
|
||||
inventory_at = Instant::now();
|
||||
@@ -1017,6 +1049,7 @@ async fn main() -> Result<()> {
|
||||
certificates,
|
||||
rotation_policy,
|
||||
catalog: agent_update_catalog::Checker::default(),
|
||||
update_observer: agent_update_apply::Observer::default(),
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1_u64;
|
||||
|
||||
Reference in New Issue
Block a user