Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s

This commit is contained in:
2026-11-16 15:25:46 +01:00
parent 6497c79a91
commit 96a2b6e5ae
44 changed files with 6986 additions and 1024 deletions
+329 -22
View File
@@ -133,6 +133,12 @@ impl<'a> FullUpdateGuard<'a> {
journal.phase(),
Phase::LedgerReturnIntent | Phase::ServicesStarted | Phase::HealthVerified
);
let restored = matches!(
journal.phase(),
Phase::RollbackLedgerReturnIntent
| Phase::RollbackServicesStarted
| Phase::RollbackHealthVerified
);
let services = service_policies(&original, &trust, target)?;
let own = services.observe(Service::Full, &budget)?;
ensure!(
@@ -148,6 +154,8 @@ impl<'a> FullUpdateGuard<'a> {
"UPDATE_ORIGINAL_FULL_CHANGED"
);
Participant::OriginalHelper
} else if restored && header.boot_id == Uuid::parse_str(&rmm_agent_linux::boot_id()?)? {
Participant::RestoredHelper
} else if target && header.boot_id == Uuid::parse_str(&rmm_agent_linux::boot_id()?)? {
Participant::NewHelper
} else {
@@ -227,8 +235,11 @@ impl<'a> FullUpdateGuard<'a> {
}
self.check_effect_window()?;
ensure!(
self.participant == Participant::NewHelper,
"UPDATE_NEW_FULL_REQUIRED"
matches!(
self.participant,
Participant::NewHelper | Participant::RestoredHelper
),
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
);
let agent = self.services.observe(Service::Agent, &self.budget)?;
let mut peer = std::mem::MaybeUninit::<libc::ucred>::uninit();
@@ -264,18 +275,30 @@ impl<'a> FullUpdateGuard<'a> {
) -> Result<()> {
self.check_effect_window()?;
ensure!(
self.participant == Participant::NewHelper,
"UPDATE_NEW_FULL_REQUIRED"
matches!(
self.participant,
Participant::NewHelper | Participant::RestoredHelper
),
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
);
let mut journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
ensure!(
journal.phase() == Phase::ServicesStarted,
matches!(
journal.phase(),
Phase::ServicesStarted | Phase::RollbackServicesStarted
),
"UPDATE_HEALTH_PHASE"
);
let event = journal.prepare(
u64::try_from(self.clock.now()?.timestamp())?,
Evidence::HealthVerified {
receipt: signed.clone(),
if self.participant == Participant::RestoredHelper {
Evidence::RollbackHealthVerified {
receipt: signed.clone(),
}
} else {
Evidence::HealthVerified {
receipt: signed.clone(),
}
},
)?;
journal.replay(&event)?; // ORIGINAL pinned health signature and all kernel fields.
@@ -286,7 +309,10 @@ impl<'a> FullUpdateGuard<'a> {
self.check_effect_window()?;
let journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
ensure!(
journal.phase() == Phase::HealthVerified,
matches!(
journal.phase(),
Phase::HealthVerified | Phase::RollbackHealthVerified
),
"UPDATE_ORIGINAL_STORED_HEALTH_REQUIRED"
);
let event_raw:Vec<u8>=db.query_row("SELECT event FROM agent_update_handoff_events WHERE grant_id=?1 ORDER BY sequence DESC LIMIT 1",[self.original.permit().grant_id.to_string()],|r|r.get(0))?;
@@ -295,8 +321,16 @@ impl<'a> FullUpdateGuard<'a> {
hash(&event_raw) == journal.head_sha256(),
"UPDATE_STORED_HEALTH_HEAD_CHANGED"
);
let Evidence::HealthVerified { receipt } = &event.evidence else {
anyhow::bail!("UPDATE_STORED_HEALTH_EVENT_REQUIRED")
let receipt = match &event.evidence {
Evidence::HealthVerified { receipt } if self.participant == Participant::NewHelper => {
receipt
}
Evidence::RollbackHealthVerified { receipt }
if self.participant == Participant::RestoredHelper =>
{
receipt
}
_ => anyhow::bail!("UPDATE_STORED_HEALTH_EVENT_REQUIRED"),
};
self.health = Some(self.health_evidence(db, receipt)?);
Ok(event_raw)
@@ -354,8 +388,11 @@ impl<'a> FullUpdateGuard<'a> {
pub fn started(&self) -> Result<(Vec<rmm_updater_handoff::StartedService>, String)> {
self.check_effect_window()?;
ensure!(
self.participant == Participant::NewHelper,
"UPDATE_NEW_FULL_REQUIRED"
matches!(
self.participant,
Participant::NewHelper | Participant::RestoredHelper
),
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
);
let mut raw = Vec::new();
let mut proofs = Vec::new();
@@ -415,7 +452,13 @@ impl<'a> FullUpdateGuard<'a> {
Ok(())
}
fn verify_owner(&self, phase: Phase, events: &[Event]) -> Result<()> {
if !matches!(phase, Phase::ServicesStarted | Phase::HealthVerified) {
if !matches!(
phase,
Phase::ServicesStarted
| Phase::HealthVerified
| Phase::RollbackServicesStarted
| Phase::RollbackHealthVerified
) {
let updater = self
.updater
.as_ref()
@@ -431,15 +474,30 @@ impl<'a> FullUpdateGuard<'a> {
"UPDATE_ORIGINAL_UPDATER_OWNER_CHANGED"
);
}
if matches!(phase, Phase::ServicesStarted | Phase::HealthVerified) {
if matches!(
phase,
Phase::ServicesStarted
| Phase::HealthVerified
| Phase::RollbackServicesStarted
| Phase::RollbackHealthVerified
) {
let started = events
.iter()
.find_map(|event| {
if let Evidence::ServicesStarted { services, .. } = &event.evidence {
.find_map(|event| match &event.evidence {
Evidence::ServicesStarted { services, .. }
if matches!(phase, Phase::ServicesStarted | Phase::HealthVerified) =>
{
Some(services)
} else {
None
}
Evidence::RollbackServicesStarted { services, .. }
if matches!(
phase,
Phase::RollbackServicesStarted | Phase::RollbackHealthVerified
) =>
{
Some(services)
}
_ => None,
})
.context("UPDATE_STARTED_PROOF_MISSING")?;
self.verify_current_services(started)?;
@@ -486,16 +544,39 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
);
self.check_effect_window()?;
ensure!(
self.participant == Participant::NewHelper,
"UPDATE_NEW_FULL_REQUIRED"
matches!(
self.participant,
Participant::NewHelper | Participant::RestoredHelper
),
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
);
self.witnesses.files.verify_pair(&self.budget)?;
if self.participant == Participant::RestoredHelper {
self.witnesses.files.verify_predecessor_pair(&self.budget)?;
} else {
self.witnesses.files.verify_pair(&self.budget)?;
}
match &event.evidence {
Evidence::ServicesStarted {
services,
ledger_identity_sha256,
observation_sha256,
}
| Evidence::RollbackServicesStarted {
services,
ledger_identity_sha256,
observation_sha256,
} => {
ensure!(
matches!(
(&event.evidence, self.participant),
(Evidence::ServicesStarted { .. }, Participant::NewHelper)
| (
Evidence::RollbackServicesStarted { .. },
Participant::RestoredHelper
)
),
"UPDATE_STARTED_ROLE"
);
let mut held = Vec::new();
let mut proofs = Vec::new();
for service in [Service::Agent, Service::General, Service::Full] {
@@ -511,7 +592,18 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
"UPDATE_ACTUAL_STARTED_PROOFS_REQUIRED"
);
}
Evidence::HealthVerified { receipt } => {
Evidence::HealthVerified { receipt } | Evidence::RollbackHealthVerified { receipt } => {
ensure!(
matches!(
(&event.evidence, self.participant),
(Evidence::HealthVerified { .. }, Participant::NewHelper)
| (
Evidence::RollbackHealthVerified { .. },
Participant::RestoredHelper
)
),
"UPDATE_HEALTH_ROLE"
);
let health = self
.health
.as_ref()
@@ -521,6 +613,14 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
"UPDATE_LOCAL_HEALTH_RECEIPT_CHANGED"
);
}
Evidence::RollbackOutcomePersisted { .. }
if self.participant == Participant::RestoredHelper =>
{
ensure!(
self.health.is_some(),
"UPDATE_ROLLBACK_LOCAL_HEALTH_REQUIRED"
);
}
// InstalledRecord/Outcome have a separate original-record CAS.
_ => anyhow::bail!("UPDATE_FULL_EVENT_NOT_YET_QUALIFIED"),
}
@@ -578,10 +678,63 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
.observe(service, &self.budget)?
.verify(&self.budget)?;
}
(
Participant::NewHelper,
NativeEffect::ConditionalRollbackStopGeneral,
NativeEffectStage::ObserveNonstart,
) => {
// The retained new Full proves the early-Full-failure branch
// was never entered. Closing this reservation records NO stop.
ensure!(
handoff.schema_version == 2,
"UPDATE_CONDITIONAL_STOP_PROFILE"
);
for service in [Service::General, Service::Full] {
self.services
.observe(service, &self.budget)?
.verify(&self.budget)?;
}
}
(
Participant::RestoredHelper,
NativeEffect::RollbackStartService { service },
NativeEffectStage::Observe,
) => {
let service = match service {
NativeService::Agent => Service::Agent,
NativeService::General => Service::General,
NativeService::Full => Service::Full,
_ => anyhow::bail!("UPDATE_WINDOWS_SERVICE_ON_LINUX"),
};
self.witnesses.files.verify_predecessor_pair(&self.budget)?;
self.services
.observe(service, &self.budget)?
.verify(&self.budget)?;
}
_ => anyhow::bail!("UPDATE_FULL_NATIVE_EFFECT_NOT_QUALIFIED"),
}
Ok(())
}
fn native_observation_sha256(
&self,
handoff: &Handoff,
delegation: &UpdateDelegation,
effect: &NativeEffect,
stage: NativeEffectStage,
) -> Result<String> {
self.verify_native_effect(handoff, delegation, effect, stage)?;
ensure!(
*effect == NativeEffect::ConditionalRollbackStopGeneral
&& stage == NativeEffectStage::ObserveNonstart,
"UPDATE_FULL_NATIVE_HASH_NOT_QUALIFIED"
);
Ok(hash(&serde_json::to_vec(&(
"conditional_general_stop_unused_v1",
handoff.owner_nonce,
handoff.attempt.grant_id,
self.services.observe(Service::Full, &self.budget)?.proof(),
))?))
}
}
struct VerifiedHealth {
@@ -985,3 +1138,157 @@ impl FullUpdateGuard<'_> {
Ok(bytes)
}
}
impl FullUpdateGuard<'_> {
pub fn finalize_restored(&mut self, db: &mut Connection) -> Result<Vec<u8>> {
ensure!(
self.participant == Participant::RestoredHelper,
"UPDATE_RESTORED_FULL_REQUIRED"
);
self.qualify_effects(db)?;
self.qualify_stored_health(db)?;
self.witnesses.files.verify_predecessor_pair(&self.budget)?;
// TrustState still pins the SAME original record and activation floor.
self.trust.recheck(&self.original, self.clock)?;
let journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
let health = self
.health
.as_ref()
.context("UPDATE_ROLLBACK_HEALTH_REQUIRED")?;
let outcome = crate::Outcome {
state: "failed".into(),
stdout: vec![],
stderr: vec![],
exit_code: None,
output_truncated: false,
reboot_required: false,
error_code: "AGENT_UPDATE_RETURNED_TO_PREDECESSOR".into(),
evidence: serde_json::json!({"postconditions_passed":false,"previous_boot_id":self.original.handoff().boot_id,
"agent_update":{"schema_version":2,"handoff_sha256":journal.handoff_sha256(),
"manifest_sha256":self.original.handoff().target.manifest_sha256,
"predecessor_record_sha256":self.original.handoff().predecessor_record_sha256,
"installed_record_sha256":self.original.handoff().predecessor_record_sha256,
"signed_health_sha256":health.signed_sha256,"rolled_back":true}}),
};
update_handoff::finalize_rollback(
db,
self,
self.original.permit(),
journal.head_sha256(),
u64::try_from(self.clock.now()?.timestamp())?,
&outcome,
)?;
Ok(db.query_row(
"SELECT outcome FROM execution_evidence WHERE grant_id=?1",
[self.original.permit().grant_id.to_string()],
|r| r.get(0),
)?)
}
pub fn restored(&self) -> bool {
self.participant == Participant::RestoredHelper
}
/// Exact ORIGINAL bytes from a signed inflight decision already applied
/// by this actual Full process under the same existing guarded ledger.
/// Missing evidence keeps the worker waiting; it never becomes readiness.
pub fn pre_stop_signed_sync(&self, db: &Connection) -> Result<Option<(Vec<u8>, Vec<u8>)>> {
use prost::Message;
use rusqlite::OptionalExtension;
self.check(db)?;
if self.participant != Participant::OriginalHelper {
return Ok(None);
}
let journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
if journal.phase() != Phase::StopIntent {
return Ok(None);
}
let id: Option<String> = db
.query_row(
"SELECT last_decision FROM helper_sync_meta WHERE id=1 AND fault IS NULL",
[],
|r| r.get(0),
)
.optional()?
.flatten();
let Some(id) = id else { return Ok(None) };
let id: Uuid = id.parse()?;
let applied:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM agent_update_sync_applied WHERE grant_id=?1 AND decision_id=?2)",
rusqlite::params![self.original.permit().grant_id.to_string(),id.to_string()],|r|r.get(0))?;
if !applied {
return Ok(None);
}
self.qualify_effects(db)?;
let round = rmm_agent_core::update_health_sync::read_round(
db,
self,
self.binding,
self.original.permit().grant_id,
id,
)?;
let decision = round.decision();
let h = self.original.handoff();
let now = self.clock.now()?;
let proof = update_handoff::current_proof(db, self, self.original.permit().grant_id)?
.context("UPDATE_PRESTOP_ORIGINAL_PROOF_REQUIRED")?;
let expected =
rmm_domain::agent_update::inflight_decision(&proof, self.trust.health_public_key())
.map_err(anyhow::Error::msg)?;
let attempts = decision
.attempts
.iter()
.filter(|a| a.job_id == h.attempt.job_id && a.attempt_id == h.attempt.attempt_id)
.collect::<Vec<_>>();
ensure!(attempts.len() == 1, "UPDATE_PRESTOP_ONE_ORIGINAL_ACK");
let ack = attempts[0];
ensure!(
decision.valid_profile(now)
&& !decision.ready
&& !decision.helper_ready
&& decision.boot_id == h.boot_id
&& decision.connection_generation == h.connection_generation
&& decision.journal_id == h.agent_journal_id
&& decision.helper_journal_id == Some(h.helper_journal_id)
&& ack.update_inflight.as_ref() == Some(&expected)
&& ack.grant_id == Some(h.attempt.grant_id)
&& ack.authorization_id == Some(h.attempt.authorization_id)
&& ack.plan_hash.as_deref() == Some(h.attempt.plan_sha256.as_str())
&& ack.job_payload_hash == h.attempt.job_payload_sha256
&& ack.original_boot_id == Some(h.boot_id)
&& ack.action == "update_agent"
&& ack.execution_identity == "root"
&& !ack.unlock
&& ack.outcome_state == "running"
&& ack.outcome_bytes.is_empty()
&& ack.outcome_sha256.is_none()
&& round.applied().applied_at >= decision.issued_at
&& round.applied().applied_at < decision.valid_until,
"UPDATE_PRESTOP_APPLIED_ORIGINAL_ACK"
);
let (raw, response): (Vec<u8>, Vec<u8>) = db.query_row(
"SELECT request,response FROM helper_sync_decisions WHERE decision_id=?1",
[id.to_string()],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
ensure!(
raw.len() <= 256 * 1024 && response.len() <= 256 * 1024,
"UPDATE_PRESTOP_WIRE_LIMIT"
);
let request = rmm_protocol::SyncRequest::decode(raw.as_slice())?;
let local = request
.attempts
.iter()
.find(|a| {
a.job_id == h.attempt.job_id.to_string()
&& a.attempt_id == h.attempt.attempt_id.to_string()
})
.context("UPDATE_PRESTOP_REQUEST_ATTEMPT")?;
ensure!(
request.journal_complete
&& local.state == "running"
&& local.consumption == "consumed"
&& local.update_handoff_bytes == proof.bytes().map_err(anyhow::Error::msg)?,
"UPDATE_PRESTOP_REQUEST_ORIGINAL_PREFIX"
);
self.check_effect_window()?;
Ok(Some((raw, response)))
}
}