Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -133,6 +133,12 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
journal.phase(),
|
||||
Phase::LedgerReturnIntent | Phase::ServicesStarted | Phase::HealthVerified
|
||||
);
|
||||
let restored = matches!(
|
||||
journal.phase(),
|
||||
Phase::RollbackLedgerReturnIntent
|
||||
| Phase::RollbackServicesStarted
|
||||
| Phase::RollbackHealthVerified
|
||||
);
|
||||
let services = service_policies(&original, &trust, target)?;
|
||||
let own = services.observe(Service::Full, &budget)?;
|
||||
ensure!(
|
||||
@@ -148,6 +154,8 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
"UPDATE_ORIGINAL_FULL_CHANGED"
|
||||
);
|
||||
Participant::OriginalHelper
|
||||
} else if restored && header.boot_id == Uuid::parse_str(&rmm_agent_linux::boot_id()?)? {
|
||||
Participant::RestoredHelper
|
||||
} else if target && header.boot_id == Uuid::parse_str(&rmm_agent_linux::boot_id()?)? {
|
||||
Participant::NewHelper
|
||||
} else {
|
||||
@@ -227,8 +235,11 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
}
|
||||
self.check_effect_window()?;
|
||||
ensure!(
|
||||
self.participant == Participant::NewHelper,
|
||||
"UPDATE_NEW_FULL_REQUIRED"
|
||||
matches!(
|
||||
self.participant,
|
||||
Participant::NewHelper | Participant::RestoredHelper
|
||||
),
|
||||
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
|
||||
);
|
||||
let agent = self.services.observe(Service::Agent, &self.budget)?;
|
||||
let mut peer = std::mem::MaybeUninit::<libc::ucred>::uninit();
|
||||
@@ -264,18 +275,30 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
) -> Result<()> {
|
||||
self.check_effect_window()?;
|
||||
ensure!(
|
||||
self.participant == Participant::NewHelper,
|
||||
"UPDATE_NEW_FULL_REQUIRED"
|
||||
matches!(
|
||||
self.participant,
|
||||
Participant::NewHelper | Participant::RestoredHelper
|
||||
),
|
||||
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
|
||||
);
|
||||
let mut journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
|
||||
ensure!(
|
||||
journal.phase() == Phase::ServicesStarted,
|
||||
matches!(
|
||||
journal.phase(),
|
||||
Phase::ServicesStarted | Phase::RollbackServicesStarted
|
||||
),
|
||||
"UPDATE_HEALTH_PHASE"
|
||||
);
|
||||
let event = journal.prepare(
|
||||
u64::try_from(self.clock.now()?.timestamp())?,
|
||||
Evidence::HealthVerified {
|
||||
receipt: signed.clone(),
|
||||
if self.participant == Participant::RestoredHelper {
|
||||
Evidence::RollbackHealthVerified {
|
||||
receipt: signed.clone(),
|
||||
}
|
||||
} else {
|
||||
Evidence::HealthVerified {
|
||||
receipt: signed.clone(),
|
||||
}
|
||||
},
|
||||
)?;
|
||||
journal.replay(&event)?; // ORIGINAL pinned health signature and all kernel fields.
|
||||
@@ -286,7 +309,10 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
self.check_effect_window()?;
|
||||
let journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
|
||||
ensure!(
|
||||
journal.phase() == Phase::HealthVerified,
|
||||
matches!(
|
||||
journal.phase(),
|
||||
Phase::HealthVerified | Phase::RollbackHealthVerified
|
||||
),
|
||||
"UPDATE_ORIGINAL_STORED_HEALTH_REQUIRED"
|
||||
);
|
||||
let event_raw:Vec<u8>=db.query_row("SELECT event FROM agent_update_handoff_events WHERE grant_id=?1 ORDER BY sequence DESC LIMIT 1",[self.original.permit().grant_id.to_string()],|r|r.get(0))?;
|
||||
@@ -295,8 +321,16 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
hash(&event_raw) == journal.head_sha256(),
|
||||
"UPDATE_STORED_HEALTH_HEAD_CHANGED"
|
||||
);
|
||||
let Evidence::HealthVerified { receipt } = &event.evidence else {
|
||||
anyhow::bail!("UPDATE_STORED_HEALTH_EVENT_REQUIRED")
|
||||
let receipt = match &event.evidence {
|
||||
Evidence::HealthVerified { receipt } if self.participant == Participant::NewHelper => {
|
||||
receipt
|
||||
}
|
||||
Evidence::RollbackHealthVerified { receipt }
|
||||
if self.participant == Participant::RestoredHelper =>
|
||||
{
|
||||
receipt
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_STORED_HEALTH_EVENT_REQUIRED"),
|
||||
};
|
||||
self.health = Some(self.health_evidence(db, receipt)?);
|
||||
Ok(event_raw)
|
||||
@@ -354,8 +388,11 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
pub fn started(&self) -> Result<(Vec<rmm_updater_handoff::StartedService>, String)> {
|
||||
self.check_effect_window()?;
|
||||
ensure!(
|
||||
self.participant == Participant::NewHelper,
|
||||
"UPDATE_NEW_FULL_REQUIRED"
|
||||
matches!(
|
||||
self.participant,
|
||||
Participant::NewHelper | Participant::RestoredHelper
|
||||
),
|
||||
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
|
||||
);
|
||||
let mut raw = Vec::new();
|
||||
let mut proofs = Vec::new();
|
||||
@@ -415,7 +452,13 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
Ok(())
|
||||
}
|
||||
fn verify_owner(&self, phase: Phase, events: &[Event]) -> Result<()> {
|
||||
if !matches!(phase, Phase::ServicesStarted | Phase::HealthVerified) {
|
||||
if !matches!(
|
||||
phase,
|
||||
Phase::ServicesStarted
|
||||
| Phase::HealthVerified
|
||||
| Phase::RollbackServicesStarted
|
||||
| Phase::RollbackHealthVerified
|
||||
) {
|
||||
let updater = self
|
||||
.updater
|
||||
.as_ref()
|
||||
@@ -431,15 +474,30 @@ impl<'a> FullUpdateGuard<'a> {
|
||||
"UPDATE_ORIGINAL_UPDATER_OWNER_CHANGED"
|
||||
);
|
||||
}
|
||||
if matches!(phase, Phase::ServicesStarted | Phase::HealthVerified) {
|
||||
if matches!(
|
||||
phase,
|
||||
Phase::ServicesStarted
|
||||
| Phase::HealthVerified
|
||||
| Phase::RollbackServicesStarted
|
||||
| Phase::RollbackHealthVerified
|
||||
) {
|
||||
let started = events
|
||||
.iter()
|
||||
.find_map(|event| {
|
||||
if let Evidence::ServicesStarted { services, .. } = &event.evidence {
|
||||
.find_map(|event| match &event.evidence {
|
||||
Evidence::ServicesStarted { services, .. }
|
||||
if matches!(phase, Phase::ServicesStarted | Phase::HealthVerified) =>
|
||||
{
|
||||
Some(services)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
Evidence::RollbackServicesStarted { services, .. }
|
||||
if matches!(
|
||||
phase,
|
||||
Phase::RollbackServicesStarted | Phase::RollbackHealthVerified
|
||||
) =>
|
||||
{
|
||||
Some(services)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.context("UPDATE_STARTED_PROOF_MISSING")?;
|
||||
self.verify_current_services(started)?;
|
||||
@@ -486,16 +544,39 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
|
||||
);
|
||||
self.check_effect_window()?;
|
||||
ensure!(
|
||||
self.participant == Participant::NewHelper,
|
||||
"UPDATE_NEW_FULL_REQUIRED"
|
||||
matches!(
|
||||
self.participant,
|
||||
Participant::NewHelper | Participant::RestoredHelper
|
||||
),
|
||||
"UPDATE_NEW_OR_RESTORED_FULL_REQUIRED"
|
||||
);
|
||||
self.witnesses.files.verify_pair(&self.budget)?;
|
||||
if self.participant == Participant::RestoredHelper {
|
||||
self.witnesses.files.verify_predecessor_pair(&self.budget)?;
|
||||
} else {
|
||||
self.witnesses.files.verify_pair(&self.budget)?;
|
||||
}
|
||||
match &event.evidence {
|
||||
Evidence::ServicesStarted {
|
||||
services,
|
||||
ledger_identity_sha256,
|
||||
observation_sha256,
|
||||
}
|
||||
| Evidence::RollbackServicesStarted {
|
||||
services,
|
||||
ledger_identity_sha256,
|
||||
observation_sha256,
|
||||
} => {
|
||||
ensure!(
|
||||
matches!(
|
||||
(&event.evidence, self.participant),
|
||||
(Evidence::ServicesStarted { .. }, Participant::NewHelper)
|
||||
| (
|
||||
Evidence::RollbackServicesStarted { .. },
|
||||
Participant::RestoredHelper
|
||||
)
|
||||
),
|
||||
"UPDATE_STARTED_ROLE"
|
||||
);
|
||||
let mut held = Vec::new();
|
||||
let mut proofs = Vec::new();
|
||||
for service in [Service::Agent, Service::General, Service::Full] {
|
||||
@@ -511,7 +592,18 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
|
||||
"UPDATE_ACTUAL_STARTED_PROOFS_REQUIRED"
|
||||
);
|
||||
}
|
||||
Evidence::HealthVerified { receipt } => {
|
||||
Evidence::HealthVerified { receipt } | Evidence::RollbackHealthVerified { receipt } => {
|
||||
ensure!(
|
||||
matches!(
|
||||
(&event.evidence, self.participant),
|
||||
(Evidence::HealthVerified { .. }, Participant::NewHelper)
|
||||
| (
|
||||
Evidence::RollbackHealthVerified { .. },
|
||||
Participant::RestoredHelper
|
||||
)
|
||||
),
|
||||
"UPDATE_HEALTH_ROLE"
|
||||
);
|
||||
let health = self
|
||||
.health
|
||||
.as_ref()
|
||||
@@ -521,6 +613,14 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
|
||||
"UPDATE_LOCAL_HEALTH_RECEIPT_CHANGED"
|
||||
);
|
||||
}
|
||||
Evidence::RollbackOutcomePersisted { .. }
|
||||
if self.participant == Participant::RestoredHelper =>
|
||||
{
|
||||
ensure!(
|
||||
self.health.is_some(),
|
||||
"UPDATE_ROLLBACK_LOCAL_HEALTH_REQUIRED"
|
||||
);
|
||||
}
|
||||
// InstalledRecord/Outcome have a separate original-record CAS.
|
||||
_ => anyhow::bail!("UPDATE_FULL_EVENT_NOT_YET_QUALIFIED"),
|
||||
}
|
||||
@@ -578,10 +678,63 @@ impl UpdateParticipantGuard for FullUpdateGuard<'_> {
|
||||
.observe(service, &self.budget)?
|
||||
.verify(&self.budget)?;
|
||||
}
|
||||
(
|
||||
Participant::NewHelper,
|
||||
NativeEffect::ConditionalRollbackStopGeneral,
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => {
|
||||
// The retained new Full proves the early-Full-failure branch
|
||||
// was never entered. Closing this reservation records NO stop.
|
||||
ensure!(
|
||||
handoff.schema_version == 2,
|
||||
"UPDATE_CONDITIONAL_STOP_PROFILE"
|
||||
);
|
||||
for service in [Service::General, Service::Full] {
|
||||
self.services
|
||||
.observe(service, &self.budget)?
|
||||
.verify(&self.budget)?;
|
||||
}
|
||||
}
|
||||
(
|
||||
Participant::RestoredHelper,
|
||||
NativeEffect::RollbackStartService { service },
|
||||
NativeEffectStage::Observe,
|
||||
) => {
|
||||
let service = match service {
|
||||
NativeService::Agent => Service::Agent,
|
||||
NativeService::General => Service::General,
|
||||
NativeService::Full => Service::Full,
|
||||
_ => anyhow::bail!("UPDATE_WINDOWS_SERVICE_ON_LINUX"),
|
||||
};
|
||||
self.witnesses.files.verify_predecessor_pair(&self.budget)?;
|
||||
self.services
|
||||
.observe(service, &self.budget)?
|
||||
.verify(&self.budget)?;
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_FULL_NATIVE_EFFECT_NOT_QUALIFIED"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn native_observation_sha256(
|
||||
&self,
|
||||
handoff: &Handoff,
|
||||
delegation: &UpdateDelegation,
|
||||
effect: &NativeEffect,
|
||||
stage: NativeEffectStage,
|
||||
) -> Result<String> {
|
||||
self.verify_native_effect(handoff, delegation, effect, stage)?;
|
||||
ensure!(
|
||||
*effect == NativeEffect::ConditionalRollbackStopGeneral
|
||||
&& stage == NativeEffectStage::ObserveNonstart,
|
||||
"UPDATE_FULL_NATIVE_HASH_NOT_QUALIFIED"
|
||||
);
|
||||
Ok(hash(&serde_json::to_vec(&(
|
||||
"conditional_general_stop_unused_v1",
|
||||
handoff.owner_nonce,
|
||||
handoff.attempt.grant_id,
|
||||
self.services.observe(Service::Full, &self.budget)?.proof(),
|
||||
))?))
|
||||
}
|
||||
}
|
||||
|
||||
struct VerifiedHealth {
|
||||
@@ -985,3 +1138,157 @@ impl FullUpdateGuard<'_> {
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl FullUpdateGuard<'_> {
|
||||
pub fn finalize_restored(&mut self, db: &mut Connection) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
self.participant == Participant::RestoredHelper,
|
||||
"UPDATE_RESTORED_FULL_REQUIRED"
|
||||
);
|
||||
self.qualify_effects(db)?;
|
||||
self.qualify_stored_health(db)?;
|
||||
self.witnesses.files.verify_predecessor_pair(&self.budget)?;
|
||||
// TrustState still pins the SAME original record and activation floor.
|
||||
self.trust.recheck(&self.original, self.clock)?;
|
||||
let journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
|
||||
let health = self
|
||||
.health
|
||||
.as_ref()
|
||||
.context("UPDATE_ROLLBACK_HEALTH_REQUIRED")?;
|
||||
let outcome = crate::Outcome {
|
||||
state: "failed".into(),
|
||||
stdout: vec![],
|
||||
stderr: vec![],
|
||||
exit_code: None,
|
||||
output_truncated: false,
|
||||
reboot_required: false,
|
||||
error_code: "AGENT_UPDATE_RETURNED_TO_PREDECESSOR".into(),
|
||||
evidence: serde_json::json!({"postconditions_passed":false,"previous_boot_id":self.original.handoff().boot_id,
|
||||
"agent_update":{"schema_version":2,"handoff_sha256":journal.handoff_sha256(),
|
||||
"manifest_sha256":self.original.handoff().target.manifest_sha256,
|
||||
"predecessor_record_sha256":self.original.handoff().predecessor_record_sha256,
|
||||
"installed_record_sha256":self.original.handoff().predecessor_record_sha256,
|
||||
"signed_health_sha256":health.signed_sha256,"rolled_back":true}}),
|
||||
};
|
||||
update_handoff::finalize_rollback(
|
||||
db,
|
||||
self,
|
||||
self.original.permit(),
|
||||
journal.head_sha256(),
|
||||
u64::try_from(self.clock.now()?.timestamp())?,
|
||||
&outcome,
|
||||
)?;
|
||||
Ok(db.query_row(
|
||||
"SELECT outcome FROM execution_evidence WHERE grant_id=?1",
|
||||
[self.original.permit().grant_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?)
|
||||
}
|
||||
pub fn restored(&self) -> bool {
|
||||
self.participant == Participant::RestoredHelper
|
||||
}
|
||||
/// Exact ORIGINAL bytes from a signed inflight decision already applied
|
||||
/// by this actual Full process under the same existing guarded ledger.
|
||||
/// Missing evidence keeps the worker waiting; it never becomes readiness.
|
||||
pub fn pre_stop_signed_sync(&self, db: &Connection) -> Result<Option<(Vec<u8>, Vec<u8>)>> {
|
||||
use prost::Message;
|
||||
use rusqlite::OptionalExtension;
|
||||
self.check(db)?;
|
||||
if self.participant != Participant::OriginalHelper {
|
||||
return Ok(None);
|
||||
}
|
||||
let journal = update_handoff::read(db, self, self.original.permit().grant_id)?;
|
||||
if journal.phase() != Phase::StopIntent {
|
||||
return Ok(None);
|
||||
}
|
||||
let id: Option<String> = db
|
||||
.query_row(
|
||||
"SELECT last_decision FROM helper_sync_meta WHERE id=1 AND fault IS NULL",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.flatten();
|
||||
let Some(id) = id else { return Ok(None) };
|
||||
let id: Uuid = id.parse()?;
|
||||
let applied:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM agent_update_sync_applied WHERE grant_id=?1 AND decision_id=?2)",
|
||||
rusqlite::params![self.original.permit().grant_id.to_string(),id.to_string()],|r|r.get(0))?;
|
||||
if !applied {
|
||||
return Ok(None);
|
||||
}
|
||||
self.qualify_effects(db)?;
|
||||
let round = rmm_agent_core::update_health_sync::read_round(
|
||||
db,
|
||||
self,
|
||||
self.binding,
|
||||
self.original.permit().grant_id,
|
||||
id,
|
||||
)?;
|
||||
let decision = round.decision();
|
||||
let h = self.original.handoff();
|
||||
let now = self.clock.now()?;
|
||||
let proof = update_handoff::current_proof(db, self, self.original.permit().grant_id)?
|
||||
.context("UPDATE_PRESTOP_ORIGINAL_PROOF_REQUIRED")?;
|
||||
let expected =
|
||||
rmm_domain::agent_update::inflight_decision(&proof, self.trust.health_public_key())
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let attempts = decision
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| a.job_id == h.attempt.job_id && a.attempt_id == h.attempt.attempt_id)
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(attempts.len() == 1, "UPDATE_PRESTOP_ONE_ORIGINAL_ACK");
|
||||
let ack = attempts[0];
|
||||
ensure!(
|
||||
decision.valid_profile(now)
|
||||
&& !decision.ready
|
||||
&& !decision.helper_ready
|
||||
&& decision.boot_id == h.boot_id
|
||||
&& decision.connection_generation == h.connection_generation
|
||||
&& decision.journal_id == h.agent_journal_id
|
||||
&& decision.helper_journal_id == Some(h.helper_journal_id)
|
||||
&& ack.update_inflight.as_ref() == Some(&expected)
|
||||
&& ack.grant_id == Some(h.attempt.grant_id)
|
||||
&& ack.authorization_id == Some(h.attempt.authorization_id)
|
||||
&& ack.plan_hash.as_deref() == Some(h.attempt.plan_sha256.as_str())
|
||||
&& ack.job_payload_hash == h.attempt.job_payload_sha256
|
||||
&& ack.original_boot_id == Some(h.boot_id)
|
||||
&& ack.action == "update_agent"
|
||||
&& ack.execution_identity == "root"
|
||||
&& !ack.unlock
|
||||
&& ack.outcome_state == "running"
|
||||
&& ack.outcome_bytes.is_empty()
|
||||
&& ack.outcome_sha256.is_none()
|
||||
&& round.applied().applied_at >= decision.issued_at
|
||||
&& round.applied().applied_at < decision.valid_until,
|
||||
"UPDATE_PRESTOP_APPLIED_ORIGINAL_ACK"
|
||||
);
|
||||
let (raw, response): (Vec<u8>, Vec<u8>) = db.query_row(
|
||||
"SELECT request,response FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
ensure!(
|
||||
raw.len() <= 256 * 1024 && response.len() <= 256 * 1024,
|
||||
"UPDATE_PRESTOP_WIRE_LIMIT"
|
||||
);
|
||||
let request = rmm_protocol::SyncRequest::decode(raw.as_slice())?;
|
||||
let local = request
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|a| {
|
||||
a.job_id == h.attempt.job_id.to_string()
|
||||
&& a.attempt_id == h.attempt.attempt_id.to_string()
|
||||
})
|
||||
.context("UPDATE_PRESTOP_REQUEST_ATTEMPT")?;
|
||||
ensure!(
|
||||
request.journal_complete
|
||||
&& local.state == "running"
|
||||
&& local.consumption == "consumed"
|
||||
&& local.update_handoff_bytes == proof.bytes().map_err(anyhow::Error::msg)?,
|
||||
"UPDATE_PRESTOP_REQUEST_ORIGINAL_PREFIX"
|
||||
);
|
||||
self.check_effect_window()?;
|
||||
Ok(Some((raw, response)))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user