Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -212,6 +212,12 @@ impl<'a, 'stopped> OwnedLedger<'a, 'stopped> {
|
||||
NativeEffect::PublishReplacement { role } => {
|
||||
self.native_observation_sha256(file_role(*role), true)?
|
||||
}
|
||||
NativeEffect::SaveRejectedCandidate { role } => {
|
||||
self.native_observation_sha256(file_role(*role), true)?
|
||||
}
|
||||
NativeEffect::RestorePredecessor { role } => {
|
||||
self.native_observation_sha256(file_role(*role), false)?
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_OWNED_OBSERVATION_NOT_ALLOWED"),
|
||||
};
|
||||
ensure!(expected == observation, "UPDATE_NATIVE_OBSERVATION_CHANGED");
|
||||
@@ -243,6 +249,19 @@ impl<'a, 'stopped> OwnedLedger<'a, 'stopped> {
|
||||
pub fn verify_pair(&self) -> Result<()> {
|
||||
self.guard.files.verify_pair(self.guard.budget)
|
||||
}
|
||||
pub fn verify_predecessor_pair(&self) -> Result<()> {
|
||||
self.guard.files.verify_predecessor_pair(self.guard.budget)
|
||||
}
|
||||
pub fn predecessor_sha256(&self, role: rmm_updater_handoff::FileRole) -> Result<String> {
|
||||
self.guard
|
||||
.original
|
||||
.handoff()
|
||||
.files
|
||||
.iter()
|
||||
.find(|f| f.role == role)
|
||||
.map(|f| f.old_sha256.clone())
|
||||
.ok_or_else(|| anyhow::anyhow!("UPDATE_OLD_FILE_NOT_BOUND"))
|
||||
}
|
||||
pub fn inflight_proof(&self) -> Result<rmm_domain::agent_update::UpdateInflightProof> {
|
||||
let journal = self.journal()?;
|
||||
let mut statement = self.db.prepare(
|
||||
@@ -291,6 +310,71 @@ impl<'a, 'stopped> OwnedLedger<'a, 'stopped> {
|
||||
old_services,
|
||||
})
|
||||
}
|
||||
pub fn prepare_rollback_return_and_close(mut self) -> Result<PreparedReservation> {
|
||||
let journal = self.journal()?;
|
||||
ensure!(
|
||||
journal.phase() == Phase::PredecessorVerified,
|
||||
"UPDATE_ROLLBACK_RETURN_REQUIRES_PREDECESSOR"
|
||||
);
|
||||
let now = self.guard.clock.now()?;
|
||||
let event = journal.prepare(
|
||||
u64::try_from(now.timestamp())?,
|
||||
Evidence::RollbackLedgerReturnIntent,
|
||||
)?;
|
||||
let claims = update_handoff::prepare_rollback_return_and_starts(
|
||||
&mut self.db,
|
||||
&self.guard,
|
||||
self.guard.original.permit().grant_id,
|
||||
journal.head_sha256(),
|
||||
&event,
|
||||
now,
|
||||
)?;
|
||||
let proof = self.inflight_proof()?;
|
||||
let ledger = self.guard.os.identity().clone();
|
||||
let old_services = self.guard.stopped.proofs();
|
||||
self.close()?;
|
||||
Ok(PreparedReservation {
|
||||
claims,
|
||||
proof,
|
||||
ledger,
|
||||
old_services,
|
||||
})
|
||||
}
|
||||
pub fn begin_rollback_after_early_full_failure(&mut self, expected_head: &str) -> Result<()> {
|
||||
let source = self.guard.stopped.early_full_failure(self.guard.budget)?;
|
||||
let start = hash(&serde_json::to_vec(source.general_start())?);
|
||||
let failure = source.failure_sha256()?;
|
||||
update_handoff::begin_rollback_after_reserved_stop(
|
||||
&mut self.db,
|
||||
&self.guard,
|
||||
update_handoff::ReservedGeneralStop {
|
||||
grant_id: self.guard.original.permit().grant_id,
|
||||
expected_head,
|
||||
general_start_observation_sha256: &start,
|
||||
general_stop_observation_sha256: source.general_stop_sha256(),
|
||||
failure_observation_sha256: &failure,
|
||||
now: self.guard.clock.now()?,
|
||||
},
|
||||
)
|
||||
}
|
||||
pub fn append_rollback_stopped(&mut self) -> Result<()> {
|
||||
let source = self.guard.stopped.early_full_failure(self.guard.budget)?;
|
||||
let observation = hash(&serde_json::to_vec(&(
|
||||
source.general_start(),
|
||||
source.general_stop_sha256(),
|
||||
source.failure(),
|
||||
))?);
|
||||
let owner = self.guard.updater.identity();
|
||||
self.append_observed(Evidence::RollbackStopped {
|
||||
process: rmm_updater_handoff::Process {
|
||||
pid: owner.pid,
|
||||
birth: owner.birth,
|
||||
image_sha256: owner.image_sha256.clone(),
|
||||
},
|
||||
ledger_identity_sha256: self.guard.os.identity().sha256()?,
|
||||
observation_sha256: observation,
|
||||
})
|
||||
}
|
||||
/// Close the actual SQLite connection BEFORE returning the OS flock. An
|
||||
/// open/failed-close connection never produces a released-ledger witness.
|
||||
pub fn close(self) -> Result<()> {
|
||||
@@ -443,6 +527,81 @@ impl UpdateParticipantGuard for OwnedGuard<'_, '_> {
|
||||
);
|
||||
}
|
||||
Evidence::LedgerReturnIntent => self.files.verify_pair(self.budget)?,
|
||||
Evidence::RollbackIntent {
|
||||
cause:
|
||||
rmm_updater_handoff::RollbackCause::StartupFailed {
|
||||
service: rmm_updater_handoff::Service::LinuxFullHelper,
|
||||
},
|
||||
failure_observation_sha256,
|
||||
} => {
|
||||
let source = self.stopped.early_full_failure(self.budget)?;
|
||||
ensure!(
|
||||
*failure_observation_sha256 == source.failure_sha256()?,
|
||||
"UPDATE_ROLLBACK_FAILURE_CHANGED"
|
||||
);
|
||||
self.files.verify_pair(self.budget)?;
|
||||
}
|
||||
Evidence::RollbackStopped {
|
||||
process,
|
||||
ledger_identity_sha256,
|
||||
observation_sha256,
|
||||
} => {
|
||||
let source = self.stopped.early_full_failure(self.budget)?;
|
||||
let owner = self.updater.identity();
|
||||
ensure!(
|
||||
process.pid == owner.pid
|
||||
&& process.birth == owner.birth
|
||||
&& process.image_sha256 == owner.image_sha256
|
||||
&& *ledger_identity_sha256 == self.os.identity().sha256()?
|
||||
&& *observation_sha256
|
||||
== hash(&serde_json::to_vec(&(
|
||||
source.general_start(),
|
||||
source.general_stop_sha256(),
|
||||
source.failure()
|
||||
))?),
|
||||
"UPDATE_ROLLBACK_STOPPED_WITNESS"
|
||||
);
|
||||
}
|
||||
Evidence::RestoreIntent { role } => self.files.verify_replaced(*role, self.budget)?,
|
||||
Evidence::CandidatePreserved {
|
||||
role,
|
||||
observation_sha256,
|
||||
} => {
|
||||
self.files.verify_candidate_saved(*role, self.budget)?;
|
||||
ensure!(
|
||||
*observation_sha256 == self.files.native_observation_sha256(*role, true)?,
|
||||
"UPDATE_REJECTED_CANDIDATE_HASH"
|
||||
);
|
||||
}
|
||||
Evidence::PredecessorRestored {
|
||||
role,
|
||||
observed_sha256,
|
||||
} => {
|
||||
self.files.verify_restored(*role, self.budget)?;
|
||||
ensure!(
|
||||
handoff
|
||||
.files
|
||||
.iter()
|
||||
.any(|f| f.role == *role && f.old_sha256 == *observed_sha256),
|
||||
"UPDATE_RESTORED_PREDECESSOR_HASH"
|
||||
);
|
||||
}
|
||||
Evidence::PredecessorVerified {
|
||||
pair,
|
||||
retained_state_sha256,
|
||||
predecessor_record_sha256,
|
||||
} => {
|
||||
self.files.verify_predecessor_pair(self.budget)?;
|
||||
ensure!(
|
||||
*pair == handoff.predecessor
|
||||
&& *retained_state_sha256 == handoff.retained_state_sha256
|
||||
&& *predecessor_record_sha256 == handoff.predecessor_record_sha256,
|
||||
"UPDATE_PREDECESSOR_BINDING"
|
||||
);
|
||||
}
|
||||
Evidence::RollbackLedgerReturnIntent => {
|
||||
self.files.verify_predecessor_pair(self.budget)?
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_EVENT_WRITER_NOT_ALLOWED"),
|
||||
}
|
||||
Ok(())
|
||||
@@ -492,8 +651,123 @@ impl UpdateParticipantGuard for OwnedGuard<'_, '_> {
|
||||
| (NativeEffect::StartService { .. }, NativeEffectStage::ReserveFutureStart) => {
|
||||
self.files.verify_pair(self.budget)?
|
||||
}
|
||||
(
|
||||
NativeEffect::ConditionalRollbackStopGeneral,
|
||||
NativeEffectStage::ReserveConditionalStop,
|
||||
) => {
|
||||
ensure!(
|
||||
handoff.schema_version == 2
|
||||
&& delegation.original_helper.role
|
||||
== rmm_domain::agent_update::ParticipantRole::LinuxFullHelper,
|
||||
"UPDATE_CONDITIONAL_STOP_PROFILE"
|
||||
);
|
||||
self.stopped.verify_still_stopped(self.budget)?;
|
||||
self.files.verify_pair(self.budget)?;
|
||||
}
|
||||
(
|
||||
NativeEffect::StartService {
|
||||
service: update_handoff::NativeService::General,
|
||||
},
|
||||
NativeEffectStage::ObserveRetainedStart,
|
||||
)
|
||||
| (NativeEffect::ConditionalRollbackStopGeneral, NativeEffectStage::Observe)
|
||||
| (
|
||||
NativeEffect::StartService {
|
||||
service:
|
||||
update_handoff::NativeService::Full | update_handoff::NativeService::Agent,
|
||||
},
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => {
|
||||
self.stopped.early_full_failure(self.budget)?;
|
||||
self.files.verify_pair(self.budget)?;
|
||||
}
|
||||
(
|
||||
NativeEffect::RollbackStopService {
|
||||
service:
|
||||
update_handoff::NativeService::Full | update_handoff::NativeService::Agent,
|
||||
},
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => {
|
||||
self.stopped.early_full_failure(self.budget)?;
|
||||
self.files.verify_pair(self.budget)?;
|
||||
}
|
||||
(NativeEffect::SaveRejectedCandidate { role: r }, NativeEffectStage::Claim) => {
|
||||
self.files.verify_replaced(role(r), self.budget)?
|
||||
}
|
||||
(NativeEffect::SaveRejectedCandidate { role: r }, NativeEffectStage::Observe)
|
||||
| (NativeEffect::RestorePredecessor { role: r }, NativeEffectStage::Claim) => {
|
||||
self.files.verify_candidate_saved(role(r), self.budget)?
|
||||
}
|
||||
(NativeEffect::RestorePredecessor { role: r }, NativeEffectStage::Observe) => {
|
||||
self.files.verify_restored(role(r), self.budget)?
|
||||
}
|
||||
(NativeEffect::RollbackReturnLedger, NativeEffectStage::Claim)
|
||||
| (NativeEffect::RollbackStartService { .. }, NativeEffectStage::ReserveFutureStart) => {
|
||||
self.files.verify_predecessor_pair(self.budget)?
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_NATIVE_OWNED_EFFECT_DENIED"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn native_observation_sha256(
|
||||
&self,
|
||||
h: &Handoff,
|
||||
d: &UpdateDelegation,
|
||||
effect: &NativeEffect,
|
||||
stage: NativeEffectStage,
|
||||
) -> Result<String> {
|
||||
self.verify_native_effect(h, d, effect, stage)?;
|
||||
let source = self.stopped.early_full_failure(self.budget)?;
|
||||
match (effect, stage) {
|
||||
(
|
||||
NativeEffect::StartService {
|
||||
service: update_handoff::NativeService::General,
|
||||
},
|
||||
NativeEffectStage::ObserveRetainedStart,
|
||||
) => Ok(hash(&serde_json::to_vec(source.general_start())?)),
|
||||
(NativeEffect::ConditionalRollbackStopGeneral, NativeEffectStage::Observe) => {
|
||||
Ok(source.general_stop_sha256().into())
|
||||
}
|
||||
(
|
||||
NativeEffect::StartService {
|
||||
service: update_handoff::NativeService::Full,
|
||||
},
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => source.failure_sha256(),
|
||||
(
|
||||
NativeEffect::RollbackStopService {
|
||||
service: update_handoff::NativeService::Full,
|
||||
},
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => Ok(hash(&serde_json::to_vec(&(
|
||||
"failed_full_stop_not_needed_v1",
|
||||
h.owner_nonce,
|
||||
h.attempt.grant_id,
|
||||
source.failure_sha256()?,
|
||||
))?)),
|
||||
(
|
||||
NativeEffect::RollbackStopService {
|
||||
service: update_handoff::NativeService::Agent,
|
||||
},
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => Ok(hash(&serde_json::to_vec(&(
|
||||
"unstarted_agent_stop_not_needed_v1",
|
||||
h.owner_nonce,
|
||||
h.attempt.grant_id,
|
||||
source.failure_sha256()?,
|
||||
))?)),
|
||||
(
|
||||
NativeEffect::StartService {
|
||||
service: update_handoff::NativeService::Agent,
|
||||
},
|
||||
NativeEffectStage::ObserveNonstart,
|
||||
) => Ok(hash(&serde_json::to_vec(&(
|
||||
"agent_start_not_dispatched_v1",
|
||||
h.owner_nonce,
|
||||
h.attempt.grant_id,
|
||||
source.failure_sha256()?,
|
||||
))?)),
|
||||
_ => anyhow::bail!("UPDATE_OWNED_NATIVE_HASH_NOT_QUALIFIED"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user