Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s

This commit is contained in:
2026-11-16 15:25:46 +01:00
parent 6497c79a91
commit 96a2b6e5ae
44 changed files with 6986 additions and 1024 deletions
+7
View File
@@ -38,6 +38,13 @@ pub trait SyncClock {
upper: own_wall,
})
}
/// Full trusted UTC interval for crypto validity. Existing point-clock
/// providers retain their contract; interval providers override atomically.
fn bounds(&self) -> Result<(DateTime<Utc>,DateTime<Utc>)> {
let reading=self.reading()?;
ensure!(reading.own_wall<=reading.upper,"CLOCK_INTERVAL_ORDER");
Ok((reading.own_wall,reading.upper))
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Channel {
+726 -19
View File
@@ -38,6 +38,7 @@ pub enum Participant {
OriginalHelper,
StableUpdater,
NewHelper,
RestoredHelper,
}
pub trait UpdateParticipantGuard: ExecutorLedgerGuard {
fn participant(&self) -> Participant;
@@ -50,6 +51,18 @@ pub trait UpdateParticipantGuard: ExecutorLedgerGuard {
) -> Result<()> {
anyhow::bail!("UPDATE_NATIVE_EFFECT_NOT_QUALIFIED")
}
/// A retained native START/conditional STOP witness must be independently
/// bound to the original once-only source/held handles. No caller hash or
/// current service-name/PID lookup is an attestation of a past operation.
fn native_observation_sha256(
&self,
_handoff: &Handoff,
_delegation: &UpdateDelegation,
_effect: &NativeEffect,
_stage: NativeEffectStage,
) -> Result<String> {
anyhow::bail!("UPDATE_RETAINED_NATIVE_OBSERVATION_NOT_QUALIFIED")
}
/// Recheck fixed service/PID/birth/image, original boot, SAME file IDs,
/// current clock/deadline and evidence-specific protected observations.
/// Health additionally compares receipt contents with the actually applied
@@ -91,7 +104,11 @@ pub fn initialize(db: &Connection, guard: &impl ExecutorLedgerGuard) -> Result<(
FOREIGN KEY(grant_id,effect_key) REFERENCES agent_update_native_claims(grant_id,effect_key));
CREATE TABLE IF NOT EXISTS agent_update_sync_applied(
grant_id TEXT NOT NULL REFERENCES agent_update_handoffs(grant_id),decision_id TEXT NOT NULL,
witness BLOB NOT NULL,witness_sha256 TEXT NOT NULL,PRIMARY KEY(grant_id,decision_id));",
witness BLOB NOT NULL,witness_sha256 TEXT NOT NULL,PRIMARY KEY(grant_id,decision_id));
CREATE TABLE IF NOT EXISTS agent_update_native_nonstarts(
grant_id TEXT NOT NULL,effect_key TEXT NOT NULL,claim_sha256 TEXT NOT NULL,
observation_sha256 TEXT NOT NULL,observed_at TEXT NOT NULL,PRIMARY KEY(grant_id,effect_key),
FOREIGN KEY(grant_id,effect_key) REFERENCES agent_update_native_claims(grant_id,effect_key));",
)?;
Ok(())
}
@@ -300,7 +317,7 @@ pub(crate) fn read_locked(db: &Connection, grant: Uuid) -> Result<Journal> {
let (effect_key, raw, digest) = row?;
let claim: NativeEffectClaim = serde_json::from_slice(&raw)?;
ensure!(
native_count <= 17
native_count <= if handoff.schema_version == 2 { 32 } else { 17 }
&& raw.len() <= 2048
&& hash(&raw) == digest
&& serde_json::to_vec(&claim)? == raw
@@ -313,6 +330,10 @@ pub(crate) fn read_locked(db: &Connection, grant: Uuid) -> Result<Journal> {
}
let invalid:u64=db.query_row("SELECT count(*) FROM agent_update_native_observations o LEFT JOIN agent_update_native_claims c ON c.grant_id=o.grant_id AND c.effect_key=o.effect_key WHERE o.grant_id=?1 AND (c.claim_sha256 IS NULL OR c.claim_sha256<>o.claim_sha256)",[grant.to_string()],|r|r.get(0))?;
ensure!(invalid == 0, "UPDATE_NATIVE_OBSERVATION_BINDING");
if handoff.schema_version == 2 {
let invalid:u64=db.query_row("SELECT count(*) FROM agent_update_native_nonstarts n LEFT JOIN agent_update_native_claims c ON c.grant_id=n.grant_id AND c.effect_key=n.effect_key LEFT JOIN agent_update_native_observations o ON o.grant_id=n.grant_id AND o.effect_key=n.effect_key WHERE n.grant_id=?1 AND (c.claim_sha256 IS NULL OR c.claim_sha256<>n.claim_sha256 OR o.claim_sha256 IS NOT NULL OR length(n.observation_sha256)<>64)",[grant.to_string()],|r|r.get(0))?;
ensure!(invalid == 0, "UPDATE_NATIVE_NONSTART_BINDING");
}
let mut journal = Journal::open(
&record,
key.try_into()
@@ -337,7 +358,9 @@ pub(crate) fn read_locked(db: &Connection, grant: Uuid) -> Result<Journal> {
);
journal.replay(&event)?;
let parsed: Event = serde_json::from_slice(&event)?;
if let Evidence::OutcomePersisted { outcome_sha256 } = parsed.evidence {
if let Evidence::OutcomePersisted { outcome_sha256 }
| Evidence::RollbackOutcomePersisted { outcome_sha256 } = parsed.evidence
{
stored_outcome_hash = Some(outcome_sha256);
}
}
@@ -440,7 +463,11 @@ pub fn append(
ensure!(
!matches!(
parsed.evidence,
Evidence::OutcomePersisted { .. } | Evidence::Unknown { .. }
Evidence::OutcomePersisted { .. }
| Evidence::RollbackOutcomePersisted { .. }
| Evidence::RollbackIntent { .. }
| Evidence::RollbackLedgerReturnIntent
| Evidence::Unknown { .. }
),
"UPDATE_TERMINAL_REQUIRES_OUTCOME_CAS"
);
@@ -481,6 +508,16 @@ fn verify_writer(
|r| r.get(0),
)?;
let handoff: Handoff = serde_json::from_slice(&raw)?;
let native_services =
if handoff.pin_transition == rmm_updater_handoff::PinTransition::WindowsCodePairClockV1 {
vec![NativeService::WindowsAgent, NativeService::WindowsHelper]
} else {
vec![
NativeService::Agent,
NativeService::General,
NativeService::Full,
]
};
let expected = match event.evidence {
Evidence::AllStopped { .. }
| Evidence::UpdaterOwnsLedger { .. }
@@ -491,6 +528,25 @@ fn verify_writer(
Evidence::ServicesStarted { .. }
| Evidence::HealthVerified { .. }
| Evidence::OutcomePersisted { .. } => Participant::NewHelper,
Evidence::RollbackStopped { .. }
| Evidence::RestoreIntent { .. }
| Evidence::CandidatePreserved { .. }
| Evidence::PredecessorRestored { .. }
| Evidence::PredecessorVerified { .. }
| Evidence::RollbackLedgerReturnIntent => Participant::StableUpdater,
Evidence::RollbackServicesStarted { .. }
| Evidence::RollbackHealthVerified { .. }
| Evidence::RollbackOutcomePersisted { .. } => Participant::RestoredHelper,
Evidence::RollbackIntent { .. } => {
ensure!(
matches!(
guard.participant(),
Participant::StableUpdater | Participant::NewHelper
),
"UPDATE_ROLLBACK_INTENT_WRITER"
);
guard.participant()
}
Evidence::Unknown { .. } => guard.participant(),
Evidence::StopIntent => anyhow::bail!("UPDATE_STOP_ALREADY_ATOMICALLY_STORED"),
};
@@ -502,11 +558,7 @@ fn verify_writer(
};
match &event.evidence {
Evidence::AllStopped { .. } => {
for service in [
NativeService::Agent,
NativeService::General,
NativeService::Full,
] {
for service in native_services {
if handoff.stop_set.contains(&native_service(service)) {
require_observed(NativeEffect::StopService { service })?;
}
@@ -518,19 +570,64 @@ fn verify_writer(
rmm_updater_handoff::FileRole::HelperBinary => NativeFileRole::HelperBinary,
rmm_updater_handoff::FileRole::LinuxFullConfig => NativeFileRole::FullConfig,
rmm_updater_handoff::FileRole::LinuxFullPin => NativeFileRole::FullPin,
rmm_updater_handoff::FileRole::WindowsSettings
if handoff.pin_transition
== rmm_updater_handoff::PinTransition::WindowsCodePairClockV1 =>
{
NativeFileRole::WindowsSettings
}
rmm_updater_handoff::FileRole::WindowsPin
if handoff.pin_transition
== rmm_updater_handoff::PinTransition::WindowsCodePairClockV1 =>
{
NativeFileRole::WindowsPin
}
rmm_updater_handoff::FileRole::WindowsTimeProfile
if handoff.pin_transition
== rmm_updater_handoff::PinTransition::WindowsCodePairClockV1 =>
{
NativeFileRole::WindowsTimeProfile
}
_ => anyhow::bail!("UPDATE_NATIVE_PLATFORM_UNQUALIFIED"),
};
require_observed(NativeEffect::PublishReplacement { role })?;
}
Evidence::ServicesStarted { .. } => {
for service in [
NativeService::General,
NativeService::Full,
NativeService::Agent,
] {
for service in native_services {
require_observed(NativeEffect::StartService { service })?;
}
}
Evidence::RollbackStopped { .. } => {
for service in native_services {
let key = native_key(&NativeEffect::RollbackStopService { service })?;
let observed:u64=db.query_row("SELECT count(*) FROM agent_update_native_observations WHERE grant_id=?1 AND effect_key=?2",params![grant.to_string(),key],|r|r.get(0))?;
let nonstart:u64=db.query_row("SELECT count(*) FROM agent_update_native_nonstarts WHERE grant_id=?1 AND effect_key=?2",params![grant.to_string(),key],|r|r.get(0))?;
let conditional = if service == NativeService::General {
db.query_row("SELECT count(*) FROM agent_update_native_observations WHERE grant_id=?1 AND effect_key=?2",params![grant.to_string(),native_key(&NativeEffect::ConditionalRollbackStopGeneral)?],|r|r.get::<_,u64>(0))?
} else {
0
};
ensure!(
observed + nonstart + conditional == 1,
"UPDATE_ROLLBACK_STOP_NOT_PROVED"
);
}
}
Evidence::CandidatePreserved { role, .. } => {
require_observed(NativeEffect::SaveRejectedCandidate {
role: to_native_file(*role),
})?
}
Evidence::PredecessorRestored { role, .. } => {
require_observed(NativeEffect::RestorePredecessor {
role: to_native_file(*role),
})?
}
Evidence::RollbackServicesStarted { .. } => {
for service in native_services {
require_observed(NativeEffect::RollbackStartService { service })?;
}
}
_ => {}
}
guard.verify_update_event(&handoff, &read_delegation(db, grant)?, event)
@@ -597,6 +694,54 @@ pub fn finalize(
tx.commit()?;
Ok(())
}
/// A recovered predecessor is a FAILED candidate update, never success for the
/// requested target. Existing consumption/outcome/PKI/floor are not reset.
pub fn finalize_rollback(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
permit: &ExecutionPermitPayload,
expected_head: &str,
observed_at: u64,
outcome: &Outcome,
) -> Result<()> {
ensure!(
guard.participant() == Participant::RestoredHelper
&& outcome.state == "failed"
&& outcome.error_code == "AGENT_UPDATE_RETURNED_TO_PREDECESSOR"
&& !outcome.reboot_required,
"UPDATE_ROLLBACK_FINAL_OUTCOME"
);
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let mut journal = read(&tx, guard, permit.grant_id)?;
ensure!(
journal.phase() == Phase::RollbackHealthVerified && journal.head_sha256() == expected_head,
"UPDATE_ROLLBACK_FINAL_HEAD"
);
let records:u64=tx.query_row("SELECT count(*) FROM agent_update_native_claims WHERE grant_id=?1 AND effect_key IN ('save_installed_record','publish_installed_record')",
[permit.grant_id.to_string()],|r|r.get(0))?;
ensure!(records == 0, "UPDATE_ROLLBACK_RECORD_ALREADY_CHANGED");
let bytes = ledger::persist_outcome_in(&tx, permit, outcome)?;
let event = journal.prepare(
observed_at,
Evidence::RollbackOutcomePersisted {
outcome_sha256: hash(&bytes),
},
)?;
let parsed: Event = serde_json::from_slice(&event)?;
verify_writer(&tx, guard, permit.grant_id, &parsed)?;
journal.replay(&event)?;
append_in(
&tx,
permit.grant_id,
expected_head,
&event,
&parsed,
&journal,
)?;
guard.check(&tx)?;
tx.commit()?;
Ok(())
}
/// Persist uncertainty once under the same original consumption. No release,
/// new start, result rewrite or transition back out of Unknown is provided.
@@ -666,6 +811,15 @@ fn phase_name(phase: Phase) -> &'static str {
Phase::LedgerReturnIntent => "ledger_return_intent",
Phase::ServicesStarted => "services_started",
Phase::HealthVerified => "health_verified",
Phase::RollbackIntent => "rollback_intent",
Phase::RollbackStopped => "rollback_stopped",
Phase::RestoreIntent => "restore_intent",
Phase::CandidatePreserved => "candidate_preserved",
Phase::PredecessorRestored => "predecessor_restored",
Phase::PredecessorVerified => "predecessor_verified",
Phase::RollbackLedgerReturnIntent => "rollback_ledger_return_intent",
Phase::RollbackServicesStarted => "rollback_services_started",
Phase::RollbackHealthVerified => "rollback_health_verified",
Phase::OutcomePersisted => "outcome_persisted",
Phase::Unknown => "unknown",
}
@@ -699,6 +853,17 @@ pub enum NativeEffect {
old_sha256: String,
new_sha256: String,
},
RollbackStopService {
service: NativeService,
},
SaveRejectedCandidate {
role: NativeFileRole,
},
RollbackStartService {
service: NativeService,
},
RollbackReturnLedger,
ConditionalRollbackStopGeneral,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -725,6 +890,9 @@ pub enum NativeEffectStage {
Claim,
Observe,
ReserveFutureStart,
ObserveNonstart,
ReserveConditionalStop,
ObserveRetainedStart,
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
@@ -755,6 +923,12 @@ impl NativeEffectClaim {
pub struct PreparedReturnClaims<const N: usize = 3> {
pub return_ledger: NativeEffectClaim,
pub starts: [NativeEffectClaim; N],
conditional_rollback_stop_general: Option<NativeEffectClaim>,
}
impl<const N: usize> PreparedReturnClaims<N> {
pub fn conditional_rollback_stop_general(&self) -> Option<&NativeEffectClaim> {
self.conditional_rollback_stop_general.as_ref()
}
}
fn native_context(
db: &Connection,
@@ -791,6 +965,18 @@ fn native_role(role: NativeFileRole) -> rmm_updater_handoff::FileRole {
NativeFileRole::WindowsTimeProfile => F::WindowsTimeProfile,
}
}
fn to_native_file(role: rmm_updater_handoff::FileRole) -> NativeFileRole {
use rmm_updater_handoff::FileRole as F;
match role {
F::AgentBinary => NativeFileRole::AgentBinary,
F::HelperBinary => NativeFileRole::HelperBinary,
F::LinuxFullConfig => NativeFileRole::FullConfig,
F::LinuxFullPin => NativeFileRole::FullPin,
F::WindowsSettings => NativeFileRole::WindowsSettings,
F::WindowsPin => NativeFileRole::WindowsPin,
F::WindowsTimeProfile => NativeFileRole::WindowsTimeProfile,
}
}
fn native_service(service: NativeService) -> rmm_updater_handoff::Service {
use rmm_updater_handoff::Service as S;
match service {
@@ -820,7 +1006,10 @@ fn insert_native_claim(
[h.attempt.grant_id.to_string()],
|r| r.get(0),
)?;
ensure!(count < 17, "UPDATE_NATIVE_EFFECT_LIMIT");
ensure!(
count < if h.schema_version == 2 { 32 } else { 17 },
"UPDATE_NATIVE_EFFECT_LIMIT"
);
let claim = NativeEffectClaim {
grant_id: h.attempt.grant_id,
owner_nonce: h.owner_nonce,
@@ -890,6 +1079,12 @@ pub fn claim_native_effect(
&& old_sha256 != new_sha256,
"UPDATE_NATIVE_INSTALLED_RECORD_BINDING"
);
if h.schema_version == 2
&& h.pin_transition == rmm_updater_handoff::PinTransition::LinuxGeneralFullV1
{
let denied:u64=tx.query_row("SELECT count(*) FROM agent_update_native_nonstarts WHERE grant_id=?1 AND effect_key=?2",params![grant.to_string(),native_key(&NativeEffect::ConditionalRollbackStopGeneral)?],|r|r.get(0))?;
ensure!(denied == 1, "UPDATE_CONDITIONAL_STOP_NOT_CLOSED");
}
if matches!(effect, NativeEffect::PublishInstalledRecord { .. }) {
let save = native_key(&NativeEffect::SaveInstalledRecord {
old_sha256: old_sha256.clone(),
@@ -900,11 +1095,60 @@ pub fn claim_native_effect(
require_matching_record_save(&tx, grant, old_sha256, new_sha256)?;
}
}
NativeEffect::RollbackStopService { service } => {
ensure!(
h.schema_version == 2
&& j.phase() == Phase::RollbackIntent
&& matches!(
guard.participant(),
Participant::NewHelper | Participant::StableUpdater
)
&& h.stop_set.contains(&native_service(*service)),
"UPDATE_ROLLBACK_STOP_ROLE"
);
}
NativeEffect::SaveRejectedCandidate { role } => {
ensure!(
h.schema_version == 2
&& guard.participant() == Participant::StableUpdater
&& j.phase() == Phase::RestoreIntent,
"UPDATE_ROLLBACK_SAVE_ROLE"
);
let last:Vec<u8>=tx.query_row("SELECT event FROM agent_update_handoff_events WHERE grant_id=?1 ORDER BY sequence DESC LIMIT 1",[grant.to_string()],|r|r.get(0))?;
ensure!(
serde_json::from_slice::<Event>(&last)?.evidence
== Evidence::RestoreIntent {
role: native_role(*role)
},
"UPDATE_ROLLBACK_SAVE_FILE"
);
}
NativeEffect::RestorePredecessor { role } => {
ensure!(
h.schema_version == 2
&& guard.participant() == Participant::StableUpdater
&& j.phase() == Phase::CandidatePreserved,
"UPDATE_ROLLBACK_RESTORE_ROLE"
);
let last:Vec<u8>=tx.query_row("SELECT event FROM agent_update_handoff_events WHERE grant_id=?1 ORDER BY sequence DESC LIMIT 1",[grant.to_string()],|r|r.get(0))?;
ensure!(
matches!(serde_json::from_slice::<Event>(&last)?.evidence,
Evidence::CandidatePreserved{role:r,..} if r==native_role(*role)),
"UPDATE_ROLLBACK_RESTORE_FILE"
);
let save = native_key(&NativeEffect::SaveRejectedCandidate { role: *role })?;
let observed:u64=tx.query_row("SELECT count(*) FROM agent_update_native_observations WHERE grant_id=?1 AND effect_key=?2",params![grant.to_string(),save],|r|r.get(0))?;
ensure!(observed == 1, "UPDATE_ROLLBACK_CANDIDATE_NOT_PRESERVED");
}
// Rollback needs its own qualified event transition; never infer it
// from an uncertain replacement or silently restart the predecessor.
NativeEffect::RestorePredecessor { .. }
| NativeEffect::StartService { .. }
| NativeEffect::ReturnLedger => anyhow::bail!("UPDATE_NATIVE_SPECIAL_TRANSITION_REQUIRED"),
NativeEffect::StartService { .. }
| NativeEffect::ReturnLedger
| NativeEffect::RollbackStartService { .. }
| NativeEffect::RollbackReturnLedger
| NativeEffect::ConditionalRollbackStopGeneral => {
anyhow::bail!("UPDATE_NATIVE_SPECIAL_TRANSITION_REQUIRED")
}
}
guard.verify_native_effect(&h, &d, effect, NativeEffectStage::Claim)?;
let result = insert_native_claim(&tx, &h, expected_head, effect.clone(), now)?;
@@ -912,6 +1156,281 @@ pub fn claim_native_effect(
tx.commit()?;
Ok(result)
}
/// Known startup/health failure only, still the original consumed attempt.
/// Nonstarts are separately attested under the actual guard; no failed start
/// is mislabeled a successful service observation and no old claim is erased.
pub fn begin_rollback(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
grant: Uuid,
expected_head: &str,
cause: &rmm_updater_handoff::RollbackCause,
failure_observation: &str,
now: DateTime<Utc>,
) -> Result<()> {
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
begin_rollback_in(
&tx,
guard,
grant,
expected_head,
cause,
failure_observation,
now,
)?;
guard.check(&tx)?;
tx.commit()?;
Ok(())
}
fn begin_rollback_in(
tx: &Connection,
guard: &impl UpdateParticipantGuard,
grant: Uuid,
expected_head: &str,
cause: &rmm_updater_handoff::RollbackCause,
failure_observation: &str,
now: DateTime<Utc>,
) -> Result<()> {
ensure!(
rmm_domain::agent_update::digest(failure_observation),
"UPDATE_ROLLBACK_FAILURE_OBSERVATION"
);
let (handoff, delegation, mut journal) = native_context(tx, guard, grant, now)?;
ensure!(
handoff.schema_version == 2
&& journal.head_sha256() == expected_head
&& matches!(
guard.participant(),
Participant::NewHelper | Participant::StableUpdater
),
"UPDATE_ROLLBACK_ORIGINAL_ROLE"
);
let raw = journal.prepare(
u64::try_from(now.timestamp())?,
Evidence::RollbackIntent {
cause: cause.clone(),
failure_observation_sha256: failure_observation.into(),
},
)?;
let event: Event = serde_json::from_slice(&raw)?;
verify_writer(tx, guard, grant, &event)?;
let mut query=tx.prepare("SELECT c.effect_key,c.claim,c.claim_sha256 FROM agent_update_native_claims c LEFT JOIN agent_update_native_observations o ON o.grant_id=c.grant_id AND o.effect_key=c.effect_key LEFT JOIN agent_update_native_nonstarts n ON n.grant_id=c.grant_id AND n.effect_key=c.effect_key WHERE c.grant_id=?1 AND o.claim_sha256 IS NULL AND n.claim_sha256 IS NULL")?;
let rows = query
.query_map([grant.to_string()], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, Vec<u8>>(1)?,
r.get::<_, String>(2)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
drop(query);
let mut actual_failed = false;
for (effect_key, raw_claim, claim_sha) in rows {
let claim: NativeEffectClaim = serde_json::from_slice(&raw_claim)?;
match &claim.effect {
NativeEffect::ReturnLedger => {}
NativeEffect::ConditionalRollbackStopGeneral => {
guard.verify_native_effect(
&handoff,
&delegation,
&claim.effect,
NativeEffectStage::ObserveNonstart,
)?;
let observation = guard.native_observation_sha256(
&handoff,
&delegation,
&claim.effect,
NativeEffectStage::ObserveNonstart,
)?;
ensure!(
rmm_domain::agent_update::digest(&observation),
"UPDATE_ROLLBACK_NONSTART_OBSERVATION"
);
tx.execute("INSERT INTO agent_update_native_nonstarts(grant_id,effect_key,claim_sha256,observation_sha256,observed_at) VALUES(?1,?2,?3,?4,?5)",params![grant.to_string(),effect_key,claim_sha,observation,now.to_rfc3339()])?;
}
NativeEffect::StartService { service } => {
ensure!(
matches!(
cause,
rmm_updater_handoff::RollbackCause::StartupFailed { .. }
) && native_key(&claim.effect)? == effect_key
&& hash(&raw_claim) == claim_sha,
"UPDATE_ROLLBACK_UNOBSERVED_START"
);
guard.verify_native_effect(
&handoff,
&delegation,
&claim.effect,
NativeEffectStage::ObserveNonstart,
)?;
let observation = guard.native_observation_sha256(
&handoff,
&delegation,
&claim.effect,
NativeEffectStage::ObserveNonstart,
)?;
ensure!(
rmm_domain::agent_update::digest(&observation),
"UPDATE_ROLLBACK_NONSTART_OBSERVATION"
);
if matches!(cause,rmm_updater_handoff::RollbackCause::StartupFailed{service:s} if *s==native_service(*service))
{
actual_failed = true;
}
tx.execute("INSERT INTO agent_update_native_nonstarts(grant_id,effect_key,claim_sha256,observation_sha256,observed_at) VALUES(?1,?2,?3,?4,?5)",params![grant.to_string(),effect_key,claim_sha,observation,now.to_rfc3339()])?;
}
_ => anyhow::bail!("UPDATE_ROLLBACK_UNCERTAIN_NATIVE_EFFECT"),
}
}
ensure!(
matches!(cause, rmm_updater_handoff::RollbackCause::HealthRejected) || actual_failed,
"UPDATE_ROLLBACK_KNOWN_FAILURE_REQUIRED"
);
journal.replay(&raw)?;
append_in(tx, grant, expected_head, &raw, &event, &journal)?;
// A service whose original once-only start is independently known not to
// have happened needs NO STOP effect. Close an exact deny-only subordinate
// claim, not a fabricated successful ControlService/systemctl observation.
let mut query=tx.prepare("SELECT claim FROM agent_update_native_claims c JOIN agent_update_native_nonstarts n ON n.grant_id=c.grant_id AND n.effect_key=c.effect_key WHERE c.grant_id=?1")?;
let nonstarts = query
.query_map([grant.to_string()], |r| r.get::<_, Vec<u8>>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
drop(query);
for raw_claim in nonstarts {
let claim: NativeEffectClaim = serde_json::from_slice(&raw_claim)?;
if let NativeEffect::StartService { service } = claim.effect {
let effect = NativeEffect::RollbackStopService { service };
guard.verify_native_effect(
&handoff,
&delegation,
&effect,
NativeEffectStage::ObserveNonstart,
)?;
let observation = guard.native_observation_sha256(
&handoff,
&delegation,
&effect,
NativeEffectStage::ObserveNonstart,
)?;
ensure!(
rmm_domain::agent_update::digest(&observation),
"UPDATE_ROLLBACK_STOP_NONSTART_HASH"
);
let tombstone =
insert_native_claim(tx, &handoff, journal.head_sha256(), effect.clone(), now)?;
tx.execute("INSERT INTO agent_update_native_nonstarts(grant_id,effect_key,claim_sha256,observation_sha256,observed_at) VALUES(?1,?2,?3,?4,?5)",params![grant.to_string(),native_key(&effect)?,hash(&serde_json::to_vec(&tombstone)?),observation,now.to_rfc3339()])?;
}
}
Ok(())
}
pub struct ReservedGeneralStop<'a> {
pub grant_id: Uuid,
pub expected_head: &'a str,
pub general_start_observation_sha256: &'a str,
pub general_stop_observation_sha256: &'a str,
pub failure_observation_sha256: &'a str,
pub now: DateTime<Utc>,
}
/// Early Full startup failure: the concrete started General process was
/// stopped using the conditional intent reserved BEFORE the original return.
/// Only the same still-live updater may record retained original START + STOP
/// observations after all readers stopped and the original lock was acquired.
pub fn begin_rollback_after_reserved_stop(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
input: ReservedGeneralStop<'_>,
) -> Result<()> {
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let (h, d, journal) = native_context(&tx, guard, input.grant_id, input.now)?;
ensure!(
h.schema_version == 2
&& h.pin_transition == rmm_updater_handoff::PinTransition::LinuxGeneralFullV1
&& guard.participant() == Participant::StableUpdater
&& journal.phase() == Phase::LedgerReturnIntent
&& journal.head_sha256() == input.expected_head,
"UPDATE_CONDITIONAL_ROLLBACK_CONTEXT"
);
for (effect, stage, observation) in [
(
NativeEffect::StartService {
service: NativeService::General,
},
NativeEffectStage::ObserveRetainedStart,
input.general_start_observation_sha256,
),
(
NativeEffect::ConditionalRollbackStopGeneral,
NativeEffectStage::Observe,
input.general_stop_observation_sha256,
),
] {
ensure!(
rmm_domain::agent_update::digest(observation),
"UPDATE_CONDITIONAL_OBSERVATION_HASH"
);
let claim = read_native_claim(&tx, guard, input.grant_id, &effect)?;
ensure!(
claim.head_sha256 == journal.head_sha256()
&& claim.owner_nonce == h.owner_nonce
&& input.now >= claim.claimed_at,
"UPDATE_CONDITIONAL_ORIGINAL_CLAIM"
);
guard.verify_native_effect(&h, &d, &effect, stage)?;
ensure!(
guard.native_observation_sha256(&h, &d, &effect, stage)? == observation,
"UPDATE_CONDITIONAL_RETAINED_OBSERVATION_CHANGED"
);
tx.execute("INSERT INTO agent_update_native_observations(grant_id,effect_key,claim_sha256,observation_sha256,observed_at) VALUES(?1,?2,?3,?4,?5)",params![input.grant_id.to_string(),native_key(&effect)?,hash(&serde_json::to_vec(&claim)?),observation,input.now.to_rfc3339()])?;
}
begin_rollback_in(
&tx,
guard,
input.grant_id,
input.expected_head,
&rmm_updater_handoff::RollbackCause::StartupFailed {
service: rmm_updater_handoff::Service::LinuxFullHelper,
},
input.failure_observation_sha256,
input.now,
)?;
guard.check(&tx)?;
tx.commit()?;
Ok(())
}
/// Deny-only close of a conditional reserve after actual new Full startup.
/// It is not a STOP observation and never dispatches/resumes the reservation.
pub fn deny_conditional_stop(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
grant: Uuid,
observation: &str,
now: DateTime<Utc>,
) -> Result<()> {
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let (h, d, journal) = native_context(&tx, guard, grant, now)?;
ensure!(
h.schema_version == 2
&& guard.participant() == Participant::NewHelper
&& matches!(
journal.phase(),
Phase::ServicesStarted | Phase::HealthVerified
),
"UPDATE_CONDITIONAL_DENY_ROLE"
);
let effect = NativeEffect::ConditionalRollbackStopGeneral;
let claim = read_native_claim(&tx, guard, grant, &effect)?;
guard.verify_native_effect(&h, &d, &effect, NativeEffectStage::ObserveNonstart)?;
ensure!(
guard.native_observation_sha256(&h, &d, &effect, NativeEffectStage::ObserveNonstart)?
== observation,
"UPDATE_CONDITIONAL_DENY_OBSERVATION"
);
tx.execute("INSERT INTO agent_update_native_nonstarts(grant_id,effect_key,claim_sha256,observation_sha256,observed_at) VALUES(?1,?2,?3,?4,?5)",params![grant.to_string(),native_key(&effect)?,hash(&serde_json::to_vec(&claim)?),observation,now.to_rfc3339()])?;
guard.check(&tx)?;
tx.commit()?;
Ok(())
}
pub fn observe_native_effect(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
@@ -966,8 +1485,43 @@ pub fn observe_native_effect(
)
}
NativeEffect::ReturnLedger | NativeEffect::RestorePredecessor { .. } => {
if matches!(claim.effect, NativeEffect::RestorePredecessor { .. }) {
ensure!(
h.schema_version == 2
&& guard.participant() == Participant::StableUpdater
&& j.phase() == Phase::CandidatePreserved
&& j.head_sha256() == claim.head_sha256,
"UPDATE_ROLLBACK_FILE_OBSERVER"
);
} else {
anyhow::bail!("UPDATE_NATIVE_OBSERVATION_NOT_SUPPORTED");
}
}
NativeEffect::RollbackStopService { .. } => ensure!(
h.schema_version == 2
&& guard.participant() == Participant::StableUpdater
&& j.phase() == Phase::RollbackIntent,
"UPDATE_ROLLBACK_STOP_OBSERVER"
),
NativeEffect::SaveRejectedCandidate { .. } => ensure!(
h.schema_version == 2
&& guard.participant() == Participant::StableUpdater
&& j.phase() == Phase::RestoreIntent
&& j.head_sha256() == claim.head_sha256,
"UPDATE_ROLLBACK_FILE_OBSERVER"
),
NativeEffect::RollbackStartService { .. } => ensure!(
h.schema_version == 2
&& guard.participant() == Participant::RestoredHelper
&& j.phase() == Phase::RollbackLedgerReturnIntent,
"UPDATE_ROLLBACK_START_OBSERVER"
),
NativeEffect::RollbackReturnLedger => {
anyhow::bail!("UPDATE_NATIVE_OBSERVATION_NOT_SUPPORTED")
}
NativeEffect::ConditionalRollbackStopGeneral => {
anyhow::bail!("UPDATE_NATIVE_SPECIAL_TRANSITION_REQUIRED")
}
}
guard.verify_native_effect(&h, &d, &claim.effect, NativeEffectStage::Observe)?;
tx.execute("INSERT INTO agent_update_native_observations(grant_id,effect_key,claim_sha256,observation_sha256,observed_at) VALUES(?1,?2,?3,?4,?5)",params![claim.grant_id.to_string(),native_key(&claim.effect)?,hash(&raw),observation_sha256,now.to_rfc3339()])?;
@@ -1079,6 +1633,16 @@ fn prepare_return_for<const N: usize>(
NativeEffectStage::ReserveFutureStart,
)?;
}
let conditional = h.schema_version == 2
&& h.pin_transition == rmm_updater_handoff::PinTransition::LinuxGeneralFullV1;
if conditional {
guard.verify_native_effect(
&h,
&d,
&NativeEffect::ConditionalRollbackStopGeneral,
NativeEffectStage::ReserveConditionalStop,
)?;
}
j.replay(return_event)?;
append_in(&tx, grant, expected_head, return_event, &parsed, &j)?;
let return_ledger =
@@ -1093,13 +1657,148 @@ fn prepare_return_for<const N: usize>(
now,
)?);
}
let conditional_rollback_stop_general = if conditional {
Some(insert_native_claim(
&tx,
&h,
j.head_sha256(),
NativeEffect::ConditionalRollbackStopGeneral,
now,
)?)
} else {
None
};
guard.check(&tx)?;
tx.commit()?;
Ok(PreparedReturnClaims {
return_ledger,
conditional_rollback_stop_general,
starts: starts
.try_into()
.map_err(|_| anyhow::anyhow!("UPDATE_NATIVE_START_COUNT"))?,
})
}
pub fn prepare_rollback_return_and_starts(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
grant: Uuid,
expected_head: &str,
return_event: &[u8],
now: DateTime<Utc>,
) -> Result<PreparedReturnClaims> {
prepare_rollback_return_for(
db,
guard,
grant,
expected_head,
return_event,
now,
[
NativeService::General,
NativeService::Full,
NativeService::Agent,
],
)
}
pub fn prepare_windows_rollback_return_and_starts(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
grant: Uuid,
expected_head: &str,
return_event: &[u8],
now: DateTime<Utc>,
) -> Result<PreparedReturnClaims<2>> {
prepare_rollback_return_for(
db,
guard,
grant,
expected_head,
return_event,
now,
[NativeService::WindowsHelper, NativeService::WindowsAgent],
)
}
fn prepare_rollback_return_for<const N: usize>(
db: &mut Connection,
guard: &impl UpdateParticipantGuard,
grant: Uuid,
expected_head: &str,
return_event: &[u8],
now: DateTime<Utc>,
sequence: [NativeService; N],
) -> Result<PreparedReturnClaims<N>> {
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
let (h, d, mut journal) = native_context(&tx, guard, grant, now)?;
ensure!(
h.schema_version == 2
&& guard.participant() == Participant::StableUpdater
&& journal.phase() == Phase::PredecessorVerified
&& journal.head_sha256() == expected_head,
"UPDATE_ROLLBACK_RETURN_ROLE"
);
ensure!(
(sequence.as_slice()
== [
NativeService::General,
NativeService::Full,
NativeService::Agent
]
&& h.pin_transition == rmm_updater_handoff::PinTransition::LinuxGeneralFullV1)
|| (sequence.as_slice() == [NativeService::WindowsHelper, NativeService::WindowsAgent]
&& h.pin_transition == rmm_updater_handoff::PinTransition::WindowsCodePairClockV1),
"UPDATE_ROLLBACK_RETURN_SERVICE_SET"
);
let missing:u64=tx.query_row("SELECT count(*) FROM agent_update_native_claims c LEFT JOIN agent_update_native_observations o ON o.grant_id=c.grant_id AND o.effect_key=c.effect_key LEFT JOIN agent_update_native_nonstarts n ON n.grant_id=c.grant_id AND n.effect_key=c.effect_key WHERE c.grant_id=?1 AND o.claim_sha256 IS NULL AND n.claim_sha256 IS NULL AND c.effect_key<>?2",
params![grant.to_string(),native_key(&NativeEffect::ReturnLedger)?],|r|r.get(0))?;
ensure!(missing == 0, "UPDATE_ROLLBACK_UNOBSERVED_EFFECT");
let event: Event = serde_json::from_slice(return_event)?;
ensure!(
event.evidence == Evidence::RollbackLedgerReturnIntent
&& event.observed_at == u64::try_from(now.timestamp())?,
"UPDATE_ROLLBACK_RETURN_EVENT"
);
verify_writer(&tx, guard, grant, &event)?;
guard.verify_native_effect(
&h,
&d,
&NativeEffect::RollbackReturnLedger,
NativeEffectStage::Claim,
)?;
for service in sequence {
guard.verify_native_effect(
&h,
&d,
&NativeEffect::RollbackStartService { service },
NativeEffectStage::ReserveFutureStart,
)?;
}
journal.replay(return_event)?;
append_in(&tx, grant, expected_head, return_event, &event, &journal)?;
let return_ledger = insert_native_claim(
&tx,
&h,
journal.head_sha256(),
NativeEffect::RollbackReturnLedger,
now,
)?;
let mut starts = Vec::new();
for service in sequence {
starts.push(insert_native_claim(
&tx,
&h,
journal.head_sha256(),
NativeEffect::RollbackStartService { service },
now,
)?);
}
guard.check(&tx)?;
tx.commit()?;
Ok(PreparedReturnClaims {
return_ledger,
starts: starts
.try_into()
.map_err(|_| anyhow::anyhow!("UPDATE_NATIVE_START_COUNT"))?,
.map_err(|_| anyhow::anyhow!("UPDATE_ROLLBACK_START_COUNT"))?,
conditional_rollback_stop_general: None,
})
}
@@ -1158,6 +1857,14 @@ pub struct NativeControlReply {
/// Only ReadOriginal; exact retained execution_evidence.start bytes,
/// bounded to 512KiB before canonical Base64url encoding, never rewritten.
pub original_start_base64url: Option<String>,
/// ReadOriginal only: exact applied helper_sync_decisions.request protobuf
/// and response JSON SignedResponse bytes. Both absent until that actual
/// ORIGINAL inflight control acknowledgement has been applied. Evidence
/// only: never a caller ready bit, replacement request or new authority.
#[serde(default,skip_serializing_if="Option::is_none")]
pub pre_stop_sync_request_base64url:Option<String>,
#[serde(default,skip_serializing_if="Option::is_none")]
pub pre_stop_sync_response_base64url:Option<String>,
}
fn require_matching_record_save(
+4 -1
View File
@@ -251,7 +251,10 @@ fn services_prefix(
let (observed, journal) = verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
ensure!(
observed.bytes()? == header.bytes()?
&& journal.phase() == Phase::ServicesStarted
&& matches!(
journal.phase(),
Phase::ServicesStarted | Phase::RollbackServicesStarted
)
&& local.grant_id == header.attempt.grant_id.to_string()
&& local.consumption == "consumed"
&& local.state == "running"