Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -40,8 +40,10 @@ const MAX_REPLY: usize = 1536 * 1024;
|
||||
pub enum Generation {
|
||||
Original,
|
||||
Replacement,
|
||||
Restored,
|
||||
}
|
||||
pub struct BridgeContext<'a> {
|
||||
pub binding: &'a rmm_agent_core::auth::Binding,
|
||||
pub handoff: &'a Handoff,
|
||||
pub delegation: &'a UpdateDelegation,
|
||||
pub services: &'a Systemd,
|
||||
@@ -62,6 +64,7 @@ pub struct Reply {
|
||||
pub original_start: Option<Vec<u8>>,
|
||||
pub claim: Option<NativeEffectClaim>,
|
||||
pub original_outcome: Option<Vec<u8>>,
|
||||
pub pre_stop_sync: Option<(Vec<u8>, Vec<u8>)>,
|
||||
}
|
||||
impl<'a> FullBridge<'a> {
|
||||
/// Prefix is either the exact initial atomic StopIntent or a complete
|
||||
@@ -137,18 +140,20 @@ impl<'a> FullBridge<'a> {
|
||||
"UPDATE_BRIDGE_CLAIM_PHASE"
|
||||
),
|
||||
NativeControlOperation::ObserveStarts { .. } => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal
|
||||
.as_ref()
|
||||
.is_some_and(|j| j.phase() == Phase::LedgerReturnIntent),
|
||||
journal.as_ref().is_some_and(|j| matches!(
|
||||
(context.generation, j.phase()),
|
||||
(Generation::Replacement, Phase::LedgerReturnIntent)
|
||||
| (Generation::Restored, Phase::RollbackLedgerReturnIntent)
|
||||
)),
|
||||
"UPDATE_BRIDGE_START_PHASE"
|
||||
),
|
||||
NativeControlOperation::Health { receipt } => {
|
||||
ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal
|
||||
.as_ref()
|
||||
.is_some_and(|j| j.phase() == Phase::ServicesStarted),
|
||||
journal.as_ref().is_some_and(|j| matches!(
|
||||
(context.generation, j.phase()),
|
||||
(Generation::Replacement, Phase::ServicesStarted)
|
||||
| (Generation::Restored, Phase::RollbackServicesStarted)
|
||||
)),
|
||||
"UPDATE_BRIDGE_HEALTH_PHASE"
|
||||
);
|
||||
journal
|
||||
@@ -156,16 +161,23 @@ impl<'a> FullBridge<'a> {
|
||||
.context("UPDATE_BRIDGE_ORIGINAL_READ_REQUIRED")?
|
||||
.prepare(
|
||||
u64::try_from(context.clock.now()?.timestamp())?,
|
||||
rmm_updater_handoff::Evidence::HealthVerified {
|
||||
receipt: receipt.clone(),
|
||||
if context.generation == Generation::Restored {
|
||||
rmm_updater_handoff::Evidence::RollbackHealthVerified {
|
||||
receipt: receipt.clone(),
|
||||
}
|
||||
} else {
|
||||
rmm_updater_handoff::Evidence::HealthVerified {
|
||||
receipt: receipt.clone(),
|
||||
}
|
||||
},
|
||||
)?;
|
||||
}
|
||||
NativeControlOperation::Finalize => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal
|
||||
.as_ref()
|
||||
.is_some_and(|j| j.phase() == Phase::HealthVerified),
|
||||
journal.as_ref().is_some_and(|j| matches!(
|
||||
(context.generation, j.phase()),
|
||||
(Generation::Replacement, Phase::HealthVerified)
|
||||
| (Generation::Restored, Phase::RollbackHealthVerified)
|
||||
)),
|
||||
"UPDATE_BRIDGE_FINAL_PHASE"
|
||||
),
|
||||
}
|
||||
@@ -227,6 +239,11 @@ impl<'a> FullBridge<'a> {
|
||||
&& peer.pid != context.delegation.original_helper.pid,
|
||||
"UPDATE_BRIDGE_REPLACEMENT_HELPER_CHANGED"
|
||||
),
|
||||
Generation::Restored => ensure!(
|
||||
peer.image_sha256 == context.handoff.predecessor.helper_sha256
|
||||
&& peer.pid != context.delegation.original_helper.pid,
|
||||
"UPDATE_BRIDGE_RESTORED_HELPER_CHANGED"
|
||||
),
|
||||
}
|
||||
let request = NativeControlRequest {
|
||||
operation: "agent_update_control_v1".into(),
|
||||
@@ -276,6 +293,21 @@ impl<'a> FullBridge<'a> {
|
||||
&& reply.handoff_sha256 == request.handoff_sha256,
|
||||
"UPDATE_BRIDGE_REPLY_BINDING"
|
||||
);
|
||||
let pre_stop_sync = match (
|
||||
&reply.pre_stop_sync_request_base64url,
|
||||
&reply.pre_stop_sync_response_base64url,
|
||||
) {
|
||||
(None, None) => None,
|
||||
(Some(raw_request), Some(response))
|
||||
if matches!(request.method, NativeControlOperation::ReadOriginal) =>
|
||||
{
|
||||
Some((
|
||||
decode(raw_request, 256 * 1024)?,
|
||||
decode(response, 256 * 1024)?,
|
||||
))
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_BRIDGE_PRESTOP_FIELDS_DENIED"),
|
||||
};
|
||||
let original_start = reply
|
||||
.original_start_base64url
|
||||
.as_deref()
|
||||
@@ -297,8 +329,13 @@ impl<'a> FullBridge<'a> {
|
||||
let outcome: ReconciledOutcome = serde_json::from_slice(raw)?;
|
||||
ensure!(
|
||||
canonical_outcome_bytes(&outcome)? == *raw
|
||||
&& outcome.state == "succeeded"
|
||||
&& !outcome.reboot_required,
|
||||
&& if context.generation == Generation::Restored {
|
||||
outcome.state == "failed"
|
||||
&& outcome.error_code == "AGENT_UPDATE_RETURNED_TO_PREDECESSOR"
|
||||
&& !outcome.reboot_required
|
||||
} else {
|
||||
outcome.state == "succeeded" && !outcome.reboot_required
|
||||
},
|
||||
"UPDATE_BRIDGE_NONCANONICAL_OR_UNSUCCESSFUL_OUTCOME"
|
||||
);
|
||||
} else {
|
||||
@@ -322,6 +359,58 @@ impl<'a> FullBridge<'a> {
|
||||
&& proof.events[..prior.events.len()] == prior.events),
|
||||
"UPDATE_BRIDGE_PROOF_ROLLBACK_OR_CHANGED"
|
||||
);
|
||||
if let Some((raw, response)) = &pre_stop_sync {
|
||||
use prost::Message;
|
||||
let req = rmm_protocol::SyncRequest::decode(raw.as_slice())?;
|
||||
let signed: rmm_agent_core::helper_reconcile::SignedResponse =
|
||||
serde_json::from_slice(response)?;
|
||||
let response = rmm_protocol::SyncResponse {
|
||||
decision_bytes: signed.decision_bytes,
|
||||
signature: signed.signature,
|
||||
};
|
||||
let decision = rmm_agent_core::reconcile::verify_response(
|
||||
context.binding,
|
||||
&req,
|
||||
&rmm_agent_core::reconcile::Channel {
|
||||
boot_id: context.handoff.boot_id,
|
||||
connection_generation: context.handoff.connection_generation,
|
||||
},
|
||||
&response,
|
||||
context.clock.now()?,
|
||||
)?;
|
||||
let expected =
|
||||
rmm_domain::agent_update::inflight_decision(proof, context.health_public_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let matching = decision
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| {
|
||||
a.job_id == context.handoff.attempt.job_id
|
||||
&& a.attempt_id == context.handoff.attempt.attempt_id
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(matching.len() == 1, "UPDATE_BRIDGE_PRESTOP_ACK_ATTEMPT");
|
||||
let ack = matching[0];
|
||||
ensure!(
|
||||
context.generation == Generation::Original
|
||||
&& journal.phase() == Phase::StopIntent
|
||||
&& req.encode_to_vec() == *raw
|
||||
&& !decision.ready
|
||||
&& !decision.helper_ready
|
||||
&& ack.update_inflight.as_ref() == Some(&expected)
|
||||
&& !ack.unlock
|
||||
&& ack.outcome_state == "running"
|
||||
&& ack.outcome_bytes.is_empty()
|
||||
&& ack.outcome_sha256.is_none()
|
||||
&& ack.grant_id == Some(context.handoff.attempt.grant_id)
|
||||
&& ack.authorization_id == Some(context.handoff.attempt.authorization_id)
|
||||
&& ack.plan_hash.as_deref()
|
||||
== Some(context.handoff.attempt.plan_sha256.as_str())
|
||||
&& ack.job_payload_hash == context.handoff.attempt.job_payload_sha256
|
||||
&& ack.original_boot_id == Some(context.handoff.boot_id),
|
||||
"UPDATE_BRIDGE_SIGNED_PRESTOP_ACK"
|
||||
);
|
||||
}
|
||||
match request.method {
|
||||
NativeControlOperation::ReadOriginal => ensure!(
|
||||
reply.claim.is_none() && original_start.is_some(),
|
||||
@@ -348,16 +437,20 @@ impl<'a> FullBridge<'a> {
|
||||
);
|
||||
}
|
||||
NativeControlOperation::ObserveStarts { .. } => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::ServicesStarted
|
||||
&& reply.claim.is_none()
|
||||
matches!(
|
||||
(context.generation, journal.phase()),
|
||||
(Generation::Replacement, Phase::ServicesStarted)
|
||||
| (Generation::Restored, Phase::RollbackServicesStarted)
|
||||
) && reply.claim.is_none()
|
||||
&& original_start.is_none(),
|
||||
"UPDATE_BRIDGE_STARTS_REPLY"
|
||||
),
|
||||
NativeControlOperation::Health { .. } => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::HealthVerified
|
||||
&& reply.claim.is_none()
|
||||
matches!(
|
||||
(context.generation, journal.phase()),
|
||||
(Generation::Replacement, Phase::HealthVerified)
|
||||
| (Generation::Restored, Phase::RollbackHealthVerified)
|
||||
) && reply.claim.is_none()
|
||||
&& original_start.is_none(),
|
||||
"UPDATE_BRIDGE_HEALTH_REPLY"
|
||||
),
|
||||
@@ -377,6 +470,7 @@ impl<'a> FullBridge<'a> {
|
||||
claim: reply.claim,
|
||||
original_start,
|
||||
original_outcome,
|
||||
pre_stop_sync,
|
||||
})
|
||||
}
|
||||
pub fn head_sha256(&self) -> Result<&str> {
|
||||
@@ -422,8 +516,21 @@ impl<'a> FullBridge<'a> {
|
||||
.context("UPDATE_BRIDGE_STARTED_EVENT_MISSING")?,
|
||||
)?;
|
||||
ensure!(
|
||||
matches!(last.evidence,rmm_updater_handoff::Evidence::ServicesStarted {services:actual,observation_sha256:hash,..}
|
||||
if actual==services && hash==observation_sha256),
|
||||
match last.evidence {
|
||||
rmm_updater_handoff::Evidence::ServicesStarted {
|
||||
services: actual,
|
||||
observation_sha256: hash,
|
||||
..
|
||||
} if self.context.generation == Generation::Replacement =>
|
||||
actual == services && hash == observation_sha256,
|
||||
rmm_updater_handoff::Evidence::RollbackServicesStarted {
|
||||
services: actual,
|
||||
observation_sha256: hash,
|
||||
..
|
||||
} if self.context.generation == Generation::Restored =>
|
||||
actual == services && hash == observation_sha256,
|
||||
_ => false,
|
||||
},
|
||||
"UPDATE_BRIDGE_STARTED_RESULT_CHANGED"
|
||||
);
|
||||
Ok(reply)
|
||||
|
||||
Reference in New Issue
Block a user