Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s

This commit is contained in:
2026-11-16 15:25:46 +01:00
parent 6497c79a91
commit 96a2b6e5ae
44 changed files with 6986 additions and 1024 deletions
+338 -17
View File
@@ -150,22 +150,117 @@ impl Systemd {
],
budget,
)?;
let text = std::str::from_utf8(&output)?;
let mut fields = BTreeMap::new();
for row in text.lines() {
let (k, v) = row
parse_service_status(&output)
}
fn failure_status(service: Service, budget: &Budget) -> Result<BTreeMap<String, String>> {
let bytes = command(
&[
"show",
service.unit(),
"-p",
"SubState",
"-p",
"ControlPID",
"-p",
"Result",
"-p",
"ExecMainCode",
"-p",
"ExecMainStatus",
"-p",
"Job",
"-p",
"ControlGroup",
],
budget,
)?;
let mut status = BTreeMap::new();
for line in std::str::from_utf8(&bytes)?.lines() {
let (k, v) = line
.split_once('=')
.ok_or_else(|| anyhow::anyhow!("UPDATE_SYSTEMD_RESPONSE"))?;
.ok_or_else(|| anyhow::anyhow!("UPDATE_FAILED_STATUS"))?;
ensure!(
fields.insert(k.into(), v.into()).is_none(),
"UPDATE_SYSTEMD_DUPLICATE"
status.insert(k.to_owned(), v.to_owned()).is_none(),
"UPDATE_FAILED_STATUS_DUPLICATE"
);
}
ensure!(status.len() == 7, "UPDATE_FAILED_STATUS_FIELDS");
Ok(status)
}
fn failed_start(&self, service: Service, budget: &Budget) -> Result<FailedStartProof> {
let pin = self
.policies
.iter()
.find(|p| p.service == service)
.ok_or_else(|| anyhow::anyhow!("UPDATE_SERVICE_NOT_PINNED"))?;
let first = Self::status(service, budget)?;
let extra = Self::failure_status(service, budget)?;
ensure!(
fields.len() == 8 && fields.get("LoadState").is_some_and(|v| v == "loaded"),
"UPDATE_SYSTEMD_LOADED"
first["ActiveState"] == "failed"
&& first["MainPID"] == "0"
&& extra["SubState"] == "failed"
&& extra["ControlPID"] == "0"
&& extra["Job"].is_empty()
&& extra["ControlGroup"].is_empty()
&& matches!(
extra["Result"].as_str(),
"exit-code"
| "signal"
| "core-dump"
| "timeout"
| "protocol"
| "resources"
| "start-limit-hit"
)
&& first["InvocationID"].len() == 32
&& first["InvocationID"].bytes().any(|b| b != b'0')
&& first["InvocationID"].bytes().all(|b| b.is_ascii_hexdigit())
&& unit_composition(&first)? == pin.unit_sha256,
"UPDATE_START_FAILURE_NOT_CLOSED"
);
Ok(fields)
ensure!(
Self::status(service, budget)? == first
&& Self::failure_status(service, budget)? == extra,
"UPDATE_START_FAILURE_CHANGED"
);
let mut status = first;
status.extend(extra);
Ok(FailedStartProof {
service,
unit_sha256: pin.unit_sha256.clone(),
status,
})
}
pub fn verify_failed_start(&self, expected: &FailedStartProof, budget: &Budget) -> Result<()> {
ensure!(
self.failed_start(expected.service, budget)? == *expected,
"UPDATE_FAILED_START_CHANGED"
);
Ok(())
}
pub fn verify_inactive(&self, service: Service, budget: &Budget) -> Result<()> {
let pin = self
.policies
.iter()
.find(|p| p.service == service)
.ok_or_else(|| anyhow::anyhow!("UPDATE_SERVICE_NOT_PINNED"))?;
let status = Self::status(service, budget)?;
let extra = Self::failure_status(service, budget)?;
ensure!(
status["ActiveState"] == "inactive"
&& status["MainPID"] == "0"
&& extra["ControlPID"] == "0"
&& extra["Job"].is_empty()
&& extra["ControlGroup"].is_empty()
&& unit_composition(&status)? == pin.unit_sha256,
"UPDATE_SERVICE_NOT_CLOSED_INACTIVE"
);
ensure!(
Self::status(service, budget)? == status
&& Self::failure_status(service, budget)? == extra,
"UPDATE_INACTIVE_SERVICE_CHANGED"
);
Ok(())
}
pub fn observe(&self, service: Service, budget: &Budget) -> Result<HeldService> {
let pin = self
@@ -265,6 +360,9 @@ impl Systemd {
let result = StoppedServices {
systemd: self,
original,
failed: None,
subsequently_exited: Vec::new(),
early_full_failure: None,
};
result.verify_still_stopped(budget)?;
Ok(result)
@@ -356,17 +454,170 @@ impl HeldService {
pub struct StoppedServices {
systemd: Systemd,
original: Vec<HeldService>,
failed: Option<FailedStartProof>,
subsequently_exited: Vec<HeldService>,
early_full_failure: Option<EarlyFullFailure>,
}
/// Concrete systemd failure of our acknowledged once-only start. A timeout,
/// lost command reply, activating/auto-restart or residual control process is
/// never represented by this value.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct FailedStartProof {
pub service: Service,
pub unit_sha256: String,
pub status: BTreeMap<String, String>,
}
pub enum Startup {
Complete {
started: Vec<HeldService>,
stopped: StoppedServices,
},
KnownFailed(KnownStartupFailure),
}
/// Retained only by the live worker that actually started and stopped the new
/// General under the original conditional reservation. Not deserializable.
pub struct EarlyFullFailure {
general_start: ServiceProof,
general_stop_sha256: String,
failure: FailedStartProof,
}
impl EarlyFullFailure {
pub fn general_start(&self) -> &ServiceProof {
&self.general_start
}
pub fn general_stop_sha256(&self) -> &str {
&self.general_stop_sha256
}
pub fn failure(&self) -> &FailedStartProof {
&self.failure
}
pub fn failure_sha256(&self) -> Result<String> {
Ok(digest(&serde_json::to_vec(&self.failure)?))
}
}
pub struct KnownStartupFailure {
stopped: StoppedServices,
target: Systemd,
started: Vec<HeldService>,
failure: FailedStartProof,
}
impl KnownStartupFailure {
pub fn failure(&self) -> &FailedStartProof {
&self.failure
}
pub fn started(&self) -> &[HeldService] {
&self.started
}
pub fn verify(&self, budget: &Budget) -> Result<()> {
budget.check()?;
self.target.verify_failed_start(&self.failure, budget)?;
for held in &self.started {
held.verify(budget)?;
}
for service in [Service::General, Service::Full, Service::Agent] {
if service == self.failure.service
|| self.started.iter().any(|s| s.proof.service == service)
{
continue;
}
self.target.verify_inactive(service, budget)?;
}
ensure!(
self.stopped
.original
.iter()
.all(|s| s.process.exited().unwrap_or(false)),
"UPDATE_OLD_PROCESS_STILL_LIVE"
);
Ok(())
}
/// An early Full failure leaves only the newly started General reader.
/// The embedding fence must hold its original first-return conditional
/// reservation. No SQLite may be opened until this exact held PID exits.
pub fn stop_reserved_general(
mut self,
budget: &Budget,
fence: &mut impl DurableEffectFence,
) -> Result<(StoppedServices, ServiceProof, String)> {
self.verify(budget)?;
ensure!(
self.failure.service == Service::Full
&& self.started.len() == 1
&& self.started[0].proof.service == Service::General,
"UPDATE_CONDITIONAL_STOP_ONLY_EARLY_FULL_FAILURE"
);
let held = &self.started[0];
let proof = held.proof.clone();
let effect = Effect::StopService {
service: Service::General,
};
fence.before_effect(&effect)?;
command(&["--no-block", "stop", Service::General.unit()], budget)?;
loop {
budget.check()?;
self.target.verify_failed_start(&self.failure, budget)?;
self.target.verify_inactive(Service::Agent, budget)?;
let state = Systemd::status(Service::General, budget)?;
ensure!(
state["MainPID"] == "0" || state["MainPID"] == proof.process.pid.to_string(),
"UPDATE_CONDITIONAL_STOP_PID_CHANGED"
);
if state["ActiveState"] == "inactive"
&& state["MainPID"] == "0"
&& held.process.exited()?
{
break;
}
thread::sleep(Duration::from_millis(20));
}
let observation = digest(&serde_json::to_vec(&(
"reserved_general_stop_v1",
&proof,
&self.failure,
))?);
fence.after_effect(&effect, &observation)?;
self.stopped.subsequently_exited.append(&mut self.started);
self.stopped.failed = Some(self.failure.clone());
self.stopped.early_full_failure = Some(EarlyFullFailure {
general_start: proof.clone(),
general_stop_sha256: observation.clone(),
failure: self.failure,
});
self.stopped.verify_still_stopped(budget)?;
Ok((self.stopped, proof, observation))
}
}
impl StoppedServices {
pub fn early_full_failure(&self, budget: &Budget) -> Result<&EarlyFullFailure> {
self.verify_still_stopped(budget)?;
let proof = self
.early_full_failure
.as_ref()
.ok_or_else(|| anyhow::anyhow!("UPDATE_NO_RETAINED_EARLY_FAILURE"))?;
ensure!(
self.subsequently_exited.len() == 1
&& self.subsequently_exited[0].proof == proof.general_start
&& self.subsequently_exited[0].process.exited()?,
"UPDATE_RETAINED_GENERAL_STOP_CHANGED"
);
Ok(proof)
}
pub fn verify_still_stopped(&self, budget: &Budget) -> Result<()> {
for pin in &self.systemd.policies {
let s = Systemd::status(pin.service, budget)?;
ensure!(
s["ActiveState"] == "inactive"
&& s["MainPID"] == "0"
&& unit_composition(&s)? == pin.unit_sha256,
"UPDATE_READER_NOT_STOPPED"
);
if let Some(failed) = self.failed.as_ref().filter(|f| f.service == pin.service) {
// Unit composition is still the provisioned original unit;
// only code/config files were part of the approved transition.
self.systemd.verify_failed_start(failed, budget)?;
} else {
ensure!(
s["ActiveState"] == "inactive"
&& s["MainPID"] == "0"
&& unit_composition(&s)? == pin.unit_sha256,
"UPDATE_READER_NOT_STOPPED"
);
}
if pin.service == Service::Apt {
ensure!(
s["UnitFileState"] == "disabled",
@@ -377,6 +628,7 @@ impl StoppedServices {
ensure!(
self.original
.iter()
.chain(self.subsequently_exited.iter())
.all(|s| s.process.exited().unwrap_or(false)),
"UPDATE_OLD_PROCESS_STILL_LIVE"
);
@@ -393,6 +645,17 @@ impl StoppedServices {
budget: &Budget,
fence: &mut impl DurableEffectFence,
) -> Result<Vec<HeldService>> {
match self.start_once_observed(new, budget, fence)? {
Startup::Complete { started, .. } => Ok(started),
Startup::KnownFailed(_) => anyhow::bail!("UPDATE_START_KNOWN_FAILED"),
}
}
pub fn start_once_observed(
self,
new: Systemd,
budget: &Budget,
fence: &mut impl DurableEffectFence,
) -> Result<Startup> {
self.verify_still_stopped(budget)?;
ensure!(
self.systemd
@@ -424,6 +687,17 @@ impl StoppedServices {
started.push(held);
break;
}
if state["ActiveState"] == "failed" {
let failure = new.failed_start(service, budget)?;
let result = KnownStartupFailure {
stopped: self,
target: new,
started,
failure,
};
result.verify(budget)?;
return Ok(Startup::KnownFailed(result));
}
ensure!(
state["ActiveState"] == "activating" || state["ActiveState"] == "active",
"UPDATE_START_UNPROVEN"
@@ -433,10 +707,40 @@ impl StoppedServices {
}
// Return in the canonical Handoff stop-set order, not actual start order.
started.sort_by_key(|s| s.proof.service);
Ok(started)
Ok(Startup::Complete {
started,
stopped: self,
})
}
}
fn parse_service_status(raw: &[u8]) -> Result<BTreeMap<String, String>> {
const FIELDS: [&str; 7] = [
"LoadState",
"ActiveState",
"MainPID",
"InvocationID",
"UnitFileState",
"FragmentPath",
"DropInPaths",
];
let mut fields = BTreeMap::new();
for row in std::str::from_utf8(raw)?.lines() {
let (k, v) = row
.split_once('=')
.ok_or_else(|| anyhow::anyhow!("UPDATE_SYSTEMD_RESPONSE"))?;
ensure!(
FIELDS.contains(&k) && fields.insert(k.into(), v.into()).is_none(),
"UPDATE_SYSTEMD_PROPERTIES"
);
}
ensure!(
fields.len() == FIELDS.len() && fields.get("LoadState").is_some_and(|s| s == "loaded"),
"UPDATE_SYSTEMD_LOADED"
);
Ok(fields)
}
fn unit_composition(fields: &BTreeMap<String, String>) -> Result<String> {
use crate::files::Directory;
use std::{ffi::OsStr, path::Path};
@@ -550,3 +854,20 @@ fn command(args: &[&str], budget: &Budget) -> Result<Vec<u8>> {
thread::sleep(Duration::from_millis(5));
}
}
#[cfg(test)]
mod status_tests {
use super::*;
#[test]
fn accepts_actual_seven_property_shape_and_denies_missing_or_extra_properties() {
let status=b"LoadState=loaded\nActiveState=active\nMainPID=123\nInvocationID=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\nUnitFileState=enabled\nFragmentPath=/etc/systemd/system/rmm-agent-remote.service\nDropInPaths=\n";
assert_eq!(parse_service_status(status).unwrap().len(), 7);
let mut extra = status.to_vec();
extra.extend_from_slice(b"Unexpected=1\n");
assert!(parse_service_status(&extra).is_err());
assert!(parse_service_status(b"LoadState=loaded\nActiveState=active\n").is_err());
let mut duplicate = status.to_vec();
duplicate.extend_from_slice(b"MainPID=321\n");
assert!(parse_service_status(&duplicate).is_err());
}
}