Wire preserved update control, health evidence and record publication
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,518 @@
|
||||
//! Bounded client of the EXISTING fixed Fullhelper control socket. No retry,
|
||||
//! SQL/event transport, caller paths, process-name or root-UID-only fallback.
|
||||
use crate::{clock::NativeClock, participants::StableParticipant};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use rmm_agent_core::{
|
||||
reconcile::SyncClock,
|
||||
update_handoff::{
|
||||
NativeControlOperation, NativeControlReply, NativeControlRequest, NativeEffect,
|
||||
NativeEffectClaim,
|
||||
},
|
||||
};
|
||||
use rmm_domain::{
|
||||
agent_update::{ParticipantRole, UpdateDelegation, UpdateInflightProof, verify_inflight_proof},
|
||||
reconciliation::{ReconciledOutcome, canonical_outcome_bytes},
|
||||
};
|
||||
use rmm_updater_handoff::{Handoff, Phase};
|
||||
use rmm_updater_linux::{
|
||||
Budget,
|
||||
files::Directory,
|
||||
services::{Service, Systemd},
|
||||
};
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
os::{
|
||||
fd::{AsRawFd, FromRawFd},
|
||||
unix::{
|
||||
fs::{FileTypeExt, MetadataExt},
|
||||
net::UnixStream,
|
||||
},
|
||||
},
|
||||
path::Path,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
const SOCKET: &str = "/run/rmm-helper-maintenance/control.sock";
|
||||
const MAX_REPLY: usize = 1536 * 1024;
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Generation {
|
||||
Original,
|
||||
Replacement,
|
||||
}
|
||||
pub struct BridgeContext<'a> {
|
||||
pub handoff: &'a Handoff,
|
||||
pub delegation: &'a UpdateDelegation,
|
||||
pub services: &'a Systemd,
|
||||
pub updater: &'a StableParticipant,
|
||||
pub clock: &'a NativeClock,
|
||||
pub budget: &'a Budget,
|
||||
pub health_public_key: [u8; 32],
|
||||
pub generation: Generation,
|
||||
}
|
||||
pub struct FullBridge<'a> {
|
||||
context: BridgeContext<'a>,
|
||||
prefix: UpdateInflightProof,
|
||||
poisoned: bool,
|
||||
finalized: bool,
|
||||
}
|
||||
pub struct Reply {
|
||||
pub proof: Option<UpdateInflightProof>,
|
||||
pub original_start: Option<Vec<u8>>,
|
||||
pub claim: Option<NativeEffectClaim>,
|
||||
pub original_outcome: Option<Vec<u8>>,
|
||||
}
|
||||
impl<'a> FullBridge<'a> {
|
||||
/// Prefix is either the exact initial atomic StopIntent or a complete
|
||||
/// retained prefix read under the original guarded ledger. Never truncate.
|
||||
pub fn new(context: BridgeContext<'a>, prefix: UpdateInflightProof) -> Result<Self> {
|
||||
let (handoff, _) = verify_inflight_proof(&prefix, context.health_public_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
handoff.bytes()? == context.handoff.bytes()?
|
||||
&& prefix.delegation == *context.delegation,
|
||||
"UPDATE_BRIDGE_PREFIX_BINDING"
|
||||
);
|
||||
Ok(Self {
|
||||
context,
|
||||
prefix,
|
||||
poisoned: false,
|
||||
finalized: false,
|
||||
})
|
||||
}
|
||||
pub fn call(&mut self, method: NativeControlOperation) -> Result<Reply> {
|
||||
let context = &self.context;
|
||||
context
|
||||
.updater
|
||||
.verify_delegation(context.delegation, context.budget)?;
|
||||
context.budget.check()?;
|
||||
let (_, journal) = verify_inflight_proof(&self.prefix, context.health_public_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
match &method {
|
||||
NativeControlOperation::ReadOriginal => {}
|
||||
NativeControlOperation::ClaimStop {
|
||||
expected_head_sha256,
|
||||
..
|
||||
} => ensure!(
|
||||
context.generation == Generation::Original
|
||||
&& journal.phase() == Phase::StopIntent
|
||||
&& *expected_head_sha256 == self.prefix.head_sha256,
|
||||
"UPDATE_BRIDGE_CLAIM_PHASE"
|
||||
),
|
||||
NativeControlOperation::ObserveStarts { .. } => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::LedgerReturnIntent,
|
||||
"UPDATE_BRIDGE_START_PHASE"
|
||||
),
|
||||
NativeControlOperation::Health { receipt } => {
|
||||
ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::ServicesStarted,
|
||||
"UPDATE_BRIDGE_HEALTH_PHASE"
|
||||
);
|
||||
journal.prepare(
|
||||
u64::try_from(context.clock.now()?.timestamp())?,
|
||||
rmm_updater_handoff::Evidence::HealthVerified {
|
||||
receipt: receipt.clone(),
|
||||
},
|
||||
)?;
|
||||
}
|
||||
NativeControlOperation::Finalize => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::HealthVerified,
|
||||
"UPDATE_BRIDGE_FINAL_PHASE"
|
||||
),
|
||||
}
|
||||
let mutating = !matches!(method, NativeControlOperation::ReadOriginal);
|
||||
ensure!(
|
||||
!self.finalized && (!mutating || !self.poisoned),
|
||||
"UPDATE_BRIDGE_NO_RETRY_OR_RESUME"
|
||||
);
|
||||
if mutating {
|
||||
self.poisoned = true;
|
||||
}
|
||||
let io_budget = IoBudget {
|
||||
clock: context.clock,
|
||||
budget: context.budget,
|
||||
finish_before: context.handoff.finish_before,
|
||||
until: Instant::now() + Duration::from_secs(5),
|
||||
};
|
||||
io_budget.remaining()?;
|
||||
ensure!(
|
||||
std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?
|
||||
.trim()
|
||||
.parse::<Uuid>()?
|
||||
== context.handoff.boot_id,
|
||||
"UPDATE_BRIDGE_BOOT_CHANGED"
|
||||
);
|
||||
let directory = Directory::open(Path::new("/run/rmm-helper-maintenance"), 0, false)?;
|
||||
let meta = std::fs::symlink_metadata(SOCKET)?;
|
||||
ensure!(
|
||||
meta.file_type().is_socket()
|
||||
&& meta.uid() == 0
|
||||
&& meta.mode() & 0o007 == 0
|
||||
&& meta.nlink() == 1,
|
||||
"UPDATE_BRIDGE_SOCKET_PERMISSIONS"
|
||||
);
|
||||
let stream = connect(&io_budget)?;
|
||||
directory.verify()?;
|
||||
let after = std::fs::symlink_metadata(SOCKET)?;
|
||||
ensure!(
|
||||
meta.dev() == after.dev()
|
||||
&& meta.ino() == after.ino()
|
||||
&& meta.mode() == after.mode()
|
||||
&& meta.uid() == after.uid()
|
||||
&& meta.gid() == after.gid(),
|
||||
"UPDATE_BRIDGE_SOCKET_CHANGED"
|
||||
);
|
||||
let helper = context.services.observe(Service::Full, context.budget)?;
|
||||
let peer = helper.authenticate_peer(&stream, context.budget)?;
|
||||
ensure!(
|
||||
peer.role == ParticipantRole::LinuxFullHelper,
|
||||
"UPDATE_BRIDGE_FULL_REQUIRED"
|
||||
);
|
||||
match context.generation {
|
||||
Generation::Original => ensure!(
|
||||
peer == context.delegation.original_helper,
|
||||
"UPDATE_BRIDGE_ORIGINAL_HELPER_CHANGED"
|
||||
),
|
||||
Generation::Replacement => ensure!(
|
||||
peer.image_sha256 == context.handoff.target.helper_sha256
|
||||
&& peer.pid != context.delegation.original_helper.pid,
|
||||
"UPDATE_BRIDGE_REPLACEMENT_HELPER_CHANGED"
|
||||
),
|
||||
}
|
||||
let request = NativeControlRequest {
|
||||
operation: "agent_update_control_v1".into(),
|
||||
schema_version: 1,
|
||||
challenge: Uuid::new_v4(),
|
||||
tenant_id: context.handoff.attempt.tenant_id,
|
||||
asset_id: context.handoff.attempt.asset_id,
|
||||
agent_id: context.handoff.attempt.agent_id,
|
||||
enrollment_generation: context.handoff.attempt.enrollment_generation,
|
||||
assignment_version: context.handoff.attempt.assignment_version,
|
||||
grant_id: context.handoff.attempt.grant_id,
|
||||
owner_nonce: context.handoff.owner_nonce,
|
||||
handoff_sha256: context.delegation.handoff_sha256.clone(),
|
||||
method,
|
||||
};
|
||||
let raw = serde_json::to_vec(&request)?;
|
||||
ensure!(
|
||||
!raw.is_empty() && raw.len() <= 32768,
|
||||
"UPDATE_BRIDGE_REQUEST_LIMIT"
|
||||
);
|
||||
let mut io = SocketIo {
|
||||
stream,
|
||||
budget: io_budget,
|
||||
};
|
||||
io.write_all(&(raw.len() as u32).to_be_bytes())?;
|
||||
io.write_all(&raw)?;
|
||||
io.flush()?;
|
||||
let mut size = [0; 4];
|
||||
io.read_exact(&mut size)?;
|
||||
let size = u32::from_be_bytes(size) as usize;
|
||||
ensure!(size > 0 && size <= MAX_REPLY, "UPDATE_BRIDGE_REPLY_LIMIT");
|
||||
let mut raw = vec![0; size];
|
||||
io.read_exact(&mut raw)?;
|
||||
let reply: NativeControlReply = serde_json::from_slice(&raw)?;
|
||||
helper.verify(context.budget)?;
|
||||
ensure!(
|
||||
reply.schema_version == 1
|
||||
&& !reply.ready
|
||||
&& reply.challenge == request.challenge
|
||||
&& reply.tenant_id == request.tenant_id
|
||||
&& reply.asset_id == request.asset_id
|
||||
&& reply.agent_id == request.agent_id
|
||||
&& reply.enrollment_generation == request.enrollment_generation
|
||||
&& reply.assignment_version == request.assignment_version
|
||||
&& reply.grant_id == request.grant_id
|
||||
&& reply.owner_nonce == request.owner_nonce
|
||||
&& reply.handoff_sha256 == request.handoff_sha256,
|
||||
"UPDATE_BRIDGE_REPLY_BINDING"
|
||||
);
|
||||
let original_start = reply
|
||||
.original_start_base64url
|
||||
.as_deref()
|
||||
.map(|v| decode(v, 512 * 1024))
|
||||
.transpose()?;
|
||||
let original_outcome = reply
|
||||
.original_outcome_base64url
|
||||
.as_deref()
|
||||
.map(|v| decode(v, 512 * 1024))
|
||||
.transpose()?;
|
||||
if matches!(request.method, NativeControlOperation::Finalize) {
|
||||
ensure!(
|
||||
reply.proof.is_none() && reply.claim.is_none() && original_start.is_none(),
|
||||
"UPDATE_BRIDGE_FINAL_REPLY"
|
||||
);
|
||||
let raw = original_outcome
|
||||
.as_ref()
|
||||
.context("UPDATE_BRIDGE_ORIGINAL_OUTCOME_MISSING")?;
|
||||
let outcome: ReconciledOutcome = serde_json::from_slice(raw)?;
|
||||
ensure!(
|
||||
canonical_outcome_bytes(&outcome)? == *raw
|
||||
&& outcome.state == "succeeded"
|
||||
&& !outcome.reboot_required,
|
||||
"UPDATE_BRIDGE_NONCANONICAL_OR_UNSUCCESSFUL_OUTCOME"
|
||||
);
|
||||
} else {
|
||||
ensure!(
|
||||
original_outcome.is_none(),
|
||||
"UPDATE_BRIDGE_UNEXPECTED_TERMINAL"
|
||||
);
|
||||
let proof = reply
|
||||
.proof
|
||||
.as_ref()
|
||||
.context("UPDATE_BRIDGE_PROOF_MISSING")?;
|
||||
let (handoff, journal) = verify_inflight_proof(proof, context.health_public_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
handoff.bytes()? == context.handoff.bytes()?
|
||||
&& proof.delegation == *context.delegation
|
||||
&& proof.events.len() >= self.prefix.events.len()
|
||||
&& proof.events[..self.prefix.events.len()] == self.prefix.events,
|
||||
"UPDATE_BRIDGE_PROOF_ROLLBACK_OR_CHANGED"
|
||||
);
|
||||
match request.method {
|
||||
NativeControlOperation::ReadOriginal => ensure!(
|
||||
reply.claim.is_none() && original_start.is_some(),
|
||||
"UPDATE_BRIDGE_ORIGINAL_REPLY"
|
||||
),
|
||||
NativeControlOperation::ClaimStop {
|
||||
ref expected_head_sha256,
|
||||
service,
|
||||
} => {
|
||||
let claim = reply
|
||||
.claim
|
||||
.as_ref()
|
||||
.context("UPDATE_BRIDGE_STOP_CLAIM_MISSING")?;
|
||||
ensure!(
|
||||
context.generation == Generation::Original
|
||||
&& journal.phase() == Phase::StopIntent
|
||||
&& claim.effect() == &NativeEffect::StopService { service }
|
||||
&& claim.grant_id() == request.grant_id
|
||||
&& claim.owner_nonce() == request.owner_nonce
|
||||
&& claim.head_sha256() == expected_head_sha256
|
||||
&& proof.head_sha256 == *expected_head_sha256
|
||||
&& original_start.is_none(),
|
||||
"UPDATE_BRIDGE_STOP_CLAIM_CHANGED"
|
||||
);
|
||||
}
|
||||
NativeControlOperation::ObserveStarts { .. } => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::ServicesStarted
|
||||
&& reply.claim.is_none()
|
||||
&& original_start.is_none(),
|
||||
"UPDATE_BRIDGE_STARTS_REPLY"
|
||||
),
|
||||
NativeControlOperation::Health { .. } => ensure!(
|
||||
context.generation == Generation::Replacement
|
||||
&& journal.phase() == Phase::HealthVerified
|
||||
&& reply.claim.is_none()
|
||||
&& original_start.is_none(),
|
||||
"UPDATE_BRIDGE_HEALTH_REPLY"
|
||||
),
|
||||
NativeControlOperation::Finalize => unreachable!(),
|
||||
}
|
||||
self.prefix = proof.clone();
|
||||
}
|
||||
context.budget.check()?;
|
||||
if matches!(request.method, NativeControlOperation::Finalize) {
|
||||
self.finalized = true;
|
||||
}
|
||||
if mutating {
|
||||
self.poisoned = false;
|
||||
}
|
||||
Ok(Reply {
|
||||
proof: reply.proof,
|
||||
claim: reply.claim,
|
||||
original_start,
|
||||
original_outcome,
|
||||
})
|
||||
}
|
||||
pub fn head_sha256(&self) -> &str {
|
||||
&self.prefix.head_sha256
|
||||
}
|
||||
pub fn observe_starts(
|
||||
&mut self,
|
||||
held: &[rmm_updater_linux::services::HeldService],
|
||||
) -> Result<Reply> {
|
||||
ensure!(
|
||||
held.len() == 3
|
||||
&& held.iter().map(|s| s.proof().service).eq([
|
||||
Service::Agent,
|
||||
Service::General,
|
||||
Service::Full
|
||||
]),
|
||||
"UPDATE_BRIDGE_STARTED_SET"
|
||||
);
|
||||
let services = held
|
||||
.iter()
|
||||
.map(|s| s.started_proof(self.context.budget))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let proofs = held.iter().map(|s| s.proof().clone()).collect::<Vec<_>>();
|
||||
let observation_sha256 = rmm_domain::jobs::hash(&serde_json::to_vec(&proofs)?);
|
||||
let reply = self.call(NativeControlOperation::ObserveStarts {
|
||||
services: services.clone(),
|
||||
observation_sha256: observation_sha256.clone(),
|
||||
})?;
|
||||
for service in held {
|
||||
service.verify(self.context.budget)?;
|
||||
}
|
||||
let proof = reply
|
||||
.proof
|
||||
.as_ref()
|
||||
.context("UPDATE_BRIDGE_STARTED_PROOF_MISSING")?;
|
||||
let raw = proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let last: rmm_updater_handoff::Event = serde_json::from_slice(
|
||||
raw.events
|
||||
.last()
|
||||
.context("UPDATE_BRIDGE_STARTED_EVENT_MISSING")?,
|
||||
)?;
|
||||
ensure!(
|
||||
matches!(last.evidence,rmm_updater_handoff::Evidence::ServicesStarted {services:actual,observation_sha256:hash,..}
|
||||
if actual==services && hash==observation_sha256),
|
||||
"UPDATE_BRIDGE_STARTED_RESULT_CHANGED"
|
||||
);
|
||||
Ok(reply)
|
||||
}
|
||||
}
|
||||
fn decode(value: &str, max: usize) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
!value.is_empty() && value.len() <= 4 * max.div_ceil(3),
|
||||
"UPDATE_BRIDGE_BASE64_LIMIT"
|
||||
);
|
||||
let raw = URL_SAFE_NO_PAD.decode(value)?;
|
||||
ensure!(
|
||||
!raw.is_empty() && raw.len() <= max && URL_SAFE_NO_PAD.encode(&raw) == value,
|
||||
"UPDATE_BRIDGE_NONCANONICAL_BASE64"
|
||||
);
|
||||
Ok(raw)
|
||||
}
|
||||
struct IoBudget<'a> {
|
||||
clock: &'a NativeClock,
|
||||
budget: &'a Budget,
|
||||
finish_before: u64,
|
||||
until: Instant,
|
||||
}
|
||||
impl IoBudget<'_> {
|
||||
fn remaining(&self) -> Result<Duration> {
|
||||
self.budget.check()?;
|
||||
let now = self.clock.now()?;
|
||||
let finish = chrono::DateTime::from_timestamp(i64::try_from(self.finish_before)?, 0)
|
||||
.context("UPDATE_BRIDGE_FINISH")?;
|
||||
let original = (finish - now).to_std()?;
|
||||
let frame = self.until.saturating_duration_since(Instant::now());
|
||||
let remaining = original.min(frame);
|
||||
ensure!(!remaining.is_zero(), "UPDATE_BRIDGE_DEADLINE");
|
||||
Ok(remaining)
|
||||
}
|
||||
}
|
||||
struct SocketIo<'a> {
|
||||
stream: UnixStream,
|
||||
budget: IoBudget<'a>,
|
||||
}
|
||||
impl Read for SocketIo<'_> {
|
||||
fn read(&mut self, bytes: &mut [u8]) -> std::io::Result<usize> {
|
||||
self.stream.set_read_timeout(Some(
|
||||
self.budget.remaining().map_err(std::io::Error::other)?,
|
||||
))?;
|
||||
self.stream.read(bytes)
|
||||
}
|
||||
}
|
||||
impl Write for SocketIo<'_> {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.stream.set_write_timeout(Some(
|
||||
self.budget.remaining().map_err(std::io::Error::other)?,
|
||||
))?;
|
||||
self.stream.write(bytes)
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
self.budget.remaining().map_err(std::io::Error::other)?;
|
||||
self.stream.flush()
|
||||
}
|
||||
}
|
||||
fn connect(budget: &IoBudget<'_>) -> Result<UnixStream> {
|
||||
// A blocking Unix connect can wait forever on a full listener backlog.
|
||||
// Use only the fixed path and an owned nonblocking socket, including during
|
||||
// connect; never switch to a network endpoint or another helper.
|
||||
let fd = unsafe {
|
||||
libc::socket(
|
||||
libc::AF_UNIX,
|
||||
libc::SOCK_STREAM | libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK,
|
||||
0,
|
||||
)
|
||||
};
|
||||
ensure!(fd >= 0, "UPDATE_BRIDGE_SOCKET_CREATE");
|
||||
let stream = unsafe { UnixStream::from_raw_fd(fd) };
|
||||
let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() };
|
||||
address.sun_family = libc::AF_UNIX as libc::sa_family_t;
|
||||
ensure!(
|
||||
SOCKET.len() < address.sun_path.len(),
|
||||
"UPDATE_BRIDGE_FIXED_PATH_LIMIT"
|
||||
);
|
||||
for (slot, byte) in address.sun_path.iter_mut().zip(SOCKET.bytes()) {
|
||||
*slot = byte as libc::c_char;
|
||||
}
|
||||
let result = unsafe {
|
||||
libc::connect(
|
||||
fd,
|
||||
(&address as *const libc::sockaddr_un).cast(),
|
||||
std::mem::size_of::<libc::sockaddr_un>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if result < 0 {
|
||||
let error = std::io::Error::last_os_error();
|
||||
ensure!(
|
||||
error.raw_os_error() == Some(libc::EINPROGRESS),
|
||||
"UPDATE_BRIDGE_CONNECT: {error}"
|
||||
);
|
||||
loop {
|
||||
let remaining = budget.remaining()?;
|
||||
let mut poll = libc::pollfd {
|
||||
fd: stream.as_raw_fd(),
|
||||
events: libc::POLLOUT,
|
||||
revents: 0,
|
||||
};
|
||||
let result = unsafe {
|
||||
libc::poll(
|
||||
&mut poll,
|
||||
1,
|
||||
i32::try_from(remaining.as_millis().min(200))?.max(1),
|
||||
)
|
||||
};
|
||||
if result < 0 {
|
||||
let error = std::io::Error::last_os_error();
|
||||
ensure!(
|
||||
error.kind() == std::io::ErrorKind::Interrupted,
|
||||
"UPDATE_BRIDGE_CONNECT_POLL: {error}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if result == 0 {
|
||||
continue;
|
||||
}
|
||||
let mut error: libc::c_int = 0;
|
||||
let mut length = std::mem::size_of_val(&error) as libc::socklen_t;
|
||||
ensure!(
|
||||
unsafe {
|
||||
libc::getsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
libc::SO_ERROR,
|
||||
(&mut error as *mut libc::c_int).cast(),
|
||||
&mut length,
|
||||
)
|
||||
} == 0
|
||||
&& error == 0
|
||||
&& poll.revents == libc::POLLOUT,
|
||||
"UPDATE_BRIDGE_CONNECT_FAILED"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
budget.remaining()?;
|
||||
stream.set_nonblocking(false)?;
|
||||
Ok(stream)
|
||||
}
|
||||
Reference in New Issue
Block a user