Implement guarded Linux updater runtime and compatible baseline tooling
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -0,0 +1,392 @@
|
||||
//! Approved artifact prefetch before a grant; Full independently admits the handoff.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use rmm_domain::{
|
||||
agent_update::{
|
||||
APPLY_CAPABILITY, ArtifactDescriptor, ArtifactRole, CHECK_CAPABILITY, Platform,
|
||||
},
|
||||
jobs::{Action, JobPayload},
|
||||
};
|
||||
use rmm_privileged_helper::{Binding, Request, control, update_admission::Accepted};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{Read, Write},
|
||||
os::unix::fs::{MetadataExt, OpenOptionsExt},
|
||||
path::{Path, PathBuf},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
const MAXIMUM: u64 = 256 * 1024 * 1024;
|
||||
const CHUNK: u64 = 512 * 1024;
|
||||
pub struct Prefetch(pub tokio::task::JoinHandle<Result<()>>);
|
||||
impl Drop for Prefetch {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
fn directory(path: &Path, create: bool) -> Result<()> {
|
||||
if create && !path.try_exists()? {
|
||||
std::fs::DirBuilder::new().mode(0o700).create(path)?;
|
||||
}
|
||||
let m = fs::symlink_metadata(path)?;
|
||||
ensure!(
|
||||
m.is_dir() && !m.file_type().is_symlink() && m.uid() == 993 && m.mode() & 0o777 == 0o700,
|
||||
"UPDATE_PREFETCH_DIRECTORY"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
fn parent(job: &JobPayload) -> Result<PathBuf> {
|
||||
ensure!(
|
||||
unsafe { libc::geteuid() } == 993,
|
||||
"UPDATE_PREFETCH_AGENT_UID"
|
||||
);
|
||||
let base = Path::new("/var/lib/rmm-agent-remote");
|
||||
directory(base, false)?;
|
||||
let base = base.join("update-downloads");
|
||||
directory(&base, true)?;
|
||||
let base = base.join(job.job_id.to_string());
|
||||
directory(&base, true)?;
|
||||
let base = base.join(job.attempt_id.to_string());
|
||||
directory(&base, true)?;
|
||||
Ok(base)
|
||||
}
|
||||
fn label(a: &ArtifactDescriptor) -> &'static str {
|
||||
match a.role {
|
||||
ArtifactRole::Agent => "agent.bin",
|
||||
ArtifactRole::Helper => "helper.bin",
|
||||
}
|
||||
}
|
||||
fn checked(path: &Path, a: &ArtifactDescriptor) -> Result<()> {
|
||||
let mut f = OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
|
||||
.open(path)?;
|
||||
let m = f.metadata()?;
|
||||
ensure!(
|
||||
m.is_file()
|
||||
&& m.uid() == 993
|
||||
&& m.mode() & 0o777 == 0o600
|
||||
&& m.nlink() == 1
|
||||
&& m.len() == a.bytes,
|
||||
"UPDATE_PREFETCH_FILE"
|
||||
);
|
||||
let mut h = Sha256::new();
|
||||
let mut n = 0;
|
||||
let mut buf = [0; 65536];
|
||||
loop {
|
||||
let size = f.read(&mut buf)?;
|
||||
if size == 0 {
|
||||
break;
|
||||
}
|
||||
n += size as u64;
|
||||
ensure!(n <= a.bytes, "UPDATE_PREFETCH_LENGTH");
|
||||
h.update(&buf[..size]);
|
||||
}
|
||||
ensure!(
|
||||
n == a.bytes && format!("{:x}", h.finalize()) == a.sha256,
|
||||
"UPDATE_PREFETCH_DIGEST"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub async fn prefetch(client: reqwest::Client, base: String, job: JobPayload) -> Result<()> {
|
||||
let Action::UpdateAgent { update } = &job.action else {
|
||||
anyhow::bail!("UPDATE_ACTION_REQUIRED")
|
||||
};
|
||||
let manifest = update.manifest().map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
manifest.validate() && manifest.platform == Platform::LinuxX64,
|
||||
"UPDATE_PREFETCH_MANIFEST"
|
||||
);
|
||||
let pjob = job.clone();
|
||||
let folder = tokio::task::spawn_blocking(move || parent(&pjob)).await??;
|
||||
let deadline = Instant::now() + Duration::from_secs(600);
|
||||
for artifact in manifest.artifacts {
|
||||
ensure!(
|
||||
artifact.bytes > 0 && artifact.bytes <= MAXIMUM,
|
||||
"UPDATE_PREFETCH_LIMIT"
|
||||
);
|
||||
let path = folder.join(label(&artifact));
|
||||
let existing = path.clone();
|
||||
let desc = artifact.clone();
|
||||
if tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
if existing.try_exists()? {
|
||||
checked(&existing, &desc)?;
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
})
|
||||
.await??
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let stage = folder.join(format!("{}.partial", label(&artifact)));
|
||||
let stage_path = stage.clone();
|
||||
let mut file = tokio::task::spawn_blocking(move || {
|
||||
OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
|
||||
.open(stage_path)
|
||||
})
|
||||
.await??;
|
||||
let mut offset = 0;
|
||||
let mut hash = Sha256::new();
|
||||
while offset < artifact.bytes {
|
||||
ensure!(
|
||||
Instant::now() < deadline && chrono::Utc::now() < job.start_before,
|
||||
"UPDATE_PREFETCH_EXPIRED"
|
||||
);
|
||||
let length = CHUNK.min(artifact.bytes - offset);
|
||||
let url = format!(
|
||||
"{base}/agent/v1/update-artifacts/{}?job_id={}&attempt_id={}&offset={offset}&length={length}",
|
||||
artifact.artifact_id, job.job_id, job.attempt_id
|
||||
);
|
||||
let mut response = client.get(&url).send().await?;
|
||||
let partial = offset != 0 || length != artifact.bytes;
|
||||
ensure!(
|
||||
response.url().as_str() == url
|
||||
&& response.status()
|
||||
== if partial {
|
||||
reqwest::StatusCode::PARTIAL_CONTENT
|
||||
} else {
|
||||
reqwest::StatusCode::OK
|
||||
},
|
||||
"UPDATE_PREFETCH_HTTP"
|
||||
);
|
||||
let h = response.headers();
|
||||
ensure!(
|
||||
h.get("x-artifact-sha256").and_then(|v| v.to_str().ok())
|
||||
== Some(artifact.sha256.as_str())
|
||||
&& h.get("content-type").and_then(|v| v.to_str().ok())
|
||||
== Some("application/octet-stream")
|
||||
&& !h.contains_key("content-encoding")
|
||||
&& response.content_length() == Some(length),
|
||||
"UPDATE_PREFETCH_HEADERS"
|
||||
);
|
||||
if partial {
|
||||
ensure!(
|
||||
h.get("content-range").and_then(|v| v.to_str().ok())
|
||||
== Some(
|
||||
format!(
|
||||
"bytes {}-{}/{}",
|
||||
offset,
|
||||
offset + length - 1,
|
||||
artifact.bytes
|
||||
)
|
||||
.as_str()
|
||||
),
|
||||
"UPDATE_PREFETCH_RANGE"
|
||||
);
|
||||
} else {
|
||||
ensure!(!h.contains_key("content-range"), "UPDATE_PREFETCH_RANGE");
|
||||
}
|
||||
let mut bytes = Vec::with_capacity(length as usize);
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
ensure!(
|
||||
bytes.len() + chunk.len() <= length as usize,
|
||||
"UPDATE_PREFETCH_BODY_LIMIT"
|
||||
);
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
ensure!(
|
||||
bytes.len() == length as usize && Instant::now() < deadline,
|
||||
"UPDATE_PREFETCH_BODY_LENGTH"
|
||||
);
|
||||
hash.update(&bytes);
|
||||
file = tokio::task::spawn_blocking(move || -> Result<File> {
|
||||
file.write_all(&bytes)?;
|
||||
Ok(file)
|
||||
})
|
||||
.await??;
|
||||
offset += length;
|
||||
}
|
||||
ensure!(
|
||||
format!("{:x}", hash.finalize()) == artifact.sha256,
|
||||
"UPDATE_PREFETCH_DIGEST"
|
||||
);
|
||||
let dir = folder.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
file.sync_all()?;
|
||||
drop(file);
|
||||
fs::hard_link(&stage, &path)?;
|
||||
fs::remove_file(stage)?;
|
||||
File::open(dir)?.sync_all()?;
|
||||
checked(&path, &artifact)
|
||||
})
|
||||
.await??;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// No Outcome is invented on acceptance; loss after write stays ambiguous.
|
||||
pub fn handoff(binding: &Binding, request: &Request, grant: uuid::Uuid) -> Result<Accepted> {
|
||||
let socket = std::env::var_os("RMM_FULL_MAINTENANCE_HELPER_SOCKET")
|
||||
.context("UPDATE_FULL_SOCKET_REQUIRED")?;
|
||||
let pin =
|
||||
std::env::var_os("RMM_FULL_MAINTENANCE_POLICY_FILE").context("UPDATE_FULL_PIN_REQUIRED")?;
|
||||
let pin = control::GeneralPin::read(Path::new(&pin), binding)?;
|
||||
ensure!(
|
||||
pin.profile == "full_maintenance_v1",
|
||||
"UPDATE_FULL_PROFILE_REQUIRED"
|
||||
);
|
||||
control::general_probe(Path::new(&socket), binding, &pin)?;
|
||||
let ack: Accepted =
|
||||
control::exchange(Path::new(&socket), request, Duration::from_secs(50), 4096)?;
|
||||
ack.matches(request, grant)?;
|
||||
Ok(ack)
|
||||
}
|
||||
|
||||
/// Opaque public metadata authenticated by the current protected Full peer,
|
||||
/// with complete guarded history verified there. No caller JSON constructor.
|
||||
#[derive(Clone)]
|
||||
pub struct GuardedRecord {
|
||||
raw: Vec<u8>,
|
||||
observation: rmm_domain::agent_update::AgentUpdateObservation,
|
||||
}
|
||||
impl GuardedRecord {
|
||||
pub fn bytes(&self, policy_raw: &[u8], keys_raw: &[u8]) -> Result<&[u8]> {
|
||||
use rmm_domain::agent_update::ObservationQuality;
|
||||
let o = &self.observation;
|
||||
let age = chrono::Utc::now() - o.observed_at;
|
||||
let p = o
|
||||
.predecessor
|
||||
.as_ref()
|
||||
.context("UPDATE_CATALOG_PREDECESSOR")?;
|
||||
let u = o.updater.as_ref().context("UPDATE_CATALOG_UPDATER")?;
|
||||
ensure!(
|
||||
o.validate()
|
||||
&& o.quality == ObservationQuality::Valid
|
||||
&& o.platform == Platform::LinuxX64
|
||||
&& age >= chrono::Duration::zero()
|
||||
&& age <= chrono::Duration::seconds(90)
|
||||
&& rmm_domain::jobs::hash(policy_raw) == u.policy_sha256
|
||||
&& rmm_domain::jobs::hash(keys_raw) == u.keyring_sha256
|
||||
&& rmm_domain::jobs::hash(&self.raw) == p.installed_record_sha256,
|
||||
"UPDATE_CATALOG_GUARDED_RECORD"
|
||||
);
|
||||
Ok(&self.raw)
|
||||
}
|
||||
}
|
||||
/// Passive cached Full observations only; never a live MCP collection.
|
||||
type Observation = rmm_domain::agent_update::AgentUpdateObservation;
|
||||
struct Observing {
|
||||
receiver: tokio::sync::oneshot::Receiver<Result<(Observation, Vec<u8>)>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
impl Drop for Observing {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
#[derive(Default)]
|
||||
pub struct Observer {
|
||||
current: Option<Observation>,
|
||||
record: Option<GuardedRecord>,
|
||||
pending: Option<Observing>,
|
||||
}
|
||||
impl Observer {
|
||||
pub fn cached(&self) -> Option<&Observation> {
|
||||
let value = self.current.as_ref()?;
|
||||
let age = chrono::Utc::now() - value.observed_at;
|
||||
(value.validate()
|
||||
&& age >= chrono::Duration::zero()
|
||||
&& age <= chrono::Duration::seconds(90))
|
||||
.then_some(value)
|
||||
}
|
||||
pub fn poll(
|
||||
&mut self,
|
||||
binding: Option<&Binding>,
|
||||
candidate: Option<(rmm_domain::agent_update::ReleaseEnvelope, u64)>,
|
||||
) -> Observation {
|
||||
if let Some(pending) = &mut self.pending {
|
||||
match pending.receiver.try_recv() {
|
||||
Ok(Ok((value, raw))) if value.validate() => {
|
||||
self.record = Some(GuardedRecord {
|
||||
raw,
|
||||
observation: value.clone(),
|
||||
});
|
||||
self.current = Some(value);
|
||||
self.pending = None;
|
||||
}
|
||||
Ok(_) | Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
|
||||
self.current = None;
|
||||
self.record = None;
|
||||
self.pending = None;
|
||||
}
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {}
|
||||
}
|
||||
}
|
||||
if self.pending.is_none()
|
||||
&& let Some(binding) = binding.cloned()
|
||||
{
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
let (envelope, generation) =
|
||||
candidate.map_or((None, None), |(e, g)| (Some(e), Some(g)));
|
||||
let value = rmm_privileged_helper::update_admission::observe_record(
|
||||
Path::new("/run/rmm-helper-maintenance/control.sock"),
|
||||
&binding,
|
||||
envelope,
|
||||
generation,
|
||||
);
|
||||
let _ = sender.send(value);
|
||||
});
|
||||
self.pending = Some(Observing { receiver, task });
|
||||
}
|
||||
self.cached()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Observation::not_configured(Platform::LinuxX64, chrono::Utc::now()))
|
||||
}
|
||||
pub fn catalog_record(&self) -> Option<GuardedRecord> {
|
||||
self.cached()?;
|
||||
self.record.clone()
|
||||
}
|
||||
pub fn capabilities(&self) -> Vec<String> {
|
||||
let Some(value) = self
|
||||
.cached()
|
||||
.filter(|v| v.quality == rmm_domain::agent_update::ObservationQuality::Valid)
|
||||
else {
|
||||
return vec![];
|
||||
};
|
||||
let (Some(updater), Some(predecessor)) = (&value.updater, &value.predecessor) else {
|
||||
return vec![];
|
||||
};
|
||||
vec![
|
||||
APPLY_CAPABILITY.into(),
|
||||
CHECK_CAPABILITY.into(),
|
||||
"update_agent".into(),
|
||||
"root:update_agent".into(),
|
||||
"helper.root.v1".into(),
|
||||
format!("updater.binary.sha256:{}", updater.binary_sha256),
|
||||
format!("updater.policy.sha256:{}", updater.policy_sha256),
|
||||
format!("helper.binary.sha256:{}", predecessor.helper_sha256),
|
||||
]
|
||||
}
|
||||
}
|
||||
pub fn supported(binding: &Binding, job: &JobPayload) -> bool {
|
||||
(|| -> Result<bool> {
|
||||
ensure!(
|
||||
unsafe { libc::geteuid() } == 993 && job.execution_identity == "root",
|
||||
"UPDATE_AGENT_PRINCIPAL"
|
||||
);
|
||||
let Action::UpdateAgent { update } = &job.action else {
|
||||
return Ok(false);
|
||||
};
|
||||
let observation = rmm_privileged_helper::update_admission::observe(
|
||||
Path::new("/run/rmm-helper-maintenance/control.sock"),
|
||||
binding,
|
||||
Some(update.release_envelope.clone()),
|
||||
Some(update.updater.keyring_generation),
|
||||
)?;
|
||||
Ok(observation.validate()
|
||||
&& observation.quality == rmm_domain::agent_update::ObservationQuality::Valid
|
||||
&& observation.predecessor.as_ref() == Some(&update.predecessor)
|
||||
&& observation.updater.as_ref() == Some(&update.updater)
|
||||
&& observation.candidate_previews.iter().any(|c| {
|
||||
c.manifest_sha256 == update.manifest_sha256
|
||||
&& c.peer_pin_transition == update.peer_pin_transition
|
||||
}))
|
||||
})()
|
||||
.unwrap_or(false)
|
||||
}
|
||||
@@ -62,7 +62,7 @@ pub fn observation(report: Report, collected_at: DateTime<Utc>) -> serde_json::V
|
||||
"error_code":error,"data":report})
|
||||
}
|
||||
struct Pending {
|
||||
receiver: oneshot::Receiver<Report>,
|
||||
receiver: oneshot::Receiver<(Report, Option<ReleaseEnvelope>)>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
impl Drop for Pending {
|
||||
@@ -73,10 +73,21 @@ impl Drop for Pending {
|
||||
#[derive(Default)]
|
||||
pub struct Checker {
|
||||
report: Option<Report>,
|
||||
release: Option<ReleaseEnvelope>,
|
||||
started: Option<Instant>,
|
||||
pending: Option<Pending>,
|
||||
}
|
||||
impl Checker {
|
||||
/// Exact authenticated envelope, for the independent read-only Full
|
||||
/// verifier. This cache never confers admission or a capability itself.
|
||||
pub fn candidate_envelope(&self, now: DateTime<Utc>) -> Option<(ReleaseEnvelope, u64)> {
|
||||
let report = self.cached(now)?;
|
||||
let candidate = report.candidate?;
|
||||
let envelope = self.release.as_ref()?;
|
||||
let decoded = envelope.decode().ok()?;
|
||||
(hash(&decoded.payload_bytes) == candidate.manifest_sha256)
|
||||
.then(|| (envelope.clone(), candidate.keyring_generation))
|
||||
}
|
||||
pub fn cached(&self, now: DateTime<Utc>) -> Option<Report> {
|
||||
self.report
|
||||
.as_ref()
|
||||
@@ -90,17 +101,20 @@ impl Checker {
|
||||
binding: Option<&Binding>,
|
||||
clock: TrustedClock,
|
||||
address: SocketAddr,
|
||||
guarded: Option<crate::agent_update_apply::GuardedRecord>,
|
||||
client: impl FnOnce() -> Result<reqwest::Client>,
|
||||
) -> Report {
|
||||
if let Some(pending) = &mut self.pending {
|
||||
match pending.receiver.try_recv() {
|
||||
Ok(report) => {
|
||||
Ok((report, release)) => {
|
||||
self.report = Some(report);
|
||||
self.release = release;
|
||||
self.pending.take();
|
||||
}
|
||||
Err(oneshot::error::TryRecvError::Empty) => {}
|
||||
Err(oneshot::error::TryRecvError::Closed) => {
|
||||
self.report = Some(empty(Status::Unavailable));
|
||||
self.release = None;
|
||||
self.pending.take();
|
||||
}
|
||||
}
|
||||
@@ -123,6 +137,7 @@ impl Checker {
|
||||
Path::new(POLICY),
|
||||
Path::new(KEYRING),
|
||||
Path::new(RECORD),
|
||||
guarded.as_ref(),
|
||||
)
|
||||
})
|
||||
.transpose();
|
||||
@@ -148,7 +163,7 @@ impl Checker {
|
||||
let report = result
|
||||
.ok()
|
||||
.and_then(Result::ok)
|
||||
.unwrap_or_else(|| empty(Status::Unavailable));
|
||||
.unwrap_or_else(|| (empty(Status::Unavailable), None));
|
||||
let _ = sender.send(report);
|
||||
});
|
||||
self.pending = Some(Pending { receiver, task });
|
||||
@@ -254,7 +269,13 @@ struct Trust {
|
||||
generation: u64,
|
||||
}
|
||||
impl Trust {
|
||||
fn load(binding: &Binding, policy: &Path, keys: &Path, record: &Path) -> Result<Option<Self>> {
|
||||
fn load(
|
||||
binding: &Binding,
|
||||
policy: &Path,
|
||||
keys: &Path,
|
||||
record: &Path,
|
||||
guarded: Option<&crate::agent_update_apply::GuardedRecord>,
|
||||
) -> Result<Option<Self>> {
|
||||
let result = (|| -> Result<Self> {
|
||||
let policy = Protected::read(policy, 65536)?;
|
||||
let (local, _) =
|
||||
@@ -264,12 +285,22 @@ impl Trust {
|
||||
"UPDATE_CATALOG_FIXED_ANCHOR_PATHS"
|
||||
);
|
||||
let keys = Protected::read(keys, 16384)?;
|
||||
let record = Protected::read(record, 32768)?;
|
||||
let result = Self::verify(binding, &policy.raw, &keys.raw, &record.raw)?;
|
||||
Ok(Self {
|
||||
files: vec![policy, keys, record],
|
||||
..result
|
||||
})
|
||||
if let Some(guarded) = guarded {
|
||||
let raw = guarded.bytes(&policy.raw, &keys.raw)?;
|
||||
let result =
|
||||
Self::verify_guarded(binding, &policy.raw, &keys.raw, raw, Some(guarded))?;
|
||||
Ok(Self {
|
||||
files: vec![policy, keys],
|
||||
..result
|
||||
})
|
||||
} else {
|
||||
let record = Protected::read(record, 32768)?;
|
||||
let result = Self::verify(binding, &policy.raw, &keys.raw, &record.raw)?;
|
||||
Ok(Self {
|
||||
files: vec![policy, keys, record],
|
||||
..result
|
||||
})
|
||||
}
|
||||
})();
|
||||
match result {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
@@ -288,6 +319,15 @@ impl Trust {
|
||||
policy_raw: &[u8],
|
||||
keys_raw: &[u8],
|
||||
record_raw: &[u8],
|
||||
) -> Result<Self> {
|
||||
Self::verify_guarded(binding, policy_raw, keys_raw, record_raw, None)
|
||||
}
|
||||
fn verify_guarded(
|
||||
binding: &Binding,
|
||||
policy_raw: &[u8],
|
||||
keys_raw: &[u8],
|
||||
record_raw: &[u8],
|
||||
guarded: Option<&crate::agent_update_apply::GuardedRecord>,
|
||||
) -> Result<Self> {
|
||||
let (policy, _) = LocalUpdatePolicy::from_bytes(policy_raw).map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
@@ -322,14 +362,23 @@ impl Trust {
|
||||
.initial_baseline
|
||||
.decode()
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
// Only the independently signed bootstrap can currently authenticate
|
||||
// the floor here. Later Updated records require the separate guarded
|
||||
// Full history observer; do not guess a floor from parsed root JSON.
|
||||
ensure!(
|
||||
baseline.payload_bytes == record_raw
|
||||
&& hash(record_raw) == policy.initial_record_sha256,
|
||||
"UPDATE_CATALOG_GUARDED_HISTORY_REQUIRED"
|
||||
hash(&baseline.payload_bytes) == policy.initial_record_sha256,
|
||||
"UPDATE_CATALOG_BASELINE_BYTES"
|
||||
);
|
||||
if let Some(guarded) = guarded {
|
||||
ensure!(
|
||||
guarded.bytes(policy_raw, keys_raw)? == record_raw,
|
||||
"UPDATE_CATALOG_GUARDED_HISTORY_REQUIRED"
|
||||
);
|
||||
} else {
|
||||
ensure!(
|
||||
baseline.payload_bytes == record_raw,
|
||||
"UPDATE_CATALOG_GUARDED_HISTORY_REQUIRED"
|
||||
);
|
||||
}
|
||||
let installed =
|
||||
InstalledReleaseRecord::from_bytes(record_raw).map_err(anyhow::Error::msg)?;
|
||||
let signer = keys
|
||||
.keys
|
||||
.iter()
|
||||
@@ -347,8 +396,7 @@ impl Trust {
|
||||
Verifier::new_without_digest(&public)?.verify_oneshot(&baseline.signature, &signed)?,
|
||||
"UPDATE_CATALOG_BASELINE_SIGNATURE"
|
||||
);
|
||||
let decoded = baseline
|
||||
.record
|
||||
let decoded = installed
|
||||
.release_envelope
|
||||
.decode()
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
@@ -358,7 +406,7 @@ impl Trust {
|
||||
protocol: decoded.manifest.protocol_major,
|
||||
state: decoded.manifest.state_schema,
|
||||
generation: keys.generation,
|
||||
record: baseline.record,
|
||||
record: installed,
|
||||
};
|
||||
result.release(&result.record.release_envelope, result.record.activated_at)?;
|
||||
Ok(result)
|
||||
@@ -458,14 +506,17 @@ async fn request(
|
||||
address: SocketAddr,
|
||||
trust: &Trust,
|
||||
clock: &TrustedClock,
|
||||
) -> Result<Report> {
|
||||
) -> Result<(Report, Option<ReleaseEnvelope>)> {
|
||||
ensure!(clock.trusted(), "UPDATE_CATALOG_CLOCK_UNTRUSTED");
|
||||
let bytes = fetch(client, address).await?;
|
||||
ensure!(clock.trusted(), "UPDATE_CATALOG_CLOCK_UNTRUSTED");
|
||||
for file in &trust.files {
|
||||
file.recheck()?;
|
||||
}
|
||||
trust.response(&bytes, Utc::now())
|
||||
let report = trust.response(&bytes, Utc::now())?;
|
||||
let response: CatalogResponse = serde_json::from_slice(&bytes)?;
|
||||
let release = response.release.map(|r| r.release_envelope);
|
||||
Ok((report, release))
|
||||
}
|
||||
|
||||
async fn fetch(client: &reqwest::Client, address: SocketAddr) -> Result<Vec<u8>> {
|
||||
@@ -750,6 +801,7 @@ mod tests {
|
||||
None,
|
||||
TrustedClock::new(),
|
||||
"127.0.0.1:1".parse().unwrap(),
|
||||
None,
|
||||
|| panic!("missing anchor must not load device TLS keys"),
|
||||
);
|
||||
assert_eq!(report.status, Status::NotConfigured);
|
||||
@@ -757,6 +809,7 @@ mod tests {
|
||||
None,
|
||||
TrustedClock::new(),
|
||||
"127.0.0.1:1".parse().unwrap(),
|
||||
None,
|
||||
|| panic!("cadence must not create client"),
|
||||
);
|
||||
assert_eq!(report.checked_at, again.checked_at);
|
||||
|
||||
@@ -222,7 +222,8 @@ pub fn context(
|
||||
let key = update::job_health_public_key(&job).map_err(anyhow::Error::msg)?;
|
||||
let (header, journal) =
|
||||
update::verify_inflight_proof(proof, key).map_err(anyhow::Error::msg)?;
|
||||
if journal.phase() != Phase::ServicesStarted {
|
||||
let rollback = journal.phase() == Phase::RollbackServicesStarted;
|
||||
if journal.phase() != Phase::ServicesStarted && !rollback {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
@@ -241,7 +242,8 @@ pub fn context(
|
||||
.context("UPDATE_HEALTH_SERVICES_STARTED")?,
|
||||
)?;
|
||||
ensure!(
|
||||
matches!(event.evidence, Evidence::ServicesStarted { .. }),
|
||||
matches!(event.evidence, Evidence::ServicesStarted { .. }) && !rollback
|
||||
|| matches!(event.evidence, Evidence::RollbackServicesStarted { .. }) && rollback,
|
||||
"UPDATE_HEALTH_SERVICES_STARTED"
|
||||
);
|
||||
let (last, generation, fault, pending): (Option<String>, u64, Option<String>, Option<String>) =
|
||||
@@ -424,8 +426,14 @@ pub async fn request(
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
journal.prepare(
|
||||
u64::try_from(now.timestamp())?,
|
||||
Evidence::HealthVerified {
|
||||
receipt: signed.clone(),
|
||||
if journal.phase() == Phase::RollbackServicesStarted {
|
||||
Evidence::RollbackHealthVerified {
|
||||
receipt: signed.clone(),
|
||||
}
|
||||
} else {
|
||||
Evidence::HealthVerified {
|
||||
receipt: signed.clone(),
|
||||
}
|
||||
},
|
||||
)?;
|
||||
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
||||
|
||||
@@ -24,6 +24,9 @@ pub fn supported(job: &JobPayload) -> bool {
|
||||
)
|
||||
}
|
||||
pub fn supported_with_binding(job: &JobPayload, binding: Option<&Binding>) -> bool {
|
||||
if matches!(job.action, Action::UpdateAgent { .. }) {
|
||||
return binding.is_some_and(|b| crate::agent_update_apply::supported(b, job));
|
||||
}
|
||||
if rmm_privileged_helper::apt::is_apt_action(job) {
|
||||
// SAFETY: geteuid has no preconditions.
|
||||
return unsafe { libc::geteuid() } != 0
|
||||
|
||||
+51
-18
@@ -1,3 +1,4 @@
|
||||
mod agent_update_apply;
|
||||
mod agent_update_catalog;
|
||||
mod agent_update_health;
|
||||
mod apt;
|
||||
@@ -62,6 +63,7 @@ struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
||||
certificates: certificate_rotation::RotationStore,
|
||||
rotation_policy: certificate_rotation::RotationPolicy,
|
||||
catalog: agent_update_catalog::Checker,
|
||||
update_observer: agent_update_apply::Observer,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
@@ -428,6 +430,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
|| binding.verify(&request, Utc::now()).is_err()
|
||||
{
|
||||
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
|
||||
} else if matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) {
|
||||
match agent_update_apply::handoff(&binding, &request, grant.grant_id) {
|
||||
// Full now owns this original. The stable updater may
|
||||
// stop us. Reconciliation alone supplies its final result.
|
||||
Ok(_) => return,
|
||||
Err(_) => rmm_privileged_helper::Outcome::unknown("UPDATE_HANDOFF_UNCONFIRMED"),
|
||||
}
|
||||
} else if job.execution_identity == "root" {
|
||||
apt::invoke_with_control(&binding, &job, &request, || {
|
||||
ensure!(
|
||||
@@ -483,6 +492,9 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
capabilities.extend(apt::capabilities(binding));
|
||||
capabilities.extend(helper_sync::general_capabilities(binding));
|
||||
if self.clock.trusted() {
|
||||
capabilities.extend(self.update_observer.capabilities());
|
||||
}
|
||||
}
|
||||
capabilities.sort();
|
||||
capabilities.dedup();
|
||||
@@ -530,18 +542,15 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if !storage::passive_collection_ready(&self.db)? {
|
||||
return Ok(());
|
||||
}
|
||||
// Until the separately qualified protected updater/helper observer is
|
||||
// provisioned, report its real absence; never infer apply readiness from
|
||||
// this schema support or the agent's own code/build version.
|
||||
let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured(
|
||||
rmm_domain::agent_update::Platform::LinuxX64,
|
||||
Utc::now(),
|
||||
);
|
||||
let address = self.address;
|
||||
let ca = &self.ca;
|
||||
let certificates = &self.certificates;
|
||||
self.catalog
|
||||
.poll(self.binding.as_ref(), self.clock.clone(), address, || {
|
||||
self.catalog.poll(
|
||||
self.binding.as_ref(),
|
||||
self.clock.clone(),
|
||||
address,
|
||||
self.update_observer.catalog_record(),
|
||||
|| {
|
||||
let identity = certificates.current(Utc::now().timestamp())?;
|
||||
https_channel::mtls_client(
|
||||
address,
|
||||
@@ -549,7 +558,12 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)
|
||||
});
|
||||
},
|
||||
);
|
||||
let observation = self.update_observer.poll(
|
||||
self.binding.as_ref(),
|
||||
self.catalog.candidate_envelope(Utc::now()),
|
||||
);
|
||||
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
|
||||
let envelope = self.envelope(
|
||||
generation,
|
||||
@@ -660,6 +674,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
let mut first_heartbeat = true;
|
||||
let mut health_task: Option<PendingUpdateHealth> = None;
|
||||
let mut health_inventory_prefix: Option<String> = None;
|
||||
let mut update_prefetch: Option<(JobOffer, agent_update_apply::Prefetch)> = None;
|
||||
let mut prefetched = std::collections::HashSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(result) = self.inventory_rx.recv() => {
|
||||
@@ -742,6 +758,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
_ = tick.tick() => {
|
||||
self.replay(&ws, generation).await?;
|
||||
if update_prefetch.as_ref().is_some_and(|(_,task)|task.0.is_finished()) {
|
||||
let (offer, mut task)=update_prefetch.take().context("UPDATE_PREFETCH_REQUIRED")?;
|
||||
match (&mut task.0).await {
|
||||
Ok(Ok(())) => {prefetched.insert((offer.job_id.clone(),offer.attempt_id.clone()));},
|
||||
_ => {self.record(generation,&offer,"",jobs::no_start("failed","UPDATE_ARTIFACT_PREFETCH_FAILED"))?;needs_sync=true;}
|
||||
}
|
||||
}
|
||||
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
||||
let update=self.update_proof()?;
|
||||
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
|
||||
@@ -749,7 +772,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if let Some(proof)=&update {
|
||||
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
|
||||
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
|
||||
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..}|rmm_updater_handoff::Evidence::RollbackServicesStarted{..})) {
|
||||
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
|
||||
// Fixed update health includes one genuinely new
|
||||
// normal base sample, never a fresh timestamp on old data.
|
||||
@@ -760,6 +783,13 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Planning pins must be refreshed before the 90-second stored-data
|
||||
// planning limit. One inflight blocking collector bounds pressure;
|
||||
// heartbeats never perform this metadata work.
|
||||
if update_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_update_observation(generation)?;
|
||||
update_at = Instant::now();
|
||||
}
|
||||
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
||||
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
@@ -773,6 +803,15 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
} else if !self.ready(generation,job.execution_identity=="root") {
|
||||
needs_sync=true;
|
||||
} else if Utc::now() >= job.not_before {
|
||||
if matches!(job.action,rmm_domain::jobs::Action::UpdateAgent{..}) && !prefetched.contains(&(offer.job_id.clone(),offer.attempt_id.clone())) {
|
||||
if update_prefetch.is_none() {
|
||||
let identity=self.certificates.current(Utc::now().timestamp())?;
|
||||
let client=https_channel::mtls_client(self.address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?;
|
||||
let base=format!("https://{}:{}",https_channel::server_name()?,self.address.port());
|
||||
update_prefetch=Some((offer.clone(),agent_update_apply::Prefetch(tokio::spawn(agent_update_apply::prefetch(client,base,job)))));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Reserve before send. A lost response must never cause
|
||||
// a second permit request for this attempt.
|
||||
jobs::reserve_request(&self.db, &offer)?;
|
||||
@@ -808,13 +847,6 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
heartbeat_at = Instant::now();
|
||||
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
}
|
||||
// Planning pins must be refreshed before the 90-second stored-data
|
||||
// planning limit. One inflight blocking collector bounds pressure;
|
||||
// heartbeats never perform this metadata work.
|
||||
if update_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_update_observation(generation)?;
|
||||
update_at = Instant::now();
|
||||
}
|
||||
if inventory_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_inventory();
|
||||
inventory_at = Instant::now();
|
||||
@@ -1017,6 +1049,7 @@ async fn main() -> Result<()> {
|
||||
certificates,
|
||||
rotation_policy,
|
||||
catalog: agent_update_catalog::Checker::default(),
|
||||
update_observer: agent_update_apply::Observer::default(),
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1_u64;
|
||||
|
||||
@@ -151,6 +151,7 @@ fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat()
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
catalog: agent_update_catalog::Checker::default(),
|
||||
update_observer: agent_update_apply::Observer::default(),
|
||||
};
|
||||
// A forged local consumption/start cannot become a purported grant result.
|
||||
agent
|
||||
@@ -539,6 +540,7 @@ async fn run(mode: wire_channel::Mode, trusted: bool, forged: bool, rounds: u64)
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
catalog: agent_update_catalog::Checker::default(),
|
||||
update_observer: agent_update_apply::Observer::default(),
|
||||
};
|
||||
for _ in 0..rounds {
|
||||
assert!(
|
||||
|
||||
Reference in New Issue
Block a user