Files
rust-discovery/crates/agent/src/agent_update_health.rs
T
2026-11-14 23:51:59 +01:00

466 lines
18 KiB
Rust

//! Evidence-only client: ORIGINAL retained inventory and two real signed rounds.
//! No collection, job, grant, readiness, helper effect or caller ACK is produced.
use anyhow::{Context, Result, ensure};
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
use chrono::{DateTime, Utc};
use prost::Message;
use rmm_agent_core::{
auth::Binding,
reconcile::{Channel, verify_response},
update_health_inventory,
};
use rmm_domain::{
agent_update::{self as update, UpdateInflightProof},
jobs::{JobPayload, hash},
reconciliation::ReconcileDecisionPayload,
};
use rmm_protocol::{Envelope, JobOffer, Payload, SyncRequest, SyncResponse};
use rmm_updater_handoff::{Event, Evidence, Handoff, HealthReceipt, Phase, SignedHealth};
use rusqlite::{Connection, OptionalExtension, params};
use serde::Serialize;
use std::{
net::SocketAddr,
time::{Duration, Instant},
};
use uuid::Uuid;
#[derive(Serialize)]
struct HealthRequest {
proof: UpdateInflightProof,
applied_sync_request_base64url: String,
confirming_sync_request_base64url: String,
inventory_envelope_base64url: String,
}
/// Opaque; constructed only by examining retained, verified local evidence.
pub struct HealthContext {
input: HealthRequest,
header: Handoff,
health_key: [u8; 32],
inventory_id: Uuid,
applied: ReconcileDecisionPayload,
applied_response_sha256: String,
confirming: ReconcileDecisionPayload,
observed_now: DateTime<Utc>,
prepared_at: Instant,
deadline: Instant,
}
impl HealthContext {
pub fn handoff(&self) -> &Handoff {
&self.header
}
pub fn health_public_key(&self) -> [u8; 32] {
self.health_key
}
}
struct StoredSync {
request: SyncRequest,
raw: Vec<u8>,
response: SyncResponse,
decision: ReconcileDecisionPayload,
}
fn stored_sync(
db: &Connection,
id: Uuid,
binding: &Binding,
channel: &Channel,
now: DateTime<Utc>,
) -> Result<Option<StoredSync>> {
// At most 64 recent control rounds, incrementally read; no unbounded journal scan.
let mut q=db.prepare("SELECT request,envelope,response FROM reconcile_requests WHERE response IS NOT NULL ORDER BY rowid DESC LIMIT 64")?;
let mut rows = q.query([])?;
while let Some(row) = rows.next()? {
let raw: Vec<u8> = row.get(0)?;
let envelope_raw: Vec<u8> = row.get(1)?;
let response_raw: Vec<u8> = row.get(2)?;
ensure!(
raw.len() <= rmm_domain::MAX_MESSAGE
&& envelope_raw.len() <= rmm_domain::MAX_MESSAGE
&& response_raw.len() <= rmm_domain::MAX_MESSAGE,
"UPDATE_HEALTH_CONTROL_SIZE"
);
let response = SyncResponse::decode(response_raw.as_slice())?;
let decision: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
if decision.decision_id != id {
continue;
}
let request = SyncRequest::decode(raw.as_slice())?;
let envelope = Envelope::decode(envelope_raw.as_slice())?;
ensure!(
request.encode_to_vec() == raw
&& response.encode_to_vec() == response_raw
&& envelope.encode_to_vec() == envelope_raw
&& rmm_protocol::valid_envelope(
&envelope,
&binding.agent_id.to_string(),
channel.connection_generation
)
&& envelope.agent_id == binding.agent_id.to_string()
&& envelope.enrollment_generation == binding.enrollment_generation
&& envelope.boot_id == channel.boot_id.to_string()
&& envelope.connection_generation == channel.connection_generation
&& matches!(envelope.payload,Some(Payload::SyncRequest(ref r)) if r==&request),
"UPDATE_HEALTH_CONTROL_BYTES"
);
let decision = verify_response(binding, &request, channel, &response, now)?;
ensure!(
serde_json::to_vec(&decision)? == response.decision_bytes,
"UPDATE_HEALTH_CONTROL_CODEC"
);
return Ok(Some(StoredSync {
request,
raw,
response,
decision,
}));
}
Ok(None)
}
fn exact_prefix(
saved: &StoredSync,
proof: &UpdateInflightProof,
header: &Handoff,
key: [u8; 32],
) -> Result<()> {
let request = &saved.request;
let d = &saved.decision;
let a = request
.attempts
.iter()
.filter(|a| {
a.job_id == header.attempt.job_id.to_string()
&& a.attempt_id == header.attempt.attempt_id.to_string()
})
.collect::<Vec<_>>();
let signed = d
.attempts
.iter()
.filter(|a| a.job_id == header.attempt.job_id && a.attempt_id == header.attempt.attempt_id)
.collect::<Vec<_>>();
ensure!(
a.len() == 1
&& signed.len() == 1
&& !d.ready
&& !d.helper_ready
&& request.journal_complete,
"UPDATE_HEALTH_INFLIGHT_REQUIRED"
);
let a = a[0];
let signed = signed[0];
let expected = update::inflight_decision(proof, key).map_err(anyhow::Error::msg)?;
ensure!(
a.update_handoff_bytes == proof.bytes().map_err(anyhow::Error::msg)?
&& a.grant_id == header.attempt.grant_id.to_string()
&& a.consumption == "consumed"
&& a.state == "running"
&& a.job_payload_hash == header.attempt.job_payload_sha256
&& a.original_boot_id == header.boot_id.to_string()
&& a.outcome_bytes.is_empty()
&& a.outcome_sha256.is_empty()
&& a.result.is_none()
&& a.result_envelope.is_none()
&& signed.update_inflight.as_ref() == Some(&expected)
&& signed.grant_id == Some(header.attempt.grant_id)
&& signed.authorization_id == Some(header.attempt.authorization_id)
&& signed.plan_hash.as_deref() == Some(header.attempt.plan_sha256.as_str())
&& signed.job_payload_hash == header.attempt.job_payload_sha256
&& signed.action == "update_agent"
&& signed.outcome_state == "running"
&& !signed.unlock
&& signed.outcome_bytes.is_empty()
&& signed.outcome_sha256.is_none()
&& request
.attempts
.iter()
.all(|other| (other.job_id == header.attempt.job_id.to_string()
&& other.attempt_id == header.attempt.attempt_id.to_string())
|| other.update_handoff_bytes.is_empty())
&& d.attempts
.iter()
.all(|other| (other.job_id == header.attempt.job_id
&& other.attempt_id == header.attempt.attempt_id)
|| (other.update_inflight.is_none()
&& matches!(
other.outcome_state.as_str(),
"accepted" | "succeeded" | "failed" | "cancelled" | "expired"
)))
&& d.journal_id == header.agent_journal_id
&& d.helper_journal_id == Some(header.helper_journal_id),
"UPDATE_HEALTH_PREFIX_MISMATCH"
);
Ok(())
}
/// `proof` comes from the independently authenticated helper snapshot. Binding
/// and key are locally provisioned; no supplied receipt/boolean proves apply.
/// `now` must be the existing qualified SyncClock, never a refreshed old sample.
pub fn context(
db: &Connection,
binding: &Binding,
proof: &UpdateInflightProof,
now: DateTime<Utc>,
) -> Result<Option<HealthContext>> {
let prepared_at = Instant::now();
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
let unverified: Handoff = serde_json::from_slice(&decoded.handoff_record)?;
let offer: Option<(Vec<u8>, String, String)> = db
.query_row(
"SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
params![
unverified.attempt.job_id.to_string(),
unverified.attempt.attempt_id.to_string()
],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.optional()?;
let Some((raw, state, grant)) = offer else {
return Ok(None);
};
let offer = JobOffer::decode(raw.as_slice())?;
let job: JobPayload = binding.job(&offer.payload_bytes)?;
let key = update::job_health_public_key(&job).map_err(anyhow::Error::msg)?;
let (header, journal) =
update::verify_inflight_proof(proof, key).map_err(anyhow::Error::msg)?;
if journal.phase() != Phase::ServicesStarted {
return Ok(None);
}
ensure!(
state == "running"
&& grant == header.attempt.grant_id.to_string()
&& update::handoff_matches_job(&header, &job)
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& offer.plan_hash == header.attempt.plan_sha256
&& offer.authorization_id == header.attempt.authorization_id.to_string(),
"UPDATE_HEALTH_ORIGINAL_JOB"
);
let event: Event = serde_json::from_slice(
decoded
.events
.last()
.context("UPDATE_HEALTH_SERVICES_STARTED")?,
)?;
ensure!(
matches!(event.evidence, Evidence::ServicesStarted { .. }),
"UPDATE_HEALTH_SERVICES_STARTED"
);
let (last, generation, fault, pending): (Option<String>, u64, Option<String>, Option<String>) =
db.query_row(
"SELECT last_decision,generation,fault,pending_request FROM reconcile_meta WHERE id=1",
[],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)?;
if fault.is_some() || last.is_none() || pending.is_some() {
return Ok(None);
}
ensure!(
generation > header.connection_generation,
"UPDATE_HEALTH_FRESH_CONNECTION"
);
let channel = Channel {
connection_generation: generation,
boot_id: header.boot_id,
};
let Some(confirming) =
stored_sync(db, Uuid::parse_str(&last.unwrap())?, binding, &channel, now)?
else {
return Ok(None);
};
if !confirming.request.previous_request_id.is_empty()
|| confirming.request.previous_decision_id.is_empty()
{
return Ok(None);
}
let prior = Uuid::parse_str(&confirming.request.previous_decision_id)?;
if confirming.request.helper_previous_decision_id != prior.to_string() {
return Ok(None);
}
let Some(applied) = stored_sync(db, prior, binding, &channel, now)? else {
return Ok(None);
};
ensure!(
confirming.request.checkpoint > applied.decision.checkpoint
&& confirming.request.helper_checkpoint > applied.decision.helper_checkpoint,
"UPDATE_HEALTH_APPLY_LINK"
);
for sync in [&applied, &confirming] {
exact_prefix(sync, proof, &header, key)?;
ensure!(
sync.decision.issued_at.timestamp() >= i64::try_from(event.observed_at)?,
"UPDATE_HEALTH_CONTROL_BEFORE_START"
);
}
let mut inventory = None;
for id in &applied.decision.acknowledged_message_ids {
let marker:Option<(String,String)>=db.query_row("SELECT handoff_sha256,prefix_sha256 FROM update_health_inventory WHERE grant_id=?1 AND message_id=?2 AND signed_decision_id=?3 AND signed_response_sha256=?4",params![header.attempt.grant_id.to_string(),id.to_string(),applied.decision.decision_id.to_string(),hash(&applied.response.encode_to_vec())],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
if let Some((handoff, prefix)) = marker {
ensure!(
handoff == journal.handoff_sha256() && prefix == journal.head_sha256(),
"UPDATE_HEALTH_RETAINED_PREFIX"
);
let envelope = update_health_inventory::read(db, header.attempt.grant_id, *id)?;
ensure!(
envelope.connection_generation == generation
&& envelope.boot_id == header.boot_id.to_string()
&& envelope.agent_id == binding.agent_id.to_string()
&& envelope.enrollment_generation == binding.enrollment_generation
&& envelope.observed_at
>= i64::try_from(event.observed_at)?
.checked_mul(1000)
.context("UPDATE_HEALTH_TIME")?
&& envelope.observed_at <= now.timestamp_millis(),
"UPDATE_HEALTH_RETAINED_BINDING"
);
let Some(Payload::InventorySnapshot(ref snapshot)) = envelope.payload else {
anyhow::bail!("UPDATE_HEALTH_BASE_REQUIRED")
};
let scope = match job.action {
rmm_domain::jobs::Action::UpdateAgent { ref update } => {
match update.manifest().map_err(anyhow::Error::msg)?.platform {
update::Platform::LinuxX64 => "linux.base",
update::Platform::WindowsX64 => "windows.base",
}
}
_ => anyhow::bail!("UPDATE_HEALTH_ACTION"),
};
ensure!(snapshot.scope == scope, "UPDATE_HEALTH_BASE_REQUIRED");
inventory = Some((*id, envelope));
break;
}
}
let Some((inventory_id, envelope)) = inventory else {
return Ok(None);
};
let until = header
.finish_before
.min(
event
.observed_at
.checked_add(180)
.context("UPDATE_HEALTH_TIME")?,
)
.min(u64::try_from(applied.decision.valid_until.timestamp())?)
.min(u64::try_from(confirming.decision.valid_until.timestamp())?);
let remaining =
(DateTime::from_timestamp(i64::try_from(until)?, 0).context("UPDATE_HEALTH_TIME")? - now)
.to_std()
.context("UPDATE_HEALTH_EXPIRED")?;
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
Ok(Some(HealthContext {
input: HealthRequest {
proof: proof.clone(),
applied_sync_request_base64url: URL_SAFE_NO_PAD.encode(&applied.raw),
confirming_sync_request_base64url: URL_SAFE_NO_PAD.encode(&confirming.raw),
inventory_envelope_base64url: URL_SAFE_NO_PAD.encode(envelope.encode_to_vec()),
},
header,
health_key: key,
inventory_id,
applied_response_sha256: hash(&applied.response.encode_to_vec()),
applied: applied.decision,
confirming: confirming.decision,
observed_now: now,
prepared_at,
deadline: prepared_at + remaining,
}))
}
/// Caller supplies the existing https_channel::mtls_client (no redirects,
/// validated chain/SNI and fixed TCP override); never a caller URL or bearer.
pub async fn request(
client: &reqwest::Client,
address: SocketAddr,
context: &HealthContext,
) -> Result<SignedHealth> {
let body = serde_json::to_vec(&context.input)?;
ensure!(body.len() <= 5 * 1024 * 1024, "UPDATE_HEALTH_REQUEST_SIZE");
let url = format!(
"https://{}:{}/agent/v1/update-health",
crate::https_channel::server_name()?,
address.port()
);
let remaining = context
.deadline
.checked_duration_since(Instant::now())
.context("UPDATE_HEALTH_EXPIRED")?;
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
let request_budget = remaining.min(Duration::from_secs(35));
let request_deadline = Instant::now() + request_budget;
let signed = tokio::time::timeout(request_budget, async {
let response = client
.post(&url)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(body)
.send()
.await?;
ensure!(response.url().as_str() == url, "UPDATE_HEALTH_REDIRECT");
let raw = receipt_bytes(response, request_deadline).await?;
let signed: SignedHealth = serde_json::from_slice(&raw)?;
ensure!(
signed.payload.len() <= 4096 && signed.signature.len() == 64,
"UPDATE_HEALTH_RECEIPT_SIZE"
);
Ok::<_, anyhow::Error>(signed)
})
.await
.context("UPDATE_HEALTH_TIMEOUT")??;
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
let now = context.observed_now + chrono::Duration::from_std(context.prepared_at.elapsed())?;
let receipt: HealthReceipt = serde_json::from_slice(&signed.payload)?;
ensure!(
receipt.signed_sync_decision_id == context.applied.decision_id
&& receipt.signed_sync_request_sha256
== hash(&URL_SAFE_NO_PAD.decode(&context.input.applied_sync_request_base64url)?)
&& receipt.signed_sync_response_sha256 == context.applied_response_sha256
&& receipt.inventory_ack_sha256 == context.applied_response_sha256
&& receipt.inventory_message_id == context.inventory_id
&& receipt.connection_generation == context.confirming.connection_generation
&& receipt.signed_sync_applied_at
== u64::try_from(context.confirming.issued_at.timestamp())?
&& receipt.valid_until <= context.header.finish_before,
"UPDATE_HEALTH_RECEIPT_BINDING"
);
let (_, journal) = update::verify_inflight_proof(&context.input.proof, context.health_key)
.map_err(anyhow::Error::msg)?;
journal.prepare(
u64::try_from(now.timestamp())?,
Evidence::HealthVerified {
receipt: signed.clone(),
},
)?;
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
Ok(signed)
}
async fn receipt_bytes(mut response: reqwest::Response, deadline: Instant) -> Result<Vec<u8>> {
ensure!(
response.status() == reqwest::StatusCode::OK,
"UPDATE_HEALTH_HTTP_STATUS"
);
ensure!(
response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
== Some("application/json")
&& !response
.headers()
.contains_key(reqwest::header::CONTENT_ENCODING)
&& response.content_length().is_none_or(|size| size <= 16384),
"UPDATE_HEALTH_HTTP_BODY"
);
let mut raw = Vec::new();
loop {
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
let chunk = response.chunk().await?;
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
let Some(chunk) = chunk else { break };
ensure!(
chunk.len() <= 16384 - raw.len(),
"UPDATE_HEALTH_RECEIPT_SIZE"
);
raw.extend_from_slice(&chunk);
}
Ok(raw)
}