474 lines
19 KiB
Rust
474 lines
19 KiB
Rust
//! Evidence-only client: ORIGINAL retained inventory and two real signed rounds.
|
|
//! No collection, job, grant, readiness, helper effect or caller ACK is produced.
|
|
use anyhow::{Context, Result, ensure};
|
|
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
|
use chrono::{DateTime, Utc};
|
|
use prost::Message;
|
|
use rmm_agent_core::{
|
|
auth::Binding,
|
|
reconcile::{Channel, verify_response},
|
|
update_health_inventory,
|
|
};
|
|
use rmm_domain::{
|
|
agent_update::{self as update, UpdateInflightProof},
|
|
jobs::{JobPayload, hash},
|
|
reconciliation::ReconcileDecisionPayload,
|
|
};
|
|
use rmm_protocol::{Envelope, JobOffer, Payload, SyncRequest, SyncResponse};
|
|
use rmm_updater_handoff::{Event, Evidence, Handoff, HealthReceipt, Phase, SignedHealth};
|
|
use rusqlite::{Connection, OptionalExtension, params};
|
|
use serde::Serialize;
|
|
use std::{
|
|
net::SocketAddr,
|
|
time::{Duration, Instant},
|
|
};
|
|
use uuid::Uuid;
|
|
|
|
#[derive(Serialize)]
|
|
struct HealthRequest {
|
|
proof: UpdateInflightProof,
|
|
applied_sync_request_base64url: String,
|
|
confirming_sync_request_base64url: String,
|
|
inventory_envelope_base64url: String,
|
|
}
|
|
/// Opaque; constructed only by examining retained, verified local evidence.
|
|
pub struct HealthContext {
|
|
input: HealthRequest,
|
|
header: Handoff,
|
|
health_key: [u8; 32],
|
|
inventory_id: Uuid,
|
|
applied: ReconcileDecisionPayload,
|
|
applied_response_sha256: String,
|
|
confirming: ReconcileDecisionPayload,
|
|
observed_now: DateTime<Utc>,
|
|
prepared_at: Instant,
|
|
deadline: Instant,
|
|
}
|
|
impl HealthContext {
|
|
pub fn handoff(&self) -> &Handoff {
|
|
&self.header
|
|
}
|
|
pub fn health_public_key(&self) -> [u8; 32] {
|
|
self.health_key
|
|
}
|
|
}
|
|
struct StoredSync {
|
|
request: SyncRequest,
|
|
raw: Vec<u8>,
|
|
response: SyncResponse,
|
|
decision: ReconcileDecisionPayload,
|
|
}
|
|
|
|
fn stored_sync(
|
|
db: &Connection,
|
|
id: Uuid,
|
|
binding: &Binding,
|
|
channel: &Channel,
|
|
now: DateTime<Utc>,
|
|
) -> Result<Option<StoredSync>> {
|
|
// At most 64 recent control rounds, incrementally read; no unbounded journal scan.
|
|
let mut q=db.prepare("SELECT request,envelope,response FROM reconcile_requests WHERE response IS NOT NULL ORDER BY rowid DESC LIMIT 64")?;
|
|
let mut rows = q.query([])?;
|
|
while let Some(row) = rows.next()? {
|
|
let raw: Vec<u8> = row.get(0)?;
|
|
let envelope_raw: Vec<u8> = row.get(1)?;
|
|
let response_raw: Vec<u8> = row.get(2)?;
|
|
ensure!(
|
|
raw.len() <= rmm_domain::MAX_MESSAGE
|
|
&& envelope_raw.len() <= rmm_domain::MAX_MESSAGE
|
|
&& response_raw.len() <= rmm_domain::MAX_MESSAGE,
|
|
"UPDATE_HEALTH_CONTROL_SIZE"
|
|
);
|
|
let response = SyncResponse::decode(response_raw.as_slice())?;
|
|
let decision: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
|
if decision.decision_id != id {
|
|
continue;
|
|
}
|
|
let request = SyncRequest::decode(raw.as_slice())?;
|
|
let envelope = Envelope::decode(envelope_raw.as_slice())?;
|
|
ensure!(
|
|
request.encode_to_vec() == raw
|
|
&& response.encode_to_vec() == response_raw
|
|
&& envelope.encode_to_vec() == envelope_raw
|
|
&& rmm_protocol::valid_envelope(
|
|
&envelope,
|
|
&binding.agent_id.to_string(),
|
|
channel.connection_generation
|
|
)
|
|
&& envelope.agent_id == binding.agent_id.to_string()
|
|
&& envelope.enrollment_generation == binding.enrollment_generation
|
|
&& envelope.boot_id == channel.boot_id.to_string()
|
|
&& envelope.connection_generation == channel.connection_generation
|
|
&& matches!(envelope.payload,Some(Payload::SyncRequest(ref r)) if r==&request),
|
|
"UPDATE_HEALTH_CONTROL_BYTES"
|
|
);
|
|
let decision = verify_response(binding, &request, channel, &response, now)?;
|
|
ensure!(
|
|
serde_json::to_vec(&decision)? == response.decision_bytes,
|
|
"UPDATE_HEALTH_CONTROL_CODEC"
|
|
);
|
|
return Ok(Some(StoredSync {
|
|
request,
|
|
raw,
|
|
response,
|
|
decision,
|
|
}));
|
|
}
|
|
Ok(None)
|
|
}
|
|
|
|
fn exact_prefix(
|
|
saved: &StoredSync,
|
|
proof: &UpdateInflightProof,
|
|
header: &Handoff,
|
|
key: [u8; 32],
|
|
) -> Result<()> {
|
|
let request = &saved.request;
|
|
let d = &saved.decision;
|
|
let a = request
|
|
.attempts
|
|
.iter()
|
|
.filter(|a| {
|
|
a.job_id == header.attempt.job_id.to_string()
|
|
&& a.attempt_id == header.attempt.attempt_id.to_string()
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let signed = d
|
|
.attempts
|
|
.iter()
|
|
.filter(|a| a.job_id == header.attempt.job_id && a.attempt_id == header.attempt.attempt_id)
|
|
.collect::<Vec<_>>();
|
|
ensure!(
|
|
a.len() == 1
|
|
&& signed.len() == 1
|
|
&& !d.ready
|
|
&& !d.helper_ready
|
|
&& request.journal_complete,
|
|
"UPDATE_HEALTH_INFLIGHT_REQUIRED"
|
|
);
|
|
let a = a[0];
|
|
let signed = signed[0];
|
|
let expected = update::inflight_decision(proof, key).map_err(anyhow::Error::msg)?;
|
|
ensure!(
|
|
a.update_handoff_bytes == proof.bytes().map_err(anyhow::Error::msg)?
|
|
&& a.grant_id == header.attempt.grant_id.to_string()
|
|
&& a.consumption == "consumed"
|
|
&& a.state == "running"
|
|
&& a.job_payload_hash == header.attempt.job_payload_sha256
|
|
&& a.original_boot_id == header.boot_id.to_string()
|
|
&& a.outcome_bytes.is_empty()
|
|
&& a.outcome_sha256.is_empty()
|
|
&& a.result.is_none()
|
|
&& a.result_envelope.is_none()
|
|
&& signed.update_inflight.as_ref() == Some(&expected)
|
|
&& signed.grant_id == Some(header.attempt.grant_id)
|
|
&& signed.authorization_id == Some(header.attempt.authorization_id)
|
|
&& signed.plan_hash.as_deref() == Some(header.attempt.plan_sha256.as_str())
|
|
&& signed.job_payload_hash == header.attempt.job_payload_sha256
|
|
&& signed.action == "update_agent"
|
|
&& signed.outcome_state == "running"
|
|
&& !signed.unlock
|
|
&& signed.outcome_bytes.is_empty()
|
|
&& signed.outcome_sha256.is_none()
|
|
&& request
|
|
.attempts
|
|
.iter()
|
|
.all(|other| (other.job_id == header.attempt.job_id.to_string()
|
|
&& other.attempt_id == header.attempt.attempt_id.to_string())
|
|
|| other.update_handoff_bytes.is_empty())
|
|
&& d.attempts
|
|
.iter()
|
|
.all(|other| (other.job_id == header.attempt.job_id
|
|
&& other.attempt_id == header.attempt.attempt_id)
|
|
|| (other.update_inflight.is_none()
|
|
&& matches!(
|
|
other.outcome_state.as_str(),
|
|
"accepted" | "succeeded" | "failed" | "cancelled" | "expired"
|
|
)))
|
|
&& d.journal_id == header.agent_journal_id
|
|
&& d.helper_journal_id == Some(header.helper_journal_id),
|
|
"UPDATE_HEALTH_PREFIX_MISMATCH"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// `proof` comes from the independently authenticated helper snapshot. Binding
|
|
/// and key are locally provisioned; no supplied receipt/boolean proves apply.
|
|
/// `now` must be the existing qualified SyncClock, never a refreshed old sample.
|
|
pub fn context(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
proof: &UpdateInflightProof,
|
|
now: DateTime<Utc>,
|
|
) -> Result<Option<HealthContext>> {
|
|
let prepared_at = Instant::now();
|
|
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
|
let unverified: Handoff = serde_json::from_slice(&decoded.handoff_record)?;
|
|
let offer: Option<(Vec<u8>, String, String)> = db
|
|
.query_row(
|
|
"SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![
|
|
unverified.attempt.job_id.to_string(),
|
|
unverified.attempt.attempt_id.to_string()
|
|
],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
|
)
|
|
.optional()?;
|
|
let Some((raw, state, grant)) = offer else {
|
|
return Ok(None);
|
|
};
|
|
let offer = JobOffer::decode(raw.as_slice())?;
|
|
let job: JobPayload = binding.job(&offer.payload_bytes)?;
|
|
let key = update::job_health_public_key(&job).map_err(anyhow::Error::msg)?;
|
|
let (header, journal) =
|
|
update::verify_inflight_proof(proof, key).map_err(anyhow::Error::msg)?;
|
|
let rollback = journal.phase() == Phase::RollbackServicesStarted;
|
|
if journal.phase() != Phase::ServicesStarted && !rollback {
|
|
return Ok(None);
|
|
}
|
|
ensure!(
|
|
state == "running"
|
|
&& grant == header.attempt.grant_id.to_string()
|
|
&& update::handoff_matches_job(&header, &job)
|
|
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
|
&& offer.plan_hash == header.attempt.plan_sha256
|
|
&& offer.authorization_id == header.attempt.authorization_id.to_string(),
|
|
"UPDATE_HEALTH_ORIGINAL_JOB"
|
|
);
|
|
let event: Event = serde_json::from_slice(
|
|
decoded
|
|
.events
|
|
.last()
|
|
.context("UPDATE_HEALTH_SERVICES_STARTED")?,
|
|
)?;
|
|
ensure!(
|
|
matches!(event.evidence, Evidence::ServicesStarted { .. }) && !rollback
|
|
|| matches!(event.evidence, Evidence::RollbackServicesStarted { .. }) && rollback,
|
|
"UPDATE_HEALTH_SERVICES_STARTED"
|
|
);
|
|
let (last, generation, fault, pending): (Option<String>, u64, Option<String>, Option<String>) =
|
|
db.query_row(
|
|
"SELECT last_decision,generation,fault,pending_request FROM reconcile_meta WHERE id=1",
|
|
[],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
|
)?;
|
|
if fault.is_some() || last.is_none() || pending.is_some() {
|
|
return Ok(None);
|
|
}
|
|
ensure!(
|
|
generation > header.connection_generation,
|
|
"UPDATE_HEALTH_FRESH_CONNECTION"
|
|
);
|
|
let channel = Channel {
|
|
connection_generation: generation,
|
|
boot_id: header.boot_id,
|
|
};
|
|
let Some(confirming) =
|
|
stored_sync(db, Uuid::parse_str(&last.unwrap())?, binding, &channel, now)?
|
|
else {
|
|
return Ok(None);
|
|
};
|
|
if !confirming.request.previous_request_id.is_empty()
|
|
|| confirming.request.previous_decision_id.is_empty()
|
|
{
|
|
return Ok(None);
|
|
}
|
|
let prior = Uuid::parse_str(&confirming.request.previous_decision_id)?;
|
|
if confirming.request.helper_previous_decision_id != prior.to_string() {
|
|
return Ok(None);
|
|
}
|
|
let Some(applied) = stored_sync(db, prior, binding, &channel, now)? else {
|
|
return Ok(None);
|
|
};
|
|
ensure!(
|
|
confirming.request.checkpoint > applied.decision.checkpoint
|
|
&& confirming.request.helper_checkpoint > applied.decision.helper_checkpoint,
|
|
"UPDATE_HEALTH_APPLY_LINK"
|
|
);
|
|
for sync in [&applied, &confirming] {
|
|
exact_prefix(sync, proof, &header, key)?;
|
|
ensure!(
|
|
sync.decision.issued_at.timestamp() >= i64::try_from(event.observed_at)?,
|
|
"UPDATE_HEALTH_CONTROL_BEFORE_START"
|
|
);
|
|
}
|
|
let mut inventory = None;
|
|
for id in &applied.decision.acknowledged_message_ids {
|
|
let marker:Option<(String,String)>=db.query_row("SELECT handoff_sha256,prefix_sha256 FROM update_health_inventory WHERE grant_id=?1 AND message_id=?2 AND signed_decision_id=?3 AND signed_response_sha256=?4",params![header.attempt.grant_id.to_string(),id.to_string(),applied.decision.decision_id.to_string(),hash(&applied.response.encode_to_vec())],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
|
|
if let Some((handoff, prefix)) = marker {
|
|
ensure!(
|
|
handoff == journal.handoff_sha256() && prefix == journal.head_sha256(),
|
|
"UPDATE_HEALTH_RETAINED_PREFIX"
|
|
);
|
|
let envelope = update_health_inventory::read(db, header.attempt.grant_id, *id)?;
|
|
ensure!(
|
|
envelope.connection_generation == generation
|
|
&& envelope.boot_id == header.boot_id.to_string()
|
|
&& envelope.agent_id == binding.agent_id.to_string()
|
|
&& envelope.enrollment_generation == binding.enrollment_generation
|
|
&& envelope.observed_at
|
|
>= i64::try_from(event.observed_at)?
|
|
.checked_mul(1000)
|
|
.context("UPDATE_HEALTH_TIME")?
|
|
&& envelope.observed_at <= now.timestamp_millis(),
|
|
"UPDATE_HEALTH_RETAINED_BINDING"
|
|
);
|
|
let Some(Payload::InventorySnapshot(ref snapshot)) = envelope.payload else {
|
|
anyhow::bail!("UPDATE_HEALTH_BASE_REQUIRED")
|
|
};
|
|
let scope = match job.action {
|
|
rmm_domain::jobs::Action::UpdateAgent { ref update } => {
|
|
match update.manifest().map_err(anyhow::Error::msg)?.platform {
|
|
update::Platform::LinuxX64 | update::Platform::LinuxArm64 => "linux.base",
|
|
update::Platform::WindowsX64 | update::Platform::WindowsGnuX64 => "windows.base",
|
|
}
|
|
}
|
|
_ => anyhow::bail!("UPDATE_HEALTH_ACTION"),
|
|
};
|
|
ensure!(snapshot.scope == scope, "UPDATE_HEALTH_BASE_REQUIRED");
|
|
inventory = Some((*id, envelope));
|
|
break;
|
|
}
|
|
}
|
|
let Some((inventory_id, envelope)) = inventory else {
|
|
return Ok(None);
|
|
};
|
|
let until = header
|
|
.finish_before
|
|
.min(
|
|
event
|
|
.observed_at
|
|
.checked_add(180)
|
|
.context("UPDATE_HEALTH_TIME")?,
|
|
)
|
|
.min(u64::try_from(applied.decision.valid_until.timestamp())?)
|
|
.min(u64::try_from(confirming.decision.valid_until.timestamp())?);
|
|
let remaining =
|
|
(DateTime::from_timestamp(i64::try_from(until)?, 0).context("UPDATE_HEALTH_TIME")? - now)
|
|
.to_std()
|
|
.context("UPDATE_HEALTH_EXPIRED")?;
|
|
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
|
|
Ok(Some(HealthContext {
|
|
input: HealthRequest {
|
|
proof: proof.clone(),
|
|
applied_sync_request_base64url: URL_SAFE_NO_PAD.encode(&applied.raw),
|
|
confirming_sync_request_base64url: URL_SAFE_NO_PAD.encode(&confirming.raw),
|
|
inventory_envelope_base64url: URL_SAFE_NO_PAD.encode(envelope.encode_to_vec()),
|
|
},
|
|
header,
|
|
health_key: key,
|
|
inventory_id,
|
|
applied_response_sha256: hash(&applied.response.encode_to_vec()),
|
|
applied: applied.decision,
|
|
confirming: confirming.decision,
|
|
observed_now: now,
|
|
prepared_at,
|
|
deadline: prepared_at + remaining,
|
|
}))
|
|
}
|
|
|
|
/// Caller supplies the existing https_channel::mtls_client (no redirects,
|
|
/// validated chain/SNI and fixed TCP override); never a caller URL or bearer.
|
|
pub async fn request(
|
|
client: &reqwest::Client,
|
|
address: SocketAddr,
|
|
context: &HealthContext,
|
|
) -> Result<SignedHealth> {
|
|
let body = serde_json::to_vec(&context.input)?;
|
|
ensure!(body.len() <= 5 * 1024 * 1024, "UPDATE_HEALTH_REQUEST_SIZE");
|
|
let url = format!(
|
|
"https://{}:{}/agent/v1/update-health",
|
|
crate::https_channel::server_name()?,
|
|
address.port()
|
|
);
|
|
let remaining = context
|
|
.deadline
|
|
.checked_duration_since(Instant::now())
|
|
.context("UPDATE_HEALTH_EXPIRED")?;
|
|
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
|
|
let request_budget = remaining.min(Duration::from_secs(35));
|
|
let request_deadline = Instant::now() + request_budget;
|
|
let signed = tokio::time::timeout(request_budget, async {
|
|
let response = client
|
|
.post(&url)
|
|
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
|
.body(body)
|
|
.send()
|
|
.await?;
|
|
ensure!(response.url().as_str() == url, "UPDATE_HEALTH_REDIRECT");
|
|
let raw = receipt_bytes(response, request_deadline).await?;
|
|
let signed: SignedHealth = serde_json::from_slice(&raw)?;
|
|
ensure!(
|
|
signed.payload.len() <= 4096 && signed.signature.len() == 64,
|
|
"UPDATE_HEALTH_RECEIPT_SIZE"
|
|
);
|
|
Ok::<_, anyhow::Error>(signed)
|
|
})
|
|
.await
|
|
.context("UPDATE_HEALTH_TIMEOUT")??;
|
|
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
|
let now = context.observed_now + chrono::Duration::from_std(context.prepared_at.elapsed())?;
|
|
let receipt: HealthReceipt = serde_json::from_slice(&signed.payload)?;
|
|
ensure!(
|
|
receipt.signed_sync_decision_id == context.applied.decision_id
|
|
&& receipt.signed_sync_request_sha256
|
|
== hash(&URL_SAFE_NO_PAD.decode(&context.input.applied_sync_request_base64url)?)
|
|
&& receipt.signed_sync_response_sha256 == context.applied_response_sha256
|
|
&& receipt.inventory_ack_sha256 == context.applied_response_sha256
|
|
&& receipt.inventory_message_id == context.inventory_id
|
|
&& receipt.connection_generation == context.confirming.connection_generation
|
|
&& receipt.signed_sync_applied_at
|
|
== u64::try_from(context.confirming.issued_at.timestamp())?
|
|
&& receipt.valid_until <= context.header.finish_before,
|
|
"UPDATE_HEALTH_RECEIPT_BINDING"
|
|
);
|
|
let (_, journal) = update::verify_inflight_proof(&context.input.proof, context.health_key)
|
|
.map_err(anyhow::Error::msg)?;
|
|
journal.prepare(
|
|
u64::try_from(now.timestamp())?,
|
|
if journal.phase() == Phase::RollbackServicesStarted {
|
|
Evidence::RollbackHealthVerified {
|
|
receipt: signed.clone(),
|
|
}
|
|
} else {
|
|
Evidence::HealthVerified {
|
|
receipt: signed.clone(),
|
|
}
|
|
},
|
|
)?;
|
|
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
|
Ok(signed)
|
|
}
|
|
|
|
async fn receipt_bytes(mut response: reqwest::Response, deadline: Instant) -> Result<Vec<u8>> {
|
|
ensure!(
|
|
response.status() == reqwest::StatusCode::OK,
|
|
"UPDATE_HEALTH_HTTP_STATUS"
|
|
);
|
|
ensure!(
|
|
response
|
|
.headers()
|
|
.get(reqwest::header::CONTENT_TYPE)
|
|
.and_then(|v| v.to_str().ok())
|
|
== Some("application/json")
|
|
&& !response
|
|
.headers()
|
|
.contains_key(reqwest::header::CONTENT_ENCODING)
|
|
&& response.content_length().is_none_or(|size| size <= 16384),
|
|
"UPDATE_HEALTH_HTTP_BODY"
|
|
);
|
|
let mut raw = Vec::new();
|
|
loop {
|
|
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
|
|
let chunk = response.chunk().await?;
|
|
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
|
|
let Some(chunk) = chunk else { break };
|
|
ensure!(
|
|
chunk.len() <= 16384 - raw.len(),
|
|
"UPDATE_HEALTH_RECEIPT_SIZE"
|
|
);
|
|
raw.extend_from_slice(&chunk);
|
|
}
|
|
Ok(raw)
|
|
}
|