Wire preserved update control, health evidence and record publication
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,465 @@
|
||||
//! Evidence-only client: ORIGINAL retained inventory and two real signed rounds.
|
||||
//! No collection, job, grant, readiness, helper effect or caller ACK is produced.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use prost::Message;
|
||||
use rmm_agent_core::{
|
||||
auth::Binding,
|
||||
reconcile::{Channel, verify_response},
|
||||
update_health_inventory,
|
||||
};
|
||||
use rmm_domain::{
|
||||
agent_update::{self as update, UpdateInflightProof},
|
||||
jobs::{JobPayload, hash},
|
||||
reconciliation::ReconcileDecisionPayload,
|
||||
};
|
||||
use rmm_protocol::{Envelope, JobOffer, Payload, SyncRequest, SyncResponse};
|
||||
use rmm_updater_handoff::{Event, Evidence, Handoff, HealthReceipt, Phase, SignedHealth};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
net::SocketAddr,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct HealthRequest {
|
||||
proof: UpdateInflightProof,
|
||||
applied_sync_request_base64url: String,
|
||||
confirming_sync_request_base64url: String,
|
||||
inventory_envelope_base64url: String,
|
||||
}
|
||||
/// Opaque; constructed only by examining retained, verified local evidence.
|
||||
pub struct HealthContext {
|
||||
input: HealthRequest,
|
||||
header: Handoff,
|
||||
health_key: [u8; 32],
|
||||
inventory_id: Uuid,
|
||||
applied: ReconcileDecisionPayload,
|
||||
applied_response_sha256: String,
|
||||
confirming: ReconcileDecisionPayload,
|
||||
observed_now: DateTime<Utc>,
|
||||
prepared_at: Instant,
|
||||
deadline: Instant,
|
||||
}
|
||||
impl HealthContext {
|
||||
pub fn handoff(&self) -> &Handoff {
|
||||
&self.header
|
||||
}
|
||||
pub fn health_public_key(&self) -> [u8; 32] {
|
||||
self.health_key
|
||||
}
|
||||
}
|
||||
struct StoredSync {
|
||||
request: SyncRequest,
|
||||
raw: Vec<u8>,
|
||||
response: SyncResponse,
|
||||
decision: ReconcileDecisionPayload,
|
||||
}
|
||||
|
||||
fn stored_sync(
|
||||
db: &Connection,
|
||||
id: Uuid,
|
||||
binding: &Binding,
|
||||
channel: &Channel,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Option<StoredSync>> {
|
||||
// At most 64 recent control rounds, incrementally read; no unbounded journal scan.
|
||||
let mut q=db.prepare("SELECT request,envelope,response FROM reconcile_requests WHERE response IS NOT NULL ORDER BY rowid DESC LIMIT 64")?;
|
||||
let mut rows = q.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let raw: Vec<u8> = row.get(0)?;
|
||||
let envelope_raw: Vec<u8> = row.get(1)?;
|
||||
let response_raw: Vec<u8> = row.get(2)?;
|
||||
ensure!(
|
||||
raw.len() <= rmm_domain::MAX_MESSAGE
|
||||
&& envelope_raw.len() <= rmm_domain::MAX_MESSAGE
|
||||
&& response_raw.len() <= rmm_domain::MAX_MESSAGE,
|
||||
"UPDATE_HEALTH_CONTROL_SIZE"
|
||||
);
|
||||
let response = SyncResponse::decode(response_raw.as_slice())?;
|
||||
let decision: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
||||
if decision.decision_id != id {
|
||||
continue;
|
||||
}
|
||||
let request = SyncRequest::decode(raw.as_slice())?;
|
||||
let envelope = Envelope::decode(envelope_raw.as_slice())?;
|
||||
ensure!(
|
||||
request.encode_to_vec() == raw
|
||||
&& response.encode_to_vec() == response_raw
|
||||
&& envelope.encode_to_vec() == envelope_raw
|
||||
&& rmm_protocol::valid_envelope(
|
||||
&envelope,
|
||||
&binding.agent_id.to_string(),
|
||||
channel.connection_generation
|
||||
)
|
||||
&& envelope.agent_id == binding.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == binding.enrollment_generation
|
||||
&& envelope.boot_id == channel.boot_id.to_string()
|
||||
&& envelope.connection_generation == channel.connection_generation
|
||||
&& matches!(envelope.payload,Some(Payload::SyncRequest(ref r)) if r==&request),
|
||||
"UPDATE_HEALTH_CONTROL_BYTES"
|
||||
);
|
||||
let decision = verify_response(binding, &request, channel, &response, now)?;
|
||||
ensure!(
|
||||
serde_json::to_vec(&decision)? == response.decision_bytes,
|
||||
"UPDATE_HEALTH_CONTROL_CODEC"
|
||||
);
|
||||
return Ok(Some(StoredSync {
|
||||
request,
|
||||
raw,
|
||||
response,
|
||||
decision,
|
||||
}));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn exact_prefix(
|
||||
saved: &StoredSync,
|
||||
proof: &UpdateInflightProof,
|
||||
header: &Handoff,
|
||||
key: [u8; 32],
|
||||
) -> Result<()> {
|
||||
let request = &saved.request;
|
||||
let d = &saved.decision;
|
||||
let a = request
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| {
|
||||
a.job_id == header.attempt.job_id.to_string()
|
||||
&& a.attempt_id == header.attempt.attempt_id.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let signed = d
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| a.job_id == header.attempt.job_id && a.attempt_id == header.attempt.attempt_id)
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(
|
||||
a.len() == 1
|
||||
&& signed.len() == 1
|
||||
&& !d.ready
|
||||
&& !d.helper_ready
|
||||
&& request.journal_complete,
|
||||
"UPDATE_HEALTH_INFLIGHT_REQUIRED"
|
||||
);
|
||||
let a = a[0];
|
||||
let signed = signed[0];
|
||||
let expected = update::inflight_decision(proof, key).map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
a.update_handoff_bytes == proof.bytes().map_err(anyhow::Error::msg)?
|
||||
&& a.grant_id == header.attempt.grant_id.to_string()
|
||||
&& a.consumption == "consumed"
|
||||
&& a.state == "running"
|
||||
&& a.job_payload_hash == header.attempt.job_payload_sha256
|
||||
&& a.original_boot_id == header.boot_id.to_string()
|
||||
&& a.outcome_bytes.is_empty()
|
||||
&& a.outcome_sha256.is_empty()
|
||||
&& a.result.is_none()
|
||||
&& a.result_envelope.is_none()
|
||||
&& signed.update_inflight.as_ref() == Some(&expected)
|
||||
&& signed.grant_id == Some(header.attempt.grant_id)
|
||||
&& signed.authorization_id == Some(header.attempt.authorization_id)
|
||||
&& signed.plan_hash.as_deref() == Some(header.attempt.plan_sha256.as_str())
|
||||
&& signed.job_payload_hash == header.attempt.job_payload_sha256
|
||||
&& signed.action == "update_agent"
|
||||
&& signed.outcome_state == "running"
|
||||
&& !signed.unlock
|
||||
&& signed.outcome_bytes.is_empty()
|
||||
&& signed.outcome_sha256.is_none()
|
||||
&& request
|
||||
.attempts
|
||||
.iter()
|
||||
.all(|other| (other.job_id == header.attempt.job_id.to_string()
|
||||
&& other.attempt_id == header.attempt.attempt_id.to_string())
|
||||
|| other.update_handoff_bytes.is_empty())
|
||||
&& d.attempts
|
||||
.iter()
|
||||
.all(|other| (other.job_id == header.attempt.job_id
|
||||
&& other.attempt_id == header.attempt.attempt_id)
|
||||
|| (other.update_inflight.is_none()
|
||||
&& matches!(
|
||||
other.outcome_state.as_str(),
|
||||
"accepted" | "succeeded" | "failed" | "cancelled" | "expired"
|
||||
)))
|
||||
&& d.journal_id == header.agent_journal_id
|
||||
&& d.helper_journal_id == Some(header.helper_journal_id),
|
||||
"UPDATE_HEALTH_PREFIX_MISMATCH"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `proof` comes from the independently authenticated helper snapshot. Binding
|
||||
/// and key are locally provisioned; no supplied receipt/boolean proves apply.
|
||||
/// `now` must be the existing qualified SyncClock, never a refreshed old sample.
|
||||
pub fn context(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
proof: &UpdateInflightProof,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Option<HealthContext>> {
|
||||
let prepared_at = Instant::now();
|
||||
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let unverified: Handoff = serde_json::from_slice(&decoded.handoff_record)?;
|
||||
let offer: Option<(Vec<u8>, String, String)> = db
|
||||
.query_row(
|
||||
"SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![
|
||||
unverified.attempt.job_id.to_string(),
|
||||
unverified.attempt.attempt_id.to_string()
|
||||
],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let Some((raw, state, grant)) = offer else {
|
||||
return Ok(None);
|
||||
};
|
||||
let offer = JobOffer::decode(raw.as_slice())?;
|
||||
let job: JobPayload = binding.job(&offer.payload_bytes)?;
|
||||
let key = update::job_health_public_key(&job).map_err(anyhow::Error::msg)?;
|
||||
let (header, journal) =
|
||||
update::verify_inflight_proof(proof, key).map_err(anyhow::Error::msg)?;
|
||||
if journal.phase() != Phase::ServicesStarted {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
state == "running"
|
||||
&& grant == header.attempt.grant_id.to_string()
|
||||
&& update::handoff_matches_job(&header, &job)
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& offer.plan_hash == header.attempt.plan_sha256
|
||||
&& offer.authorization_id == header.attempt.authorization_id.to_string(),
|
||||
"UPDATE_HEALTH_ORIGINAL_JOB"
|
||||
);
|
||||
let event: Event = serde_json::from_slice(
|
||||
decoded
|
||||
.events
|
||||
.last()
|
||||
.context("UPDATE_HEALTH_SERVICES_STARTED")?,
|
||||
)?;
|
||||
ensure!(
|
||||
matches!(event.evidence, Evidence::ServicesStarted { .. }),
|
||||
"UPDATE_HEALTH_SERVICES_STARTED"
|
||||
);
|
||||
let (last, generation, fault, pending): (Option<String>, u64, Option<String>, Option<String>) =
|
||||
db.query_row(
|
||||
"SELECT last_decision,generation,fault,pending_request FROM reconcile_meta WHERE id=1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)?;
|
||||
if fault.is_some() || last.is_none() || pending.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
generation > header.connection_generation,
|
||||
"UPDATE_HEALTH_FRESH_CONNECTION"
|
||||
);
|
||||
let channel = Channel {
|
||||
connection_generation: generation,
|
||||
boot_id: header.boot_id,
|
||||
};
|
||||
let Some(confirming) =
|
||||
stored_sync(db, Uuid::parse_str(&last.unwrap())?, binding, &channel, now)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !confirming.request.previous_request_id.is_empty()
|
||||
|| confirming.request.previous_decision_id.is_empty()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let prior = Uuid::parse_str(&confirming.request.previous_decision_id)?;
|
||||
if confirming.request.helper_previous_decision_id != prior.to_string() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(applied) = stored_sync(db, prior, binding, &channel, now)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
ensure!(
|
||||
confirming.request.checkpoint > applied.decision.checkpoint
|
||||
&& confirming.request.helper_checkpoint > applied.decision.helper_checkpoint,
|
||||
"UPDATE_HEALTH_APPLY_LINK"
|
||||
);
|
||||
for sync in [&applied, &confirming] {
|
||||
exact_prefix(sync, proof, &header, key)?;
|
||||
ensure!(
|
||||
sync.decision.issued_at.timestamp() >= i64::try_from(event.observed_at)?,
|
||||
"UPDATE_HEALTH_CONTROL_BEFORE_START"
|
||||
);
|
||||
}
|
||||
let mut inventory = None;
|
||||
for id in &applied.decision.acknowledged_message_ids {
|
||||
let marker:Option<(String,String)>=db.query_row("SELECT handoff_sha256,prefix_sha256 FROM update_health_inventory WHERE grant_id=?1 AND message_id=?2 AND signed_decision_id=?3 AND signed_response_sha256=?4",params![header.attempt.grant_id.to_string(),id.to_string(),applied.decision.decision_id.to_string(),hash(&applied.response.encode_to_vec())],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
|
||||
if let Some((handoff, prefix)) = marker {
|
||||
ensure!(
|
||||
handoff == journal.handoff_sha256() && prefix == journal.head_sha256(),
|
||||
"UPDATE_HEALTH_RETAINED_PREFIX"
|
||||
);
|
||||
let envelope = update_health_inventory::read(db, header.attempt.grant_id, *id)?;
|
||||
ensure!(
|
||||
envelope.connection_generation == generation
|
||||
&& envelope.boot_id == header.boot_id.to_string()
|
||||
&& envelope.agent_id == binding.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == binding.enrollment_generation
|
||||
&& envelope.observed_at
|
||||
>= i64::try_from(event.observed_at)?
|
||||
.checked_mul(1000)
|
||||
.context("UPDATE_HEALTH_TIME")?
|
||||
&& envelope.observed_at <= now.timestamp_millis(),
|
||||
"UPDATE_HEALTH_RETAINED_BINDING"
|
||||
);
|
||||
let Some(Payload::InventorySnapshot(ref snapshot)) = envelope.payload else {
|
||||
anyhow::bail!("UPDATE_HEALTH_BASE_REQUIRED")
|
||||
};
|
||||
let scope = match job.action {
|
||||
rmm_domain::jobs::Action::UpdateAgent { ref update } => {
|
||||
match update.manifest().map_err(anyhow::Error::msg)?.platform {
|
||||
update::Platform::LinuxX64 => "linux.base",
|
||||
update::Platform::WindowsX64 => "windows.base",
|
||||
}
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_HEALTH_ACTION"),
|
||||
};
|
||||
ensure!(snapshot.scope == scope, "UPDATE_HEALTH_BASE_REQUIRED");
|
||||
inventory = Some((*id, envelope));
|
||||
break;
|
||||
}
|
||||
}
|
||||
let Some((inventory_id, envelope)) = inventory else {
|
||||
return Ok(None);
|
||||
};
|
||||
let until = header
|
||||
.finish_before
|
||||
.min(
|
||||
event
|
||||
.observed_at
|
||||
.checked_add(180)
|
||||
.context("UPDATE_HEALTH_TIME")?,
|
||||
)
|
||||
.min(u64::try_from(applied.decision.valid_until.timestamp())?)
|
||||
.min(u64::try_from(confirming.decision.valid_until.timestamp())?);
|
||||
let remaining =
|
||||
(DateTime::from_timestamp(i64::try_from(until)?, 0).context("UPDATE_HEALTH_TIME")? - now)
|
||||
.to_std()
|
||||
.context("UPDATE_HEALTH_EXPIRED")?;
|
||||
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
|
||||
Ok(Some(HealthContext {
|
||||
input: HealthRequest {
|
||||
proof: proof.clone(),
|
||||
applied_sync_request_base64url: URL_SAFE_NO_PAD.encode(&applied.raw),
|
||||
confirming_sync_request_base64url: URL_SAFE_NO_PAD.encode(&confirming.raw),
|
||||
inventory_envelope_base64url: URL_SAFE_NO_PAD.encode(envelope.encode_to_vec()),
|
||||
},
|
||||
header,
|
||||
health_key: key,
|
||||
inventory_id,
|
||||
applied_response_sha256: hash(&applied.response.encode_to_vec()),
|
||||
applied: applied.decision,
|
||||
confirming: confirming.decision,
|
||||
observed_now: now,
|
||||
prepared_at,
|
||||
deadline: prepared_at + remaining,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Caller supplies the existing https_channel::mtls_client (no redirects,
|
||||
/// validated chain/SNI and fixed TCP override); never a caller URL or bearer.
|
||||
pub async fn request(
|
||||
client: &reqwest::Client,
|
||||
address: SocketAddr,
|
||||
context: &HealthContext,
|
||||
) -> Result<SignedHealth> {
|
||||
let body = serde_json::to_vec(&context.input)?;
|
||||
ensure!(body.len() <= 5 * 1024 * 1024, "UPDATE_HEALTH_REQUEST_SIZE");
|
||||
let url = format!(
|
||||
"https://{}:{}/agent/v1/update-health",
|
||||
crate::https_channel::server_name()?,
|
||||
address.port()
|
||||
);
|
||||
let remaining = context
|
||||
.deadline
|
||||
.checked_duration_since(Instant::now())
|
||||
.context("UPDATE_HEALTH_EXPIRED")?;
|
||||
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
|
||||
let request_budget = remaining.min(Duration::from_secs(35));
|
||||
let request_deadline = Instant::now() + request_budget;
|
||||
let signed = tokio::time::timeout(request_budget, async {
|
||||
let response = client
|
||||
.post(&url)
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
ensure!(response.url().as_str() == url, "UPDATE_HEALTH_REDIRECT");
|
||||
let raw = receipt_bytes(response, request_deadline).await?;
|
||||
let signed: SignedHealth = serde_json::from_slice(&raw)?;
|
||||
ensure!(
|
||||
signed.payload.len() <= 4096 && signed.signature.len() == 64,
|
||||
"UPDATE_HEALTH_RECEIPT_SIZE"
|
||||
);
|
||||
Ok::<_, anyhow::Error>(signed)
|
||||
})
|
||||
.await
|
||||
.context("UPDATE_HEALTH_TIMEOUT")??;
|
||||
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
||||
let now = context.observed_now + chrono::Duration::from_std(context.prepared_at.elapsed())?;
|
||||
let receipt: HealthReceipt = serde_json::from_slice(&signed.payload)?;
|
||||
ensure!(
|
||||
receipt.signed_sync_decision_id == context.applied.decision_id
|
||||
&& receipt.signed_sync_request_sha256
|
||||
== hash(&URL_SAFE_NO_PAD.decode(&context.input.applied_sync_request_base64url)?)
|
||||
&& receipt.signed_sync_response_sha256 == context.applied_response_sha256
|
||||
&& receipt.inventory_ack_sha256 == context.applied_response_sha256
|
||||
&& receipt.inventory_message_id == context.inventory_id
|
||||
&& receipt.connection_generation == context.confirming.connection_generation
|
||||
&& receipt.signed_sync_applied_at
|
||||
== u64::try_from(context.confirming.issued_at.timestamp())?
|
||||
&& receipt.valid_until <= context.header.finish_before,
|
||||
"UPDATE_HEALTH_RECEIPT_BINDING"
|
||||
);
|
||||
let (_, journal) = update::verify_inflight_proof(&context.input.proof, context.health_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
journal.prepare(
|
||||
u64::try_from(now.timestamp())?,
|
||||
Evidence::HealthVerified {
|
||||
receipt: signed.clone(),
|
||||
},
|
||||
)?;
|
||||
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
||||
Ok(signed)
|
||||
}
|
||||
|
||||
async fn receipt_bytes(mut response: reqwest::Response, deadline: Instant) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
response.status() == reqwest::StatusCode::OK,
|
||||
"UPDATE_HEALTH_HTTP_STATUS"
|
||||
);
|
||||
ensure!(
|
||||
response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
== Some("application/json")
|
||||
&& !response
|
||||
.headers()
|
||||
.contains_key(reqwest::header::CONTENT_ENCODING)
|
||||
&& response.content_length().is_none_or(|size| size <= 16384),
|
||||
"UPDATE_HEALTH_HTTP_BODY"
|
||||
);
|
||||
let mut raw = Vec::new();
|
||||
loop {
|
||||
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
|
||||
let chunk = response.chunk().await?;
|
||||
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
|
||||
let Some(chunk) = chunk else { break };
|
||||
ensure!(
|
||||
chunk.len() <= 16384 - raw.len(),
|
||||
"UPDATE_HEALTH_RECEIPT_SIZE"
|
||||
);
|
||||
raw.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(raw)
|
||||
}
|
||||
Reference in New Issue
Block a user