Wire preserved update control, health evidence and record publication
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,465 @@
|
||||
//! Evidence-only client: ORIGINAL retained inventory and two real signed rounds.
|
||||
//! No collection, job, grant, readiness, helper effect or caller ACK is produced.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use prost::Message;
|
||||
use rmm_agent_core::{
|
||||
auth::Binding,
|
||||
reconcile::{Channel, verify_response},
|
||||
update_health_inventory,
|
||||
};
|
||||
use rmm_domain::{
|
||||
agent_update::{self as update, UpdateInflightProof},
|
||||
jobs::{JobPayload, hash},
|
||||
reconciliation::ReconcileDecisionPayload,
|
||||
};
|
||||
use rmm_protocol::{Envelope, JobOffer, Payload, SyncRequest, SyncResponse};
|
||||
use rmm_updater_handoff::{Event, Evidence, Handoff, HealthReceipt, Phase, SignedHealth};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
net::SocketAddr,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct HealthRequest {
|
||||
proof: UpdateInflightProof,
|
||||
applied_sync_request_base64url: String,
|
||||
confirming_sync_request_base64url: String,
|
||||
inventory_envelope_base64url: String,
|
||||
}
|
||||
/// Opaque; constructed only by examining retained, verified local evidence.
|
||||
pub struct HealthContext {
|
||||
input: HealthRequest,
|
||||
header: Handoff,
|
||||
health_key: [u8; 32],
|
||||
inventory_id: Uuid,
|
||||
applied: ReconcileDecisionPayload,
|
||||
applied_response_sha256: String,
|
||||
confirming: ReconcileDecisionPayload,
|
||||
observed_now: DateTime<Utc>,
|
||||
prepared_at: Instant,
|
||||
deadline: Instant,
|
||||
}
|
||||
impl HealthContext {
|
||||
pub fn handoff(&self) -> &Handoff {
|
||||
&self.header
|
||||
}
|
||||
pub fn health_public_key(&self) -> [u8; 32] {
|
||||
self.health_key
|
||||
}
|
||||
}
|
||||
struct StoredSync {
|
||||
request: SyncRequest,
|
||||
raw: Vec<u8>,
|
||||
response: SyncResponse,
|
||||
decision: ReconcileDecisionPayload,
|
||||
}
|
||||
|
||||
fn stored_sync(
|
||||
db: &Connection,
|
||||
id: Uuid,
|
||||
binding: &Binding,
|
||||
channel: &Channel,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Option<StoredSync>> {
|
||||
// At most 64 recent control rounds, incrementally read; no unbounded journal scan.
|
||||
let mut q=db.prepare("SELECT request,envelope,response FROM reconcile_requests WHERE response IS NOT NULL ORDER BY rowid DESC LIMIT 64")?;
|
||||
let mut rows = q.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let raw: Vec<u8> = row.get(0)?;
|
||||
let envelope_raw: Vec<u8> = row.get(1)?;
|
||||
let response_raw: Vec<u8> = row.get(2)?;
|
||||
ensure!(
|
||||
raw.len() <= rmm_domain::MAX_MESSAGE
|
||||
&& envelope_raw.len() <= rmm_domain::MAX_MESSAGE
|
||||
&& response_raw.len() <= rmm_domain::MAX_MESSAGE,
|
||||
"UPDATE_HEALTH_CONTROL_SIZE"
|
||||
);
|
||||
let response = SyncResponse::decode(response_raw.as_slice())?;
|
||||
let decision: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
||||
if decision.decision_id != id {
|
||||
continue;
|
||||
}
|
||||
let request = SyncRequest::decode(raw.as_slice())?;
|
||||
let envelope = Envelope::decode(envelope_raw.as_slice())?;
|
||||
ensure!(
|
||||
request.encode_to_vec() == raw
|
||||
&& response.encode_to_vec() == response_raw
|
||||
&& envelope.encode_to_vec() == envelope_raw
|
||||
&& rmm_protocol::valid_envelope(
|
||||
&envelope,
|
||||
&binding.agent_id.to_string(),
|
||||
channel.connection_generation
|
||||
)
|
||||
&& envelope.agent_id == binding.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == binding.enrollment_generation
|
||||
&& envelope.boot_id == channel.boot_id.to_string()
|
||||
&& envelope.connection_generation == channel.connection_generation
|
||||
&& matches!(envelope.payload,Some(Payload::SyncRequest(ref r)) if r==&request),
|
||||
"UPDATE_HEALTH_CONTROL_BYTES"
|
||||
);
|
||||
let decision = verify_response(binding, &request, channel, &response, now)?;
|
||||
ensure!(
|
||||
serde_json::to_vec(&decision)? == response.decision_bytes,
|
||||
"UPDATE_HEALTH_CONTROL_CODEC"
|
||||
);
|
||||
return Ok(Some(StoredSync {
|
||||
request,
|
||||
raw,
|
||||
response,
|
||||
decision,
|
||||
}));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn exact_prefix(
|
||||
saved: &StoredSync,
|
||||
proof: &UpdateInflightProof,
|
||||
header: &Handoff,
|
||||
key: [u8; 32],
|
||||
) -> Result<()> {
|
||||
let request = &saved.request;
|
||||
let d = &saved.decision;
|
||||
let a = request
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| {
|
||||
a.job_id == header.attempt.job_id.to_string()
|
||||
&& a.attempt_id == header.attempt.attempt_id.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let signed = d
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| a.job_id == header.attempt.job_id && a.attempt_id == header.attempt.attempt_id)
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(
|
||||
a.len() == 1
|
||||
&& signed.len() == 1
|
||||
&& !d.ready
|
||||
&& !d.helper_ready
|
||||
&& request.journal_complete,
|
||||
"UPDATE_HEALTH_INFLIGHT_REQUIRED"
|
||||
);
|
||||
let a = a[0];
|
||||
let signed = signed[0];
|
||||
let expected = update::inflight_decision(proof, key).map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
a.update_handoff_bytes == proof.bytes().map_err(anyhow::Error::msg)?
|
||||
&& a.grant_id == header.attempt.grant_id.to_string()
|
||||
&& a.consumption == "consumed"
|
||||
&& a.state == "running"
|
||||
&& a.job_payload_hash == header.attempt.job_payload_sha256
|
||||
&& a.original_boot_id == header.boot_id.to_string()
|
||||
&& a.outcome_bytes.is_empty()
|
||||
&& a.outcome_sha256.is_empty()
|
||||
&& a.result.is_none()
|
||||
&& a.result_envelope.is_none()
|
||||
&& signed.update_inflight.as_ref() == Some(&expected)
|
||||
&& signed.grant_id == Some(header.attempt.grant_id)
|
||||
&& signed.authorization_id == Some(header.attempt.authorization_id)
|
||||
&& signed.plan_hash.as_deref() == Some(header.attempt.plan_sha256.as_str())
|
||||
&& signed.job_payload_hash == header.attempt.job_payload_sha256
|
||||
&& signed.action == "update_agent"
|
||||
&& signed.outcome_state == "running"
|
||||
&& !signed.unlock
|
||||
&& signed.outcome_bytes.is_empty()
|
||||
&& signed.outcome_sha256.is_none()
|
||||
&& request
|
||||
.attempts
|
||||
.iter()
|
||||
.all(|other| (other.job_id == header.attempt.job_id.to_string()
|
||||
&& other.attempt_id == header.attempt.attempt_id.to_string())
|
||||
|| other.update_handoff_bytes.is_empty())
|
||||
&& d.attempts
|
||||
.iter()
|
||||
.all(|other| (other.job_id == header.attempt.job_id
|
||||
&& other.attempt_id == header.attempt.attempt_id)
|
||||
|| (other.update_inflight.is_none()
|
||||
&& matches!(
|
||||
other.outcome_state.as_str(),
|
||||
"accepted" | "succeeded" | "failed" | "cancelled" | "expired"
|
||||
)))
|
||||
&& d.journal_id == header.agent_journal_id
|
||||
&& d.helper_journal_id == Some(header.helper_journal_id),
|
||||
"UPDATE_HEALTH_PREFIX_MISMATCH"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `proof` comes from the independently authenticated helper snapshot. Binding
|
||||
/// and key are locally provisioned; no supplied receipt/boolean proves apply.
|
||||
/// `now` must be the existing qualified SyncClock, never a refreshed old sample.
|
||||
pub fn context(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
proof: &UpdateInflightProof,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Option<HealthContext>> {
|
||||
let prepared_at = Instant::now();
|
||||
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let unverified: Handoff = serde_json::from_slice(&decoded.handoff_record)?;
|
||||
let offer: Option<(Vec<u8>, String, String)> = db
|
||||
.query_row(
|
||||
"SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![
|
||||
unverified.attempt.job_id.to_string(),
|
||||
unverified.attempt.attempt_id.to_string()
|
||||
],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let Some((raw, state, grant)) = offer else {
|
||||
return Ok(None);
|
||||
};
|
||||
let offer = JobOffer::decode(raw.as_slice())?;
|
||||
let job: JobPayload = binding.job(&offer.payload_bytes)?;
|
||||
let key = update::job_health_public_key(&job).map_err(anyhow::Error::msg)?;
|
||||
let (header, journal) =
|
||||
update::verify_inflight_proof(proof, key).map_err(anyhow::Error::msg)?;
|
||||
if journal.phase() != Phase::ServicesStarted {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
state == "running"
|
||||
&& grant == header.attempt.grant_id.to_string()
|
||||
&& update::handoff_matches_job(&header, &job)
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& offer.plan_hash == header.attempt.plan_sha256
|
||||
&& offer.authorization_id == header.attempt.authorization_id.to_string(),
|
||||
"UPDATE_HEALTH_ORIGINAL_JOB"
|
||||
);
|
||||
let event: Event = serde_json::from_slice(
|
||||
decoded
|
||||
.events
|
||||
.last()
|
||||
.context("UPDATE_HEALTH_SERVICES_STARTED")?,
|
||||
)?;
|
||||
ensure!(
|
||||
matches!(event.evidence, Evidence::ServicesStarted { .. }),
|
||||
"UPDATE_HEALTH_SERVICES_STARTED"
|
||||
);
|
||||
let (last, generation, fault, pending): (Option<String>, u64, Option<String>, Option<String>) =
|
||||
db.query_row(
|
||||
"SELECT last_decision,generation,fault,pending_request FROM reconcile_meta WHERE id=1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)?;
|
||||
if fault.is_some() || last.is_none() || pending.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
generation > header.connection_generation,
|
||||
"UPDATE_HEALTH_FRESH_CONNECTION"
|
||||
);
|
||||
let channel = Channel {
|
||||
connection_generation: generation,
|
||||
boot_id: header.boot_id,
|
||||
};
|
||||
let Some(confirming) =
|
||||
stored_sync(db, Uuid::parse_str(&last.unwrap())?, binding, &channel, now)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !confirming.request.previous_request_id.is_empty()
|
||||
|| confirming.request.previous_decision_id.is_empty()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let prior = Uuid::parse_str(&confirming.request.previous_decision_id)?;
|
||||
if confirming.request.helper_previous_decision_id != prior.to_string() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(applied) = stored_sync(db, prior, binding, &channel, now)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
ensure!(
|
||||
confirming.request.checkpoint > applied.decision.checkpoint
|
||||
&& confirming.request.helper_checkpoint > applied.decision.helper_checkpoint,
|
||||
"UPDATE_HEALTH_APPLY_LINK"
|
||||
);
|
||||
for sync in [&applied, &confirming] {
|
||||
exact_prefix(sync, proof, &header, key)?;
|
||||
ensure!(
|
||||
sync.decision.issued_at.timestamp() >= i64::try_from(event.observed_at)?,
|
||||
"UPDATE_HEALTH_CONTROL_BEFORE_START"
|
||||
);
|
||||
}
|
||||
let mut inventory = None;
|
||||
for id in &applied.decision.acknowledged_message_ids {
|
||||
let marker:Option<(String,String)>=db.query_row("SELECT handoff_sha256,prefix_sha256 FROM update_health_inventory WHERE grant_id=?1 AND message_id=?2 AND signed_decision_id=?3 AND signed_response_sha256=?4",params![header.attempt.grant_id.to_string(),id.to_string(),applied.decision.decision_id.to_string(),hash(&applied.response.encode_to_vec())],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
|
||||
if let Some((handoff, prefix)) = marker {
|
||||
ensure!(
|
||||
handoff == journal.handoff_sha256() && prefix == journal.head_sha256(),
|
||||
"UPDATE_HEALTH_RETAINED_PREFIX"
|
||||
);
|
||||
let envelope = update_health_inventory::read(db, header.attempt.grant_id, *id)?;
|
||||
ensure!(
|
||||
envelope.connection_generation == generation
|
||||
&& envelope.boot_id == header.boot_id.to_string()
|
||||
&& envelope.agent_id == binding.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == binding.enrollment_generation
|
||||
&& envelope.observed_at
|
||||
>= i64::try_from(event.observed_at)?
|
||||
.checked_mul(1000)
|
||||
.context("UPDATE_HEALTH_TIME")?
|
||||
&& envelope.observed_at <= now.timestamp_millis(),
|
||||
"UPDATE_HEALTH_RETAINED_BINDING"
|
||||
);
|
||||
let Some(Payload::InventorySnapshot(ref snapshot)) = envelope.payload else {
|
||||
anyhow::bail!("UPDATE_HEALTH_BASE_REQUIRED")
|
||||
};
|
||||
let scope = match job.action {
|
||||
rmm_domain::jobs::Action::UpdateAgent { ref update } => {
|
||||
match update.manifest().map_err(anyhow::Error::msg)?.platform {
|
||||
update::Platform::LinuxX64 => "linux.base",
|
||||
update::Platform::WindowsX64 => "windows.base",
|
||||
}
|
||||
}
|
||||
_ => anyhow::bail!("UPDATE_HEALTH_ACTION"),
|
||||
};
|
||||
ensure!(snapshot.scope == scope, "UPDATE_HEALTH_BASE_REQUIRED");
|
||||
inventory = Some((*id, envelope));
|
||||
break;
|
||||
}
|
||||
}
|
||||
let Some((inventory_id, envelope)) = inventory else {
|
||||
return Ok(None);
|
||||
};
|
||||
let until = header
|
||||
.finish_before
|
||||
.min(
|
||||
event
|
||||
.observed_at
|
||||
.checked_add(180)
|
||||
.context("UPDATE_HEALTH_TIME")?,
|
||||
)
|
||||
.min(u64::try_from(applied.decision.valid_until.timestamp())?)
|
||||
.min(u64::try_from(confirming.decision.valid_until.timestamp())?);
|
||||
let remaining =
|
||||
(DateTime::from_timestamp(i64::try_from(until)?, 0).context("UPDATE_HEALTH_TIME")? - now)
|
||||
.to_std()
|
||||
.context("UPDATE_HEALTH_EXPIRED")?;
|
||||
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
|
||||
Ok(Some(HealthContext {
|
||||
input: HealthRequest {
|
||||
proof: proof.clone(),
|
||||
applied_sync_request_base64url: URL_SAFE_NO_PAD.encode(&applied.raw),
|
||||
confirming_sync_request_base64url: URL_SAFE_NO_PAD.encode(&confirming.raw),
|
||||
inventory_envelope_base64url: URL_SAFE_NO_PAD.encode(envelope.encode_to_vec()),
|
||||
},
|
||||
header,
|
||||
health_key: key,
|
||||
inventory_id,
|
||||
applied_response_sha256: hash(&applied.response.encode_to_vec()),
|
||||
applied: applied.decision,
|
||||
confirming: confirming.decision,
|
||||
observed_now: now,
|
||||
prepared_at,
|
||||
deadline: prepared_at + remaining,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Caller supplies the existing https_channel::mtls_client (no redirects,
|
||||
/// validated chain/SNI and fixed TCP override); never a caller URL or bearer.
|
||||
pub async fn request(
|
||||
client: &reqwest::Client,
|
||||
address: SocketAddr,
|
||||
context: &HealthContext,
|
||||
) -> Result<SignedHealth> {
|
||||
let body = serde_json::to_vec(&context.input)?;
|
||||
ensure!(body.len() <= 5 * 1024 * 1024, "UPDATE_HEALTH_REQUEST_SIZE");
|
||||
let url = format!(
|
||||
"https://{}:{}/agent/v1/update-health",
|
||||
crate::https_channel::server_name()?,
|
||||
address.port()
|
||||
);
|
||||
let remaining = context
|
||||
.deadline
|
||||
.checked_duration_since(Instant::now())
|
||||
.context("UPDATE_HEALTH_EXPIRED")?;
|
||||
ensure!(!remaining.is_zero(), "UPDATE_HEALTH_EXPIRED");
|
||||
let request_budget = remaining.min(Duration::from_secs(35));
|
||||
let request_deadline = Instant::now() + request_budget;
|
||||
let signed = tokio::time::timeout(request_budget, async {
|
||||
let response = client
|
||||
.post(&url)
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
ensure!(response.url().as_str() == url, "UPDATE_HEALTH_REDIRECT");
|
||||
let raw = receipt_bytes(response, request_deadline).await?;
|
||||
let signed: SignedHealth = serde_json::from_slice(&raw)?;
|
||||
ensure!(
|
||||
signed.payload.len() <= 4096 && signed.signature.len() == 64,
|
||||
"UPDATE_HEALTH_RECEIPT_SIZE"
|
||||
);
|
||||
Ok::<_, anyhow::Error>(signed)
|
||||
})
|
||||
.await
|
||||
.context("UPDATE_HEALTH_TIMEOUT")??;
|
||||
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
||||
let now = context.observed_now + chrono::Duration::from_std(context.prepared_at.elapsed())?;
|
||||
let receipt: HealthReceipt = serde_json::from_slice(&signed.payload)?;
|
||||
ensure!(
|
||||
receipt.signed_sync_decision_id == context.applied.decision_id
|
||||
&& receipt.signed_sync_request_sha256
|
||||
== hash(&URL_SAFE_NO_PAD.decode(&context.input.applied_sync_request_base64url)?)
|
||||
&& receipt.signed_sync_response_sha256 == context.applied_response_sha256
|
||||
&& receipt.inventory_ack_sha256 == context.applied_response_sha256
|
||||
&& receipt.inventory_message_id == context.inventory_id
|
||||
&& receipt.connection_generation == context.confirming.connection_generation
|
||||
&& receipt.signed_sync_applied_at
|
||||
== u64::try_from(context.confirming.issued_at.timestamp())?
|
||||
&& receipt.valid_until <= context.header.finish_before,
|
||||
"UPDATE_HEALTH_RECEIPT_BINDING"
|
||||
);
|
||||
let (_, journal) = update::verify_inflight_proof(&context.input.proof, context.health_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
journal.prepare(
|
||||
u64::try_from(now.timestamp())?,
|
||||
Evidence::HealthVerified {
|
||||
receipt: signed.clone(),
|
||||
},
|
||||
)?;
|
||||
ensure!(Instant::now() < context.deadline, "UPDATE_HEALTH_EXPIRED");
|
||||
Ok(signed)
|
||||
}
|
||||
|
||||
async fn receipt_bytes(mut response: reqwest::Response, deadline: Instant) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
response.status() == reqwest::StatusCode::OK,
|
||||
"UPDATE_HEALTH_HTTP_STATUS"
|
||||
);
|
||||
ensure!(
|
||||
response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
== Some("application/json")
|
||||
&& !response
|
||||
.headers()
|
||||
.contains_key(reqwest::header::CONTENT_ENCODING)
|
||||
&& response.content_length().is_none_or(|size| size <= 16384),
|
||||
"UPDATE_HEALTH_HTTP_BODY"
|
||||
);
|
||||
let mut raw = Vec::new();
|
||||
loop {
|
||||
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
|
||||
let chunk = response.chunk().await?;
|
||||
ensure!(Instant::now() < deadline, "UPDATE_HEALTH_TIMEOUT");
|
||||
let Some(chunk) = chunk else { break };
|
||||
ensure!(
|
||||
chunk.len() <= 16384 - raw.len(),
|
||||
"UPDATE_HEALTH_RECEIPT_SIZE"
|
||||
);
|
||||
raw.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(raw)
|
||||
}
|
||||
@@ -299,3 +299,83 @@ pub fn general_supported(binding: &Binding, job: &rmm_domain::jobs::JobPayload)
|
||||
})()
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
impl Helpers {
|
||||
/// Evidence-only delivery to the actual new Fullhelper. The independent
|
||||
/// helper reads its own applied control/retained inventory before writing.
|
||||
pub fn submit_update_health(
|
||||
&self,
|
||||
header: &rmm_updater_handoff::Handoff,
|
||||
key: [u8; 32],
|
||||
receipt: &rmm_updater_handoff::SignedHealth,
|
||||
) -> Result<()> {
|
||||
use rmm_agent_core::update_handoff::{
|
||||
NativeControlOperation, NativeControlReply, NativeControlRequest,
|
||||
};
|
||||
let endpoint = self
|
||||
.endpoints
|
||||
.iter()
|
||||
.find(|e| e.profile == "full_maintenance_v1")
|
||||
.context("UPDATE_FULL_HELPER_REQUIRED")?;
|
||||
ensure!(
|
||||
endpoint.socket == Path::new("/run/rmm-helper-maintenance/control.sock"),
|
||||
"UPDATE_FIXED_FULL_SOCKET_REQUIRED"
|
||||
);
|
||||
let challenge = Uuid::new_v4();
|
||||
let request = NativeControlRequest {
|
||||
operation: "agent_update_control_v1".into(),
|
||||
schema_version: 1,
|
||||
challenge,
|
||||
tenant_id: self.binding.tenant_id,
|
||||
asset_id: self.binding.asset_id,
|
||||
agent_id: self.binding.agent_id,
|
||||
enrollment_generation: self.binding.enrollment_generation,
|
||||
assignment_version: self.binding.assignment_version,
|
||||
grant_id: header.attempt.grant_id,
|
||||
owner_nonce: header.owner_nonce,
|
||||
handoff_sha256: rmm_domain::jobs::hash(&header.bytes()?),
|
||||
method: NativeControlOperation::Health {
|
||||
receipt: receipt.clone(),
|
||||
},
|
||||
};
|
||||
let reply: NativeControlReply = control::exchange(
|
||||
&endpoint.socket,
|
||||
&request,
|
||||
std::time::Duration::from_secs(10),
|
||||
1536 * 1024,
|
||||
)?;
|
||||
ensure!(
|
||||
reply.schema_version == 1
|
||||
&& reply.challenge == challenge
|
||||
&& reply.tenant_id == request.tenant_id
|
||||
&& reply.asset_id == request.asset_id
|
||||
&& reply.agent_id == request.agent_id
|
||||
&& reply.enrollment_generation == request.enrollment_generation
|
||||
&& reply.assignment_version == request.assignment_version
|
||||
&& reply.grant_id == request.grant_id
|
||||
&& reply.owner_nonce == request.owner_nonce
|
||||
&& reply.handoff_sha256 == request.handoff_sha256
|
||||
&& !reply.ready
|
||||
&& reply.claim.is_none()
|
||||
&& reply.original_start_base64url.is_none()
|
||||
&& reply.original_outcome_base64url.is_none(),
|
||||
"UPDATE_HEALTH_REPLY_BINDING"
|
||||
);
|
||||
let proof = reply.proof.context("UPDATE_HEALTH_REPLY_PROOF_REQUIRED")?;
|
||||
let (bound, journal) = rmm_domain::agent_update::verify_inflight_proof(&proof, key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
bound.bytes()? == header.bytes()?
|
||||
&& journal.phase() == rmm_updater_handoff::Phase::HealthVerified,
|
||||
"UPDATE_HEALTH_REPLY_ORIGINAL_PHASE"
|
||||
);
|
||||
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let event: rmm_updater_handoff::Event =
|
||||
serde_json::from_slice(decoded.events.last().context("UPDATE_HEALTH_REPLY_EVENT")?)?;
|
||||
ensure!(
|
||||
matches!(event.evidence,rmm_updater_handoff::Evidence::HealthVerified{receipt:ref saved} if saved==receipt),
|
||||
"UPDATE_HEALTH_REPLY_RECEIPT_CHANGED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
+100
-6
@@ -1,3 +1,4 @@
|
||||
mod agent_update_health;
|
||||
mod apt;
|
||||
mod bootstrap;
|
||||
mod certificate_rotation;
|
||||
@@ -100,6 +101,12 @@ impl EffectFence {
|
||||
&& (!root || self.root.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
struct PendingUpdateHealth(tokio::task::JoinHandle<Result<()>>);
|
||||
impl Drop for PendingUpdateHealth {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
type Completion = (
|
||||
JobOffer,
|
||||
rmm_domain::jobs::ExecutionPermitPayload,
|
||||
@@ -217,6 +224,51 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn update_proof(&self) -> Result<Option<rmm_domain::agent_update::UpdateInflightProof>> {
|
||||
self.binding
|
||||
.as_ref()
|
||||
.map(|binding| {
|
||||
rmm_agent_core::update_health_inventory::current_proof(&self.db, binding)
|
||||
})
|
||||
.transpose()
|
||||
.map(Option::flatten)
|
||||
}
|
||||
fn start_update_health(
|
||||
&self,
|
||||
proof: &rmm_domain::agent_update::UpdateInflightProof,
|
||||
) -> Result<Option<PendingUpdateHealth>> {
|
||||
if !self.clock.trusted() {
|
||||
return Ok(None);
|
||||
}
|
||||
let binding = self.binding.as_ref().context("UPDATE_BINDING_REQUIRED")?;
|
||||
let Some(context) =
|
||||
agent_update_health::context(&self.db, binding, proof, self.sync_clock.now()?)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let header = context.handoff().clone();
|
||||
let key = context.health_public_key();
|
||||
let identity = self.certificates.current(Utc::now().timestamp())?;
|
||||
let client = https_channel::mtls_client(
|
||||
self.address,
|
||||
&self.ca,
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)?;
|
||||
let helpers = self
|
||||
.helpers
|
||||
.clone()
|
||||
.context("UPDATE_FULL_HELPER_REQUIRED")?;
|
||||
let address = self.address;
|
||||
Ok(Some(PendingUpdateHealth(tokio::spawn(async move {
|
||||
let receipt = agent_update_health::request(&client, address, &context).await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
helpers.submit_update_health(&header, key, &receipt)
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}))))
|
||||
}
|
||||
fn record(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
@@ -245,14 +297,25 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(&self.db, &offer)?;
|
||||
} else {
|
||||
let saved =
|
||||
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?;
|
||||
if matches!(
|
||||
binding.job(&offer.payload_bytes)?.action,
|
||||
rmm_domain::jobs::Action::UpdateAgent { .. }
|
||||
) && saved.is_none()
|
||||
{
|
||||
// This original may have intentionally handed off and
|
||||
// restarted us. Gates stay closed until a genuine helper
|
||||
// capture proves inflight; no Unknown is ever undone.
|
||||
continue;
|
||||
}
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&grant,
|
||||
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?
|
||||
.unwrap_or_else(|| {
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
|
||||
}),
|
||||
saved.unwrap_or_else(|| {
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
|
||||
}),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
@@ -567,6 +630,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
let mut rotation_at = Instant::now();
|
||||
let mut capability_at = Instant::now();
|
||||
let mut first_heartbeat = true;
|
||||
let mut health_task: Option<PendingUpdateHealth> = None;
|
||||
let mut health_inventory_prefix: Option<String> = None;
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(result) = self.inventory_rx.recv() => {
|
||||
@@ -574,8 +639,19 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if let Ok((data,observed_at)) = result {
|
||||
let inventory = self.inventory_envelope(generation,data,observed_at)?;
|
||||
jobs::event(&self.db,&inventory)?;
|
||||
if let Some(binding)=&self.binding
|
||||
&& let Err(error)=rmm_agent_core::update_health_inventory::retain_current(&self.db,&inventory,binding) {
|
||||
report_closed_gate(&error);self.close_gate();needs_sync=true;
|
||||
}
|
||||
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
|
||||
}
|
||||
Some(result)=async {
|
||||
match health_task.as_mut() {Some(task)=>Some((&mut task.0).await),None=>std::future::pending().await}
|
||||
} => {
|
||||
health_task.take();
|
||||
if let Err(error)=result.map_err(anyhow::Error::from).and_then(std::convert::identity) {report_closed_gate(&error);}
|
||||
self.close_gate();needs_sync=true;
|
||||
},
|
||||
message = ws.receive() => {
|
||||
let message = message?;
|
||||
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
|
||||
@@ -639,9 +715,26 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
_ = tick.tick() => {
|
||||
self.replay(&ws, generation).await?;
|
||||
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
||||
let update=self.update_proof()?;
|
||||
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
|
||||
let own_update=update.is_some() || pending_update;
|
||||
if let Some(proof)=&update {
|
||||
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
|
||||
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
|
||||
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
|
||||
// Fixed update health includes one genuinely new
|
||||
// normal base sample, never a fresh timestamp on old data.
|
||||
self.queue_inventory();health_inventory_prefix=Some(proof.head_sha256.clone());
|
||||
}
|
||||
if health_task.is_none() && self.prepared.is_none() {
|
||||
match self.start_update_health(proof) {Ok(task)=>health_task=task,Err(error)=>{report_closed_gate(&error);self.close_gate();needs_sync=true;}}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
||||
if self.prepared.is_some() && !busy && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && !busy && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
|
||||
@@ -703,6 +796,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
if self
|
||||
.certificates
|
||||
.due(self.rotation_policy, Utc::now().timestamp())?
|
||||
&& self.update_proof()?.is_none()
|
||||
{
|
||||
self.close_gate();
|
||||
drop(ws);
|
||||
|
||||
Reference in New Issue
Block a user