Wire preserved update control, health evidence and record publication
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -791,6 +791,48 @@ impl<V: Ed25519Verifier> HelperSync<V> {
|
||||
boot: current_boot,
|
||||
}),
|
||||
)?;
|
||||
if decision
|
||||
.attempts
|
||||
.iter()
|
||||
.any(|d| d.update_inflight.is_some())
|
||||
{
|
||||
use prost::Message;
|
||||
let applied_at = clock.now()?;
|
||||
ensure!(
|
||||
applied_at >= now && applied_at < decision.valid_until,
|
||||
"UPDATE_SYNC_EXPIRED_BEFORE_APPLY"
|
||||
);
|
||||
let response_raw = rmm_protocol::SyncResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
}
|
||||
.encode_to_vec();
|
||||
for decision_attempt in decision
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|d| d.update_inflight.is_some())
|
||||
{
|
||||
let row = after
|
||||
.iter()
|
||||
.find(|r| {
|
||||
r.job_id == decision_attempt.job_id
|
||||
&& r.attempt_id == decision_attempt.attempt_id
|
||||
&& !r.update_handoff_bytes.is_empty()
|
||||
})
|
||||
.context("UPDATE_APPLIED_ORIGINAL_ROW_REQUIRED")?;
|
||||
crate::update_handoff::record_applied_sync(
|
||||
&tx,
|
||||
&crate::update_handoff::AppliedUpdateSync {
|
||||
schema_version: 1,
|
||||
grant_id: row.grant_id,
|
||||
decision_id: decision.decision_id,
|
||||
request_sha256: hash(raw_request),
|
||||
response_protobuf_sha256: hash(&response_raw),
|
||||
applied_at,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
}
|
||||
let fingerprint = ledger_fingerprint(&tx)?;
|
||||
tx.execute("INSERT INTO helper_sync_decisions(decision_id,checkpoint,response,request,fingerprint) VALUES(?1,?2,?3,?4,?5)",params![decision.decision_id.to_string(),decision.helper_checkpoint,serde_json::to_vec(response)?,raw_request,fingerprint])?;
|
||||
tx.execute("UPDATE helper_sync_meta SET last_decision=?1,fault=COALESCE(fault,?2),agent_journal_id=?3,agent_checkpoint=?4 WHERE id=1",params![decision.decision_id.to_string(),fault,decision.journal_id.to_string(),decision.checkpoint])?;
|
||||
@@ -1291,11 +1333,22 @@ fn ledger_fingerprint(db: &Connection) -> Result<String> {
|
||||
Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<NativeObservationRow>>>()?;
|
||||
let mut applied=db.prepare("SELECT grant_id,decision_id,witness_sha256 FROM agent_update_sync_applied ORDER BY grant_id,decision_id")?;
|
||||
let applied = applied
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
return Ok(hash(&serde_json::to_vec(&(
|
||||
rows,
|
||||
updates,
|
||||
claims,
|
||||
observations,
|
||||
applied,
|
||||
))?));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,3 +394,27 @@ pub fn permit_request(offer: &JobOffer, permit: &ExecutionPermit) -> Result<Requ
|
||||
authorization_id: Uuid::parse_str(&offer.authorization_id)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read-only control scheduling after an updater intentionally restarted the
|
||||
/// agent. This is NOT a handoff proof, approval, start grant or effect lease.
|
||||
/// A fresh independently authenticated helper snapshot still decides whether
|
||||
/// the exact original is inflight or must become Unknown; existing Unknown is
|
||||
/// never returned as a candidate and no local outcome is changed here.
|
||||
pub fn needs_update_control(
|
||||
db: &Connection,
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
) -> Result<bool> {
|
||||
let mut candidates = 0;
|
||||
for (offer, grant) in interrupted(db, ledger, binding)? {
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
if !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) || grant.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if recover_interrupted_outcome(ledger, binding, &offer, &grant)?.is_none() {
|
||||
candidates += 1;
|
||||
}
|
||||
}
|
||||
ensure!(candidates <= 1, "UPDATE_MULTIPLE_PENDING_CONTROLS");
|
||||
Ok(candidates == 1)
|
||||
}
|
||||
|
||||
@@ -9,3 +9,4 @@ pub mod jobs;
|
||||
pub mod ledger;
|
||||
pub mod reconcile;
|
||||
pub mod update_handoff;
|
||||
pub mod update_health_inventory;
|
||||
|
||||
@@ -306,7 +306,13 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
merge_helper(&mut attempts, h)?;
|
||||
}
|
||||
ensure!(attempts.len() <= 256, "SYNC_ATTEMPT_CAPACITY_NO_COMPACTION");
|
||||
let ids = pending_ids(&tx)?;
|
||||
let mut ids = pending_ids(&tx)?;
|
||||
if let Some(update) = inflight.first() {
|
||||
let grant = Uuid::parse_str(&update.grant_id)?;
|
||||
ids.extend(crate::update_health_inventory::pending_ids(&tx, grant)?);
|
||||
ids.sort();
|
||||
ids.dedup();
|
||||
}
|
||||
ensure!(ids.len() <= 256, "SYNC_RECEIPT_CAPACITY");
|
||||
let next: i64 = tx.query_row(
|
||||
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
||||
@@ -562,6 +568,19 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
"UPDATE reconcile_requests SET response=?2 WHERE id=?1 AND response IS NULL",
|
||||
params![decision.request_id.to_string(), response.encode_to_vec()],
|
||||
)?;
|
||||
for local in prepared
|
||||
.request
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| !a.update_handoff_bytes.is_empty())
|
||||
{
|
||||
crate::update_health_inventory::acknowledge_signed(
|
||||
&tx,
|
||||
Uuid::parse_str(&local.grant_id)?,
|
||||
&decision,
|
||||
response,
|
||||
)?;
|
||||
}
|
||||
tx.execute("UPDATE reconcile_meta SET last_decision=?1,pending_request=NULL,helper_id=?2,helper_checkpoint=?3,fault=COALESCE(fault,?4),high_sequence=?5 WHERE id=1",
|
||||
params![decision.decision_id.to_string(),decision.helper_journal_id.map(|id|id.to_string()),decision.helper_checkpoint,fault,seq])?;
|
||||
tx.commit()?;
|
||||
@@ -692,6 +711,7 @@ pub fn initialize(db: &Connection) -> Result<()> {
|
||||
CREATE TABLE IF NOT EXISTS reconcile_results(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,revision INTEGER NOT NULL,envelope BLOB NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS reconcile_receipts(id TEXT PRIMARY KEY,decision_id TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS reconcile_outcomes(job TEXT NOT NULL,attempt TEXT NOT NULL,outcome BLOB NOT NULL,decision_id TEXT NOT NULL,PRIMARY KEY(job,attempt));")?;
|
||||
crate::update_health_inventory::initialize(db)?;
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO reconcile_meta(id,journal_id) VALUES(1,?1)",
|
||||
[Uuid::new_v4().to_string()],
|
||||
|
||||
@@ -33,6 +33,8 @@ pub trait UpdateAdmissionGuard: ExecutorLedgerGuard {
|
||||
/// Role comes from an authenticated concrete local participant, not IPC data.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Participant {
|
||||
/// Authenticated Full reader; never a native effect participant.
|
||||
Observer,
|
||||
OriginalHelper,
|
||||
StableUpdater,
|
||||
NewHelper,
|
||||
@@ -64,6 +66,11 @@ pub trait UpdateParticipantGuard: ExecutorLedgerGuard {
|
||||
/// executor guard. This does not alter old schema objects, rows or file IDs.
|
||||
pub fn initialize(db: &Connection, guard: &impl ExecutorLedgerGuard) -> Result<()> {
|
||||
guard.check(db)?;
|
||||
db.execute_batch("PRAGMA foreign_keys=ON;")?;
|
||||
ensure!(
|
||||
db.query_row("PRAGMA foreign_keys", [], |r| r.get::<_, bool>(0))?,
|
||||
"UPDATE_LEDGER_FOREIGN_KEYS_REQUIRED"
|
||||
);
|
||||
db.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS agent_update_handoffs(
|
||||
grant_id TEXT PRIMARY KEY REFERENCES consumed(grant_id), record BLOB NOT NULL,
|
||||
@@ -81,7 +88,10 @@ pub fn initialize(db: &Connection, guard: &impl ExecutorLedgerGuard) -> Result<(
|
||||
CREATE TABLE IF NOT EXISTS agent_update_native_observations(
|
||||
grant_id TEXT NOT NULL,effect_key TEXT NOT NULL,claim_sha256 TEXT NOT NULL,
|
||||
observation_sha256 TEXT NOT NULL,observed_at TEXT NOT NULL,PRIMARY KEY(grant_id,effect_key),
|
||||
FOREIGN KEY(grant_id,effect_key) REFERENCES agent_update_native_claims(grant_id,effect_key));",
|
||||
FOREIGN KEY(grant_id,effect_key) REFERENCES agent_update_native_claims(grant_id,effect_key));
|
||||
CREATE TABLE IF NOT EXISTS agent_update_sync_applied(
|
||||
grant_id TEXT NOT NULL REFERENCES agent_update_handoffs(grant_id),decision_id TEXT NOT NULL,
|
||||
witness BLOB NOT NULL,witness_sha256 TEXT NOT NULL,PRIMARY KEY(grant_id,decision_id));",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1060,10 +1070,14 @@ pub enum NativeControlOperation {
|
||||
expected_head_sha256: String,
|
||||
service: NativeService,
|
||||
},
|
||||
ObserveStarts,
|
||||
ObserveStarts {
|
||||
services: Vec<rmm_updater_handoff::StartedService>,
|
||||
observation_sha256: String,
|
||||
},
|
||||
Health {
|
||||
receipt: rmm_updater_handoff::SignedHealth,
|
||||
},
|
||||
Finalize,
|
||||
}
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -1079,7 +1093,8 @@ pub struct NativeControlReply {
|
||||
pub owner_nonce: Uuid,
|
||||
pub handoff_sha256: String,
|
||||
pub ready: bool,
|
||||
pub proof: rmm_domain::agent_update::UpdateInflightProof,
|
||||
pub proof: Option<rmm_domain::agent_update::UpdateInflightProof>,
|
||||
pub original_outcome_base64url: Option<String>,
|
||||
pub claim: Option<NativeEffectClaim>,
|
||||
/// Only ReadOriginal; exact retained execution_evidence.start bytes,
|
||||
/// bounded to 512KiB before canonical Base64url encoding, never rewritten.
|
||||
@@ -1159,3 +1174,97 @@ mod native_record_tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the exact immutable native reservation, never a dispatch token.
|
||||
pub fn read_native_claim(
|
||||
db: &Connection,
|
||||
guard: &impl ExecutorLedgerGuard,
|
||||
grant: Uuid,
|
||||
effect: &NativeEffect,
|
||||
) -> Result<NativeEffectClaim> {
|
||||
read(db, guard, grant)?;
|
||||
let raw: Vec<u8> = db.query_row(
|
||||
"SELECT claim FROM agent_update_native_claims WHERE grant_id=?1 AND effect_key=?2",
|
||||
params![grant.to_string(), native_key(effect)?],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let claim: NativeEffectClaim = serde_json::from_slice(&raw)?;
|
||||
ensure!(
|
||||
claim.grant_id == grant && claim.effect == *effect,
|
||||
"UPDATE_NATIVE_ORIGINAL_CLAIM_REQUIRED"
|
||||
);
|
||||
Ok(claim)
|
||||
}
|
||||
/// No new capture/checkpoint or readiness is generated by this read.
|
||||
pub fn current_proof(
|
||||
db: &Connection,
|
||||
guard: &impl ExecutorLedgerGuard,
|
||||
grant: Uuid,
|
||||
) -> Result<Option<rmm_domain::agent_update::UpdateInflightProof>> {
|
||||
read(db, guard, grant)?;
|
||||
proof_locked(db, grant)
|
||||
}
|
||||
/// Actual local control application, recorded by HelperSync in the SAME
|
||||
/// transaction as its verified original decision. Never supplied by IPC.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AppliedUpdateSync {
|
||||
pub schema_version: u32,
|
||||
pub grant_id: Uuid,
|
||||
pub decision_id: Uuid,
|
||||
pub request_sha256: String,
|
||||
pub response_protobuf_sha256: String,
|
||||
pub applied_at: DateTime<Utc>,
|
||||
}
|
||||
pub(crate) fn record_applied_sync(db: &Connection, witness: &AppliedUpdateSync) -> Result<()> {
|
||||
ensure!(
|
||||
witness.schema_version == 1
|
||||
&& !witness.grant_id.is_nil()
|
||||
&& !witness.decision_id.is_nil()
|
||||
&& rmm_domain::agent_update::digest(&witness.request_sha256)
|
||||
&& rmm_domain::agent_update::digest(&witness.response_protobuf_sha256),
|
||||
"UPDATE_APPLIED_SYNC_SHAPE"
|
||||
);
|
||||
let raw = serde_json::to_vec(witness)?;
|
||||
db.execute("INSERT INTO agent_update_sync_applied(grant_id,decision_id,witness,witness_sha256) VALUES(?1,?2,?3,?4)",params![witness.grant_id.to_string(),witness.decision_id.to_string(),raw,hash(&raw)])?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn read_applied_sync(
|
||||
db: &Connection,
|
||||
guard: &impl ExecutorLedgerGuard,
|
||||
grant: Uuid,
|
||||
decision: Uuid,
|
||||
) -> Result<AppliedUpdateSync> {
|
||||
use prost::Message;
|
||||
guard.check(db)?;
|
||||
let (raw,digest):(Vec<u8>,String)=db.query_row("SELECT witness,witness_sha256 FROM agent_update_sync_applied WHERE grant_id=?1 AND decision_id=?2",params![grant.to_string(),decision.to_string()],|r|Ok((r.get(0)?,r.get(1)?)))?;
|
||||
let witness: AppliedUpdateSync = serde_json::from_slice(&raw)?;
|
||||
ensure!(
|
||||
raw.len() <= 2048
|
||||
&& hash(&raw) == digest
|
||||
&& serde_json::to_vec(&witness)? == raw
|
||||
&& witness.schema_version == 1
|
||||
&& witness.grant_id == grant
|
||||
&& witness.decision_id == decision,
|
||||
"UPDATE_APPLIED_SYNC_CHANGED"
|
||||
);
|
||||
let (saved_response, request): (Vec<u8>, Vec<u8>) = db.query_row(
|
||||
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[decision.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
let response: crate::helper_reconcile::SignedResponse =
|
||||
serde_json::from_slice(&saved_response)?;
|
||||
let protobuf = rmm_protocol::SyncResponse {
|
||||
decision_bytes: response.decision_bytes,
|
||||
signature: response.signature,
|
||||
}
|
||||
.encode_to_vec();
|
||||
ensure!(
|
||||
hash(&protobuf) == witness.response_protobuf_sha256
|
||||
&& hash(&request) == witness.request_sha256,
|
||||
"UPDATE_APPLIED_SYNC_ORIGINAL_BYTES_CHANGED"
|
||||
);
|
||||
guard.check(db)?;
|
||||
Ok(witness)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
//! Retain at most two ORIGINAL base-inventory envelopes for one update grant.
|
||||
//! Caller holds the existing agent journal lock and provisions its bounded DB.
|
||||
//! The health key must come from the independently verified ORIGINAL update job;
|
||||
//! neither a network response nor a caller-selected key establishes that trust.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use prost::Message;
|
||||
use rmm_domain::{
|
||||
agent_update::UpdateInflightProof, jobs::hash, reconciliation::ReconcileDecisionPayload,
|
||||
};
|
||||
use rmm_protocol::{Envelope, Payload, SyncResponse};
|
||||
use rmm_updater_handoff::{Event, Evidence, Phase, Service};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn initialize(db: &Connection) -> Result<()> {
|
||||
db.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS update_health_inventory(
|
||||
grant_id TEXT NOT NULL,message_id TEXT NOT NULL UNIQUE,
|
||||
handoff_sha256 TEXT NOT NULL,prefix_sha256 TEXT NOT NULL,
|
||||
envelope BLOB NOT NULL,envelope_sha256 TEXT NOT NULL,
|
||||
signed_decision_id TEXT,signed_response_sha256 TEXT,
|
||||
PRIMARY KEY(grant_id,message_id),
|
||||
CHECK((signed_decision_id IS NULL)=(signed_response_sha256 IS NULL)));",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct CurrentUpdate {
|
||||
proof: UpdateInflightProof,
|
||||
key: [u8; 32],
|
||||
generation: u64,
|
||||
valid_until: chrono::DateTime<Utc>,
|
||||
}
|
||||
/// Read-only scheduling of control for an ORIGINAL locally applied update.
|
||||
/// Historical signature validation here opens no current start/effect lease;
|
||||
/// it lets a fresh helper capture adjudicate a now-lost owner or expired job.
|
||||
pub fn current_proof(
|
||||
db: &Connection,
|
||||
binding: &crate::auth::Binding,
|
||||
) -> Result<Option<UpdateInflightProof>> {
|
||||
Ok(current_update(db, binding)?.map(|current| current.proof))
|
||||
}
|
||||
fn current_update(
|
||||
db: &Connection,
|
||||
binding: &crate::auth::Binding,
|
||||
) -> Result<Option<CurrentUpdate>> {
|
||||
let (last, generation, journal, fault): (Option<String>, u64, String, Option<String>) = db
|
||||
.query_row(
|
||||
"SELECT last_decision,generation,journal_id,fault FROM reconcile_meta WHERE id=1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)?;
|
||||
let Some(last) = last else { return Ok(None) };
|
||||
if fault.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut q=db.prepare("SELECT request,response FROM reconcile_requests WHERE response IS NOT NULL ORDER BY rowid DESC LIMIT 64")?;
|
||||
let mut rows = q.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let response_raw: Vec<u8> = row.get(1)?;
|
||||
ensure!(
|
||||
response_raw.len() <= rmm_domain::MAX_MESSAGE,
|
||||
"UPDATE_HEALTH_CONTROL_SIZE"
|
||||
);
|
||||
let response = SyncResponse::decode(response_raw.as_slice())?;
|
||||
let d: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
||||
if d.decision_id.to_string() != last {
|
||||
continue;
|
||||
}
|
||||
// prepare() fences a reconnect to its new generation BEFORE its
|
||||
// response is applied. The previous applied decision is then expected
|
||||
// old context, never an error or current update authority.
|
||||
if d.connection_generation < generation {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
d.connection_generation == generation,
|
||||
"UPDATE_HEALTH_CONTROL_GENERATION_ROLLBACK"
|
||||
);
|
||||
let now = Utc::now();
|
||||
if d.issued_at > now {
|
||||
return Ok(None);
|
||||
}
|
||||
let raw: Vec<u8> = row.get(0)?;
|
||||
ensure!(
|
||||
raw.len() <= rmm_domain::MAX_MESSAGE,
|
||||
"UPDATE_HEALTH_CONTROL_SIZE"
|
||||
);
|
||||
let request = rmm_protocol::SyncRequest::decode(raw.as_slice())?;
|
||||
ensure!(
|
||||
request.encode_to_vec() == raw
|
||||
&& response.encode_to_vec() == response_raw
|
||||
&& serde_json::to_vec(&d)? == response.decision_bytes,
|
||||
"UPDATE_HEALTH_CONTROL_CODEC"
|
||||
);
|
||||
let channel = crate::reconcile::Channel {
|
||||
connection_generation: generation,
|
||||
boot_id: d.boot_id,
|
||||
};
|
||||
let d =
|
||||
crate::reconcile::verify_response(binding, &request, &channel, &response, d.issued_at)?;
|
||||
let updates = d
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| a.update_inflight.is_some())
|
||||
.collect::<Vec<_>>();
|
||||
if updates.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
updates.len() == 1
|
||||
&& !d.ready
|
||||
&& !d.helper_ready
|
||||
&& d.journal_id.to_string() == journal,
|
||||
"UPDATE_HEALTH_INFLIGHT_REQUIRED"
|
||||
);
|
||||
let signed = updates[0];
|
||||
let local = request
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| {
|
||||
a.job_id == signed.job_id.to_string()
|
||||
&& a.attempt_id == signed.attempt_id.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(local.len() == 1, "UPDATE_HEALTH_LOCAL_ATTEMPT");
|
||||
let local = local[0];
|
||||
let (offer_raw, state, grant): (Vec<u8>, String, String) = db.query_row(
|
||||
"SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![signed.job_id.to_string(), signed.attempt_id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)?;
|
||||
let offer = rmm_protocol::JobOffer::decode(offer_raw.as_slice())?;
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
let key =
|
||||
rmm_domain::agent_update::job_health_public_key(&job).map_err(anyhow::Error::msg)?;
|
||||
let proof = UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let (header, _phase) = rmm_domain::agent_update::verify_inflight_proof(&proof, key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let expected =
|
||||
rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
state == "running"
|
||||
&& local.state == "running"
|
||||
&& local.consumption == "consumed"
|
||||
&& grant == header.attempt.grant_id.to_string()
|
||||
&& local.grant_id == grant
|
||||
&& signed.grant_id == Some(header.attempt.grant_id)
|
||||
&& signed.update_inflight.as_ref() == Some(&expected)
|
||||
&& !signed.unlock
|
||||
&& signed.outcome_state == "running"
|
||||
&& signed.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& offer.job_payload_hash == signed.job_payload_hash
|
||||
&& local.job_payload_hash == signed.job_payload_hash
|
||||
&& local.original_boot_id == header.boot_id.to_string()
|
||||
&& local.outcome_bytes.is_empty()
|
||||
&& local.outcome_sha256.is_empty()
|
||||
&& local.result.is_none()
|
||||
&& local.result_envelope.is_none()
|
||||
&& signed.outcome_bytes.is_empty()
|
||||
&& signed.outcome_sha256.is_none()
|
||||
&& signed.authorization_id == Some(header.attempt.authorization_id)
|
||||
&& signed.plan_hash.as_deref() == Some(header.attempt.plan_sha256.as_str())
|
||||
&& offer.plan_hash == header.attempt.plan_sha256
|
||||
&& offer.authorization_id == header.attempt.authorization_id.to_string()
|
||||
&& rmm_domain::agent_update::handoff_matches_job(&header, &job),
|
||||
"UPDATE_HEALTH_ORIGINAL_BINDING"
|
||||
);
|
||||
return Ok(Some(CurrentUpdate {
|
||||
proof,
|
||||
key,
|
||||
generation,
|
||||
valid_until: d.valid_until,
|
||||
}));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Retain only a genuinely collected, fresh base snapshot against an already
|
||||
/// applied ServicesStarted control. Old control may schedule read-only Sync,
|
||||
/// but may never make stale inventory into a current health observation.
|
||||
pub fn retain_current(
|
||||
db: &Connection,
|
||||
envelope: &Envelope,
|
||||
binding: &crate::auth::Binding,
|
||||
) -> Result<bool> {
|
||||
if !matches!(envelope.payload,Some(Payload::InventorySnapshot(ref s)) if matches!(s.scope.as_str(),"linux.base"|"windows.base"))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(current) = current_update(db, binding)? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if Utc::now() >= current.valid_until {
|
||||
return Ok(false);
|
||||
}
|
||||
let (_, journal) = rmm_domain::agent_update::verify_inflight_proof(¤t.proof, current.key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
if journal.phase() != Phase::ServicesStarted {
|
||||
return Ok(false);
|
||||
}
|
||||
ensure!(
|
||||
envelope.connection_generation == current.generation
|
||||
&& envelope.agent_id == binding.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == binding.enrollment_generation,
|
||||
"UPDATE_HEALTH_CURRENT_CHANNEL"
|
||||
);
|
||||
retain(db, envelope, ¤t.proof, current.key)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Queue integration calls this only for an actually collected new snapshot.
|
||||
/// Verification independently excludes pre-start/base-relabelled update data.
|
||||
pub fn retain(
|
||||
db: &Connection,
|
||||
envelope: &Envelope,
|
||||
proof: &UpdateInflightProof,
|
||||
health_key: [u8; 32],
|
||||
) -> Result<()> {
|
||||
let (header, journal) = rmm_domain::agent_update::verify_inflight_proof(proof, health_key)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
journal.phase() == Phase::ServicesStarted,
|
||||
"UPDATE_HEALTH_NOT_STARTED"
|
||||
);
|
||||
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
||||
let last: Event =
|
||||
serde_json::from_slice(decoded.events.last().context("UPDATE_HEALTH_START_EVENT")?)?;
|
||||
ensure!(
|
||||
matches!(last.evidence, Evidence::ServicesStarted { .. }),
|
||||
"UPDATE_HEALTH_START_EVENT"
|
||||
);
|
||||
let scope = if header.stop_set.contains(&Service::WindowsAgent) {
|
||||
"windows.base"
|
||||
} else {
|
||||
"linux.base"
|
||||
};
|
||||
let Payload::InventorySnapshot(snapshot) = envelope
|
||||
.payload
|
||||
.as_ref()
|
||||
.context("UPDATE_HEALTH_INVENTORY")?
|
||||
else {
|
||||
anyhow::bail!("UPDATE_HEALTH_BASE_REQUIRED")
|
||||
};
|
||||
let now = Utc::now();
|
||||
let observed = chrono::DateTime::from_timestamp_millis(envelope.observed_at)
|
||||
.context("UPDATE_HEALTH_TIME")?;
|
||||
let journal_id: String = db.query_row(
|
||||
"SELECT journal_id FROM reconcile_meta WHERE id=1",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
journal_id == header.agent_journal_id.to_string()
|
||||
&& envelope.protocol_major == 1
|
||||
&& envelope.agent_id == header.attempt.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == header.attempt.enrollment_generation
|
||||
&& envelope.boot_id == header.boot_id.to_string()
|
||||
&& envelope.connection_generation > header.connection_generation
|
||||
&& envelope.sequence > 0
|
||||
&& observed.timestamp() >= i64::try_from(last.observed_at)?
|
||||
&& observed <= now
|
||||
&& now.timestamp() < i64::try_from(header.finish_before)?
|
||||
&& now.timestamp()
|
||||
< i64::try_from(
|
||||
last.observed_at
|
||||
.checked_add(180)
|
||||
.context("UPDATE_HEALTH_TIME")?
|
||||
)?
|
||||
&& snapshot.scope == scope
|
||||
&& snapshot.schema_version == 1
|
||||
&& snapshot.chunk_count == 1
|
||||
&& snapshot.chunk_index == 0
|
||||
&& snapshot.completeness == "partial"
|
||||
&& snapshot.sha256 == hex::decode(hash(&snapshot.data))?,
|
||||
"UPDATE_HEALTH_INVENTORY_BINDING"
|
||||
);
|
||||
rmm_domain::inventory_document::validate_for_scope(&snapshot.data, now, scope)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let id = Uuid::parse_str(&envelope.message_id)?;
|
||||
ensure!(
|
||||
!id.is_nil() && id.to_string() == envelope.message_id,
|
||||
"UPDATE_HEALTH_MESSAGE_ID"
|
||||
);
|
||||
let raw = envelope.encode_to_vec();
|
||||
ensure!(
|
||||
raw.len() <= rmm_domain::MAX_MESSAGE,
|
||||
"UPDATE_HEALTH_INVENTORY_SIZE"
|
||||
);
|
||||
store(
|
||||
db,
|
||||
header.attempt.grant_id,
|
||||
id,
|
||||
journal.handoff_sha256(),
|
||||
journal.head_sha256(),
|
||||
&raw,
|
||||
)
|
||||
}
|
||||
|
||||
fn store(
|
||||
db: &Connection,
|
||||
grant: Uuid,
|
||||
id: Uuid,
|
||||
handoff: &str,
|
||||
prefix: &str,
|
||||
raw: &[u8],
|
||||
) -> Result<()> {
|
||||
let existing:Option<(String,String,Vec<u8>,String)>=db.query_row("SELECT handoff_sha256,prefix_sha256,envelope,envelope_sha256 FROM update_health_inventory WHERE grant_id=?1 AND message_id=?2",params![grant.to_string(),id.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?))).optional()?;
|
||||
if let Some((h, p, b, digest)) = existing {
|
||||
ensure!(
|
||||
h == handoff && p == prefix && b == raw && digest == hash(raw),
|
||||
"UPDATE_HEALTH_RETAIN_CONFLICT"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let inserted=db.execute("INSERT INTO update_health_inventory(grant_id,message_id,handoff_sha256,prefix_sha256,envelope,envelope_sha256) SELECT ?1,?2,?3,?4,?5,?6 WHERE (SELECT count(*) FROM update_health_inventory WHERE grant_id=?1)<2",params![grant.to_string(),id.to_string(),handoff,prefix,raw,hash(raw)])?;
|
||||
ensure!(inserted == 1, "UPDATE_HEALTH_SAMPLE_LIMIT");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Only the exact current typed original inflight may request these IDs.
|
||||
pub fn pending_ids(db: &Connection, grant: Uuid) -> Result<Vec<String>> {
|
||||
let mut q=db.prepare("SELECT message_id FROM update_health_inventory WHERE grant_id=?1 AND signed_decision_id IS NULL ORDER BY rowid LIMIT 2")?;
|
||||
Ok(q.query_map([grant.to_string()], |r| r.get(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||
}
|
||||
|
||||
/// INTERNAL normal SignedSync apply hook, after independent signature/binding
|
||||
/// verification and durable response/receipt insertion in the same transaction.
|
||||
/// A plain transport ACK cannot call this path or supply a boolean assertion.
|
||||
pub(crate) fn acknowledge_signed(
|
||||
db: &Connection,
|
||||
grant: Uuid,
|
||||
decision: &ReconcileDecisionPayload,
|
||||
response: &SyncResponse,
|
||||
) -> Result<()> {
|
||||
ensure!(
|
||||
serde_json::to_vec(decision)? == response.decision_bytes,
|
||||
"UPDATE_HEALTH_SIGNED_BYTES"
|
||||
);
|
||||
let saved: Option<Vec<u8>> = db
|
||||
.query_row(
|
||||
"SELECT response FROM reconcile_requests WHERE id=?1",
|
||||
[decision.request_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.flatten();
|
||||
ensure!(
|
||||
saved.as_deref() == Some(response.encode_to_vec().as_slice()),
|
||||
"UPDATE_HEALTH_APPLY_NOT_RETAINED"
|
||||
);
|
||||
for id in &decision.acknowledged_message_ids {
|
||||
let receipt: Option<String> = db
|
||||
.query_row(
|
||||
"SELECT decision_id FROM reconcile_receipts WHERE id=?1",
|
||||
[id.to_string()],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
ensure!(
|
||||
receipt.as_deref() == Some(decision.decision_id.to_string().as_str()),
|
||||
"UPDATE_HEALTH_SIGNED_RECEIPT_MISSING"
|
||||
);
|
||||
db.execute("UPDATE update_health_inventory SET signed_decision_id=?3,signed_response_sha256=?4 WHERE grant_id=?1 AND message_id=?2 AND signed_decision_id IS NULL",params![grant.to_string(),id.to_string(),decision.decision_id.to_string(),hash(&response.encode_to_vec())])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Original raw protobuf bytes remain retained after BOTH kinds of ACK.
|
||||
pub fn read(db: &Connection, grant: Uuid, message: Uuid) -> Result<Envelope> {
|
||||
let (raw,digest):(Vec<u8>,String)=db.query_row("SELECT envelope,envelope_sha256 FROM update_health_inventory WHERE grant_id=?1 AND message_id=?2",params![grant.to_string(),message.to_string()],|r|Ok((r.get(0)?,r.get(1)?)))?;
|
||||
ensure!(
|
||||
raw.len() <= rmm_domain::MAX_MESSAGE && hash(&raw) == digest,
|
||||
"UPDATE_HEALTH_RAW_CHANGED"
|
||||
);
|
||||
let envelope = Envelope::decode(raw.as_slice())?;
|
||||
ensure!(
|
||||
envelope.encode_to_vec() == raw && envelope.message_id == message.to_string(),
|
||||
"UPDATE_HEALTH_RAW_CHANGED"
|
||||
);
|
||||
Ok(envelope)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn ordinary_ack_preserves_original_raw_and_signed_sync_pending_id() {
|
||||
let mut db = Connection::open_in_memory().unwrap();
|
||||
crate::reconcile::initialize(&db).unwrap();
|
||||
initialize(&db).unwrap();
|
||||
db.execute_batch("CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB NOT NULL);")
|
||||
.unwrap();
|
||||
let grant = Uuid::new_v4();
|
||||
let id = Uuid::new_v4();
|
||||
let envelope = Envelope {
|
||||
message_id: id.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let raw = envelope.encode_to_vec();
|
||||
store(&db, grant, id, "handoff", "prefix", &raw).unwrap();
|
||||
crate::jobs::event(&db, &envelope).unwrap();
|
||||
crate::jobs::ack(&mut db, &id.to_string()).unwrap();
|
||||
assert!(crate::jobs::events(&db).unwrap().is_empty());
|
||||
assert_eq!(pending_ids(&db, grant).unwrap(), vec![id.to_string()]);
|
||||
assert_eq!(read(&db, grant, id).unwrap().encode_to_vec(), raw);
|
||||
assert!(read(&db, Uuid::new_v4(), id).is_err());
|
||||
// Internal hook contract only: these synthetic bytes are not a
|
||||
// signature qualification. The normal verifier must run before the hook.
|
||||
let decision: ReconcileDecisionPayload = serde_json::from_value(serde_json::json!({
|
||||
"domain":"rmm.reconciliation-decision","schema_version":1,
|
||||
"decision_id":Uuid::new_v4(),"request_id":Uuid::new_v4(),"request_hash":"11".repeat(32),
|
||||
"tenant_id":grant,"asset_id":grant,"agent_id":grant,"enrollment_generation":1,
|
||||
"assignment_version":1,"connection_generation":2,"boot_id":grant,"journal_id":grant,
|
||||
"checkpoint":3,"helper_journal_id":grant,"helper_checkpoint":3,
|
||||
"issued_at":Utc::now(),"valid_until":Utc::now()+chrono::Duration::seconds(60),
|
||||
"ready":false,"helper_ready":false,"reason":"codec-only","attempts":[],
|
||||
"acknowledged_message_ids":[id],"key_id":"22".repeat(32)
|
||||
}))
|
||||
.unwrap();
|
||||
let response = SyncResponse {
|
||||
decision_bytes: serde_json::to_vec(&decision).unwrap(),
|
||||
signature: vec![1; 64],
|
||||
};
|
||||
assert!(acknowledge_signed(&db, grant, &decision, &response).is_err());
|
||||
db.execute("INSERT INTO reconcile_requests(id,request,envelope,generation,response) VALUES(?1,?2,?2,2,?3)",params![decision.request_id.to_string(),Vec::<u8>::new(),response.encode_to_vec()]).unwrap();
|
||||
assert!(acknowledge_signed(&db, grant, &decision, &response).is_err());
|
||||
db.execute(
|
||||
"INSERT INTO reconcile_receipts(id,decision_id) VALUES(?1,?2)",
|
||||
params![id.to_string(), decision.decision_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
acknowledge_signed(&db, grant, &decision, &response).unwrap();
|
||||
assert!(pending_ids(&db, grant).unwrap().is_empty());
|
||||
assert_eq!(read(&db, grant, id).unwrap().encode_to_vec(), raw);
|
||||
}
|
||||
#[test]
|
||||
fn retain_is_create_once_bounded_to_two_samples_and_detects_tampering() {
|
||||
let db = Connection::open_in_memory().unwrap();
|
||||
initialize(&db).unwrap();
|
||||
let grant = Uuid::new_v4();
|
||||
let id = Uuid::new_v4();
|
||||
let raw = Envelope {
|
||||
message_id: id.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec();
|
||||
store(&db, grant, id, "handoff", "prefix", &raw).unwrap();
|
||||
store(&db, grant, id, "handoff", "prefix", &raw).unwrap();
|
||||
assert!(store(&db, grant, id, "different", "prefix", &raw).is_err());
|
||||
store(&db, grant, Uuid::new_v4(), "handoff", "prefix", &[1]).unwrap();
|
||||
assert!(store(&db, grant, Uuid::new_v4(), "handoff", "prefix", &[2]).is_err());
|
||||
db.execute(
|
||||
"UPDATE update_health_inventory SET envelope=?2 WHERE message_id=?1",
|
||||
params![id.to_string(), vec![8_u8]],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(read(&db, grant, id).is_err());
|
||||
assert_eq!(pending_ids(&db, grant).unwrap().len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -1777,6 +1777,23 @@ fn update_inflight_negative_control_keeps_every_start_closed_and_owner_loss_is_t
|
||||
.apply(&mut f.db, &guard, &signed, &raw, &f.clock, &f.boot)
|
||||
.unwrap();
|
||||
assert!(!applied.ready && applied.unlocked_grants.is_empty());
|
||||
let local_apply = rmm_agent_core::update_handoff::read_applied_sync(
|
||||
&f.db,
|
||||
&guard,
|
||||
handoff.attempt.grant_id,
|
||||
decision.decision_id,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(local_apply.applied_at, f.clock.now);
|
||||
assert_eq!(local_apply.request_sha256, hash(&raw));
|
||||
let proto = rmm_protocol::SyncResponse {
|
||||
decision_bytes: signed.decision_bytes.clone(),
|
||||
signature: signed.signature.clone(),
|
||||
};
|
||||
assert_eq!(
|
||||
local_apply.response_protobuf_sha256,
|
||||
hash(&prost::Message::encode_to_vec(&proto))
|
||||
);
|
||||
assert!(!fresh.can_start(&f.db, &guard, 8, &f.clock, &f.boot));
|
||||
assert_eq!(
|
||||
f.db.query_row::<u32, _, _>(
|
||||
@@ -1974,3 +1991,63 @@ fn native_stop_intent_is_original_head_bound_create_once_and_transactional() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restarted_update_can_only_schedule_control_from_its_verified_original_not_unknown() {
|
||||
use rmm_agent_core::{jobs, ledger};
|
||||
let (f, _, request, _, _) = update_case();
|
||||
let app = Connection::open_in_memory().unwrap();
|
||||
app.execute_batch("CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER DEFAULT 0,grant_id TEXT DEFAULT '',cancel_requested INTEGER DEFAULT 0,PRIMARY KEY(job,attempt));").unwrap();
|
||||
let mut own = Connection::open_in_memory().unwrap();
|
||||
ledger::initialize_ledger(&own).unwrap();
|
||||
let payload: rmm_domain::jobs::JobPayload = serde_json::from_slice(&request.job_bytes).unwrap();
|
||||
let permit: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes).unwrap();
|
||||
let offer = rmm_protocol::JobOffer {
|
||||
job_id: payload.job_id.to_string(),
|
||||
attempt_id: payload.attempt_id.to_string(),
|
||||
authorization_id: request.authorization_id.to_string(),
|
||||
plan_hash: request.plan_hash.clone(),
|
||||
job_payload_hash: hash(&request.job_bytes),
|
||||
payload_bytes: request.job_bytes.clone(),
|
||||
};
|
||||
jobs::accept(&app, &f.binding, &offer, |job| {
|
||||
matches!(job.action, Action::UpdateAgent { .. })
|
||||
})
|
||||
.unwrap();
|
||||
assert!(!jobs::needs_update_control(&app, &own, &f.binding).unwrap());
|
||||
ledger::begin_execution(&mut own, &permit, &request, f.boot.0, f.clock.now).unwrap();
|
||||
jobs::transition(&app, &offer, "running", &permit.grant_id.to_string()).unwrap();
|
||||
let original: Vec<u8> = own
|
||||
.query_row("SELECT start FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert!(jobs::needs_update_control(&app, &own, &f.binding).unwrap());
|
||||
assert_eq!(
|
||||
app.query_row::<String, _, _>("SELECT state FROM job_attempts", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
"running"
|
||||
);
|
||||
assert_eq!(
|
||||
own.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM execution_evidence WHERE outcome IS NOT NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
own.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
|
||||
.unwrap();
|
||||
assert!(jobs::needs_update_control(&app, &own, &f.binding).is_err());
|
||||
own.execute(
|
||||
"UPDATE consumed SET agent=?1",
|
||||
[f.binding.agent_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
jobs::transition(&app, &offer, "unknown", &permit.grant_id.to_string()).unwrap();
|
||||
assert!(!jobs::needs_update_control(&app, &own, &f.binding).unwrap());
|
||||
assert_eq!(
|
||||
own.query_row::<Vec<u8>, _, _>("SELECT start FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
original
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user