fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-13 17:43:38 +01:00
parent 998f16fde4
commit 359233c64d
21 changed files with 3030 additions and 30 deletions
@@ -0,0 +1,458 @@
//! Fixed bootstrap verification of one signed no-effect checkpoint repair.
use crate::Binding;
use anyhow::{Context, Result, ensure};
use chrono::Utc;
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rusqlite::{Connection, OptionalExtension, params, types::ValueRef};
use serde_json::{Value, json};
pub const AUDIT: &str = "checkpoint_repair_receipts";
pub fn verify_historical_decision(
binding: &Binding,
signed: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
ensure!(
signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64,
"REPAIR_DECISION_SIZE"
);
let raw = hex::decode(&binding.public_key_hex)?;
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?
.verify_oneshot(&signed.signature, &signed.decision_bytes)?,
"REPAIR_HISTORICAL_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
ensure!(
d.valid_profile(d.issued_at)
&& d.key_id == hash(&raw)
&& d.tenant_id == binding.tenant_id
&& d.asset_id == binding.asset_id
&& d.agent_id == binding.agent_id
&& d.enrollment_generation == binding.enrollment_generation
&& d.assignment_version == binding.assignment_version,
"REPAIR_HISTORICAL_IDENTITY"
);
Ok(d)
}
pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result<Receipt> {
ensure!(
signed.payload.len() <= 16384 && signed.signature.len() == 64,
"REPAIR_RECEIPT_SIZE"
);
let raw = hex::decode(&binding.public_key_hex)?;
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?,
"REPAIR_RECEIPT_SIGNATURE"
);
let r: Receipt = serde_json::from_slice(&signed.payload)?;
ensure!(
r.valid_profile(Utc::now())
&& r.key_id == hash(&raw)
&& r.tenant_id == binding.tenant_id
&& r.asset_id == binding.asset_id
&& r.agent_id == binding.agent_id
&& r.enrollment_generation == binding.enrollment_generation
&& r.assignment_version == binding.assignment_version,
"REPAIR_RECEIPT_BINDING"
);
Ok(r)
}
/// Full database inventory with typed values; byte fields are bounded hashes.
/// The append-only repair archive is excluded; only the repaired fault cell is
/// normalized for exact idempotent replay of the same signed receipt.
pub fn inventory(db: &Connection, meta: &str) -> Result<Value> {
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY");
let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?;
ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM");
let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?;
let names = q
.query_map([AUDIT], |r| r.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut tables = serde_json::Map::new();
let mut count = 0;
for name in names {
ensure!(
name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
"REPAIR_TABLE_NAME"
);
let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?;
let columns = s
.column_names()
.into_iter()
.map(str::to_owned)
.collect::<Vec<_>>();
let mut rows = vec![];
let mut cursor = s.query([])?;
while let Some(row) = cursor.next()? {
count += 1;
ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT");
let mut cells = vec![];
for (i, col) in columns.iter().enumerate() {
let cell = if name == meta && col == "fault" {
let v = row.get_ref(i)?;
ensure!(
matches!(v, ValueRef::Null)
|| matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")),
"REPAIR_DIFFERENT_FAULT"
);
json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"})
} else {
match row.get_ref(i)? {
ValueRef::Null => json!({"type":"null"}),
ValueRef::Integer(v) => json!({"type":"integer","value":v}),
ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"),
ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}),
ValueRef::Blob(v) => {
json!({"type":"blob","sha256":hash(v),"length":v.len()})
}
}
};
cells.push(cell);
}
rows.push(cells);
}
tables.insert(name, json!({"columns":columns,"rows":rows}));
}
Ok(Value::Object(tables))
}
pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result<bool> {
let exists: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
[AUDIT],
|r| r.get(0),
)?;
if !exists {
return Ok(false);
}
let r: Receipt = serde_json::from_slice(&signed.payload)?;
let old: Option<Vec<u8>> = db
.query_row(
"SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1",
[r.receipt_id.to_string()],
|v| v.get(0),
)
.optional()?;
if let Some(old) = old {
ensure!(
old == serde_json::to_vec(signed)?,
"REPAIR_RECEIPT_CONFLICT"
);
Ok(true)
} else {
Ok(false)
}
}
pub fn commit_fault_repair(
db: &mut Connection,
meta: &str,
signed: &SignedReceipt,
expected_hash: &str,
additional: Option<(&str, Value)>,
) -> Result<()> {
ensure!(
matches!(meta, "reconcile_meta" | "helper_sync_meta"),
"REPAIR_META_TABLE"
);
let r: Receipt = serde_json::from_slice(&signed.payload)?;
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let mut actual = inventory(&tx, meta)?;
if let Some((key, value)) = additional {
actual
.as_object_mut()
.context("REPAIR_INVENTORY_OBJECT")?
.insert(key.into(), value);
}
ensure!(
inventory_sha256(&actual)? == expected_hash,
"REPAIR_WITNESS_CHANGED"
);
let fault: Option<String> =
tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| {
v.get(0)
})?;
if already_applied(&tx, signed)? {
ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED");
tx.commit()?;
return Ok(());
}
ensure!(
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"),
"REPAIR_EXACT_FAULT_REQUIRED"
);
tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?;
tx.execute(
"INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)",
params![
r.receipt_id.to_string(),
serde_json::to_vec(signed)?,
fault,
expected_hash,
Utc::now().to_rfc3339()
],
)?;
ensure!(
tx.execute(
&format!(
"UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'"
),
[]
)? == 1,
"REPAIR_FAULT_CHANGED"
);
let mut after = inventory(&tx, meta)?;
if let Some(value) = actual.get("agent_consumption") {
after
.as_object_mut()
.context("REPAIR_INVENTORY_OBJECT")?
.insert("agent_consumption".into(), value.clone());
}
ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED");
tx.commit()?;
Ok(())
}
pub fn verify_helper(
db: &Connection,
binding: &Binding,
signed: &SignedReceipt,
e: &Evidence,
) -> Result<()> {
let r = verify_receipt(binding, signed)?;
ensure!(
e.schema_version == 1
&& e.tenant_id == binding.tenant_id
&& e.asset_id == binding.asset_id
&& e.agent_id == binding.agent_id
&& e.enrollment_generation == binding.enrollment_generation
&& e.assignment_version == binding.assignment_version,
"REPAIR_HELPER_EVIDENCE_IDENTITY"
);
crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?;
let inv = inventory(db, "helper_sync_meta")?;
ensure!(
inventory_sha256(&inv)? == r.helper_inventory_sha256
&& inv == e.helper_inventory
&& helper_ledger_sha256(&inv)? == r.helper_ledger_sha256,
"REPAIR_HELPER_INVENTORY"
);
let (journal, checkpoint, decision, fault): (String, u64, String, Option<String>) = db
.query_row(
"SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1",
[],
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)),
)?;
ensure!(
journal == r.helper_journal_id.to_string()
&& checkpoint == r.helper_checkpoint
&& decision == r.rejected_decision_id.to_string()
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|| (fault.is_none() && already_applied(db, signed)?)),
"REPAIR_HELPER_HEAD"
);
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
[decision],
|v| Ok((v.get(0)?, v.get(1)?)),
)?;
let stored: SignedDecision = serde_json::from_slice(&response)?;
ensure!(
stored == e.rejected_response
&& request == e.rejected_request
&& hash(&request) == r.rejected_request_sha256
&& hash(&stored.decision_bytes) == r.rejected_decision_sha256,
"REPAIR_HELPER_REJECTION"
);
let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?;
ensure!(
rejection.reason == r.fault
&& !rejection.ready
&& !rejection.helper_ready
&& !rejection.attempts.iter().any(|v| v.unlock),
"REPAIR_REJECTION_HAS_AUTHORITY"
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use openssl::{pkey::Private, sign::Signer};
use uuid::Uuid;
fn fixture() -> (Binding, PKey<Private>, Receipt) {
let key = PKey::generate_ed25519().unwrap();
let id = Uuid::new_v4();
let now = Utc::now();
let binding = Binding {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
};
let r = Receipt {
domain: "rmm.sync.unsent-request-repair".into(),
schema_version: 1,
receipt_id: Uuid::new_v4(),
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
issued_at: now,
valid_until: now + chrono::Duration::minutes(15),
key_id: hash(&key.raw_public_key().unwrap()),
fault: "CHECKPOINT_CHAIN_MISMATCH".into(),
agent_journal_id: id,
agent_checkpoint: 13,
helper_journal_id: id,
helper_checkpoint: 22,
rejected_decision_id: id,
rejected_decision_sha256: hash(b"denied"),
rejected_request_id: id,
rejected_request_sha256: hash(b"request"),
uncommitted_request_id: id,
uncommitted_request_sha256: hash(b"unsent"),
previous_valid_decision_id: id,
previous_valid_decision_sha256: hash(b"previous"),
agent_inventory_sha256: hash(b"agent"),
helper_inventory_sha256: hash(b"helper"),
helper_ledger_sha256: hash(b"ledger"),
ready: false,
helper_ready: false,
unlock: false,
};
(binding, key, r)
}
fn sign(key: &PKey<Private>, r: &Receipt) -> SignedReceipt {
let payload = serde_json::to_vec(r).unwrap();
let signature = Signer::new_without_digest(key)
.unwrap()
.sign_oneshot_to_vec(&payload)
.unwrap();
SignedReceipt { payload, signature }
}
#[test]
fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() {
let (binding, key, r) = fixture();
assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok());
let mut bad = sign(&key, &r);
bad.payload[0] ^= 1;
assert!(verify_receipt(&binding, &bad).is_err());
for changed in [0, 1, 2, 3, 4, 5] {
let mut bad = r.clone();
match changed {
0 => bad.ready = true,
1 => bad.helper_ready = true,
2 => bad.unlock = true,
3 => bad.agent_id = Uuid::new_v4(),
4 => bad.domain = "rmm.execution-permit".into(),
_ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16),
};
assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err());
}
let mut expired = r.clone();
expired.issued_at = Utc::now() - chrono::Duration::minutes(2);
expired.valid_until = Utc::now() - chrono::Duration::minutes(1);
assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err());
}
fn database() -> Connection {
let db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap();
db
}
#[test]
fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() {
let (_, key, mut r) = fixture();
let mut db = database();
let before = inventory(&db, "helper_sync_meta").unwrap();
r.helper_inventory_sha256 = inventory_sha256(&before).unwrap();
r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap();
let signed = sign(&key, &r);
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)
.unwrap();
assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before);
assert!(
db.query_row("SELECT result IS NULL FROM consumed", [], |v| v
.get::<_, bool>(0))
.unwrap()
);
assert_eq!(
db.query_row("SELECT outcome FROM execution_evidence", [], |v| v
.get::<_, Vec<u8>>(0))
.unwrap(),
b"unknown"
);
assert_eq!(
db.query_row(
"SELECT old_fault FROM checkpoint_repair_receipts",
[],
|v| v.get::<_, String>(0)
)
.unwrap(),
"CHECKPOINT_CHAIN_MISMATCH"
);
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)
.unwrap();
assert_eq!(
db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v
.get::<_, u64>(0))
.unwrap(),
1
);
db.execute("UPDATE consumed SET result=x'01'", []).unwrap();
assert!(
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None
)
.is_err()
);
}
#[test]
fn changed_witness_or_other_fault_is_never_cleared() {
let (_, key, r) = fixture();
let mut db = database();
let signed = sign(&key, &r);
assert!(
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None
)
.is_err()
);
db.execute(
"UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'",
[],
)
.unwrap();
assert!(inventory(&db, "helper_sync_meta").is_err());
assert!(!already_applied(&db, &signed).unwrap());
let db = database();
db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap();
assert!(inventory(&db, "helper_sync_meta").is_err());
}
}
+1
View File
@@ -3,6 +3,7 @@
extern crate self as rmm_privileged_helper;
pub mod apt;
pub mod apt_reconcile;
pub mod checkpoint_repair;
pub mod control;
pub mod readiness;
pub mod reconcile;
+143
View File
@@ -681,6 +681,142 @@ fn validate_history(
}
Ok(())
}
/// Bootstrap-only historical verification; this never creates a lease.
pub fn verify_checkpoint_repair_history(
db: &Connection,
binding: &Binding,
links: Option<(
&rmm_domain::checkpoint_repair::Receipt,
&rmm_domain::checkpoint_repair::Evidence,
)>,
) -> Result<()> {
let meta = metadata(db)?;
validate_history(db, binding, &meta, meta.generation)?;
let mut statement = db.prepare(
"SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint",
)?;
let entries = statement
.query_map([], |r| {
Ok((
r.get::<_, Vec<u8>>(0)?,
r.get::<_, Vec<u8>>(1)?,
r.get::<_, u64>(2)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT");
let mut previous = None;
let mut previous_checkpoint = 0;
let mut previous_agent_checkpoint = 0;
for (signed, raw, checkpoint) in entries {
let response: SignedResponse = serde_json::from_slice(&signed)?;
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
let decision = verify(binding, &response, parsed.issued_at)?;
let request = RequestView::decode(&raw)?;
let captured: Vec<u8> = db.query_row(
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
[checkpoint],
|r| r.get(0),
)?;
let capture: Snapshot = serde_json::from_slice(&captured)?;
ensure!(
decision.helper_journal_id == Some(meta.journal_id)
&& decision.helper_checkpoint == checkpoint
&& request.helper_journal_id == meta.journal_id.to_string()
&& request.helper_checkpoint == checkpoint
&& request.assignment_version == binding.assignment_version
&& request.journal_complete
&& decision.request_hash == hash(&raw)
&& decision.request_id.to_string() == request.request_id
&& decision.journal_id.to_string() == request.journal_id
&& decision.checkpoint == request.checkpoint
&& capture.journal_id == meta.journal_id
&& capture.checkpoint == checkpoint
&& capture.boot_id == decision.boot_id
&& capture.captured_at <= decision.issued_at
&& capture.previous_decision_id == previous
&& checkpoint > previous_checkpoint
&& decision.checkpoint >= previous_agent_checkpoint
&& decision.connection_generation <= meta.generation
&& request.helper_previous_decision_id
== capture
.previous_decision_id
.map(|v| v.to_string())
.unwrap_or_default(),
"REPAIR_HELPER_HISTORY_BINDING"
);
previous = Some(decision.decision_id);
previous_checkpoint = checkpoint;
previous_agent_checkpoint = decision.checkpoint;
}
let _ = load_rows(db, binding, false)?;
if let Some((r, e)) = links {
verify_checkpoint_repair_links(db, binding, r, e)?;
}
Ok(())
}
fn verify_checkpoint_repair_links(
db: &Connection,
binding: &Binding,
r: &rmm_domain::checkpoint_repair::Receipt,
e: &rmm_domain::checkpoint_repair::Evidence,
) -> Result<()> {
let x = RequestView::decode(&e.uncommitted_request)?;
let y = RequestView::decode(&e.rejected_request)?;
let a: ReconcileDecisionPayload =
serde_json::from_slice(&e.previous_valid_response.decision_bytes)?;
let a = verify(
binding,
&SignedResponse {
decision_bytes: e.previous_valid_response.decision_bytes.clone(),
signature: e.previous_valid_response.signature.clone(),
},
a.issued_at,
)?;
let old: Vec<u8> = db.query_row(
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
[a.decision_id.to_string()],
|v| v.get(0),
)?;
let old: SignedResponse = serde_json::from_slice(&old)?;
ensure!(
old.decision_bytes == e.previous_valid_response.decision_bytes
&& old.signature == e.previous_valid_response.signature
&& a.decision_id == r.previous_valid_decision_id
&& hash(&old.decision_bytes) == r.previous_valid_decision_sha256
&& a.reason == "ATTEMPT_RECONCILIATION_REQUIRED"
&& !a.ready
&& !a.helper_ready
&& x.request_id == r.uncommitted_request_id.to_string()
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
&& y.request_id == r.rejected_request_id.to_string()
&& hash(&e.rejected_request) == r.rejected_request_sha256
&& x.previous_decision_id == a.decision_id.to_string()
&& x.previous_request_id.is_empty()
&& x.helper_previous_decision_id == a.decision_id.to_string()
&& y.helper_previous_decision_id == a.decision_id.to_string()
&& y.previous_decision_id.is_empty()
&& y.previous_request_id == x.request_id
&& x.journal_id == r.agent_journal_id.to_string()
&& y.journal_id == x.journal_id
&& a.journal_id == r.agent_journal_id
&& x.helper_journal_id == r.helper_journal_id.to_string()
&& y.helper_journal_id == x.helper_journal_id
&& a.helper_journal_id == Some(r.helper_journal_id)
&& a.checkpoint < x.checkpoint
&& x.checkpoint < y.checkpoint
&& y.checkpoint == r.agent_checkpoint
&& a.helper_checkpoint < x.helper_checkpoint
&& x.helper_checkpoint < y.helper_checkpoint
&& y.helper_checkpoint == r.helper_checkpoint
&& x.assignment_version == binding.assignment_version
&& y.assignment_version == binding.assignment_version
&& x.journal_complete
&& y.journal_complete,
"REPAIR_EXACT_UNSENT_CHAIN_REQUIRED"
);
Ok(())
}
fn persist_fault(db: &Connection, error: &anyhow::Error) {
let text = error.to_string();
if [
@@ -991,6 +1127,8 @@ struct RequestView {
helper_journal_id: String,
helper_checkpoint: u64,
helper_previous_decision_id: String,
previous_decision_id: String,
previous_request_id: String,
}
#[derive(Default)]
struct AttemptView {
@@ -1089,6 +1227,11 @@ impl RequestView {
if !s.is_empty() {
Uuid::parse_str(&s)?;
}
if tag == 4 {
r.previous_decision_id = s;
} else {
r.previous_request_id = s;
}
}
5 => r.assignment_version = number(value)?,
6 => {