Files
rust-discovery/crates/privileged-helper/src/checkpoint_repair.rs
T
2026-11-13 17:43:38 +01:00

459 lines
17 KiB
Rust

//! Fixed bootstrap verification of one signed no-effect checkpoint repair.
use crate::Binding;
use anyhow::{Context, Result, ensure};
use chrono::Utc;
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rusqlite::{Connection, OptionalExtension, params, types::ValueRef};
use serde_json::{Value, json};
pub const AUDIT: &str = "checkpoint_repair_receipts";
pub fn verify_historical_decision(
binding: &Binding,
signed: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
ensure!(
signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64,
"REPAIR_DECISION_SIZE"
);
let raw = hex::decode(&binding.public_key_hex)?;
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?
.verify_oneshot(&signed.signature, &signed.decision_bytes)?,
"REPAIR_HISTORICAL_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
ensure!(
d.valid_profile(d.issued_at)
&& d.key_id == hash(&raw)
&& d.tenant_id == binding.tenant_id
&& d.asset_id == binding.asset_id
&& d.agent_id == binding.agent_id
&& d.enrollment_generation == binding.enrollment_generation
&& d.assignment_version == binding.assignment_version,
"REPAIR_HISTORICAL_IDENTITY"
);
Ok(d)
}
pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result<Receipt> {
ensure!(
signed.payload.len() <= 16384 && signed.signature.len() == 64,
"REPAIR_RECEIPT_SIZE"
);
let raw = hex::decode(&binding.public_key_hex)?;
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?,
"REPAIR_RECEIPT_SIGNATURE"
);
let r: Receipt = serde_json::from_slice(&signed.payload)?;
ensure!(
r.valid_profile(Utc::now())
&& r.key_id == hash(&raw)
&& r.tenant_id == binding.tenant_id
&& r.asset_id == binding.asset_id
&& r.agent_id == binding.agent_id
&& r.enrollment_generation == binding.enrollment_generation
&& r.assignment_version == binding.assignment_version,
"REPAIR_RECEIPT_BINDING"
);
Ok(r)
}
/// Full database inventory with typed values; byte fields are bounded hashes.
/// The append-only repair archive is excluded; only the repaired fault cell is
/// normalized for exact idempotent replay of the same signed receipt.
pub fn inventory(db: &Connection, meta: &str) -> Result<Value> {
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY");
let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?;
ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM");
let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?;
let names = q
.query_map([AUDIT], |r| r.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut tables = serde_json::Map::new();
let mut count = 0;
for name in names {
ensure!(
name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
"REPAIR_TABLE_NAME"
);
let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?;
let columns = s
.column_names()
.into_iter()
.map(str::to_owned)
.collect::<Vec<_>>();
let mut rows = vec![];
let mut cursor = s.query([])?;
while let Some(row) = cursor.next()? {
count += 1;
ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT");
let mut cells = vec![];
for (i, col) in columns.iter().enumerate() {
let cell = if name == meta && col == "fault" {
let v = row.get_ref(i)?;
ensure!(
matches!(v, ValueRef::Null)
|| matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")),
"REPAIR_DIFFERENT_FAULT"
);
json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"})
} else {
match row.get_ref(i)? {
ValueRef::Null => json!({"type":"null"}),
ValueRef::Integer(v) => json!({"type":"integer","value":v}),
ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"),
ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}),
ValueRef::Blob(v) => {
json!({"type":"blob","sha256":hash(v),"length":v.len()})
}
}
};
cells.push(cell);
}
rows.push(cells);
}
tables.insert(name, json!({"columns":columns,"rows":rows}));
}
Ok(Value::Object(tables))
}
pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result<bool> {
let exists: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
[AUDIT],
|r| r.get(0),
)?;
if !exists {
return Ok(false);
}
let r: Receipt = serde_json::from_slice(&signed.payload)?;
let old: Option<Vec<u8>> = db
.query_row(
"SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1",
[r.receipt_id.to_string()],
|v| v.get(0),
)
.optional()?;
if let Some(old) = old {
ensure!(
old == serde_json::to_vec(signed)?,
"REPAIR_RECEIPT_CONFLICT"
);
Ok(true)
} else {
Ok(false)
}
}
pub fn commit_fault_repair(
db: &mut Connection,
meta: &str,
signed: &SignedReceipt,
expected_hash: &str,
additional: Option<(&str, Value)>,
) -> Result<()> {
ensure!(
matches!(meta, "reconcile_meta" | "helper_sync_meta"),
"REPAIR_META_TABLE"
);
let r: Receipt = serde_json::from_slice(&signed.payload)?;
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let mut actual = inventory(&tx, meta)?;
if let Some((key, value)) = additional {
actual
.as_object_mut()
.context("REPAIR_INVENTORY_OBJECT")?
.insert(key.into(), value);
}
ensure!(
inventory_sha256(&actual)? == expected_hash,
"REPAIR_WITNESS_CHANGED"
);
let fault: Option<String> =
tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| {
v.get(0)
})?;
if already_applied(&tx, signed)? {
ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED");
tx.commit()?;
return Ok(());
}
ensure!(
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"),
"REPAIR_EXACT_FAULT_REQUIRED"
);
tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?;
tx.execute(
"INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)",
params![
r.receipt_id.to_string(),
serde_json::to_vec(signed)?,
fault,
expected_hash,
Utc::now().to_rfc3339()
],
)?;
ensure!(
tx.execute(
&format!(
"UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'"
),
[]
)? == 1,
"REPAIR_FAULT_CHANGED"
);
let mut after = inventory(&tx, meta)?;
if let Some(value) = actual.get("agent_consumption") {
after
.as_object_mut()
.context("REPAIR_INVENTORY_OBJECT")?
.insert("agent_consumption".into(), value.clone());
}
ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED");
tx.commit()?;
Ok(())
}
pub fn verify_helper(
db: &Connection,
binding: &Binding,
signed: &SignedReceipt,
e: &Evidence,
) -> Result<()> {
let r = verify_receipt(binding, signed)?;
ensure!(
e.schema_version == 1
&& e.tenant_id == binding.tenant_id
&& e.asset_id == binding.asset_id
&& e.agent_id == binding.agent_id
&& e.enrollment_generation == binding.enrollment_generation
&& e.assignment_version == binding.assignment_version,
"REPAIR_HELPER_EVIDENCE_IDENTITY"
);
crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?;
let inv = inventory(db, "helper_sync_meta")?;
ensure!(
inventory_sha256(&inv)? == r.helper_inventory_sha256
&& inv == e.helper_inventory
&& helper_ledger_sha256(&inv)? == r.helper_ledger_sha256,
"REPAIR_HELPER_INVENTORY"
);
let (journal, checkpoint, decision, fault): (String, u64, String, Option<String>) = db
.query_row(
"SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1",
[],
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)),
)?;
ensure!(
journal == r.helper_journal_id.to_string()
&& checkpoint == r.helper_checkpoint
&& decision == r.rejected_decision_id.to_string()
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|| (fault.is_none() && already_applied(db, signed)?)),
"REPAIR_HELPER_HEAD"
);
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
[decision],
|v| Ok((v.get(0)?, v.get(1)?)),
)?;
let stored: SignedDecision = serde_json::from_slice(&response)?;
ensure!(
stored == e.rejected_response
&& request == e.rejected_request
&& hash(&request) == r.rejected_request_sha256
&& hash(&stored.decision_bytes) == r.rejected_decision_sha256,
"REPAIR_HELPER_REJECTION"
);
let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?;
ensure!(
rejection.reason == r.fault
&& !rejection.ready
&& !rejection.helper_ready
&& !rejection.attempts.iter().any(|v| v.unlock),
"REPAIR_REJECTION_HAS_AUTHORITY"
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use openssl::{pkey::Private, sign::Signer};
use uuid::Uuid;
fn fixture() -> (Binding, PKey<Private>, Receipt) {
let key = PKey::generate_ed25519().unwrap();
let id = Uuid::new_v4();
let now = Utc::now();
let binding = Binding {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
};
let r = Receipt {
domain: "rmm.sync.unsent-request-repair".into(),
schema_version: 1,
receipt_id: Uuid::new_v4(),
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
issued_at: now,
valid_until: now + chrono::Duration::minutes(15),
key_id: hash(&key.raw_public_key().unwrap()),
fault: "CHECKPOINT_CHAIN_MISMATCH".into(),
agent_journal_id: id,
agent_checkpoint: 13,
helper_journal_id: id,
helper_checkpoint: 22,
rejected_decision_id: id,
rejected_decision_sha256: hash(b"denied"),
rejected_request_id: id,
rejected_request_sha256: hash(b"request"),
uncommitted_request_id: id,
uncommitted_request_sha256: hash(b"unsent"),
previous_valid_decision_id: id,
previous_valid_decision_sha256: hash(b"previous"),
agent_inventory_sha256: hash(b"agent"),
helper_inventory_sha256: hash(b"helper"),
helper_ledger_sha256: hash(b"ledger"),
ready: false,
helper_ready: false,
unlock: false,
};
(binding, key, r)
}
fn sign(key: &PKey<Private>, r: &Receipt) -> SignedReceipt {
let payload = serde_json::to_vec(r).unwrap();
let signature = Signer::new_without_digest(key)
.unwrap()
.sign_oneshot_to_vec(&payload)
.unwrap();
SignedReceipt { payload, signature }
}
#[test]
fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() {
let (binding, key, r) = fixture();
assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok());
let mut bad = sign(&key, &r);
bad.payload[0] ^= 1;
assert!(verify_receipt(&binding, &bad).is_err());
for changed in [0, 1, 2, 3, 4, 5] {
let mut bad = r.clone();
match changed {
0 => bad.ready = true,
1 => bad.helper_ready = true,
2 => bad.unlock = true,
3 => bad.agent_id = Uuid::new_v4(),
4 => bad.domain = "rmm.execution-permit".into(),
_ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16),
};
assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err());
}
let mut expired = r.clone();
expired.issued_at = Utc::now() - chrono::Duration::minutes(2);
expired.valid_until = Utc::now() - chrono::Duration::minutes(1);
assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err());
}
fn database() -> Connection {
let db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap();
db
}
#[test]
fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() {
let (_, key, mut r) = fixture();
let mut db = database();
let before = inventory(&db, "helper_sync_meta").unwrap();
r.helper_inventory_sha256 = inventory_sha256(&before).unwrap();
r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap();
let signed = sign(&key, &r);
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)
.unwrap();
assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before);
assert!(
db.query_row("SELECT result IS NULL FROM consumed", [], |v| v
.get::<_, bool>(0))
.unwrap()
);
assert_eq!(
db.query_row("SELECT outcome FROM execution_evidence", [], |v| v
.get::<_, Vec<u8>>(0))
.unwrap(),
b"unknown"
);
assert_eq!(
db.query_row(
"SELECT old_fault FROM checkpoint_repair_receipts",
[],
|v| v.get::<_, String>(0)
)
.unwrap(),
"CHECKPOINT_CHAIN_MISMATCH"
);
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)
.unwrap();
assert_eq!(
db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v
.get::<_, u64>(0))
.unwrap(),
1
);
db.execute("UPDATE consumed SET result=x'01'", []).unwrap();
assert!(
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None
)
.is_err()
);
}
#[test]
fn changed_witness_or_other_fault_is_never_cleared() {
let (_, key, r) = fixture();
let mut db = database();
let signed = sign(&key, &r);
assert!(
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None
)
.is_err()
);
db.execute(
"UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'",
[],
)
.unwrap();
assert!(inventory(&db, "helper_sync_meta").is_err());
assert!(!already_applied(&db, &signed).unwrap());
let db = database();
db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap();
assert!(inventory(&db, "helper_sync_meta").is_err());
}
}