fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,458 @@
|
||||
//! Fixed bootstrap verification of one signed no-effect checkpoint repair.
|
||||
use crate::Binding;
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use openssl::{
|
||||
pkey::{Id, PKey},
|
||||
sign::Verifier,
|
||||
};
|
||||
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
|
||||
use rusqlite::{Connection, OptionalExtension, params, types::ValueRef};
|
||||
use serde_json::{Value, json};
|
||||
pub const AUDIT: &str = "checkpoint_repair_receipts";
|
||||
pub fn verify_historical_decision(
|
||||
binding: &Binding,
|
||||
signed: &SignedDecision,
|
||||
) -> Result<ReconcileDecisionPayload> {
|
||||
ensure!(
|
||||
signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64,
|
||||
"REPAIR_DECISION_SIZE"
|
||||
);
|
||||
let raw = hex::decode(&binding.public_key_hex)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?
|
||||
.verify_oneshot(&signed.signature, &signed.decision_bytes)?,
|
||||
"REPAIR_HISTORICAL_SIGNATURE"
|
||||
);
|
||||
let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
|
||||
ensure!(
|
||||
d.valid_profile(d.issued_at)
|
||||
&& d.key_id == hash(&raw)
|
||||
&& d.tenant_id == binding.tenant_id
|
||||
&& d.asset_id == binding.asset_id
|
||||
&& d.agent_id == binding.agent_id
|
||||
&& d.enrollment_generation == binding.enrollment_generation
|
||||
&& d.assignment_version == binding.assignment_version,
|
||||
"REPAIR_HISTORICAL_IDENTITY"
|
||||
);
|
||||
Ok(d)
|
||||
}
|
||||
pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result<Receipt> {
|
||||
ensure!(
|
||||
signed.payload.len() <= 16384 && signed.signature.len() == 64,
|
||||
"REPAIR_RECEIPT_SIZE"
|
||||
);
|
||||
let raw = hex::decode(&binding.public_key_hex)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?,
|
||||
"REPAIR_RECEIPT_SIGNATURE"
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
ensure!(
|
||||
r.valid_profile(Utc::now())
|
||||
&& r.key_id == hash(&raw)
|
||||
&& r.tenant_id == binding.tenant_id
|
||||
&& r.asset_id == binding.asset_id
|
||||
&& r.agent_id == binding.agent_id
|
||||
&& r.enrollment_generation == binding.enrollment_generation
|
||||
&& r.assignment_version == binding.assignment_version,
|
||||
"REPAIR_RECEIPT_BINDING"
|
||||
);
|
||||
Ok(r)
|
||||
}
|
||||
/// Full database inventory with typed values; byte fields are bounded hashes.
|
||||
/// The append-only repair archive is excluded; only the repaired fault cell is
|
||||
/// normalized for exact idempotent replay of the same signed receipt.
|
||||
pub fn inventory(db: &Connection, meta: &str) -> Result<Value> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY");
|
||||
let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?;
|
||||
ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM");
|
||||
let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?;
|
||||
let names = q
|
||||
.query_map([AUDIT], |r| r.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let mut tables = serde_json::Map::new();
|
||||
let mut count = 0;
|
||||
for name in names {
|
||||
ensure!(
|
||||
name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
|
||||
"REPAIR_TABLE_NAME"
|
||||
);
|
||||
let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?;
|
||||
let columns = s
|
||||
.column_names()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
let mut rows = vec![];
|
||||
let mut cursor = s.query([])?;
|
||||
while let Some(row) = cursor.next()? {
|
||||
count += 1;
|
||||
ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT");
|
||||
let mut cells = vec![];
|
||||
for (i, col) in columns.iter().enumerate() {
|
||||
let cell = if name == meta && col == "fault" {
|
||||
let v = row.get_ref(i)?;
|
||||
ensure!(
|
||||
matches!(v, ValueRef::Null)
|
||||
|| matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")),
|
||||
"REPAIR_DIFFERENT_FAULT"
|
||||
);
|
||||
json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"})
|
||||
} else {
|
||||
match row.get_ref(i)? {
|
||||
ValueRef::Null => json!({"type":"null"}),
|
||||
ValueRef::Integer(v) => json!({"type":"integer","value":v}),
|
||||
ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"),
|
||||
ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}),
|
||||
ValueRef::Blob(v) => {
|
||||
json!({"type":"blob","sha256":hash(v),"length":v.len()})
|
||||
}
|
||||
}
|
||||
};
|
||||
cells.push(cell);
|
||||
}
|
||||
rows.push(cells);
|
||||
}
|
||||
tables.insert(name, json!({"columns":columns,"rows":rows}));
|
||||
}
|
||||
Ok(Value::Object(tables))
|
||||
}
|
||||
pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result<bool> {
|
||||
let exists: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
|
||||
[AUDIT],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if !exists {
|
||||
return Ok(false);
|
||||
}
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let old: Option<Vec<u8>> = db
|
||||
.query_row(
|
||||
"SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1",
|
||||
[r.receipt_id.to_string()],
|
||||
|v| v.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some(old) = old {
|
||||
ensure!(
|
||||
old == serde_json::to_vec(signed)?,
|
||||
"REPAIR_RECEIPT_CONFLICT"
|
||||
);
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
pub fn commit_fault_repair(
|
||||
db: &mut Connection,
|
||||
meta: &str,
|
||||
signed: &SignedReceipt,
|
||||
expected_hash: &str,
|
||||
additional: Option<(&str, Value)>,
|
||||
) -> Result<()> {
|
||||
ensure!(
|
||||
matches!(meta, "reconcile_meta" | "helper_sync_meta"),
|
||||
"REPAIR_META_TABLE"
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
let mut actual = inventory(&tx, meta)?;
|
||||
if let Some((key, value)) = additional {
|
||||
actual
|
||||
.as_object_mut()
|
||||
.context("REPAIR_INVENTORY_OBJECT")?
|
||||
.insert(key.into(), value);
|
||||
}
|
||||
ensure!(
|
||||
inventory_sha256(&actual)? == expected_hash,
|
||||
"REPAIR_WITNESS_CHANGED"
|
||||
);
|
||||
let fault: Option<String> =
|
||||
tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| {
|
||||
v.get(0)
|
||||
})?;
|
||||
if already_applied(&tx, signed)? {
|
||||
ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED");
|
||||
tx.commit()?;
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"),
|
||||
"REPAIR_EXACT_FAULT_REQUIRED"
|
||||
);
|
||||
tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?;
|
||||
tx.execute(
|
||||
"INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)",
|
||||
params![
|
||||
r.receipt_id.to_string(),
|
||||
serde_json::to_vec(signed)?,
|
||||
fault,
|
||||
expected_hash,
|
||||
Utc::now().to_rfc3339()
|
||||
],
|
||||
)?;
|
||||
ensure!(
|
||||
tx.execute(
|
||||
&format!(
|
||||
"UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'"
|
||||
),
|
||||
[]
|
||||
)? == 1,
|
||||
"REPAIR_FAULT_CHANGED"
|
||||
);
|
||||
let mut after = inventory(&tx, meta)?;
|
||||
if let Some(value) = actual.get("agent_consumption") {
|
||||
after
|
||||
.as_object_mut()
|
||||
.context("REPAIR_INVENTORY_OBJECT")?
|
||||
.insert("agent_consumption".into(), value.clone());
|
||||
}
|
||||
ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED");
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn verify_helper(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
signed: &SignedReceipt,
|
||||
e: &Evidence,
|
||||
) -> Result<()> {
|
||||
let r = verify_receipt(binding, signed)?;
|
||||
ensure!(
|
||||
e.schema_version == 1
|
||||
&& e.tenant_id == binding.tenant_id
|
||||
&& e.asset_id == binding.asset_id
|
||||
&& e.agent_id == binding.agent_id
|
||||
&& e.enrollment_generation == binding.enrollment_generation
|
||||
&& e.assignment_version == binding.assignment_version,
|
||||
"REPAIR_HELPER_EVIDENCE_IDENTITY"
|
||||
);
|
||||
crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?;
|
||||
let inv = inventory(db, "helper_sync_meta")?;
|
||||
ensure!(
|
||||
inventory_sha256(&inv)? == r.helper_inventory_sha256
|
||||
&& inv == e.helper_inventory
|
||||
&& helper_ledger_sha256(&inv)? == r.helper_ledger_sha256,
|
||||
"REPAIR_HELPER_INVENTORY"
|
||||
);
|
||||
let (journal, checkpoint, decision, fault): (String, u64, String, Option<String>) = db
|
||||
.query_row(
|
||||
"SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1",
|
||||
[],
|
||||
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)),
|
||||
)?;
|
||||
ensure!(
|
||||
journal == r.helper_journal_id.to_string()
|
||||
&& checkpoint == r.helper_checkpoint
|
||||
&& decision == r.rejected_decision_id.to_string()
|
||||
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|
||||
|| (fault.is_none() && already_applied(db, signed)?)),
|
||||
"REPAIR_HELPER_HEAD"
|
||||
);
|
||||
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
|
||||
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[decision],
|
||||
|v| Ok((v.get(0)?, v.get(1)?)),
|
||||
)?;
|
||||
let stored: SignedDecision = serde_json::from_slice(&response)?;
|
||||
ensure!(
|
||||
stored == e.rejected_response
|
||||
&& request == e.rejected_request
|
||||
&& hash(&request) == r.rejected_request_sha256
|
||||
&& hash(&stored.decision_bytes) == r.rejected_decision_sha256,
|
||||
"REPAIR_HELPER_REJECTION"
|
||||
);
|
||||
let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?;
|
||||
ensure!(
|
||||
rejection.reason == r.fault
|
||||
&& !rejection.ready
|
||||
&& !rejection.helper_ready
|
||||
&& !rejection.attempts.iter().any(|v| v.unlock),
|
||||
"REPAIR_REJECTION_HAS_AUTHORITY"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use openssl::{pkey::Private, sign::Signer};
|
||||
use uuid::Uuid;
|
||||
fn fixture() -> (Binding, PKey<Private>, Receipt) {
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let id = Uuid::new_v4();
|
||||
let now = Utc::now();
|
||||
let binding = Binding {
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
|
||||
};
|
||||
let r = Receipt {
|
||||
domain: "rmm.sync.unsent-request-repair".into(),
|
||||
schema_version: 1,
|
||||
receipt_id: Uuid::new_v4(),
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
issued_at: now,
|
||||
valid_until: now + chrono::Duration::minutes(15),
|
||||
key_id: hash(&key.raw_public_key().unwrap()),
|
||||
fault: "CHECKPOINT_CHAIN_MISMATCH".into(),
|
||||
agent_journal_id: id,
|
||||
agent_checkpoint: 13,
|
||||
helper_journal_id: id,
|
||||
helper_checkpoint: 22,
|
||||
rejected_decision_id: id,
|
||||
rejected_decision_sha256: hash(b"denied"),
|
||||
rejected_request_id: id,
|
||||
rejected_request_sha256: hash(b"request"),
|
||||
uncommitted_request_id: id,
|
||||
uncommitted_request_sha256: hash(b"unsent"),
|
||||
previous_valid_decision_id: id,
|
||||
previous_valid_decision_sha256: hash(b"previous"),
|
||||
agent_inventory_sha256: hash(b"agent"),
|
||||
helper_inventory_sha256: hash(b"helper"),
|
||||
helper_ledger_sha256: hash(b"ledger"),
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
unlock: false,
|
||||
};
|
||||
(binding, key, r)
|
||||
}
|
||||
fn sign(key: &PKey<Private>, r: &Receipt) -> SignedReceipt {
|
||||
let payload = serde_json::to_vec(r).unwrap();
|
||||
let signature = Signer::new_without_digest(key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&payload)
|
||||
.unwrap();
|
||||
SignedReceipt { payload, signature }
|
||||
}
|
||||
#[test]
|
||||
fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() {
|
||||
let (binding, key, r) = fixture();
|
||||
assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok());
|
||||
let mut bad = sign(&key, &r);
|
||||
bad.payload[0] ^= 1;
|
||||
assert!(verify_receipt(&binding, &bad).is_err());
|
||||
for changed in [0, 1, 2, 3, 4, 5] {
|
||||
let mut bad = r.clone();
|
||||
match changed {
|
||||
0 => bad.ready = true,
|
||||
1 => bad.helper_ready = true,
|
||||
2 => bad.unlock = true,
|
||||
3 => bad.agent_id = Uuid::new_v4(),
|
||||
4 => bad.domain = "rmm.execution-permit".into(),
|
||||
_ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16),
|
||||
};
|
||||
assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err());
|
||||
}
|
||||
let mut expired = r.clone();
|
||||
expired.issued_at = Utc::now() - chrono::Duration::minutes(2);
|
||||
expired.valid_until = Utc::now() - chrono::Duration::minutes(1);
|
||||
assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err());
|
||||
}
|
||||
fn database() -> Connection {
|
||||
let db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap();
|
||||
db
|
||||
}
|
||||
#[test]
|
||||
fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() {
|
||||
let (_, key, mut r) = fixture();
|
||||
let mut db = database();
|
||||
let before = inventory(&db, "helper_sync_meta").unwrap();
|
||||
r.helper_inventory_sha256 = inventory_sha256(&before).unwrap();
|
||||
r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap();
|
||||
let signed = sign(&key, &r);
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before);
|
||||
assert!(
|
||||
db.query_row("SELECT result IS NULL FROM consumed", [], |v| v
|
||||
.get::<_, bool>(0))
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT outcome FROM execution_evidence", [], |v| v
|
||||
.get::<_, Vec<u8>>(0))
|
||||
.unwrap(),
|
||||
b"unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row(
|
||||
"SELECT old_fault FROM checkpoint_repair_receipts",
|
||||
[],
|
||||
|v| v.get::<_, String>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
"CHECKPOINT_CHAIN_MISMATCH"
|
||||
);
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v
|
||||
.get::<_, u64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
db.execute("UPDATE consumed SET result=x'01'", []).unwrap();
|
||||
assert!(
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn changed_witness_or_other_fault_is_never_cleared() {
|
||||
let (_, key, r) = fixture();
|
||||
let mut db = database();
|
||||
let signed = sign(&key, &r);
|
||||
assert!(
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
db.execute(
|
||||
"UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(inventory(&db, "helper_sync_meta").is_err());
|
||||
assert!(!already_applied(&db, &signed).unwrap());
|
||||
let db = database();
|
||||
db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap();
|
||||
assert!(inventory(&db, "helper_sync_meta").is_err());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user