fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,458 @@
|
||||
//! Fixed bootstrap verification of one signed no-effect checkpoint repair.
|
||||
use crate::Binding;
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use openssl::{
|
||||
pkey::{Id, PKey},
|
||||
sign::Verifier,
|
||||
};
|
||||
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
|
||||
use rusqlite::{Connection, OptionalExtension, params, types::ValueRef};
|
||||
use serde_json::{Value, json};
|
||||
pub const AUDIT: &str = "checkpoint_repair_receipts";
|
||||
pub fn verify_historical_decision(
|
||||
binding: &Binding,
|
||||
signed: &SignedDecision,
|
||||
) -> Result<ReconcileDecisionPayload> {
|
||||
ensure!(
|
||||
signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64,
|
||||
"REPAIR_DECISION_SIZE"
|
||||
);
|
||||
let raw = hex::decode(&binding.public_key_hex)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?
|
||||
.verify_oneshot(&signed.signature, &signed.decision_bytes)?,
|
||||
"REPAIR_HISTORICAL_SIGNATURE"
|
||||
);
|
||||
let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
|
||||
ensure!(
|
||||
d.valid_profile(d.issued_at)
|
||||
&& d.key_id == hash(&raw)
|
||||
&& d.tenant_id == binding.tenant_id
|
||||
&& d.asset_id == binding.asset_id
|
||||
&& d.agent_id == binding.agent_id
|
||||
&& d.enrollment_generation == binding.enrollment_generation
|
||||
&& d.assignment_version == binding.assignment_version,
|
||||
"REPAIR_HISTORICAL_IDENTITY"
|
||||
);
|
||||
Ok(d)
|
||||
}
|
||||
pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result<Receipt> {
|
||||
ensure!(
|
||||
signed.payload.len() <= 16384 && signed.signature.len() == 64,
|
||||
"REPAIR_RECEIPT_SIZE"
|
||||
);
|
||||
let raw = hex::decode(&binding.public_key_hex)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?,
|
||||
"REPAIR_RECEIPT_SIGNATURE"
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
ensure!(
|
||||
r.valid_profile(Utc::now())
|
||||
&& r.key_id == hash(&raw)
|
||||
&& r.tenant_id == binding.tenant_id
|
||||
&& r.asset_id == binding.asset_id
|
||||
&& r.agent_id == binding.agent_id
|
||||
&& r.enrollment_generation == binding.enrollment_generation
|
||||
&& r.assignment_version == binding.assignment_version,
|
||||
"REPAIR_RECEIPT_BINDING"
|
||||
);
|
||||
Ok(r)
|
||||
}
|
||||
/// Full database inventory with typed values; byte fields are bounded hashes.
|
||||
/// The append-only repair archive is excluded; only the repaired fault cell is
|
||||
/// normalized for exact idempotent replay of the same signed receipt.
|
||||
pub fn inventory(db: &Connection, meta: &str) -> Result<Value> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY");
|
||||
let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?;
|
||||
ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM");
|
||||
let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?;
|
||||
let names = q
|
||||
.query_map([AUDIT], |r| r.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let mut tables = serde_json::Map::new();
|
||||
let mut count = 0;
|
||||
for name in names {
|
||||
ensure!(
|
||||
name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
|
||||
"REPAIR_TABLE_NAME"
|
||||
);
|
||||
let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?;
|
||||
let columns = s
|
||||
.column_names()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
let mut rows = vec![];
|
||||
let mut cursor = s.query([])?;
|
||||
while let Some(row) = cursor.next()? {
|
||||
count += 1;
|
||||
ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT");
|
||||
let mut cells = vec![];
|
||||
for (i, col) in columns.iter().enumerate() {
|
||||
let cell = if name == meta && col == "fault" {
|
||||
let v = row.get_ref(i)?;
|
||||
ensure!(
|
||||
matches!(v, ValueRef::Null)
|
||||
|| matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")),
|
||||
"REPAIR_DIFFERENT_FAULT"
|
||||
);
|
||||
json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"})
|
||||
} else {
|
||||
match row.get_ref(i)? {
|
||||
ValueRef::Null => json!({"type":"null"}),
|
||||
ValueRef::Integer(v) => json!({"type":"integer","value":v}),
|
||||
ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"),
|
||||
ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}),
|
||||
ValueRef::Blob(v) => {
|
||||
json!({"type":"blob","sha256":hash(v),"length":v.len()})
|
||||
}
|
||||
}
|
||||
};
|
||||
cells.push(cell);
|
||||
}
|
||||
rows.push(cells);
|
||||
}
|
||||
tables.insert(name, json!({"columns":columns,"rows":rows}));
|
||||
}
|
||||
Ok(Value::Object(tables))
|
||||
}
|
||||
pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result<bool> {
|
||||
let exists: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
|
||||
[AUDIT],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if !exists {
|
||||
return Ok(false);
|
||||
}
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let old: Option<Vec<u8>> = db
|
||||
.query_row(
|
||||
"SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1",
|
||||
[r.receipt_id.to_string()],
|
||||
|v| v.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some(old) = old {
|
||||
ensure!(
|
||||
old == serde_json::to_vec(signed)?,
|
||||
"REPAIR_RECEIPT_CONFLICT"
|
||||
);
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
pub fn commit_fault_repair(
|
||||
db: &mut Connection,
|
||||
meta: &str,
|
||||
signed: &SignedReceipt,
|
||||
expected_hash: &str,
|
||||
additional: Option<(&str, Value)>,
|
||||
) -> Result<()> {
|
||||
ensure!(
|
||||
matches!(meta, "reconcile_meta" | "helper_sync_meta"),
|
||||
"REPAIR_META_TABLE"
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
let mut actual = inventory(&tx, meta)?;
|
||||
if let Some((key, value)) = additional {
|
||||
actual
|
||||
.as_object_mut()
|
||||
.context("REPAIR_INVENTORY_OBJECT")?
|
||||
.insert(key.into(), value);
|
||||
}
|
||||
ensure!(
|
||||
inventory_sha256(&actual)? == expected_hash,
|
||||
"REPAIR_WITNESS_CHANGED"
|
||||
);
|
||||
let fault: Option<String> =
|
||||
tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| {
|
||||
v.get(0)
|
||||
})?;
|
||||
if already_applied(&tx, signed)? {
|
||||
ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED");
|
||||
tx.commit()?;
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"),
|
||||
"REPAIR_EXACT_FAULT_REQUIRED"
|
||||
);
|
||||
tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?;
|
||||
tx.execute(
|
||||
"INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)",
|
||||
params![
|
||||
r.receipt_id.to_string(),
|
||||
serde_json::to_vec(signed)?,
|
||||
fault,
|
||||
expected_hash,
|
||||
Utc::now().to_rfc3339()
|
||||
],
|
||||
)?;
|
||||
ensure!(
|
||||
tx.execute(
|
||||
&format!(
|
||||
"UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'"
|
||||
),
|
||||
[]
|
||||
)? == 1,
|
||||
"REPAIR_FAULT_CHANGED"
|
||||
);
|
||||
let mut after = inventory(&tx, meta)?;
|
||||
if let Some(value) = actual.get("agent_consumption") {
|
||||
after
|
||||
.as_object_mut()
|
||||
.context("REPAIR_INVENTORY_OBJECT")?
|
||||
.insert("agent_consumption".into(), value.clone());
|
||||
}
|
||||
ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED");
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn verify_helper(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
signed: &SignedReceipt,
|
||||
e: &Evidence,
|
||||
) -> Result<()> {
|
||||
let r = verify_receipt(binding, signed)?;
|
||||
ensure!(
|
||||
e.schema_version == 1
|
||||
&& e.tenant_id == binding.tenant_id
|
||||
&& e.asset_id == binding.asset_id
|
||||
&& e.agent_id == binding.agent_id
|
||||
&& e.enrollment_generation == binding.enrollment_generation
|
||||
&& e.assignment_version == binding.assignment_version,
|
||||
"REPAIR_HELPER_EVIDENCE_IDENTITY"
|
||||
);
|
||||
crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?;
|
||||
let inv = inventory(db, "helper_sync_meta")?;
|
||||
ensure!(
|
||||
inventory_sha256(&inv)? == r.helper_inventory_sha256
|
||||
&& inv == e.helper_inventory
|
||||
&& helper_ledger_sha256(&inv)? == r.helper_ledger_sha256,
|
||||
"REPAIR_HELPER_INVENTORY"
|
||||
);
|
||||
let (journal, checkpoint, decision, fault): (String, u64, String, Option<String>) = db
|
||||
.query_row(
|
||||
"SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1",
|
||||
[],
|
||||
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)),
|
||||
)?;
|
||||
ensure!(
|
||||
journal == r.helper_journal_id.to_string()
|
||||
&& checkpoint == r.helper_checkpoint
|
||||
&& decision == r.rejected_decision_id.to_string()
|
||||
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|
||||
|| (fault.is_none() && already_applied(db, signed)?)),
|
||||
"REPAIR_HELPER_HEAD"
|
||||
);
|
||||
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
|
||||
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[decision],
|
||||
|v| Ok((v.get(0)?, v.get(1)?)),
|
||||
)?;
|
||||
let stored: SignedDecision = serde_json::from_slice(&response)?;
|
||||
ensure!(
|
||||
stored == e.rejected_response
|
||||
&& request == e.rejected_request
|
||||
&& hash(&request) == r.rejected_request_sha256
|
||||
&& hash(&stored.decision_bytes) == r.rejected_decision_sha256,
|
||||
"REPAIR_HELPER_REJECTION"
|
||||
);
|
||||
let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?;
|
||||
ensure!(
|
||||
rejection.reason == r.fault
|
||||
&& !rejection.ready
|
||||
&& !rejection.helper_ready
|
||||
&& !rejection.attempts.iter().any(|v| v.unlock),
|
||||
"REPAIR_REJECTION_HAS_AUTHORITY"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use openssl::{pkey::Private, sign::Signer};
|
||||
use uuid::Uuid;
|
||||
fn fixture() -> (Binding, PKey<Private>, Receipt) {
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let id = Uuid::new_v4();
|
||||
let now = Utc::now();
|
||||
let binding = Binding {
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
|
||||
};
|
||||
let r = Receipt {
|
||||
domain: "rmm.sync.unsent-request-repair".into(),
|
||||
schema_version: 1,
|
||||
receipt_id: Uuid::new_v4(),
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
issued_at: now,
|
||||
valid_until: now + chrono::Duration::minutes(15),
|
||||
key_id: hash(&key.raw_public_key().unwrap()),
|
||||
fault: "CHECKPOINT_CHAIN_MISMATCH".into(),
|
||||
agent_journal_id: id,
|
||||
agent_checkpoint: 13,
|
||||
helper_journal_id: id,
|
||||
helper_checkpoint: 22,
|
||||
rejected_decision_id: id,
|
||||
rejected_decision_sha256: hash(b"denied"),
|
||||
rejected_request_id: id,
|
||||
rejected_request_sha256: hash(b"request"),
|
||||
uncommitted_request_id: id,
|
||||
uncommitted_request_sha256: hash(b"unsent"),
|
||||
previous_valid_decision_id: id,
|
||||
previous_valid_decision_sha256: hash(b"previous"),
|
||||
agent_inventory_sha256: hash(b"agent"),
|
||||
helper_inventory_sha256: hash(b"helper"),
|
||||
helper_ledger_sha256: hash(b"ledger"),
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
unlock: false,
|
||||
};
|
||||
(binding, key, r)
|
||||
}
|
||||
fn sign(key: &PKey<Private>, r: &Receipt) -> SignedReceipt {
|
||||
let payload = serde_json::to_vec(r).unwrap();
|
||||
let signature = Signer::new_without_digest(key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&payload)
|
||||
.unwrap();
|
||||
SignedReceipt { payload, signature }
|
||||
}
|
||||
#[test]
|
||||
fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() {
|
||||
let (binding, key, r) = fixture();
|
||||
assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok());
|
||||
let mut bad = sign(&key, &r);
|
||||
bad.payload[0] ^= 1;
|
||||
assert!(verify_receipt(&binding, &bad).is_err());
|
||||
for changed in [0, 1, 2, 3, 4, 5] {
|
||||
let mut bad = r.clone();
|
||||
match changed {
|
||||
0 => bad.ready = true,
|
||||
1 => bad.helper_ready = true,
|
||||
2 => bad.unlock = true,
|
||||
3 => bad.agent_id = Uuid::new_v4(),
|
||||
4 => bad.domain = "rmm.execution-permit".into(),
|
||||
_ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16),
|
||||
};
|
||||
assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err());
|
||||
}
|
||||
let mut expired = r.clone();
|
||||
expired.issued_at = Utc::now() - chrono::Duration::minutes(2);
|
||||
expired.valid_until = Utc::now() - chrono::Duration::minutes(1);
|
||||
assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err());
|
||||
}
|
||||
fn database() -> Connection {
|
||||
let db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap();
|
||||
db
|
||||
}
|
||||
#[test]
|
||||
fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() {
|
||||
let (_, key, mut r) = fixture();
|
||||
let mut db = database();
|
||||
let before = inventory(&db, "helper_sync_meta").unwrap();
|
||||
r.helper_inventory_sha256 = inventory_sha256(&before).unwrap();
|
||||
r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap();
|
||||
let signed = sign(&key, &r);
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before);
|
||||
assert!(
|
||||
db.query_row("SELECT result IS NULL FROM consumed", [], |v| v
|
||||
.get::<_, bool>(0))
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT outcome FROM execution_evidence", [], |v| v
|
||||
.get::<_, Vec<u8>>(0))
|
||||
.unwrap(),
|
||||
b"unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row(
|
||||
"SELECT old_fault FROM checkpoint_repair_receipts",
|
||||
[],
|
||||
|v| v.get::<_, String>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
"CHECKPOINT_CHAIN_MISMATCH"
|
||||
);
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v
|
||||
.get::<_, u64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
db.execute("UPDATE consumed SET result=x'01'", []).unwrap();
|
||||
assert!(
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn changed_witness_or_other_fault_is_never_cleared() {
|
||||
let (_, key, r) = fixture();
|
||||
let mut db = database();
|
||||
let signed = sign(&key, &r);
|
||||
assert!(
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
db.execute(
|
||||
"UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(inventory(&db, "helper_sync_meta").is_err());
|
||||
assert!(!already_applied(&db, &signed).unwrap());
|
||||
let db = database();
|
||||
db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap();
|
||||
assert!(inventory(&db, "helper_sync_meta").is_err());
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
extern crate self as rmm_privileged_helper;
|
||||
pub mod apt;
|
||||
pub mod apt_reconcile;
|
||||
pub mod checkpoint_repair;
|
||||
pub mod control;
|
||||
pub mod readiness;
|
||||
pub mod reconcile;
|
||||
|
||||
@@ -681,6 +681,142 @@ fn validate_history(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Bootstrap-only historical verification; this never creates a lease.
|
||||
pub fn verify_checkpoint_repair_history(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
links: Option<(
|
||||
&rmm_domain::checkpoint_repair::Receipt,
|
||||
&rmm_domain::checkpoint_repair::Evidence,
|
||||
)>,
|
||||
) -> Result<()> {
|
||||
let meta = metadata(db)?;
|
||||
validate_history(db, binding, &meta, meta.generation)?;
|
||||
let mut statement = db.prepare(
|
||||
"SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint",
|
||||
)?;
|
||||
let entries = statement
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, Vec<u8>>(0)?,
|
||||
r.get::<_, Vec<u8>>(1)?,
|
||||
r.get::<_, u64>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT");
|
||||
let mut previous = None;
|
||||
let mut previous_checkpoint = 0;
|
||||
let mut previous_agent_checkpoint = 0;
|
||||
for (signed, raw, checkpoint) in entries {
|
||||
let response: SignedResponse = serde_json::from_slice(&signed)?;
|
||||
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
||||
let decision = verify(binding, &response, parsed.issued_at)?;
|
||||
let request = RequestView::decode(&raw)?;
|
||||
let captured: Vec<u8> = db.query_row(
|
||||
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
|
||||
[checkpoint],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let capture: Snapshot = serde_json::from_slice(&captured)?;
|
||||
ensure!(
|
||||
decision.helper_journal_id == Some(meta.journal_id)
|
||||
&& decision.helper_checkpoint == checkpoint
|
||||
&& request.helper_journal_id == meta.journal_id.to_string()
|
||||
&& request.helper_checkpoint == checkpoint
|
||||
&& request.assignment_version == binding.assignment_version
|
||||
&& request.journal_complete
|
||||
&& decision.request_hash == hash(&raw)
|
||||
&& decision.request_id.to_string() == request.request_id
|
||||
&& decision.journal_id.to_string() == request.journal_id
|
||||
&& decision.checkpoint == request.checkpoint
|
||||
&& capture.journal_id == meta.journal_id
|
||||
&& capture.checkpoint == checkpoint
|
||||
&& capture.boot_id == decision.boot_id
|
||||
&& capture.captured_at <= decision.issued_at
|
||||
&& capture.previous_decision_id == previous
|
||||
&& checkpoint > previous_checkpoint
|
||||
&& decision.checkpoint >= previous_agent_checkpoint
|
||||
&& decision.connection_generation <= meta.generation
|
||||
&& request.helper_previous_decision_id
|
||||
== capture
|
||||
.previous_decision_id
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default(),
|
||||
"REPAIR_HELPER_HISTORY_BINDING"
|
||||
);
|
||||
previous = Some(decision.decision_id);
|
||||
previous_checkpoint = checkpoint;
|
||||
previous_agent_checkpoint = decision.checkpoint;
|
||||
}
|
||||
let _ = load_rows(db, binding, false)?;
|
||||
if let Some((r, e)) = links {
|
||||
verify_checkpoint_repair_links(db, binding, r, e)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn verify_checkpoint_repair_links(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
r: &rmm_domain::checkpoint_repair::Receipt,
|
||||
e: &rmm_domain::checkpoint_repair::Evidence,
|
||||
) -> Result<()> {
|
||||
let x = RequestView::decode(&e.uncommitted_request)?;
|
||||
let y = RequestView::decode(&e.rejected_request)?;
|
||||
let a: ReconcileDecisionPayload =
|
||||
serde_json::from_slice(&e.previous_valid_response.decision_bytes)?;
|
||||
let a = verify(
|
||||
binding,
|
||||
&SignedResponse {
|
||||
decision_bytes: e.previous_valid_response.decision_bytes.clone(),
|
||||
signature: e.previous_valid_response.signature.clone(),
|
||||
},
|
||||
a.issued_at,
|
||||
)?;
|
||||
let old: Vec<u8> = db.query_row(
|
||||
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[a.decision_id.to_string()],
|
||||
|v| v.get(0),
|
||||
)?;
|
||||
let old: SignedResponse = serde_json::from_slice(&old)?;
|
||||
ensure!(
|
||||
old.decision_bytes == e.previous_valid_response.decision_bytes
|
||||
&& old.signature == e.previous_valid_response.signature
|
||||
&& a.decision_id == r.previous_valid_decision_id
|
||||
&& hash(&old.decision_bytes) == r.previous_valid_decision_sha256
|
||||
&& a.reason == "ATTEMPT_RECONCILIATION_REQUIRED"
|
||||
&& !a.ready
|
||||
&& !a.helper_ready
|
||||
&& x.request_id == r.uncommitted_request_id.to_string()
|
||||
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
|
||||
&& y.request_id == r.rejected_request_id.to_string()
|
||||
&& hash(&e.rejected_request) == r.rejected_request_sha256
|
||||
&& x.previous_decision_id == a.decision_id.to_string()
|
||||
&& x.previous_request_id.is_empty()
|
||||
&& x.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.previous_decision_id.is_empty()
|
||||
&& y.previous_request_id == x.request_id
|
||||
&& x.journal_id == r.agent_journal_id.to_string()
|
||||
&& y.journal_id == x.journal_id
|
||||
&& a.journal_id == r.agent_journal_id
|
||||
&& x.helper_journal_id == r.helper_journal_id.to_string()
|
||||
&& y.helper_journal_id == x.helper_journal_id
|
||||
&& a.helper_journal_id == Some(r.helper_journal_id)
|
||||
&& a.checkpoint < x.checkpoint
|
||||
&& x.checkpoint < y.checkpoint
|
||||
&& y.checkpoint == r.agent_checkpoint
|
||||
&& a.helper_checkpoint < x.helper_checkpoint
|
||||
&& x.helper_checkpoint < y.helper_checkpoint
|
||||
&& y.helper_checkpoint == r.helper_checkpoint
|
||||
&& x.assignment_version == binding.assignment_version
|
||||
&& y.assignment_version == binding.assignment_version
|
||||
&& x.journal_complete
|
||||
&& y.journal_complete,
|
||||
"REPAIR_EXACT_UNSENT_CHAIN_REQUIRED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn persist_fault(db: &Connection, error: &anyhow::Error) {
|
||||
let text = error.to_string();
|
||||
if [
|
||||
@@ -991,6 +1127,8 @@ struct RequestView {
|
||||
helper_journal_id: String,
|
||||
helper_checkpoint: u64,
|
||||
helper_previous_decision_id: String,
|
||||
previous_decision_id: String,
|
||||
previous_request_id: String,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct AttemptView {
|
||||
@@ -1089,6 +1227,11 @@ impl RequestView {
|
||||
if !s.is_empty() {
|
||||
Uuid::parse_str(&s)?;
|
||||
}
|
||||
if tag == 4 {
|
||||
r.previous_decision_id = s;
|
||||
} else {
|
||||
r.previous_request_id = s;
|
||||
}
|
||||
}
|
||||
5 => r.assignment_version = number(value)?,
|
||||
6 => {
|
||||
|
||||
Reference in New Issue
Block a user