fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -681,6 +681,142 @@ fn validate_history(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Bootstrap-only historical verification; this never creates a lease.
|
||||
pub fn verify_checkpoint_repair_history(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
links: Option<(
|
||||
&rmm_domain::checkpoint_repair::Receipt,
|
||||
&rmm_domain::checkpoint_repair::Evidence,
|
||||
)>,
|
||||
) -> Result<()> {
|
||||
let meta = metadata(db)?;
|
||||
validate_history(db, binding, &meta, meta.generation)?;
|
||||
let mut statement = db.prepare(
|
||||
"SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint",
|
||||
)?;
|
||||
let entries = statement
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, Vec<u8>>(0)?,
|
||||
r.get::<_, Vec<u8>>(1)?,
|
||||
r.get::<_, u64>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT");
|
||||
let mut previous = None;
|
||||
let mut previous_checkpoint = 0;
|
||||
let mut previous_agent_checkpoint = 0;
|
||||
for (signed, raw, checkpoint) in entries {
|
||||
let response: SignedResponse = serde_json::from_slice(&signed)?;
|
||||
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
||||
let decision = verify(binding, &response, parsed.issued_at)?;
|
||||
let request = RequestView::decode(&raw)?;
|
||||
let captured: Vec<u8> = db.query_row(
|
||||
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
|
||||
[checkpoint],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let capture: Snapshot = serde_json::from_slice(&captured)?;
|
||||
ensure!(
|
||||
decision.helper_journal_id == Some(meta.journal_id)
|
||||
&& decision.helper_checkpoint == checkpoint
|
||||
&& request.helper_journal_id == meta.journal_id.to_string()
|
||||
&& request.helper_checkpoint == checkpoint
|
||||
&& request.assignment_version == binding.assignment_version
|
||||
&& request.journal_complete
|
||||
&& decision.request_hash == hash(&raw)
|
||||
&& decision.request_id.to_string() == request.request_id
|
||||
&& decision.journal_id.to_string() == request.journal_id
|
||||
&& decision.checkpoint == request.checkpoint
|
||||
&& capture.journal_id == meta.journal_id
|
||||
&& capture.checkpoint == checkpoint
|
||||
&& capture.boot_id == decision.boot_id
|
||||
&& capture.captured_at <= decision.issued_at
|
||||
&& capture.previous_decision_id == previous
|
||||
&& checkpoint > previous_checkpoint
|
||||
&& decision.checkpoint >= previous_agent_checkpoint
|
||||
&& decision.connection_generation <= meta.generation
|
||||
&& request.helper_previous_decision_id
|
||||
== capture
|
||||
.previous_decision_id
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default(),
|
||||
"REPAIR_HELPER_HISTORY_BINDING"
|
||||
);
|
||||
previous = Some(decision.decision_id);
|
||||
previous_checkpoint = checkpoint;
|
||||
previous_agent_checkpoint = decision.checkpoint;
|
||||
}
|
||||
let _ = load_rows(db, binding, false)?;
|
||||
if let Some((r, e)) = links {
|
||||
verify_checkpoint_repair_links(db, binding, r, e)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn verify_checkpoint_repair_links(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
r: &rmm_domain::checkpoint_repair::Receipt,
|
||||
e: &rmm_domain::checkpoint_repair::Evidence,
|
||||
) -> Result<()> {
|
||||
let x = RequestView::decode(&e.uncommitted_request)?;
|
||||
let y = RequestView::decode(&e.rejected_request)?;
|
||||
let a: ReconcileDecisionPayload =
|
||||
serde_json::from_slice(&e.previous_valid_response.decision_bytes)?;
|
||||
let a = verify(
|
||||
binding,
|
||||
&SignedResponse {
|
||||
decision_bytes: e.previous_valid_response.decision_bytes.clone(),
|
||||
signature: e.previous_valid_response.signature.clone(),
|
||||
},
|
||||
a.issued_at,
|
||||
)?;
|
||||
let old: Vec<u8> = db.query_row(
|
||||
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[a.decision_id.to_string()],
|
||||
|v| v.get(0),
|
||||
)?;
|
||||
let old: SignedResponse = serde_json::from_slice(&old)?;
|
||||
ensure!(
|
||||
old.decision_bytes == e.previous_valid_response.decision_bytes
|
||||
&& old.signature == e.previous_valid_response.signature
|
||||
&& a.decision_id == r.previous_valid_decision_id
|
||||
&& hash(&old.decision_bytes) == r.previous_valid_decision_sha256
|
||||
&& a.reason == "ATTEMPT_RECONCILIATION_REQUIRED"
|
||||
&& !a.ready
|
||||
&& !a.helper_ready
|
||||
&& x.request_id == r.uncommitted_request_id.to_string()
|
||||
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
|
||||
&& y.request_id == r.rejected_request_id.to_string()
|
||||
&& hash(&e.rejected_request) == r.rejected_request_sha256
|
||||
&& x.previous_decision_id == a.decision_id.to_string()
|
||||
&& x.previous_request_id.is_empty()
|
||||
&& x.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.previous_decision_id.is_empty()
|
||||
&& y.previous_request_id == x.request_id
|
||||
&& x.journal_id == r.agent_journal_id.to_string()
|
||||
&& y.journal_id == x.journal_id
|
||||
&& a.journal_id == r.agent_journal_id
|
||||
&& x.helper_journal_id == r.helper_journal_id.to_string()
|
||||
&& y.helper_journal_id == x.helper_journal_id
|
||||
&& a.helper_journal_id == Some(r.helper_journal_id)
|
||||
&& a.checkpoint < x.checkpoint
|
||||
&& x.checkpoint < y.checkpoint
|
||||
&& y.checkpoint == r.agent_checkpoint
|
||||
&& a.helper_checkpoint < x.helper_checkpoint
|
||||
&& x.helper_checkpoint < y.helper_checkpoint
|
||||
&& y.helper_checkpoint == r.helper_checkpoint
|
||||
&& x.assignment_version == binding.assignment_version
|
||||
&& y.assignment_version == binding.assignment_version
|
||||
&& x.journal_complete
|
||||
&& y.journal_complete,
|
||||
"REPAIR_EXACT_UNSENT_CHAIN_REQUIRED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn persist_fault(db: &Connection, error: &anyhow::Error) {
|
||||
let text = error.to_string();
|
||||
if [
|
||||
@@ -991,6 +1127,8 @@ struct RequestView {
|
||||
helper_journal_id: String,
|
||||
helper_checkpoint: u64,
|
||||
helper_previous_decision_id: String,
|
||||
previous_decision_id: String,
|
||||
previous_request_id: String,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct AttemptView {
|
||||
@@ -1089,6 +1227,11 @@ impl RequestView {
|
||||
if !s.is_empty() {
|
||||
Uuid::parse_str(&s)?;
|
||||
}
|
||||
if tag == 4 {
|
||||
r.previous_decision_id = s;
|
||||
} else {
|
||||
r.previous_request_id = s;
|
||||
}
|
||||
}
|
||||
5 => r.assignment_version = number(value)?,
|
||||
6 => {
|
||||
|
||||
Reference in New Issue
Block a user