Recover Windows helper sync after acknowledged result delivery
This commit is contained in:
@@ -365,7 +365,6 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
&& original.helper_previous_decision_id == h.to_string()
|
||||
&& original.previous_request_id.is_empty()
|
||||
&& original.journal_complete
|
||||
&& original.durable_message_ids.is_empty()
|
||||
&& decision.issued_at <= now
|
||||
&& decision.connection_generation <= channel.connection_generation
|
||||
&& previous.connection_generation <= decision.connection_generation
|
||||
@@ -379,6 +378,7 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
&& helper.checkpoint >= decision.helper_checkpoint,
|
||||
"CLOSED_CONTROL_EXACT_PARTIAL_EDGE"
|
||||
);
|
||||
verify_closed_result_messages(db, &original, &decision)?;
|
||||
let current = attempts(db, ledger, &self.binding)?;
|
||||
ensure!(
|
||||
current.len() == original.attempts.len()
|
||||
@@ -1171,6 +1171,81 @@ pub fn archive_events(db: &Connection) -> Result<()> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// A partial control edge can contain the result that closed an attempt, and
|
||||
/// an earlier unknown report for that same grant. Only account for archived
|
||||
/// messages here. Never ACK an event, change its state or replay a result.
|
||||
fn verify_closed_result_messages(
|
||||
db: &Connection,
|
||||
request: &SyncRequest,
|
||||
decision: &ReconcileDecisionPayload,
|
||||
) -> Result<()> {
|
||||
for id in &request.durable_message_ids {
|
||||
let raw: Vec<u8> = db.query_row(
|
||||
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
||||
[id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let envelope = Envelope::decode(raw.as_slice())?;
|
||||
let Some(Payload::JobResult(result)) = &envelope.payload else {
|
||||
anyhow::bail!("CLOSED_CONTROL_EXPECTED_RESULT");
|
||||
};
|
||||
let attempt = request
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|a| {
|
||||
a.job_id == result.job_id
|
||||
&& a.attempt_id == result.attempt_id
|
||||
&& a.grant_id == result.grant_id
|
||||
})
|
||||
.context("CLOSED_CONTROL_ARCHIVED_ATTEMPT")?;
|
||||
let final_envelope = attempt
|
||||
.result_envelope
|
||||
.as_ref()
|
||||
.context("CLOSED_CONTROL_FINAL_ENVELOPE")?;
|
||||
let final_id = Uuid::parse_str(&attempt.result_message_id)?;
|
||||
let persisted: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM reconcile_receipts WHERE id=?1 AND decision_id=?2)",
|
||||
params![attempt.result_message_id, decision.decision_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
persisted
|
||||
&& decision.acknowledged_message_ids.contains(&final_id)
|
||||
&& request
|
||||
.durable_message_ids
|
||||
.contains(&attempt.result_message_id)
|
||||
&& final_envelope.message_id == attempt.result_message_id
|
||||
&& envelope.message_id == *id
|
||||
&& envelope.encode_to_vec() == raw
|
||||
&& rmm_protocol::valid_envelope(
|
||||
&envelope,
|
||||
&decision.agent_id.to_string(),
|
||||
final_envelope.connection_generation
|
||||
)
|
||||
&& envelope.boot_id == final_envelope.boot_id
|
||||
&& matches!(
|
||||
attempt.state.as_str(),
|
||||
"succeeded" | "failed" | "cancelled" | "expired"
|
||||
)
|
||||
&& !result.grant_id.is_empty(),
|
||||
"CLOSED_CONTROL_FINAL_RESULT_RECEIPT"
|
||||
);
|
||||
ensure!(
|
||||
if *id == attempt.result_message_id {
|
||||
envelope == *final_envelope
|
||||
&& result.revision == attempt.revision
|
||||
&& result.state == attempt.state
|
||||
&& canonical_outcome_bytes(&outcome(result)?)? == attempt.outcome_bytes
|
||||
} else {
|
||||
result.state == "unknown"
|
||||
&& result.revision < attempt.revision
|
||||
&& envelope.sequence < final_envelope.sequence
|
||||
},
|
||||
"CLOSED_CONTROL_CONFLICTING_ARCHIVED_RESULT"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn pending_ids(db: &Connection) -> Result<Vec<String>> {
|
||||
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?;
|
||||
let rows = statement
|
||||
|
||||
Reference in New Issue
Block a user