Recover Windows helper sync after acknowledged result delivery
This commit is contained in:
Generated
+1
-1
@@ -2213,7 +2213,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rmm-agent-windows"
|
name = "rmm-agent-windows"
|
||||||
version = "0.1.15"
|
version = "0.1.16"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
|
|||||||
@@ -365,7 +365,6 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
|||||||
&& original.helper_previous_decision_id == h.to_string()
|
&& original.helper_previous_decision_id == h.to_string()
|
||||||
&& original.previous_request_id.is_empty()
|
&& original.previous_request_id.is_empty()
|
||||||
&& original.journal_complete
|
&& original.journal_complete
|
||||||
&& original.durable_message_ids.is_empty()
|
|
||||||
&& decision.issued_at <= now
|
&& decision.issued_at <= now
|
||||||
&& decision.connection_generation <= channel.connection_generation
|
&& decision.connection_generation <= channel.connection_generation
|
||||||
&& previous.connection_generation <= decision.connection_generation
|
&& previous.connection_generation <= decision.connection_generation
|
||||||
@@ -379,6 +378,7 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
|||||||
&& helper.checkpoint >= decision.helper_checkpoint,
|
&& helper.checkpoint >= decision.helper_checkpoint,
|
||||||
"CLOSED_CONTROL_EXACT_PARTIAL_EDGE"
|
"CLOSED_CONTROL_EXACT_PARTIAL_EDGE"
|
||||||
);
|
);
|
||||||
|
verify_closed_result_messages(db, &original, &decision)?;
|
||||||
let current = attempts(db, ledger, &self.binding)?;
|
let current = attempts(db, ledger, &self.binding)?;
|
||||||
ensure!(
|
ensure!(
|
||||||
current.len() == original.attempts.len()
|
current.len() == original.attempts.len()
|
||||||
@@ -1171,6 +1171,81 @@ pub fn archive_events(db: &Connection) -> Result<()> {
|
|||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
/// A partial control edge can contain the result that closed an attempt, and
|
||||||
|
/// an earlier unknown report for that same grant. Only account for archived
|
||||||
|
/// messages here. Never ACK an event, change its state or replay a result.
|
||||||
|
fn verify_closed_result_messages(
|
||||||
|
db: &Connection,
|
||||||
|
request: &SyncRequest,
|
||||||
|
decision: &ReconcileDecisionPayload,
|
||||||
|
) -> Result<()> {
|
||||||
|
for id in &request.durable_message_ids {
|
||||||
|
let raw: Vec<u8> = db.query_row(
|
||||||
|
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
||||||
|
[id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
let envelope = Envelope::decode(raw.as_slice())?;
|
||||||
|
let Some(Payload::JobResult(result)) = &envelope.payload else {
|
||||||
|
anyhow::bail!("CLOSED_CONTROL_EXPECTED_RESULT");
|
||||||
|
};
|
||||||
|
let attempt = request
|
||||||
|
.attempts
|
||||||
|
.iter()
|
||||||
|
.find(|a| {
|
||||||
|
a.job_id == result.job_id
|
||||||
|
&& a.attempt_id == result.attempt_id
|
||||||
|
&& a.grant_id == result.grant_id
|
||||||
|
})
|
||||||
|
.context("CLOSED_CONTROL_ARCHIVED_ATTEMPT")?;
|
||||||
|
let final_envelope = attempt
|
||||||
|
.result_envelope
|
||||||
|
.as_ref()
|
||||||
|
.context("CLOSED_CONTROL_FINAL_ENVELOPE")?;
|
||||||
|
let final_id = Uuid::parse_str(&attempt.result_message_id)?;
|
||||||
|
let persisted: bool = db.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM reconcile_receipts WHERE id=?1 AND decision_id=?2)",
|
||||||
|
params![attempt.result_message_id, decision.decision_id.to_string()],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
ensure!(
|
||||||
|
persisted
|
||||||
|
&& decision.acknowledged_message_ids.contains(&final_id)
|
||||||
|
&& request
|
||||||
|
.durable_message_ids
|
||||||
|
.contains(&attempt.result_message_id)
|
||||||
|
&& final_envelope.message_id == attempt.result_message_id
|
||||||
|
&& envelope.message_id == *id
|
||||||
|
&& envelope.encode_to_vec() == raw
|
||||||
|
&& rmm_protocol::valid_envelope(
|
||||||
|
&envelope,
|
||||||
|
&decision.agent_id.to_string(),
|
||||||
|
final_envelope.connection_generation
|
||||||
|
)
|
||||||
|
&& envelope.boot_id == final_envelope.boot_id
|
||||||
|
&& matches!(
|
||||||
|
attempt.state.as_str(),
|
||||||
|
"succeeded" | "failed" | "cancelled" | "expired"
|
||||||
|
)
|
||||||
|
&& !result.grant_id.is_empty(),
|
||||||
|
"CLOSED_CONTROL_FINAL_RESULT_RECEIPT"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
if *id == attempt.result_message_id {
|
||||||
|
envelope == *final_envelope
|
||||||
|
&& result.revision == attempt.revision
|
||||||
|
&& result.state == attempt.state
|
||||||
|
&& canonical_outcome_bytes(&outcome(result)?)? == attempt.outcome_bytes
|
||||||
|
} else {
|
||||||
|
result.state == "unknown"
|
||||||
|
&& result.revision < attempt.revision
|
||||||
|
&& envelope.sequence < final_envelope.sequence
|
||||||
|
},
|
||||||
|
"CLOSED_CONTROL_CONFLICTING_ARCHIVED_RESULT"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
fn pending_ids(db: &Connection) -> Result<Vec<String>> {
|
fn pending_ids(db: &Connection) -> Result<Vec<String>> {
|
||||||
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?;
|
let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?;
|
||||||
let rows = statement
|
let rows = statement
|
||||||
|
|||||||
@@ -1091,6 +1091,103 @@ fn windows_unconfirmed_helper_head_cannot_replace_a_committed_decision() {
|
|||||||
);
|
);
|
||||||
assert!(!reopened.can_start(&f.channel, false, &f.clock));
|
assert!(!reopened.can_start(&f.channel, false, &f.clock));
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn missed_system_ack_after_result_delivery_requires_archived_definitive_proof() {
|
||||||
|
let mut f = Fixture::new();
|
||||||
|
f.helper_identity = HelperIdentity::System;
|
||||||
|
let mut c = f.client();
|
||||||
|
let mut helper = HelperSnapshot {
|
||||||
|
journal_id: Uuid::new_v4(),
|
||||||
|
checkpoint: 1,
|
||||||
|
previous_decision_id: None,
|
||||||
|
attempts: vec![],
|
||||||
|
};
|
||||||
|
let first = c
|
||||||
|
.prepare(
|
||||||
|
&mut f.db,
|
||||||
|
&f.ledger,
|
||||||
|
f.channel.clone(),
|
||||||
|
Some(&helper),
|
||||||
|
&f.clock,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let anchor = f.decision(&first, true);
|
||||||
|
f.apply(&mut c, &anchor).unwrap();
|
||||||
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||||
|
helper.previous_decision_id = Some(anchor.decision_id);
|
||||||
|
c.confirm_helper(&helper).unwrap();
|
||||||
|
helper.checkpoint += 1;
|
||||||
|
let (offer, grant, report) = f.add("unknown", true, Some(success()), script());
|
||||||
|
let mut prior = report.unwrap();
|
||||||
|
let Some(Payload::JobResult(result)) = &mut prior.payload else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
result.state = "unknown".into();
|
||||||
|
f.db.execute(
|
||||||
|
"UPDATE job_events SET payload=?1 WHERE id=?2",
|
||||||
|
params![prior.encode_to_vec(), prior.message_id],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let final_report =
|
||||||
|
persist_original_fixture_result(&mut f, &offer, &grant.unwrap(), completion_outcome());
|
||||||
|
let edge = c
|
||||||
|
.prepare(
|
||||||
|
&mut f.db,
|
||||||
|
&f.ledger,
|
||||||
|
f.channel.clone(),
|
||||||
|
Some(&helper),
|
||||||
|
&f.clock,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(edge.request.durable_message_ids.len(), 2);
|
||||||
|
let mut signed = f.decision(&edge, true);
|
||||||
|
signed
|
||||||
|
.acknowledged_message_ids
|
||||||
|
.push(Uuid::parse_str(&final_report.message_id).unwrap());
|
||||||
|
f.apply(&mut c, &signed).unwrap();
|
||||||
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||||
|
helper.checkpoint += 1;
|
||||||
|
f.channel.connection_generation += 1;
|
||||||
|
let mut fresh = f.client();
|
||||||
|
let prepare = |c: &mut ClientSync, f: &mut Fixture| {
|
||||||
|
c.prepare_with_closed_control_idle(
|
||||||
|
&mut f.db,
|
||||||
|
&f.ledger,
|
||||||
|
f.channel.clone(),
|
||||||
|
Some(&helper),
|
||||||
|
&f.clock,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert!(prepare(&mut fresh, &mut f).is_err()); // prior report still pending
|
||||||
|
rmm_agent_core::jobs::ack(&mut f.db, &prior.message_id).unwrap();
|
||||||
|
let mut conflicting = prior.clone();
|
||||||
|
if let Some(Payload::JobResult(r)) = &mut conflicting.payload {
|
||||||
|
r.state = "failed".into();
|
||||||
|
}
|
||||||
|
f.db.execute(
|
||||||
|
"UPDATE reconcile_results SET envelope=?1 WHERE id=?2",
|
||||||
|
params![conflicting.encode_to_vec(), prior.message_id],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(prepare(&mut fresh, &mut f).is_err());
|
||||||
|
f.db.execute(
|
||||||
|
"UPDATE reconcile_results SET envelope=?1 WHERE id=?2",
|
||||||
|
params![prior.encode_to_vec(), prior.message_id],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let next = prepare(&mut fresh, &mut f).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
next.request.previous_decision_id,
|
||||||
|
signed.decision_id.to_string()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
next.request.helper_previous_decision_id,
|
||||||
|
anchor.decision_id.to_string()
|
||||||
|
);
|
||||||
|
assert!(!fresh.can_start(&f.channel, true, &f.clock));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn independent_helper_checkpoint_regression_is_sticky() {
|
fn independent_helper_checkpoint_regression_is_sticky() {
|
||||||
let mut f = Fixture::new();
|
let mut f = Fixture::new();
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "rmm-agent-windows"
|
name = "rmm-agent-windows"
|
||||||
version = "0.1.15"
|
version = "0.1.16"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user