diff --git a/Cargo.lock b/Cargo.lock index 4e29cbe..d4fb689 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2213,7 +2213,7 @@ dependencies = [ [[package]] name = "rmm-agent-windows" -version = "0.1.15" +version = "0.1.16" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/crates/agent-core/src/reconcile.rs b/crates/agent-core/src/reconcile.rs index 276aa73..983aa48 100644 --- a/crates/agent-core/src/reconcile.rs +++ b/crates/agent-core/src/reconcile.rs @@ -365,7 +365,6 @@ impl ClientSync { && original.helper_previous_decision_id == h.to_string() && original.previous_request_id.is_empty() && original.journal_complete - && original.durable_message_ids.is_empty() && decision.issued_at <= now && decision.connection_generation <= channel.connection_generation && previous.connection_generation <= decision.connection_generation @@ -379,6 +378,7 @@ impl ClientSync { && helper.checkpoint >= decision.helper_checkpoint, "CLOSED_CONTROL_EXACT_PARTIAL_EDGE" ); + verify_closed_result_messages(db, &original, &decision)?; let current = attempts(db, ledger, &self.binding)?; ensure!( current.len() == original.attempts.len() @@ -1171,6 +1171,81 @@ pub fn archive_events(db: &Connection) -> Result<()> { } Ok(()) } +/// A partial control edge can contain the result that closed an attempt, and +/// an earlier unknown report for that same grant. Only account for archived +/// messages here. Never ACK an event, change its state or replay a result. +fn verify_closed_result_messages( + db: &Connection, + request: &SyncRequest, + decision: &ReconcileDecisionPayload, +) -> Result<()> { + for id in &request.durable_message_ids { + let raw: Vec = db.query_row( + "SELECT envelope FROM reconcile_results WHERE id=?1", + [id], + |r| r.get(0), + )?; + let envelope = Envelope::decode(raw.as_slice())?; + let Some(Payload::JobResult(result)) = &envelope.payload else { + anyhow::bail!("CLOSED_CONTROL_EXPECTED_RESULT"); + }; + let attempt = request + .attempts + .iter() + .find(|a| { + a.job_id == result.job_id + && a.attempt_id == result.attempt_id + && a.grant_id == result.grant_id + }) + .context("CLOSED_CONTROL_ARCHIVED_ATTEMPT")?; + let final_envelope = attempt + .result_envelope + .as_ref() + .context("CLOSED_CONTROL_FINAL_ENVELOPE")?; + let final_id = Uuid::parse_str(&attempt.result_message_id)?; + let persisted: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM reconcile_receipts WHERE id=?1 AND decision_id=?2)", + params![attempt.result_message_id, decision.decision_id.to_string()], + |r| r.get(0), + )?; + ensure!( + persisted + && decision.acknowledged_message_ids.contains(&final_id) + && request + .durable_message_ids + .contains(&attempt.result_message_id) + && final_envelope.message_id == attempt.result_message_id + && envelope.message_id == *id + && envelope.encode_to_vec() == raw + && rmm_protocol::valid_envelope( + &envelope, + &decision.agent_id.to_string(), + final_envelope.connection_generation + ) + && envelope.boot_id == final_envelope.boot_id + && matches!( + attempt.state.as_str(), + "succeeded" | "failed" | "cancelled" | "expired" + ) + && !result.grant_id.is_empty(), + "CLOSED_CONTROL_FINAL_RESULT_RECEIPT" + ); + ensure!( + if *id == attempt.result_message_id { + envelope == *final_envelope + && result.revision == attempt.revision + && result.state == attempt.state + && canonical_outcome_bytes(&outcome(result)?)? == attempt.outcome_bytes + } else { + result.state == "unknown" + && result.revision < attempt.revision + && envelope.sequence < final_envelope.sequence + }, + "CLOSED_CONTROL_CONFLICTING_ARCHIVED_RESULT" + ); + } + Ok(()) +} fn pending_ids(db: &Connection) -> Result> { let mut statement = db.prepare("SELECT id,payload FROM job_events ORDER BY id")?; let rows = statement diff --git a/crates/agent-core/tests/reconcile.rs b/crates/agent-core/tests/reconcile.rs index b6dc331..98125be 100644 --- a/crates/agent-core/tests/reconcile.rs +++ b/crates/agent-core/tests/reconcile.rs @@ -1091,6 +1091,103 @@ fn windows_unconfirmed_helper_head_cannot_replace_a_committed_decision() { ); assert!(!reopened.can_start(&f.channel, false, &f.clock)); } +#[test] +fn missed_system_ack_after_result_delivery_requires_archived_definitive_proof() { + let mut f = Fixture::new(); + f.helper_identity = HelperIdentity::System; + let mut c = f.client(); + let mut helper = HelperSnapshot { + journal_id: Uuid::new_v4(), + checkpoint: 1, + previous_decision_id: None, + attempts: vec![], + }; + let first = c + .prepare( + &mut f.db, + &f.ledger, + f.channel.clone(), + Some(&helper), + &f.clock, + ) + .unwrap(); + let anchor = f.decision(&first, true); + f.apply(&mut c, &anchor).unwrap(); + c.apply_consumption(&mut f.ledger, &f.clock).unwrap(); + helper.previous_decision_id = Some(anchor.decision_id); + c.confirm_helper(&helper).unwrap(); + helper.checkpoint += 1; + let (offer, grant, report) = f.add("unknown", true, Some(success()), script()); + let mut prior = report.unwrap(); + let Some(Payload::JobResult(result)) = &mut prior.payload else { + unreachable!() + }; + result.state = "unknown".into(); + f.db.execute( + "UPDATE job_events SET payload=?1 WHERE id=?2", + params![prior.encode_to_vec(), prior.message_id], + ) + .unwrap(); + let final_report = + persist_original_fixture_result(&mut f, &offer, &grant.unwrap(), completion_outcome()); + let edge = c + .prepare( + &mut f.db, + &f.ledger, + f.channel.clone(), + Some(&helper), + &f.clock, + ) + .unwrap(); + assert_eq!(edge.request.durable_message_ids.len(), 2); + let mut signed = f.decision(&edge, true); + signed + .acknowledged_message_ids + .push(Uuid::parse_str(&final_report.message_id).unwrap()); + f.apply(&mut c, &signed).unwrap(); + c.apply_consumption(&mut f.ledger, &f.clock).unwrap(); + helper.checkpoint += 1; + f.channel.connection_generation += 1; + let mut fresh = f.client(); + let prepare = |c: &mut ClientSync, f: &mut Fixture| { + c.prepare_with_closed_control_idle( + &mut f.db, + &f.ledger, + f.channel.clone(), + Some(&helper), + &f.clock, + true, + ) + }; + assert!(prepare(&mut fresh, &mut f).is_err()); // prior report still pending + rmm_agent_core::jobs::ack(&mut f.db, &prior.message_id).unwrap(); + let mut conflicting = prior.clone(); + if let Some(Payload::JobResult(r)) = &mut conflicting.payload { + r.state = "failed".into(); + } + f.db.execute( + "UPDATE reconcile_results SET envelope=?1 WHERE id=?2", + params![conflicting.encode_to_vec(), prior.message_id], + ) + .unwrap(); + assert!(prepare(&mut fresh, &mut f).is_err()); + f.db.execute( + "UPDATE reconcile_results SET envelope=?1 WHERE id=?2", + params![prior.encode_to_vec(), prior.message_id], + ) + .unwrap(); + let next = prepare(&mut fresh, &mut f).unwrap(); + assert_eq!( + next.request.previous_decision_id, + signed.decision_id.to_string() + ); + assert_eq!( + next.request.helper_previous_decision_id, + anchor.decision_id.to_string() + ); + assert!(!fresh.can_start(&f.channel, true, &f.clock)); +} + #[test] fn independent_helper_checkpoint_regression_is_sticky() { let mut f = Fixture::new(); diff --git a/crates/agent-windows/Cargo.toml b/crates/agent-windows/Cargo.toml index b2b99b1..9b42dba 100644 --- a/crates/agent-windows/Cargo.toml +++ b/crates/agent-windows/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rmm-agent-windows" -version = "0.1.15" +version = "0.1.16" edition.workspace = true license.workspace = true