Make start evidence atomic and recover only bound durable outcomes
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -1232,3 +1232,234 @@ fn system_helper_import_requires_explicit_system_identity_and_never_root_alias()
|
||||
let changed = c.with_helper_identity(HelperIdentity::Root);
|
||||
assert!(!changed.can_start(&f.channel, true, &f.clock));
|
||||
}
|
||||
|
||||
fn persist_original_fixture_result(
|
||||
f: &mut Fixture,
|
||||
offer: &JobOffer,
|
||||
grant: &ExecutionPermitPayload,
|
||||
outcome: rmm_agent_core::auth::Outcome,
|
||||
) -> Envelope {
|
||||
let tx = f.db.transaction().unwrap();
|
||||
let revision =
|
||||
rmm_agent_core::jobs::transition(&tx, offer, &outcome.state, &grant.grant_id.to_string())
|
||||
.unwrap();
|
||||
let sequence: i64 = tx
|
||||
.query_row(
|
||||
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
let envelope = Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: f.binding.agent_id.to_string(),
|
||||
enrollment_generation: f.binding.enrollment_generation,
|
||||
connection_generation: f.channel.connection_generation,
|
||||
boot_id: f.channel.boot_id.to_string(),
|
||||
sequence: sequence.try_into().unwrap(),
|
||||
observed_at: f.clock.now.timestamp_millis(),
|
||||
payload: Some(Payload::JobResult(rmm_agent_core::jobs::result(
|
||||
offer,
|
||||
revision,
|
||||
&grant.grant_id.to_string(),
|
||||
outcome,
|
||||
))),
|
||||
};
|
||||
rmm_agent_core::jobs::event(&tx, &envelope).unwrap();
|
||||
tx.commit().unwrap();
|
||||
envelope
|
||||
}
|
||||
fn completion_outcome() -> rmm_agent_core::auth::Outcome {
|
||||
serde_json::from_slice(&canonical_outcome_bytes(&success()).unwrap()).unwrap()
|
||||
}
|
||||
#[test]
|
||||
fn crash_after_durable_outcome_before_result_recovers_original_then_signed_sync_closes() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
||||
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
||||
assert_eq!(interrupted.len(), 1);
|
||||
let original = rmm_agent_core::jobs::recover_interrupted_outcome(
|
||||
&f.ledger,
|
||||
&f.binding,
|
||||
&offer,
|
||||
&interrupted[0].1,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let envelope = persist_original_fixture_result(&mut f, &offer, &grant, original);
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let mut client = f.client();
|
||||
let prepared = f.prepare(&mut client);
|
||||
let decision = f.decision(&prepared, true);
|
||||
assert!(decision.attempts[0].unlock);
|
||||
assert_eq!(
|
||||
prepared.request.attempts[0].result_message_id,
|
||||
envelope.message_id
|
||||
);
|
||||
f.apply(&mut client, &decision).unwrap();
|
||||
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn crash_after_result_commit_before_finish_retains_exact_result_and_local_prior() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
||||
let original = persist_original_fixture_result(&mut f, &offer, &grant, completion_outcome());
|
||||
assert!(
|
||||
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
rmm_agent_core::jobs::events(&f.db).unwrap(),
|
||||
vec![original.clone()]
|
||||
);
|
||||
let mut client = f.client();
|
||||
let p = f.prepare(&mut client);
|
||||
let d = f.decision(&p, true);
|
||||
f.apply(&mut client, &d).unwrap();
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
let archived: Vec<u8> =
|
||||
f.db.query_row(
|
||||
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
||||
[&original.message_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(archived, original.encode_to_vec());
|
||||
}
|
||||
#[test]
|
||||
fn crash_before_running_event_recovers_real_consumption_without_inventing_completion() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
f.db.execute("UPDATE job_attempts SET state='requesting',grant_id=''", [])
|
||||
.unwrap();
|
||||
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
||||
assert_eq!(interrupted[0].1, grant.grant_id.to_string());
|
||||
assert!(
|
||||
rmm_agent_core::jobs::recover_interrupted_outcome(
|
||||
&f.ledger,
|
||||
&f.binding,
|
||||
&offer,
|
||||
&interrupted[0].1
|
||||
)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
persist_original_fixture_result(
|
||||
&mut f,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_agent_core::auth::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
);
|
||||
let mut c = f.client();
|
||||
let p = f.prepare(&mut c);
|
||||
let d = f.decision(&p, false);
|
||||
assert!(!d.attempts[0].unlock);
|
||||
f.apply(&mut c, &d).unwrap();
|
||||
assert!(!c.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let missing: Option<Vec<u8>> = f
|
||||
.ledger
|
||||
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert!(missing.is_none());
|
||||
}
|
||||
#[test]
|
||||
fn signed_unlock_still_rejects_a_missing_local_outcome_in_legacy_crash_window() {
|
||||
let mut f = Fixture::new();
|
||||
let (_, grant, _) = f.add("succeeded", true, Some(success()), script());
|
||||
let grant = grant.unwrap();
|
||||
f.ledger
|
||||
.execute("UPDATE execution_evidence SET outcome=NULL", [])
|
||||
.unwrap();
|
||||
let mut c = f.client();
|
||||
let p = f.prepare(&mut c);
|
||||
let d = f.decision(&p, true);
|
||||
assert!(d.attempts[0].unlock);
|
||||
f.apply(&mut c, &d).unwrap();
|
||||
assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err());
|
||||
assert!(!c.can_start(&f.channel, false, &f.clock));
|
||||
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn original_completion_after_reconnect_unknown_is_recovered_without_reexecuting() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
let mut previous = f.client();
|
||||
let previous_request = f.prepare(&mut previous);
|
||||
assert_eq!(previous_request.request.attempts[0].state, "unknown");
|
||||
assert!(
|
||||
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
// The already-running executor completes after the transport fenced it.
|
||||
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
||||
drop(previous);
|
||||
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
||||
assert_eq!(interrupted.len(), 1);
|
||||
let saved = rmm_agent_core::jobs::recover_interrupted_outcome(
|
||||
&f.ledger,
|
||||
&f.binding,
|
||||
&offer,
|
||||
&interrupted[0].1,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let original = persist_original_fixture_result(&mut f, &offer, &grant, saved);
|
||||
let mut process = f.client();
|
||||
let p = f.prepare(&mut process);
|
||||
let d = f.decision(&p, true);
|
||||
assert_eq!(p.request.attempts[0].result_message_id, original.message_id);
|
||||
f.apply(&mut process, &d).unwrap();
|
||||
process.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
assert!(process.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user