Make start evidence atomic and recover only bound durable outcomes
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-14 02:01:17 +01:00
parent de3b408c34
commit 74736483df
13 changed files with 764 additions and 87 deletions
+126
View File
@@ -290,3 +290,129 @@ fn portable_acceptance_requires_platform_capability_and_exact_redelivery() {
assert!(jobs::next(&db).unwrap().is_none());
assert!(jobs::transition(&db, &offer, "running", "").is_err());
}
#[test]
fn atomic_begin_execution_rolls_back_both_rows_on_missing_witness_or_start_insert_failure() {
let (_, request, permit, now) = fixture();
let mut db = Connection::open_in_memory().unwrap();
ledger::initialize_ledger(&db).unwrap();
assert!(ledger::begin_execution(&mut db, &permit, &request, Uuid::nil(), now).is_err());
assert_eq!(
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap(),
0
);
db.execute_batch("CREATE TRIGGER refuse_start BEFORE INSERT ON execution_evidence BEGIN SELECT RAISE(ABORT,'fixture storage failure'); END;").unwrap();
assert!(ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).is_err());
assert_eq!(
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap(),
0
);
assert_eq!(
db.query_row::<u32, _, _>("SELECT count(*) FROM execution_evidence", [], |r| r.get(0))
.unwrap(),
0
);
db.execute_batch("DROP TRIGGER refuse_start").unwrap();
ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).unwrap();
assert!(ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).is_err());
assert_eq!(
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap(),
1
);
}
#[test]
fn completion_cas_preserves_original_bytes_and_recovers_only_verified_bound_evidence() {
let (binding, request, permit, now) = fixture();
let mut db = Connection::open_in_memory().unwrap();
ledger::initialize_ledger(&db).unwrap();
ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).unwrap();
let offer = JobOffer {
job_id: permit.job_id.to_string(),
attempt_id: permit.attempt_id.to_string(),
payload_bytes: request.job_bytes.clone(),
job_payload_hash: permit.job_payload_hash.clone(),
plan_hash: permit.plan_hash.clone(),
authorization_id: permit.authorization_id.to_string(),
};
assert!(
jobs::recover_interrupted_outcome(&db, &binding, &offer, &permit.grant_id.to_string())
.unwrap()
.is_none()
);
let mut original = Outcome::unknown("");
original.state = "succeeded".into();
original.stdout = b"Hello\n".to_vec();
original.exit_code = Some(0);
original.evidence = serde_json::json!({"postconditions_passed":true});
ledger::persist_outcome(&db, &permit, &original).unwrap();
let saved: Vec<u8> = db
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
.unwrap();
ledger::persist_outcome(&db, &permit, &original).unwrap();
assert_eq!(
saved,
db.query_row::<Vec<u8>, _, _>("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
.unwrap()
);
assert_eq!(
db.query_row::<u32, _, _>(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get(0)
)
.unwrap(),
1
);
assert!(ledger::persist_outcome(&db, &permit, &Outcome::unknown("late uncertainty")).is_err());
assert!(ledger::finish(&db, &permit, &Outcome::unknown("late uncertainty")).is_err());
let recovered =
jobs::recover_interrupted_outcome(&db, &binding, &offer, &permit.grant_id.to_string())
.unwrap()
.unwrap();
assert_eq!(serde_json::to_vec(&recovered).unwrap(), saved);
let mut forged = binding.clone();
forged.public_key_hex =
hex::encode(SigningKey::from_bytes(&[31; 32]).verifying_key().as_bytes());
assert!(
jobs::recover_interrupted_outcome(&db, &forged, &offer, &permit.grant_id.to_string())
.is_err()
);
let mut changed = offer.clone();
changed.authorization_id = Uuid::new_v4().to_string();
assert!(
jobs::recover_interrupted_outcome(&db, &binding, &changed, &permit.grant_id.to_string())
.is_err()
);
let mut changed = permit.clone();
changed.assignment_version += 1;
assert!(ledger::persist_outcome(&db, &changed, &original).is_err());
ledger::finish(&db, &permit, &original).unwrap();
ledger::finish(&db, &permit, &original).unwrap();
assert_eq!(
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap(),
1
);
assert!(ledger::consume(&mut db, &permit).is_err());
let mut corrupted: serde_json::Value = db
.query_row("SELECT start FROM execution_evidence", [], |r| {
r.get::<_, Vec<u8>>(0)
})
.map(|b| serde_json::from_slice(&b).unwrap())
.unwrap();
let changed = corrupted["request"]["signature"][0].as_u64().unwrap() ^ 1;
corrupted["request"]["signature"][0] = serde_json::json!(changed);
db.execute(
"UPDATE execution_evidence SET start=?1",
[serde_json::to_vec(&corrupted).unwrap()],
)
.unwrap();
assert!(
jobs::recover_interrupted_outcome(&db, &binding, &offer, &permit.grant_id.to_string())
.is_err()
);
}
+231
View File
@@ -1232,3 +1232,234 @@ fn system_helper_import_requires_explicit_system_identity_and_never_root_alias()
let changed = c.with_helper_identity(HelperIdentity::Root);
assert!(!changed.can_start(&f.channel, true, &f.clock));
}
fn persist_original_fixture_result(
f: &mut Fixture,
offer: &JobOffer,
grant: &ExecutionPermitPayload,
outcome: rmm_agent_core::auth::Outcome,
) -> Envelope {
let tx = f.db.transaction().unwrap();
let revision =
rmm_agent_core::jobs::transition(&tx, offer, &outcome.state, &grant.grant_id.to_string())
.unwrap();
let sequence: i64 = tx
.query_row(
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
[],
|r| r.get(0),
)
.unwrap();
let envelope = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: f.binding.agent_id.to_string(),
enrollment_generation: f.binding.enrollment_generation,
connection_generation: f.channel.connection_generation,
boot_id: f.channel.boot_id.to_string(),
sequence: sequence.try_into().unwrap(),
observed_at: f.clock.now.timestamp_millis(),
payload: Some(Payload::JobResult(rmm_agent_core::jobs::result(
offer,
revision,
&grant.grant_id.to_string(),
outcome,
))),
};
rmm_agent_core::jobs::event(&tx, &envelope).unwrap();
tx.commit().unwrap();
envelope
}
fn completion_outcome() -> rmm_agent_core::auth::Outcome {
serde_json::from_slice(&canonical_outcome_bytes(&success()).unwrap()).unwrap()
}
#[test]
fn crash_after_durable_outcome_before_result_recovers_original_then_signed_sync_closes() {
let mut f = Fixture::new();
let (offer, grant, _) = f.add("running", true, None, script());
let grant = grant.unwrap();
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
assert_eq!(interrupted.len(), 1);
let original = rmm_agent_core::jobs::recover_interrupted_outcome(
&f.ledger,
&f.binding,
&offer,
&interrupted[0].1,
)
.unwrap()
.unwrap();
let envelope = persist_original_fixture_result(&mut f, &offer, &grant, original);
assert_eq!(
f.ledger
.query_row::<u32, _, _>(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get(0)
)
.unwrap(),
1
);
let mut client = f.client();
let prepared = f.prepare(&mut client);
let decision = f.decision(&prepared, true);
assert!(decision.attempts[0].unlock);
assert_eq!(
prepared.request.attempts[0].result_message_id,
envelope.message_id
);
f.apply(&mut client, &decision).unwrap();
assert!(!client.can_start(&f.channel, false, &f.clock));
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
assert_eq!(
f.ledger
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap(),
1
);
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
}
#[test]
fn crash_after_result_commit_before_finish_retains_exact_result_and_local_prior() {
let mut f = Fixture::new();
let (offer, grant, _) = f.add("running", true, None, script());
let grant = grant.unwrap();
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
let original = persist_original_fixture_result(&mut f, &offer, &grant, completion_outcome());
assert!(
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
.unwrap()
.is_empty()
);
assert_eq!(
rmm_agent_core::jobs::events(&f.db).unwrap(),
vec![original.clone()]
);
let mut client = f.client();
let p = f.prepare(&mut client);
let d = f.decision(&p, true);
f.apply(&mut client, &d).unwrap();
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
let archived: Vec<u8> =
f.db.query_row(
"SELECT envelope FROM reconcile_results WHERE id=?1",
[&original.message_id],
|r| r.get(0),
)
.unwrap();
assert_eq!(archived, original.encode_to_vec());
}
#[test]
fn crash_before_running_event_recovers_real_consumption_without_inventing_completion() {
let mut f = Fixture::new();
let (offer, grant, _) = f.add("running", true, None, script());
let grant = grant.unwrap();
f.db.execute("UPDATE job_attempts SET state='requesting',grant_id=''", [])
.unwrap();
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
assert_eq!(interrupted[0].1, grant.grant_id.to_string());
assert!(
rmm_agent_core::jobs::recover_interrupted_outcome(
&f.ledger,
&f.binding,
&offer,
&interrupted[0].1
)
.unwrap()
.is_none()
);
persist_original_fixture_result(
&mut f,
&offer,
&grant,
rmm_agent_core::auth::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
);
let mut c = f.client();
let p = f.prepare(&mut c);
let d = f.decision(&p, false);
assert!(!d.attempts[0].unlock);
f.apply(&mut c, &d).unwrap();
assert!(!c.can_start(&f.channel, false, &f.clock));
assert_eq!(
f.ledger
.query_row::<u32, _, _>(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get(0)
)
.unwrap(),
1
);
let missing: Option<Vec<u8>> = f
.ledger
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
.unwrap();
assert!(missing.is_none());
}
#[test]
fn signed_unlock_still_rejects_a_missing_local_outcome_in_legacy_crash_window() {
let mut f = Fixture::new();
let (_, grant, _) = f.add("succeeded", true, Some(success()), script());
let grant = grant.unwrap();
f.ledger
.execute("UPDATE execution_evidence SET outcome=NULL", [])
.unwrap();
let mut c = f.client();
let p = f.prepare(&mut c);
let d = f.decision(&p, true);
assert!(d.attempts[0].unlock);
f.apply(&mut c, &d).unwrap();
assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err());
assert!(!c.can_start(&f.channel, false, &f.clock));
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
}
#[test]
fn original_completion_after_reconnect_unknown_is_recovered_without_reexecuting() {
let mut f = Fixture::new();
let (offer, grant, _) = f.add("running", true, None, script());
let grant = grant.unwrap();
let mut previous = f.client();
let previous_request = f.prepare(&mut previous);
assert_eq!(previous_request.request.attempts[0].state, "unknown");
assert!(
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
.unwrap()
.is_empty()
);
// The already-running executor completes after the transport fenced it.
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
drop(previous);
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
assert_eq!(interrupted.len(), 1);
let saved = rmm_agent_core::jobs::recover_interrupted_outcome(
&f.ledger,
&f.binding,
&offer,
&interrupted[0].1,
)
.unwrap()
.unwrap();
let original = persist_original_fixture_result(&mut f, &offer, &grant, saved);
let mut process = f.client();
let p = f.prepare(&mut process);
let d = f.decision(&p, true);
assert_eq!(p.request.attempts[0].result_message_id, original.message_id);
f.apply(&mut process, &d).unwrap();
process.apply_consumption(&mut f.ledger, &f.clock).unwrap();
assert!(process.can_start(&f.channel, false, &f.clock));
assert_eq!(
f.ledger
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap(),
1
);
assert!(
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
.unwrap()
.is_empty()
);
}