Reconcile a human closure when the helper has no result
Rust development slice / linux (push) Failing after 9s
Rust development slice / linux (push) Failing after 9s
This commit is contained in:
@@ -1485,6 +1485,207 @@ fn persist_original_fixture_result(
|
||||
fn completion_outcome() -> rmm_agent_core::auth::Outcome {
|
||||
serde_json::from_slice(&canonical_outcome_bytes(&success()).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn human_closure_reconciles_agent_interruption_and_a_helper_without_result() {
|
||||
use rmm_agent_core::helper_reconcile::{self, BootSource, ExecutorLedgerGuard, HelperSync};
|
||||
|
||||
struct TestGuard;
|
||||
impl ExecutorLedgerGuard for TestGuard {
|
||||
fn check(&self, db: &Connection) -> anyhow::Result<()> {
|
||||
anyhow::ensure!(
|
||||
db.is_autocommit(),
|
||||
"fixture ledger must be outside a transaction"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
struct TestBoot(Uuid);
|
||||
impl BootSource for TestBoot {
|
||||
fn current_boot(&self) -> anyhow::Result<Uuid> {
|
||||
Ok(self.0)
|
||||
}
|
||||
}
|
||||
fn map(snapshot: &helper_reconcile::Snapshot) -> HelperSnapshot {
|
||||
HelperSnapshot {
|
||||
journal_id: snapshot.journal_id,
|
||||
checkpoint: snapshot.checkpoint,
|
||||
previous_decision_id: snapshot.previous_decision_id,
|
||||
attempts: snapshot
|
||||
.attempts
|
||||
.iter()
|
||||
.map(|a| SyncAttempt {
|
||||
job_id: a.job_id.to_string(),
|
||||
attempt_id: a.attempt_id.to_string(),
|
||||
job_payload_hash: a.job_payload_hash.clone(),
|
||||
grant_id: a.grant_id.to_string(),
|
||||
consumption: a.consumption.clone(),
|
||||
state: a.state.clone(),
|
||||
revision: a.revision,
|
||||
original_boot_id: a.original_boot_id.to_string(),
|
||||
outcome_sha256: a.outcome_sha256.clone(),
|
||||
outcome_bytes: a.outcome_bytes.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
let closed = canonical_outcome_bytes(&ReconciledOutcome {
|
||||
state: "closed_unknown".into(),
|
||||
stdout: vec![],
|
||||
stderr: vec![],
|
||||
exit_code: None,
|
||||
output_truncated: false,
|
||||
reboot_required: false,
|
||||
evidence: serde_json::json!({
|
||||
"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"retry":false
|
||||
}),
|
||||
error_code: "OPERATOR_CLOSED_UNKNOWN".into(),
|
||||
})
|
||||
.unwrap();
|
||||
for interrupt_after_projection in [false, true] {
|
||||
let mut f = Fixture::new();
|
||||
reconcile::initialize(&f.db).unwrap();
|
||||
let (offer, grant, _) = f.add(
|
||||
"running",
|
||||
true,
|
||||
None,
|
||||
Action::SetServiceState {
|
||||
unit: "rmm-acceptance.service".into(),
|
||||
state: "running".into(),
|
||||
},
|
||||
);
|
||||
let grant = grant.unwrap();
|
||||
let archive = persist_original_fixture_result(
|
||||
&mut f,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_agent_core::auth::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
);
|
||||
rmm_agent_core::jobs::ack(&mut f.db, &archive.message_id).unwrap();
|
||||
let start: Vec<u8> = f
|
||||
.ledger
|
||||
.query_row("SELECT start FROM execution_evidence", [], |row| row.get(0))
|
||||
.unwrap();
|
||||
let start: serde_json::Value = serde_json::from_slice(&start).unwrap();
|
||||
let request: Request = serde_json::from_value(start["request"].clone()).unwrap();
|
||||
let boot = TestBoot(f.channel.boot_id);
|
||||
let guard = TestGuard;
|
||||
let mut helper_db = Connection::open_in_memory().unwrap();
|
||||
rmm_agent_core::ledger::initialize_ledger(&helper_db).unwrap();
|
||||
rmm_agent_core::ledger::begin_execution(
|
||||
&mut helper_db,
|
||||
&grant,
|
||||
&request,
|
||||
boot.0,
|
||||
f.clock.now,
|
||||
)
|
||||
.unwrap();
|
||||
let mut helper = HelperSync::new(f.binding.clone(), &f.clock).unwrap();
|
||||
let mut client = f.client();
|
||||
|
||||
for round in 0..if interrupt_after_projection { 3 } else { 2 } {
|
||||
let snapshot = map(&helper
|
||||
.snapshot(
|
||||
&mut helper_db,
|
||||
&guard,
|
||||
f.channel.connection_generation,
|
||||
&f.clock,
|
||||
&boot,
|
||||
)
|
||||
.unwrap());
|
||||
assert_eq!(snapshot.attempts[0].state, "unknown");
|
||||
assert!(snapshot.attempts[0].outcome_bytes.is_empty());
|
||||
let prepared = client
|
||||
.prepare(
|
||||
&mut f.db,
|
||||
&f.ledger,
|
||||
f.channel.clone(),
|
||||
Some(&snapshot),
|
||||
&f.clock,
|
||||
)
|
||||
.unwrap();
|
||||
let agent_attempt = &prepared.request.attempts[0];
|
||||
assert!(!agent_attempt.outcome_bytes.is_empty());
|
||||
let mut d = f.decision(&prepared, round != 0);
|
||||
d.issued_at = f.clock.now;
|
||||
if round != 0 {
|
||||
let attempt = &mut d.attempts[0];
|
||||
attempt.revision = 3;
|
||||
attempt.outcome_state = "closed_unknown".into();
|
||||
attempt.reason = "OPERATOR_CLOSED_UNKNOWN".into();
|
||||
attempt.unlock = true;
|
||||
attempt.outcome_sha256 = Some(hash(&closed));
|
||||
if agent_attempt.state == "unknown" {
|
||||
attempt.outcome_bytes = closed.clone();
|
||||
}
|
||||
}
|
||||
let signed = f.sign(&d);
|
||||
client.apply(&mut f.db, &signed, &f.clock).unwrap();
|
||||
if interrupt_after_projection && round == 1 {
|
||||
client = f.client();
|
||||
continue;
|
||||
}
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
let raw = prepared.request.encode_to_vec();
|
||||
let response = helper_reconcile::SignedResponse {
|
||||
decision_bytes: signed.decision_bytes,
|
||||
signature: signed.signature,
|
||||
};
|
||||
let applied = helper
|
||||
.apply(&mut helper_db, &guard, &response, &raw, &f.clock, &boot)
|
||||
.unwrap();
|
||||
if round == 0 {
|
||||
assert!(!applied.ready && applied.unlocked_grants.is_empty());
|
||||
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||
} else {
|
||||
assert!(applied.ready);
|
||||
assert_eq!(applied.unlocked_grants, vec![grant.grant_id]);
|
||||
assert!(
|
||||
helper
|
||||
.apply(&mut helper_db, &guard, &response, &raw, &f.clock, &boot)
|
||||
.unwrap()
|
||||
.unlocked_grants
|
||||
.is_empty()
|
||||
);
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
client.confirm_helper(&map(&applied.snapshot)).unwrap();
|
||||
assert!(client.can_start(&f.channel, true, &f.clock));
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
f.db.query_row::<String, _, _>("SELECT state FROM job_attempts", [], |rec| rec.get(0))
|
||||
.unwrap(),
|
||||
"closed_unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
f.db.query_row::<Vec<u8>, _, _>("SELECT envelope FROM reconcile_results", [], |rec| {
|
||||
rec.get(0)
|
||||
})
|
||||
.unwrap(),
|
||||
archive.encode_to_vec()
|
||||
);
|
||||
for ledger in [&f.ledger, &helper_db] {
|
||||
assert_eq!(
|
||||
ledger
|
||||
.query_row::<Vec<u8>, _, _>("SELECT result FROM consumed", [], |rec| rec.get(0))
|
||||
.unwrap(),
|
||||
closed
|
||||
);
|
||||
assert_eq!(
|
||||
ledger
|
||||
.query_row::<(Vec<u8>, Vec<u8>), _, _>(
|
||||
"SELECT original_outcome,closure_outcome FROM unknown_closure_originals",
|
||||
[],
|
||||
|rec| Ok((rec.get(0)?, rec.get(1)?))
|
||||
)
|
||||
.unwrap(),
|
||||
(vec![], closed.clone())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crash_after_durable_outcome_before_result_recovers_original_then_signed_sync_closes() {
|
||||
let mut f = Fixture::new();
|
||||
|
||||
Reference in New Issue
Block a user