Make start evidence atomic and recover only bound durable outcomes
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-14 02:01:17 +01:00
parent de3b408c34
commit 74736483df
13 changed files with 764 additions and 87 deletions
+167 -45
View File
@@ -72,60 +72,56 @@ pub fn interrupted_with(
verifier: &impl crate::crypto::Ed25519Verifier,
) -> Result<Vec<(JobOffer, String)>> {
let mut statement = db.prepare(
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
"SELECT offer,grant_id,state FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown') ORDER BY job,attempt",
)?;
let rows = statement.query_map([], |r| {
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
Ok((
r.get::<_, Vec<u8>>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})?;
let mut interrupted = Vec::new();
for row in rows {
let (bytes, grant) = row?;
let (bytes, mut grant, state) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"INTERRUPTED_OFFER_BINDING"
);
validate_interrupted_offer(&offer, &job)?;
// begin_execution may have committed before the running journal event.
// Recover its actual grant, never invent one from an accepted offer.
let (count, stored): (u64, Option<String>) = ledger.query_row(
"SELECT count(*),MIN(grant_id) FROM consumed WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
ensure!(count <= 1, "INTERRUPTED_MULTIPLE_CONSUMPTIONS");
if let Some(stored) = stored {
ensure!(
grant.is_empty() || grant == stored,
"INTERRUPTED_GRANT_CONFLICT"
);
grant = stored;
}
if state == "unknown" {
// A reconnect may have closed the running journal while the same
// original executor was still finishing. Only its already-durable
// terminal completion can repair this event gap; uncertainty stays.
if !grant.is_empty()
&& recover_interrupted_outcome_with(ledger, binding, &offer, &grant, verifier)?
.is_some_and(|saved| {
matches!(
saved.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
)
})
{
interrupted.push((offer, grant));
}
continue;
}
if !grant.is_empty() {
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
[&grant],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
)
.optional()?;
let (agent, generation, job_id, attempt, start) =
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == offer.job_id
&& attempt == offer.attempt_id,
"INTERRUPTED_CONSUMPTION_BINDING"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: Request = serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
// Verify the historical signed request at its original valid time;
// this is evidence only, never permission to start it again.
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
ensure!(
verified == permit
&& permit.grant_id.to_string() == grant
&& request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"INTERRUPTED_START_BINDING"
);
verified_interrupted_start(ledger, binding, &offer, &grant, verifier)?;
} else {
// A request may have committed remotely without its reply arriving.
// It can only become local unknown, never a fabricated grant/result.
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
@@ -138,6 +134,132 @@ pub fn interrupted_with(
Ok(interrupted)
}
fn validate_interrupted_offer(offer: &JobOffer, job: &JobPayload) -> Result<()> {
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"INTERRUPTED_OFFER_BINDING"
);
Ok(())
}
type InterruptedStartRow = (
String,
u64,
String,
String,
Vec<u8>,
Option<Vec<u8>>,
Option<Vec<u8>>,
);
type InterruptedOutcomes = (Option<Vec<u8>>, Option<Vec<u8>>);
fn verified_interrupted_start(
ledger: &Connection,
binding: &Binding,
offer: &JobOffer,
grant: &str,
verifier: &impl crate::crypto::Ed25519Verifier,
) -> Result<InterruptedOutcomes> {
let job = binding.job(&offer.payload_bytes)?;
validate_interrupted_offer(offer, &job)?;
ensure!(
Uuid::parse_str(grant).is_ok_and(|id| !id.is_nil()),
"INTERRUPTED_GRANT_ID"
);
let row:Option<InterruptedStartRow>=ledger.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[grant],
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
let (agent, generation, job_id, attempt, start, outcome, result) =
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == offer.job_id
&& attempt == offer.attempt_id,
"INTERRUPTED_CONSUMPTION_BINDING"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let boot = start["previous_boot_id"]
.as_str()
.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_ORIGINAL_BOOT_MISSING"))?;
ensure!(
Uuid::parse_str(boot).is_ok_and(|id| !id.is_nil()),
"INTERRUPTED_ORIGINAL_BOOT_MISSING"
);
let _: chrono::DateTime<chrono::Utc> = serde_json::from_value(start["reserved_at"].clone())?;
ensure!(
start["reauthentication_required"] == true && start["postconditions_passed"] == false,
"INTERRUPTED_START_WITNESS_PROFILE"
);
let request: Request = serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
// Historical verification is evidence only; it can NEVER start another job.
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
ensure!(
verified == permit
&& permit.grant_id.to_string() == grant
&& request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"INTERRUPTED_START_BINDING"
);
Ok((outcome, result))
}
/// Recover only a completion already durably recorded by the original executor.
/// No outcome means uncertainty, not nonstart or a permission to execute again.
/// Caller re-journalizes this evidence, never calls finish/unlocks at startup.
pub fn recover_interrupted_outcome(
ledger: &Connection,
binding: &Binding,
offer: &JobOffer,
grant: &str,
) -> Result<Option<Outcome>> {
recover_interrupted_outcome_with(
ledger,
binding,
offer,
grant,
&crate::crypto::DefaultVerifier::default(),
)
}
pub fn recover_interrupted_outcome_with(
ledger: &Connection,
binding: &Binding,
offer: &JobOffer,
grant: &str,
verifier: &impl crate::crypto::Ed25519Verifier,
) -> Result<Option<Outcome>> {
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
let (outcome, result) = verified_interrupted_start(ledger, binding, offer, grant, verifier)?;
let canonical = |bytes: &[u8]| -> Result<Vec<u8>> {
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
ReconciledOutcome,
>(bytes)?)?)
};
if let (Some(a), Some(b)) = (&outcome, &result) {
ensure!(
canonical(a)? == canonical(b)?,
"INTERRUPTED_OUTCOME_CONFLICT"
);
}
let Some(bytes) = outcome.or(result) else {
return Ok(None);
};
let saved: Outcome = serde_json::from_slice(&canonical(&bytes)?)?;
ensure!(
matches!(
saved.state.as_str(),
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
),
"INTERRUPTED_OUTCOME_STATE"
);
Ok(Some(saved))
}
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
ensure!(
db.execute(