fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:11:45 +01:00
parent da7eb8109c
commit c512bd6fb2
36 changed files with 2556 additions and 38 deletions
+86
View File
@@ -92,6 +92,92 @@ fn hex_hash(s: &str) -> bool {
s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
/// executor thread from the previous process can still deliver its outcome.
/// This does not execute, release consumption or invent a successful outcome.
pub fn interrupted(
db: &Connection,
ledger: &Connection,
binding: &Binding,
) -> Result<Vec<(JobOffer, String)>> {
let mut statement = db.prepare(
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
)?;
let rows = statement.query_map([], |r| {
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
})?;
let mut interrupted = Vec::new();
for row in rows {
let (bytes, grant) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"INTERRUPTED_OFFER_BINDING"
);
if !grant.is_empty() {
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
[&grant],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
)
.optional()?;
let (agent, generation, job_id, attempt, start) =
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == offer.job_id
&& attempt == offer.attempt_id,
"INTERRUPTED_CONSUMPTION_BINDING"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: Request = serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
// Verify the historical signed request at its original valid time;
// this is evidence only, never permission to start it again.
let (_, verified) = binding.verify(&request, permit.not_before)?;
ensure!(
verified == permit
&& permit.grant_id.to_string() == grant
&& request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"INTERRUPTED_START_BINDING"
);
} else {
// A request may have committed remotely without its reply arriving.
// It can only become local unknown, never a fabricated grant/result.
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
}
interrupted.push((offer, grant));
}
Ok(interrupted)
}
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
ensure!(
db.execute(
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
params![offer.job_id, offer.attempt_id],
)? == 1,
"INTERRUPTED_REQUEST_STATE"
);
Ok(())
}
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");