Make start evidence atomic and recover only bound durable outcomes
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-14 02:01:17 +01:00
parent de3b408c34
commit 74736483df
13 changed files with 764 additions and 87 deletions
+141 -19
View File
@@ -1,8 +1,13 @@
//! Connection-level ledger rules, with OS path/lock/ACL handling left to adapters.
use crate::auth::{Outcome, Request};
use anyhow::{Result, ensure};
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Utc};
use rmm_domain::jobs::{ExecutionPermitPayload, JobPayload};
use rmm_domain::{
jobs::hash,
reconciliation::{ReconciledOutcome, canonical_outcome_bytes},
};
use rusqlite::{OptionalExtension, TransactionBehavior, params};
use uuid::Uuid;
/// Finite 1 GiB bound for durable agent/helper history. Unexpected page sizes
@@ -29,18 +34,24 @@ pub fn initialize_ledger(db: &rusqlite::Connection) -> Result<()> {
Ok(())
}
/// Legacy split reservation API retained for old fixtures/readers. Production
/// callers use begin_execution so a crash cannot lose the original start witness.
pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
// Unknown effects retain the global device lock, including across restart.
let unresolved: i64 = tx.query_row(
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
consume_in(&tx, p)?;
tx.commit()?;
Ok(())
}
fn consume_in(db: &rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
let unresolved: i64 = db.query_row(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get(0),
)?;
ensure!(unresolved == 0, "UNRESOLVED_EFFECT_RESOURCE_LOCKED");
tx.execute(
db.execute(
"INSERT INTO consumed(grant_id,agent,generation,job,attempt) VALUES(?1,?2,?3,?4,?5)",
rusqlite::params![
params![
p.grant_id.to_string(),
p.agent_id.to_string(),
p.enrollment_generation,
@@ -48,29 +59,140 @@ pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Res
p.attempt_id.to_string()
],
)?;
Ok(())
}
/// Caller has already checked the signature, qualified clock, fresh Sync and
/// policy. Persist consumption and the exact legacy start witness atomically.
pub fn begin_execution(
db: &mut rusqlite::Connection,
permit: &ExecutionPermitPayload,
request: &Request,
previous_boot_id: Uuid,
recorded_at: DateTime<Utc>,
) -> Result<()> {
validate_start_request(permit, request)?;
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
consume_in(&tx, permit)?;
save_start(&tx, permit, request, previous_boot_id, recorded_at)?;
tx.commit()?;
Ok(())
}
fn validate_start_request(p: &ExecutionPermitPayload, request: &Request) -> Result<()> {
ensure!(
request.job_bytes.len() <= 65536
&& request.permit_bytes.len() <= 8192
&& request.signature.len() == 64,
"START_REQUEST_SIZE"
);
let saved: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?;
let j: JobPayload = serde_json::from_slice(&request.job_bytes)?;
ensure!(
saved == *p
&& request.authorization_id == p.authorization_id
&& request.plan_hash == p.plan_hash
&& p.job_payload_hash == hash(&request.job_bytes)
&& p.tenant_id == j.tenant_id
&& p.asset_id == j.asset_id
&& p.agent_id == j.agent_id
&& p.enrollment_generation == j.enrollment_generation
&& p.assignment_version == j.assignment_version
&& p.job_id == j.job_id
&& p.attempt_id == j.attempt_id
&& p.action == j.action.name()
&& p.execution_identity == j.execution_identity
&& p.max_runtime_seconds == j.max_runtime_seconds,
"STORED_START_REQUEST_BINDING"
);
Ok(())
}
fn canonical(bytes: &[u8]) -> Result<Vec<u8>> {
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
ReconciledOutcome,
>(bytes)?)?)
}
/// Compare-and-set the original completion evidence WITHOUT releasing a lock.
/// Existing identical bytes are preserved; contradictory completions fail closed.
pub fn persist_outcome(
db: &rusqlite::Connection,
p: &ExecutionPermitPayload,
outcome: &Outcome,
) -> Result<()> {
let tx = db.unchecked_transaction()?;
persist_outcome_in(&tx, p, outcome)?;
tx.commit()?;
Ok(())
}
fn persist_outcome_in(
db: &rusqlite::Connection,
p: &ExecutionPermitPayload,
outcome: &Outcome,
) -> Result<Vec<u8>> {
ensure!(
matches!(
outcome.state.as_str(),
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
),
"INVALID_STORED_OUTCOME_STATE"
);
type CompletionRow = (
String,
u64,
String,
String,
Vec<u8>,
Option<Vec<u8>>,
Option<Vec<u8>>,
);
let row:Option<CompletionRow>=db.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[p.grant_id.to_string()],
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
let (agent, generation, job, attempt, start, prior, finished) =
row.context("STORED_COMPLETION_START_MISSING")?;
ensure!(
agent == p.agent_id.to_string()
&& generation == p.enrollment_generation
&& job == p.job_id.to_string()
&& attempt == p.attempt_id.to_string(),
"STORED_COMPLETION_CONSUMPTION_BINDING"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: Request = serde_json::from_value(start["request"].clone())?;
validate_start_request(p, &request)?;
let bytes = serde_json::to_vec(outcome)?;
let expected = canonical(&bytes)?;
for previous in [&prior, &finished].into_iter().flatten() {
ensure!(
canonical(previous)? == expected,
"STORED_COMPLETION_CONFLICT"
);
}
if prior.is_none() {
ensure!(
db.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1 AND outcome IS NULL",
params![p.grant_id.to_string(), bytes]
)? == 1,
"STORED_COMPLETION_CAS_CONFLICT"
);
}
Ok(prior.unwrap_or(bytes))
}
/// Called only AFTER the corresponding journal result committed. A crash before
/// this call retains the original outcome for the normal signed reconciliation.
pub fn finish(
db: &rusqlite::Connection,
p: &ExecutionPermitPayload,
outcome: &Outcome,
) -> Result<()> {
db.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
rusqlite::params![p.grant_id.to_string(), serde_json::to_vec(outcome)?],
)?;
// Unknown / awaiting reboot keep the resource reserved until a real sync
// protocol exists. Never unlock on a process exit or a network reconnect.
let tx = db.unchecked_transaction()?;
let bytes = persist_outcome_in(&tx, p, outcome)?;
if outcome.state != "unknown" && outcome.state != "awaiting_reboot" {
ensure!(
db.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1 AND result IS NULL",
rusqlite::params![p.grant_id.to_string(), serde_json::to_vec(outcome)?]
)? == 1,
"CONSUMPTION_NOT_FOUND"
);
tx.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1 AND result IS NULL",
params![p.grant_id.to_string(), bytes],
)?;
}
tx.commit()?;
Ok(())
}