Make start evidence atomic and recover only bound durable outcomes
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
+167
-45
@@ -72,60 +72,56 @@ pub fn interrupted_with(
|
||||
verifier: &impl crate::crypto::Ed25519Verifier,
|
||||
) -> Result<Vec<(JobOffer, String)>> {
|
||||
let mut statement = db.prepare(
|
||||
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
|
||||
"SELECT offer,grant_id,state FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown') ORDER BY job,attempt",
|
||||
)?;
|
||||
let rows = statement.query_map([], |r| {
|
||||
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
|
||||
Ok((
|
||||
r.get::<_, Vec<u8>>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut interrupted = Vec::new();
|
||||
for row in rows {
|
||||
let (bytes, grant) = row?;
|
||||
let (bytes, mut grant, state) = row?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
validate_interrupted_offer(&offer, &job)?;
|
||||
// begin_execution may have committed before the running journal event.
|
||||
// Recover its actual grant, never invent one from an accepted offer.
|
||||
let (count, stored): (u64, Option<String>) = ledger.query_row(
|
||||
"SELECT count(*),MIN(grant_id) FROM consumed WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
ensure!(count <= 1, "INTERRUPTED_MULTIPLE_CONSUMPTIONS");
|
||||
if let Some(stored) = stored {
|
||||
ensure!(
|
||||
grant.is_empty() || grant == stored,
|
||||
"INTERRUPTED_GRANT_CONFLICT"
|
||||
);
|
||||
grant = stored;
|
||||
}
|
||||
if state == "unknown" {
|
||||
// A reconnect may have closed the running journal while the same
|
||||
// original executor was still finishing. Only its already-durable
|
||||
// terminal completion can repair this event gap; uncertainty stays.
|
||||
if !grant.is_empty()
|
||||
&& recover_interrupted_outcome_with(ledger, binding, &offer, &grant, verifier)?
|
||||
.is_some_and(|saved| {
|
||||
matches!(
|
||||
saved.state.as_str(),
|
||||
"succeeded" | "failed" | "cancelled" | "expired"
|
||||
)
|
||||
})
|
||||
{
|
||||
interrupted.push((offer, grant));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !grant.is_empty() {
|
||||
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
|
||||
.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[&grant],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (agent, generation, job_id, attempt, start) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Verify the historical signed request at its original valid time;
|
||||
// this is evidence only, never permission to start it again.
|
||||
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
verified_interrupted_start(ledger, binding, &offer, &grant, verifier)?;
|
||||
} else {
|
||||
// A request may have committed remotely without its reply arriving.
|
||||
// It can only become local unknown, never a fabricated grant/result.
|
||||
let state: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
@@ -138,6 +134,132 @@ pub fn interrupted_with(
|
||||
Ok(interrupted)
|
||||
}
|
||||
|
||||
fn validate_interrupted_offer(offer: &JobOffer, job: &JobPayload) -> Result<()> {
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
type InterruptedStartRow = (
|
||||
String,
|
||||
u64,
|
||||
String,
|
||||
String,
|
||||
Vec<u8>,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
);
|
||||
type InterruptedOutcomes = (Option<Vec<u8>>, Option<Vec<u8>>);
|
||||
fn verified_interrupted_start(
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
verifier: &impl crate::crypto::Ed25519Verifier,
|
||||
) -> Result<InterruptedOutcomes> {
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
validate_interrupted_offer(offer, &job)?;
|
||||
ensure!(
|
||||
Uuid::parse_str(grant).is_ok_and(|id| !id.is_nil()),
|
||||
"INTERRUPTED_GRANT_ID"
|
||||
);
|
||||
let row:Option<InterruptedStartRow>=ledger.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[grant],
|
||||
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
||||
let (agent, generation, job_id, attempt, start, outcome, result) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let boot = start["previous_boot_id"]
|
||||
.as_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_ORIGINAL_BOOT_MISSING"))?;
|
||||
ensure!(
|
||||
Uuid::parse_str(boot).is_ok_and(|id| !id.is_nil()),
|
||||
"INTERRUPTED_ORIGINAL_BOOT_MISSING"
|
||||
);
|
||||
let _: chrono::DateTime<chrono::Utc> = serde_json::from_value(start["reserved_at"].clone())?;
|
||||
ensure!(
|
||||
start["reauthentication_required"] == true && start["postconditions_passed"] == false,
|
||||
"INTERRUPTED_START_WITNESS_PROFILE"
|
||||
);
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Historical verification is evidence only; it can NEVER start another job.
|
||||
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
Ok((outcome, result))
|
||||
}
|
||||
/// Recover only a completion already durably recorded by the original executor.
|
||||
/// No outcome means uncertainty, not nonstart or a permission to execute again.
|
||||
/// Caller re-journalizes this evidence, never calls finish/unlocks at startup.
|
||||
pub fn recover_interrupted_outcome(
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
) -> Result<Option<Outcome>> {
|
||||
recover_interrupted_outcome_with(
|
||||
ledger,
|
||||
binding,
|
||||
offer,
|
||||
grant,
|
||||
&crate::crypto::DefaultVerifier::default(),
|
||||
)
|
||||
}
|
||||
pub fn recover_interrupted_outcome_with(
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
verifier: &impl crate::crypto::Ed25519Verifier,
|
||||
) -> Result<Option<Outcome>> {
|
||||
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
|
||||
let (outcome, result) = verified_interrupted_start(ledger, binding, offer, grant, verifier)?;
|
||||
let canonical = |bytes: &[u8]| -> Result<Vec<u8>> {
|
||||
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
|
||||
ReconciledOutcome,
|
||||
>(bytes)?)?)
|
||||
};
|
||||
if let (Some(a), Some(b)) = (&outcome, &result) {
|
||||
ensure!(
|
||||
canonical(a)? == canonical(b)?,
|
||||
"INTERRUPTED_OUTCOME_CONFLICT"
|
||||
);
|
||||
}
|
||||
let Some(bytes) = outcome.or(result) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let saved: Outcome = serde_json::from_slice(&canonical(&bytes)?)?;
|
||||
ensure!(
|
||||
matches!(
|
||||
saved.state.as_str(),
|
||||
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
|
||||
),
|
||||
"INTERRUPTED_OUTCOME_STATE"
|
||||
);
|
||||
Ok(Some(saved))
|
||||
}
|
||||
|
||||
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
||||
ensure!(
|
||||
db.execute(
|
||||
|
||||
+141
-19
@@ -1,8 +1,13 @@
|
||||
//! Connection-level ledger rules, with OS path/lock/ACL handling left to adapters.
|
||||
use crate::auth::{Outcome, Request};
|
||||
use anyhow::{Result, ensure};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rmm_domain::jobs::{ExecutionPermitPayload, JobPayload};
|
||||
use rmm_domain::{
|
||||
jobs::hash,
|
||||
reconciliation::{ReconciledOutcome, canonical_outcome_bytes},
|
||||
};
|
||||
use rusqlite::{OptionalExtension, TransactionBehavior, params};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Finite 1 GiB bound for durable agent/helper history. Unexpected page sizes
|
||||
@@ -29,18 +34,24 @@ pub fn initialize_ledger(db: &rusqlite::Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Legacy split reservation API retained for old fixtures/readers. Production
|
||||
/// callers use begin_execution so a crash cannot lose the original start witness.
|
||||
pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
|
||||
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
// Unknown effects retain the global device lock, including across restart.
|
||||
let unresolved: i64 = tx.query_row(
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
consume_in(&tx, p)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn consume_in(db: &rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
|
||||
let unresolved: i64 = db.query_row(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(unresolved == 0, "UNRESOLVED_EFFECT_RESOURCE_LOCKED");
|
||||
tx.execute(
|
||||
db.execute(
|
||||
"INSERT INTO consumed(grant_id,agent,generation,job,attempt) VALUES(?1,?2,?3,?4,?5)",
|
||||
rusqlite::params![
|
||||
params![
|
||||
p.grant_id.to_string(),
|
||||
p.agent_id.to_string(),
|
||||
p.enrollment_generation,
|
||||
@@ -48,29 +59,140 @@ pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Res
|
||||
p.attempt_id.to_string()
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
/// Caller has already checked the signature, qualified clock, fresh Sync and
|
||||
/// policy. Persist consumption and the exact legacy start witness atomically.
|
||||
pub fn begin_execution(
|
||||
db: &mut rusqlite::Connection,
|
||||
permit: &ExecutionPermitPayload,
|
||||
request: &Request,
|
||||
previous_boot_id: Uuid,
|
||||
recorded_at: DateTime<Utc>,
|
||||
) -> Result<()> {
|
||||
validate_start_request(permit, request)?;
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
consume_in(&tx, permit)?;
|
||||
save_start(&tx, permit, request, previous_boot_id, recorded_at)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn validate_start_request(p: &ExecutionPermitPayload, request: &Request) -> Result<()> {
|
||||
ensure!(
|
||||
request.job_bytes.len() <= 65536
|
||||
&& request.permit_bytes.len() <= 8192
|
||||
&& request.signature.len() == 64,
|
||||
"START_REQUEST_SIZE"
|
||||
);
|
||||
let saved: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?;
|
||||
let j: JobPayload = serde_json::from_slice(&request.job_bytes)?;
|
||||
ensure!(
|
||||
saved == *p
|
||||
&& request.authorization_id == p.authorization_id
|
||||
&& request.plan_hash == p.plan_hash
|
||||
&& p.job_payload_hash == hash(&request.job_bytes)
|
||||
&& p.tenant_id == j.tenant_id
|
||||
&& p.asset_id == j.asset_id
|
||||
&& p.agent_id == j.agent_id
|
||||
&& p.enrollment_generation == j.enrollment_generation
|
||||
&& p.assignment_version == j.assignment_version
|
||||
&& p.job_id == j.job_id
|
||||
&& p.attempt_id == j.attempt_id
|
||||
&& p.action == j.action.name()
|
||||
&& p.execution_identity == j.execution_identity
|
||||
&& p.max_runtime_seconds == j.max_runtime_seconds,
|
||||
"STORED_START_REQUEST_BINDING"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn canonical(bytes: &[u8]) -> Result<Vec<u8>> {
|
||||
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
|
||||
ReconciledOutcome,
|
||||
>(bytes)?)?)
|
||||
}
|
||||
/// Compare-and-set the original completion evidence WITHOUT releasing a lock.
|
||||
/// Existing identical bytes are preserved; contradictory completions fail closed.
|
||||
pub fn persist_outcome(
|
||||
db: &rusqlite::Connection,
|
||||
p: &ExecutionPermitPayload,
|
||||
outcome: &Outcome,
|
||||
) -> Result<()> {
|
||||
let tx = db.unchecked_transaction()?;
|
||||
persist_outcome_in(&tx, p, outcome)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn persist_outcome_in(
|
||||
db: &rusqlite::Connection,
|
||||
p: &ExecutionPermitPayload,
|
||||
outcome: &Outcome,
|
||||
) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
matches!(
|
||||
outcome.state.as_str(),
|
||||
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
|
||||
),
|
||||
"INVALID_STORED_OUTCOME_STATE"
|
||||
);
|
||||
type CompletionRow = (
|
||||
String,
|
||||
u64,
|
||||
String,
|
||||
String,
|
||||
Vec<u8>,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
);
|
||||
let row:Option<CompletionRow>=db.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[p.grant_id.to_string()],
|
||||
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
||||
let (agent, generation, job, attempt, start, prior, finished) =
|
||||
row.context("STORED_COMPLETION_START_MISSING")?;
|
||||
ensure!(
|
||||
agent == p.agent_id.to_string()
|
||||
&& generation == p.enrollment_generation
|
||||
&& job == p.job_id.to_string()
|
||||
&& attempt == p.attempt_id.to_string(),
|
||||
"STORED_COMPLETION_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
validate_start_request(p, &request)?;
|
||||
let bytes = serde_json::to_vec(outcome)?;
|
||||
let expected = canonical(&bytes)?;
|
||||
for previous in [&prior, &finished].into_iter().flatten() {
|
||||
ensure!(
|
||||
canonical(previous)? == expected,
|
||||
"STORED_COMPLETION_CONFLICT"
|
||||
);
|
||||
}
|
||||
if prior.is_none() {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1 AND outcome IS NULL",
|
||||
params![p.grant_id.to_string(), bytes]
|
||||
)? == 1,
|
||||
"STORED_COMPLETION_CAS_CONFLICT"
|
||||
);
|
||||
}
|
||||
Ok(prior.unwrap_or(bytes))
|
||||
}
|
||||
/// Called only AFTER the corresponding journal result committed. A crash before
|
||||
/// this call retains the original outcome for the normal signed reconciliation.
|
||||
pub fn finish(
|
||||
db: &rusqlite::Connection,
|
||||
p: &ExecutionPermitPayload,
|
||||
outcome: &Outcome,
|
||||
) -> Result<()> {
|
||||
db.execute(
|
||||
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
|
||||
rusqlite::params![p.grant_id.to_string(), serde_json::to_vec(outcome)?],
|
||||
)?;
|
||||
// Unknown / awaiting reboot keep the resource reserved until a real sync
|
||||
// protocol exists. Never unlock on a process exit or a network reconnect.
|
||||
let tx = db.unchecked_transaction()?;
|
||||
let bytes = persist_outcome_in(&tx, p, outcome)?;
|
||||
if outcome.state != "unknown" && outcome.state != "awaiting_reboot" {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE consumed SET result=?2 WHERE grant_id=?1 AND result IS NULL",
|
||||
rusqlite::params![p.grant_id.to_string(), serde_json::to_vec(outcome)?]
|
||||
)? == 1,
|
||||
"CONSUMPTION_NOT_FOUND"
|
||||
);
|
||||
tx.execute(
|
||||
"UPDATE consumed SET result=?2 WHERE grant_id=?1 AND result IS NULL",
|
||||
params![p.grant_id.to_string(), bytes],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user