421 lines
16 KiB
Rust
421 lines
16 KiB
Rust
//! Durable job persistence without capability discovery or OS execution.
|
|
//! Caller performs fresh Sync/Clock/permit checks before consuming or executing.
|
|
use crate::auth::{Binding, Outcome, Request};
|
|
use anyhow::{Result, ensure};
|
|
use prost::Message;
|
|
use rmm_domain::jobs::{JobPayload, hash};
|
|
use rmm_protocol::*;
|
|
use rusqlite::{Connection, OptionalExtension, params};
|
|
use uuid::Uuid;
|
|
|
|
pub fn accept(
|
|
db: &Connection,
|
|
binding: &Binding,
|
|
offer: &JobOffer,
|
|
supported: impl FnOnce(&JobPayload) -> bool,
|
|
) -> Result<bool> {
|
|
let job = binding.job(&offer.payload_bytes)?;
|
|
ensure!(
|
|
offer.job_id == job.job_id.to_string()
|
|
&& offer.attempt_id == job.attempt_id.to_string()
|
|
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
|
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
|
&& offer.plan_hash.len() == 64
|
|
&& hex_hash(&offer.plan_hash),
|
|
"OFFER_BINDING"
|
|
);
|
|
ensure!(supported(&job), "UNSUPPORTED_CAPABILITY");
|
|
let bytes = offer.encode_to_vec();
|
|
let existing: Option<Vec<u8>> = db
|
|
.query_row(
|
|
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![offer.job_id, offer.attempt_id],
|
|
|r| r.get(0),
|
|
)
|
|
.optional()?;
|
|
if let Some(old) = existing {
|
|
ensure!(old == bytes, "OFFER_CONFLICT");
|
|
return Ok(false);
|
|
}
|
|
let count: i64 = db.query_row("SELECT count(*) FROM job_attempts", [], |r| r.get(0))?;
|
|
ensure!(count < 10000, "JOURNAL_CAPACITY_NEW_EFFECTS_BLOCKED");
|
|
db.execute(
|
|
"INSERT INTO job_attempts(job,attempt,offer,state) VALUES(?1,?2,?3,'accepted')",
|
|
params![offer.job_id, offer.attempt_id, bytes],
|
|
)?;
|
|
Ok(true)
|
|
}
|
|
fn hex_hash(s: &str) -> bool {
|
|
s.bytes()
|
|
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
|
}
|
|
|
|
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
|
|
/// executor thread from the previous process can still deliver its outcome.
|
|
/// This does not execute, release consumption or invent a successful outcome.
|
|
pub fn interrupted(
|
|
db: &Connection,
|
|
ledger: &Connection,
|
|
binding: &Binding,
|
|
) -> Result<Vec<(JobOffer, String)>> {
|
|
interrupted_with(
|
|
db,
|
|
ledger,
|
|
binding,
|
|
&crate::crypto::DefaultVerifier::default(),
|
|
)
|
|
}
|
|
pub fn interrupted_with(
|
|
db: &Connection,
|
|
ledger: &Connection,
|
|
binding: &Binding,
|
|
verifier: &impl crate::crypto::Ed25519Verifier,
|
|
) -> Result<Vec<(JobOffer, String)>> {
|
|
let mut statement = db.prepare(
|
|
"SELECT offer,grant_id,state FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown') ORDER BY job,attempt",
|
|
)?;
|
|
let rows = statement.query_map([], |r| {
|
|
Ok((
|
|
r.get::<_, Vec<u8>>(0)?,
|
|
r.get::<_, String>(1)?,
|
|
r.get::<_, String>(2)?,
|
|
))
|
|
})?;
|
|
let mut interrupted = Vec::new();
|
|
for row in rows {
|
|
let (bytes, mut grant, state) = row?;
|
|
let offer = JobOffer::decode(bytes.as_slice())?;
|
|
let job = binding.job(&offer.payload_bytes)?;
|
|
validate_interrupted_offer(&offer, &job)?;
|
|
// begin_execution may have committed before the running journal event.
|
|
// Recover its actual grant, never invent one from an accepted offer.
|
|
let (count, stored): (u64, Option<String>) = ledger.query_row(
|
|
"SELECT count(*),MIN(grant_id) FROM consumed WHERE job=?1 AND attempt=?2",
|
|
params![offer.job_id, offer.attempt_id],
|
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
)?;
|
|
ensure!(count <= 1, "INTERRUPTED_MULTIPLE_CONSUMPTIONS");
|
|
if let Some(stored) = stored {
|
|
ensure!(
|
|
grant.is_empty() || grant == stored,
|
|
"INTERRUPTED_GRANT_CONFLICT"
|
|
);
|
|
grant = stored;
|
|
}
|
|
if state == "unknown" {
|
|
// A reconnect may have closed the running journal while the same
|
|
// original executor was still finishing. Only its already-durable
|
|
// terminal completion can repair this event gap; uncertainty stays.
|
|
if !grant.is_empty()
|
|
&& recover_interrupted_outcome_with(ledger, binding, &offer, &grant, verifier)?
|
|
.is_some_and(|saved| {
|
|
matches!(
|
|
saved.state.as_str(),
|
|
"succeeded" | "failed" | "cancelled" | "expired"
|
|
)
|
|
})
|
|
{
|
|
interrupted.push((offer, grant));
|
|
}
|
|
continue;
|
|
}
|
|
if !grant.is_empty() {
|
|
verified_interrupted_start(ledger, binding, &offer, &grant, verifier)?;
|
|
} else {
|
|
let state: String = db.query_row(
|
|
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![offer.job_id, offer.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
|
|
}
|
|
interrupted.push((offer, grant));
|
|
}
|
|
Ok(interrupted)
|
|
}
|
|
|
|
fn validate_interrupted_offer(offer: &JobOffer, job: &JobPayload) -> Result<()> {
|
|
ensure!(
|
|
offer.job_id == job.job_id.to_string()
|
|
&& offer.attempt_id == job.attempt_id.to_string()
|
|
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
|
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
|
&& offer.plan_hash.len() == 64
|
|
&& hex_hash(&offer.plan_hash),
|
|
"INTERRUPTED_OFFER_BINDING"
|
|
);
|
|
Ok(())
|
|
}
|
|
type InterruptedStartRow = (
|
|
String,
|
|
u64,
|
|
String,
|
|
String,
|
|
Vec<u8>,
|
|
Option<Vec<u8>>,
|
|
Option<Vec<u8>>,
|
|
);
|
|
type InterruptedOutcomes = (Option<Vec<u8>>, Option<Vec<u8>>);
|
|
fn verified_interrupted_start(
|
|
ledger: &Connection,
|
|
binding: &Binding,
|
|
offer: &JobOffer,
|
|
grant: &str,
|
|
verifier: &impl crate::crypto::Ed25519Verifier,
|
|
) -> Result<InterruptedOutcomes> {
|
|
let job = binding.job(&offer.payload_bytes)?;
|
|
validate_interrupted_offer(offer, &job)?;
|
|
ensure!(
|
|
Uuid::parse_str(grant).is_ok_and(|id| !id.is_nil()),
|
|
"INTERRUPTED_GRANT_ID"
|
|
);
|
|
let row:Option<InterruptedStartRow>=ledger.query_row(
|
|
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[grant],
|
|
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
|
let (agent, generation, job_id, attempt, start, outcome, result) =
|
|
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
|
ensure!(
|
|
agent == binding.agent_id.to_string()
|
|
&& generation == binding.enrollment_generation
|
|
&& job_id == offer.job_id
|
|
&& attempt == offer.attempt_id,
|
|
"INTERRUPTED_CONSUMPTION_BINDING"
|
|
);
|
|
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
|
let boot = start["previous_boot_id"]
|
|
.as_str()
|
|
.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_ORIGINAL_BOOT_MISSING"))?;
|
|
ensure!(
|
|
Uuid::parse_str(boot).is_ok_and(|id| !id.is_nil()),
|
|
"INTERRUPTED_ORIGINAL_BOOT_MISSING"
|
|
);
|
|
let _: chrono::DateTime<chrono::Utc> = serde_json::from_value(start["reserved_at"].clone())?;
|
|
ensure!(
|
|
start["reauthentication_required"] == true && start["postconditions_passed"] == false,
|
|
"INTERRUPTED_START_WITNESS_PROFILE"
|
|
);
|
|
let request: Request = serde_json::from_value(start["request"].clone())?;
|
|
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
|
serde_json::from_slice(&request.permit_bytes)?;
|
|
// Historical verification is evidence only; it can NEVER start another job.
|
|
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
|
|
ensure!(
|
|
verified == permit
|
|
&& permit.grant_id.to_string() == grant
|
|
&& request.job_bytes == offer.payload_bytes
|
|
&& request.plan_hash == offer.plan_hash
|
|
&& request.authorization_id.to_string() == offer.authorization_id,
|
|
"INTERRUPTED_START_BINDING"
|
|
);
|
|
Ok((outcome, result))
|
|
}
|
|
/// Recover only a completion already durably recorded by the original executor.
|
|
/// No outcome means uncertainty, not nonstart or a permission to execute again.
|
|
/// Caller re-journalizes this evidence, never calls finish/unlocks at startup.
|
|
pub fn recover_interrupted_outcome(
|
|
ledger: &Connection,
|
|
binding: &Binding,
|
|
offer: &JobOffer,
|
|
grant: &str,
|
|
) -> Result<Option<Outcome>> {
|
|
recover_interrupted_outcome_with(
|
|
ledger,
|
|
binding,
|
|
offer,
|
|
grant,
|
|
&crate::crypto::DefaultVerifier::default(),
|
|
)
|
|
}
|
|
pub fn recover_interrupted_outcome_with(
|
|
ledger: &Connection,
|
|
binding: &Binding,
|
|
offer: &JobOffer,
|
|
grant: &str,
|
|
verifier: &impl crate::crypto::Ed25519Verifier,
|
|
) -> Result<Option<Outcome>> {
|
|
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
|
|
let (outcome, result) = verified_interrupted_start(ledger, binding, offer, grant, verifier)?;
|
|
let canonical = |bytes: &[u8]| -> Result<Vec<u8>> {
|
|
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
|
|
ReconciledOutcome,
|
|
>(bytes)?)?)
|
|
};
|
|
if let (Some(a), Some(b)) = (&outcome, &result) {
|
|
ensure!(
|
|
canonical(a)? == canonical(b)?,
|
|
"INTERRUPTED_OUTCOME_CONFLICT"
|
|
);
|
|
}
|
|
let Some(bytes) = outcome.or(result) else {
|
|
return Ok(None);
|
|
};
|
|
let saved: Outcome = serde_json::from_slice(&canonical(&bytes)?)?;
|
|
ensure!(
|
|
matches!(
|
|
saved.state.as_str(),
|
|
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
|
|
),
|
|
"INTERRUPTED_OUTCOME_STATE"
|
|
);
|
|
Ok(Some(saved))
|
|
}
|
|
|
|
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
|
ensure!(
|
|
db.execute(
|
|
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
|
|
params![offer.job_id, offer.attempt_id],
|
|
)? == 1,
|
|
"INTERRUPTED_REQUEST_STATE"
|
|
);
|
|
Ok(())
|
|
}
|
|
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
|
|
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
|
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
|
|
db.execute(
|
|
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
|
|
params![envelope.message_id, envelope.encode_to_vec()],
|
|
)?;
|
|
Ok(())
|
|
}
|
|
pub fn events(db: &Connection) -> Result<Vec<Envelope>> {
|
|
let mut statement = db.prepare("SELECT payload FROM job_events ORDER BY rowid LIMIT 100")?;
|
|
let bytes = statement
|
|
.query_map([], |r| r.get::<_, Vec<u8>>(0))?
|
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
|
bytes
|
|
.into_iter()
|
|
.map(|b| Ok(Envelope::decode(b.as_slice())?))
|
|
.collect()
|
|
}
|
|
pub fn ack(db: &mut Connection, id: &str) -> Result<()> {
|
|
let tx = db.transaction()?;
|
|
crate::reconcile::archive_events(&tx)?;
|
|
tx.execute("DELETE FROM job_events WHERE id=?1", [id])?;
|
|
tx.commit()?;
|
|
Ok(())
|
|
}
|
|
pub fn next(db: &Connection) -> Result<Option<JobOffer>> {
|
|
next_at(db, chrono::Utc::now())
|
|
}
|
|
/// Scheduling only; the qualified clock/Sync/permit gate still precedes effects.
|
|
pub fn next_at(db: &Connection, now: chrono::DateTime<chrono::Utc>) -> Result<Option<JobOffer>> {
|
|
// One device-wide effect at a time; unknown/reserved attempts retain locks.
|
|
let busy: i64 = db.query_row("SELECT count(*) FROM job_attempts WHERE state IN ('requesting','running','unknown','awaiting_reboot')", [], |r| r.get(0))?;
|
|
if busy != 0 {
|
|
return Ok(None);
|
|
}
|
|
let mut statement = db.prepare("SELECT offer FROM job_attempts WHERE state='accepted' AND cancel_requested=0 ORDER BY rowid")?;
|
|
let rows = statement.query_map([], |r| r.get::<_, Vec<u8>>(0))?;
|
|
for bytes in rows {
|
|
let offer = JobOffer::decode(bytes?.as_slice())?;
|
|
let job: JobPayload = serde_json::from_slice(&offer.payload_bytes)?;
|
|
if now >= job.not_before || now >= job.start_before {
|
|
return Ok(Some(offer));
|
|
}
|
|
}
|
|
Ok(None)
|
|
}
|
|
pub fn requesting(db: &Connection) -> Result<Option<JobOffer>> {
|
|
let bytes: Option<Vec<u8>> = db
|
|
.query_row(
|
|
"SELECT offer FROM job_attempts WHERE state='requesting' LIMIT 1",
|
|
[],
|
|
|r| r.get(0),
|
|
)
|
|
.optional()?;
|
|
bytes
|
|
.map(|b| Ok(JobOffer::decode(b.as_slice())?))
|
|
.transpose()
|
|
}
|
|
pub fn reserve_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
|
ensure!(db.execute("UPDATE job_attempts SET state='requesting' WHERE job=?1 AND attempt=?2 AND state='accepted' AND cancel_requested=0",
|
|
params![offer.job_id,offer.attempt_id])? == 1, "ATTEMPT_NOT_STARTABLE");
|
|
Ok(())
|
|
}
|
|
pub fn transition(db: &Connection, offer: &JobOffer, state: &str, grant: &str) -> Result<u64> {
|
|
let previous: String = db.query_row(
|
|
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![offer.job_id, offer.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
let allowed = match previous.as_str() {
|
|
"accepted" | "requesting" => matches!(
|
|
state,
|
|
"running" | "failed" | "cancelled" | "expired" | "unknown"
|
|
),
|
|
"running" | "verifying" => matches!(
|
|
state,
|
|
"verifying" | "succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
|
|
),
|
|
// Only a definitive completion of the already-running executor calls
|
|
// this reconciliation path. No new effect is started from unknown.
|
|
"unknown" | "awaiting_reboot" => matches!(
|
|
state,
|
|
"succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
|
|
),
|
|
_ => false,
|
|
};
|
|
ensure!(allowed, "INVALID_LOCAL_STATE_TRANSITION");
|
|
let rev: i64 = db.query_row("UPDATE job_attempts SET state=?3,grant_id=?4,revision=revision+1 WHERE job=?1 AND attempt=?2 RETURNING revision",
|
|
params![offer.job_id,offer.attempt_id,state,grant], |r| r.get(0))?;
|
|
Ok(u64::try_from(rev)?)
|
|
}
|
|
pub fn result(offer: &JobOffer, revision: u64, grant: &str, outcome: Outcome) -> JobResult {
|
|
JobResult {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
revision,
|
|
state: outcome.state,
|
|
grant_id: grant.into(),
|
|
stdout: outcome.stdout,
|
|
stderr: outcome.stderr,
|
|
output_truncated: outcome.output_truncated,
|
|
reboot_required: outcome.reboot_required,
|
|
exit_code: outcome.exit_code,
|
|
evidence_json: outcome.evidence.to_string(),
|
|
error_code: outcome.error_code,
|
|
}
|
|
}
|
|
pub fn no_start(state: &str, code: &str) -> Outcome {
|
|
Outcome {
|
|
state: state.into(),
|
|
evidence: serde_json::json!({"not_started":true,"postconditions_passed":false}),
|
|
..Outcome::unknown(code)
|
|
}
|
|
}
|
|
pub fn permit_request(offer: &JobOffer, permit: &ExecutionPermit) -> Result<Request> {
|
|
Ok(Request {
|
|
job_bytes: offer.payload_bytes.clone(),
|
|
permit_bytes: permit.payload_bytes.clone(),
|
|
signature: permit.signature.clone(),
|
|
plan_hash: offer.plan_hash.clone(),
|
|
authorization_id: Uuid::parse_str(&offer.authorization_id)?,
|
|
})
|
|
}
|
|
|
|
/// Read-only control scheduling after an updater intentionally restarted the
|
|
/// agent. This is NOT a handoff proof, approval, start grant or effect lease.
|
|
/// A fresh independently authenticated helper snapshot still decides whether
|
|
/// the exact original is inflight or must become Unknown; existing Unknown is
|
|
/// never returned as a candidate and no local outcome is changed here.
|
|
pub fn needs_update_control(
|
|
db: &Connection,
|
|
ledger: &Connection,
|
|
binding: &Binding,
|
|
) -> Result<bool> {
|
|
let mut candidates = 0;
|
|
for (offer, grant) in interrupted(db, ledger, binding)? {
|
|
let job = binding.job(&offer.payload_bytes)?;
|
|
if !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) || grant.is_empty() {
|
|
continue;
|
|
}
|
|
if recover_interrupted_outcome(ledger, binding, &offer, &grant)?.is_none() {
|
|
candidates += 1;
|
|
}
|
|
}
|
|
ensure!(candidates <= 1, "UPDATE_MULTIPLE_PENDING_CONTROLS");
|
|
Ok(candidates == 1)
|
|
}
|