//! Durable job persistence without capability discovery or OS execution. //! Caller performs fresh Sync/Clock/permit checks before consuming or executing. use crate::auth::{Binding, Outcome, Request}; use anyhow::{Result, ensure}; use prost::Message; use rmm_domain::jobs::{JobPayload, hash}; use rmm_protocol::*; use rusqlite::{Connection, OptionalExtension, params}; use uuid::Uuid; pub fn accept( db: &Connection, binding: &Binding, offer: &JobOffer, supported: impl FnOnce(&JobPayload) -> bool, ) -> Result { let job = binding.job(&offer.payload_bytes)?; ensure!( offer.job_id == job.job_id.to_string() && offer.attempt_id == job.attempt_id.to_string() && offer.job_payload_hash == hash(&offer.payload_bytes) && Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil()) && offer.plan_hash.len() == 64 && hex_hash(&offer.plan_hash), "OFFER_BINDING" ); ensure!(supported(&job), "UNSUPPORTED_CAPABILITY"); let bytes = offer.encode_to_vec(); let existing: Option> = db .query_row( "SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2", params![offer.job_id, offer.attempt_id], |r| r.get(0), ) .optional()?; if let Some(old) = existing { ensure!(old == bytes, "OFFER_CONFLICT"); return Ok(false); } let count: i64 = db.query_row("SELECT count(*) FROM job_attempts", [], |r| r.get(0))?; ensure!(count < 10000, "JOURNAL_CAPACITY_NEW_EFFECTS_BLOCKED"); db.execute( "INSERT INTO job_attempts(job,attempt,offer,state) VALUES(?1,?2,?3,'accepted')", params![offer.job_id, offer.attempt_id, bytes], )?; Ok(true) } fn hex_hash(s: &str) -> bool { s.bytes() .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } /// Inspect ONLY at a new process startup, never on a transport reconnect: no /// executor thread from the previous process can still deliver its outcome. /// This does not execute, release consumption or invent a successful outcome. pub fn interrupted( db: &Connection, ledger: &Connection, binding: &Binding, ) -> Result> { interrupted_with( db, ledger, binding, &crate::crypto::DefaultVerifier::default(), ) } pub fn interrupted_with( db: &Connection, ledger: &Connection, binding: &Binding, verifier: &impl crate::crypto::Ed25519Verifier, ) -> Result> { let mut statement = db.prepare( "SELECT offer,grant_id,state FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown') ORDER BY job,attempt", )?; let rows = statement.query_map([], |r| { Ok(( r.get::<_, Vec>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, )) })?; let mut interrupted = Vec::new(); for row in rows { let (bytes, mut grant, state) = row?; let offer = JobOffer::decode(bytes.as_slice())?; let job = binding.job(&offer.payload_bytes)?; validate_interrupted_offer(&offer, &job)?; // begin_execution may have committed before the running journal event. // Recover its actual grant, never invent one from an accepted offer. let (count, stored): (u64, Option) = ledger.query_row( "SELECT count(*),MIN(grant_id) FROM consumed WHERE job=?1 AND attempt=?2", params![offer.job_id, offer.attempt_id], |r| Ok((r.get(0)?, r.get(1)?)), )?; ensure!(count <= 1, "INTERRUPTED_MULTIPLE_CONSUMPTIONS"); if let Some(stored) = stored { ensure!( grant.is_empty() || grant == stored, "INTERRUPTED_GRANT_CONFLICT" ); grant = stored; } if state == "unknown" { // A reconnect may have closed the running journal while the same // original executor was still finishing. Only its already-durable // terminal completion can repair this event gap; uncertainty stays. if !grant.is_empty() && recover_interrupted_outcome_with(ledger, binding, &offer, &grant, verifier)? .is_some_and(|saved| { matches!( saved.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired" ) }) { interrupted.push((offer, grant)); } continue; } if !grant.is_empty() { verified_interrupted_start(ledger, binding, &offer, &grant, verifier)?; } else { let state: String = db.query_row( "SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2", params![offer.job_id, offer.attempt_id], |r| r.get(0), )?; ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING"); } interrupted.push((offer, grant)); } Ok(interrupted) } fn validate_interrupted_offer(offer: &JobOffer, job: &JobPayload) -> Result<()> { ensure!( offer.job_id == job.job_id.to_string() && offer.attempt_id == job.attempt_id.to_string() && offer.job_payload_hash == hash(&offer.payload_bytes) && Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil()) && offer.plan_hash.len() == 64 && hex_hash(&offer.plan_hash), "INTERRUPTED_OFFER_BINDING" ); Ok(()) } type InterruptedStartRow = ( String, u64, String, String, Vec, Option>, Option>, ); type InterruptedOutcomes = (Option>, Option>); fn verified_interrupted_start( ledger: &Connection, binding: &Binding, offer: &JobOffer, grant: &str, verifier: &impl crate::crypto::Ed25519Verifier, ) -> Result { let job = binding.job(&offer.payload_bytes)?; validate_interrupted_offer(offer, &job)?; ensure!( Uuid::parse_str(grant).is_ok_and(|id| !id.is_nil()), "INTERRUPTED_GRANT_ID" ); let row:Option=ledger.query_row( "SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[grant], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?; let (agent, generation, job_id, attempt, start, outcome, result) = row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?; ensure!( agent == binding.agent_id.to_string() && generation == binding.enrollment_generation && job_id == offer.job_id && attempt == offer.attempt_id, "INTERRUPTED_CONSUMPTION_BINDING" ); let start: serde_json::Value = serde_json::from_slice(&start)?; let boot = start["previous_boot_id"] .as_str() .ok_or_else(|| anyhow::anyhow!("INTERRUPTED_ORIGINAL_BOOT_MISSING"))?; ensure!( Uuid::parse_str(boot).is_ok_and(|id| !id.is_nil()), "INTERRUPTED_ORIGINAL_BOOT_MISSING" ); let _: chrono::DateTime = serde_json::from_value(start["reserved_at"].clone())?; ensure!( start["reauthentication_required"] == true && start["postconditions_passed"] == false, "INTERRUPTED_START_WITNESS_PROFILE" ); let request: Request = serde_json::from_value(start["request"].clone())?; let permit: rmm_domain::jobs::ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?; // Historical verification is evidence only; it can NEVER start another job. let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?; ensure!( verified == permit && permit.grant_id.to_string() == grant && request.job_bytes == offer.payload_bytes && request.plan_hash == offer.plan_hash && request.authorization_id.to_string() == offer.authorization_id, "INTERRUPTED_START_BINDING" ); Ok((outcome, result)) } /// Recover only a completion already durably recorded by the original executor. /// No outcome means uncertainty, not nonstart or a permission to execute again. /// Caller re-journalizes this evidence, never calls finish/unlocks at startup. pub fn recover_interrupted_outcome( ledger: &Connection, binding: &Binding, offer: &JobOffer, grant: &str, ) -> Result> { recover_interrupted_outcome_with( ledger, binding, offer, grant, &crate::crypto::DefaultVerifier::default(), ) } pub fn recover_interrupted_outcome_with( ledger: &Connection, binding: &Binding, offer: &JobOffer, grant: &str, verifier: &impl crate::crypto::Ed25519Verifier, ) -> Result> { use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes}; let (outcome, result) = verified_interrupted_start(ledger, binding, offer, grant, verifier)?; let canonical = |bytes: &[u8]| -> Result> { Ok(canonical_outcome_bytes(&serde_json::from_slice::< ReconciledOutcome, >(bytes)?)?) }; if let (Some(a), Some(b)) = (&outcome, &result) { ensure!( canonical(a)? == canonical(b)?, "INTERRUPTED_OUTCOME_CONFLICT" ); } let Some(bytes) = outcome.or(result) else { return Ok(None); }; let saved: Outcome = serde_json::from_slice(&canonical(&bytes)?)?; ensure!( matches!( saved.state.as_str(), "unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired" ), "INTERRUPTED_OUTCOME_STATE" ); Ok(Some(saved)) } pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> { ensure!( db.execute( "UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''", params![offer.job_id, offer.attempt_id], )? == 1, "INTERRUPTED_REQUEST_STATE" ); Ok(()) } pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> { let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?; ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT"); db.execute( "INSERT INTO job_events(id,payload) VALUES(?1,?2)", params![envelope.message_id, envelope.encode_to_vec()], )?; Ok(()) } pub fn events(db: &Connection) -> Result> { let mut statement = db.prepare("SELECT payload FROM job_events ORDER BY rowid LIMIT 100")?; let bytes = statement .query_map([], |r| r.get::<_, Vec>(0))? .collect::>>()?; bytes .into_iter() .map(|b| Ok(Envelope::decode(b.as_slice())?)) .collect() } pub fn ack(db: &mut Connection, id: &str) -> Result<()> { let tx = db.transaction()?; crate::reconcile::archive_events(&tx)?; tx.execute("DELETE FROM job_events WHERE id=?1", [id])?; tx.commit()?; Ok(()) } pub fn next(db: &Connection) -> Result> { next_at(db, chrono::Utc::now()) } /// Scheduling only; the qualified clock/Sync/permit gate still precedes effects. pub fn next_at(db: &Connection, now: chrono::DateTime) -> Result> { // One device-wide effect at a time; unknown/reserved attempts retain locks. let busy: i64 = db.query_row("SELECT count(*) FROM job_attempts WHERE state IN ('requesting','running','unknown','awaiting_reboot')", [], |r| r.get(0))?; if busy != 0 { return Ok(None); } let mut statement = db.prepare("SELECT offer FROM job_attempts WHERE state='accepted' AND cancel_requested=0 ORDER BY rowid")?; let rows = statement.query_map([], |r| r.get::<_, Vec>(0))?; for bytes in rows { let offer = JobOffer::decode(bytes?.as_slice())?; let job: JobPayload = serde_json::from_slice(&offer.payload_bytes)?; if now >= job.not_before || now >= job.start_before { return Ok(Some(offer)); } } Ok(None) } pub fn requesting(db: &Connection) -> Result> { let bytes: Option> = db .query_row( "SELECT offer FROM job_attempts WHERE state='requesting' LIMIT 1", [], |r| r.get(0), ) .optional()?; bytes .map(|b| Ok(JobOffer::decode(b.as_slice())?)) .transpose() } pub fn reserve_request(db: &Connection, offer: &JobOffer) -> Result<()> { ensure!(db.execute("UPDATE job_attempts SET state='requesting' WHERE job=?1 AND attempt=?2 AND state='accepted' AND cancel_requested=0", params![offer.job_id,offer.attempt_id])? == 1, "ATTEMPT_NOT_STARTABLE"); Ok(()) } pub fn transition(db: &Connection, offer: &JobOffer, state: &str, grant: &str) -> Result { let previous: String = db.query_row( "SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2", params![offer.job_id, offer.attempt_id], |r| r.get(0), )?; let allowed = match previous.as_str() { "accepted" | "requesting" => matches!( state, "running" | "failed" | "cancelled" | "expired" | "unknown" ), "running" | "verifying" => matches!( state, "verifying" | "succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot" ), // Only a definitive completion of the already-running executor calls // this reconciliation path. No new effect is started from unknown. "unknown" | "awaiting_reboot" => matches!( state, "succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot" ), _ => false, }; ensure!(allowed, "INVALID_LOCAL_STATE_TRANSITION"); let rev: i64 = db.query_row("UPDATE job_attempts SET state=?3,grant_id=?4,revision=revision+1 WHERE job=?1 AND attempt=?2 RETURNING revision", params![offer.job_id,offer.attempt_id,state,grant], |r| r.get(0))?; Ok(u64::try_from(rev)?) } pub fn result(offer: &JobOffer, revision: u64, grant: &str, outcome: Outcome) -> JobResult { JobResult { job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(), revision, state: outcome.state, grant_id: grant.into(), stdout: outcome.stdout, stderr: outcome.stderr, output_truncated: outcome.output_truncated, reboot_required: outcome.reboot_required, exit_code: outcome.exit_code, evidence_json: outcome.evidence.to_string(), error_code: outcome.error_code, } } pub fn no_start(state: &str, code: &str) -> Outcome { Outcome { state: state.into(), evidence: serde_json::json!({"not_started":true,"postconditions_passed":false}), ..Outcome::unknown(code) } } pub fn permit_request(offer: &JobOffer, permit: &ExecutionPermit) -> Result { Ok(Request { job_bytes: offer.payload_bytes.clone(), permit_bytes: permit.payload_bytes.clone(), signature: permit.signature.clone(), plan_hash: offer.plan_hash.clone(), authorization_id: Uuid::parse_str(&offer.authorization_id)?, }) } /// Read-only control scheduling after an updater intentionally restarted the /// agent. This is NOT a handoff proof, approval, start grant or effect lease. /// A fresh independently authenticated helper snapshot still decides whether /// the exact original is inflight or must become Unknown; existing Unknown is /// never returned as a candidate and no local outcome is changed here. pub fn needs_update_control( db: &Connection, ledger: &Connection, binding: &Binding, ) -> Result { let mut candidates = 0; for (offer, grant) in interrupted(db, ledger, binding)? { let job = binding.job(&offer.payload_bytes)?; if !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) || grant.is_empty() { continue; } if recover_interrupted_outcome(ledger, binding, &offer, &grant)?.is_none() { candidates += 1; } } ensure!(candidates <= 1, "UPDATE_MULTIPLE_PENDING_CONTROLS"); Ok(candidates == 1) }