use anyhow::{Context, Result, ensure}; use chrono::{Duration, Utc}; use prost::Message; use rmm_domain::jobs::*; use rmm_protocol::*; use serde_json::{Value, json}; use sqlx::{Postgres, Row, Transaction}; use uuid::Uuid; /// Called in the same tenant transaction as durable message receipt and ACK. pub async fn event( tx: &mut Transaction<'_, Postgres>, tenant: Uuid, agent: Uuid, e: &Envelope, ) -> Result<()> { let bytes = e.payload.as_ref().context("missing payload")?; let mut identity = e.clone(); identity.connection_generation = 0; let digest = hash(&identity.encode_to_vec()); let message = Uuid::parse_str(&e.message_id)?; if let Some(old) = sqlx::query_scalar::<_, String>("SELECT payload_hash FROM job_receipts WHERE message_id=$1") .bind(message) .fetch_optional(&mut **tx) .await? { ensure!(old == digest, "receipt identity conflict"); return Ok(()); } match bytes { Payload::JobAccepted(a) => { ensure!( !a.journal_proof.is_empty() && a.journal_proof.len() <= 256, "journal proof required" ); let row=sqlx::query("SELECT state,payload_hash FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(&a.job_id)?).bind(Uuid::parse_str(&a.attempt_id)?).bind(agent).fetch_one(&mut **tx).await?; ensure!( row.get::("payload_hash") == a.job_payload_hash, "payload mismatch" ); let state: String = row.get("state"); if state == "queued" { sqlx::query("UPDATE jobs SET state='accepted',journal_proof=$2 WHERE job_id=$1") .bind(Uuid::parse_str(&a.job_id)?) .bind(&a.journal_proof) .execute(&mut **tx) .await?; } sqlx::query( "UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1", ) .bind(Uuid::parse_str(&a.job_id)?) .execute(&mut **tx) .await?; } Payload::JobStateChanged(s) => { let row = locked_job(tx, agent, &s.job_id, &s.attempt_id).await?; let old_revision: i64 = row.get("revision"); let old: String = row.get("state"); check_grant(tx, &s.job_id, &s.attempt_id, &s.grant_id).await?; let old_state: JobState = serde_json::from_value(json!(old))?; let next: JobState = serde_json::from_value(json!(s.state))?; ensure!(!next.terminal(), "terminal states require result evidence"); if old_state.terminal() || old_state == JobState::Unknown || i64::try_from(s.revision)? < old_revision { // ACK durable but late evidence without regressing the ledger. } else if i64::try_from(s.revision)? == old_revision { ensure!(old == s.state, "state revision conflict"); } else { // Terminal results require evidence and must use JobResult. ensure!( !next.terminal() && old_state.allows(next), "invalid state transition" ); sqlx::query("UPDATE jobs SET state=$2,revision=$3 WHERE job_id=$1") .bind(Uuid::parse_str(&s.job_id)?) .bind(&s.state) .bind(i64::try_from(s.revision)?) .execute(&mut **tx) .await?; } } Payload::JobResult(r) => { ensure!( r.evidence_json.len() <= 65536 && r.error_code.len() <= 128, "evidence bound" ); let row = locked_job(tx, agent, &r.job_id, &r.attempt_id).await?; let payload: JobPayload = serde_json::from_slice(&row.get::, _>("payload"))?; ensure!( r.stdout.len() + r.stderr.len() <= payload.max_output_bytes as usize, "output bound" ); let evidence: Value = serde_json::from_str(&r.evidence_json)?; let outcome: JobState = serde_json::from_value(json!(r.state))?; ensure!( matches!( outcome, JobState::Succeeded | JobState::Failed | JobState::Cancelled | JobState::Expired | JobState::Unknown | JobState::AwaitingReboot ), "invalid result state" ); let result = json!({"stdout":r.stdout,"stderr":r.stderr,"output_truncated":r.output_truncated,"reboot_required":r.reboot_required,"exit_code":r.exit_code,"evidence":evidence,"error_code":r.error_code,"grant_id":r.grant_id}); let old_revision: i64 = row.get("revision"); if i64::try_from(r.revision)? <= old_revision { ensure!( row.get::("state") == r.state && row.get::, _>("result").as_ref() == Some(&result), "result identity conflict" ); } else { let previous: JobState = serde_json::from_value(json!(row.get::("state")))?; let nonstart = !row.get::("start_reserved") && r.grant_id.is_empty() && evidence.get("not_started") == Some(&json!(true)); ensure!( !previous.terminal() || (previous == outcome && nonstart && row.get::, _>("result").is_none()), "terminal outcome cannot be reset" ); if previous == JobState::Unknown { ensure!( matches!( outcome, JobState::Succeeded | JobState::Failed | JobState::Cancelled ), "unknown needs definitive reconciliation evidence" ); } if r.grant_id.is_empty() { ensure!( nonstart && matches!( previous, JobState::Queued | JobState::Accepted | JobState::Cancelled | JobState::Expired ) && matches!( outcome, JobState::Cancelled | JobState::Expired | JobState::Failed ), "ungranted outcome needs definite nonstart evidence" ); } else { check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?; ensure!( outcome != JobState::Expired, "grant reservation cannot prove expiry without effect" ); } if outcome == JobState::Cancelled { ensure!( evidence.get("not_started") == Some(&json!(true)) || evidence.get("abort_confirmed") == Some(&json!(true)), "cancellation needs actual nonstart/abort evidence" ); } if outcome == JobState::Failed { ensure!( !r.error_code.is_empty(), "known failure requires structured reason" ); } if outcome == JobState::Succeeded { ensure!( evidence.get("postconditions_passed") == Some(&json!(true)), "postcondition evidence required" ); if let Action::RunScript { expected_stdout, .. } = &payload.action { ensure!(r.exit_code == Some(0), "script exit failure"); if let Some(expected) = expected_stdout { ensure!(r.stdout == expected.as_bytes(), "expected output mismatch"); } } if matches!(payload.action, Action::Reboot { .. }) { crate::synchronization::verify_reboot_success( tx, tenant, agent, e, &payload, r, &evidence, ) .await?; } } sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1") .bind(Uuid::parse_str(&r.job_id)?) .bind(&r.state) .bind(i64::try_from(r.revision)?) .bind(result) .execute(&mut **tx) .await?; if previous == JobState::Unknown { sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','job_reconciled',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"outcome":r.state,"evidence_source":"authenticated_durable_job_result"})).execute(&mut **tx).await?; } } } _ => anyhow::bail!("not a durable job event"), } sqlx::query("INSERT INTO job_receipts VALUES($1,$2,$3,$4)") .bind(tenant) .bind(agent) .bind(message) .bind(digest) .execute(&mut **tx) .await?; Ok(()) } async fn locked_job( tx: &mut Transaction<'_, Postgres>, agent: Uuid, job: &str, attempt: &str, ) -> Result { Ok(sqlx::query("SELECT state,revision,result,payload,start_reserved FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(agent).fetch_one(&mut **tx).await?) } async fn check_grant( tx: &mut Transaction<'_, Postgres>, job: &str, attempt: &str, grant: &str, ) -> Result<()> { let ok:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM execution_permits WHERE job_id=$1 AND attempt_id=$2 AND grant_id=$3)").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(Uuid::parse_str(grant)?).fetch_one(&mut **tx).await?; ensure!(ok, "missing bound grant"); Ok(()) } pub async fn permit( state: &crate::State, tenant: Uuid, agent: Uuid, e: &Envelope, r: &ExecutionPermitRequest, ) -> Result { let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?; // Tenant serial lock covers rollout counters, stop check, and reservations. sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))") .bind(format!("starts:{tenant}")) .execute(&mut *tx) .await?; crate::agent_service::control_lock(&mut tx, tenant, agent).await?; let denied = |code: &str| ExecutionPermit { payload_bytes: vec![], signature: vec![], denial_code: code.into(), }; let row=sqlx::query("SELECT j.*,a.authorization_id,a.revoked_at,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,r.approval_state,i.enrollment_generation,i.assignment_version,i.asset_id AS bound_asset,i.capabilities,i.revoked_at AS device_revoked,i.connection_generation,t.changes_stopped FROM jobs j JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 FOR UPDATE OF j").bind(Uuid::parse_str(&r.job_id)?).bind(Uuid::parse_str(&r.attempt_id)?).bind(agent).fetch_optional(&mut *tx).await?; let Some(row) = row else { return Ok(denied("NOT_FOUND")); }; let payload: JobPayload = serde_json::from_slice(&row.get::, _>("payload"))?; let plan: PlanPayload = serde_json::from_slice(&row.get::, _>("plan_payload"))?; let capabilities: Value = row.get("capabilities"); if row.get::("bound_asset") != payload.asset_id || !capabilities .as_array() .is_some_and(|c| c.iter().any(|v| v.as_str() == Some(payload.action.name()))) || hash(&row.get::, _>("plan_payload")) != row.get::("plan_hash") || !plan.jobs.iter().any(|j| { j.payload.job_id == payload.job_id && j.payload.attempt_id == payload.attempt_id && j.job_payload_hash == r.job_payload_hash }) { return Ok(denied("BINDING_MISMATCH")); } if row.get::("start_reserved") { return Ok(denied("START_ALREADY_RESERVED")); } if row.get::("state") != "accepted" || row.get::, _>("journal_proof").is_none() { return Ok(denied("JOURNAL_ACCEPTANCE_REQUIRED")); } if row.get::("changes_stopped") || row.get::("cancel_requested") || row.get::("approval_state") != "approved" || row .get::>, _>("revoked_at") .is_some() || row .get::>, _>("device_revoked") .is_some() || !crate::jobs::requester_enabled(state, &plan.principal, tenant) || !crate::jobs::requester_enabled(state, &row.get::("approver"), tenant) { return Ok(denied("AUTHORIZATION_REVOKED")); } if row.get::("enrollment_generation") as u64 != payload.enrollment_generation || row.get::("assignment_version") as u64 != payload.assignment_version || r.assignment_version != payload.assignment_version || row.get::("connection_generation") as u64 != e.connection_generation || r.job_payload_hash != row.get::("payload_hash") || hash(&encode(&payload)?) != r.job_payload_hash { return Ok(denied("BINDING_MISMATCH")); } let now = Utc::now(); if !payload.action.validate() { return Ok(denied("APT_MANIFEST_REQUIRED")); } if payload.execution_identity == "root" { let helper_ready: bool = sqlx::query_scalar( "SELECT helper_sync_ready FROM agent_installations WHERE agent_id=$1", ) .bind(agent) .fetch_one(&mut *tx) .await?; if !helper_ready || crate::root_policy::authorize(&mut tx, &plan.principal, &payload) .await .is_err() || crate::root_policy::authorize(&mut tx, &row.get::("approver"), &payload) .await .is_err() { return Ok(denied("ROOT_SCOPE_OR_HELPER_SYNC_REQUIRED")); } } let sync: (bool, bool) = sqlx::query_as( "SELECT sync_required,sync_ready FROM agent_installations WHERE agent_id=$1", ) .bind(agent) .fetch_one(&mut *tx) .await?; if sync.0 && !sync.1 { return Ok(denied("SYNC_REQUIRED")); } let restore_blocked: bool = sqlx::query_scalar("SELECT restore_blocked FROM agent_installations WHERE agent_id=$1") .bind(agent) .fetch_one(&mut *tx) .await?; if restore_blocked { return Ok(denied("RESTORE_RECONCILIATION_REQUIRED")); } if crate::catalog::check(&mut tx, &payload.action) .await .is_err() { return Ok(denied("RECIPE_BINDING_MISMATCH")); } let references = crate::catalog::references(&mut tx, &payload.action) .await .map_err(|_| anyhow::anyhow!("catalog unavailable"))?; if references .iter() .any(|r| !plan.recipe_references.contains(r)) { return Ok(denied("RECIPE_BINDING_MISMATCH")); } let artifact_ids = plan .artifact_references .iter() .filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id) .map(|r| r.artifact.artifact_id) .collect::>(); let artifacts = crate::catalog::resolve_artifacts(&mut tx, tenant, &artifact_ids) .await .map_err(|_| anyhow::anyhow!("artifact unavailable"))?; if plan .artifact_references .iter() .filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id) .any(|r| !artifacts.contains(&r.artifact)) { return Ok(denied("ARTIFACT_BINDING_MISMATCH")); } let approver: String = row.get("approver"); if matches!(payload.action, Action::QueryLogs { .. }) && !crate::permissions::allowed(&mut tx, &plan.principal, "logs:read").await? { return Ok(denied("AUTHORIZATION_REVOKED")); } if !crate::permissions::allowed(&mut tx, &approver, "plans:approve").await? || !crate::permissions::allowed(&mut tx, &plan.principal, "operations:submit").await? || (matches!(payload.action, Action::RunScript { .. }) && !crate::permissions::allowed(&mut tx, &plan.principal, "scripts:execute").await?) { return Ok(denied("AUTHORIZATION_REVOKED")); } let limits: (i32, i32) = sqlx::query_as("SELECT starts_per_hour,max_parallel FROM tenant_limits") .fetch_one(&mut *tx) .await?; let hourly: i64 = sqlx::query_scalar( "SELECT count(*) FROM execution_permits WHERE created_at>now()-interval '1 hour'", ) .fetch_one(&mut *tx) .await?; let concurrent:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired')").fetch_one(&mut *tx).await?; if hourly >= i64::from(limits.0) || concurrent >= i64::from(limits.1) { return Ok(denied("QUOTA_EXCEEDED")); } if let Action::Reboot { dependency_group, .. } = &payload.action { let busy:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired') AND (convert_from(payload,'UTF8')::jsonb->'action'->>'dependency_group'=$1 OR convert_from(payload,'UTF8')::jsonb->'action'->>'action'='reboot'))").bind(dependency_group).fetch_one(&mut *tx).await?; if busy { return Ok(denied("RESOURCE_LOCKED")); } } if now >= payload.start_before { return Ok(denied("PLAN_EXPIRED")); } if now < payload.not_before { return Ok(denied("MAINTENANCE_WINDOW")); } let failures: i64 = sqlx::query_scalar( "SELECT count(*) FROM jobs WHERE operation_id=$1 AND state IN ('failed','unknown')", ) .bind(row.get::("operation_id")) .fetch_one(&mut *tx) .await?; let active:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE operation_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(row.get::("operation_id")).fetch_one(&mut *tx).await?; if failures >= i64::from(plan.halt_after_failures) || active >= i64::from(plan.max_parallel) { return Ok(denied("ROLLOUT_PAUSED")); } let locked:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired'))").bind(agent).fetch_one(&mut *tx).await?; if locked { return Ok(denied("RESOURCE_LOCKED")); } let grant = ExecutionPermitPayload { domain: "rmm.execution-permit".into(), schema_version: 1, grant_id: Uuid::new_v4(), authorization_id: row.get("authorization_id"), tenant_id: tenant, asset_id: payload.asset_id, agent_id: agent, enrollment_generation: payload.enrollment_generation, assignment_version: payload.assignment_version, job_id: payload.job_id, attempt_id: payload.attempt_id, plan_hash: row.get("plan_hash"), job_payload_hash: r.job_payload_hash.clone(), action: payload.action.name().into(), execution_identity: payload.execution_identity, not_before: now, start_before: std::cmp::min(now + Duration::seconds(60), payload.start_before), max_runtime_seconds: payload.max_runtime_seconds, key_id: state.job_signer.key_id.clone(), }; let bytes = encode(&grant)?; let signature = state.job_signer.sign(&bytes)?; sqlx::query("INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(grant.grant_id).bind(grant.job_id).bind(grant.attempt_id).bind(&bytes).bind(&signature).execute(&mut *tx).await?; sqlx::query("UPDATE jobs SET start_reserved=true WHERE job_id=$1") .bind(grant.job_id) .execute(&mut *tx) .await?; sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','start_reserved',$3,now())") .bind(tenant) .bind(Uuid::new_v4()) .bind(json!({"job_id":r.job_id,"grant_id":grant.grant_id,"boot_id":e.boot_id})) .execute(&mut *tx) .await?; tx.commit().await?; Ok(ExecutionPermit { payload_bytes: bytes, signature, denial_code: String::new(), }) } pub async fn next( state: &crate::State, tenant: Uuid, agent: Uuid, generation: u64, ) -> Result> { let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?; // Never invent nonstart once a grant was reserved. The durable ledger is // authoritative even when every start packet was lost. crate::scheduler::maintain(&mut tx, Some(agent)).await?; let active:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE agent_id=$1 AND connection_generation=$2 AND revoked_at IS NULL AND NOT restore_blocked AND (NOT sync_required OR sync_ready))").bind(agent).bind(i64::try_from(generation)?).fetch_one(&mut *tx).await?; if !active { return Ok(None); } if let Some(r)=sqlx::query("SELECT job_id,attempt_id FROM jobs WHERE agent_id=$1 AND cancel_requested AND state IN ('accepted','running','verifying','awaiting_reboot') ORDER BY job_id LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?{ tx.commit().await?;return Ok(Some(Payload::CancelRequest(CancelRequest{job_id:r.get::("job_id").to_string(),attempt_id:r.get::("attempt_id").to_string(),reason:"operator_cancel_requested".into()}))) } let row=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload,j.payload_hash,a.authorization_id,p.plan_hash FROM outbox b JOIN jobs j USING(tenant_id,job_id,attempt_id) JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.agent_id=$1 AND j.state='queued' AND NOT j.cancel_requested AND NOT t.changes_stopped AND a.revoked_at IS NULL AND r.approval_state='approved' AND p.expires_at>now() AND b.acknowledged_at IS NULL AND (b.lease_until IS NULL OR b.lease_until("job_id")) .execute(&mut *tx) .await?; let offer = JobOffer { job_id: row.get::("job_id").to_string(), attempt_id: row.get::("attempt_id").to_string(), authorization_id: row.get::("authorization_id").to_string(), plan_hash: row.get("plan_hash"), job_payload_hash: row.get("payload_hash"), payload_bytes: row.get("payload"), }; tx.commit().await?; Ok(Some(Payload::JobOffer(offer))) }