use anyhow::{Context, Result, ensure}; use chrono::{Duration, Utc}; use prost::Message; use rmm_domain::jobs::*; use rmm_protocol::*; use serde_json::{Value, json}; use sqlx::{Postgres, Row, Transaction}; use uuid::Uuid; /// Called in the same tenant transaction as durable message receipt and ACK. pub async fn event( tx: &mut Transaction<'_, Postgres>, tenant: Uuid, agent: Uuid, e: &Envelope, ) -> Result<()> { let bytes = e.payload.as_ref().context("missing payload")?; let mut identity = e.clone(); identity.connection_generation = 0; let digest = hash(&identity.encode_to_vec()); let message = Uuid::parse_str(&e.message_id)?; if let Some(old) = sqlx::query_scalar::<_, String>("SELECT payload_hash FROM job_receipts WHERE message_id=$1") .bind(message) .fetch_optional(&mut **tx) .await? { ensure!(old == digest, "receipt identity conflict"); return Ok(()); } match bytes { Payload::JobAccepted(a) => { ensure!( !a.journal_proof.is_empty() && a.journal_proof.len() <= 256, "journal proof required" ); let row=sqlx::query("SELECT state,payload_hash FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(&a.job_id)?).bind(Uuid::parse_str(&a.attempt_id)?).bind(agent).fetch_one(&mut **tx).await?; ensure!( row.get::("payload_hash") == a.job_payload_hash, "payload mismatch" ); let state: String = row.get("state"); if state == "queued" { sqlx::query("UPDATE jobs SET state='accepted',journal_proof=$2 WHERE job_id=$1") .bind(Uuid::parse_str(&a.job_id)?) .bind(&a.journal_proof) .execute(&mut **tx) .await?; } sqlx::query( "UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1", ) .bind(Uuid::parse_str(&a.job_id)?) .execute(&mut **tx) .await?; } Payload::JobStateChanged(s) => { let row = locked_job(tx, agent, &s.job_id, &s.attempt_id).await?; let old_revision: i64 = row.get("revision"); let old: String = row.get("state"); check_grant(tx, &s.job_id, &s.attempt_id, &s.grant_id).await?; let old_state: JobState = serde_json::from_value(json!(old))?; let next: JobState = serde_json::from_value(json!(s.state))?; ensure!(!next.terminal(), "terminal states require result evidence"); if old_state.terminal() || old_state == JobState::Unknown || i64::try_from(s.revision)? < old_revision { // ACK durable but late evidence without regressing the ledger. } else if i64::try_from(s.revision)? == old_revision { ensure!(old == s.state, "state revision conflict"); } else { // Terminal results require evidence and must use JobResult. ensure!( !next.terminal() && old_state.allows(next), "invalid state transition" ); sqlx::query("UPDATE jobs SET state=$2,revision=$3 WHERE job_id=$1") .bind(Uuid::parse_str(&s.job_id)?) .bind(&s.state) .bind(i64::try_from(s.revision)?) .execute(&mut **tx) .await?; } } Payload::JobResult(r) => { ensure!( r.evidence_json.len() <= 65536 && r.error_code.len() <= 128, "evidence bound" ); let row = locked_job(tx, agent, &r.job_id, &r.attempt_id).await?; let payload: JobPayload = serde_json::from_slice(&row.get::, _>("payload"))?; ensure!( r.stdout.len() + r.stderr.len() <= payload.max_output_bytes as usize, "output bound" ); let evidence: Value = serde_json::from_str(&r.evidence_json)?; let outcome: JobState = serde_json::from_value(json!(r.state))?; ensure!( matches!( outcome, JobState::Succeeded | JobState::Failed | JobState::Cancelled | JobState::Expired | JobState::Unknown | JobState::AwaitingReboot ), "invalid result state" ); let result = json!({"stdout":r.stdout,"stderr":r.stderr,"output_truncated":r.output_truncated,"reboot_required":r.reboot_required,"exit_code":r.exit_code,"evidence":evidence,"error_code":r.error_code,"grant_id":r.grant_id}); let old_revision: i64 = row.get("revision"); if i64::try_from(r.revision)? < old_revision && payload.execution_identity == "system" && row.get::("start_reserved") && superseded_delegation_placeholder(r, &evidence) { // Sync may import the helper's authoritative record before the // agent's earlier content-free delegation placeholder arrives. // ACK it durably without regressing the ledger; any other stale // evidence must still match exactly. check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?; sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','superseded_delegation_placeholder_acknowledged',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"placeholder_revision":r.revision,"current_revision":old_revision,"current_state":row.get::("state"),"message_id":e.message_id})).execute(&mut **tx).await?; } else if i64::try_from(r.revision)? <= old_revision { ensure!( row.get::("state") == r.state && row.get::, _>("result").as_ref() == Some(&result), "result identity conflict" ); } else { let previous: JobState = serde_json::from_value(json!(row.get::("state")))?; let nonstart = !row.get::("start_reserved") && r.grant_id.is_empty() && evidence.get("not_started") == Some(&json!(true)); ensure!( !previous.terminal() || (previous == outcome && nonstart && row.get::, _>("result").is_none()), "terminal outcome cannot be reset" ); if previous == JobState::Unknown { ensure!( matches!( outcome, JobState::Succeeded | JobState::Failed | JobState::Cancelled | JobState::Unknown ), "unknown needs bound unknown or definitive reconciliation evidence" ); } if r.grant_id.is_empty() { ensure!( nonstart && matches!( previous, JobState::Queued | JobState::Accepted | JobState::Cancelled | JobState::Expired ) && matches!( outcome, JobState::Cancelled | JobState::Expired | JobState::Failed ), "ungranted outcome needs definite nonstart evidence" ); } else { check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?; ensure!( outcome != JobState::Expired, "grant reservation cannot prove expiry without effect" ); } if outcome == JobState::Cancelled { ensure!( evidence.get("not_started") == Some(&json!(true)) || evidence.get("abort_confirmed") == Some(&json!(true)), "cancellation needs actual nonstart/abort evidence" ); } if outcome == JobState::Failed { ensure!( !r.error_code.is_empty(), "known failure requires structured reason" ); } if r.error_code == rmm_domain::windows_system_nonstart::ERROR_CODE { crate::windows_system_nonstart::validate_result(tx, tenant, agent, r, &evidence).await?; } if outcome == JobState::Succeeded { ensure!( evidence.get("postconditions_passed") == Some(&json!(true)), "postcondition evidence required" ); if let Action::RunScript { expected_stdout, .. } = &payload.action { ensure!(r.exit_code == Some(0), "script exit failure"); if let Some(expected) = expected_stdout { ensure!(r.stdout == expected.as_bytes(), "expected output mismatch"); } } script_profile_evidence(&payload.action, &evidence)?; windows_event_log_success_evidence(&payload, r, &evidence)?; if matches!(payload.action, Action::Reboot { .. }) { crate::synchronization::verify_reboot_success( tx, tenant, agent, e, &payload, r, &evidence, ) .await?; } } sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1") .bind(Uuid::parse_str(&r.job_id)?) .bind(&r.state) .bind(i64::try_from(r.revision)?) .bind(result) .execute(&mut **tx) .await?; // A newly persisted unknown result may follow a Sync decision // that already marked the attempt unknown. Receipt/ACK records // that evidence; it does not reconcile or release the attempt. if previous == JobState::Unknown && outcome != JobState::Unknown { sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','job_reconciled',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"outcome":r.state,"evidence_source":"authenticated_durable_job_result"})).execute(&mut **tx).await?; } } } _ => anyhow::bail!("not a durable job event"), } sqlx::query("INSERT INTO job_receipts VALUES($1,$2,$3,$4)") .bind(tenant) .bind(agent) .bind(message) .bind(digest) .execute(&mut **tx) .await?; Ok(()) } /// Exact agent-side SYSTEM delegation placeholder: unknown, no output, no /// exit code and no evidence beyond the failed postcondition marker. fn superseded_delegation_placeholder(r: &JobResult, evidence: &Value) -> bool { r.state == "unknown" && r.error_code == "SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT" && !r.grant_id.is_empty() && r.stdout.is_empty() && r.stderr.is_empty() && r.exit_code.is_none() && !r.output_truncated && !r.reboot_required && *evidence == json!({"postconditions_passed": false}) } fn windows_event_log_success_evidence( job: &JobPayload, result: &JobResult, evidence: &Value, ) -> Result<()> { if !job.action.is_windows_event_log_fixture() { return Ok(()); } let Action::QueryLogs { unit, since, until, max_lines, } = &job.action else { anyhow::bail!("fixed event log action required") }; let observed_since: chrono::DateTime = serde_json::from_value( evidence .get("start_utc") .cloned() .context("event log start missing")?, )?; let observed_until: chrono::DateTime = serde_json::from_value( evidence .get("end_utc") .cloned() .context("event log end missing")?, )?; let complete = evidence.get("completeness") == Some(&json!("complete")) && evidence.get("truncated") == Some(&json!(false)) && evidence.get("error_code") == Some(&Value::Null); let partial = evidence.get("completeness") == Some(&json!("partial")) && evidence.get("truncated") == Some(&json!(true)) && matches!( evidence.get("error_code").and_then(Value::as_str), Some("EVENT_LOG_EVENT_LIMIT" | "EVENT_LOG_BYTE_LIMIT" | "EVENT_LOG_EVENT_SIZE_LIMIT") ); ensure!( !result.reboot_required && evidence.get("truncated") == Some(&json!(result.output_truncated)) && job.execution_identity == "agent_service" && job.action.validate() && job .action .valid_job_limits(job.max_runtime_seconds, job.max_output_bytes) && evidence.get("source_id") == Some(&json!(unit)) && evidence.get("channel") == Some(&json!(WINDOWS_EVENT_LOG_FIXTURE_CHANNEL)) && evidence.get("provider") == Some(&json!(WINDOWS_EVENT_LOG_FIXTURE_PROVIDER)) && observed_since == *since && observed_until == *until && evidence.get("max_events") == Some(&json!(max_lines)) && evidence.get("max_output_bytes") == Some(&json!(job.max_output_bytes)) && evidence.get("quality") == Some(&json!("valid")) && evidence.get("win32_error") == Some(&Value::Null) && evidence.get("bookmark_updated") == Some(&json!(false)) && evidence.get("process_tree_bounded") == Some(&json!(true)) && evidence.get("execution_sid") == Some(&json!("S-1-5-19")) && (complete || partial), "fixed Windows Event Log evidence mismatch" ); Ok(()) } // The profile records the requested process option, never an effective GPO policy. fn script_profile_evidence(action: &Action, evidence: &Value) -> Result<()> { if action.full_maintenance() { let Action::RunScript { script_utf8, .. } = action else { unreachable!() }; ensure!( evidence["privileged_script_profile"] == "full_maintenance_v1" && evidence["script_sha256"] == hash(script_utf8.as_bytes()) && match action.script_platform() { Some("linux") => evidence["execution_uid"] == 0, Some("windows") => evidence["execution_sid"] == "S-1-5-18", _ => false, }, "full maintenance identity/profile evidence mismatch" ); } if let Action::RunScript { script_utf8, windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned), .. } = action { ensure!( evidence.get("windows_execution_profile") == Some(&json!("powershell_51_remote_signed")) && evidence.get("execution_policy") == Some(&json!("process_remote_signed_requested")) && evidence.get("execution_policy_requested") == Some(&json!("RemoteSigned")) && evidence.get("execution_policy_scope") == Some(&json!("process")) && evidence.get("script_sha256") == Some(&json!(hash(script_utf8.as_bytes()))), "requested Windows process profile evidence mismatch" ); } Ok(()) } #[cfg(test)] mod script_profile_tests { use super::*; #[test] fn windows_event_log_success_binds_request_and_keeps_limit_partial_honest() { let id = Uuid::new_v4(); let now = Utc::now(); let job = JobPayload { schema_version: 1, tenant_id: id, asset_id: id, agent_id: id, enrollment_generation: 1, assignment_version: 1, job_id: id, attempt_id: id, action: Action::QueryLogs { unit: WINDOWS_EVENT_LOG_FIXTURE_SOURCE.into(), since: now - Duration::hours(1), until: now, max_lines: 1000, }, execution_identity: "agent_service".into(), not_before: now, start_before: now + Duration::minutes(1), max_runtime_seconds: 30, max_output_bytes: 4096, }; let evidence = json!({"source_id":WINDOWS_EVENT_LOG_FIXTURE_SOURCE,"channel":"Application","provider":"RustDiscovery-MVP-Test","start_utc":now-Duration::hours(1),"end_utc":now,"max_events":1000,"max_output_bytes":4096,"quality":"valid","completeness":"complete","truncated":false,"error_code":null,"win32_error":null,"bookmark_updated":false,"process_tree_bounded":true,"execution_sid":"S-1-5-19"}); let result = JobResult::default(); windows_event_log_success_evidence(&job, &result, &evidence).unwrap(); for conflicting in [ JobResult { reboot_required: true, ..Default::default() }, JobResult { output_truncated: true, ..Default::default() }, ] { assert!(windows_event_log_success_evidence(&job, &conflicting, &evidence).is_err()); } for field in [ "source_id", "channel", "provider", "start_utc", "end_utc", "max_events", "max_output_bytes", "quality", "win32_error", "bookmark_updated", "process_tree_bounded", "execution_sid", ] { let mut bad = evidence.clone(); bad[field] = json!("different"); assert!( windows_event_log_success_evidence(&job, &result, &bad).is_err(), "{field}" ); } for code in [ "EVENT_LOG_EVENT_LIMIT", "EVENT_LOG_BYTE_LIMIT", "EVENT_LOG_EVENT_SIZE_LIMIT", ] { let mut partial = evidence.clone(); partial["completeness"] = json!("partial"); partial["truncated"] = json!(true); partial["error_code"] = json!(code); let partial_result = JobResult { output_truncated: true, ..Default::default() }; windows_event_log_success_evidence(&job, &partial_result, &partial).unwrap(); assert!(windows_event_log_success_evidence(&job, &result, &partial).is_err()); partial["error_code"] = json!("EVENT_LOG_TIMEOUT"); assert!(windows_event_log_success_evidence(&job, &partial_result, &partial).is_err()); partial["quality"] = json!("permission_denied"); assert!(windows_event_log_success_evidence(&job, &partial_result, &partial).is_err()); } let mut permission_denied = evidence.clone(); permission_denied["win32_error"] = json!(5); assert!(windows_event_log_success_evidence(&job, &result, &permission_denied).is_err()); let mut false_complete = evidence.clone(); false_complete["truncated"] = json!(true); assert!(windows_event_log_success_evidence(&job, &result, &false_complete).is_err()); } #[test] fn requested_profile_success_evidence_is_exact_without_effective_policy_claim() { let mut action = Action::RunScript { interpreter: "powershell".into(), script_utf8: "Write-Output 'hi'\r\n".into(), arguments: vec![], expected_stdout: None, windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned), privileged_script_profile: None, }; let evidence = json!({"windows_execution_profile":"powershell_51_remote_signed", "execution_policy":"process_remote_signed_requested", "execution_policy_requested":"RemoteSigned", "execution_policy_scope":"process", "script_sha256":hash(b"Write-Output 'hi'\r\n")}); script_profile_evidence(&action, &evidence).unwrap(); for field in [ "windows_execution_profile", "execution_policy", "execution_policy_requested", "execution_policy_scope", "script_sha256", ] { let mut changed = evidence.clone(); changed.as_object_mut().unwrap().remove(field); assert!( script_profile_evidence(&action, &changed).is_err(), "{field}" ); changed[field] = json!("different"); assert!( script_profile_evidence(&action, &changed).is_err(), "{field}" ); } if let Action::RunScript { windows_execution_profile, .. } = &mut action { *windows_execution_profile = None; } script_profile_evidence(&action, &json!({})).unwrap(); } } async fn locked_job( tx: &mut Transaction<'_, Postgres>, agent: Uuid, job: &str, attempt: &str, ) -> Result { Ok(sqlx::query("SELECT state,revision,result,payload,start_reserved FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(agent).fetch_one(&mut **tx).await?) } async fn check_grant( tx: &mut Transaction<'_, Postgres>, job: &str, attempt: &str, grant: &str, ) -> Result<()> { let ok:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM execution_permits WHERE job_id=$1 AND attempt_id=$2 AND grant_id=$3)").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(Uuid::parse_str(grant)?).fetch_one(&mut **tx).await?; ensure!(ok, "missing bound grant"); Ok(()) } pub async fn permit( state: &crate::State, tenant: Uuid, agent: Uuid, e: &Envelope, r: &ExecutionPermitRequest, ) -> Result { let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?; // Tenant serial lock covers rollout counters, stop check, and reservations. sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))") .bind(format!("starts:{tenant}")) .execute(&mut *tx) .await?; crate::agent_service::control_lock(&mut tx, tenant, agent).await?; let denied = |code: &str| ExecutionPermit { payload_bytes: vec![], signature: vec![], denial_code: code.into(), unreserved_request: None, }; let row=sqlx::query("SELECT j.*,a.authorization_id,a.revoked_at,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,r.approval_state,i.enrollment_generation,i.assignment_version,i.asset_id AS bound_asset,i.capabilities,i.revoked_at AS device_revoked,i.connection_generation,t.changes_stopped FROM jobs j JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 FOR UPDATE OF j").bind(Uuid::parse_str(&r.job_id)?).bind(Uuid::parse_str(&r.attempt_id)?).bind(agent).fetch_optional(&mut *tx).await?; let Some(row) = row else { return Ok(denied("NOT_FOUND")); }; let payload: JobPayload = serde_json::from_slice(&row.get::, _>("payload"))?; let plan: PlanPayload = serde_json::from_slice(&row.get::, _>("plan_payload"))?; let capabilities: Vec = serde_json::from_value(row.get("capabilities"))?; if crate::windows_msi_catalog::validate_job_binding(&plan, &payload).is_err() || (matches!(payload.action, Action::UpdateAgent { .. }) && crate::agent_updates::validate_job_binding(&plan, &payload).is_err()) { return Ok(denied("WINDOWS_MSI_BINDING_MISMATCH")); } if row.get::("bound_asset") != payload.asset_id || !payload .action .available_for(&payload.execution_identity, &capabilities) || hash(&row.get::, _>("plan_payload")) != row.get::("plan_hash") || !plan.jobs.iter().any(|j| { j.payload.job_id == payload.job_id && j.payload.attempt_id == payload.attempt_id && j.job_payload_hash == r.job_payload_hash }) { return Ok(denied("BINDING_MISMATCH")); } if row.get::("start_reserved") { return Ok(denied("START_ALREADY_RESERVED")); } if row.get::("state") != "accepted" || row.get::, _>("journal_proof").is_none() { return Ok(denied("JOURNAL_ACCEPTANCE_REQUIRED")); } if row.get::("changes_stopped") || row.get::("cancel_requested") || row.get::("approval_state") != "approved" || row .get::>, _>("revoked_at") .is_some() || row .get::>, _>("device_revoked") .is_some() || !crate::jobs::requester_enabled(state, &mut tx, &plan.principal, tenant).await? || !crate::jobs::requester_enabled(state, &mut tx, &row.get::("approver"), tenant).await? { return Ok(denied("AUTHORIZATION_REVOKED")); } if row.get::("enrollment_generation") as u64 != payload.enrollment_generation || row.get::("assignment_version") as u64 != payload.assignment_version || r.assignment_version != payload.assignment_version || row.get::("connection_generation") as u64 != e.connection_generation || r.job_payload_hash != row.get::("payload_hash") || hash(&encode(&payload)?) != r.job_payload_hash { return Ok(denied("BINDING_MISMATCH")); } // The locked row belongs to this request and has no start reservation. // Echo the binding only after both checks, so a client can safely return // this particular offer to scheduling after a fresh signed reconciliation. let denied = |code: &str| ExecutionPermit { unreserved_request: Some(r.clone()), ..denied(code) }; let now = Utc::now(); if !payload .action .valid_job_limits(payload.max_runtime_seconds, payload.max_output_bytes) { return Ok(denied("INVALID_JOB_LIMITS")); } if !payload.action.validate() { return Ok(denied("APT_MANIFEST_REQUIRED")); } if crate::root_policy::needs_helper(&payload.execution_identity) { let helper_ready: bool = sqlx::query_scalar( "SELECT i.helper_sync_ready AND EXISTS(SELECT 1 FROM agent_sync_state s WHERE s.tenant_id=i.tenant_id AND s.agent_id=i.agent_id AND s.helper_journal_id IS NOT NULL) FROM agent_installations i WHERE i.agent_id=$1", ) .bind(agent) .fetch_one(&mut *tx) .await?; if !helper_ready || crate::root_policy::authorize(&mut tx, &plan.principal, &payload) .await .is_err() || crate::root_policy::authorize(&mut tx, &row.get::("approver"), &payload) .await .is_err() { return Ok(denied(if payload.execution_identity == "system" { "SYSTEM_SCOPE_OR_HELPER_SYNC_REQUIRED" } else { "ROOT_SCOPE_OR_HELPER_SYNC_REQUIRED" })); } } let sync: (bool, bool) = sqlx::query_as( "SELECT sync_required,sync_ready FROM agent_installations WHERE agent_id=$1", ) .bind(agent) .fetch_one(&mut *tx) .await?; if sync.0 && !sync.1 { return Ok(denied("SYNC_REQUIRED")); } let restore_blocked: bool = sqlx::query_scalar("SELECT restore_blocked FROM agent_installations WHERE agent_id=$1") .bind(agent) .fetch_one(&mut *tx) .await?; if restore_blocked { return Ok(denied("RESTORE_RECONCILIATION_REQUIRED")); } if crate::catalog::check(&mut tx, &payload.action) .await .is_err() { return Ok(denied("RECIPE_BINDING_MISMATCH")); } let references = crate::catalog::references(&mut tx, &payload.action) .await .map_err(|_| anyhow::anyhow!("catalog unavailable"))?; if references .iter() .any(|r| !plan.recipe_references.contains(r)) { return Ok(denied("RECIPE_BINDING_MISMATCH")); } let artifact_ids = plan .artifact_references .iter() .filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id) .map(|r| r.artifact.artifact_id) .collect::>(); let artifacts = if let Action::UpdateAgent { update } = &payload.action { crate::agent_updates::resolve_artifacts( &mut tx, tenant, &update .manifest() .map_err(|_| anyhow::anyhow!("invalid agent release"))? .artifacts, ) .await } else { crate::catalog::resolve_artifacts(&mut tx, tenant, &artifact_ids).await } .map_err(|_| anyhow::anyhow!("artifact unavailable"))?; if plan .artifact_references .iter() .filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id) .any(|r| !artifacts.contains(&r.artifact)) { return Ok(denied("ARTIFACT_BINDING_MISMATCH")); } let approver: String = row.get("approver"); if matches!(payload.action, Action::QueryLogs { .. }) && !crate::permissions::allowed(&mut tx, &plan.principal, "logs:read").await? { return Ok(denied("AUTHORIZATION_REVOKED")); } if !crate::permissions::allowed(&mut tx, &approver, "plans:approve").await? || !crate::permissions::allowed(&mut tx, &plan.principal, "operations:submit").await? || (matches!(payload.action, Action::RunScript { .. }) && (!crate::permissions::allowed(&mut tx, &plan.principal, "scripts:execute").await? || !crate::permissions::allowed(&mut tx, &approver, "scripts:execute").await?)) || (matches!(payload.action, Action::UpdateAgent { .. }) && (!crate::permissions::allowed(&mut tx, &plan.principal, "updates:apply").await? || !crate::permissions::allowed(&mut tx, &approver, "updates:apply").await?)) { return Ok(denied("AUTHORIZATION_REVOKED")); } let limits: (i32, i32) = sqlx::query_as("SELECT starts_per_hour,max_parallel FROM tenant_limits") .fetch_one(&mut *tx) .await?; let hourly: i64 = sqlx::query_scalar( "SELECT count(*) FROM execution_permits WHERE created_at>now()-interval '1 hour'", ) .fetch_one(&mut *tx) .await?; let concurrent:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").fetch_one(&mut *tx).await?; if hourly >= i64::from(limits.0) || concurrent >= i64::from(limits.1) { return Ok(denied("QUOTA_EXCEEDED")); } if let Action::Reboot { dependency_group, .. } = &payload.action { let busy:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown') AND (convert_from(payload,'UTF8')::jsonb->'action'->>'dependency_group'=$1 OR convert_from(payload,'UTF8')::jsonb->'action'->>'action'='reboot'))").bind(dependency_group).fetch_one(&mut *tx).await?; if busy { return Ok(denied("RESOURCE_LOCKED")); } } if now >= payload.start_before { return Ok(denied("PLAN_EXPIRED")); } if now < payload.not_before { return Ok(denied("MAINTENANCE_WINDOW")); } let failures: i64 = sqlx::query_scalar( "SELECT count(*) FROM jobs WHERE operation_id=$1 AND state IN ('failed','unknown')", ) .bind(row.get::("operation_id")) .fetch_one(&mut *tx) .await?; let active:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE operation_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").bind(row.get::("operation_id")).fetch_one(&mut *tx).await?; if failures >= i64::from(plan.halt_after_failures) || active >= i64::from(plan.max_parallel) { return Ok(denied("ROLLOUT_PAUSED")); } let locked:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))").bind(agent).fetch_one(&mut *tx).await?; if locked { return Ok(denied("RESOURCE_LOCKED")); } let grant = ExecutionPermitPayload { agent_update_plan_id: matches!(payload.action, Action::UpdateAgent { .. }) .then_some(plan.plan_id), domain: "rmm.execution-permit".into(), schema_version: 1, grant_id: Uuid::new_v4(), authorization_id: row.get("authorization_id"), tenant_id: tenant, asset_id: payload.asset_id, agent_id: agent, enrollment_generation: payload.enrollment_generation, assignment_version: payload.assignment_version, job_id: payload.job_id, attempt_id: payload.attempt_id, plan_hash: row.get("plan_hash"), job_payload_hash: r.job_payload_hash.clone(), action: payload.action.name().into(), execution_identity: payload.execution_identity, not_before: now, start_before: std::cmp::min(now + Duration::seconds(60), payload.start_before), max_runtime_seconds: payload.max_runtime_seconds, key_id: state.job_signer.key_id.clone(), }; let bytes = encode(&grant)?; let signature = state.job_signer.sign(&bytes)?; sqlx::query("INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(grant.grant_id).bind(grant.job_id).bind(grant.attempt_id).bind(&bytes).bind(&signature).execute(&mut *tx).await?; sqlx::query("UPDATE jobs SET start_reserved=true WHERE job_id=$1") .bind(grant.job_id) .execute(&mut *tx) .await?; sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','start_reserved',$3,now())") .bind(tenant) .bind(Uuid::new_v4()) .bind(json!({"job_id":r.job_id,"grant_id":grant.grant_id,"boot_id":e.boot_id})) .execute(&mut *tx) .await?; tx.commit().await?; Ok(ExecutionPermit { payload_bytes: bytes, signature, denial_code: String::new(), unreserved_request: None, }) } pub async fn next( state: &crate::State, tenant: Uuid, agent: Uuid, generation: u64, ) -> Result> { let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?; // Never invent nonstart once a grant was reserved. The durable ledger is // authoritative even when every start packet was lost. crate::scheduler::maintain(&mut tx, Some(agent)).await?; let active:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE agent_id=$1 AND connection_generation=$2 AND revoked_at IS NULL AND NOT restore_blocked AND (NOT sync_required OR sync_ready))").bind(agent).bind(i64::try_from(generation)?).fetch_one(&mut *tx).await?; if !active { return Ok(None); } if let Some(r)=sqlx::query("SELECT job_id,attempt_id FROM jobs WHERE agent_id=$1 AND cancel_requested AND state IN ('accepted','running','verifying','awaiting_reboot') ORDER BY job_id LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?{ tx.commit().await?;return Ok(Some(Payload::CancelRequest(CancelRequest{job_id:r.get::("job_id").to_string(),attempt_id:r.get::("attempt_id").to_string(),reason:"operator_cancel_requested".into()}))) } let row=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload,j.payload_hash,a.authorization_id,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,i.capabilities FROM outbox b JOIN jobs j USING(tenant_id,job_id,attempt_id) JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN tenants t ON t.tenant_id=j.tenant_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id WHERE j.agent_id=$1 AND j.state='queued' AND NOT j.cancel_requested AND NOT t.changes_stopped AND a.revoked_at IS NULL AND r.approval_state='approved' AND p.expires_at>now() AND b.acknowledged_at IS NULL AND (b.lease_until IS NULL OR b.lease_until, _>("payload"))?; let caps: Vec = serde_json::from_value(row.get("capabilities"))?; if payload.action.windows_msi().is_some() || matches!(payload.action, Action::UpdateAgent { .. }) { let raw: Vec = row.get("plan_payload"); let plan: PlanPayload = serde_json::from_slice(&raw)?; if hash(&raw) != row.get::("plan_hash") || crate::windows_msi_catalog::validate_job_binding(&plan, &payload).is_err() || (matches!(payload.action, Action::UpdateAgent { .. }) && crate::agent_updates::validate_job_binding(&plan, &payload).is_err()) || crate::catalog::check(&mut tx, &payload.action) .await .is_err() { tx.commit().await?; return Ok(None); } } if !payload .action .available_for(&payload.execution_identity, &caps) || !payload .action .valid_job_limits(payload.max_runtime_seconds, payload.max_output_bytes) { tx.commit().await?; return Ok(None); } if crate::root_policy::needs_helper(&payload.execution_identity) { let plan: PlanPayload = serde_json::from_slice(&row.get::, _>("plan_payload"))?; if crate::root_policy::authorize(&mut tx, &plan.principal, &payload) .await .is_err() || crate::root_policy::authorize(&mut tx, &row.get::("approver"), &payload) .await .is_err() { tx.commit().await?; return Ok(None); } } sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1") .bind(row.get::("job_id")) .execute(&mut *tx) .await?; let offer = JobOffer { job_id: row.get::("job_id").to_string(), attempt_id: row.get::("attempt_id").to_string(), authorization_id: row.get::("authorization_id").to_string(), plan_hash: row.get("plan_hash"), job_payload_hash: row.get("payload_hash"), payload_bytes: row.get("payload"), }; tx.commit().await?; Ok(Some(Payload::JobOffer(offer))) } #[cfg(test)] mod tests { use super::*; #[test] fn only_the_exact_delegation_placeholder_is_superseded() { let placeholder = JobResult { job_id: Uuid::new_v4().to_string(), attempt_id: Uuid::new_v4().to_string(), grant_id: Uuid::new_v4().to_string(), revision: 1, state: "unknown".into(), error_code: "SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT".into(), evidence_json: json!({"postconditions_passed": false}).to_string(), ..Default::default() }; let evidence = json!({"postconditions_passed": false}); assert!(superseded_delegation_placeholder(&placeholder, &evidence)); for case in 0..8 { let mut other = placeholder.clone(); let mut other_evidence = evidence.clone(); match case { 0 => other.state = "failed".into(), 1 => other.error_code = "SYSTEM_SCRIPT_EFFECT_UNRESOLVED".into(), 2 => other.grant_id.clear(), 3 => other.stdout = b"x".to_vec(), 4 => other.exit_code = Some(0), 5 => other.output_truncated = true, 6 => other.reboot_required = true, _ => other_evidence = json!({"postconditions_passed": true}), } assert!(!superseded_delegation_placeholder(&other, &other_evidence)); } } #[tokio::test] #[ignore = "requires dedicated isolated PostgreSQL with all migrations"] async fn unknown_followup_result_is_durable_without_reconciliation_or_new_grant() { let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap()) .await .unwrap(); let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap()) .await .unwrap(); rmm_persistence::verify_application_role(&app) .await .unwrap(); let tenant = Uuid::new_v4(); let agent = Uuid::new_v4(); let asset = Uuid::new_v4(); let job = Uuid::new_v4(); let attempt = Uuid::new_v4(); let grant = Uuid::new_v4(); let plan = Uuid::new_v4(); let operation = Uuid::new_v4(); let authorization = Uuid::new_v4(); let now = Utc::now(); let payload = JobPayload { schema_version: 1, tenant_id: tenant, asset_id: asset, agent_id: agent, enrollment_generation: 1, assignment_version: 1, job_id: job, attempt_id: attempt, action: Action::RefreshInventory { scope: "linux.base".into(), }, execution_identity: "agent_service".into(), not_before: now, start_before: now + Duration::minutes(5), max_runtime_seconds: 30, max_output_bytes: 65536, }; let bytes = serde_json::to_vec(&payload).unwrap(); sqlx::query("INSERT INTO tenants VALUES($1,'isolated unknown-result test')") .bind(tenant) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO assets VALUES($1,$2)") .bind(tenant) .bind(asset) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO operations VALUES($1,$2,$3)") .bind(tenant) .bind(operation) .bind(authorization) .execute(&admin) .await .unwrap(); // This is an isolated ledger fixture, not an approval or executable job // created through a deployed application. Sync already marked it unknown. sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,revision,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',2,true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)") .bind(tenant) .bind(grant) .bind(job) .bind(attempt) .bind(b"original-bound-grant".as_slice()) .bind(vec![0u8; 64]) .bind(now) .execute(&admin) .await .unwrap(); let mut result = JobResult { job_id: job.to_string(), attempt_id: attempt.to_string(), grant_id: grant.to_string(), revision: 3, state: "unknown".into(), error_code: "AGENT_PROCESS_INTERRUPTED".into(), evidence_json: json!({"effect_complete":false,"postconditions_passed":false}) .to_string(), ..Default::default() }; let mut envelope = Envelope { message_id: Uuid::new_v4().to_string(), agent_id: agent.to_string(), connection_generation: 1, payload: Some(Payload::JobResult(result.clone())), ..Default::default() }; let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); event(&mut tx, tenant, agent, &envelope).await.unwrap(); tx.commit().await.unwrap(); // The exact durable message is ACKable after reconnect without another // receipt or any reinterpretation of its still-unknown effect. envelope.connection_generation = 2; let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); event(&mut tx, tenant, agent, &envelope).await.unwrap(); tx.commit().await.unwrap(); result.revision = 4; result.error_code = "EFFECT_STILL_UNRESOLVED".into(); envelope.message_id = Uuid::new_v4().to_string(); envelope.payload = Some(Payload::JobResult(result.clone())); let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); event(&mut tx, tenant, agent, &envelope).await.unwrap(); tx.commit().await.unwrap(); for case in 0..7 { let mut invalid = result.clone(); invalid.revision = 5; let mut target_tenant = tenant; let mut target_agent = agent; match case { 0 => invalid.grant_id.clear(), 1 => invalid.grant_id = Uuid::new_v4().to_string(), 2 => invalid.attempt_id = Uuid::new_v4().to_string(), 3 => { invalid.revision = 3; invalid.error_code = "CONFLICTING_STALE_EVIDENCE".into(); } 4 => invalid.state = "awaiting_reboot".into(), 5 => target_tenant = Uuid::new_v4(), _ => target_agent = Uuid::new_v4(), } let mut invalid_envelope = envelope.clone(); invalid_envelope.message_id = Uuid::new_v4().to_string(); invalid_envelope.payload = Some(Payload::JobResult(invalid)); let mut tx = rmm_persistence::tenant_tx(&app, target_tenant) .await .unwrap(); assert!( event(&mut tx, target_tenant, target_agent, &invalid_envelope) .await .is_err(), "negative case {case}" ); tx.rollback().await.unwrap(); } let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); let row = locked_job(&mut tx, agent, &job.to_string(), &attempt.to_string()) .await .unwrap(); assert_eq!(row.get::("state"), "unknown"); assert_eq!(row.get::("revision"), 4); assert!(row.get::("start_reserved")); assert_eq!(row.get::, _>("payload"), bytes); let stored = row.get::("result"); assert_eq!(stored["grant_id"], grant.to_string()); assert_eq!(stored["error_code"], "EFFECT_STILL_UNRESOLVED"); assert_eq!(stored["evidence"]["postconditions_passed"], false); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts") .fetch_one(&mut *tx) .await .unwrap(), 2 ); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT count(*) FROM execution_permits") .fetch_one(&mut *tx) .await .unwrap(), 1 ); assert_eq!( sqlx::query_scalar::<_, Vec>( "SELECT payload FROM execution_permits WHERE grant_id=$1" ) .bind(grant) .fetch_one(&mut *tx) .await .unwrap(), b"original-bound-grant" ); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT count(*) FROM outbox") .fetch_one(&mut *tx) .await .unwrap(), 0 ); assert_eq!( sqlx::query_scalar::<_, i64>( "SELECT count(*) FROM audit_events WHERE event_type='job_reconciled'" ) .fetch_one(&mut *tx) .await .unwrap(), 0 ); tx.commit().await.unwrap(); } }