Files
rust-discovery/crates/server/src/job_gateway.rs
T

1103 lines
50 KiB
Rust

use anyhow::{Context, Result, ensure};
use chrono::{Duration, Utc};
use prost::Message;
use rmm_domain::jobs::*;
use rmm_protocol::*;
use serde_json::{Value, json};
use sqlx::{Postgres, Row, Transaction};
use uuid::Uuid;
/// Called in the same tenant transaction as durable message receipt and ACK.
pub async fn event(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
) -> Result<()> {
let bytes = e.payload.as_ref().context("missing payload")?;
let mut identity = e.clone();
identity.connection_generation = 0;
let digest = hash(&identity.encode_to_vec());
let message = Uuid::parse_str(&e.message_id)?;
if let Some(old) =
sqlx::query_scalar::<_, String>("SELECT payload_hash FROM job_receipts WHERE message_id=$1")
.bind(message)
.fetch_optional(&mut **tx)
.await?
{
ensure!(old == digest, "receipt identity conflict");
return Ok(());
}
match bytes {
Payload::JobAccepted(a) => {
ensure!(
!a.journal_proof.is_empty() && a.journal_proof.len() <= 256,
"journal proof required"
);
let row=sqlx::query("SELECT state,payload_hash FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(&a.job_id)?).bind(Uuid::parse_str(&a.attempt_id)?).bind(agent).fetch_one(&mut **tx).await?;
ensure!(
row.get::<String, _>("payload_hash") == a.job_payload_hash,
"payload mismatch"
);
let state: String = row.get("state");
if state == "queued" {
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=$2 WHERE job_id=$1")
.bind(Uuid::parse_str(&a.job_id)?)
.bind(&a.journal_proof)
.execute(&mut **tx)
.await?;
}
sqlx::query(
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
)
.bind(Uuid::parse_str(&a.job_id)?)
.execute(&mut **tx)
.await?;
}
Payload::JobStateChanged(s) => {
let row = locked_job(tx, agent, &s.job_id, &s.attempt_id).await?;
let old_revision: i64 = row.get("revision");
let old: String = row.get("state");
check_grant(tx, &s.job_id, &s.attempt_id, &s.grant_id).await?;
let old_state: JobState = serde_json::from_value(json!(old))?;
let next: JobState = serde_json::from_value(json!(s.state))?;
ensure!(!next.terminal(), "terminal states require result evidence");
if old_state.terminal()
|| old_state == JobState::Unknown
|| i64::try_from(s.revision)? < old_revision
{
// ACK durable but late evidence without regressing the ledger.
} else if i64::try_from(s.revision)? == old_revision {
ensure!(old == s.state, "state revision conflict");
} else {
// Terminal results require evidence and must use JobResult.
ensure!(
!next.terminal() && old_state.allows(next),
"invalid state transition"
);
sqlx::query("UPDATE jobs SET state=$2,revision=$3 WHERE job_id=$1")
.bind(Uuid::parse_str(&s.job_id)?)
.bind(&s.state)
.bind(i64::try_from(s.revision)?)
.execute(&mut **tx)
.await?;
}
}
Payload::JobResult(r) => {
ensure!(
r.evidence_json.len() <= 65536 && r.error_code.len() <= 128,
"evidence bound"
);
let row = locked_job(tx, agent, &r.job_id, &r.attempt_id).await?;
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
ensure!(
r.stdout.len() + r.stderr.len() <= payload.max_output_bytes as usize,
"output bound"
);
let evidence: Value = serde_json::from_str(&r.evidence_json)?;
let outcome: JobState = serde_json::from_value(json!(r.state))?;
ensure!(
matches!(
outcome,
JobState::Succeeded
| JobState::Failed
| JobState::Cancelled
| JobState::Expired
| JobState::Unknown
| JobState::AwaitingReboot
),
"invalid result state"
);
let result = json!({"stdout":r.stdout,"stderr":r.stderr,"output_truncated":r.output_truncated,"reboot_required":r.reboot_required,"exit_code":r.exit_code,"evidence":evidence,"error_code":r.error_code,"grant_id":r.grant_id});
let old_revision: i64 = row.get("revision");
if i64::try_from(r.revision)? < old_revision
&& payload.execution_identity == "system"
&& row.get::<bool, _>("start_reserved")
&& superseded_delegation_placeholder(r, &evidence)
{
// Sync may import the helper's authoritative record before the
// agent's earlier content-free delegation placeholder arrives.
// ACK it durably without regressing the ledger; any other stale
// evidence must still match exactly.
check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?;
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','superseded_delegation_placeholder_acknowledged',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"placeholder_revision":r.revision,"current_revision":old_revision,"current_state":row.get::<String, _>("state"),"message_id":e.message_id})).execute(&mut **tx).await?;
} else if i64::try_from(r.revision)? <= old_revision {
ensure!(
row.get::<String, _>("state") == r.state
&& row.get::<Option<Value>, _>("result").as_ref() == Some(&result),
"result identity conflict"
);
} else {
let previous: JobState =
serde_json::from_value(json!(row.get::<String, _>("state")))?;
let nonstart = !row.get::<bool, _>("start_reserved")
&& r.grant_id.is_empty()
&& evidence.get("not_started") == Some(&json!(true));
ensure!(
!previous.terminal()
|| (previous == outcome
&& nonstart
&& row.get::<Option<Value>, _>("result").is_none()),
"terminal outcome cannot be reset"
);
if previous == JobState::Unknown {
ensure!(
matches!(
outcome,
JobState::Succeeded
| JobState::Failed
| JobState::Cancelled
| JobState::Unknown
),
"unknown needs bound unknown or definitive reconciliation evidence"
);
}
if r.grant_id.is_empty() {
ensure!(
nonstart
&& matches!(
previous,
JobState::Queued
| JobState::Accepted
| JobState::Cancelled
| JobState::Expired
)
&& matches!(
outcome,
JobState::Cancelled | JobState::Expired | JobState::Failed
),
"ungranted outcome needs definite nonstart evidence"
);
} else {
check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?;
ensure!(
outcome != JobState::Expired,
"grant reservation cannot prove expiry without effect"
);
}
if outcome == JobState::Cancelled {
ensure!(
evidence.get("not_started") == Some(&json!(true))
|| evidence.get("abort_confirmed") == Some(&json!(true)),
"cancellation needs actual nonstart/abort evidence"
);
}
if outcome == JobState::Failed {
ensure!(
!r.error_code.is_empty(),
"known failure requires structured reason"
);
}
if r.error_code == rmm_domain::windows_system_nonstart::ERROR_CODE {
crate::windows_system_nonstart::validate_result(tx, tenant, agent, r, &evidence).await?;
}
if outcome == JobState::Succeeded {
ensure!(
evidence.get("postconditions_passed") == Some(&json!(true)),
"postcondition evidence required"
);
if let Action::RunScript {
expected_stdout, ..
} = &payload.action
{
ensure!(r.exit_code == Some(0), "script exit failure");
if let Some(expected) = expected_stdout {
ensure!(r.stdout == expected.as_bytes(), "expected output mismatch");
}
}
script_profile_evidence(&payload.action, &evidence)?;
windows_event_log_success_evidence(&payload, r, &evidence)?;
if matches!(payload.action, Action::Reboot { .. }) {
crate::synchronization::verify_reboot_success(
tx, tenant, agent, e, &payload, r, &evidence,
)
.await?;
}
}
sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1")
.bind(Uuid::parse_str(&r.job_id)?)
.bind(&r.state)
.bind(i64::try_from(r.revision)?)
.bind(result)
.execute(&mut **tx)
.await?;
// A newly persisted unknown result may follow a Sync decision
// that already marked the attempt unknown. Receipt/ACK records
// that evidence; it does not reconcile or release the attempt.
if previous == JobState::Unknown && outcome != JobState::Unknown {
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','job_reconciled',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"outcome":r.state,"evidence_source":"authenticated_durable_job_result"})).execute(&mut **tx).await?;
}
}
}
_ => anyhow::bail!("not a durable job event"),
}
sqlx::query("INSERT INTO job_receipts VALUES($1,$2,$3,$4)")
.bind(tenant)
.bind(agent)
.bind(message)
.bind(digest)
.execute(&mut **tx)
.await?;
Ok(())
}
/// Exact agent-side SYSTEM delegation placeholder: unknown, no output, no
/// exit code and no evidence beyond the failed postcondition marker.
fn superseded_delegation_placeholder(r: &JobResult, evidence: &Value) -> bool {
r.state == "unknown"
&& r.error_code == "SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"
&& !r.grant_id.is_empty()
&& r.stdout.is_empty()
&& r.stderr.is_empty()
&& r.exit_code.is_none()
&& !r.output_truncated
&& !r.reboot_required
&& *evidence == json!({"postconditions_passed": false})
}
fn windows_event_log_success_evidence(
job: &JobPayload,
result: &JobResult,
evidence: &Value,
) -> Result<()> {
if !job.action.is_windows_event_log_fixture() {
return Ok(());
}
let Action::QueryLogs {
unit,
since,
until,
max_lines,
} = &job.action
else {
anyhow::bail!("fixed event log action required")
};
let observed_since: chrono::DateTime<Utc> = serde_json::from_value(
evidence
.get("start_utc")
.cloned()
.context("event log start missing")?,
)?;
let observed_until: chrono::DateTime<Utc> = serde_json::from_value(
evidence
.get("end_utc")
.cloned()
.context("event log end missing")?,
)?;
let complete = evidence.get("completeness") == Some(&json!("complete"))
&& evidence.get("truncated") == Some(&json!(false))
&& evidence.get("error_code") == Some(&Value::Null);
let partial = evidence.get("completeness") == Some(&json!("partial"))
&& evidence.get("truncated") == Some(&json!(true))
&& matches!(
evidence.get("error_code").and_then(Value::as_str),
Some("EVENT_LOG_EVENT_LIMIT" | "EVENT_LOG_BYTE_LIMIT" | "EVENT_LOG_EVENT_SIZE_LIMIT")
);
ensure!(
!result.reboot_required
&& evidence.get("truncated") == Some(&json!(result.output_truncated))
&& job.execution_identity == "agent_service"
&& job.action.validate()
&& job
.action
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
&& evidence.get("source_id") == Some(&json!(unit))
&& evidence.get("channel") == Some(&json!(WINDOWS_EVENT_LOG_FIXTURE_CHANNEL))
&& evidence.get("provider") == Some(&json!(WINDOWS_EVENT_LOG_FIXTURE_PROVIDER))
&& observed_since == *since
&& observed_until == *until
&& evidence.get("max_events") == Some(&json!(max_lines))
&& evidence.get("max_output_bytes") == Some(&json!(job.max_output_bytes))
&& evidence.get("quality") == Some(&json!("valid"))
&& evidence.get("win32_error") == Some(&Value::Null)
&& evidence.get("bookmark_updated") == Some(&json!(false))
&& evidence.get("process_tree_bounded") == Some(&json!(true))
&& evidence.get("execution_sid") == Some(&json!("S-1-5-19"))
&& (complete || partial),
"fixed Windows Event Log evidence mismatch"
);
Ok(())
}
// The profile records the requested process option, never an effective GPO policy.
fn script_profile_evidence(action: &Action, evidence: &Value) -> Result<()> {
if action.full_maintenance() {
let Action::RunScript { script_utf8, .. } = action else {
unreachable!()
};
ensure!(
evidence["privileged_script_profile"] == "full_maintenance_v1"
&& evidence["script_sha256"] == hash(script_utf8.as_bytes())
&& match action.script_platform() {
Some("linux") => evidence["execution_uid"] == 0,
Some("windows") => evidence["execution_sid"] == "S-1-5-18",
_ => false,
},
"full maintenance identity/profile evidence mismatch"
);
}
if let Action::RunScript {
script_utf8,
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
..
} = action
{
ensure!(
evidence.get("windows_execution_profile")
== Some(&json!("powershell_51_remote_signed"))
&& evidence.get("execution_policy")
== Some(&json!("process_remote_signed_requested"))
&& evidence.get("execution_policy_requested") == Some(&json!("RemoteSigned"))
&& evidence.get("execution_policy_scope") == Some(&json!("process"))
&& evidence.get("script_sha256") == Some(&json!(hash(script_utf8.as_bytes()))),
"requested Windows process profile evidence mismatch"
);
}
Ok(())
}
#[cfg(test)]
mod script_profile_tests {
use super::*;
#[test]
fn windows_event_log_success_binds_request_and_keeps_limit_partial_honest() {
let id = Uuid::new_v4();
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
job_id: id,
attempt_id: id,
action: Action::QueryLogs {
unit: WINDOWS_EVENT_LOG_FIXTURE_SOURCE.into(),
since: now - Duration::hours(1),
until: now,
max_lines: 1000,
},
execution_identity: "agent_service".into(),
not_before: now,
start_before: now + Duration::minutes(1),
max_runtime_seconds: 30,
max_output_bytes: 4096,
};
let evidence = json!({"source_id":WINDOWS_EVENT_LOG_FIXTURE_SOURCE,"channel":"Application","provider":"RustDiscovery-MVP-Test","start_utc":now-Duration::hours(1),"end_utc":now,"max_events":1000,"max_output_bytes":4096,"quality":"valid","completeness":"complete","truncated":false,"error_code":null,"win32_error":null,"bookmark_updated":false,"process_tree_bounded":true,"execution_sid":"S-1-5-19"});
let result = JobResult::default();
windows_event_log_success_evidence(&job, &result, &evidence).unwrap();
for conflicting in [
JobResult {
reboot_required: true,
..Default::default()
},
JobResult {
output_truncated: true,
..Default::default()
},
] {
assert!(windows_event_log_success_evidence(&job, &conflicting, &evidence).is_err());
}
for field in [
"source_id",
"channel",
"provider",
"start_utc",
"end_utc",
"max_events",
"max_output_bytes",
"quality",
"win32_error",
"bookmark_updated",
"process_tree_bounded",
"execution_sid",
] {
let mut bad = evidence.clone();
bad[field] = json!("different");
assert!(
windows_event_log_success_evidence(&job, &result, &bad).is_err(),
"{field}"
);
}
for code in [
"EVENT_LOG_EVENT_LIMIT",
"EVENT_LOG_BYTE_LIMIT",
"EVENT_LOG_EVENT_SIZE_LIMIT",
] {
let mut partial = evidence.clone();
partial["completeness"] = json!("partial");
partial["truncated"] = json!(true);
partial["error_code"] = json!(code);
let partial_result = JobResult {
output_truncated: true,
..Default::default()
};
windows_event_log_success_evidence(&job, &partial_result, &partial).unwrap();
assert!(windows_event_log_success_evidence(&job, &result, &partial).is_err());
partial["error_code"] = json!("EVENT_LOG_TIMEOUT");
assert!(windows_event_log_success_evidence(&job, &partial_result, &partial).is_err());
partial["quality"] = json!("permission_denied");
assert!(windows_event_log_success_evidence(&job, &partial_result, &partial).is_err());
}
let mut permission_denied = evidence.clone();
permission_denied["win32_error"] = json!(5);
assert!(windows_event_log_success_evidence(&job, &result, &permission_denied).is_err());
let mut false_complete = evidence.clone();
false_complete["truncated"] = json!(true);
assert!(windows_event_log_success_evidence(&job, &result, &false_complete).is_err());
}
#[test]
fn requested_profile_success_evidence_is_exact_without_effective_policy_claim() {
let mut action = Action::RunScript {
interpreter: "powershell".into(),
script_utf8: "Write-Output 'hi'\r\n".into(),
arguments: vec![],
expected_stdout: None,
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
privileged_script_profile: None,
};
let evidence = json!({"windows_execution_profile":"powershell_51_remote_signed",
"execution_policy":"process_remote_signed_requested", "execution_policy_requested":"RemoteSigned",
"execution_policy_scope":"process", "script_sha256":hash(b"Write-Output 'hi'\r\n")});
script_profile_evidence(&action, &evidence).unwrap();
for field in [
"windows_execution_profile",
"execution_policy",
"execution_policy_requested",
"execution_policy_scope",
"script_sha256",
] {
let mut changed = evidence.clone();
changed.as_object_mut().unwrap().remove(field);
assert!(
script_profile_evidence(&action, &changed).is_err(),
"{field}"
);
changed[field] = json!("different");
assert!(
script_profile_evidence(&action, &changed).is_err(),
"{field}"
);
}
if let Action::RunScript {
windows_execution_profile,
..
} = &mut action
{
*windows_execution_profile = None;
}
script_profile_evidence(&action, &json!({})).unwrap();
}
}
async fn locked_job(
tx: &mut Transaction<'_, Postgres>,
agent: Uuid,
job: &str,
attempt: &str,
) -> Result<sqlx::postgres::PgRow> {
Ok(sqlx::query("SELECT state,revision,result,payload,start_reserved FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(agent).fetch_one(&mut **tx).await?)
}
async fn check_grant(
tx: &mut Transaction<'_, Postgres>,
job: &str,
attempt: &str,
grant: &str,
) -> Result<()> {
let ok:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM execution_permits WHERE job_id=$1 AND attempt_id=$2 AND grant_id=$3)").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(Uuid::parse_str(grant)?).fetch_one(&mut **tx).await?;
ensure!(ok, "missing bound grant");
Ok(())
}
pub async fn permit(
state: &crate::State,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
r: &ExecutionPermitRequest,
) -> Result<ExecutionPermit> {
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
// Tenant serial lock covers rollout counters, stop check, and reservations.
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("starts:{tenant}"))
.execute(&mut *tx)
.await?;
crate::agent_service::control_lock(&mut tx, tenant, agent).await?;
let denied = |code: &str| ExecutionPermit {
payload_bytes: vec![],
signature: vec![],
denial_code: code.into(),
unreserved_request: None,
};
let row=sqlx::query("SELECT j.*,a.authorization_id,a.revoked_at,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,r.approval_state,i.enrollment_generation,i.assignment_version,i.asset_id AS bound_asset,i.capabilities,i.revoked_at AS device_revoked,i.connection_generation,t.changes_stopped FROM jobs j JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 FOR UPDATE OF j").bind(Uuid::parse_str(&r.job_id)?).bind(Uuid::parse_str(&r.attempt_id)?).bind(agent).fetch_optional(&mut *tx).await?;
let Some(row) = row else {
return Ok(denied("NOT_FOUND"));
};
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("plan_payload"))?;
let capabilities: Vec<String> = serde_json::from_value(row.get("capabilities"))?;
if crate::windows_msi_catalog::validate_job_binding(&plan, &payload).is_err()
|| (matches!(payload.action, Action::UpdateAgent { .. })
&& crate::agent_updates::validate_job_binding(&plan, &payload).is_err())
{
return Ok(denied("WINDOWS_MSI_BINDING_MISMATCH"));
}
if row.get::<Uuid, _>("bound_asset") != payload.asset_id
|| !payload
.action
.available_for(&payload.execution_identity, &capabilities)
|| hash(&row.get::<Vec<u8>, _>("plan_payload")) != row.get::<String, _>("plan_hash")
|| !plan.jobs.iter().any(|j| {
j.payload.job_id == payload.job_id
&& j.payload.attempt_id == payload.attempt_id
&& j.job_payload_hash == r.job_payload_hash
})
{
return Ok(denied("BINDING_MISMATCH"));
}
if row.get::<bool, _>("start_reserved") {
return Ok(denied("START_ALREADY_RESERVED"));
}
if row.get::<String, _>("state") != "accepted"
|| row.get::<Option<String>, _>("journal_proof").is_none()
{
return Ok(denied("JOURNAL_ACCEPTANCE_REQUIRED"));
}
if row.get::<bool, _>("changes_stopped")
|| row.get::<bool, _>("cancel_requested")
|| row.get::<String, _>("approval_state") != "approved"
|| row
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
.is_some()
|| row
.get::<Option<chrono::DateTime<Utc>>, _>("device_revoked")
.is_some()
|| !crate::jobs::requester_enabled(state, &mut tx, &plan.principal, tenant).await?
|| !crate::jobs::requester_enabled(state, &mut tx, &row.get::<String, _>("approver"), tenant).await?
{
return Ok(denied("AUTHORIZATION_REVOKED"));
}
if row.get::<i64, _>("enrollment_generation") as u64 != payload.enrollment_generation
|| row.get::<i64, _>("assignment_version") as u64 != payload.assignment_version
|| r.assignment_version != payload.assignment_version
|| row.get::<i64, _>("connection_generation") as u64 != e.connection_generation
|| r.job_payload_hash != row.get::<String, _>("payload_hash")
|| hash(&encode(&payload)?) != r.job_payload_hash
{
return Ok(denied("BINDING_MISMATCH"));
}
// The locked row belongs to this request and has no start reservation.
// Echo the binding only after both checks, so a client can safely return
// this particular offer to scheduling after a fresh signed reconciliation.
let denied = |code: &str| ExecutionPermit {
unreserved_request: Some(r.clone()),
..denied(code)
};
let now = Utc::now();
if !payload
.action
.valid_job_limits(payload.max_runtime_seconds, payload.max_output_bytes)
{
return Ok(denied("INVALID_JOB_LIMITS"));
}
if !payload.action.validate() {
return Ok(denied("APT_MANIFEST_REQUIRED"));
}
if crate::root_policy::needs_helper(&payload.execution_identity) {
let helper_ready: bool = sqlx::query_scalar(
"SELECT i.helper_sync_ready AND EXISTS(SELECT 1 FROM agent_sync_state s WHERE s.tenant_id=i.tenant_id AND s.agent_id=i.agent_id AND s.helper_journal_id IS NOT NULL) FROM agent_installations i WHERE i.agent_id=$1",
)
.bind(agent)
.fetch_one(&mut *tx)
.await?;
if !helper_ready
|| crate::root_policy::authorize(&mut tx, &plan.principal, &payload)
.await
.is_err()
|| crate::root_policy::authorize(&mut tx, &row.get::<String, _>("approver"), &payload)
.await
.is_err()
{
return Ok(denied(if payload.execution_identity == "system" {
"SYSTEM_SCOPE_OR_HELPER_SYNC_REQUIRED"
} else {
"ROOT_SCOPE_OR_HELPER_SYNC_REQUIRED"
}));
}
}
let sync: (bool, bool) = sqlx::query_as(
"SELECT sync_required,sync_ready FROM agent_installations WHERE agent_id=$1",
)
.bind(agent)
.fetch_one(&mut *tx)
.await?;
if sync.0 && !sync.1 {
return Ok(denied("SYNC_REQUIRED"));
}
let restore_blocked: bool =
sqlx::query_scalar("SELECT restore_blocked FROM agent_installations WHERE agent_id=$1")
.bind(agent)
.fetch_one(&mut *tx)
.await?;
if restore_blocked {
return Ok(denied("RESTORE_RECONCILIATION_REQUIRED"));
}
if crate::catalog::check(&mut tx, &payload.action)
.await
.is_err()
{
return Ok(denied("RECIPE_BINDING_MISMATCH"));
}
let references = crate::catalog::references(&mut tx, &payload.action)
.await
.map_err(|_| anyhow::anyhow!("catalog unavailable"))?;
if references
.iter()
.any(|r| !plan.recipe_references.contains(r))
{
return Ok(denied("RECIPE_BINDING_MISMATCH"));
}
let artifact_ids = plan
.artifact_references
.iter()
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
.map(|r| r.artifact.artifact_id)
.collect::<Vec<_>>();
let artifacts = if let Action::UpdateAgent { update } = &payload.action {
crate::agent_updates::resolve_artifacts(
&mut tx,
tenant,
&update
.manifest()
.map_err(|_| anyhow::anyhow!("invalid agent release"))?
.artifacts,
)
.await
} else {
crate::catalog::resolve_artifacts(&mut tx, tenant, &artifact_ids).await
}
.map_err(|_| anyhow::anyhow!("artifact unavailable"))?;
if plan
.artifact_references
.iter()
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
.any(|r| !artifacts.contains(&r.artifact))
{
return Ok(denied("ARTIFACT_BINDING_MISMATCH"));
}
let approver: String = row.get("approver");
if matches!(payload.action, Action::QueryLogs { .. })
&& !crate::permissions::allowed(&mut tx, &plan.principal, "logs:read").await?
{
return Ok(denied("AUTHORIZATION_REVOKED"));
}
if !crate::permissions::allowed(&mut tx, &approver, "plans:approve").await?
|| !crate::permissions::allowed(&mut tx, &plan.principal, "operations:submit").await?
|| (matches!(payload.action, Action::RunScript { .. })
&& (!crate::permissions::allowed(&mut tx, &plan.principal, "scripts:execute").await?
|| !crate::permissions::allowed(&mut tx, &approver, "scripts:execute").await?))
|| (matches!(payload.action, Action::UpdateAgent { .. })
&& (!crate::permissions::allowed(&mut tx, &plan.principal, "updates:apply").await?
|| !crate::permissions::allowed(&mut tx, &approver, "updates:apply").await?))
{
return Ok(denied("AUTHORIZATION_REVOKED"));
}
let limits: (i32, i32) =
sqlx::query_as("SELECT starts_per_hour,max_parallel FROM tenant_limits")
.fetch_one(&mut *tx)
.await?;
let hourly: i64 = sqlx::query_scalar(
"SELECT count(*) FROM execution_permits WHERE created_at>now()-interval '1 hour'",
)
.fetch_one(&mut *tx)
.await?;
let concurrent:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").fetch_one(&mut *tx).await?;
if hourly >= i64::from(limits.0) || concurrent >= i64::from(limits.1) {
return Ok(denied("QUOTA_EXCEEDED"));
}
if let Action::Reboot {
dependency_group, ..
} = &payload.action
{
let busy:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown') AND (convert_from(payload,'UTF8')::jsonb->'action'->>'dependency_group'=$1 OR convert_from(payload,'UTF8')::jsonb->'action'->>'action'='reboot'))").bind(dependency_group).fetch_one(&mut *tx).await?;
if busy {
return Ok(denied("RESOURCE_LOCKED"));
}
}
if now >= payload.start_before {
return Ok(denied("PLAN_EXPIRED"));
}
if now < payload.not_before {
return Ok(denied("MAINTENANCE_WINDOW"));
}
let failures: i64 = sqlx::query_scalar(
"SELECT count(*) FROM jobs WHERE operation_id=$1 AND state IN ('failed','unknown')",
)
.bind(row.get::<Uuid, _>("operation_id"))
.fetch_one(&mut *tx)
.await?;
let active:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE operation_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").bind(row.get::<Uuid,_>("operation_id")).fetch_one(&mut *tx).await?;
if failures >= i64::from(plan.halt_after_failures) || active >= i64::from(plan.max_parallel) {
return Ok(denied("ROLLOUT_PAUSED"));
}
let locked:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))").bind(agent).fetch_one(&mut *tx).await?;
if locked {
return Ok(denied("RESOURCE_LOCKED"));
}
let grant = ExecutionPermitPayload {
agent_update_plan_id: matches!(payload.action, Action::UpdateAgent { .. })
.then_some(plan.plan_id),
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: row.get("authorization_id"),
tenant_id: tenant,
asset_id: payload.asset_id,
agent_id: agent,
enrollment_generation: payload.enrollment_generation,
assignment_version: payload.assignment_version,
job_id: payload.job_id,
attempt_id: payload.attempt_id,
plan_hash: row.get("plan_hash"),
job_payload_hash: r.job_payload_hash.clone(),
action: payload.action.name().into(),
execution_identity: payload.execution_identity,
not_before: now,
start_before: std::cmp::min(now + Duration::seconds(60), payload.start_before),
max_runtime_seconds: payload.max_runtime_seconds,
key_id: state.job_signer.key_id.clone(),
};
let bytes = encode(&grant)?;
let signature = state.job_signer.sign(&bytes)?;
sqlx::query("INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(grant.grant_id).bind(grant.job_id).bind(grant.attempt_id).bind(&bytes).bind(&signature).execute(&mut *tx).await?;
sqlx::query("UPDATE jobs SET start_reserved=true WHERE job_id=$1")
.bind(grant.job_id)
.execute(&mut *tx)
.await?;
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','start_reserved',$3,now())")
.bind(tenant)
.bind(Uuid::new_v4())
.bind(json!({"job_id":r.job_id,"grant_id":grant.grant_id,"boot_id":e.boot_id}))
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(ExecutionPermit {
payload_bytes: bytes,
signature,
denial_code: String::new(),
unreserved_request: None,
})
}
pub async fn next(
state: &crate::State,
tenant: Uuid,
agent: Uuid,
generation: u64,
) -> Result<Option<Payload>> {
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
// Never invent nonstart once a grant was reserved. The durable ledger is
// authoritative even when every start packet was lost.
crate::scheduler::maintain(&mut tx, Some(agent)).await?;
let active:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE agent_id=$1 AND connection_generation=$2 AND revoked_at IS NULL AND NOT restore_blocked AND (NOT sync_required OR sync_ready))").bind(agent).bind(i64::try_from(generation)?).fetch_one(&mut *tx).await?;
if !active {
return Ok(None);
}
if let Some(r)=sqlx::query("SELECT job_id,attempt_id FROM jobs WHERE agent_id=$1 AND cancel_requested AND state IN ('accepted','running','verifying','awaiting_reboot') ORDER BY job_id LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?{
tx.commit().await?;return Ok(Some(Payload::CancelRequest(CancelRequest{job_id:r.get::<Uuid,_>("job_id").to_string(),attempt_id:r.get::<Uuid,_>("attempt_id").to_string(),reason:"operator_cancel_requested".into()})))
}
let row=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload,j.payload_hash,a.authorization_id,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,i.capabilities FROM outbox b JOIN jobs j USING(tenant_id,job_id,attempt_id) JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN tenants t ON t.tenant_id=j.tenant_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id WHERE j.agent_id=$1 AND j.state='queued' AND NOT j.cancel_requested AND NOT t.changes_stopped AND a.revoked_at IS NULL AND r.approval_state='approved' AND p.expires_at>now() AND b.acknowledged_at IS NULL AND (b.lease_until IS NULL OR b.lease_until<now()) ORDER BY j.job_id FOR UPDATE OF b SKIP LOCKED LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?;
let Some(row) = row else {
tx.commit().await?;
return Ok(None);
};
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
let caps: Vec<String> = serde_json::from_value(row.get("capabilities"))?;
if payload.action.windows_msi().is_some()
|| matches!(payload.action, Action::UpdateAgent { .. })
{
let raw: Vec<u8> = row.get("plan_payload");
let plan: PlanPayload = serde_json::from_slice(&raw)?;
if hash(&raw) != row.get::<String, _>("plan_hash")
|| crate::windows_msi_catalog::validate_job_binding(&plan, &payload).is_err()
|| (matches!(payload.action, Action::UpdateAgent { .. })
&& crate::agent_updates::validate_job_binding(&plan, &payload).is_err())
|| crate::catalog::check(&mut tx, &payload.action)
.await
.is_err()
{
tx.commit().await?;
return Ok(None);
}
}
if !payload
.action
.available_for(&payload.execution_identity, &caps)
|| !payload
.action
.valid_job_limits(payload.max_runtime_seconds, payload.max_output_bytes)
{
tx.commit().await?;
return Ok(None);
}
if crate::root_policy::needs_helper(&payload.execution_identity) {
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("plan_payload"))?;
if crate::root_policy::authorize(&mut tx, &plan.principal, &payload)
.await
.is_err()
|| crate::root_policy::authorize(&mut tx, &row.get::<String, _>("approver"), &payload)
.await
.is_err()
{
tx.commit().await?;
return Ok(None);
}
}
sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1")
.bind(row.get::<Uuid, _>("job_id"))
.execute(&mut *tx)
.await?;
let offer = JobOffer {
job_id: row.get::<Uuid, _>("job_id").to_string(),
attempt_id: row.get::<Uuid, _>("attempt_id").to_string(),
authorization_id: row.get::<Uuid, _>("authorization_id").to_string(),
plan_hash: row.get("plan_hash"),
job_payload_hash: row.get("payload_hash"),
payload_bytes: row.get("payload"),
};
tx.commit().await?;
Ok(Some(Payload::JobOffer(offer)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_the_exact_delegation_placeholder_is_superseded() {
let placeholder = JobResult {
job_id: Uuid::new_v4().to_string(),
attempt_id: Uuid::new_v4().to_string(),
grant_id: Uuid::new_v4().to_string(),
revision: 1,
state: "unknown".into(),
error_code: "SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT".into(),
evidence_json: json!({"postconditions_passed": false}).to_string(),
..Default::default()
};
let evidence = json!({"postconditions_passed": false});
assert!(superseded_delegation_placeholder(&placeholder, &evidence));
for case in 0..8 {
let mut other = placeholder.clone();
let mut other_evidence = evidence.clone();
match case {
0 => other.state = "failed".into(),
1 => other.error_code = "SYSTEM_SCRIPT_EFFECT_UNRESOLVED".into(),
2 => other.grant_id.clear(),
3 => other.stdout = b"x".to_vec(),
4 => other.exit_code = Some(0),
5 => other.output_truncated = true,
6 => other.reboot_required = true,
_ => other_evidence = json!({"postconditions_passed": true}),
}
assert!(!superseded_delegation_placeholder(&other, &other_evidence));
}
}
#[tokio::test]
#[ignore = "requires dedicated isolated PostgreSQL with all migrations"]
async fn unknown_followup_result_is_durable_without_reconciliation_or_new_grant() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await
.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await
.unwrap();
rmm_persistence::verify_application_role(&app)
.await
.unwrap();
let tenant = Uuid::new_v4();
let agent = Uuid::new_v4();
let asset = Uuid::new_v4();
let job = Uuid::new_v4();
let attempt = Uuid::new_v4();
let grant = Uuid::new_v4();
let plan = Uuid::new_v4();
let operation = Uuid::new_v4();
let authorization = Uuid::new_v4();
let now = Utc::now();
let payload = JobPayload {
schema_version: 1,
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
job_id: job,
attempt_id: attempt,
action: Action::RefreshInventory {
scope: "linux.base".into(),
},
execution_identity: "agent_service".into(),
not_before: now,
start_before: now + Duration::minutes(5),
max_runtime_seconds: 30,
max_output_bytes: 65536,
};
let bytes = serde_json::to_vec(&payload).unwrap();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated unknown-result test')")
.bind(tenant)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets VALUES($1,$2)")
.bind(tenant)
.bind(asset)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
.bind(tenant)
.bind(operation)
.bind(authorization)
.execute(&admin)
.await
.unwrap();
// This is an isolated ledger fixture, not an approval or executable job
// created through a deployed application. Sync already marked it unknown.
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,revision,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',2,true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)")
.bind(tenant)
.bind(grant)
.bind(job)
.bind(attempt)
.bind(b"original-bound-grant".as_slice())
.bind(vec![0u8; 64])
.bind(now)
.execute(&admin)
.await
.unwrap();
let mut result = JobResult {
job_id: job.to_string(),
attempt_id: attempt.to_string(),
grant_id: grant.to_string(),
revision: 3,
state: "unknown".into(),
error_code: "AGENT_PROCESS_INTERRUPTED".into(),
evidence_json: json!({"effect_complete":false,"postconditions_passed":false})
.to_string(),
..Default::default()
};
let mut envelope = Envelope {
message_id: Uuid::new_v4().to_string(),
agent_id: agent.to_string(),
connection_generation: 1,
payload: Some(Payload::JobResult(result.clone())),
..Default::default()
};
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
event(&mut tx, tenant, agent, &envelope).await.unwrap();
tx.commit().await.unwrap();
// The exact durable message is ACKable after reconnect without another
// receipt or any reinterpretation of its still-unknown effect.
envelope.connection_generation = 2;
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
event(&mut tx, tenant, agent, &envelope).await.unwrap();
tx.commit().await.unwrap();
result.revision = 4;
result.error_code = "EFFECT_STILL_UNRESOLVED".into();
envelope.message_id = Uuid::new_v4().to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
event(&mut tx, tenant, agent, &envelope).await.unwrap();
tx.commit().await.unwrap();
for case in 0..7 {
let mut invalid = result.clone();
invalid.revision = 5;
let mut target_tenant = tenant;
let mut target_agent = agent;
match case {
0 => invalid.grant_id.clear(),
1 => invalid.grant_id = Uuid::new_v4().to_string(),
2 => invalid.attempt_id = Uuid::new_v4().to_string(),
3 => {
invalid.revision = 3;
invalid.error_code = "CONFLICTING_STALE_EVIDENCE".into();
}
4 => invalid.state = "awaiting_reboot".into(),
5 => target_tenant = Uuid::new_v4(),
_ => target_agent = Uuid::new_v4(),
}
let mut invalid_envelope = envelope.clone();
invalid_envelope.message_id = Uuid::new_v4().to_string();
invalid_envelope.payload = Some(Payload::JobResult(invalid));
let mut tx = rmm_persistence::tenant_tx(&app, target_tenant)
.await
.unwrap();
assert!(
event(&mut tx, target_tenant, target_agent, &invalid_envelope)
.await
.is_err(),
"negative case {case}"
);
tx.rollback().await.unwrap();
}
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
let row = locked_job(&mut tx, agent, &job.to_string(), &attempt.to_string())
.await
.unwrap();
assert_eq!(row.get::<String, _>("state"), "unknown");
assert_eq!(row.get::<i64, _>("revision"), 4);
assert!(row.get::<bool, _>("start_reserved"));
assert_eq!(row.get::<Vec<u8>, _>("payload"), bytes);
let stored = row.get::<Value, _>("result");
assert_eq!(stored["grant_id"], grant.to_string());
assert_eq!(stored["error_code"], "EFFECT_STILL_UNRESOLVED");
assert_eq!(stored["evidence"]["postconditions_passed"], false);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
.fetch_one(&mut *tx)
.await
.unwrap(),
2
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM execution_permits")
.fetch_one(&mut *tx)
.await
.unwrap(),
1
);
assert_eq!(
sqlx::query_scalar::<_, Vec<u8>>(
"SELECT payload FROM execution_permits WHERE grant_id=$1"
)
.bind(grant)
.fetch_one(&mut *tx)
.await
.unwrap(),
b"original-bound-grant"
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM outbox")
.fetch_one(&mut *tx)
.await
.unwrap(),
0
);
assert_eq!(
sqlx::query_scalar::<_, i64>(
"SELECT count(*) FROM audit_events WHERE event_type='job_reconciled'"
)
.fetch_one(&mut *tx)
.await
.unwrap(),
0
);
tx.commit().await.unwrap();
}
}