Files
rust-discovery/crates/server/src/job_gateway.rs
T
2026-11-13 19:49:12 +01:00

514 lines
24 KiB
Rust

use anyhow::{Context, Result, ensure};
use chrono::{Duration, Utc};
use prost::Message;
use rmm_domain::jobs::*;
use rmm_protocol::*;
use serde_json::{Value, json};
use sqlx::{Postgres, Row, Transaction};
use uuid::Uuid;
/// Called in the same tenant transaction as durable message receipt and ACK.
pub async fn event(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
) -> Result<()> {
let bytes = e.payload.as_ref().context("missing payload")?;
let mut identity = e.clone();
identity.connection_generation = 0;
let digest = hash(&identity.encode_to_vec());
let message = Uuid::parse_str(&e.message_id)?;
if let Some(old) =
sqlx::query_scalar::<_, String>("SELECT payload_hash FROM job_receipts WHERE message_id=$1")
.bind(message)
.fetch_optional(&mut **tx)
.await?
{
ensure!(old == digest, "receipt identity conflict");
return Ok(());
}
match bytes {
Payload::JobAccepted(a) => {
ensure!(
!a.journal_proof.is_empty() && a.journal_proof.len() <= 256,
"journal proof required"
);
let row=sqlx::query("SELECT state,payload_hash FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(&a.job_id)?).bind(Uuid::parse_str(&a.attempt_id)?).bind(agent).fetch_one(&mut **tx).await?;
ensure!(
row.get::<String, _>("payload_hash") == a.job_payload_hash,
"payload mismatch"
);
let state: String = row.get("state");
if state == "queued" {
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=$2 WHERE job_id=$1")
.bind(Uuid::parse_str(&a.job_id)?)
.bind(&a.journal_proof)
.execute(&mut **tx)
.await?;
}
sqlx::query(
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
)
.bind(Uuid::parse_str(&a.job_id)?)
.execute(&mut **tx)
.await?;
}
Payload::JobStateChanged(s) => {
let row = locked_job(tx, agent, &s.job_id, &s.attempt_id).await?;
let old_revision: i64 = row.get("revision");
let old: String = row.get("state");
check_grant(tx, &s.job_id, &s.attempt_id, &s.grant_id).await?;
let old_state: JobState = serde_json::from_value(json!(old))?;
let next: JobState = serde_json::from_value(json!(s.state))?;
ensure!(!next.terminal(), "terminal states require result evidence");
if old_state.terminal()
|| old_state == JobState::Unknown
|| i64::try_from(s.revision)? < old_revision
{
// ACK durable but late evidence without regressing the ledger.
} else if i64::try_from(s.revision)? == old_revision {
ensure!(old == s.state, "state revision conflict");
} else {
// Terminal results require evidence and must use JobResult.
ensure!(
!next.terminal() && old_state.allows(next),
"invalid state transition"
);
sqlx::query("UPDATE jobs SET state=$2,revision=$3 WHERE job_id=$1")
.bind(Uuid::parse_str(&s.job_id)?)
.bind(&s.state)
.bind(i64::try_from(s.revision)?)
.execute(&mut **tx)
.await?;
}
}
Payload::JobResult(r) => {
ensure!(
r.evidence_json.len() <= 65536 && r.error_code.len() <= 128,
"evidence bound"
);
let row = locked_job(tx, agent, &r.job_id, &r.attempt_id).await?;
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
ensure!(
r.stdout.len() + r.stderr.len() <= payload.max_output_bytes as usize,
"output bound"
);
let evidence: Value = serde_json::from_str(&r.evidence_json)?;
let outcome: JobState = serde_json::from_value(json!(r.state))?;
ensure!(
matches!(
outcome,
JobState::Succeeded
| JobState::Failed
| JobState::Cancelled
| JobState::Expired
| JobState::Unknown
| JobState::AwaitingReboot
),
"invalid result state"
);
let result = json!({"stdout":r.stdout,"stderr":r.stderr,"output_truncated":r.output_truncated,"reboot_required":r.reboot_required,"exit_code":r.exit_code,"evidence":evidence,"error_code":r.error_code,"grant_id":r.grant_id});
let old_revision: i64 = row.get("revision");
if i64::try_from(r.revision)? <= old_revision {
ensure!(
row.get::<String, _>("state") == r.state
&& row.get::<Option<Value>, _>("result").as_ref() == Some(&result),
"result identity conflict"
);
} else {
let previous: JobState =
serde_json::from_value(json!(row.get::<String, _>("state")))?;
let nonstart = !row.get::<bool, _>("start_reserved")
&& r.grant_id.is_empty()
&& evidence.get("not_started") == Some(&json!(true));
ensure!(
!previous.terminal()
|| (previous == outcome
&& nonstart
&& row.get::<Option<Value>, _>("result").is_none()),
"terminal outcome cannot be reset"
);
if previous == JobState::Unknown {
ensure!(
matches!(
outcome,
JobState::Succeeded | JobState::Failed | JobState::Cancelled
),
"unknown needs definitive reconciliation evidence"
);
}
if r.grant_id.is_empty() {
ensure!(
nonstart
&& matches!(
previous,
JobState::Queued
| JobState::Accepted
| JobState::Cancelled
| JobState::Expired
)
&& matches!(
outcome,
JobState::Cancelled | JobState::Expired | JobState::Failed
),
"ungranted outcome needs definite nonstart evidence"
);
} else {
check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?;
ensure!(
outcome != JobState::Expired,
"grant reservation cannot prove expiry without effect"
);
}
if outcome == JobState::Cancelled {
ensure!(
evidence.get("not_started") == Some(&json!(true))
|| evidence.get("abort_confirmed") == Some(&json!(true)),
"cancellation needs actual nonstart/abort evidence"
);
}
if outcome == JobState::Failed {
ensure!(
!r.error_code.is_empty(),
"known failure requires structured reason"
);
}
if outcome == JobState::Succeeded {
ensure!(
evidence.get("postconditions_passed") == Some(&json!(true)),
"postcondition evidence required"
);
if let Action::RunScript {
expected_stdout, ..
} = &payload.action
{
ensure!(r.exit_code == Some(0), "script exit failure");
if let Some(expected) = expected_stdout {
ensure!(r.stdout == expected.as_bytes(), "expected output mismatch");
}
}
if matches!(payload.action, Action::Reboot { .. }) {
crate::synchronization::verify_reboot_success(
tx, tenant, agent, e, &payload, r, &evidence,
)
.await?;
}
}
sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1")
.bind(Uuid::parse_str(&r.job_id)?)
.bind(&r.state)
.bind(i64::try_from(r.revision)?)
.bind(result)
.execute(&mut **tx)
.await?;
if previous == JobState::Unknown {
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','job_reconciled',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"outcome":r.state,"evidence_source":"authenticated_durable_job_result"})).execute(&mut **tx).await?;
}
}
}
_ => anyhow::bail!("not a durable job event"),
}
sqlx::query("INSERT INTO job_receipts VALUES($1,$2,$3,$4)")
.bind(tenant)
.bind(agent)
.bind(message)
.bind(digest)
.execute(&mut **tx)
.await?;
Ok(())
}
async fn locked_job(
tx: &mut Transaction<'_, Postgres>,
agent: Uuid,
job: &str,
attempt: &str,
) -> Result<sqlx::postgres::PgRow> {
Ok(sqlx::query("SELECT state,revision,result,payload,start_reserved FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(agent).fetch_one(&mut **tx).await?)
}
async fn check_grant(
tx: &mut Transaction<'_, Postgres>,
job: &str,
attempt: &str,
grant: &str,
) -> Result<()> {
let ok:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM execution_permits WHERE job_id=$1 AND attempt_id=$2 AND grant_id=$3)").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(Uuid::parse_str(grant)?).fetch_one(&mut **tx).await?;
ensure!(ok, "missing bound grant");
Ok(())
}
pub async fn permit(
state: &crate::State,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
r: &ExecutionPermitRequest,
) -> Result<ExecutionPermit> {
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
// Tenant serial lock covers rollout counters, stop check, and reservations.
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("starts:{tenant}"))
.execute(&mut *tx)
.await?;
crate::agent_service::control_lock(&mut tx, tenant, agent).await?;
let denied = |code: &str| ExecutionPermit {
payload_bytes: vec![],
signature: vec![],
denial_code: code.into(),
};
let row=sqlx::query("SELECT j.*,a.authorization_id,a.revoked_at,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,r.approval_state,i.enrollment_generation,i.assignment_version,i.asset_id AS bound_asset,i.capabilities,i.revoked_at AS device_revoked,i.connection_generation,t.changes_stopped FROM jobs j JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 FOR UPDATE OF j").bind(Uuid::parse_str(&r.job_id)?).bind(Uuid::parse_str(&r.attempt_id)?).bind(agent).fetch_optional(&mut *tx).await?;
let Some(row) = row else {
return Ok(denied("NOT_FOUND"));
};
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("plan_payload"))?;
let capabilities: Value = row.get("capabilities");
if row.get::<Uuid, _>("bound_asset") != payload.asset_id
|| !capabilities
.as_array()
.is_some_and(|c| c.iter().any(|v| v.as_str() == Some(payload.action.name())))
|| hash(&row.get::<Vec<u8>, _>("plan_payload")) != row.get::<String, _>("plan_hash")
|| !plan.jobs.iter().any(|j| {
j.payload.job_id == payload.job_id
&& j.payload.attempt_id == payload.attempt_id
&& j.job_payload_hash == r.job_payload_hash
})
{
return Ok(denied("BINDING_MISMATCH"));
}
if row.get::<bool, _>("start_reserved") {
return Ok(denied("START_ALREADY_RESERVED"));
}
if row.get::<String, _>("state") != "accepted"
|| row.get::<Option<String>, _>("journal_proof").is_none()
{
return Ok(denied("JOURNAL_ACCEPTANCE_REQUIRED"));
}
if row.get::<bool, _>("changes_stopped")
|| row.get::<bool, _>("cancel_requested")
|| row.get::<String, _>("approval_state") != "approved"
|| row
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
.is_some()
|| row
.get::<Option<chrono::DateTime<Utc>>, _>("device_revoked")
.is_some()
|| !crate::jobs::requester_enabled(state, &plan.principal, tenant)
|| !crate::jobs::requester_enabled(state, &row.get::<String, _>("approver"), tenant)
{
return Ok(denied("AUTHORIZATION_REVOKED"));
}
if row.get::<i64, _>("enrollment_generation") as u64 != payload.enrollment_generation
|| row.get::<i64, _>("assignment_version") as u64 != payload.assignment_version
|| r.assignment_version != payload.assignment_version
|| row.get::<i64, _>("connection_generation") as u64 != e.connection_generation
|| r.job_payload_hash != row.get::<String, _>("payload_hash")
|| hash(&encode(&payload)?) != r.job_payload_hash
{
return Ok(denied("BINDING_MISMATCH"));
}
let now = Utc::now();
if !payload.action.validate() {
return Ok(denied("APT_MANIFEST_REQUIRED"));
}
if payload.execution_identity == "root" {
let helper_ready: bool = sqlx::query_scalar(
"SELECT helper_sync_ready FROM agent_installations WHERE agent_id=$1",
)
.bind(agent)
.fetch_one(&mut *tx)
.await?;
if !helper_ready
|| crate::root_policy::authorize(&mut tx, &plan.principal, &payload)
.await
.is_err()
|| crate::root_policy::authorize(&mut tx, &row.get::<String, _>("approver"), &payload)
.await
.is_err()
{
return Ok(denied("ROOT_SCOPE_OR_HELPER_SYNC_REQUIRED"));
}
}
let sync: (bool, bool) = sqlx::query_as(
"SELECT sync_required,sync_ready FROM agent_installations WHERE agent_id=$1",
)
.bind(agent)
.fetch_one(&mut *tx)
.await?;
if sync.0 && !sync.1 {
return Ok(denied("SYNC_REQUIRED"));
}
let restore_blocked: bool =
sqlx::query_scalar("SELECT restore_blocked FROM agent_installations WHERE agent_id=$1")
.bind(agent)
.fetch_one(&mut *tx)
.await?;
if restore_blocked {
return Ok(denied("RESTORE_RECONCILIATION_REQUIRED"));
}
if crate::catalog::check(&mut tx, &payload.action)
.await
.is_err()
{
return Ok(denied("RECIPE_BINDING_MISMATCH"));
}
let references = crate::catalog::references(&mut tx, &payload.action)
.await
.map_err(|_| anyhow::anyhow!("catalog unavailable"))?;
if references
.iter()
.any(|r| !plan.recipe_references.contains(r))
{
return Ok(denied("RECIPE_BINDING_MISMATCH"));
}
let artifact_ids = plan
.artifact_references
.iter()
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
.map(|r| r.artifact.artifact_id)
.collect::<Vec<_>>();
let artifacts = crate::catalog::resolve_artifacts(&mut tx, tenant, &artifact_ids)
.await
.map_err(|_| anyhow::anyhow!("artifact unavailable"))?;
if plan
.artifact_references
.iter()
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
.any(|r| !artifacts.contains(&r.artifact))
{
return Ok(denied("ARTIFACT_BINDING_MISMATCH"));
}
let approver: String = row.get("approver");
if matches!(payload.action, Action::QueryLogs { .. })
&& !crate::permissions::allowed(&mut tx, &plan.principal, "logs:read").await?
{
return Ok(denied("AUTHORIZATION_REVOKED"));
}
if !crate::permissions::allowed(&mut tx, &approver, "plans:approve").await?
|| !crate::permissions::allowed(&mut tx, &plan.principal, "operations:submit").await?
|| (matches!(payload.action, Action::RunScript { .. })
&& !crate::permissions::allowed(&mut tx, &plan.principal, "scripts:execute").await?)
{
return Ok(denied("AUTHORIZATION_REVOKED"));
}
let limits: (i32, i32) =
sqlx::query_as("SELECT starts_per_hour,max_parallel FROM tenant_limits")
.fetch_one(&mut *tx)
.await?;
let hourly: i64 = sqlx::query_scalar(
"SELECT count(*) FROM execution_permits WHERE created_at>now()-interval '1 hour'",
)
.fetch_one(&mut *tx)
.await?;
let concurrent:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired')").fetch_one(&mut *tx).await?;
if hourly >= i64::from(limits.0) || concurrent >= i64::from(limits.1) {
return Ok(denied("QUOTA_EXCEEDED"));
}
if let Action::Reboot {
dependency_group, ..
} = &payload.action
{
let busy:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired') AND (convert_from(payload,'UTF8')::jsonb->'action'->>'dependency_group'=$1 OR convert_from(payload,'UTF8')::jsonb->'action'->>'action'='reboot'))").bind(dependency_group).fetch_one(&mut *tx).await?;
if busy {
return Ok(denied("RESOURCE_LOCKED"));
}
}
if now >= payload.start_before {
return Ok(denied("PLAN_EXPIRED"));
}
if now < payload.not_before {
return Ok(denied("MAINTENANCE_WINDOW"));
}
let failures: i64 = sqlx::query_scalar(
"SELECT count(*) FROM jobs WHERE operation_id=$1 AND state IN ('failed','unknown')",
)
.bind(row.get::<Uuid, _>("operation_id"))
.fetch_one(&mut *tx)
.await?;
let active:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE operation_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(row.get::<Uuid,_>("operation_id")).fetch_one(&mut *tx).await?;
if failures >= i64::from(plan.halt_after_failures) || active >= i64::from(plan.max_parallel) {
return Ok(denied("ROLLOUT_PAUSED"));
}
let locked:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired'))").bind(agent).fetch_one(&mut *tx).await?;
if locked {
return Ok(denied("RESOURCE_LOCKED"));
}
let grant = ExecutionPermitPayload {
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: row.get("authorization_id"),
tenant_id: tenant,
asset_id: payload.asset_id,
agent_id: agent,
enrollment_generation: payload.enrollment_generation,
assignment_version: payload.assignment_version,
job_id: payload.job_id,
attempt_id: payload.attempt_id,
plan_hash: row.get("plan_hash"),
job_payload_hash: r.job_payload_hash.clone(),
action: payload.action.name().into(),
execution_identity: payload.execution_identity,
not_before: now,
start_before: std::cmp::min(now + Duration::seconds(60), payload.start_before),
max_runtime_seconds: payload.max_runtime_seconds,
key_id: state.job_signer.key_id.clone(),
};
let bytes = encode(&grant)?;
let signature = state.job_signer.sign(&bytes)?;
sqlx::query("INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(grant.grant_id).bind(grant.job_id).bind(grant.attempt_id).bind(&bytes).bind(&signature).execute(&mut *tx).await?;
sqlx::query("UPDATE jobs SET start_reserved=true WHERE job_id=$1")
.bind(grant.job_id)
.execute(&mut *tx)
.await?;
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','start_reserved',$3,now())")
.bind(tenant)
.bind(Uuid::new_v4())
.bind(json!({"job_id":r.job_id,"grant_id":grant.grant_id,"boot_id":e.boot_id}))
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(ExecutionPermit {
payload_bytes: bytes,
signature,
denial_code: String::new(),
})
}
pub async fn next(
state: &crate::State,
tenant: Uuid,
agent: Uuid,
generation: u64,
) -> Result<Option<Payload>> {
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
// Never invent nonstart once a grant was reserved. The durable ledger is
// authoritative even when every start packet was lost.
crate::scheduler::maintain(&mut tx, Some(agent)).await?;
let active:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE agent_id=$1 AND connection_generation=$2 AND revoked_at IS NULL AND NOT restore_blocked AND (NOT sync_required OR sync_ready))").bind(agent).bind(i64::try_from(generation)?).fetch_one(&mut *tx).await?;
if !active {
return Ok(None);
}
if let Some(r)=sqlx::query("SELECT job_id,attempt_id FROM jobs WHERE agent_id=$1 AND cancel_requested AND state IN ('accepted','running','verifying','awaiting_reboot') ORDER BY job_id LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?{
tx.commit().await?;return Ok(Some(Payload::CancelRequest(CancelRequest{job_id:r.get::<Uuid,_>("job_id").to_string(),attempt_id:r.get::<Uuid,_>("attempt_id").to_string(),reason:"operator_cancel_requested".into()})))
}
let row=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload,j.payload_hash,a.authorization_id,p.plan_hash FROM outbox b JOIN jobs j USING(tenant_id,job_id,attempt_id) JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.agent_id=$1 AND j.state='queued' AND NOT j.cancel_requested AND NOT t.changes_stopped AND a.revoked_at IS NULL AND r.approval_state='approved' AND p.expires_at>now() AND b.acknowledged_at IS NULL AND (b.lease_until IS NULL OR b.lease_until<now()) ORDER BY j.job_id FOR UPDATE OF b SKIP LOCKED LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?;
let Some(row) = row else {
tx.commit().await?;
return Ok(None);
};
sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1")
.bind(row.get::<Uuid, _>("job_id"))
.execute(&mut *tx)
.await?;
let offer = JobOffer {
job_id: row.get::<Uuid, _>("job_id").to_string(),
attempt_id: row.get::<Uuid, _>("attempt_id").to_string(),
authorization_id: row.get::<Uuid, _>("authorization_id").to_string(),
plan_hash: row.get("plan_hash"),
job_payload_hash: row.get("payload_hash"),
payload_bytes: row.get("payload"),
};
tx.commit().await?;
Ok(Some(Payload::JobOffer(offer)))
}