514 lines
24 KiB
Rust
514 lines
24 KiB
Rust
use anyhow::{Context, Result, ensure};
|
|
use chrono::{Duration, Utc};
|
|
use prost::Message;
|
|
use rmm_domain::jobs::*;
|
|
use rmm_protocol::*;
|
|
use serde_json::{Value, json};
|
|
use sqlx::{Postgres, Row, Transaction};
|
|
use uuid::Uuid;
|
|
|
|
/// Called in the same tenant transaction as durable message receipt and ACK.
|
|
pub async fn event(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
) -> Result<()> {
|
|
let bytes = e.payload.as_ref().context("missing payload")?;
|
|
let mut identity = e.clone();
|
|
identity.connection_generation = 0;
|
|
let digest = hash(&identity.encode_to_vec());
|
|
let message = Uuid::parse_str(&e.message_id)?;
|
|
if let Some(old) =
|
|
sqlx::query_scalar::<_, String>("SELECT payload_hash FROM job_receipts WHERE message_id=$1")
|
|
.bind(message)
|
|
.fetch_optional(&mut **tx)
|
|
.await?
|
|
{
|
|
ensure!(old == digest, "receipt identity conflict");
|
|
return Ok(());
|
|
}
|
|
match bytes {
|
|
Payload::JobAccepted(a) => {
|
|
ensure!(
|
|
!a.journal_proof.is_empty() && a.journal_proof.len() <= 256,
|
|
"journal proof required"
|
|
);
|
|
let row=sqlx::query("SELECT state,payload_hash FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(&a.job_id)?).bind(Uuid::parse_str(&a.attempt_id)?).bind(agent).fetch_one(&mut **tx).await?;
|
|
ensure!(
|
|
row.get::<String, _>("payload_hash") == a.job_payload_hash,
|
|
"payload mismatch"
|
|
);
|
|
let state: String = row.get("state");
|
|
if state == "queued" {
|
|
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=$2 WHERE job_id=$1")
|
|
.bind(Uuid::parse_str(&a.job_id)?)
|
|
.bind(&a.journal_proof)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
}
|
|
sqlx::query(
|
|
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
|
|
)
|
|
.bind(Uuid::parse_str(&a.job_id)?)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
}
|
|
Payload::JobStateChanged(s) => {
|
|
let row = locked_job(tx, agent, &s.job_id, &s.attempt_id).await?;
|
|
let old_revision: i64 = row.get("revision");
|
|
let old: String = row.get("state");
|
|
check_grant(tx, &s.job_id, &s.attempt_id, &s.grant_id).await?;
|
|
let old_state: JobState = serde_json::from_value(json!(old))?;
|
|
let next: JobState = serde_json::from_value(json!(s.state))?;
|
|
ensure!(!next.terminal(), "terminal states require result evidence");
|
|
if old_state.terminal()
|
|
|| old_state == JobState::Unknown
|
|
|| i64::try_from(s.revision)? < old_revision
|
|
{
|
|
// ACK durable but late evidence without regressing the ledger.
|
|
} else if i64::try_from(s.revision)? == old_revision {
|
|
ensure!(old == s.state, "state revision conflict");
|
|
} else {
|
|
// Terminal results require evidence and must use JobResult.
|
|
ensure!(
|
|
!next.terminal() && old_state.allows(next),
|
|
"invalid state transition"
|
|
);
|
|
sqlx::query("UPDATE jobs SET state=$2,revision=$3 WHERE job_id=$1")
|
|
.bind(Uuid::parse_str(&s.job_id)?)
|
|
.bind(&s.state)
|
|
.bind(i64::try_from(s.revision)?)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
}
|
|
}
|
|
Payload::JobResult(r) => {
|
|
ensure!(
|
|
r.evidence_json.len() <= 65536 && r.error_code.len() <= 128,
|
|
"evidence bound"
|
|
);
|
|
let row = locked_job(tx, agent, &r.job_id, &r.attempt_id).await?;
|
|
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
|
ensure!(
|
|
r.stdout.len() + r.stderr.len() <= payload.max_output_bytes as usize,
|
|
"output bound"
|
|
);
|
|
let evidence: Value = serde_json::from_str(&r.evidence_json)?;
|
|
let outcome: JobState = serde_json::from_value(json!(r.state))?;
|
|
ensure!(
|
|
matches!(
|
|
outcome,
|
|
JobState::Succeeded
|
|
| JobState::Failed
|
|
| JobState::Cancelled
|
|
| JobState::Expired
|
|
| JobState::Unknown
|
|
| JobState::AwaitingReboot
|
|
),
|
|
"invalid result state"
|
|
);
|
|
let result = json!({"stdout":r.stdout,"stderr":r.stderr,"output_truncated":r.output_truncated,"reboot_required":r.reboot_required,"exit_code":r.exit_code,"evidence":evidence,"error_code":r.error_code,"grant_id":r.grant_id});
|
|
let old_revision: i64 = row.get("revision");
|
|
if i64::try_from(r.revision)? <= old_revision {
|
|
ensure!(
|
|
row.get::<String, _>("state") == r.state
|
|
&& row.get::<Option<Value>, _>("result").as_ref() == Some(&result),
|
|
"result identity conflict"
|
|
);
|
|
} else {
|
|
let previous: JobState =
|
|
serde_json::from_value(json!(row.get::<String, _>("state")))?;
|
|
let nonstart = !row.get::<bool, _>("start_reserved")
|
|
&& r.grant_id.is_empty()
|
|
&& evidence.get("not_started") == Some(&json!(true));
|
|
ensure!(
|
|
!previous.terminal()
|
|
|| (previous == outcome
|
|
&& nonstart
|
|
&& row.get::<Option<Value>, _>("result").is_none()),
|
|
"terminal outcome cannot be reset"
|
|
);
|
|
if previous == JobState::Unknown {
|
|
ensure!(
|
|
matches!(
|
|
outcome,
|
|
JobState::Succeeded | JobState::Failed | JobState::Cancelled
|
|
),
|
|
"unknown needs definitive reconciliation evidence"
|
|
);
|
|
}
|
|
if r.grant_id.is_empty() {
|
|
ensure!(
|
|
nonstart
|
|
&& matches!(
|
|
previous,
|
|
JobState::Queued
|
|
| JobState::Accepted
|
|
| JobState::Cancelled
|
|
| JobState::Expired
|
|
)
|
|
&& matches!(
|
|
outcome,
|
|
JobState::Cancelled | JobState::Expired | JobState::Failed
|
|
),
|
|
"ungranted outcome needs definite nonstart evidence"
|
|
);
|
|
} else {
|
|
check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?;
|
|
ensure!(
|
|
outcome != JobState::Expired,
|
|
"grant reservation cannot prove expiry without effect"
|
|
);
|
|
}
|
|
if outcome == JobState::Cancelled {
|
|
ensure!(
|
|
evidence.get("not_started") == Some(&json!(true))
|
|
|| evidence.get("abort_confirmed") == Some(&json!(true)),
|
|
"cancellation needs actual nonstart/abort evidence"
|
|
);
|
|
}
|
|
if outcome == JobState::Failed {
|
|
ensure!(
|
|
!r.error_code.is_empty(),
|
|
"known failure requires structured reason"
|
|
);
|
|
}
|
|
if outcome == JobState::Succeeded {
|
|
ensure!(
|
|
evidence.get("postconditions_passed") == Some(&json!(true)),
|
|
"postcondition evidence required"
|
|
);
|
|
if let Action::RunScript {
|
|
expected_stdout, ..
|
|
} = &payload.action
|
|
{
|
|
ensure!(r.exit_code == Some(0), "script exit failure");
|
|
if let Some(expected) = expected_stdout {
|
|
ensure!(r.stdout == expected.as_bytes(), "expected output mismatch");
|
|
}
|
|
}
|
|
if matches!(payload.action, Action::Reboot { .. }) {
|
|
crate::synchronization::verify_reboot_success(
|
|
tx, tenant, agent, e, &payload, r, &evidence,
|
|
)
|
|
.await?;
|
|
}
|
|
}
|
|
sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1")
|
|
.bind(Uuid::parse_str(&r.job_id)?)
|
|
.bind(&r.state)
|
|
.bind(i64::try_from(r.revision)?)
|
|
.bind(result)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
if previous == JobState::Unknown {
|
|
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','job_reconciled',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"outcome":r.state,"evidence_source":"authenticated_durable_job_result"})).execute(&mut **tx).await?;
|
|
}
|
|
}
|
|
}
|
|
_ => anyhow::bail!("not a durable job event"),
|
|
}
|
|
sqlx::query("INSERT INTO job_receipts VALUES($1,$2,$3,$4)")
|
|
.bind(tenant)
|
|
.bind(agent)
|
|
.bind(message)
|
|
.bind(digest)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
async fn locked_job(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
agent: Uuid,
|
|
job: &str,
|
|
attempt: &str,
|
|
) -> Result<sqlx::postgres::PgRow> {
|
|
Ok(sqlx::query("SELECT state,revision,result,payload,start_reserved FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(agent).fetch_one(&mut **tx).await?)
|
|
}
|
|
async fn check_grant(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
job: &str,
|
|
attempt: &str,
|
|
grant: &str,
|
|
) -> Result<()> {
|
|
let ok:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM execution_permits WHERE job_id=$1 AND attempt_id=$2 AND grant_id=$3)").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(Uuid::parse_str(grant)?).fetch_one(&mut **tx).await?;
|
|
ensure!(ok, "missing bound grant");
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn permit(
|
|
state: &crate::State,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
r: &ExecutionPermitRequest,
|
|
) -> Result<ExecutionPermit> {
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
// Tenant serial lock covers rollout counters, stop check, and reservations.
|
|
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
|
.bind(format!("starts:{tenant}"))
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
crate::agent_service::control_lock(&mut tx, tenant, agent).await?;
|
|
let denied = |code: &str| ExecutionPermit {
|
|
payload_bytes: vec![],
|
|
signature: vec![],
|
|
denial_code: code.into(),
|
|
};
|
|
let row=sqlx::query("SELECT j.*,a.authorization_id,a.revoked_at,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,r.approval_state,i.enrollment_generation,i.assignment_version,i.asset_id AS bound_asset,i.capabilities,i.revoked_at AS device_revoked,i.connection_generation,t.changes_stopped FROM jobs j JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 FOR UPDATE OF j").bind(Uuid::parse_str(&r.job_id)?).bind(Uuid::parse_str(&r.attempt_id)?).bind(agent).fetch_optional(&mut *tx).await?;
|
|
let Some(row) = row else {
|
|
return Ok(denied("NOT_FOUND"));
|
|
};
|
|
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
|
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("plan_payload"))?;
|
|
let capabilities: Value = row.get("capabilities");
|
|
if row.get::<Uuid, _>("bound_asset") != payload.asset_id
|
|
|| !capabilities
|
|
.as_array()
|
|
.is_some_and(|c| c.iter().any(|v| v.as_str() == Some(payload.action.name())))
|
|
|| hash(&row.get::<Vec<u8>, _>("plan_payload")) != row.get::<String, _>("plan_hash")
|
|
|| !plan.jobs.iter().any(|j| {
|
|
j.payload.job_id == payload.job_id
|
|
&& j.payload.attempt_id == payload.attempt_id
|
|
&& j.job_payload_hash == r.job_payload_hash
|
|
})
|
|
{
|
|
return Ok(denied("BINDING_MISMATCH"));
|
|
}
|
|
if row.get::<bool, _>("start_reserved") {
|
|
return Ok(denied("START_ALREADY_RESERVED"));
|
|
}
|
|
if row.get::<String, _>("state") != "accepted"
|
|
|| row.get::<Option<String>, _>("journal_proof").is_none()
|
|
{
|
|
return Ok(denied("JOURNAL_ACCEPTANCE_REQUIRED"));
|
|
}
|
|
if row.get::<bool, _>("changes_stopped")
|
|
|| row.get::<bool, _>("cancel_requested")
|
|
|| row.get::<String, _>("approval_state") != "approved"
|
|
|| row
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
|
|
.is_some()
|
|
|| row
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("device_revoked")
|
|
.is_some()
|
|
|| !crate::jobs::requester_enabled(state, &plan.principal, tenant)
|
|
|| !crate::jobs::requester_enabled(state, &row.get::<String, _>("approver"), tenant)
|
|
{
|
|
return Ok(denied("AUTHORIZATION_REVOKED"));
|
|
}
|
|
if row.get::<i64, _>("enrollment_generation") as u64 != payload.enrollment_generation
|
|
|| row.get::<i64, _>("assignment_version") as u64 != payload.assignment_version
|
|
|| r.assignment_version != payload.assignment_version
|
|
|| row.get::<i64, _>("connection_generation") as u64 != e.connection_generation
|
|
|| r.job_payload_hash != row.get::<String, _>("payload_hash")
|
|
|| hash(&encode(&payload)?) != r.job_payload_hash
|
|
{
|
|
return Ok(denied("BINDING_MISMATCH"));
|
|
}
|
|
let now = Utc::now();
|
|
if !payload.action.validate() {
|
|
return Ok(denied("APT_MANIFEST_REQUIRED"));
|
|
}
|
|
if payload.execution_identity == "root" {
|
|
let helper_ready: bool = sqlx::query_scalar(
|
|
"SELECT helper_sync_ready FROM agent_installations WHERE agent_id=$1",
|
|
)
|
|
.bind(agent)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if !helper_ready
|
|
|| crate::root_policy::authorize(&mut tx, &plan.principal, &payload)
|
|
.await
|
|
.is_err()
|
|
|| crate::root_policy::authorize(&mut tx, &row.get::<String, _>("approver"), &payload)
|
|
.await
|
|
.is_err()
|
|
{
|
|
return Ok(denied("ROOT_SCOPE_OR_HELPER_SYNC_REQUIRED"));
|
|
}
|
|
}
|
|
let sync: (bool, bool) = sqlx::query_as(
|
|
"SELECT sync_required,sync_ready FROM agent_installations WHERE agent_id=$1",
|
|
)
|
|
.bind(agent)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if sync.0 && !sync.1 {
|
|
return Ok(denied("SYNC_REQUIRED"));
|
|
}
|
|
let restore_blocked: bool =
|
|
sqlx::query_scalar("SELECT restore_blocked FROM agent_installations WHERE agent_id=$1")
|
|
.bind(agent)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if restore_blocked {
|
|
return Ok(denied("RESTORE_RECONCILIATION_REQUIRED"));
|
|
}
|
|
if crate::catalog::check(&mut tx, &payload.action)
|
|
.await
|
|
.is_err()
|
|
{
|
|
return Ok(denied("RECIPE_BINDING_MISMATCH"));
|
|
}
|
|
let references = crate::catalog::references(&mut tx, &payload.action)
|
|
.await
|
|
.map_err(|_| anyhow::anyhow!("catalog unavailable"))?;
|
|
if references
|
|
.iter()
|
|
.any(|r| !plan.recipe_references.contains(r))
|
|
{
|
|
return Ok(denied("RECIPE_BINDING_MISMATCH"));
|
|
}
|
|
let artifact_ids = plan
|
|
.artifact_references
|
|
.iter()
|
|
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
|
|
.map(|r| r.artifact.artifact_id)
|
|
.collect::<Vec<_>>();
|
|
let artifacts = crate::catalog::resolve_artifacts(&mut tx, tenant, &artifact_ids)
|
|
.await
|
|
.map_err(|_| anyhow::anyhow!("artifact unavailable"))?;
|
|
if plan
|
|
.artifact_references
|
|
.iter()
|
|
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
|
|
.any(|r| !artifacts.contains(&r.artifact))
|
|
{
|
|
return Ok(denied("ARTIFACT_BINDING_MISMATCH"));
|
|
}
|
|
let approver: String = row.get("approver");
|
|
if matches!(payload.action, Action::QueryLogs { .. })
|
|
&& !crate::permissions::allowed(&mut tx, &plan.principal, "logs:read").await?
|
|
{
|
|
return Ok(denied("AUTHORIZATION_REVOKED"));
|
|
}
|
|
if !crate::permissions::allowed(&mut tx, &approver, "plans:approve").await?
|
|
|| !crate::permissions::allowed(&mut tx, &plan.principal, "operations:submit").await?
|
|
|| (matches!(payload.action, Action::RunScript { .. })
|
|
&& !crate::permissions::allowed(&mut tx, &plan.principal, "scripts:execute").await?)
|
|
{
|
|
return Ok(denied("AUTHORIZATION_REVOKED"));
|
|
}
|
|
let limits: (i32, i32) =
|
|
sqlx::query_as("SELECT starts_per_hour,max_parallel FROM tenant_limits")
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
let hourly: i64 = sqlx::query_scalar(
|
|
"SELECT count(*) FROM execution_permits WHERE created_at>now()-interval '1 hour'",
|
|
)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
let concurrent:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired')").fetch_one(&mut *tx).await?;
|
|
if hourly >= i64::from(limits.0) || concurrent >= i64::from(limits.1) {
|
|
return Ok(denied("QUOTA_EXCEEDED"));
|
|
}
|
|
if let Action::Reboot {
|
|
dependency_group, ..
|
|
} = &payload.action
|
|
{
|
|
let busy:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired') AND (convert_from(payload,'UTF8')::jsonb->'action'->>'dependency_group'=$1 OR convert_from(payload,'UTF8')::jsonb->'action'->>'action'='reboot'))").bind(dependency_group).fetch_one(&mut *tx).await?;
|
|
if busy {
|
|
return Ok(denied("RESOURCE_LOCKED"));
|
|
}
|
|
}
|
|
if now >= payload.start_before {
|
|
return Ok(denied("PLAN_EXPIRED"));
|
|
}
|
|
if now < payload.not_before {
|
|
return Ok(denied("MAINTENANCE_WINDOW"));
|
|
}
|
|
let failures: i64 = sqlx::query_scalar(
|
|
"SELECT count(*) FROM jobs WHERE operation_id=$1 AND state IN ('failed','unknown')",
|
|
)
|
|
.bind(row.get::<Uuid, _>("operation_id"))
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
let active:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE operation_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(row.get::<Uuid,_>("operation_id")).fetch_one(&mut *tx).await?;
|
|
if failures >= i64::from(plan.halt_after_failures) || active >= i64::from(plan.max_parallel) {
|
|
return Ok(denied("ROLLOUT_PAUSED"));
|
|
}
|
|
let locked:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired'))").bind(agent).fetch_one(&mut *tx).await?;
|
|
if locked {
|
|
return Ok(denied("RESOURCE_LOCKED"));
|
|
}
|
|
let grant = ExecutionPermitPayload {
|
|
domain: "rmm.execution-permit".into(),
|
|
schema_version: 1,
|
|
grant_id: Uuid::new_v4(),
|
|
authorization_id: row.get("authorization_id"),
|
|
tenant_id: tenant,
|
|
asset_id: payload.asset_id,
|
|
agent_id: agent,
|
|
enrollment_generation: payload.enrollment_generation,
|
|
assignment_version: payload.assignment_version,
|
|
job_id: payload.job_id,
|
|
attempt_id: payload.attempt_id,
|
|
plan_hash: row.get("plan_hash"),
|
|
job_payload_hash: r.job_payload_hash.clone(),
|
|
action: payload.action.name().into(),
|
|
execution_identity: payload.execution_identity,
|
|
not_before: now,
|
|
start_before: std::cmp::min(now + Duration::seconds(60), payload.start_before),
|
|
max_runtime_seconds: payload.max_runtime_seconds,
|
|
key_id: state.job_signer.key_id.clone(),
|
|
};
|
|
let bytes = encode(&grant)?;
|
|
let signature = state.job_signer.sign(&bytes)?;
|
|
sqlx::query("INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(grant.grant_id).bind(grant.job_id).bind(grant.attempt_id).bind(&bytes).bind(&signature).execute(&mut *tx).await?;
|
|
sqlx::query("UPDATE jobs SET start_reserved=true WHERE job_id=$1")
|
|
.bind(grant.job_id)
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','start_reserved',$3,now())")
|
|
.bind(tenant)
|
|
.bind(Uuid::new_v4())
|
|
.bind(json!({"job_id":r.job_id,"grant_id":grant.grant_id,"boot_id":e.boot_id}))
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
tx.commit().await?;
|
|
Ok(ExecutionPermit {
|
|
payload_bytes: bytes,
|
|
signature,
|
|
denial_code: String::new(),
|
|
})
|
|
}
|
|
pub async fn next(
|
|
state: &crate::State,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
generation: u64,
|
|
) -> Result<Option<Payload>> {
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
// Never invent nonstart once a grant was reserved. The durable ledger is
|
|
// authoritative even when every start packet was lost.
|
|
crate::scheduler::maintain(&mut tx, Some(agent)).await?;
|
|
let active:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE agent_id=$1 AND connection_generation=$2 AND revoked_at IS NULL AND NOT restore_blocked AND (NOT sync_required OR sync_ready))").bind(agent).bind(i64::try_from(generation)?).fetch_one(&mut *tx).await?;
|
|
if !active {
|
|
return Ok(None);
|
|
}
|
|
if let Some(r)=sqlx::query("SELECT job_id,attempt_id FROM jobs WHERE agent_id=$1 AND cancel_requested AND state IN ('accepted','running','verifying','awaiting_reboot') ORDER BY job_id LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?{
|
|
tx.commit().await?;return Ok(Some(Payload::CancelRequest(CancelRequest{job_id:r.get::<Uuid,_>("job_id").to_string(),attempt_id:r.get::<Uuid,_>("attempt_id").to_string(),reason:"operator_cancel_requested".into()})))
|
|
}
|
|
let row=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload,j.payload_hash,a.authorization_id,p.plan_hash FROM outbox b JOIN jobs j USING(tenant_id,job_id,attempt_id) JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.agent_id=$1 AND j.state='queued' AND NOT j.cancel_requested AND NOT t.changes_stopped AND a.revoked_at IS NULL AND r.approval_state='approved' AND p.expires_at>now() AND b.acknowledged_at IS NULL AND (b.lease_until IS NULL OR b.lease_until<now()) ORDER BY j.job_id FOR UPDATE OF b SKIP LOCKED LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?;
|
|
let Some(row) = row else {
|
|
tx.commit().await?;
|
|
return Ok(None);
|
|
};
|
|
sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1")
|
|
.bind(row.get::<Uuid, _>("job_id"))
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
let offer = JobOffer {
|
|
job_id: row.get::<Uuid, _>("job_id").to_string(),
|
|
attempt_id: row.get::<Uuid, _>("attempt_id").to_string(),
|
|
authorization_id: row.get::<Uuid, _>("authorization_id").to_string(),
|
|
plan_hash: row.get("plan_hash"),
|
|
job_payload_hash: row.get("payload_hash"),
|
|
payload_bytes: row.get("payload"),
|
|
};
|
|
tx.commit().await?;
|
|
Ok(Some(Payload::JobOffer(offer)))
|
|
}
|