1103 lines
50 KiB
Rust
1103 lines
50 KiB
Rust
use anyhow::{Context, Result, ensure};
|
|
use chrono::{Duration, Utc};
|
|
use prost::Message;
|
|
use rmm_domain::jobs::*;
|
|
use rmm_protocol::*;
|
|
use serde_json::{Value, json};
|
|
use sqlx::{Postgres, Row, Transaction};
|
|
use uuid::Uuid;
|
|
|
|
/// Called in the same tenant transaction as durable message receipt and ACK.
|
|
pub async fn event(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
) -> Result<()> {
|
|
let bytes = e.payload.as_ref().context("missing payload")?;
|
|
let mut identity = e.clone();
|
|
identity.connection_generation = 0;
|
|
let digest = hash(&identity.encode_to_vec());
|
|
let message = Uuid::parse_str(&e.message_id)?;
|
|
if let Some(old) =
|
|
sqlx::query_scalar::<_, String>("SELECT payload_hash FROM job_receipts WHERE message_id=$1")
|
|
.bind(message)
|
|
.fetch_optional(&mut **tx)
|
|
.await?
|
|
{
|
|
ensure!(old == digest, "receipt identity conflict");
|
|
return Ok(());
|
|
}
|
|
match bytes {
|
|
Payload::JobAccepted(a) => {
|
|
ensure!(
|
|
!a.journal_proof.is_empty() && a.journal_proof.len() <= 256,
|
|
"journal proof required"
|
|
);
|
|
let row=sqlx::query("SELECT state,payload_hash FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(&a.job_id)?).bind(Uuid::parse_str(&a.attempt_id)?).bind(agent).fetch_one(&mut **tx).await?;
|
|
ensure!(
|
|
row.get::<String, _>("payload_hash") == a.job_payload_hash,
|
|
"payload mismatch"
|
|
);
|
|
let state: String = row.get("state");
|
|
if state == "queued" {
|
|
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=$2 WHERE job_id=$1")
|
|
.bind(Uuid::parse_str(&a.job_id)?)
|
|
.bind(&a.journal_proof)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
}
|
|
sqlx::query(
|
|
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
|
|
)
|
|
.bind(Uuid::parse_str(&a.job_id)?)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
}
|
|
Payload::JobStateChanged(s) => {
|
|
let row = locked_job(tx, agent, &s.job_id, &s.attempt_id).await?;
|
|
let old_revision: i64 = row.get("revision");
|
|
let old: String = row.get("state");
|
|
check_grant(tx, &s.job_id, &s.attempt_id, &s.grant_id).await?;
|
|
let old_state: JobState = serde_json::from_value(json!(old))?;
|
|
let next: JobState = serde_json::from_value(json!(s.state))?;
|
|
ensure!(!next.terminal(), "terminal states require result evidence");
|
|
if old_state.terminal()
|
|
|| old_state == JobState::Unknown
|
|
|| i64::try_from(s.revision)? < old_revision
|
|
{
|
|
// ACK durable but late evidence without regressing the ledger.
|
|
} else if i64::try_from(s.revision)? == old_revision {
|
|
ensure!(old == s.state, "state revision conflict");
|
|
} else {
|
|
// Terminal results require evidence and must use JobResult.
|
|
ensure!(
|
|
!next.terminal() && old_state.allows(next),
|
|
"invalid state transition"
|
|
);
|
|
sqlx::query("UPDATE jobs SET state=$2,revision=$3 WHERE job_id=$1")
|
|
.bind(Uuid::parse_str(&s.job_id)?)
|
|
.bind(&s.state)
|
|
.bind(i64::try_from(s.revision)?)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
}
|
|
}
|
|
Payload::JobResult(r) => {
|
|
ensure!(
|
|
r.evidence_json.len() <= 65536 && r.error_code.len() <= 128,
|
|
"evidence bound"
|
|
);
|
|
let row = locked_job(tx, agent, &r.job_id, &r.attempt_id).await?;
|
|
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
|
ensure!(
|
|
r.stdout.len() + r.stderr.len() <= payload.max_output_bytes as usize,
|
|
"output bound"
|
|
);
|
|
let evidence: Value = serde_json::from_str(&r.evidence_json)?;
|
|
let outcome: JobState = serde_json::from_value(json!(r.state))?;
|
|
ensure!(
|
|
matches!(
|
|
outcome,
|
|
JobState::Succeeded
|
|
| JobState::Failed
|
|
| JobState::Cancelled
|
|
| JobState::Expired
|
|
| JobState::Unknown
|
|
| JobState::AwaitingReboot
|
|
),
|
|
"invalid result state"
|
|
);
|
|
let result = json!({"stdout":r.stdout,"stderr":r.stderr,"output_truncated":r.output_truncated,"reboot_required":r.reboot_required,"exit_code":r.exit_code,"evidence":evidence,"error_code":r.error_code,"grant_id":r.grant_id});
|
|
let old_revision: i64 = row.get("revision");
|
|
if i64::try_from(r.revision)? < old_revision
|
|
&& payload.execution_identity == "system"
|
|
&& row.get::<bool, _>("start_reserved")
|
|
&& superseded_delegation_placeholder(r, &evidence)
|
|
{
|
|
// Sync may import the helper's authoritative record before the
|
|
// agent's earlier content-free delegation placeholder arrives.
|
|
// ACK it durably without regressing the ledger; any other stale
|
|
// evidence must still match exactly.
|
|
check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?;
|
|
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','superseded_delegation_placeholder_acknowledged',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"placeholder_revision":r.revision,"current_revision":old_revision,"current_state":row.get::<String, _>("state"),"message_id":e.message_id})).execute(&mut **tx).await?;
|
|
} else if i64::try_from(r.revision)? <= old_revision {
|
|
ensure!(
|
|
row.get::<String, _>("state") == r.state
|
|
&& row.get::<Option<Value>, _>("result").as_ref() == Some(&result),
|
|
"result identity conflict"
|
|
);
|
|
} else {
|
|
let previous: JobState =
|
|
serde_json::from_value(json!(row.get::<String, _>("state")))?;
|
|
let nonstart = !row.get::<bool, _>("start_reserved")
|
|
&& r.grant_id.is_empty()
|
|
&& evidence.get("not_started") == Some(&json!(true));
|
|
ensure!(
|
|
!previous.terminal()
|
|
|| (previous == outcome
|
|
&& nonstart
|
|
&& row.get::<Option<Value>, _>("result").is_none()),
|
|
"terminal outcome cannot be reset"
|
|
);
|
|
if previous == JobState::Unknown {
|
|
ensure!(
|
|
matches!(
|
|
outcome,
|
|
JobState::Succeeded
|
|
| JobState::Failed
|
|
| JobState::Cancelled
|
|
| JobState::Unknown
|
|
),
|
|
"unknown needs bound unknown or definitive reconciliation evidence"
|
|
);
|
|
}
|
|
if r.grant_id.is_empty() {
|
|
ensure!(
|
|
nonstart
|
|
&& matches!(
|
|
previous,
|
|
JobState::Queued
|
|
| JobState::Accepted
|
|
| JobState::Cancelled
|
|
| JobState::Expired
|
|
)
|
|
&& matches!(
|
|
outcome,
|
|
JobState::Cancelled | JobState::Expired | JobState::Failed
|
|
),
|
|
"ungranted outcome needs definite nonstart evidence"
|
|
);
|
|
} else {
|
|
check_grant(tx, &r.job_id, &r.attempt_id, &r.grant_id).await?;
|
|
ensure!(
|
|
outcome != JobState::Expired,
|
|
"grant reservation cannot prove expiry without effect"
|
|
);
|
|
}
|
|
if outcome == JobState::Cancelled {
|
|
ensure!(
|
|
evidence.get("not_started") == Some(&json!(true))
|
|
|| evidence.get("abort_confirmed") == Some(&json!(true)),
|
|
"cancellation needs actual nonstart/abort evidence"
|
|
);
|
|
}
|
|
if outcome == JobState::Failed {
|
|
ensure!(
|
|
!r.error_code.is_empty(),
|
|
"known failure requires structured reason"
|
|
);
|
|
}
|
|
if r.error_code == rmm_domain::windows_system_nonstart::ERROR_CODE {
|
|
crate::windows_system_nonstart::validate_result(tx, tenant, agent, r, &evidence).await?;
|
|
}
|
|
if outcome == JobState::Succeeded {
|
|
ensure!(
|
|
evidence.get("postconditions_passed") == Some(&json!(true)),
|
|
"postcondition evidence required"
|
|
);
|
|
if let Action::RunScript {
|
|
expected_stdout, ..
|
|
} = &payload.action
|
|
{
|
|
ensure!(r.exit_code == Some(0), "script exit failure");
|
|
if let Some(expected) = expected_stdout {
|
|
ensure!(r.stdout == expected.as_bytes(), "expected output mismatch");
|
|
}
|
|
}
|
|
script_profile_evidence(&payload.action, &evidence)?;
|
|
windows_event_log_success_evidence(&payload, r, &evidence)?;
|
|
if matches!(payload.action, Action::Reboot { .. }) {
|
|
crate::synchronization::verify_reboot_success(
|
|
tx, tenant, agent, e, &payload, r, &evidence,
|
|
)
|
|
.await?;
|
|
}
|
|
}
|
|
sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1")
|
|
.bind(Uuid::parse_str(&r.job_id)?)
|
|
.bind(&r.state)
|
|
.bind(i64::try_from(r.revision)?)
|
|
.bind(result)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
// A newly persisted unknown result may follow a Sync decision
|
|
// that already marked the attempt unknown. Receipt/ACK records
|
|
// that evidence; it does not reconcile or release the attempt.
|
|
if previous == JobState::Unknown && outcome != JobState::Unknown {
|
|
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','job_reconciled',$3,now())").bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":r.job_id,"attempt_id":r.attempt_id,"grant_id":r.grant_id,"outcome":r.state,"evidence_source":"authenticated_durable_job_result"})).execute(&mut **tx).await?;
|
|
}
|
|
}
|
|
}
|
|
_ => anyhow::bail!("not a durable job event"),
|
|
}
|
|
sqlx::query("INSERT INTO job_receipts VALUES($1,$2,$3,$4)")
|
|
.bind(tenant)
|
|
.bind(agent)
|
|
.bind(message)
|
|
.bind(digest)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
/// Exact agent-side SYSTEM delegation placeholder: unknown, no output, no
|
|
/// exit code and no evidence beyond the failed postcondition marker.
|
|
fn superseded_delegation_placeholder(r: &JobResult, evidence: &Value) -> bool {
|
|
r.state == "unknown"
|
|
&& r.error_code == "SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"
|
|
&& !r.grant_id.is_empty()
|
|
&& r.stdout.is_empty()
|
|
&& r.stderr.is_empty()
|
|
&& r.exit_code.is_none()
|
|
&& !r.output_truncated
|
|
&& !r.reboot_required
|
|
&& *evidence == json!({"postconditions_passed": false})
|
|
}
|
|
fn windows_event_log_success_evidence(
|
|
job: &JobPayload,
|
|
result: &JobResult,
|
|
evidence: &Value,
|
|
) -> Result<()> {
|
|
if !job.action.is_windows_event_log_fixture() {
|
|
return Ok(());
|
|
}
|
|
let Action::QueryLogs {
|
|
unit,
|
|
since,
|
|
until,
|
|
max_lines,
|
|
} = &job.action
|
|
else {
|
|
anyhow::bail!("fixed event log action required")
|
|
};
|
|
let observed_since: chrono::DateTime<Utc> = serde_json::from_value(
|
|
evidence
|
|
.get("start_utc")
|
|
.cloned()
|
|
.context("event log start missing")?,
|
|
)?;
|
|
let observed_until: chrono::DateTime<Utc> = serde_json::from_value(
|
|
evidence
|
|
.get("end_utc")
|
|
.cloned()
|
|
.context("event log end missing")?,
|
|
)?;
|
|
let complete = evidence.get("completeness") == Some(&json!("complete"))
|
|
&& evidence.get("truncated") == Some(&json!(false))
|
|
&& evidence.get("error_code") == Some(&Value::Null);
|
|
let partial = evidence.get("completeness") == Some(&json!("partial"))
|
|
&& evidence.get("truncated") == Some(&json!(true))
|
|
&& matches!(
|
|
evidence.get("error_code").and_then(Value::as_str),
|
|
Some("EVENT_LOG_EVENT_LIMIT" | "EVENT_LOG_BYTE_LIMIT" | "EVENT_LOG_EVENT_SIZE_LIMIT")
|
|
);
|
|
ensure!(
|
|
!result.reboot_required
|
|
&& evidence.get("truncated") == Some(&json!(result.output_truncated))
|
|
&& job.execution_identity == "agent_service"
|
|
&& job.action.validate()
|
|
&& job
|
|
.action
|
|
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
|
|
&& evidence.get("source_id") == Some(&json!(unit))
|
|
&& evidence.get("channel") == Some(&json!(WINDOWS_EVENT_LOG_FIXTURE_CHANNEL))
|
|
&& evidence.get("provider") == Some(&json!(WINDOWS_EVENT_LOG_FIXTURE_PROVIDER))
|
|
&& observed_since == *since
|
|
&& observed_until == *until
|
|
&& evidence.get("max_events") == Some(&json!(max_lines))
|
|
&& evidence.get("max_output_bytes") == Some(&json!(job.max_output_bytes))
|
|
&& evidence.get("quality") == Some(&json!("valid"))
|
|
&& evidence.get("win32_error") == Some(&Value::Null)
|
|
&& evidence.get("bookmark_updated") == Some(&json!(false))
|
|
&& evidence.get("process_tree_bounded") == Some(&json!(true))
|
|
&& evidence.get("execution_sid") == Some(&json!("S-1-5-19"))
|
|
&& (complete || partial),
|
|
"fixed Windows Event Log evidence mismatch"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
// The profile records the requested process option, never an effective GPO policy.
|
|
fn script_profile_evidence(action: &Action, evidence: &Value) -> Result<()> {
|
|
if action.full_maintenance() {
|
|
let Action::RunScript { script_utf8, .. } = action else {
|
|
unreachable!()
|
|
};
|
|
ensure!(
|
|
evidence["privileged_script_profile"] == "full_maintenance_v1"
|
|
&& evidence["script_sha256"] == hash(script_utf8.as_bytes())
|
|
&& match action.script_platform() {
|
|
Some("linux") => evidence["execution_uid"] == 0,
|
|
Some("windows") => evidence["execution_sid"] == "S-1-5-18",
|
|
_ => false,
|
|
},
|
|
"full maintenance identity/profile evidence mismatch"
|
|
);
|
|
}
|
|
if let Action::RunScript {
|
|
script_utf8,
|
|
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
|
|
..
|
|
} = action
|
|
{
|
|
ensure!(
|
|
evidence.get("windows_execution_profile")
|
|
== Some(&json!("powershell_51_remote_signed"))
|
|
&& evidence.get("execution_policy")
|
|
== Some(&json!("process_remote_signed_requested"))
|
|
&& evidence.get("execution_policy_requested") == Some(&json!("RemoteSigned"))
|
|
&& evidence.get("execution_policy_scope") == Some(&json!("process"))
|
|
&& evidence.get("script_sha256") == Some(&json!(hash(script_utf8.as_bytes()))),
|
|
"requested Windows process profile evidence mismatch"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod script_profile_tests {
|
|
use super::*;
|
|
#[test]
|
|
fn windows_event_log_success_binds_request_and_keeps_limit_partial_honest() {
|
|
let id = Uuid::new_v4();
|
|
let now = Utc::now();
|
|
let job = JobPayload {
|
|
schema_version: 1,
|
|
tenant_id: id,
|
|
asset_id: id,
|
|
agent_id: id,
|
|
enrollment_generation: 1,
|
|
assignment_version: 1,
|
|
job_id: id,
|
|
attempt_id: id,
|
|
action: Action::QueryLogs {
|
|
unit: WINDOWS_EVENT_LOG_FIXTURE_SOURCE.into(),
|
|
since: now - Duration::hours(1),
|
|
until: now,
|
|
max_lines: 1000,
|
|
},
|
|
execution_identity: "agent_service".into(),
|
|
not_before: now,
|
|
start_before: now + Duration::minutes(1),
|
|
max_runtime_seconds: 30,
|
|
max_output_bytes: 4096,
|
|
};
|
|
let evidence = json!({"source_id":WINDOWS_EVENT_LOG_FIXTURE_SOURCE,"channel":"Application","provider":"RustDiscovery-MVP-Test","start_utc":now-Duration::hours(1),"end_utc":now,"max_events":1000,"max_output_bytes":4096,"quality":"valid","completeness":"complete","truncated":false,"error_code":null,"win32_error":null,"bookmark_updated":false,"process_tree_bounded":true,"execution_sid":"S-1-5-19"});
|
|
let result = JobResult::default();
|
|
windows_event_log_success_evidence(&job, &result, &evidence).unwrap();
|
|
for conflicting in [
|
|
JobResult {
|
|
reboot_required: true,
|
|
..Default::default()
|
|
},
|
|
JobResult {
|
|
output_truncated: true,
|
|
..Default::default()
|
|
},
|
|
] {
|
|
assert!(windows_event_log_success_evidence(&job, &conflicting, &evidence).is_err());
|
|
}
|
|
for field in [
|
|
"source_id",
|
|
"channel",
|
|
"provider",
|
|
"start_utc",
|
|
"end_utc",
|
|
"max_events",
|
|
"max_output_bytes",
|
|
"quality",
|
|
"win32_error",
|
|
"bookmark_updated",
|
|
"process_tree_bounded",
|
|
"execution_sid",
|
|
] {
|
|
let mut bad = evidence.clone();
|
|
bad[field] = json!("different");
|
|
assert!(
|
|
windows_event_log_success_evidence(&job, &result, &bad).is_err(),
|
|
"{field}"
|
|
);
|
|
}
|
|
for code in [
|
|
"EVENT_LOG_EVENT_LIMIT",
|
|
"EVENT_LOG_BYTE_LIMIT",
|
|
"EVENT_LOG_EVENT_SIZE_LIMIT",
|
|
] {
|
|
let mut partial = evidence.clone();
|
|
partial["completeness"] = json!("partial");
|
|
partial["truncated"] = json!(true);
|
|
partial["error_code"] = json!(code);
|
|
let partial_result = JobResult {
|
|
output_truncated: true,
|
|
..Default::default()
|
|
};
|
|
windows_event_log_success_evidence(&job, &partial_result, &partial).unwrap();
|
|
assert!(windows_event_log_success_evidence(&job, &result, &partial).is_err());
|
|
partial["error_code"] = json!("EVENT_LOG_TIMEOUT");
|
|
assert!(windows_event_log_success_evidence(&job, &partial_result, &partial).is_err());
|
|
partial["quality"] = json!("permission_denied");
|
|
assert!(windows_event_log_success_evidence(&job, &partial_result, &partial).is_err());
|
|
}
|
|
let mut permission_denied = evidence.clone();
|
|
permission_denied["win32_error"] = json!(5);
|
|
assert!(windows_event_log_success_evidence(&job, &result, &permission_denied).is_err());
|
|
let mut false_complete = evidence.clone();
|
|
false_complete["truncated"] = json!(true);
|
|
assert!(windows_event_log_success_evidence(&job, &result, &false_complete).is_err());
|
|
}
|
|
#[test]
|
|
fn requested_profile_success_evidence_is_exact_without_effective_policy_claim() {
|
|
let mut action = Action::RunScript {
|
|
interpreter: "powershell".into(),
|
|
script_utf8: "Write-Output 'hi'\r\n".into(),
|
|
arguments: vec![],
|
|
expected_stdout: None,
|
|
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
|
|
privileged_script_profile: None,
|
|
};
|
|
let evidence = json!({"windows_execution_profile":"powershell_51_remote_signed",
|
|
"execution_policy":"process_remote_signed_requested", "execution_policy_requested":"RemoteSigned",
|
|
"execution_policy_scope":"process", "script_sha256":hash(b"Write-Output 'hi'\r\n")});
|
|
script_profile_evidence(&action, &evidence).unwrap();
|
|
for field in [
|
|
"windows_execution_profile",
|
|
"execution_policy",
|
|
"execution_policy_requested",
|
|
"execution_policy_scope",
|
|
"script_sha256",
|
|
] {
|
|
let mut changed = evidence.clone();
|
|
changed.as_object_mut().unwrap().remove(field);
|
|
assert!(
|
|
script_profile_evidence(&action, &changed).is_err(),
|
|
"{field}"
|
|
);
|
|
changed[field] = json!("different");
|
|
assert!(
|
|
script_profile_evidence(&action, &changed).is_err(),
|
|
"{field}"
|
|
);
|
|
}
|
|
if let Action::RunScript {
|
|
windows_execution_profile,
|
|
..
|
|
} = &mut action
|
|
{
|
|
*windows_execution_profile = None;
|
|
}
|
|
script_profile_evidence(&action, &json!({})).unwrap();
|
|
}
|
|
}
|
|
|
|
async fn locked_job(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
agent: Uuid,
|
|
job: &str,
|
|
attempt: &str,
|
|
) -> Result<sqlx::postgres::PgRow> {
|
|
Ok(sqlx::query("SELECT state,revision,result,payload,start_reserved FROM jobs WHERE job_id=$1 AND attempt_id=$2 AND agent_id=$3 FOR UPDATE").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(agent).fetch_one(&mut **tx).await?)
|
|
}
|
|
async fn check_grant(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
job: &str,
|
|
attempt: &str,
|
|
grant: &str,
|
|
) -> Result<()> {
|
|
let ok:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM execution_permits WHERE job_id=$1 AND attempt_id=$2 AND grant_id=$3)").bind(Uuid::parse_str(job)?).bind(Uuid::parse_str(attempt)?).bind(Uuid::parse_str(grant)?).fetch_one(&mut **tx).await?;
|
|
ensure!(ok, "missing bound grant");
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn permit(
|
|
state: &crate::State,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
r: &ExecutionPermitRequest,
|
|
) -> Result<ExecutionPermit> {
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
// Tenant serial lock covers rollout counters, stop check, and reservations.
|
|
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
|
.bind(format!("starts:{tenant}"))
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
crate::agent_service::control_lock(&mut tx, tenant, agent).await?;
|
|
let denied = |code: &str| ExecutionPermit {
|
|
payload_bytes: vec![],
|
|
signature: vec![],
|
|
denial_code: code.into(),
|
|
unreserved_request: None,
|
|
};
|
|
let row=sqlx::query("SELECT j.*,a.authorization_id,a.revoked_at,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,r.approval_state,i.enrollment_generation,i.assignment_version,i.asset_id AS bound_asset,i.capabilities,i.revoked_at AS device_revoked,i.connection_generation,t.changes_stopped FROM jobs j JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id JOIN tenants t ON t.tenant_id=j.tenant_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 FOR UPDATE OF j").bind(Uuid::parse_str(&r.job_id)?).bind(Uuid::parse_str(&r.attempt_id)?).bind(agent).fetch_optional(&mut *tx).await?;
|
|
let Some(row) = row else {
|
|
return Ok(denied("NOT_FOUND"));
|
|
};
|
|
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
|
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("plan_payload"))?;
|
|
let capabilities: Vec<String> = serde_json::from_value(row.get("capabilities"))?;
|
|
if crate::windows_msi_catalog::validate_job_binding(&plan, &payload).is_err()
|
|
|| (matches!(payload.action, Action::UpdateAgent { .. })
|
|
&& crate::agent_updates::validate_job_binding(&plan, &payload).is_err())
|
|
{
|
|
return Ok(denied("WINDOWS_MSI_BINDING_MISMATCH"));
|
|
}
|
|
if row.get::<Uuid, _>("bound_asset") != payload.asset_id
|
|
|| !payload
|
|
.action
|
|
.available_for(&payload.execution_identity, &capabilities)
|
|
|| hash(&row.get::<Vec<u8>, _>("plan_payload")) != row.get::<String, _>("plan_hash")
|
|
|| !plan.jobs.iter().any(|j| {
|
|
j.payload.job_id == payload.job_id
|
|
&& j.payload.attempt_id == payload.attempt_id
|
|
&& j.job_payload_hash == r.job_payload_hash
|
|
})
|
|
{
|
|
return Ok(denied("BINDING_MISMATCH"));
|
|
}
|
|
if row.get::<bool, _>("start_reserved") {
|
|
return Ok(denied("START_ALREADY_RESERVED"));
|
|
}
|
|
if row.get::<String, _>("state") != "accepted"
|
|
|| row.get::<Option<String>, _>("journal_proof").is_none()
|
|
{
|
|
return Ok(denied("JOURNAL_ACCEPTANCE_REQUIRED"));
|
|
}
|
|
if row.get::<bool, _>("changes_stopped")
|
|
|| row.get::<bool, _>("cancel_requested")
|
|
|| row.get::<String, _>("approval_state") != "approved"
|
|
|| row
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
|
|
.is_some()
|
|
|| row
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("device_revoked")
|
|
.is_some()
|
|
|| !crate::jobs::requester_enabled(state, &mut tx, &plan.principal, tenant).await?
|
|
|| !crate::jobs::requester_enabled(state, &mut tx, &row.get::<String, _>("approver"), tenant).await?
|
|
{
|
|
return Ok(denied("AUTHORIZATION_REVOKED"));
|
|
}
|
|
if row.get::<i64, _>("enrollment_generation") as u64 != payload.enrollment_generation
|
|
|| row.get::<i64, _>("assignment_version") as u64 != payload.assignment_version
|
|
|| r.assignment_version != payload.assignment_version
|
|
|| row.get::<i64, _>("connection_generation") as u64 != e.connection_generation
|
|
|| r.job_payload_hash != row.get::<String, _>("payload_hash")
|
|
|| hash(&encode(&payload)?) != r.job_payload_hash
|
|
{
|
|
return Ok(denied("BINDING_MISMATCH"));
|
|
}
|
|
// The locked row belongs to this request and has no start reservation.
|
|
// Echo the binding only after both checks, so a client can safely return
|
|
// this particular offer to scheduling after a fresh signed reconciliation.
|
|
let denied = |code: &str| ExecutionPermit {
|
|
unreserved_request: Some(r.clone()),
|
|
..denied(code)
|
|
};
|
|
let now = Utc::now();
|
|
if !payload
|
|
.action
|
|
.valid_job_limits(payload.max_runtime_seconds, payload.max_output_bytes)
|
|
{
|
|
return Ok(denied("INVALID_JOB_LIMITS"));
|
|
}
|
|
if !payload.action.validate() {
|
|
return Ok(denied("APT_MANIFEST_REQUIRED"));
|
|
}
|
|
if crate::root_policy::needs_helper(&payload.execution_identity) {
|
|
let helper_ready: bool = sqlx::query_scalar(
|
|
"SELECT i.helper_sync_ready AND EXISTS(SELECT 1 FROM agent_sync_state s WHERE s.tenant_id=i.tenant_id AND s.agent_id=i.agent_id AND s.helper_journal_id IS NOT NULL) FROM agent_installations i WHERE i.agent_id=$1",
|
|
)
|
|
.bind(agent)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if !helper_ready
|
|
|| crate::root_policy::authorize(&mut tx, &plan.principal, &payload)
|
|
.await
|
|
.is_err()
|
|
|| crate::root_policy::authorize(&mut tx, &row.get::<String, _>("approver"), &payload)
|
|
.await
|
|
.is_err()
|
|
{
|
|
return Ok(denied(if payload.execution_identity == "system" {
|
|
"SYSTEM_SCOPE_OR_HELPER_SYNC_REQUIRED"
|
|
} else {
|
|
"ROOT_SCOPE_OR_HELPER_SYNC_REQUIRED"
|
|
}));
|
|
}
|
|
}
|
|
let sync: (bool, bool) = sqlx::query_as(
|
|
"SELECT sync_required,sync_ready FROM agent_installations WHERE agent_id=$1",
|
|
)
|
|
.bind(agent)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if sync.0 && !sync.1 {
|
|
return Ok(denied("SYNC_REQUIRED"));
|
|
}
|
|
let restore_blocked: bool =
|
|
sqlx::query_scalar("SELECT restore_blocked FROM agent_installations WHERE agent_id=$1")
|
|
.bind(agent)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if restore_blocked {
|
|
return Ok(denied("RESTORE_RECONCILIATION_REQUIRED"));
|
|
}
|
|
if crate::catalog::check(&mut tx, &payload.action)
|
|
.await
|
|
.is_err()
|
|
{
|
|
return Ok(denied("RECIPE_BINDING_MISMATCH"));
|
|
}
|
|
let references = crate::catalog::references(&mut tx, &payload.action)
|
|
.await
|
|
.map_err(|_| anyhow::anyhow!("catalog unavailable"))?;
|
|
if references
|
|
.iter()
|
|
.any(|r| !plan.recipe_references.contains(r))
|
|
{
|
|
return Ok(denied("RECIPE_BINDING_MISMATCH"));
|
|
}
|
|
let artifact_ids = plan
|
|
.artifact_references
|
|
.iter()
|
|
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
|
|
.map(|r| r.artifact.artifact_id)
|
|
.collect::<Vec<_>>();
|
|
let artifacts = if let Action::UpdateAgent { update } = &payload.action {
|
|
crate::agent_updates::resolve_artifacts(
|
|
&mut tx,
|
|
tenant,
|
|
&update
|
|
.manifest()
|
|
.map_err(|_| anyhow::anyhow!("invalid agent release"))?
|
|
.artifacts,
|
|
)
|
|
.await
|
|
} else {
|
|
crate::catalog::resolve_artifacts(&mut tx, tenant, &artifact_ids).await
|
|
}
|
|
.map_err(|_| anyhow::anyhow!("artifact unavailable"))?;
|
|
if plan
|
|
.artifact_references
|
|
.iter()
|
|
.filter(|r| r.job_id == payload.job_id && r.attempt_id == payload.attempt_id)
|
|
.any(|r| !artifacts.contains(&r.artifact))
|
|
{
|
|
return Ok(denied("ARTIFACT_BINDING_MISMATCH"));
|
|
}
|
|
let approver: String = row.get("approver");
|
|
if matches!(payload.action, Action::QueryLogs { .. })
|
|
&& !crate::permissions::allowed(&mut tx, &plan.principal, "logs:read").await?
|
|
{
|
|
return Ok(denied("AUTHORIZATION_REVOKED"));
|
|
}
|
|
if !crate::permissions::allowed(&mut tx, &approver, "plans:approve").await?
|
|
|| !crate::permissions::allowed(&mut tx, &plan.principal, "operations:submit").await?
|
|
|| (matches!(payload.action, Action::RunScript { .. })
|
|
&& (!crate::permissions::allowed(&mut tx, &plan.principal, "scripts:execute").await?
|
|
|| !crate::permissions::allowed(&mut tx, &approver, "scripts:execute").await?))
|
|
|| (matches!(payload.action, Action::UpdateAgent { .. })
|
|
&& (!crate::permissions::allowed(&mut tx, &plan.principal, "updates:apply").await?
|
|
|| !crate::permissions::allowed(&mut tx, &approver, "updates:apply").await?))
|
|
{
|
|
return Ok(denied("AUTHORIZATION_REVOKED"));
|
|
}
|
|
let limits: (i32, i32) =
|
|
sqlx::query_as("SELECT starts_per_hour,max_parallel FROM tenant_limits")
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
let hourly: i64 = sqlx::query_scalar(
|
|
"SELECT count(*) FROM execution_permits WHERE created_at>now()-interval '1 hour'",
|
|
)
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
let concurrent:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").fetch_one(&mut *tx).await?;
|
|
if hourly >= i64::from(limits.0) || concurrent >= i64::from(limits.1) {
|
|
return Ok(denied("QUOTA_EXCEEDED"));
|
|
}
|
|
if let Action::Reboot {
|
|
dependency_group, ..
|
|
} = &payload.action
|
|
{
|
|
let busy:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown') AND (convert_from(payload,'UTF8')::jsonb->'action'->>'dependency_group'=$1 OR convert_from(payload,'UTF8')::jsonb->'action'->>'action'='reboot'))").bind(dependency_group).fetch_one(&mut *tx).await?;
|
|
if busy {
|
|
return Ok(denied("RESOURCE_LOCKED"));
|
|
}
|
|
}
|
|
if now >= payload.start_before {
|
|
return Ok(denied("PLAN_EXPIRED"));
|
|
}
|
|
if now < payload.not_before {
|
|
return Ok(denied("MAINTENANCE_WINDOW"));
|
|
}
|
|
let failures: i64 = sqlx::query_scalar(
|
|
"SELECT count(*) FROM jobs WHERE operation_id=$1 AND state IN ('failed','unknown')",
|
|
)
|
|
.bind(row.get::<Uuid, _>("operation_id"))
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
let active:i64=sqlx::query_scalar("SELECT count(*) FROM jobs WHERE operation_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").bind(row.get::<Uuid,_>("operation_id")).fetch_one(&mut *tx).await?;
|
|
if failures >= i64::from(plan.halt_after_failures) || active >= i64::from(plan.max_parallel) {
|
|
return Ok(denied("ROLLOUT_PAUSED"));
|
|
}
|
|
let locked:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))").bind(agent).fetch_one(&mut *tx).await?;
|
|
if locked {
|
|
return Ok(denied("RESOURCE_LOCKED"));
|
|
}
|
|
let grant = ExecutionPermitPayload {
|
|
agent_update_plan_id: matches!(payload.action, Action::UpdateAgent { .. })
|
|
.then_some(plan.plan_id),
|
|
domain: "rmm.execution-permit".into(),
|
|
schema_version: 1,
|
|
grant_id: Uuid::new_v4(),
|
|
authorization_id: row.get("authorization_id"),
|
|
tenant_id: tenant,
|
|
asset_id: payload.asset_id,
|
|
agent_id: agent,
|
|
enrollment_generation: payload.enrollment_generation,
|
|
assignment_version: payload.assignment_version,
|
|
job_id: payload.job_id,
|
|
attempt_id: payload.attempt_id,
|
|
plan_hash: row.get("plan_hash"),
|
|
job_payload_hash: r.job_payload_hash.clone(),
|
|
action: payload.action.name().into(),
|
|
execution_identity: payload.execution_identity,
|
|
not_before: now,
|
|
start_before: std::cmp::min(now + Duration::seconds(60), payload.start_before),
|
|
max_runtime_seconds: payload.max_runtime_seconds,
|
|
key_id: state.job_signer.key_id.clone(),
|
|
};
|
|
let bytes = encode(&grant)?;
|
|
let signature = state.job_signer.sign(&bytes)?;
|
|
sqlx::query("INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(grant.grant_id).bind(grant.job_id).bind(grant.attempt_id).bind(&bytes).bind(&signature).execute(&mut *tx).await?;
|
|
sqlx::query("UPDATE jobs SET start_reserved=true WHERE job_id=$1")
|
|
.bind(grant.job_id)
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','start_reserved',$3,now())")
|
|
.bind(tenant)
|
|
.bind(Uuid::new_v4())
|
|
.bind(json!({"job_id":r.job_id,"grant_id":grant.grant_id,"boot_id":e.boot_id}))
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
tx.commit().await?;
|
|
Ok(ExecutionPermit {
|
|
payload_bytes: bytes,
|
|
signature,
|
|
denial_code: String::new(),
|
|
unreserved_request: None,
|
|
})
|
|
}
|
|
pub async fn next(
|
|
state: &crate::State,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
generation: u64,
|
|
) -> Result<Option<Payload>> {
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
// Never invent nonstart once a grant was reserved. The durable ledger is
|
|
// authoritative even when every start packet was lost.
|
|
crate::scheduler::maintain(&mut tx, Some(agent)).await?;
|
|
let active:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE agent_id=$1 AND connection_generation=$2 AND revoked_at IS NULL AND NOT restore_blocked AND (NOT sync_required OR sync_ready))").bind(agent).bind(i64::try_from(generation)?).fetch_one(&mut *tx).await?;
|
|
if !active {
|
|
return Ok(None);
|
|
}
|
|
if let Some(r)=sqlx::query("SELECT job_id,attempt_id FROM jobs WHERE agent_id=$1 AND cancel_requested AND state IN ('accepted','running','verifying','awaiting_reboot') ORDER BY job_id LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?{
|
|
tx.commit().await?;return Ok(Some(Payload::CancelRequest(CancelRequest{job_id:r.get::<Uuid,_>("job_id").to_string(),attempt_id:r.get::<Uuid,_>("attempt_id").to_string(),reason:"operator_cancel_requested".into()})))
|
|
}
|
|
let row=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload,j.payload_hash,a.authorization_id,a.principal AS approver,p.plan_hash,p.payload AS plan_payload,i.capabilities FROM outbox b JOIN jobs j USING(tenant_id,job_id,attempt_id) JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id JOIN approval_requests r ON r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id JOIN operation_plans p ON p.tenant_id=j.tenant_id AND p.plan_id=r.plan_id JOIN tenants t ON t.tenant_id=j.tenant_id JOIN agent_installations i ON i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id WHERE j.agent_id=$1 AND j.state='queued' AND NOT j.cancel_requested AND NOT t.changes_stopped AND a.revoked_at IS NULL AND r.approval_state='approved' AND p.expires_at>now() AND b.acknowledged_at IS NULL AND (b.lease_until IS NULL OR b.lease_until<now()) ORDER BY j.job_id FOR UPDATE OF b SKIP LOCKED LIMIT 1").bind(agent).fetch_optional(&mut *tx).await?;
|
|
let Some(row) = row else {
|
|
tx.commit().await?;
|
|
return Ok(None);
|
|
};
|
|
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
|
let caps: Vec<String> = serde_json::from_value(row.get("capabilities"))?;
|
|
if payload.action.windows_msi().is_some()
|
|
|| matches!(payload.action, Action::UpdateAgent { .. })
|
|
{
|
|
let raw: Vec<u8> = row.get("plan_payload");
|
|
let plan: PlanPayload = serde_json::from_slice(&raw)?;
|
|
if hash(&raw) != row.get::<String, _>("plan_hash")
|
|
|| crate::windows_msi_catalog::validate_job_binding(&plan, &payload).is_err()
|
|
|| (matches!(payload.action, Action::UpdateAgent { .. })
|
|
&& crate::agent_updates::validate_job_binding(&plan, &payload).is_err())
|
|
|| crate::catalog::check(&mut tx, &payload.action)
|
|
.await
|
|
.is_err()
|
|
{
|
|
tx.commit().await?;
|
|
return Ok(None);
|
|
}
|
|
}
|
|
|
|
if !payload
|
|
.action
|
|
.available_for(&payload.execution_identity, &caps)
|
|
|| !payload
|
|
.action
|
|
.valid_job_limits(payload.max_runtime_seconds, payload.max_output_bytes)
|
|
{
|
|
tx.commit().await?;
|
|
return Ok(None);
|
|
}
|
|
if crate::root_policy::needs_helper(&payload.execution_identity) {
|
|
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("plan_payload"))?;
|
|
if crate::root_policy::authorize(&mut tx, &plan.principal, &payload)
|
|
.await
|
|
.is_err()
|
|
|| crate::root_policy::authorize(&mut tx, &row.get::<String, _>("approver"), &payload)
|
|
.await
|
|
.is_err()
|
|
{
|
|
tx.commit().await?;
|
|
return Ok(None);
|
|
}
|
|
}
|
|
sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1")
|
|
.bind(row.get::<Uuid, _>("job_id"))
|
|
.execute(&mut *tx)
|
|
.await?;
|
|
let offer = JobOffer {
|
|
job_id: row.get::<Uuid, _>("job_id").to_string(),
|
|
attempt_id: row.get::<Uuid, _>("attempt_id").to_string(),
|
|
authorization_id: row.get::<Uuid, _>("authorization_id").to_string(),
|
|
plan_hash: row.get("plan_hash"),
|
|
job_payload_hash: row.get("payload_hash"),
|
|
payload_bytes: row.get("payload"),
|
|
};
|
|
tx.commit().await?;
|
|
Ok(Some(Payload::JobOffer(offer)))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn only_the_exact_delegation_placeholder_is_superseded() {
|
|
let placeholder = JobResult {
|
|
job_id: Uuid::new_v4().to_string(),
|
|
attempt_id: Uuid::new_v4().to_string(),
|
|
grant_id: Uuid::new_v4().to_string(),
|
|
revision: 1,
|
|
state: "unknown".into(),
|
|
error_code: "SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT".into(),
|
|
evidence_json: json!({"postconditions_passed": false}).to_string(),
|
|
..Default::default()
|
|
};
|
|
let evidence = json!({"postconditions_passed": false});
|
|
assert!(superseded_delegation_placeholder(&placeholder, &evidence));
|
|
for case in 0..8 {
|
|
let mut other = placeholder.clone();
|
|
let mut other_evidence = evidence.clone();
|
|
match case {
|
|
0 => other.state = "failed".into(),
|
|
1 => other.error_code = "SYSTEM_SCRIPT_EFFECT_UNRESOLVED".into(),
|
|
2 => other.grant_id.clear(),
|
|
3 => other.stdout = b"x".to_vec(),
|
|
4 => other.exit_code = Some(0),
|
|
5 => other.output_truncated = true,
|
|
6 => other.reboot_required = true,
|
|
_ => other_evidence = json!({"postconditions_passed": true}),
|
|
}
|
|
assert!(!superseded_delegation_placeholder(&other, &other_evidence));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires dedicated isolated PostgreSQL with all migrations"]
|
|
async fn unknown_followup_result_is_durable_without_reconciliation_or_new_grant() {
|
|
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
|
|
.await
|
|
.unwrap();
|
|
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
|
|
.await
|
|
.unwrap();
|
|
rmm_persistence::verify_application_role(&app)
|
|
.await
|
|
.unwrap();
|
|
let tenant = Uuid::new_v4();
|
|
let agent = Uuid::new_v4();
|
|
let asset = Uuid::new_v4();
|
|
let job = Uuid::new_v4();
|
|
let attempt = Uuid::new_v4();
|
|
let grant = Uuid::new_v4();
|
|
let plan = Uuid::new_v4();
|
|
let operation = Uuid::new_v4();
|
|
let authorization = Uuid::new_v4();
|
|
let now = Utc::now();
|
|
let payload = JobPayload {
|
|
schema_version: 1,
|
|
tenant_id: tenant,
|
|
asset_id: asset,
|
|
agent_id: agent,
|
|
enrollment_generation: 1,
|
|
assignment_version: 1,
|
|
job_id: job,
|
|
attempt_id: attempt,
|
|
action: Action::RefreshInventory {
|
|
scope: "linux.base".into(),
|
|
},
|
|
execution_identity: "agent_service".into(),
|
|
not_before: now,
|
|
start_before: now + Duration::minutes(5),
|
|
max_runtime_seconds: 30,
|
|
max_output_bytes: 65536,
|
|
};
|
|
let bytes = serde_json::to_vec(&payload).unwrap();
|
|
sqlx::query("INSERT INTO tenants VALUES($1,'isolated unknown-result test')")
|
|
.bind(tenant)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO assets VALUES($1,$2)")
|
|
.bind(tenant)
|
|
.bind(asset)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
|
|
.bind(tenant)
|
|
.bind(operation)
|
|
.bind(authorization)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
// This is an isolated ledger fixture, not an approval or executable job
|
|
// created through a deployed application. Sync already marked it unknown.
|
|
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,revision,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',2,true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)")
|
|
.bind(tenant)
|
|
.bind(grant)
|
|
.bind(job)
|
|
.bind(attempt)
|
|
.bind(b"original-bound-grant".as_slice())
|
|
.bind(vec![0u8; 64])
|
|
.bind(now)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
let mut result = JobResult {
|
|
job_id: job.to_string(),
|
|
attempt_id: attempt.to_string(),
|
|
grant_id: grant.to_string(),
|
|
revision: 3,
|
|
state: "unknown".into(),
|
|
error_code: "AGENT_PROCESS_INTERRUPTED".into(),
|
|
evidence_json: json!({"effect_complete":false,"postconditions_passed":false})
|
|
.to_string(),
|
|
..Default::default()
|
|
};
|
|
let mut envelope = Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: agent.to_string(),
|
|
connection_generation: 1,
|
|
payload: Some(Payload::JobResult(result.clone())),
|
|
..Default::default()
|
|
};
|
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
|
event(&mut tx, tenant, agent, &envelope).await.unwrap();
|
|
tx.commit().await.unwrap();
|
|
// The exact durable message is ACKable after reconnect without another
|
|
// receipt or any reinterpretation of its still-unknown effect.
|
|
envelope.connection_generation = 2;
|
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
|
event(&mut tx, tenant, agent, &envelope).await.unwrap();
|
|
tx.commit().await.unwrap();
|
|
result.revision = 4;
|
|
result.error_code = "EFFECT_STILL_UNRESOLVED".into();
|
|
envelope.message_id = Uuid::new_v4().to_string();
|
|
envelope.payload = Some(Payload::JobResult(result.clone()));
|
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
|
event(&mut tx, tenant, agent, &envelope).await.unwrap();
|
|
tx.commit().await.unwrap();
|
|
for case in 0..7 {
|
|
let mut invalid = result.clone();
|
|
invalid.revision = 5;
|
|
let mut target_tenant = tenant;
|
|
let mut target_agent = agent;
|
|
match case {
|
|
0 => invalid.grant_id.clear(),
|
|
1 => invalid.grant_id = Uuid::new_v4().to_string(),
|
|
2 => invalid.attempt_id = Uuid::new_v4().to_string(),
|
|
3 => {
|
|
invalid.revision = 3;
|
|
invalid.error_code = "CONFLICTING_STALE_EVIDENCE".into();
|
|
}
|
|
4 => invalid.state = "awaiting_reboot".into(),
|
|
5 => target_tenant = Uuid::new_v4(),
|
|
_ => target_agent = Uuid::new_v4(),
|
|
}
|
|
let mut invalid_envelope = envelope.clone();
|
|
invalid_envelope.message_id = Uuid::new_v4().to_string();
|
|
invalid_envelope.payload = Some(Payload::JobResult(invalid));
|
|
let mut tx = rmm_persistence::tenant_tx(&app, target_tenant)
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
event(&mut tx, target_tenant, target_agent, &invalid_envelope)
|
|
.await
|
|
.is_err(),
|
|
"negative case {case}"
|
|
);
|
|
tx.rollback().await.unwrap();
|
|
}
|
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
|
let row = locked_job(&mut tx, agent, &job.to_string(), &attempt.to_string())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(row.get::<String, _>("state"), "unknown");
|
|
assert_eq!(row.get::<i64, _>("revision"), 4);
|
|
assert!(row.get::<bool, _>("start_reserved"));
|
|
assert_eq!(row.get::<Vec<u8>, _>("payload"), bytes);
|
|
let stored = row.get::<Value, _>("result");
|
|
assert_eq!(stored["grant_id"], grant.to_string());
|
|
assert_eq!(stored["error_code"], "EFFECT_STILL_UNRESOLVED");
|
|
assert_eq!(stored["evidence"]["postconditions_passed"], false);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
2
|
|
);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM execution_permits")
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
1
|
|
);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, Vec<u8>>(
|
|
"SELECT payload FROM execution_permits WHERE grant_id=$1"
|
|
)
|
|
.bind(grant)
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
b"original-bound-grant"
|
|
);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM outbox")
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
0
|
|
);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, i64>(
|
|
"SELECT count(*) FROM audit_events WHERE event_type='job_reconciled'"
|
|
)
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
0
|
|
);
|
|
tx.commit().await.unwrap();
|
|
}
|
|
}
|