feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s

This commit is contained in:
BuTzZ
2026-11-12 22:06:25 +01:00
commit 6c3203d725
194 changed files with 49234 additions and 0 deletions
+923
View File
@@ -0,0 +1,923 @@
//! Shared application services. HTTP/MCP adapters only establish a Principal and
//! delegate here. No transport-specific approval shortcut exists.
use crate::auth::Principal;
use axum::{
Json, Router,
extract::{Path, State},
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
routing::{get, post},
};
use chrono::{DateTime, Utc};
use rmm_domain::jobs::*;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sqlx::{Postgres, Row, Transaction};
use uuid::Uuid;
#[derive(Debug)]
pub struct Error(pub StatusCode, pub &'static str);
impl From<sqlx::Error> for Error {
fn from(_: sqlx::Error) -> Self {
Self(StatusCode::SERVICE_UNAVAILABLE, "DEPENDENCY_UNAVAILABLE")
}
}
impl From<anyhow::Error> for Error {
fn from(_: anyhow::Error) -> Self {
Self(StatusCode::SERVICE_UNAVAILABLE, "DEPENDENCY_UNAVAILABLE")
}
}
impl From<serde_json::Error> for Error {
fn from(_: serde_json::Error) -> Self {
Self(StatusCode::BAD_REQUEST, "VALIDATION_ERROR")
}
}
impl IntoResponse for Error {
fn into_response(self) -> Response {
(self.0,Json(json!({"code":self.1,"message":"operation could not be completed","retryable":self.0.is_server_error(),"correlation_id":Uuid::new_v4(),"details":{}}))).into_response()
}
}
type Result<T> = std::result::Result<T, Error>;
fn fail(code: &'static str) -> Error {
Error(StatusCode::CONFLICT, code)
}
fn missing() -> Error {
Error(StatusCode::NOT_FOUND, "NOT_FOUND")
}
pub fn scope(p: &Principal, tenant: Uuid, write: bool) -> Result<()> {
if p.tenant != tenant {
return Err(missing());
}
if write && !p.operator {
return Err(Error(StatusCode::FORBIDDEN, "FORBIDDEN"));
}
Ok(())
}
async fn principal(state: &crate::State, h: &HeaderMap, write: bool) -> Result<Principal> {
state
.auth
.principal(state, h, write)
.await
.map_err(|s| Error(s, crate::auth::error_code(s)))
}
pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
r.route("/api/v1/tenants/{tenant}/operation-plans", post(plan_http))
.route(
"/api/v1/tenants/{tenant}/operation-plans/{plan}",
get(plan_get_http),
)
.route(
"/api/v1/tenants/{tenant}/operation-plans/{plan}/submit",
post(submit_http),
)
.route(
"/api/v1/tenants/{tenant}/operation-plans/{plan}/authorizations",
post(authorize_http),
)
.route("/api/v1/tenants/{tenant}/operations", get(operations_http))
.route(
"/api/v1/tenants/{tenant}/operations/{operation}",
get(operation_http),
)
.route("/api/v1/tenants/{tenant}/jobs/{job}", get(job_http))
.route(
"/api/v1/tenants/{tenant}/jobs/{job}/cancel",
post(cancel_http),
)
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Targets {
pub asset_ids: Vec<Uuid>,
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Schedule {
pub not_before: DateTime<Utc>,
pub start_before: DateTime<Utc>,
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Rollout {
pub max_parallel: u32,
pub halt_after_failures: u32,
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct PlanInput {
pub targets: Targets,
pub action: Action,
pub execution_identity: String,
pub schedule: Schedule,
pub max_runtime_seconds: u32,
pub max_output_bytes: u32,
pub rollout: Rollout,
pub reason: String,
#[serde(default)]
pub artifact_ids: Vec<Uuid>,
}
#[derive(Clone, Deserialize, schemars::JsonSchema)]
#[serde(untagged)]
pub enum PlanRequest {
Typed(Box<PlanInput>),
Web(Box<WebPlanInput>),
}
#[derive(Clone, Deserialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct WebPlanInput {
tenant_id: Uuid,
action: String,
targets: Targets,
parameters: WebParameters,
schedule: WebSchedule,
rollout: Rollout,
reason: String,
#[serde(default)]
artifact_ids: Vec<Uuid>,
}
#[derive(Clone, Deserialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
struct WebSchedule {
not_before: DateTime<Utc>,
start_before: DateTime<Utc>,
offline_policy: String,
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
struct Variant {
os_family: String,
interpreter: String,
encoding: String,
script_utf8: String,
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
struct WebParameters {
execution_identity: String,
timeout_seconds: u32,
max_output_bytes: u32,
variants: Option<Vec<Variant>>,
arguments: Option<Vec<String>>,
expected_stdout: Option<String>,
scope: Option<String>,
unit: Option<String>,
state: Option<String>,
since: Option<DateTime<Utc>>,
until: Option<DateTime<Utc>>,
max_lines: Option<u32>,
provider: Option<String>,
package: Option<String>,
version: Option<String>,
architecture: Option<String>,
recipe_version: Option<String>,
recipe_id: Option<String>,
apt_manifest: Option<rmm_domain::apt_transaction::AptExecutionManifest>,
installation_context: Option<String>,
packages: Option<Vec<String>>,
reboot_policy: Option<String>,
reason: Option<String>,
dependency_group: Option<String>,
reconnect_timeout_seconds: Option<u32>,
}
impl PlanRequest {
pub fn resolve(self, tenant: Uuid) -> Result<PlanInput> {
let w = match self {
Self::Typed(t) => return Ok(*t),
Self::Web(w) => *w,
};
if w.tenant_id != tenant {
return Err(missing());
}
if w.schedule.offline_policy != "wait_until_deadline" {
return Err(fail("POLICY_DENIED"));
}
let parameters = serde_json::to_value(&w.parameters).map_err(Error::from)?;
if w.action != "run_script" && w.parameters.variants.is_some() {
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
let p = w.parameters;
let required =
|value: Option<String>| value.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
let action = match w.action.as_str() {
"run_script" => {
let mut variants = p
.variants
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?;
if variants.len() != 1 {
return Err(fail("AMBIGUOUS_TARGET"));
}
let v = variants.remove(0);
if v.os_family != "linux" || v.encoding != "utf-8" {
return Err(fail("UNSUPPORTED_CAPABILITY"));
}
Action::RunScript {
interpreter: v.interpreter,
script_utf8: v.script_utf8,
arguments: p.arguments.unwrap_or_default(),
expected_stdout: p.expected_stdout,
}
}
"refresh_inventory" => Action::RefreshInventory {
scope: required(p.scope)?,
},
"query_logs" => Action::QueryLogs {
unit: required(p.unit)?,
since: p
.since
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
until: p
.until
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
max_lines: p
.max_lines
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
},
"set_service_state" => Action::SetServiceState {
unit: required(p.unit)?,
state: required(p.state)?,
},
"ensure_package" => Action::EnsurePackage {
provider: required(p.provider)?,
package: required(p.package)?,
version: required(p.version)?,
architecture: required(p.architecture)?,
recipe_version: required(p.recipe_version)?,
installation_context: required(p.installation_context)?,
recipe_id: p.recipe_id,
apt_manifest: p.apt_manifest,
},
"apply_updates" => Action::ApplyUpdates {
provider: required(p.provider)?,
packages: p
.packages
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
reboot_policy: required(p.reboot_policy)?,
apt_manifest: p.apt_manifest,
},
"apt_native_acceptance" => Action::AptNativeAcceptance {
recipe_id: required(p.recipe_id)?,
recipe_version: required(p.recipe_version)?,
apt_manifest: p
.apt_manifest
.ok_or(Error(StatusCode::BAD_REQUEST, "APT_MANIFEST_REQUIRED"))?,
},
"reboot" => Action::Reboot {
reason: required(p.reason)?,
dependency_group: required(p.dependency_group)?,
reconnect_timeout_seconds: p
.reconnect_timeout_seconds
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
},
_ => return Err(fail("UNSUPPORTED_CAPABILITY")),
};
// Reject irrelevant non-null fields instead of silently discarding an
// effect that the browser displayed. Common limits bind into JobPayload.
let action_view = serde_json::to_value(&action)?;
for (key, value) in parameters.as_object().into_iter().flatten() {
if !value.is_null()
&& !matches!(
key.as_str(),
"execution_identity" | "timeout_seconds" | "max_output_bytes" | "variants"
)
&& action_view.get(key).is_none()
{
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
}
Ok(PlanInput {
targets: w.targets,
action,
execution_identity: p.execution_identity,
schedule: Schedule {
not_before: w.schedule.not_before,
start_before: w.schedule.start_before,
},
max_runtime_seconds: p.timeout_seconds,
max_output_bytes: p.max_output_bytes,
rollout: w.rollout,
reason: w.reason,
artifact_ids: w.artifact_ids,
})
}
}
async fn policy(tx: &mut Transaction<'_, Postgres>) -> Result<()> {
let stopped: bool = sqlx::query_scalar("SELECT changes_stopped FROM tenants")
.fetch_one(&mut **tx)
.await?;
if stopped {
return Err(fail("POLICY_DENIED"));
}
Ok(())
}
pub async fn plan(
state: &crate::State,
p: &Principal,
tenant: Uuid,
input: PlanInput,
) -> Result<Value> {
scope(p, tenant, true)?;
let now = Utc::now();
if input.targets.asset_ids.is_empty()
|| input.targets.asset_ids.len() > 100
|| !input.action.validate_plan_shape()
|| !matches!(input.execution_identity.as_str(), "agent_service" | "root")
|| !(1..=3600).contains(&input.max_runtime_seconds)
|| input.max_output_bytes == 0
|| input.max_output_bytes > 262144
|| input.schedule.start_before <= now
|| input.schedule.not_before >= input.schedule.start_before
|| input.schedule.start_before > now + chrono::Duration::days(7)
|| input.reason.is_empty()
|| input.reason.len() > 2048
|| !(1..=10).contains(&input.rollout.max_parallel)
|| input.rollout.halt_after_failures == 0
{
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
if matches!(
input.action,
Action::AptNativeAcceptance { .. }
| Action::EnsurePackage {
apt_manifest: Some(_),
..
}
| Action::ApplyUpdates {
apt_manifest: Some(_),
..
}
) && input.execution_identity != "root"
{
return Err(Error(StatusCode::FORBIDDEN, "ROOT_IDENTITY_REQUIRED"));
}
// Conservative policy: reboot is always serial, no provider-triggered reboot.
if matches!(input.action, Action::Reboot { .. }) && input.rollout.max_parallel != 1 {
return Err(fail("POLICY_DENIED"));
}
let mut ids = input.targets.asset_ids;
ids.sort();
ids.dedup();
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
policy(&mut tx).await?;
crate::permissions::require_principal(&mut tx, p, "operations:plan").await?;
if input.execution_identity == "root" && !p.permits_scope("root:execute") {
return Err(Error(StatusCode::FORBIDDEN, "FORBIDDEN"));
}
if matches!(input.action, Action::AptNativeAcceptance { .. }) {
crate::permissions::require_principal(&mut tx, p, "apt:native_acceptance").await?;
}
if matches!(input.action, Action::RunScript { .. }) {
crate::permissions::require_principal(&mut tx, p, "scripts:execute").await?;
}
if matches!(input.action, Action::QueryLogs { .. }) {
crate::permissions::require_principal(&mut tx, p, "logs:read").await?;
}
let recipe_references = crate::catalog::references(&mut tx, &input.action).await?;
let artifacts = crate::catalog::resolve_artifacts(&mut tx, tenant, &input.artifact_ids).await?;
let mut jobs = Vec::new();
for asset in ids {
let rows=sqlx::query("SELECT agent_id,enrollment_generation,assignment_version,capabilities FROM agent_installations WHERE asset_id=$1 AND revoked_at IS NULL ORDER BY agent_id FOR SHARE").bind(asset).fetch_all(&mut *tx).await?;
if rows.len() != 1 {
return Err(fail(if rows.is_empty() {
"NOT_FOUND"
} else {
"AMBIGUOUS_TARGET"
}));
}
let row = &rows[0];
let caps: Value = row.get("capabilities");
if !caps
.as_array()
.is_some_and(|c| c.iter().any(|v| v.as_str() == Some(input.action.name())))
{
return Err(fail("UNSUPPORTED_CAPABILITY"));
}
let payload = JobPayload {
schema_version: 1,
tenant_id: tenant,
asset_id: asset,
agent_id: row.get("agent_id"),
enrollment_generation: row.get::<i64, _>("enrollment_generation") as u64,
assignment_version: row.get::<i64, _>("assignment_version") as u64,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: input.action.clone(),
execution_identity: input.execution_identity.clone(),
not_before: input.schedule.not_before,
start_before: input.schedule.start_before,
max_runtime_seconds: input.max_runtime_seconds,
max_output_bytes: input.max_output_bytes,
};
let script_sha256 = if let Action::RunScript { script_utf8, .. } = &payload.action {
Some(hash(script_utf8.as_bytes()))
} else {
None
};
crate::root_policy::authorize(&mut tx, &p.name, &payload).await?;
if encode(&payload)?.len() > 65536 {
return Err(Error(StatusCode::PAYLOAD_TOO_LARGE, "QUOTA_EXCEEDED"));
}
jobs.push(PlannedJob {
job_payload_hash: hash(&encode(&payload)?),
payload,
script_sha256,
});
}
let artifact_references = jobs
.iter()
.flat_map(|job| {
artifacts.iter().map(|artifact| JobArtifactReference {
job_id: job.payload.job_id,
attempt_id: job.payload.attempt_id,
artifact: artifact.clone(),
})
})
.collect();
let payload = PlanPayload {
schema_version: 1,
plan_id: Uuid::new_v4(),
tenant_id: tenant,
principal: p.name.clone(),
reason: input.reason,
jobs,
max_parallel: input.rollout.max_parallel,
halt_after_failures: input.rollout.halt_after_failures,
recipe_references,
artifact_references,
};
let bytes = encode(&payload)?;
let plan_hash = hash(&bytes);
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(payload.plan_id).bind(&p.name).bind(&bytes).bind(&plan_hash).bind(input.schedule.start_before).execute(&mut *tx).await?;
tx.commit().await?;
plan_view(&payload, &plan_hash, None, &bytes)
}
fn plan_view(
plan: &PlanPayload,
hash: &str,
approval: Option<String>,
bytes: &[u8],
) -> Result<Value> {
let jobs=plan.jobs.iter().map(|j|json!({"job_id":j.payload.job_id,"attempt_id":j.payload.attempt_id,"asset_id":j.payload.asset_id,"agent_id":j.payload.agent_id,"job_payload_hash":j.job_payload_hash,"script_sha256":j.script_sha256,"payload":j.payload})).collect::<Vec<_>>();
Ok(
json!({"plan_id":plan.plan_id,"plan_hash":hash,"payload":plan,"plan_payload":plan,"plan_payload_bytes":bytes,"resolved_targets":plan.jobs.iter().map(|j|json!({"asset_id":j.payload.asset_id,"agent_id":j.payload.agent_id})).collect::<Vec<_>>(),"excluded_targets":[],"jobs":jobs,"authorization_status":if plan.jobs.iter().all(|j|j.payload.action.validate()){"required"}else{"unsupported_manifest_required"},"approval_state":approval,"expires_at":plan.jobs.first().map(|j|j.payload.start_before)}),
)
}
async fn load_plan(
tx: &mut Transaction<'_, Postgres>,
id: Uuid,
) -> Result<(PlanPayload, String, Vec<u8>)> {
let r = sqlx::query("SELECT payload,plan_hash FROM operation_plans WHERE plan_id=$1")
.bind(id)
.fetch_optional(&mut **tx)
.await?
.ok_or_else(missing)?;
let bytes: Vec<u8> = r.get("payload");
let h: String = r.get("plan_hash");
if hash(&bytes) != h {
return Err(fail("PLAN_HASH_MISMATCH"));
}
Ok((serde_json::from_slice(&bytes)?, h, bytes))
}
async fn recheck(tx: &mut Transaction<'_, Postgres>, plan: &PlanPayload) -> Result<()> {
policy(tx).await?;
let current_artifacts = crate::catalog::resolve_artifacts(
tx,
plan.tenant_id,
&plan
.artifact_references
.iter()
.map(|r| r.artifact.artifact_id)
.collect::<Vec<_>>(),
)
.await?;
for reference in &plan.artifact_references {
if !current_artifacts.contains(&reference.artifact)
|| !plan.jobs.iter().any(|j| {
j.payload.job_id == reference.job_id && j.payload.attempt_id == reference.attempt_id
})
{
return Err(fail("ARTIFACT_BINDING_MISMATCH"));
}
}
crate::permissions::require(tx, &plan.principal, "operations:plan").await?;
if plan
.jobs
.iter()
.any(|j| matches!(j.payload.action, Action::RunScript { .. }))
{
crate::permissions::require(tx, &plan.principal, "scripts:execute").await?;
}
if plan
.jobs
.iter()
.any(|j| matches!(j.payload.action, Action::QueryLogs { .. }))
{
crate::permissions::require(tx, &plan.principal, "logs:read").await?;
}
let mut references = Vec::new();
for job in &plan.jobs {
if !job.payload.action.validate() {
return Err(Error(StatusCode::CONFLICT, "APT_MANIFEST_REQUIRED"));
}
crate::root_policy::authorize(tx, &plan.principal, &job.payload).await?;
crate::catalog::check(tx, &job.payload.action).await?;
references.extend(crate::catalog::references(tx, &job.payload.action).await?);
if job.payload.start_before <= Utc::now() {
return Err(fail("PLAN_EXPIRED"));
}
let r=sqlx::query("SELECT enrollment_generation,assignment_version,capabilities FROM agent_installations WHERE agent_id=$1 AND asset_id=$2 AND revoked_at IS NULL FOR SHARE").bind(job.payload.agent_id).bind(job.payload.asset_id).fetch_optional(&mut **tx).await?.ok_or_else(||fail("AUTHORIZATION_REVOKED"))?;
if r.get::<i64, _>("assignment_version") as u64 != job.payload.assignment_version
|| r.get::<i64, _>("enrollment_generation") as u64 != job.payload.enrollment_generation
{
return Err(fail("AUTHORIZATION_REVOKED"));
}
let caps: Value = r.get("capabilities");
if !caps.as_array().is_some_and(|c| {
c.iter()
.any(|v| v.as_str() == Some(job.payload.action.name()))
}) {
return Err(fail("UNSUPPORTED_CAPABILITY"));
}
if hash(&encode(&job.payload)?) != job.job_payload_hash {
return Err(fail("PLAN_HASH_MISMATCH"));
}
}
references.sort();
references.dedup();
if references != plan.recipe_references {
return Err(fail("RECIPE_BINDING_MISMATCH"));
}
Ok(())
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Submit {
pub idempotency_key: String,
}
pub async fn submit(
state: &crate::State,
p: &Principal,
tenant: Uuid,
id: Uuid,
input: Submit,
) -> Result<Value> {
scope(p, tenant, true)?;
if input.idempotency_key.is_empty() || input.idempotency_key.len() > 128 {
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
// Serialize idempotency and same-plan submit, including competing principals.
crate::permissions::require_principal(&mut tx, p, "operations:submit").await?;
crate::permissions::require_principal(&mut tx, p, "operations:read").await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!(
"submit:{tenant}:{}:{}",
p.name, input.idempotency_key
))
.execute(&mut *tx)
.await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("plan-submit:{tenant}:{id}"))
.execute(&mut *tx)
.await?;
sqlx::query("SELECT plan_id FROM operation_plans WHERE plan_id=$1")
.bind(id)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(missing)?;
let request_hash = hash(id.as_bytes());
let (plan, _, _) = load_plan(&mut tx, id).await?;
for job in &plan.jobs {
if matches!(job.payload.action, Action::RunScript { .. }) {
crate::permissions::require_principal(&mut tx, p, "scripts:execute").await?;
}
if matches!(job.payload.action, Action::QueryLogs { .. }) {
crate::permissions::require_principal(&mut tx, p, "logs:read").await?;
}
if job.payload.execution_identity == "root" {
crate::permissions::require_principal(&mut tx, p, "root:execute").await?;
}
if matches!(job.payload.action, Action::AptNativeAcceptance { .. }) {
crate::permissions::require_principal(&mut tx, p, "apt:native_acceptance").await?;
}
}
if let Some(row)=sqlx::query("SELECT request_hash,operation_id FROM operation_submissions WHERE principal=$1 AND idempotency_key=$2").bind(&p.name).bind(&input.idempotency_key).fetch_optional(&mut *tx).await? {
if row.get::<String,_>("request_hash")!=request_hash{return Err(fail("IDEMPOTENCY_CONFLICT"))}
let op=row.get("operation_id");tx.commit().await?;return operation(state,p,tenant,op).await;
}
if let Some(op) =
sqlx::query_scalar::<_, Uuid>("SELECT operation_id FROM approval_requests WHERE plan_id=$1")
.bind(id)
.fetch_optional(&mut *tx)
.await?
{
sqlx::query("INSERT INTO operation_submissions VALUES($1,$2,$3,$4,$5)")
.bind(tenant)
.bind(&p.name)
.bind(&input.idempotency_key)
.bind(&request_hash)
.bind(op)
.execute(&mut *tx)
.await?;
tx.commit().await?;
return operation(state, p, tenant, op).await;
}
recheck(&mut tx, &plan).await?;
let op = Uuid::new_v4();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(op).bind(id).bind(&p.name).bind(&input.idempotency_key).bind(&request_hash).execute(&mut *tx).await?;
sqlx::query("INSERT INTO operation_submissions VALUES($1,$2,$3,$4,$5)")
.bind(tenant)
.bind(&p.name)
.bind(&input.idempotency_key)
.bind(&request_hash)
.bind(op)
.execute(&mut *tx)
.await?;
tx.commit().await?;
operation(state, p, tenant, op).await
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Authorize {
plan_hash: String,
#[serde(default = "approve_decision")]
decision: String,
}
fn approve_decision() -> String {
"approve".into()
}
async fn authorize_http(
State(state): State<crate::State>,
h: HeaderMap,
Path((tenant, id)): Path<(Uuid, Uuid)>,
Json(input): Json<Authorize>,
) -> Result<Json<Value>> {
let p = principal(&state, &h, true).await?;
scope(&p, tenant, true)?;
let session = p
.session_hash
.as_ref()
.ok_or(Error(StatusCode::FORBIDDEN, "INTERACTIVE_AUTH_REQUIRED"))?;
if !matches!(input.decision.as_str(), "approve" | "reject") {
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
let row = sqlx::query(
"SELECT operation_id,approval_state FROM approval_requests WHERE plan_id=$1 FOR UPDATE",
)
.bind(id)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(missing)?;
crate::permissions::require_principal(&mut tx, &p, "plans:approve").await?;
let op: Uuid = row.get("operation_id");
let status: String = row.get("approval_state");
let (plan, ph, _) = load_plan(&mut tx, id).await?;
if ph != input.plan_hash {
return Err(fail("PLAN_HASH_MISMATCH"));
}
if status == "approved" && input.decision == "approve" {
tx.commit().await?;
return Ok(Json(operation(&state, &p, tenant, op).await?));
}
if status != "pending_approval" {
return Err(fail("AUTHORIZATION_REVOKED"));
}
if input.decision == "reject" {
sqlx::query("UPDATE approval_requests SET approval_state='rejected' WHERE operation_id=$1")
.bind(op)
.execute(&mut *tx)
.await?;
} else {
recheck(&mut tx, &plan).await?;
// Both requester and approver retain explicit operator rights: the only
// shipped dev principals are fixed-scope and cannot grant memberships.
if !requester_enabled(&state, &plan.principal, tenant) {
return Err(fail("AUTHORIZATION_REVOKED"));
}
for job in &plan.jobs {
crate::root_policy::authorize(&mut tx, &p.name, &job.payload).await?;
}
let authorization = Uuid::new_v4();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,1)").bind(tenant).bind(authorization).bind(op).bind(id).bind(&ph).bind(&p.name).bind(session).bind(p.approval_origin()).bind(p.testsimulation).execute(&mut *tx).await?;
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
.bind(tenant)
.bind(op)
.bind(authorization)
.execute(&mut *tx)
.await?;
for j in &plan.jobs {
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)").bind(tenant).bind(j.payload.job_id).bind(j.payload.attempt_id).bind(op).bind(j.payload.agent_id).bind(j.payload.asset_id).bind(encode(&j.payload)?).bind(&j.job_payload_hash).bind(j.payload.action.resource()).execute(&mut *tx).await?;
sqlx::query("INSERT INTO outbox(tenant_id,job_id,attempt_id) VALUES($1,$2,$3)")
.bind(tenant)
.bind(j.payload.job_id)
.bind(j.payload.attempt_id)
.execute(&mut *tx)
.await?;
}
sqlx::query("UPDATE approval_requests SET approval_state='approved' WHERE operation_id=$1")
.bind(op)
.execute(&mut *tx)
.await?;
}
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'plan_authorization',$4,now())").bind(tenant).bind(Uuid::new_v4()).bind(&p.name).bind(json!({"plan_id":id,"plan_hash":ph,"operation_id":op,"decision":input.decision,"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation,"profile":std::env::var("RMM_PROFILE").unwrap_or_default()})).execute(&mut *tx).await?;
tx.commit().await?;
Ok(Json(operation(&state, &p, tenant, op).await?))
}
pub fn requester_enabled(state: &crate::State, name: &str, tenant: Uuid) -> bool {
state.auth.operator_enabled(name, tenant)
}
pub async fn operation(
state: &crate::State,
p: &Principal,
tenant: Uuid,
id: Uuid,
) -> Result<Value> {
scope(p, tenant, false)?;
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
crate::permissions::require_principal(&mut tx, p, "operations:read").await?;
let row=sqlx::query("SELECT r.plan_id,r.approval_state,p.payload,p.plan_hash,p.expires_at FROM approval_requests r JOIN operation_plans p USING(tenant_id,plan_id) WHERE operation_id=$1").bind(id).fetch_optional(&mut *tx).await?.ok_or_else(missing)?;
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
let mut approval: String = row.get("approval_state");
if approval == "pending_approval" && row.get::<DateTime<Utc>, _>("expires_at") <= Utc::now() {
approval = "expired".into();
sqlx::query("UPDATE approval_requests SET approval_state='expired' WHERE operation_id=$1 AND approval_state='pending_approval'").bind(id).execute(&mut *tx).await?;
}
let jobs=sqlx::query("SELECT job_id,attempt_id,state,cancel_requested,revision,pending_reason FROM jobs WHERE operation_id=$1 ORDER BY job_id LIMIT 101").bind(id).fetch_all(&mut *tx).await?;
let mut counts = serde_json::Map::new();
for j in &jobs {
let s: String = j.get("state");
let n = counts.get(&s).and_then(Value::as_u64).unwrap_or(0) + 1;
counts.insert(s, json!(n));
}
let n = jobs.len() as u64;
let c = |s: &str| counts.get(s).and_then(Value::as_u64).unwrap_or(0);
let execution = if n == 0 {
None
} else {
Some(if c("unknown") > 0 {
"needs_attention"
} else if c("succeeded") == n {
"succeeded"
} else if c("succeeded") + c("failed") + c("cancelled") + c("expired") == n {
if c("succeeded") > 0 {
"partially_succeeded"
} else if c("cancelled") == n {
"cancelled"
} else {
"failed"
}
} else if c("running") + c("verifying") + c("awaiting_reboot") > 0 {
"running"
} else {
"queued"
})
};
let items = jobs.iter().map(job_summary).collect::<Vec<_>>();
tx.commit().await?;
Ok(
json!({"operation_id":id,"plan_id":plan.plan_id,"plan_hash":row.get::<String,_>("plan_hash"),"approval_state":approval,"state":approval,"planned_job_count":plan.jobs.len(),"job_count":n,"execution_state":execution,"counts":counts,"jobs":items}),
)
}
fn job_view(r: &sqlx::postgres::PgRow) -> Value {
let mut result = job_summary(r);
result["result"] = r.get::<Option<Value>, _>("result").unwrap_or(Value::Null);
result
}
fn job_summary(r: &sqlx::postgres::PgRow) -> Value {
json!({"job_id":r.get::<Uuid,_>("job_id"),"attempt_id":r.get::<Uuid,_>("attempt_id"),"state":r.get::<String,_>("state"),"cancel_requested":r.get::<bool,_>("cancel_requested"),"revision":r.get::<i64,_>("revision"),"pending_reason":r.get::<Option<String>,_>("pending_reason")})
}
pub async fn job(state: &crate::State, p: &Principal, tenant: Uuid, id: Uuid) -> Result<Value> {
scope(p, tenant, false)?;
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
crate::permissions::require_principal(&mut tx, p, "operations:read").await?;
let r = sqlx::query(
"SELECT job_id,attempt_id,state,cancel_requested,revision,result,pending_reason FROM jobs WHERE job_id=$1",
)
.bind(id)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(missing)?;
tx.commit().await?;
Ok(job_view(&r))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Cancel {
pub reason: String,
}
pub async fn cancel(
state: &crate::State,
p: &Principal,
tenant: Uuid,
id: Uuid,
input: Cancel,
) -> Result<Value> {
scope(p, tenant, true)?;
if input.reason.is_empty() || input.reason.len() > 1024 {
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
sqlx::query("UPDATE jobs SET cancel_requested=true,state=CASE WHEN state='queued' AND NOT start_reserved THEN 'cancelled' ELSE state END WHERE job_id=$1").bind(id).execute(&mut *tx).await?;
crate::permissions::require_principal(&mut tx, p, "jobs:cancel").await?;
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'cancel_requested',$4,now())")
.bind(tenant)
.bind(Uuid::new_v4())
.bind(&p.name)
.bind(json!({"job_id":id,"reason":input.reason}))
.execute(&mut *tx)
.await?;
tx.commit().await?;
job(state, p, tenant, id).await
}
async fn plan_http(
State(s): State<crate::State>,
h: HeaderMap,
Path(t): Path<Uuid>,
Json(i): Json<PlanRequest>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, true).await?;
Ok(Json(plan(&s, &p, t, i.resolve(t)?).await?))
}
async fn submit_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
Json(i): Json<Submit>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, true).await?;
Ok(Json(submit(&s, &p, t, id, i).await?))
}
async fn plan_get_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, false).await?;
scope(&p, t, false)?;
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
let (plan, hash, bytes) = load_plan(&mut tx, id).await?;
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
let approval: Option<String> =
sqlx::query_scalar("SELECT approval_state FROM approval_requests WHERE plan_id=$1")
.bind(id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(plan_view(&plan, &hash, approval, &bytes)?))
}
async fn operation_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, false).await?;
Ok(Json(operation(&s, &p, t, id).await?))
}
async fn job_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, false).await?;
Ok(Json(job(&s, &p, t, id).await?))
}
async fn cancel_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
Json(i): Json<Cancel>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, true).await?;
Ok(Json(cancel(&s, &p, t, id, i).await?))
}
async fn operations_http(
State(s): State<crate::State>,
h: HeaderMap,
Path(t): Path<Uuid>,
axum::extract::Query(query): axum::extract::Query<crate::pagination::PageQuery>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, false).await?;
scope(&p, t, false)?;
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
let mut page = crate::pagination::Cursor::parse(&s, &p, "operations:operation_id:asc", query)
.map_err(|status| Error(status, "VALIDATION_ERROR"))?;
let ids:Vec<Uuid>=sqlx::query_scalar("SELECT operation_id FROM approval_requests WHERE operation_id>$1 AND created_at<=$2 ORDER BY operation_id LIMIT $3").bind(page.last).bind(page.as_of).bind(i64::from(page.limit)+1).fetch_all(&mut *tx).await?;
tx.commit().await?;
let truncated = ids.len() > usize::from(page.limit);
let next = if truncated {
Some(
page.next(&s, ids[usize::from(page.limit) - 1])
.map_err(|status| Error(status, "VALIDATION_ERROR"))?,
)
} else {
None
};
let mut items = Vec::new();
for id in ids.into_iter().take(usize::from(page.limit)) {
items.push(operation(&s, &p, t, id).await?)
}
Ok(Json(
json!({"items":items,"next_cursor":next,"completeness":if truncated{"partial"}else{"complete"},"truncated":truncated,"as_of":page.as_of}),
))
}