feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,923 @@
|
||||
//! Shared application services. HTTP/MCP adapters only establish a Principal and
|
||||
//! delegate here. No transport-specific approval shortcut exists.
|
||||
use crate::auth::Principal;
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rmm_domain::jobs::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::{Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Error(pub StatusCode, pub &'static str);
|
||||
impl From<sqlx::Error> for Error {
|
||||
fn from(_: sqlx::Error) -> Self {
|
||||
Self(StatusCode::SERVICE_UNAVAILABLE, "DEPENDENCY_UNAVAILABLE")
|
||||
}
|
||||
}
|
||||
impl From<anyhow::Error> for Error {
|
||||
fn from(_: anyhow::Error) -> Self {
|
||||
Self(StatusCode::SERVICE_UNAVAILABLE, "DEPENDENCY_UNAVAILABLE")
|
||||
}
|
||||
}
|
||||
impl From<serde_json::Error> for Error {
|
||||
fn from(_: serde_json::Error) -> Self {
|
||||
Self(StatusCode::BAD_REQUEST, "VALIDATION_ERROR")
|
||||
}
|
||||
}
|
||||
impl IntoResponse for Error {
|
||||
fn into_response(self) -> Response {
|
||||
(self.0,Json(json!({"code":self.1,"message":"operation could not be completed","retryable":self.0.is_server_error(),"correlation_id":Uuid::new_v4(),"details":{}}))).into_response()
|
||||
}
|
||||
}
|
||||
type Result<T> = std::result::Result<T, Error>;
|
||||
fn fail(code: &'static str) -> Error {
|
||||
Error(StatusCode::CONFLICT, code)
|
||||
}
|
||||
fn missing() -> Error {
|
||||
Error(StatusCode::NOT_FOUND, "NOT_FOUND")
|
||||
}
|
||||
pub fn scope(p: &Principal, tenant: Uuid, write: bool) -> Result<()> {
|
||||
if p.tenant != tenant {
|
||||
return Err(missing());
|
||||
}
|
||||
if write && !p.operator {
|
||||
return Err(Error(StatusCode::FORBIDDEN, "FORBIDDEN"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn principal(state: &crate::State, h: &HeaderMap, write: bool) -> Result<Principal> {
|
||||
state
|
||||
.auth
|
||||
.principal(state, h, write)
|
||||
.await
|
||||
.map_err(|s| Error(s, crate::auth::error_code(s)))
|
||||
}
|
||||
pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
|
||||
r.route("/api/v1/tenants/{tenant}/operation-plans", post(plan_http))
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/operation-plans/{plan}",
|
||||
get(plan_get_http),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/operation-plans/{plan}/submit",
|
||||
post(submit_http),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/operation-plans/{plan}/authorizations",
|
||||
post(authorize_http),
|
||||
)
|
||||
.route("/api/v1/tenants/{tenant}/operations", get(operations_http))
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/operations/{operation}",
|
||||
get(operation_http),
|
||||
)
|
||||
.route("/api/v1/tenants/{tenant}/jobs/{job}", get(job_http))
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/jobs/{job}/cancel",
|
||||
post(cancel_http),
|
||||
)
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Targets {
|
||||
pub asset_ids: Vec<Uuid>,
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Schedule {
|
||||
pub not_before: DateTime<Utc>,
|
||||
pub start_before: DateTime<Utc>,
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Rollout {
|
||||
pub max_parallel: u32,
|
||||
pub halt_after_failures: u32,
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PlanInput {
|
||||
pub targets: Targets,
|
||||
pub action: Action,
|
||||
pub execution_identity: String,
|
||||
pub schedule: Schedule,
|
||||
pub max_runtime_seconds: u32,
|
||||
pub max_output_bytes: u32,
|
||||
pub rollout: Rollout,
|
||||
pub reason: String,
|
||||
#[serde(default)]
|
||||
pub artifact_ids: Vec<Uuid>,
|
||||
}
|
||||
#[derive(Clone, Deserialize, schemars::JsonSchema)]
|
||||
#[serde(untagged)]
|
||||
pub enum PlanRequest {
|
||||
Typed(Box<PlanInput>),
|
||||
Web(Box<WebPlanInput>),
|
||||
}
|
||||
#[derive(Clone, Deserialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WebPlanInput {
|
||||
tenant_id: Uuid,
|
||||
action: String,
|
||||
targets: Targets,
|
||||
parameters: WebParameters,
|
||||
schedule: WebSchedule,
|
||||
rollout: Rollout,
|
||||
reason: String,
|
||||
#[serde(default)]
|
||||
artifact_ids: Vec<Uuid>,
|
||||
}
|
||||
#[derive(Clone, Deserialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct WebSchedule {
|
||||
not_before: DateTime<Utc>,
|
||||
start_before: DateTime<Utc>,
|
||||
offline_policy: String,
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Variant {
|
||||
os_family: String,
|
||||
interpreter: String,
|
||||
encoding: String,
|
||||
script_utf8: String,
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct WebParameters {
|
||||
execution_identity: String,
|
||||
timeout_seconds: u32,
|
||||
max_output_bytes: u32,
|
||||
variants: Option<Vec<Variant>>,
|
||||
arguments: Option<Vec<String>>,
|
||||
expected_stdout: Option<String>,
|
||||
scope: Option<String>,
|
||||
unit: Option<String>,
|
||||
state: Option<String>,
|
||||
since: Option<DateTime<Utc>>,
|
||||
until: Option<DateTime<Utc>>,
|
||||
max_lines: Option<u32>,
|
||||
provider: Option<String>,
|
||||
package: Option<String>,
|
||||
version: Option<String>,
|
||||
architecture: Option<String>,
|
||||
recipe_version: Option<String>,
|
||||
recipe_id: Option<String>,
|
||||
apt_manifest: Option<rmm_domain::apt_transaction::AptExecutionManifest>,
|
||||
installation_context: Option<String>,
|
||||
packages: Option<Vec<String>>,
|
||||
reboot_policy: Option<String>,
|
||||
reason: Option<String>,
|
||||
dependency_group: Option<String>,
|
||||
reconnect_timeout_seconds: Option<u32>,
|
||||
}
|
||||
impl PlanRequest {
|
||||
pub fn resolve(self, tenant: Uuid) -> Result<PlanInput> {
|
||||
let w = match self {
|
||||
Self::Typed(t) => return Ok(*t),
|
||||
Self::Web(w) => *w,
|
||||
};
|
||||
if w.tenant_id != tenant {
|
||||
return Err(missing());
|
||||
}
|
||||
if w.schedule.offline_policy != "wait_until_deadline" {
|
||||
return Err(fail("POLICY_DENIED"));
|
||||
}
|
||||
let parameters = serde_json::to_value(&w.parameters).map_err(Error::from)?;
|
||||
if w.action != "run_script" && w.parameters.variants.is_some() {
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
let p = w.parameters;
|
||||
let required =
|
||||
|value: Option<String>| value.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
let action = match w.action.as_str() {
|
||||
"run_script" => {
|
||||
let mut variants = p
|
||||
.variants
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?;
|
||||
if variants.len() != 1 {
|
||||
return Err(fail("AMBIGUOUS_TARGET"));
|
||||
}
|
||||
let v = variants.remove(0);
|
||||
if v.os_family != "linux" || v.encoding != "utf-8" {
|
||||
return Err(fail("UNSUPPORTED_CAPABILITY"));
|
||||
}
|
||||
Action::RunScript {
|
||||
interpreter: v.interpreter,
|
||||
script_utf8: v.script_utf8,
|
||||
arguments: p.arguments.unwrap_or_default(),
|
||||
expected_stdout: p.expected_stdout,
|
||||
}
|
||||
}
|
||||
"refresh_inventory" => Action::RefreshInventory {
|
||||
scope: required(p.scope)?,
|
||||
},
|
||||
"query_logs" => Action::QueryLogs {
|
||||
unit: required(p.unit)?,
|
||||
since: p
|
||||
.since
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
|
||||
until: p
|
||||
.until
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
|
||||
max_lines: p
|
||||
.max_lines
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
|
||||
},
|
||||
"set_service_state" => Action::SetServiceState {
|
||||
unit: required(p.unit)?,
|
||||
state: required(p.state)?,
|
||||
},
|
||||
"ensure_package" => Action::EnsurePackage {
|
||||
provider: required(p.provider)?,
|
||||
package: required(p.package)?,
|
||||
version: required(p.version)?,
|
||||
architecture: required(p.architecture)?,
|
||||
recipe_version: required(p.recipe_version)?,
|
||||
installation_context: required(p.installation_context)?,
|
||||
recipe_id: p.recipe_id,
|
||||
apt_manifest: p.apt_manifest,
|
||||
},
|
||||
"apply_updates" => Action::ApplyUpdates {
|
||||
provider: required(p.provider)?,
|
||||
packages: p
|
||||
.packages
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
|
||||
reboot_policy: required(p.reboot_policy)?,
|
||||
apt_manifest: p.apt_manifest,
|
||||
},
|
||||
"apt_native_acceptance" => Action::AptNativeAcceptance {
|
||||
recipe_id: required(p.recipe_id)?,
|
||||
recipe_version: required(p.recipe_version)?,
|
||||
apt_manifest: p
|
||||
.apt_manifest
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "APT_MANIFEST_REQUIRED"))?,
|
||||
},
|
||||
"reboot" => Action::Reboot {
|
||||
reason: required(p.reason)?,
|
||||
dependency_group: required(p.dependency_group)?,
|
||||
reconnect_timeout_seconds: p
|
||||
.reconnect_timeout_seconds
|
||||
.ok_or(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"))?,
|
||||
},
|
||||
_ => return Err(fail("UNSUPPORTED_CAPABILITY")),
|
||||
};
|
||||
// Reject irrelevant non-null fields instead of silently discarding an
|
||||
// effect that the browser displayed. Common limits bind into JobPayload.
|
||||
let action_view = serde_json::to_value(&action)?;
|
||||
for (key, value) in parameters.as_object().into_iter().flatten() {
|
||||
if !value.is_null()
|
||||
&& !matches!(
|
||||
key.as_str(),
|
||||
"execution_identity" | "timeout_seconds" | "max_output_bytes" | "variants"
|
||||
)
|
||||
&& action_view.get(key).is_none()
|
||||
{
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
}
|
||||
Ok(PlanInput {
|
||||
targets: w.targets,
|
||||
action,
|
||||
execution_identity: p.execution_identity,
|
||||
schedule: Schedule {
|
||||
not_before: w.schedule.not_before,
|
||||
start_before: w.schedule.start_before,
|
||||
},
|
||||
max_runtime_seconds: p.timeout_seconds,
|
||||
max_output_bytes: p.max_output_bytes,
|
||||
rollout: w.rollout,
|
||||
reason: w.reason,
|
||||
artifact_ids: w.artifact_ids,
|
||||
})
|
||||
}
|
||||
}
|
||||
async fn policy(tx: &mut Transaction<'_, Postgres>) -> Result<()> {
|
||||
let stopped: bool = sqlx::query_scalar("SELECT changes_stopped FROM tenants")
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
if stopped {
|
||||
return Err(fail("POLICY_DENIED"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub async fn plan(
|
||||
state: &crate::State,
|
||||
p: &Principal,
|
||||
tenant: Uuid,
|
||||
input: PlanInput,
|
||||
) -> Result<Value> {
|
||||
scope(p, tenant, true)?;
|
||||
let now = Utc::now();
|
||||
if input.targets.asset_ids.is_empty()
|
||||
|| input.targets.asset_ids.len() > 100
|
||||
|| !input.action.validate_plan_shape()
|
||||
|| !matches!(input.execution_identity.as_str(), "agent_service" | "root")
|
||||
|| !(1..=3600).contains(&input.max_runtime_seconds)
|
||||
|| input.max_output_bytes == 0
|
||||
|| input.max_output_bytes > 262144
|
||||
|| input.schedule.start_before <= now
|
||||
|| input.schedule.not_before >= input.schedule.start_before
|
||||
|| input.schedule.start_before > now + chrono::Duration::days(7)
|
||||
|| input.reason.is_empty()
|
||||
|| input.reason.len() > 2048
|
||||
|| !(1..=10).contains(&input.rollout.max_parallel)
|
||||
|| input.rollout.halt_after_failures == 0
|
||||
{
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
if matches!(
|
||||
input.action,
|
||||
Action::AptNativeAcceptance { .. }
|
||||
| Action::EnsurePackage {
|
||||
apt_manifest: Some(_),
|
||||
..
|
||||
}
|
||||
| Action::ApplyUpdates {
|
||||
apt_manifest: Some(_),
|
||||
..
|
||||
}
|
||||
) && input.execution_identity != "root"
|
||||
{
|
||||
return Err(Error(StatusCode::FORBIDDEN, "ROOT_IDENTITY_REQUIRED"));
|
||||
}
|
||||
// Conservative policy: reboot is always serial, no provider-triggered reboot.
|
||||
if matches!(input.action, Action::Reboot { .. }) && input.rollout.max_parallel != 1 {
|
||||
return Err(fail("POLICY_DENIED"));
|
||||
}
|
||||
let mut ids = input.targets.asset_ids;
|
||||
ids.sort();
|
||||
ids.dedup();
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
policy(&mut tx).await?;
|
||||
crate::permissions::require_principal(&mut tx, p, "operations:plan").await?;
|
||||
if input.execution_identity == "root" && !p.permits_scope("root:execute") {
|
||||
return Err(Error(StatusCode::FORBIDDEN, "FORBIDDEN"));
|
||||
}
|
||||
if matches!(input.action, Action::AptNativeAcceptance { .. }) {
|
||||
crate::permissions::require_principal(&mut tx, p, "apt:native_acceptance").await?;
|
||||
}
|
||||
if matches!(input.action, Action::RunScript { .. }) {
|
||||
crate::permissions::require_principal(&mut tx, p, "scripts:execute").await?;
|
||||
}
|
||||
if matches!(input.action, Action::QueryLogs { .. }) {
|
||||
crate::permissions::require_principal(&mut tx, p, "logs:read").await?;
|
||||
}
|
||||
let recipe_references = crate::catalog::references(&mut tx, &input.action).await?;
|
||||
let artifacts = crate::catalog::resolve_artifacts(&mut tx, tenant, &input.artifact_ids).await?;
|
||||
let mut jobs = Vec::new();
|
||||
for asset in ids {
|
||||
let rows=sqlx::query("SELECT agent_id,enrollment_generation,assignment_version,capabilities FROM agent_installations WHERE asset_id=$1 AND revoked_at IS NULL ORDER BY agent_id FOR SHARE").bind(asset).fetch_all(&mut *tx).await?;
|
||||
if rows.len() != 1 {
|
||||
return Err(fail(if rows.is_empty() {
|
||||
"NOT_FOUND"
|
||||
} else {
|
||||
"AMBIGUOUS_TARGET"
|
||||
}));
|
||||
}
|
||||
let row = &rows[0];
|
||||
let caps: Value = row.get("capabilities");
|
||||
if !caps
|
||||
.as_array()
|
||||
.is_some_and(|c| c.iter().any(|v| v.as_str() == Some(input.action.name())))
|
||||
{
|
||||
return Err(fail("UNSUPPORTED_CAPABILITY"));
|
||||
}
|
||||
let payload = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: tenant,
|
||||
asset_id: asset,
|
||||
agent_id: row.get("agent_id"),
|
||||
enrollment_generation: row.get::<i64, _>("enrollment_generation") as u64,
|
||||
assignment_version: row.get::<i64, _>("assignment_version") as u64,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: input.action.clone(),
|
||||
execution_identity: input.execution_identity.clone(),
|
||||
not_before: input.schedule.not_before,
|
||||
start_before: input.schedule.start_before,
|
||||
max_runtime_seconds: input.max_runtime_seconds,
|
||||
max_output_bytes: input.max_output_bytes,
|
||||
};
|
||||
let script_sha256 = if let Action::RunScript { script_utf8, .. } = &payload.action {
|
||||
Some(hash(script_utf8.as_bytes()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
crate::root_policy::authorize(&mut tx, &p.name, &payload).await?;
|
||||
if encode(&payload)?.len() > 65536 {
|
||||
return Err(Error(StatusCode::PAYLOAD_TOO_LARGE, "QUOTA_EXCEEDED"));
|
||||
}
|
||||
jobs.push(PlannedJob {
|
||||
job_payload_hash: hash(&encode(&payload)?),
|
||||
payload,
|
||||
script_sha256,
|
||||
});
|
||||
}
|
||||
let artifact_references = jobs
|
||||
.iter()
|
||||
.flat_map(|job| {
|
||||
artifacts.iter().map(|artifact| JobArtifactReference {
|
||||
job_id: job.payload.job_id,
|
||||
attempt_id: job.payload.attempt_id,
|
||||
artifact: artifact.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let payload = PlanPayload {
|
||||
schema_version: 1,
|
||||
plan_id: Uuid::new_v4(),
|
||||
tenant_id: tenant,
|
||||
principal: p.name.clone(),
|
||||
reason: input.reason,
|
||||
jobs,
|
||||
max_parallel: input.rollout.max_parallel,
|
||||
halt_after_failures: input.rollout.halt_after_failures,
|
||||
recipe_references,
|
||||
artifact_references,
|
||||
};
|
||||
let bytes = encode(&payload)?;
|
||||
let plan_hash = hash(&bytes);
|
||||
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(payload.plan_id).bind(&p.name).bind(&bytes).bind(&plan_hash).bind(input.schedule.start_before).execute(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
plan_view(&payload, &plan_hash, None, &bytes)
|
||||
}
|
||||
fn plan_view(
|
||||
plan: &PlanPayload,
|
||||
hash: &str,
|
||||
approval: Option<String>,
|
||||
bytes: &[u8],
|
||||
) -> Result<Value> {
|
||||
let jobs=plan.jobs.iter().map(|j|json!({"job_id":j.payload.job_id,"attempt_id":j.payload.attempt_id,"asset_id":j.payload.asset_id,"agent_id":j.payload.agent_id,"job_payload_hash":j.job_payload_hash,"script_sha256":j.script_sha256,"payload":j.payload})).collect::<Vec<_>>();
|
||||
Ok(
|
||||
json!({"plan_id":plan.plan_id,"plan_hash":hash,"payload":plan,"plan_payload":plan,"plan_payload_bytes":bytes,"resolved_targets":plan.jobs.iter().map(|j|json!({"asset_id":j.payload.asset_id,"agent_id":j.payload.agent_id})).collect::<Vec<_>>(),"excluded_targets":[],"jobs":jobs,"authorization_status":if plan.jobs.iter().all(|j|j.payload.action.validate()){"required"}else{"unsupported_manifest_required"},"approval_state":approval,"expires_at":plan.jobs.first().map(|j|j.payload.start_before)}),
|
||||
)
|
||||
}
|
||||
async fn load_plan(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
id: Uuid,
|
||||
) -> Result<(PlanPayload, String, Vec<u8>)> {
|
||||
let r = sqlx::query("SELECT payload,plan_hash FROM operation_plans WHERE plan_id=$1")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?
|
||||
.ok_or_else(missing)?;
|
||||
let bytes: Vec<u8> = r.get("payload");
|
||||
let h: String = r.get("plan_hash");
|
||||
if hash(&bytes) != h {
|
||||
return Err(fail("PLAN_HASH_MISMATCH"));
|
||||
}
|
||||
Ok((serde_json::from_slice(&bytes)?, h, bytes))
|
||||
}
|
||||
async fn recheck(tx: &mut Transaction<'_, Postgres>, plan: &PlanPayload) -> Result<()> {
|
||||
policy(tx).await?;
|
||||
let current_artifacts = crate::catalog::resolve_artifacts(
|
||||
tx,
|
||||
plan.tenant_id,
|
||||
&plan
|
||||
.artifact_references
|
||||
.iter()
|
||||
.map(|r| r.artifact.artifact_id)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.await?;
|
||||
for reference in &plan.artifact_references {
|
||||
if !current_artifacts.contains(&reference.artifact)
|
||||
|| !plan.jobs.iter().any(|j| {
|
||||
j.payload.job_id == reference.job_id && j.payload.attempt_id == reference.attempt_id
|
||||
})
|
||||
{
|
||||
return Err(fail("ARTIFACT_BINDING_MISMATCH"));
|
||||
}
|
||||
}
|
||||
crate::permissions::require(tx, &plan.principal, "operations:plan").await?;
|
||||
if plan
|
||||
.jobs
|
||||
.iter()
|
||||
.any(|j| matches!(j.payload.action, Action::RunScript { .. }))
|
||||
{
|
||||
crate::permissions::require(tx, &plan.principal, "scripts:execute").await?;
|
||||
}
|
||||
if plan
|
||||
.jobs
|
||||
.iter()
|
||||
.any(|j| matches!(j.payload.action, Action::QueryLogs { .. }))
|
||||
{
|
||||
crate::permissions::require(tx, &plan.principal, "logs:read").await?;
|
||||
}
|
||||
let mut references = Vec::new();
|
||||
for job in &plan.jobs {
|
||||
if !job.payload.action.validate() {
|
||||
return Err(Error(StatusCode::CONFLICT, "APT_MANIFEST_REQUIRED"));
|
||||
}
|
||||
crate::root_policy::authorize(tx, &plan.principal, &job.payload).await?;
|
||||
crate::catalog::check(tx, &job.payload.action).await?;
|
||||
references.extend(crate::catalog::references(tx, &job.payload.action).await?);
|
||||
if job.payload.start_before <= Utc::now() {
|
||||
return Err(fail("PLAN_EXPIRED"));
|
||||
}
|
||||
let r=sqlx::query("SELECT enrollment_generation,assignment_version,capabilities FROM agent_installations WHERE agent_id=$1 AND asset_id=$2 AND revoked_at IS NULL FOR SHARE").bind(job.payload.agent_id).bind(job.payload.asset_id).fetch_optional(&mut **tx).await?.ok_or_else(||fail("AUTHORIZATION_REVOKED"))?;
|
||||
if r.get::<i64, _>("assignment_version") as u64 != job.payload.assignment_version
|
||||
|| r.get::<i64, _>("enrollment_generation") as u64 != job.payload.enrollment_generation
|
||||
{
|
||||
return Err(fail("AUTHORIZATION_REVOKED"));
|
||||
}
|
||||
let caps: Value = r.get("capabilities");
|
||||
if !caps.as_array().is_some_and(|c| {
|
||||
c.iter()
|
||||
.any(|v| v.as_str() == Some(job.payload.action.name()))
|
||||
}) {
|
||||
return Err(fail("UNSUPPORTED_CAPABILITY"));
|
||||
}
|
||||
if hash(&encode(&job.payload)?) != job.job_payload_hash {
|
||||
return Err(fail("PLAN_HASH_MISMATCH"));
|
||||
}
|
||||
}
|
||||
references.sort();
|
||||
references.dedup();
|
||||
if references != plan.recipe_references {
|
||||
return Err(fail("RECIPE_BINDING_MISMATCH"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Submit {
|
||||
pub idempotency_key: String,
|
||||
}
|
||||
pub async fn submit(
|
||||
state: &crate::State,
|
||||
p: &Principal,
|
||||
tenant: Uuid,
|
||||
id: Uuid,
|
||||
input: Submit,
|
||||
) -> Result<Value> {
|
||||
scope(p, tenant, true)?;
|
||||
if input.idempotency_key.is_empty() || input.idempotency_key.len() > 128 {
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
// Serialize idempotency and same-plan submit, including competing principals.
|
||||
crate::permissions::require_principal(&mut tx, p, "operations:submit").await?;
|
||||
crate::permissions::require_principal(&mut tx, p, "operations:read").await?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
||||
.bind(format!(
|
||||
"submit:{tenant}:{}:{}",
|
||||
p.name, input.idempotency_key
|
||||
))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
||||
.bind(format!("plan-submit:{tenant}:{id}"))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("SELECT plan_id FROM operation_plans WHERE plan_id=$1")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.ok_or_else(missing)?;
|
||||
let request_hash = hash(id.as_bytes());
|
||||
let (plan, _, _) = load_plan(&mut tx, id).await?;
|
||||
for job in &plan.jobs {
|
||||
if matches!(job.payload.action, Action::RunScript { .. }) {
|
||||
crate::permissions::require_principal(&mut tx, p, "scripts:execute").await?;
|
||||
}
|
||||
if matches!(job.payload.action, Action::QueryLogs { .. }) {
|
||||
crate::permissions::require_principal(&mut tx, p, "logs:read").await?;
|
||||
}
|
||||
if job.payload.execution_identity == "root" {
|
||||
crate::permissions::require_principal(&mut tx, p, "root:execute").await?;
|
||||
}
|
||||
if matches!(job.payload.action, Action::AptNativeAcceptance { .. }) {
|
||||
crate::permissions::require_principal(&mut tx, p, "apt:native_acceptance").await?;
|
||||
}
|
||||
}
|
||||
if let Some(row)=sqlx::query("SELECT request_hash,operation_id FROM operation_submissions WHERE principal=$1 AND idempotency_key=$2").bind(&p.name).bind(&input.idempotency_key).fetch_optional(&mut *tx).await? {
|
||||
if row.get::<String,_>("request_hash")!=request_hash{return Err(fail("IDEMPOTENCY_CONFLICT"))}
|
||||
let op=row.get("operation_id");tx.commit().await?;return operation(state,p,tenant,op).await;
|
||||
}
|
||||
if let Some(op) =
|
||||
sqlx::query_scalar::<_, Uuid>("SELECT operation_id FROM approval_requests WHERE plan_id=$1")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
{
|
||||
sqlx::query("INSERT INTO operation_submissions VALUES($1,$2,$3,$4,$5)")
|
||||
.bind(tenant)
|
||||
.bind(&p.name)
|
||||
.bind(&input.idempotency_key)
|
||||
.bind(&request_hash)
|
||||
.bind(op)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
return operation(state, p, tenant, op).await;
|
||||
}
|
||||
recheck(&mut tx, &plan).await?;
|
||||
let op = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash) VALUES($1,$2,$3,$4,$5,$6)").bind(tenant).bind(op).bind(id).bind(&p.name).bind(&input.idempotency_key).bind(&request_hash).execute(&mut *tx).await?;
|
||||
sqlx::query("INSERT INTO operation_submissions VALUES($1,$2,$3,$4,$5)")
|
||||
.bind(tenant)
|
||||
.bind(&p.name)
|
||||
.bind(&input.idempotency_key)
|
||||
.bind(&request_hash)
|
||||
.bind(op)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
operation(state, p, tenant, op).await
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Authorize {
|
||||
plan_hash: String,
|
||||
#[serde(default = "approve_decision")]
|
||||
decision: String,
|
||||
}
|
||||
fn approve_decision() -> String {
|
||||
"approve".into()
|
||||
}
|
||||
async fn authorize_http(
|
||||
State(state): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((tenant, id)): Path<(Uuid, Uuid)>,
|
||||
Json(input): Json<Authorize>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&state, &h, true).await?;
|
||||
scope(&p, tenant, true)?;
|
||||
let session = p
|
||||
.session_hash
|
||||
.as_ref()
|
||||
.ok_or(Error(StatusCode::FORBIDDEN, "INTERACTIVE_AUTH_REQUIRED"))?;
|
||||
if !matches!(input.decision.as_str(), "approve" | "reject") {
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
let row = sqlx::query(
|
||||
"SELECT operation_id,approval_state FROM approval_requests WHERE plan_id=$1 FOR UPDATE",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.ok_or_else(missing)?;
|
||||
crate::permissions::require_principal(&mut tx, &p, "plans:approve").await?;
|
||||
let op: Uuid = row.get("operation_id");
|
||||
let status: String = row.get("approval_state");
|
||||
let (plan, ph, _) = load_plan(&mut tx, id).await?;
|
||||
if ph != input.plan_hash {
|
||||
return Err(fail("PLAN_HASH_MISMATCH"));
|
||||
}
|
||||
if status == "approved" && input.decision == "approve" {
|
||||
tx.commit().await?;
|
||||
return Ok(Json(operation(&state, &p, tenant, op).await?));
|
||||
}
|
||||
if status != "pending_approval" {
|
||||
return Err(fail("AUTHORIZATION_REVOKED"));
|
||||
}
|
||||
if input.decision == "reject" {
|
||||
sqlx::query("UPDATE approval_requests SET approval_state='rejected' WHERE operation_id=$1")
|
||||
.bind(op)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
} else {
|
||||
recheck(&mut tx, &plan).await?;
|
||||
// Both requester and approver retain explicit operator rights: the only
|
||||
// shipped dev principals are fixed-scope and cannot grant memberships.
|
||||
if !requester_enabled(&state, &plan.principal, tenant) {
|
||||
return Err(fail("AUTHORIZATION_REVOKED"));
|
||||
}
|
||||
for job in &plan.jobs {
|
||||
crate::root_policy::authorize(&mut tx, &p.name, &job.payload).await?;
|
||||
}
|
||||
let authorization = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,1)").bind(tenant).bind(authorization).bind(op).bind(id).bind(&ph).bind(&p.name).bind(session).bind(p.approval_origin()).bind(p.testsimulation).execute(&mut *tx).await?;
|
||||
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
|
||||
.bind(tenant)
|
||||
.bind(op)
|
||||
.bind(authorization)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
for j in &plan.jobs {
|
||||
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)").bind(tenant).bind(j.payload.job_id).bind(j.payload.attempt_id).bind(op).bind(j.payload.agent_id).bind(j.payload.asset_id).bind(encode(&j.payload)?).bind(&j.job_payload_hash).bind(j.payload.action.resource()).execute(&mut *tx).await?;
|
||||
sqlx::query("INSERT INTO outbox(tenant_id,job_id,attempt_id) VALUES($1,$2,$3)")
|
||||
.bind(tenant)
|
||||
.bind(j.payload.job_id)
|
||||
.bind(j.payload.attempt_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
sqlx::query("UPDATE approval_requests SET approval_state='approved' WHERE operation_id=$1")
|
||||
.bind(op)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'plan_authorization',$4,now())").bind(tenant).bind(Uuid::new_v4()).bind(&p.name).bind(json!({"plan_id":id,"plan_hash":ph,"operation_id":op,"decision":input.decision,"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation,"profile":std::env::var("RMM_PROFILE").unwrap_or_default()})).execute(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
Ok(Json(operation(&state, &p, tenant, op).await?))
|
||||
}
|
||||
pub fn requester_enabled(state: &crate::State, name: &str, tenant: Uuid) -> bool {
|
||||
state.auth.operator_enabled(name, tenant)
|
||||
}
|
||||
pub async fn operation(
|
||||
state: &crate::State,
|
||||
p: &Principal,
|
||||
tenant: Uuid,
|
||||
id: Uuid,
|
||||
) -> Result<Value> {
|
||||
scope(p, tenant, false)?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
crate::permissions::require_principal(&mut tx, p, "operations:read").await?;
|
||||
let row=sqlx::query("SELECT r.plan_id,r.approval_state,p.payload,p.plan_hash,p.expires_at FROM approval_requests r JOIN operation_plans p USING(tenant_id,plan_id) WHERE operation_id=$1").bind(id).fetch_optional(&mut *tx).await?.ok_or_else(missing)?;
|
||||
let plan: PlanPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
||||
let mut approval: String = row.get("approval_state");
|
||||
if approval == "pending_approval" && row.get::<DateTime<Utc>, _>("expires_at") <= Utc::now() {
|
||||
approval = "expired".into();
|
||||
sqlx::query("UPDATE approval_requests SET approval_state='expired' WHERE operation_id=$1 AND approval_state='pending_approval'").bind(id).execute(&mut *tx).await?;
|
||||
}
|
||||
let jobs=sqlx::query("SELECT job_id,attempt_id,state,cancel_requested,revision,pending_reason FROM jobs WHERE operation_id=$1 ORDER BY job_id LIMIT 101").bind(id).fetch_all(&mut *tx).await?;
|
||||
let mut counts = serde_json::Map::new();
|
||||
for j in &jobs {
|
||||
let s: String = j.get("state");
|
||||
let n = counts.get(&s).and_then(Value::as_u64).unwrap_or(0) + 1;
|
||||
counts.insert(s, json!(n));
|
||||
}
|
||||
let n = jobs.len() as u64;
|
||||
let c = |s: &str| counts.get(s).and_then(Value::as_u64).unwrap_or(0);
|
||||
let execution = if n == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(if c("unknown") > 0 {
|
||||
"needs_attention"
|
||||
} else if c("succeeded") == n {
|
||||
"succeeded"
|
||||
} else if c("succeeded") + c("failed") + c("cancelled") + c("expired") == n {
|
||||
if c("succeeded") > 0 {
|
||||
"partially_succeeded"
|
||||
} else if c("cancelled") == n {
|
||||
"cancelled"
|
||||
} else {
|
||||
"failed"
|
||||
}
|
||||
} else if c("running") + c("verifying") + c("awaiting_reboot") > 0 {
|
||||
"running"
|
||||
} else {
|
||||
"queued"
|
||||
})
|
||||
};
|
||||
let items = jobs.iter().map(job_summary).collect::<Vec<_>>();
|
||||
tx.commit().await?;
|
||||
Ok(
|
||||
json!({"operation_id":id,"plan_id":plan.plan_id,"plan_hash":row.get::<String,_>("plan_hash"),"approval_state":approval,"state":approval,"planned_job_count":plan.jobs.len(),"job_count":n,"execution_state":execution,"counts":counts,"jobs":items}),
|
||||
)
|
||||
}
|
||||
fn job_view(r: &sqlx::postgres::PgRow) -> Value {
|
||||
let mut result = job_summary(r);
|
||||
result["result"] = r.get::<Option<Value>, _>("result").unwrap_or(Value::Null);
|
||||
result
|
||||
}
|
||||
fn job_summary(r: &sqlx::postgres::PgRow) -> Value {
|
||||
json!({"job_id":r.get::<Uuid,_>("job_id"),"attempt_id":r.get::<Uuid,_>("attempt_id"),"state":r.get::<String,_>("state"),"cancel_requested":r.get::<bool,_>("cancel_requested"),"revision":r.get::<i64,_>("revision"),"pending_reason":r.get::<Option<String>,_>("pending_reason")})
|
||||
}
|
||||
pub async fn job(state: &crate::State, p: &Principal, tenant: Uuid, id: Uuid) -> Result<Value> {
|
||||
scope(p, tenant, false)?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
crate::permissions::require_principal(&mut tx, p, "operations:read").await?;
|
||||
let r = sqlx::query(
|
||||
"SELECT job_id,attempt_id,state,cancel_requested,revision,result,pending_reason FROM jobs WHERE job_id=$1",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.ok_or_else(missing)?;
|
||||
tx.commit().await?;
|
||||
Ok(job_view(&r))
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Cancel {
|
||||
pub reason: String,
|
||||
}
|
||||
pub async fn cancel(
|
||||
state: &crate::State,
|
||||
p: &Principal,
|
||||
tenant: Uuid,
|
||||
id: Uuid,
|
||||
input: Cancel,
|
||||
) -> Result<Value> {
|
||||
scope(p, tenant, true)?;
|
||||
if input.reason.is_empty() || input.reason.len() > 1024 {
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
sqlx::query("UPDATE jobs SET cancel_requested=true,state=CASE WHEN state='queued' AND NOT start_reserved THEN 'cancelled' ELSE state END WHERE job_id=$1").bind(id).execute(&mut *tx).await?;
|
||||
crate::permissions::require_principal(&mut tx, p, "jobs:cancel").await?;
|
||||
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'cancel_requested',$4,now())")
|
||||
.bind(tenant)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(&p.name)
|
||||
.bind(json!({"job_id":id,"reason":input.reason}))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
job(state, p, tenant, id).await
|
||||
}
|
||||
async fn plan_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path(t): Path<Uuid>,
|
||||
Json(i): Json<PlanRequest>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, true).await?;
|
||||
Ok(Json(plan(&s, &p, t, i.resolve(t)?).await?))
|
||||
}
|
||||
async fn submit_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((t, id)): Path<(Uuid, Uuid)>,
|
||||
Json(i): Json<Submit>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, true).await?;
|
||||
Ok(Json(submit(&s, &p, t, id, i).await?))
|
||||
}
|
||||
async fn plan_get_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((t, id)): Path<(Uuid, Uuid)>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, false).await?;
|
||||
scope(&p, t, false)?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
|
||||
let (plan, hash, bytes) = load_plan(&mut tx, id).await?;
|
||||
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
|
||||
let approval: Option<String> =
|
||||
sqlx::query_scalar("SELECT approval_state FROM approval_requests WHERE plan_id=$1")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok(Json(plan_view(&plan, &hash, approval, &bytes)?))
|
||||
}
|
||||
async fn operation_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((t, id)): Path<(Uuid, Uuid)>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, false).await?;
|
||||
Ok(Json(operation(&s, &p, t, id).await?))
|
||||
}
|
||||
async fn job_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((t, id)): Path<(Uuid, Uuid)>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, false).await?;
|
||||
Ok(Json(job(&s, &p, t, id).await?))
|
||||
}
|
||||
async fn cancel_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((t, id)): Path<(Uuid, Uuid)>,
|
||||
Json(i): Json<Cancel>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, true).await?;
|
||||
Ok(Json(cancel(&s, &p, t, id, i).await?))
|
||||
}
|
||||
async fn operations_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path(t): Path<Uuid>,
|
||||
axum::extract::Query(query): axum::extract::Query<crate::pagination::PageQuery>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, false).await?;
|
||||
scope(&p, t, false)?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
|
||||
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
|
||||
let mut page = crate::pagination::Cursor::parse(&s, &p, "operations:operation_id:asc", query)
|
||||
.map_err(|status| Error(status, "VALIDATION_ERROR"))?;
|
||||
let ids:Vec<Uuid>=sqlx::query_scalar("SELECT operation_id FROM approval_requests WHERE operation_id>$1 AND created_at<=$2 ORDER BY operation_id LIMIT $3").bind(page.last).bind(page.as_of).bind(i64::from(page.limit)+1).fetch_all(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
let truncated = ids.len() > usize::from(page.limit);
|
||||
let next = if truncated {
|
||||
Some(
|
||||
page.next(&s, ids[usize::from(page.limit) - 1])
|
||||
.map_err(|status| Error(status, "VALIDATION_ERROR"))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut items = Vec::new();
|
||||
for id in ids.into_iter().take(usize::from(page.limit)) {
|
||||
items.push(operation(&s, &p, t, id).await?)
|
||||
}
|
||||
Ok(Json(
|
||||
json!({"items":items,"next_cursor":next,"completeness":if truncated{"partial"}else{"complete"},"truncated":truncated,"as_of":page.as_of}),
|
||||
))
|
||||
}
|
||||
Reference in New Issue
Block a user