Files
rust-discovery/crates/server/src/dev_delivery_faults.rs
T
2026-11-13 19:11:45 +01:00

572 lines
20 KiB
Rust

//! Narrow dev/test wire faults after normal authorization/durable ingest.
//! Never creates jobs, grants or approvals. Claims and observations are bounded,
//! insert-only audit slots; a restart cannot arm the same fault a second time.
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Duration, Utc};
use prost::Message;
use rmm_domain::jobs::{Action, JobPayload, hash};
use rmm_protocol::{Envelope, Payload};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sqlx::{PgPool, Postgres, Row, Transaction};
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
use uuid::Uuid;
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum Mode {
DuplicateOfferOnce,
DropResultAckOnce,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Config {
schema_version: u32,
fault_id: Uuid,
tenant_id: Uuid,
agent_id: Uuid,
job_id: Uuid,
attempt_id: Uuid,
plan_sha256: String,
job_payload_sha256: String,
mode: Mode,
transport: String,
issued_at: DateTime<Utc>,
expires_at: DateTime<Utc>,
}
impl Config {
fn validate(&self) -> Result<()> {
ensure!(
self.schema_version == 1
&& [
self.fault_id,
self.tenant_id,
self.agent_id,
self.job_id,
self.attempt_id
]
.iter()
.all(|id| !id.is_nil())
&& valid_hash(&self.plan_sha256)
&& valid_hash(&self.job_payload_sha256)
&& matches!(self.transport.as_str(), "wss" | "https")
&& self.issued_at.timestamp() > 0
&& self.expires_at > self.issued_at
&& self.expires_at - self.issued_at <= Duration::minutes(15),
"DEV_DELIVERY_FAULT_CONFIG"
);
Ok(())
}
fn scope(&self, tenant: Uuid, agent: Uuid, transport: &str) -> bool {
self.tenant_id == tenant && self.agent_id == agent && self.transport == transport
}
fn job(&self, bytes: &[u8], payload_hash: &str, plan_hash: &str) -> Result<bool> {
if payload_hash != self.job_payload_sha256 || plan_hash != self.plan_sha256 {
return Ok(false);
}
ensure!(
hash(bytes) == self.job_payload_sha256,
"DEV_FAULT_PAYLOAD_HASH"
);
let job: JobPayload = serde_json::from_slice(bytes)?;
Ok(job.schema_version == 1
&& job.tenant_id == self.tenant_id
&& job.agent_id == self.agent_id
&& job.job_id == self.job_id
&& job.attempt_id == self.attempt_id
&& job.execution_identity == "agent_service"
&& job.enrollment_generation > 0
&& job.assignment_version > 0
&& job.max_runtime_seconds > 0
&& job.max_runtime_seconds <= 60
&& job.max_output_bytes <= 65536
&& job.action.validate()
&& matches!(job.action, Action::RunScript { .. })
&& self.expires_at <= job.start_before)
}
}
fn valid_hash(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
&& value.bytes().any(|b| b != b'0')
}
fn configured_path(
flag: Option<&str>,
path: Option<&str>,
profile: &str,
dev: &str,
) -> Result<Option<PathBuf>> {
match (flag, path) {
(None | Some("false"), None) => Ok(None),
(Some("true"), Some(path)) => {
ensure!(
profile == "development" && dev == "true",
"DEV_DELIVERY_FAULT_PROFILE"
);
ensure!(
path == "/root-policy/delivery-fault.json",
"DEV_DELIVERY_FAULT_PATH"
);
Ok(Some(PathBuf::from(path)))
}
_ => anyhow::bail!("DEV_DELIVERY_FAULT_EXPLICIT_OPT_IN"),
}
}
#[derive(Clone)]
struct Loaded {
config: Config,
config_sha256: String,
}
fn load() -> Result<Option<Loaded>> {
let flag = std::env::var("RMM_DEV_DELIVERY_FAULTS").ok();
let path = std::env::var("RMM_DEV_DELIVERY_FAULT_FILE").ok();
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
return Ok(None);
};
let parent = path.parent().context("DEV_FAULT_PARENT")?;
let directory = fs::symlink_metadata(parent)?;
let metadata = fs::symlink_metadata(&path)?;
ensure!(
directory.is_dir()
&& directory.uid() == 10001
&& directory.mode() & 0o777 == 0o700
&& metadata.is_file()
&& metadata.uid() == 10001
&& metadata.mode() & 0o777 == 0o600
&& metadata.nlink() == 1
&& metadata.len() <= 8192,
"DEV_FAULT_PRIVATE_CONFIG"
);
// /root-policy is the existing operator-provisioned read-only bind mount.
let bytes = fs::read(&path)?;
ensure!(bytes.len() <= 8192, "DEV_FAULT_CONFIG_SIZE");
let config: Config = serde_json::from_slice(&bytes)?;
config.validate()?;
if Utc::now() < config.issued_at || Utc::now() >= config.expires_at {
return Ok(None);
}
Ok(Some(Loaded {
config,
config_sha256: hash(&bytes),
}))
}
pub(crate) fn validate_environment() -> Result<()> {
super::dev_transport_faults::validate_environment()?;
load().map(|_| ())
}
fn slot_id(fault: Uuid, slot: &str) -> Uuid {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(format!("rmm.dev-delivery-fault.v1:{fault}:{slot}"));
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
}
fn envelope_hash(envelope: &Envelope) -> String {
let mut normalized = envelope.clone();
normalized.connection_generation = 0;
hash(&normalized.encode_to_vec())
}
fn data(loaded: &Loaded, envelope: &Envelope, detail: &str) -> Value {
let config = &loaded.config;
json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
"fault_id":config.fault_id,"mode":config.mode,"transport":config.transport,
"agent_id":config.agent_id,"job_id":config.job_id,"attempt_id":config.attempt_id,
"plan_sha256":config.plan_sha256,"job_payload_sha256":config.job_payload_sha256,
"config_sha256":loaded.config_sha256,"message_id":envelope.message_id,
"normalized_envelope_sha256":envelope_hash(envelope),"detail":detail})
}
async fn transaction<'a>(pool: &'a PgPool, loaded: &Loaded) -> Result<Transaction<'a, Postgres>> {
let mut tx = rmm_persistence::tenant_tx(pool, loaded.config.tenant_id).await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("dev-delivery-fault:{}", loaded.config.fault_id))
.execute(&mut *tx)
.await?;
Ok(tx)
}
async fn get_slot(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
slot: &str,
) -> Result<Option<Value>> {
let value: Option<Value> = sqlx::query_scalar("SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_delivery_fault_adapter'")
.bind(slot_id(loaded.config.fault_id, slot)).fetch_optional(&mut **tx).await?;
if let Some(value) = &value {
ensure!(
value.get("config_sha256") == Some(&json!(loaded.config_sha256)),
"DEV_FAULT_ID_REUSED_WITH_CHANGED_CONFIG"
);
}
Ok(value)
}
async fn insert_slot(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
slot: &str,
value: Value,
) -> Result<bool> {
if get_slot(tx, loaded, slot).await?.is_some() {
return Ok(false);
}
let rows = sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_delivery_fault_adapter',$3,$4) ON CONFLICT(tenant_id,event_id) DO NOTHING")
.bind(loaded.config.tenant_id).bind(slot_id(loaded.config.fault_id, slot))
.bind(format!("dev_delivery_fault.{slot}")).bind(value).execute(&mut **tx).await?.rows_affected();
Ok(rows == 1)
}
async fn authorized_job(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
message: Option<&Envelope>,
) -> Result<bool> {
let c = &loaded.config;
let row = sqlx::query("SELECT j.payload,j.payload_hash,p.plan_hash FROM jobs j JOIN operations o USING(tenant_id,operation_id) JOIN approval_requests r USING(tenant_id,operation_id) JOIN operation_plans p USING(tenant_id,plan_id) JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 AND r.approval_state='approved' AND a.testsimulation AND a.approval_origin='dev_test_browser_simulation' AND a.principal='simulated-testprincipal-a'")
.bind(c.job_id).bind(c.attempt_id).bind(c.agent_id).fetch_optional(&mut **tx).await?;
let Some(row) = row else {
return Ok(false);
};
if !c.job(
&row.get::<Vec<u8>, _>("payload"),
&row.get::<String, _>("payload_hash"),
&row.get::<String, _>("plan_hash"),
)? {
return Ok(false);
}
if let Some(message) = message {
let receipt: Option<String> = sqlx::query_scalar(
"SELECT payload_hash FROM job_receipts WHERE message_id=$1 AND agent_id=$2",
)
.bind(Uuid::parse_str(&message.message_id)?)
.bind(c.agent_id)
.fetch_optional(&mut **tx)
.await?;
ensure!(
receipt.as_deref() == Some(envelope_hash(message).as_str()),
"DEV_FAULT_DURABLE_RECEIPT_REQUIRED"
);
}
Ok(true)
}
pub(crate) struct DuplicateClaim {
loaded: Loaded,
envelope: Envelope,
}
/// Called only for work returned by normal postapproval `job_gateway::next`.
pub(crate) async fn duplicate_offer(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
envelope: &Envelope,
transport: &str,
) -> Result<Option<DuplicateClaim>> {
let Some(Payload::JobOffer(offer)) = &envelope.payload else {
return Ok(None);
};
let Some(loaded) = load()? else {
return Ok(None);
};
let c = &loaded.config;
if c.mode != Mode::DuplicateOfferOnce
|| !c.scope(tenant, agent, transport)
|| offer.job_id != c.job_id.to_string()
|| offer.attempt_id != c.attempt_id.to_string()
|| !c.job(
&offer.payload_bytes,
&offer.job_payload_hash,
&offer.plan_hash,
)?
{
return Ok(None);
}
let mut tx = transaction(pool, &loaded).await?;
let claimed = authorized_job(&mut tx, &loaded, None).await?
&& insert_slot(
&mut tx,
&loaded,
"claim",
data(&loaded, envelope, "duplicate_offer_reserved_before_send"),
)
.await?;
tx.commit().await?;
Ok(claimed.then(|| DuplicateClaim {
loaded,
envelope: envelope.clone(),
}))
}
/// WSS calls after both sends complete. HTTP labels construction honestly; the
/// two later distinct postcommit JobAccepted slots prove client observation.
pub(crate) async fn duplicate_emitted(pool: &PgPool, claim: DuplicateClaim) -> Result<()> {
let mut tx = transaction(pool, &claim.loaded).await?;
ensure!(
get_slot(&mut tx, &claim.loaded, "claim").await?.is_some(),
"DEV_FAULT_CLAIM_REQUIRED"
);
let detail = if claim.loaded.config.transport == "wss" {
"same_encoded_offer_sent_twice"
} else {
"same_encoded_offer_in_http_response_twice"
};
insert_slot(
&mut tx,
&claim.loaded,
"emitted",
data(&claim.loaded, &claim.envelope, detail),
)
.await?;
tx.commit().await?;
Ok(())
}
/// Called AFTER durable ingest returned its concrete ACK. Return true only to
/// omit that single matching result ACK; every subsequent replay is ACKed.
pub(crate) async fn after_ingest(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
incoming: &Envelope,
reply: &Envelope,
transport: &str,
) -> Result<bool> {
let (job, attempt, is_result) = match &incoming.payload {
Some(Payload::JobAccepted(value)) => (&value.job_id, &value.attempt_id, false),
Some(Payload::JobResult(value)) if value.state == "succeeded" => {
(&value.job_id, &value.attempt_id, true)
}
_ => return Ok(false),
};
let Some(loaded) = load()? else {
return Ok(false);
};
let c = &loaded.config;
if !c.scope(tenant, agent, transport)
|| job != &c.job_id.to_string()
|| attempt != &c.attempt_id.to_string()
{
return Ok(false);
}
let Some(Payload::Ack(ack)) = &reply.payload else {
return Ok(false);
};
ensure!(
ack.message_id == incoming.message_id,
"DEV_FAULT_CONCRETE_ACK_REQUIRED"
);
let mut tx = transaction(pool, &loaded).await?;
if !authorized_job(&mut tx, &loaded, Some(incoming)).await? {
tx.commit().await?;
return Ok(false);
}
let claim = get_slot(&mut tx, &loaded, "claim").await?;
let mut omit = false;
if is_result && c.mode == Mode::DropResultAckOnce {
if let Some(first) = claim {
if first.get("message_id") == Some(&json!(incoming.message_id))
&& first.get("normalized_envelope_sha256") == Some(&json!(envelope_hash(incoming)))
{
insert_slot(
&mut tx,
&loaded,
"replay",
data(
&loaded,
incoming,
"same_original_result_replayed_after_ack_omission",
),
)
.await?;
}
} else {
omit = insert_slot(
&mut tx,
&loaded,
"claim",
data(
&loaded,
incoming,
"one_concrete_result_ack_omitted_after_durable_commit",
),
)
.await?;
}
} else if !is_result && c.mode == Mode::DuplicateOfferOnce && claim.is_some() {
if let Some(first) = get_slot(&mut tx, &loaded, "accepted_1").await? {
if first.get("message_id") != Some(&json!(incoming.message_id)) {
insert_slot(
&mut tx,
&loaded,
"accepted_2",
data(&loaded, incoming, "second_distinct_client_acceptance"),
)
.await?;
}
} else {
insert_slot(
&mut tx,
&loaded,
"accepted_1",
data(&loaded, incoming, "first_client_acceptance"),
)
.await?;
}
}
tx.commit().await?;
Ok(omit)
}
#[cfg(test)]
mod tests {
use super::*;
fn config() -> Config {
let now = Utc::now();
Config {
schema_version: 1,
fault_id: Uuid::new_v4(),
tenant_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
plan_sha256: "a".repeat(64),
job_payload_sha256: "b".repeat(64),
mode: Mode::DuplicateOfferOnce,
transport: "wss".into(),
issued_at: now,
expires_at: now + Duration::minutes(10),
}
}
#[test]
fn opt_in_is_explicit_and_never_production() {
assert!(
configured_path(None, None, "production", "false")
.unwrap()
.is_none()
);
assert!(
configured_path(
Some("true"),
Some("/root-policy/delivery-fault.json"),
"production",
"false"
)
.is_err()
);
assert!(configured_path(Some("true"), None, "development", "true").is_err());
assert!(
configured_path(
None,
Some("/root-policy/delivery-fault.json"),
"development",
"true"
)
.is_err()
);
assert!(configured_path(Some("true"), Some("/etc/passwd"), "development", "true").is_err());
assert!(
configured_path(
Some("true"),
Some("/root-policy/delivery-fault.json"),
"development",
"true"
)
.unwrap()
.is_some()
);
}
#[test]
fn narrow_schema_rejects_wildcards_nil_placeholders_and_long_lifetime() {
let c = config();
c.validate().unwrap();
let mut altered = c.clone();
altered.expires_at = altered.issued_at + Duration::minutes(16);
assert!(altered.validate().is_err());
altered = c.clone();
altered.agent_id = Uuid::nil();
assert!(altered.validate().is_err());
altered = c.clone();
altered.job_payload_sha256 = "0".repeat(64);
assert!(altered.validate().is_err());
altered = c.clone();
altered.transport = "*".into();
assert!(altered.validate().is_err());
let mut raw = serde_json::to_value(c).unwrap();
raw["approve"] = json!(true);
assert!(serde_json::from_value::<Config>(raw).is_err());
}
#[test]
fn audit_slots_are_stable_bounded_and_distinct() {
let id = Uuid::new_v4();
assert_eq!(slot_id(id, "claim"), slot_id(id, "claim"));
assert_ne!(slot_id(id, "claim"), slot_id(id, "replay"));
assert_ne!(slot_id(id, "claim"), slot_id(Uuid::new_v4(), "claim"));
}
#[test]
fn exact_signed_job_scope_cannot_fault_root_other_targets_or_other_actions() {
let mut c = config();
let mut job = JobPayload {
schema_version: 1,
tenant_id: c.tenant_id,
asset_id: Uuid::new_v4(),
agent_id: c.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: c.job_id,
attempt_id: c.attempt_id,
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf hello\\n".into(),
arguments: vec![],
expected_stdout: None,
},
execution_identity: "agent_service".into(),
not_before: c.issued_at,
start_before: c.expires_at,
max_runtime_seconds: 60,
max_output_bytes: 4096,
};
let mut bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
assert!(!c.scope(c.tenant_id, Uuid::new_v4(), "wss"));
assert!(!c.scope(c.tenant_id, c.agent_id, "https"));
assert!(
!c.job(&bytes, &c.job_payload_sha256, &"c".repeat(64))
.unwrap()
);
job.execution_identity = "root".into();
bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
job.execution_identity = "agent_service".into();
job.action = Action::RefreshInventory {
scope: "linux.base".into(),
};
bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
}
#[test]
fn durable_replay_only_normalizes_connection_generation() {
let e = Envelope {
message_id: Uuid::new_v4().to_string(),
connection_generation: 3,
..Default::default()
};
let mut repeated = e.clone();
repeated.connection_generation = 4;
assert_eq!(envelope_hash(&e), envelope_hash(&repeated));
repeated.message_id = Uuid::new_v4().to_string();
assert_ne!(envelope_hash(&e), envelope_hash(&repeated));
}
}