//! Narrow dev/test wire faults after normal authorization/durable ingest. //! Never creates jobs, grants or approvals. Claims and observations are bounded, //! insert-only audit slots; a restart cannot arm the same fault a second time. use anyhow::{Context, Result, ensure}; use chrono::{DateTime, Duration, Utc}; use prost::Message; use rmm_domain::jobs::{Action, JobPayload, hash}; use rmm_protocol::{Envelope, Payload}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sqlx::{PgPool, Postgres, Row, Transaction}; use std::{fs, os::unix::fs::MetadataExt, path::PathBuf}; use uuid::Uuid; #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] enum Mode { DuplicateOfferOnce, DropResultAckOnce, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] struct Config { schema_version: u32, fault_id: Uuid, tenant_id: Uuid, agent_id: Uuid, job_id: Uuid, attempt_id: Uuid, plan_sha256: String, job_payload_sha256: String, mode: Mode, transport: String, issued_at: DateTime, expires_at: DateTime, } impl Config { fn validate(&self) -> Result<()> { ensure!( self.schema_version == 1 && [ self.fault_id, self.tenant_id, self.agent_id, self.job_id, self.attempt_id ] .iter() .all(|id| !id.is_nil()) && valid_hash(&self.plan_sha256) && valid_hash(&self.job_payload_sha256) && matches!(self.transport.as_str(), "wss" | "https") && self.issued_at.timestamp() > 0 && self.expires_at > self.issued_at && self.expires_at - self.issued_at <= Duration::minutes(15), "DEV_DELIVERY_FAULT_CONFIG" ); Ok(()) } fn scope(&self, tenant: Uuid, agent: Uuid, transport: &str) -> bool { self.tenant_id == tenant && self.agent_id == agent && self.transport == transport } fn job(&self, bytes: &[u8], payload_hash: &str, plan_hash: &str) -> Result { if payload_hash != self.job_payload_sha256 || plan_hash != self.plan_sha256 { return Ok(false); } ensure!( hash(bytes) == self.job_payload_sha256, "DEV_FAULT_PAYLOAD_HASH" ); let job: JobPayload = serde_json::from_slice(bytes)?; Ok(job.schema_version == 1 && job.tenant_id == self.tenant_id && job.agent_id == self.agent_id && job.job_id == self.job_id && job.attempt_id == self.attempt_id && job.execution_identity == "agent_service" && job.enrollment_generation > 0 && job.assignment_version > 0 && job.max_runtime_seconds > 0 && job.max_runtime_seconds <= 60 && job.max_output_bytes <= 65536 && job.action.validate() && matches!(job.action, Action::RunScript { .. }) && self.expires_at <= job.start_before) } } fn valid_hash(value: &str) -> bool { value.len() == 64 && value .bytes() .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) && value.bytes().any(|b| b != b'0') } fn configured_path( flag: Option<&str>, path: Option<&str>, profile: &str, dev: &str, ) -> Result> { match (flag, path) { (None | Some("false"), None) => Ok(None), (Some("true"), Some(path)) => { ensure!( profile == "development" && dev == "true", "DEV_DELIVERY_FAULT_PROFILE" ); ensure!( path == "/root-policy/delivery-fault.json", "DEV_DELIVERY_FAULT_PATH" ); Ok(Some(PathBuf::from(path))) } _ => anyhow::bail!("DEV_DELIVERY_FAULT_EXPLICIT_OPT_IN"), } } #[derive(Clone)] struct Loaded { config: Config, config_sha256: String, } fn load() -> Result> { let flag = std::env::var("RMM_DEV_DELIVERY_FAULTS").ok(); let path = std::env::var("RMM_DEV_DELIVERY_FAULT_FILE").ok(); let profile = std::env::var("RMM_PROFILE").unwrap_or_default(); let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default(); let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else { return Ok(None); }; let parent = path.parent().context("DEV_FAULT_PARENT")?; let directory = fs::symlink_metadata(parent)?; let metadata = fs::symlink_metadata(&path)?; ensure!( directory.is_dir() && directory.uid() == 10001 && directory.mode() & 0o777 == 0o700 && metadata.is_file() && metadata.uid() == 10001 && metadata.mode() & 0o777 == 0o600 && metadata.nlink() == 1 && metadata.len() <= 8192, "DEV_FAULT_PRIVATE_CONFIG" ); // /root-policy is the existing operator-provisioned read-only bind mount. let bytes = fs::read(&path)?; ensure!(bytes.len() <= 8192, "DEV_FAULT_CONFIG_SIZE"); let config: Config = serde_json::from_slice(&bytes)?; config.validate()?; if Utc::now() < config.issued_at || Utc::now() >= config.expires_at { return Ok(None); } Ok(Some(Loaded { config, config_sha256: hash(&bytes), })) } pub(crate) fn validate_environment() -> Result<()> { super::dev_transport_faults::validate_environment()?; load().map(|_| ()) } fn slot_id(fault: Uuid, slot: &str) -> Uuid { use sha2::{Digest, Sha256}; let digest = Sha256::digest(format!("rmm.dev-delivery-fault.v1:{fault}:{slot}")); Uuid::from_slice(&digest[..16]).expect("fixed digest prefix") } fn envelope_hash(envelope: &Envelope) -> String { let mut normalized = envelope.clone(); normalized.connection_generation = 0; hash(&normalized.encode_to_vec()) } fn data(loaded: &Loaded, envelope: &Envelope, detail: &str) -> Value { let config = &loaded.config; json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true, "fault_id":config.fault_id,"mode":config.mode,"transport":config.transport, "agent_id":config.agent_id,"job_id":config.job_id,"attempt_id":config.attempt_id, "plan_sha256":config.plan_sha256,"job_payload_sha256":config.job_payload_sha256, "config_sha256":loaded.config_sha256,"message_id":envelope.message_id, "normalized_envelope_sha256":envelope_hash(envelope),"detail":detail}) } async fn transaction<'a>(pool: &'a PgPool, loaded: &Loaded) -> Result> { let mut tx = rmm_persistence::tenant_tx(pool, loaded.config.tenant_id).await?; sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))") .bind(format!("dev-delivery-fault:{}", loaded.config.fault_id)) .execute(&mut *tx) .await?; Ok(tx) } async fn get_slot( tx: &mut Transaction<'_, Postgres>, loaded: &Loaded, slot: &str, ) -> Result> { let value: Option = sqlx::query_scalar("SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_delivery_fault_adapter'") .bind(slot_id(loaded.config.fault_id, slot)).fetch_optional(&mut **tx).await?; if let Some(value) = &value { ensure!( value.get("config_sha256") == Some(&json!(loaded.config_sha256)), "DEV_FAULT_ID_REUSED_WITH_CHANGED_CONFIG" ); } Ok(value) } async fn insert_slot( tx: &mut Transaction<'_, Postgres>, loaded: &Loaded, slot: &str, value: Value, ) -> Result { if get_slot(tx, loaded, slot).await?.is_some() { return Ok(false); } let rows = sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_delivery_fault_adapter',$3,$4) ON CONFLICT(tenant_id,event_id) DO NOTHING") .bind(loaded.config.tenant_id).bind(slot_id(loaded.config.fault_id, slot)) .bind(format!("dev_delivery_fault.{slot}")).bind(value).execute(&mut **tx).await?.rows_affected(); Ok(rows == 1) } async fn authorized_job( tx: &mut Transaction<'_, Postgres>, loaded: &Loaded, message: Option<&Envelope>, ) -> Result { let c = &loaded.config; let row = sqlx::query("SELECT j.payload,j.payload_hash,p.plan_hash FROM jobs j JOIN operations o USING(tenant_id,operation_id) JOIN approval_requests r USING(tenant_id,operation_id) JOIN operation_plans p USING(tenant_id,plan_id) JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 AND r.approval_state='approved' AND a.testsimulation AND a.approval_origin='dev_test_browser_simulation' AND a.principal='simulated-testprincipal-a'") .bind(c.job_id).bind(c.attempt_id).bind(c.agent_id).fetch_optional(&mut **tx).await?; let Some(row) = row else { return Ok(false); }; if !c.job( &row.get::, _>("payload"), &row.get::("payload_hash"), &row.get::("plan_hash"), )? { return Ok(false); } if let Some(message) = message { let receipt: Option = sqlx::query_scalar( "SELECT payload_hash FROM job_receipts WHERE message_id=$1 AND agent_id=$2", ) .bind(Uuid::parse_str(&message.message_id)?) .bind(c.agent_id) .fetch_optional(&mut **tx) .await?; ensure!( receipt.as_deref() == Some(envelope_hash(message).as_str()), "DEV_FAULT_DURABLE_RECEIPT_REQUIRED" ); } Ok(true) } pub(crate) struct DuplicateClaim { loaded: Loaded, envelope: Envelope, } /// Called only for work returned by normal postapproval `job_gateway::next`. pub(crate) async fn duplicate_offer( pool: &PgPool, tenant: Uuid, agent: Uuid, envelope: &Envelope, transport: &str, ) -> Result> { let Some(Payload::JobOffer(offer)) = &envelope.payload else { return Ok(None); }; let Some(loaded) = load()? else { return Ok(None); }; let c = &loaded.config; if c.mode != Mode::DuplicateOfferOnce || !c.scope(tenant, agent, transport) || offer.job_id != c.job_id.to_string() || offer.attempt_id != c.attempt_id.to_string() || !c.job( &offer.payload_bytes, &offer.job_payload_hash, &offer.plan_hash, )? { return Ok(None); } let mut tx = transaction(pool, &loaded).await?; let claimed = authorized_job(&mut tx, &loaded, None).await? && insert_slot( &mut tx, &loaded, "claim", data(&loaded, envelope, "duplicate_offer_reserved_before_send"), ) .await?; tx.commit().await?; Ok(claimed.then(|| DuplicateClaim { loaded, envelope: envelope.clone(), })) } /// WSS calls after both sends complete. HTTP labels construction honestly; the /// two later distinct postcommit JobAccepted slots prove client observation. pub(crate) async fn duplicate_emitted(pool: &PgPool, claim: DuplicateClaim) -> Result<()> { let mut tx = transaction(pool, &claim.loaded).await?; ensure!( get_slot(&mut tx, &claim.loaded, "claim").await?.is_some(), "DEV_FAULT_CLAIM_REQUIRED" ); let detail = if claim.loaded.config.transport == "wss" { "same_encoded_offer_sent_twice" } else { "same_encoded_offer_in_http_response_twice" }; insert_slot( &mut tx, &claim.loaded, "emitted", data(&claim.loaded, &claim.envelope, detail), ) .await?; tx.commit().await?; Ok(()) } /// Called AFTER durable ingest returned its concrete ACK. Return true only to /// omit that single matching result ACK; every subsequent replay is ACKed. pub(crate) async fn after_ingest( pool: &PgPool, tenant: Uuid, agent: Uuid, incoming: &Envelope, reply: &Envelope, transport: &str, ) -> Result { let (job, attempt, is_result) = match &incoming.payload { Some(Payload::JobAccepted(value)) => (&value.job_id, &value.attempt_id, false), Some(Payload::JobResult(value)) if value.state == "succeeded" => { (&value.job_id, &value.attempt_id, true) } _ => return Ok(false), }; let Some(loaded) = load()? else { return Ok(false); }; let c = &loaded.config; if !c.scope(tenant, agent, transport) || job != &c.job_id.to_string() || attempt != &c.attempt_id.to_string() { return Ok(false); } let Some(Payload::Ack(ack)) = &reply.payload else { return Ok(false); }; ensure!( ack.message_id == incoming.message_id, "DEV_FAULT_CONCRETE_ACK_REQUIRED" ); let mut tx = transaction(pool, &loaded).await?; if !authorized_job(&mut tx, &loaded, Some(incoming)).await? { tx.commit().await?; return Ok(false); } let claim = get_slot(&mut tx, &loaded, "claim").await?; let mut omit = false; if is_result && c.mode == Mode::DropResultAckOnce { if let Some(first) = claim { if first.get("message_id") == Some(&json!(incoming.message_id)) && first.get("normalized_envelope_sha256") == Some(&json!(envelope_hash(incoming))) { insert_slot( &mut tx, &loaded, "replay", data( &loaded, incoming, "same_original_result_replayed_after_ack_omission", ), ) .await?; } } else { omit = insert_slot( &mut tx, &loaded, "claim", data( &loaded, incoming, "one_concrete_result_ack_omitted_after_durable_commit", ), ) .await?; } } else if !is_result && c.mode == Mode::DuplicateOfferOnce && claim.is_some() { if let Some(first) = get_slot(&mut tx, &loaded, "accepted_1").await? { if first.get("message_id") != Some(&json!(incoming.message_id)) { insert_slot( &mut tx, &loaded, "accepted_2", data(&loaded, incoming, "second_distinct_client_acceptance"), ) .await?; } } else { insert_slot( &mut tx, &loaded, "accepted_1", data(&loaded, incoming, "first_client_acceptance"), ) .await?; } } tx.commit().await?; Ok(omit) } #[cfg(test)] mod tests { use super::*; fn config() -> Config { let now = Utc::now(); Config { schema_version: 1, fault_id: Uuid::new_v4(), tenant_id: Uuid::new_v4(), agent_id: Uuid::new_v4(), job_id: Uuid::new_v4(), attempt_id: Uuid::new_v4(), plan_sha256: "a".repeat(64), job_payload_sha256: "b".repeat(64), mode: Mode::DuplicateOfferOnce, transport: "wss".into(), issued_at: now, expires_at: now + Duration::minutes(10), } } #[test] fn opt_in_is_explicit_and_never_production() { assert!( configured_path(None, None, "production", "false") .unwrap() .is_none() ); assert!( configured_path( Some("true"), Some("/root-policy/delivery-fault.json"), "production", "false" ) .is_err() ); assert!(configured_path(Some("true"), None, "development", "true").is_err()); assert!( configured_path( None, Some("/root-policy/delivery-fault.json"), "development", "true" ) .is_err() ); assert!(configured_path(Some("true"), Some("/etc/passwd"), "development", "true").is_err()); assert!( configured_path( Some("true"), Some("/root-policy/delivery-fault.json"), "development", "true" ) .unwrap() .is_some() ); } #[test] fn narrow_schema_rejects_wildcards_nil_placeholders_and_long_lifetime() { let c = config(); c.validate().unwrap(); let mut altered = c.clone(); altered.expires_at = altered.issued_at + Duration::minutes(16); assert!(altered.validate().is_err()); altered = c.clone(); altered.agent_id = Uuid::nil(); assert!(altered.validate().is_err()); altered = c.clone(); altered.job_payload_sha256 = "0".repeat(64); assert!(altered.validate().is_err()); altered = c.clone(); altered.transport = "*".into(); assert!(altered.validate().is_err()); let mut raw = serde_json::to_value(c).unwrap(); raw["approve"] = json!(true); assert!(serde_json::from_value::(raw).is_err()); } #[test] fn audit_slots_are_stable_bounded_and_distinct() { let id = Uuid::new_v4(); assert_eq!(slot_id(id, "claim"), slot_id(id, "claim")); assert_ne!(slot_id(id, "claim"), slot_id(id, "replay")); assert_ne!(slot_id(id, "claim"), slot_id(Uuid::new_v4(), "claim")); } #[test] fn exact_signed_job_scope_cannot_fault_root_other_targets_or_other_actions() { let mut c = config(); let mut job = JobPayload { schema_version: 1, tenant_id: c.tenant_id, asset_id: Uuid::new_v4(), agent_id: c.agent_id, enrollment_generation: 1, assignment_version: 1, job_id: c.job_id, attempt_id: c.attempt_id, action: Action::RunScript { interpreter: "posix_sh".into(), script_utf8: "printf hello\\n".into(), arguments: vec![], expected_stdout: None, }, execution_identity: "agent_service".into(), not_before: c.issued_at, start_before: c.expires_at, max_runtime_seconds: 60, max_output_bytes: 4096, }; let mut bytes = serde_json::to_vec(&job).unwrap(); c.job_payload_sha256 = hash(&bytes); assert!( c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256) .unwrap() ); assert!(!c.scope(c.tenant_id, Uuid::new_v4(), "wss")); assert!(!c.scope(c.tenant_id, c.agent_id, "https")); assert!( !c.job(&bytes, &c.job_payload_sha256, &"c".repeat(64)) .unwrap() ); job.execution_identity = "root".into(); bytes = serde_json::to_vec(&job).unwrap(); c.job_payload_sha256 = hash(&bytes); assert!( !c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256) .unwrap() ); job.execution_identity = "agent_service".into(); job.action = Action::RefreshInventory { scope: "linux.base".into(), }; bytes = serde_json::to_vec(&job).unwrap(); c.job_payload_sha256 = hash(&bytes); assert!( !c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256) .unwrap() ); } #[test] fn durable_replay_only_normalizes_connection_generation() { let e = Envelope { message_id: Uuid::new_v4().to_string(), connection_generation: 3, ..Default::default() }; let mut repeated = e.clone(); repeated.connection_generation = 4; assert_eq!(envelope_hash(&e), envelope_hash(&repeated)); repeated.message_id = Uuid::new_v4().to_string(); assert_ne!(envelope_hash(&e), envelope_hash(&repeated)); } }