fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:11:45 +01:00
parent da7eb8109c
commit c512bd6fb2
36 changed files with 2556 additions and 38 deletions
+86
View File
@@ -92,6 +92,92 @@ fn hex_hash(s: &str) -> bool {
s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
/// executor thread from the previous process can still deliver its outcome.
/// This does not execute, release consumption or invent a successful outcome.
pub fn interrupted(
db: &Connection,
ledger: &Connection,
binding: &Binding,
) -> Result<Vec<(JobOffer, String)>> {
let mut statement = db.prepare(
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
)?;
let rows = statement.query_map([], |r| {
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
})?;
let mut interrupted = Vec::new();
for row in rows {
let (bytes, grant) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"INTERRUPTED_OFFER_BINDING"
);
if !grant.is_empty() {
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
[&grant],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
)
.optional()?;
let (agent, generation, job_id, attempt, start) =
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == offer.job_id
&& attempt == offer.attempt_id,
"INTERRUPTED_CONSUMPTION_BINDING"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: Request = serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
// Verify the historical signed request at its original valid time;
// this is evidence only, never permission to start it again.
let (_, verified) = binding.verify(&request, permit.not_before)?;
ensure!(
verified == permit
&& permit.grant_id.to_string() == grant
&& request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"INTERRUPTED_START_BINDING"
);
} else {
// A request may have committed remotely without its reply arriving.
// It can only become local unknown, never a fabricated grant/result.
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
}
interrupted.push((offer, grant));
}
Ok(interrupted)
}
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
ensure!(
db.execute(
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
params![offer.job_id, offer.attempt_id],
)? == 1,
"INTERRUPTED_REQUEST_STATE"
);
Ok(())
}
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
+24
View File
@@ -209,6 +209,29 @@ impl<C: reconcile::SyncClock> Agent<C> {
tx.commit()?;
Ok(())
}
/// New process only. Persist honest uncertainty before any helper snapshot
/// can block; the ordinary first-channel replay publishes it before Sync.
fn recover_interrupted(&mut self) -> Result<()> {
let Some(binding) = &self.binding else {
return Ok(());
};
let interrupted = jobs::interrupted(&self.db, &self.ledger, binding)?;
for (offer, grant) in interrupted {
if grant.is_empty() {
jobs::interrupted_request(&self.db, &offer)?;
} else {
self.record(
0,
&offer,
&grant,
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
)?;
}
}
// No finish/unlock here. Only normal bound reconciliation can release
// a consumed attempt once independent definitive evidence is available.
Ok(())
}
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
// A large replay backlog must not monopolize heartbeat/cancel handling.
// On backpressure timeout, discard this channel rather than continuing
@@ -789,6 +812,7 @@ async fn main() -> Result<()> {
certificates,
rotation_policy,
};
agent.recover_interrupted()?;
let mut backoff = 1_u64;
loop {
let started = Instant::now();
+175
View File
@@ -18,6 +18,181 @@ use std::{
struct Clock(bool);
#[test]
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
use openssl::sign::Signer;
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
use rmm_privileged_helper::{Binding, Request};
use rusqlite::params;
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
storage::prepare_dir(&dir).unwrap();
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf 'fixture only; never executed\\n'".into(),
arguments: vec![],
expected_stdout: Some("fixture only; never executed\n".into()),
},
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(5),
max_runtime_seconds: 120,
max_output_bytes: 4096,
};
let permit = ExecutionPermitPayload {
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: Uuid::new_v4(),
tenant_id: job.tenant_id,
asset_id: job.asset_id,
agent_id: job.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: job.job_id,
attempt_id: job.attempt_id,
plan_hash: hash(b"fixed test plan; no native approval"),
job_payload_hash: hash(&encode(&job).unwrap()),
action: "run_script".into(),
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::seconds(59),
max_runtime_seconds: 120,
key_id: hash(&public),
};
let mut request = Request {
job_bytes: encode(&job).unwrap(),
permit_bytes: encode(&permit).unwrap(),
signature: vec![],
plan_hash: permit.plan_hash.clone(),
authorization_id: permit.authorization_id,
};
request.signature = Signer::new_without_digest(&key)
.unwrap()
.sign_oneshot_to_vec(&request.permit_bytes)
.unwrap();
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: permit.authorization_id.to_string(),
plan_hash: permit.plan_hash.clone(),
job_payload_hash: permit.job_payload_hash.clone(),
payload_bytes: request.job_bytes.clone(),
};
let db = storage::open(&dir).unwrap();
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
let mut queued_job = job.clone();
queued_job.job_id = Uuid::new_v4();
queued_job.attempt_id = Uuid::new_v4();
let mut queued = offer.clone();
queued.job_id = queued_job.job_id.to_string();
queued.attempt_id = queued_job.attempt_id.to_string();
queued.payload_bytes = encode(&queued_job).unwrap();
queued.job_payload_hash = hash(&queued.payload_bytes);
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
let (tx, rx) = tokio::sync::mpsc::channel(1);
let (itx, irx) = tokio::sync::mpsc::channel(1);
let certificates = certificate_rotation::RotationStore::open(
&dir.join("certificates"),
&[],
binding.agent_id,
1,
)
.unwrap();
let mut agent = Agent {
address: "127.0.0.1:1".parse().unwrap(),
ca: vec![],
id: binding.agent_id.to_string(),
boot: rmm_agent_linux::boot_id().unwrap(),
instance: Uuid::new_v4().to_string(),
db,
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding: Some(binding.clone()),
ledger,
completed_tx: tx,
completed_rx: rx,
inventory_tx: itx,
inventory_rx: irx,
inventory_inflight: false,
sync: Some(reconcile::ClientSync::new(binding.clone())),
sync_clock: Clock(true),
helpers: None,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode: wire_channel::Mode::Wss,
certificates,
rotation_policy: certificate_rotation::RotationPolicy::default(),
};
// A forged local consumption/start cannot become a purported grant result.
agent
.ledger
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
.unwrap();
assert!(agent.recover_interrupted().is_err());
assert!(jobs::events(&agent.db).unwrap().is_empty());
agent
.ledger
.execute(
"UPDATE consumed SET agent=?1",
[binding.agent_id.to_string()],
)
.unwrap();
agent.recover_interrupted().unwrap();
let original = jobs::events(&agent.db).unwrap();
assert_eq!(original.len(), 1);
let Some(Payload::JobResult(result)) = &original[0].payload else {
panic!("real durable result required")
};
assert_eq!(result.state, "unknown");
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
assert_eq!(result.grant_id, permit.grant_id.to_string());
assert_eq!(result.revision, 2);
assert!(result.stdout.is_empty() && result.exit_code.is_none());
assert!(!agent.fence.allows(1, true));
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
assert_eq!(jobs::events(&reopened).unwrap(), original);
let requesting: (String, u64) = reopened
.query_row(
"SELECT state,revision FROM job_attempts WHERE job=?1",
[&queued.job_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(requesting, ("unknown".into(), 1));
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert_eq!(retained, (1, 1, 1));
// The same process/restart cannot invent another revision or message ID.
agent.recover_interrupted().unwrap();
assert_eq!(jobs::events(&agent.db).unwrap(), original);
drop(reopened);
drop(agent);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
let gate = EffectFence::default();
assert!(!gate.allows(1, false));
+26 -2
View File
@@ -65,8 +65,9 @@ pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
let uid = unsafe { libc::geteuid() };
rmm_privileged_helper::private_path(&path, uid, false)?;
let db = rusqlite::Connection::open(path)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
db.execute_batch("PRAGMA max_page_count=16384;
rmm_privileged_helper::bounded_journal_capacity(&db)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
db.execute_batch("
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
@@ -106,6 +107,29 @@ pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
#[cfg(test)]
mod passive_pressure_tests {
#[test]
fn agent_journal_capacity_is_finite_and_preserves_sequence_on_reopen() {
let dir = std::env::temp_dir().join(format!("rmm-capacity-{}", uuid::Uuid::new_v4()));
super::prepare_dir(&dir).unwrap();
let mut db = super::open(&dir).unwrap();
assert_eq!(super::next_sequence(&mut db).unwrap(), 1);
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
drop(db);
let mut db = super::open(&dir).unwrap();
assert_eq!(
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
.unwrap(),
4096
);
assert_eq!(
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
.unwrap(),
65536
);
assert_eq!(super::next_sequence(&mut db).unwrap(), 2);
drop(db);
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn pending_durable_traffic_backpressures_collection_without_deletion() {
let db = rusqlite::Connection::open_in_memory().unwrap();
+13 -1
View File
@@ -204,6 +204,17 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
f.try_lock_exclusive()?;
Ok(f)
}
/// Finite 256 MiB bound for durable agent/helper history. Unexpected page sizes
/// are refused rather than silently multiplying the byte limit. No compaction
/// or tombstone deletion occurs here; SQLITE_FULL must still fail closed.
pub fn bounded_journal_capacity(db: &rusqlite::Connection) -> Result<()> {
let page_size: u32 = db.query_row("PRAGMA page_size", [], |r| r.get(0))?;
ensure!(page_size == 4096, "JOURNAL_PAGE_SIZE_UNSUPPORTED");
let maximum: u32 = db.query_row("PRAGMA max_page_count=65536", [], |r| r.get(0))?;
ensure!(maximum == 65536, "JOURNAL_STORAGE_CAPACITY_BLOCKED");
Ok(())
}
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
use std::os::unix::fs::OpenOptionsExt;
let path = dir.join("consumption.sqlite");
@@ -218,7 +229,8 @@ pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
f.sync_all()?;
private_path(&path, uid, false)?;
let db = rusqlite::Connection::open(path)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=16384;
bounded_journal_capacity(&db)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE;
CREATE TABLE IF NOT EXISTS consumed(grant_id TEXT PRIMARY KEY, agent TEXT NOT NULL, generation INTEGER NOT NULL,
job TEXT NOT NULL, attempt TEXT NOT NULL, result BLOB, UNIQUE(agent,generation,job,attempt));
CREATE TABLE IF NOT EXISTS execution_evidence(grant_id TEXT PRIMARY KEY, start BLOB NOT NULL, outcome BLOB);")?;
+59 -10
View File
@@ -472,7 +472,30 @@ fn handle(
stream.write_all(&bytes)?;
Ok(())
}
fn safe_error_code(error: &anyhow::Error) -> String {
if error.chain().any(|cause| {
matches!(cause.downcast_ref::<rusqlite::Error>(),
Some(rusqlite::Error::SqliteFailure(code, _)) if code.code == rusqlite::ErrorCode::DiskFull)
}) {
return "JOURNAL_STORAGE_CAPACITY_BLOCKED".into();
}
let text = error.to_string();
if text.len() <= 96
&& text
.bytes()
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
{
text
} else {
"HELPER_VALIDATION_OR_IO_FAILURE".into()
}
}
fn main() -> Result<()> {
run().map_err(|error| anyhow::anyhow!(safe_error_code(&error)))
}
fn run() -> Result<()> {
let args: Vec<_> = std::env::args_os().skip(1).collect();
if args == [std::ffi::OsString::from("--help")] {
println!(
@@ -607,16 +630,7 @@ fn main() -> Result<()> {
&mut sync,
&sync_clock,
) {
let text = error.to_string();
let code = if text.len() <= 96
&& text
.bytes()
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
{
text.as_str()
} else {
"HELPER_VALIDATION_OR_IO_FAILURE"
};
let code = safe_error_code(&error);
eprintln!("helper request rejected: {code}");
}
}
@@ -627,6 +641,41 @@ fn main() -> Result<()> {
mod tests {
use super::*;
#[test]
fn sqlite_full_is_sanitized_and_failed_transaction_preserves_history() {
let db = rusqlite::Connection::open_in_memory().unwrap();
rmm_privileged_helper::bounded_journal_capacity(&db).unwrap();
db.execute_batch("CREATE TABLE witness(value BLOB);INSERT INTO witness VALUES('retained');PRAGMA max_page_count=4;").unwrap();
db.execute_batch("BEGIN;INSERT INTO witness VALUES('must roll back');")
.unwrap();
let error = db
.execute("INSERT INTO witness VALUES(zeroblob(1048576))", [])
.unwrap_err();
let wrapped = anyhow::Error::new(error).context("private SQL context must never leak");
assert_eq!(
safe_error_code(&wrapped),
"JOURNAL_STORAGE_CAPACITY_BLOCKED"
);
assert_eq!(
db.query_row("SELECT count(*) FROM witness", [], |r| r.get::<_, u32>(0))
.unwrap(),
1
);
assert_eq!(
db.query_row("PRAGMA integrity_check", [], |r| r.get::<_, String>(0))
.unwrap(),
"ok"
);
assert_eq!(
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
.unwrap(),
4
);
assert_eq!(
safe_error_code(&anyhow::anyhow!("private SQL text")),
"HELPER_VALIDATION_OR_IO_FAILURE"
);
}
#[test]
fn apt_only_profile_denies_scripts_services_reboot_and_wrong_shared_state() {
let id = uuid::Uuid::new_v4();
let mut config:Config=serde_json::from_value(serde_json::json!({"profile":"apt_only",
+51
View File
@@ -85,6 +85,57 @@ fn directory() -> std::path::PathBuf {
path
}
#[test]
fn journal_capacity_is_finite_and_retains_original_consumption_on_reopen() {
let dir = directory();
let uid = unsafe { libc::geteuid() };
let mut db = open_ledger(&dir, uid).unwrap();
let (binding, request, _, now) = fixture();
let (_, permit) = binding.verify(&request, now).unwrap();
consume(&mut db, &permit).unwrap();
save_start(&db, &permit, &request).unwrap();
let original: Vec<u8> = db
.query_row("SELECT start FROM execution_evidence", [], |r| r.get(0))
.unwrap();
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
drop(db);
let mut db = open_ledger(&dir, uid).unwrap();
assert_eq!(
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
.unwrap(),
4096
);
assert_eq!(
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
.unwrap(),
65536
);
let preserved: (String, Option<Vec<u8>>, Vec<u8>) = db.query_row("SELECT c.grant_id,c.result,e.start FROM consumed c JOIN execution_evidence e USING(grant_id)", [], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert_eq!(preserved, (permit.grant_id.to_string(), None, original));
assert!(consume(&mut db, &permit).is_err());
drop(db);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn journal_capacity_rejects_unexpected_page_size_without_rewriting_history() {
let dir = directory();
let db = rusqlite::Connection::open(dir.join("odd.sqlite")).unwrap();
db.execute_batch("PRAGMA page_size=8192;CREATE TABLE witness(value TEXT);INSERT INTO witness VALUES('retained');").unwrap();
assert!(bounded_journal_capacity(&db).is_err());
assert_eq!(
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
.unwrap(),
8192
);
assert_eq!(
db.query_row("SELECT value FROM witness", [], |r| r.get::<_, String>(0))
.unwrap(),
"retained"
);
drop(db);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn exact_signature_and_every_permit_binding() {
let (binding, request, key, now) = fixture();
assert!(binding.verify(&request, now).is_ok());
+1
View File
@@ -157,6 +157,7 @@ fn load() -> Result<Option<Loaded>> {
}))
}
pub(crate) fn validate_environment() -> Result<()> {
super::dev_transport_faults::validate_environment()?;
load().map(|_| ())
}
+246
View File
@@ -0,0 +1,246 @@
//! Development-only, exact-device WSS handshake rejection after normal mTLS
//! binding. HTTPS and every execution/approval path remain untouched.
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Duration, Utc};
use rmm_domain::jobs::hash;
use serde::Deserialize;
use serde_json::{Value, json};
use sqlx::PgPool;
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
use uuid::Uuid;
#[derive(Clone, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct Config {
schema_version: u32,
fault_id: Uuid,
tenant_id: Uuid,
agent_id: Uuid,
issued_at: DateTime<Utc>,
expires_at: DateTime<Utc>,
}
impl Config {
fn validate(&self) -> Result<()> {
ensure!(
self.schema_version == 1
&& !self.fault_id.is_nil()
&& !self.tenant_id.is_nil()
&& !self.agent_id.is_nil()
&& self.issued_at.timestamp() > 0
&& self.expires_at > self.issued_at
&& self.expires_at - self.issued_at <= Duration::minutes(15),
"DEV_TRANSPORT_FAULT_CONFIG"
);
Ok(())
}
fn active(&self, tenant: Uuid, agent: Uuid, now: DateTime<Utc>) -> bool {
self.tenant_id == tenant
&& self.agent_id == agent
&& now >= self.issued_at
&& now < self.expires_at
}
}
fn configured_path(
flag: Option<&str>,
path: Option<&str>,
profile: &str,
dev: &str,
) -> Result<Option<PathBuf>> {
match (flag, path) {
(None | Some("false"), None) => Ok(None),
(Some("true"), Some("/root-policy/transport-fault.json")) => {
ensure!(
profile == "development" && dev == "true",
"DEV_TRANSPORT_FAULT_PROFILE"
);
Ok(Some(PathBuf::from("/root-policy/transport-fault.json")))
}
_ => anyhow::bail!("DEV_TRANSPORT_FAULT_EXPLICIT_OPT_IN"),
}
}
fn load() -> Result<Option<(Config, String)>> {
let flag = std::env::var("RMM_DEV_TRANSPORT_FAULTS").ok();
let path = std::env::var("RMM_DEV_TRANSPORT_FAULT_FILE").ok();
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
return Ok(None);
};
let directory = fs::symlink_metadata(path.parent().context("DEV_TRANSPORT_FAULT_PARENT")?)?;
let metadata = fs::symlink_metadata(&path)?;
ensure!(
directory.is_dir()
&& directory.uid() == 10001
&& directory.mode() & 0o777 == 0o700
&& metadata.is_file()
&& metadata.uid() == 10001
&& metadata.mode() & 0o777 == 0o600
&& metadata.nlink() == 1
&& metadata.len() <= 4096,
"DEV_TRANSPORT_FAULT_PRIVATE_CONFIG"
);
let raw = fs::read(path)?;
ensure!(raw.len() <= 4096, "DEV_TRANSPORT_FAULT_CONFIG_SIZE");
let config: Config = serde_json::from_slice(&raw)?;
config.validate()?;
Ok(Some((config, hash(&raw))))
}
pub(crate) fn validate_environment() -> Result<()> {
load().map(|_| ())
}
fn slot_id(fault: Uuid, slot: u8) -> Uuid {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(format!("rmm.dev-transport-fault.v1:{fault}:{slot}"));
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
}
fn same_configuration(value: &Value, sha256: &str) -> Result<()> {
ensure!(
value.get("config_sha256") == Some(&json!(sha256)),
"DEV_TRANSPORT_FAULT_ID_REUSED"
);
Ok(())
}
/// Caller has already checked certificate expiry and authoritative device_binding.
/// No upgrade/ClientHello/JobOffer occurs on this path. At most two narrow durable
/// observations are inserted; every matching WSS request stays denied until expiry.
pub(crate) async fn reject_wss(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
certificate_sha256: &str,
) -> Result<bool> {
let Some((config, sha256)) = load()? else {
return Ok(false);
};
if !config.active(tenant, agent, Utc::now()) {
return Ok(false);
}
ensure!(
certificate_sha256.len() == 64
&& certificate_sha256
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
"DEV_TRANSPORT_FAULT_PEER_FINGERPRINT"
);
let mut tx = rmm_persistence::tenant_tx(pool, tenant).await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("dev-transport-fault:{}", config.fault_id))
.execute(&mut *tx)
.await?;
for slot in 1..=2 {
let id = slot_id(config.fault_id, slot);
let existing: Option<Value> = sqlx::query_scalar(
"SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_transport_fault_adapter'",
)
.bind(id)
.fetch_optional(&mut *tx)
.await?;
if let Some(value) = existing {
same_configuration(&value, &sha256)?;
} else {
let value = json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
"fault_id":config.fault_id,"agent_id":agent,"tenant_id":tenant,"slot":slot,
"config_sha256":sha256,"certificate_sha256":certificate_sha256,
"handshake_id":Uuid::new_v4(),"status":503,
"detail":"authenticated_mtls_device_wss_rejected_before_upgrade"});
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_transport_fault_adapter','dev_transport_fault.wss_rejected',$3) ON CONFLICT(tenant_id,event_id) DO NOTHING")
.bind(tenant).bind(id).bind(value).execute(&mut *tx).await?;
break;
}
}
tx.commit().await?;
Ok(true)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn opt_in_is_explicit_and_production_activation_is_rejected() {
assert!(
configured_path(None, None, "production", "false")
.unwrap()
.is_none()
);
assert!(
configured_path(Some("false"), None, "production", "false")
.unwrap()
.is_none()
);
for (flag, path, profile, dev) in [
(
Some("true"),
Some("/root-policy/transport-fault.json"),
"production",
"true",
),
(
Some("true"),
Some("/root-policy/transport-fault.json"),
"development",
"false",
),
(
Some("false"),
Some("/root-policy/transport-fault.json"),
"development",
"true",
),
(Some("true"), None, "development", "true"),
(Some("true"), Some("/etc/passwd"), "development", "true"),
] {
assert!(configured_path(flag, path, profile, dev).is_err());
}
assert!(
configured_path(
Some("true"),
Some("/root-policy/transport-fault.json"),
"development",
"true"
)
.unwrap()
.is_some()
);
}
#[test]
fn exact_scope_and_bounded_window_never_affect_another_device() {
let now = Utc::now();
let mut config = Config {
schema_version: 1,
fault_id: Uuid::new_v4(),
tenant_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
issued_at: now,
expires_at: now + Duration::minutes(15),
};
config.validate().unwrap();
assert!(config.active(config.tenant_id, config.agent_id, now));
assert!(!config.active(Uuid::new_v4(), config.agent_id, now));
assert!(!config.active(config.tenant_id, Uuid::new_v4(), now));
assert!(!config.active(
config.tenant_id,
config.agent_id,
now - Duration::seconds(1)
));
assert!(!config.active(config.tenant_id, config.agent_id, config.expires_at));
config.expires_at += Duration::milliseconds(1);
assert!(config.validate().is_err());
config.expires_at = now;
assert!(config.validate().is_err());
config.expires_at = now + Duration::minutes(1);
config.agent_id = Uuid::nil();
assert!(config.validate().is_err());
}
#[test]
fn slots_are_bounded_stable_and_cannot_change_configuration() {
let fault = Uuid::new_v4();
assert_eq!(slot_id(fault, 1), slot_id(fault, 1));
assert_ne!(slot_id(fault, 1), slot_id(fault, 2));
assert_ne!(slot_id(fault, 1), slot_id(Uuid::new_v4(), 1));
let value = json!({"config_sha256":"a".repeat(64)});
same_configuration(&value, &"a".repeat(64)).unwrap();
assert!(same_configuration(&value, &"b".repeat(64)).is_err());
assert!(same_configuration(&json!({}), &"a".repeat(64)).is_err());
}
}
+9
View File
@@ -92,6 +92,12 @@ pub async fn connect(
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
let (tenant, agent) = binding.ok_or(StatusCode::UNAUTHORIZED)?;
if dev_transport_faults::reject_wss(&state.pool, tenant, agent, &fingerprint)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
return Err(StatusCode::SERVICE_UNAVAILABLE);
}
Ok(ws
.max_message_size(MAX_MESSAGE)
.max_frame_size(MAX_MESSAGE)
@@ -226,3 +232,6 @@ async fn channel(
}
#[path = "dev_delivery_faults.rs"]
pub(crate) mod dev_delivery_faults;
#[path = "dev_transport_faults.rs"]
pub(crate) mod dev_transport_faults;