fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -92,6 +92,92 @@ fn hex_hash(s: &str) -> bool {
|
||||
s.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}
|
||||
|
||||
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
|
||||
/// executor thread from the previous process can still deliver its outcome.
|
||||
/// This does not execute, release consumption or invent a successful outcome.
|
||||
pub fn interrupted(
|
||||
db: &Connection,
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
) -> Result<Vec<(JobOffer, String)>> {
|
||||
let mut statement = db.prepare(
|
||||
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
|
||||
)?;
|
||||
let rows = statement.query_map([], |r| {
|
||||
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
|
||||
})?;
|
||||
let mut interrupted = Vec::new();
|
||||
for row in rows {
|
||||
let (bytes, grant) = row?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
if !grant.is_empty() {
|
||||
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
|
||||
.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[&grant],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (agent, generation, job_id, attempt, start) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Verify the historical signed request at its original valid time;
|
||||
// this is evidence only, never permission to start it again.
|
||||
let (_, verified) = binding.verify(&request, permit.not_before)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
} else {
|
||||
// A request may have committed remotely without its reply arriving.
|
||||
// It can only become local unknown, never a fabricated grant/result.
|
||||
let state: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
|
||||
}
|
||||
interrupted.push((offer, grant));
|
||||
}
|
||||
Ok(interrupted)
|
||||
}
|
||||
|
||||
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
)? == 1,
|
||||
"INTERRUPTED_REQUEST_STATE"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
|
||||
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
||||
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
|
||||
|
||||
@@ -209,6 +209,29 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
/// New process only. Persist honest uncertainty before any helper snapshot
|
||||
/// can block; the ordinary first-channel replay publishes it before Sync.
|
||||
fn recover_interrupted(&mut self) -> Result<()> {
|
||||
let Some(binding) = &self.binding else {
|
||||
return Ok(());
|
||||
};
|
||||
let interrupted = jobs::interrupted(&self.db, &self.ledger, binding)?;
|
||||
for (offer, grant) in interrupted {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(&self.db, &offer)?;
|
||||
} else {
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
// No finish/unlock here. Only normal bound reconciliation can release
|
||||
// a consumed attempt once independent definitive evidence is available.
|
||||
Ok(())
|
||||
}
|
||||
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
// A large replay backlog must not monopolize heartbeat/cancel handling.
|
||||
// On backpressure timeout, discard this channel rather than continuing
|
||||
@@ -789,6 +812,7 @@ async fn main() -> Result<()> {
|
||||
certificates,
|
||||
rotation_policy,
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1_u64;
|
||||
loop {
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -18,6 +18,181 @@ use std::{
|
||||
|
||||
struct Clock(bool);
|
||||
#[test]
|
||||
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
|
||||
use openssl::sign::Signer;
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
|
||||
use rmm_privileged_helper::{Binding, Request};
|
||||
use rusqlite::params;
|
||||
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&dir).unwrap();
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let public = key.raw_public_key().unwrap();
|
||||
let binding = Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
};
|
||||
let now = Utc::now();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::RunScript {
|
||||
interpreter: "posix_sh".into(),
|
||||
script_utf8: "printf 'fixture only; never executed\\n'".into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: Some("fixture only; never executed\n".into()),
|
||||
},
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::minutes(5),
|
||||
max_runtime_seconds: 120,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let permit = ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: Uuid::new_v4(),
|
||||
authorization_id: Uuid::new_v4(),
|
||||
tenant_id: job.tenant_id,
|
||||
asset_id: job.asset_id,
|
||||
agent_id: job.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
plan_hash: hash(b"fixed test plan; no native approval"),
|
||||
job_payload_hash: hash(&encode(&job).unwrap()),
|
||||
action: "run_script".into(),
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(59),
|
||||
max_runtime_seconds: 120,
|
||||
key_id: hash(&public),
|
||||
};
|
||||
let mut request = Request {
|
||||
job_bytes: encode(&job).unwrap(),
|
||||
permit_bytes: encode(&permit).unwrap(),
|
||||
signature: vec![],
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
authorization_id: permit.authorization_id,
|
||||
};
|
||||
request.signature = Signer::new_without_digest(&key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&request.permit_bytes)
|
||||
.unwrap();
|
||||
let offer = JobOffer {
|
||||
job_id: job.job_id.to_string(),
|
||||
attempt_id: job.attempt_id.to_string(),
|
||||
authorization_id: permit.authorization_id.to_string(),
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
job_payload_hash: permit.job_payload_hash.clone(),
|
||||
payload_bytes: request.job_bytes.clone(),
|
||||
};
|
||||
let db = storage::open(&dir).unwrap();
|
||||
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
|
||||
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
|
||||
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
|
||||
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
|
||||
let mut queued_job = job.clone();
|
||||
queued_job.job_id = Uuid::new_v4();
|
||||
queued_job.attempt_id = Uuid::new_v4();
|
||||
let mut queued = offer.clone();
|
||||
queued.job_id = queued_job.job_id.to_string();
|
||||
queued.attempt_id = queued_job.attempt_id.to_string();
|
||||
queued.payload_bytes = encode(&queued_job).unwrap();
|
||||
queued.job_payload_hash = hash(&queued.payload_bytes);
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
|
||||
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
let (itx, irx) = tokio::sync::mpsc::channel(1);
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&dir.join("certificates"),
|
||||
&[],
|
||||
binding.agent_id,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
let mut agent = Agent {
|
||||
address: "127.0.0.1:1".parse().unwrap(),
|
||||
ca: vec![],
|
||||
id: binding.agent_id.to_string(),
|
||||
boot: rmm_agent_linux::boot_id().unwrap(),
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db,
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding: Some(binding.clone()),
|
||||
ledger,
|
||||
completed_tx: tx,
|
||||
completed_rx: rx,
|
||||
inventory_tx: itx,
|
||||
inventory_rx: irx,
|
||||
inventory_inflight: false,
|
||||
sync: Some(reconcile::ClientSync::new(binding.clone())),
|
||||
sync_clock: Clock(true),
|
||||
helpers: None,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode: wire_channel::Mode::Wss,
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
};
|
||||
// A forged local consumption/start cannot become a purported grant result.
|
||||
agent
|
||||
.ledger
|
||||
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
|
||||
.unwrap();
|
||||
assert!(agent.recover_interrupted().is_err());
|
||||
assert!(jobs::events(&agent.db).unwrap().is_empty());
|
||||
agent
|
||||
.ledger
|
||||
.execute(
|
||||
"UPDATE consumed SET agent=?1",
|
||||
[binding.agent_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
agent.recover_interrupted().unwrap();
|
||||
let original = jobs::events(&agent.db).unwrap();
|
||||
assert_eq!(original.len(), 1);
|
||||
let Some(Payload::JobResult(result)) = &original[0].payload else {
|
||||
panic!("real durable result required")
|
||||
};
|
||||
assert_eq!(result.state, "unknown");
|
||||
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
|
||||
assert_eq!(result.grant_id, permit.grant_id.to_string());
|
||||
assert_eq!(result.revision, 2);
|
||||
assert!(result.stdout.is_empty() && result.exit_code.is_none());
|
||||
assert!(!agent.fence.allows(1, true));
|
||||
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
|
||||
assert_eq!(jobs::events(&reopened).unwrap(), original);
|
||||
let requesting: (String, u64) = reopened
|
||||
.query_row(
|
||||
"SELECT state,revision FROM job_attempts WHERE job=?1",
|
||||
[&queued.job_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(requesting, ("unknown".into(), 1));
|
||||
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(retained, (1, 1, 1));
|
||||
// The same process/restart cannot invent another revision or message ID.
|
||||
agent.recover_interrupted().unwrap();
|
||||
assert_eq!(jobs::events(&agent.db).unwrap(), original);
|
||||
drop(reopened);
|
||||
drop(agent);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
|
||||
let gate = EffectFence::default();
|
||||
assert!(!gate.allows(1, false));
|
||||
|
||||
@@ -65,8 +65,9 @@ pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
rmm_privileged_helper::private_path(&path, uid, false)?;
|
||||
let db = rusqlite::Connection::open(path)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("PRAGMA max_page_count=16384;
|
||||
rmm_privileged_helper::bounded_journal_capacity(&db)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("
|
||||
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
|
||||
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
|
||||
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
|
||||
@@ -106,6 +107,29 @@ pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod passive_pressure_tests {
|
||||
#[test]
|
||||
fn agent_journal_capacity_is_finite_and_preserves_sequence_on_reopen() {
|
||||
let dir = std::env::temp_dir().join(format!("rmm-capacity-{}", uuid::Uuid::new_v4()));
|
||||
super::prepare_dir(&dir).unwrap();
|
||||
let mut db = super::open(&dir).unwrap();
|
||||
assert_eq!(super::next_sequence(&mut db).unwrap(), 1);
|
||||
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
|
||||
drop(db);
|
||||
let mut db = super::open(&dir).unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4096
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
65536
|
||||
);
|
||||
assert_eq!(super::next_sequence(&mut db).unwrap(), 2);
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn pending_durable_traffic_backpressures_collection_without_deletion() {
|
||||
let db = rusqlite::Connection::open_in_memory().unwrap();
|
||||
|
||||
@@ -204,6 +204,17 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
f.try_lock_exclusive()?;
|
||||
Ok(f)
|
||||
}
|
||||
/// Finite 256 MiB bound for durable agent/helper history. Unexpected page sizes
|
||||
/// are refused rather than silently multiplying the byte limit. No compaction
|
||||
/// or tombstone deletion occurs here; SQLITE_FULL must still fail closed.
|
||||
pub fn bounded_journal_capacity(db: &rusqlite::Connection) -> Result<()> {
|
||||
let page_size: u32 = db.query_row("PRAGMA page_size", [], |r| r.get(0))?;
|
||||
ensure!(page_size == 4096, "JOURNAL_PAGE_SIZE_UNSUPPORTED");
|
||||
let maximum: u32 = db.query_row("PRAGMA max_page_count=65536", [], |r| r.get(0))?;
|
||||
ensure!(maximum == 65536, "JOURNAL_STORAGE_CAPACITY_BLOCKED");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
let path = dir.join("consumption.sqlite");
|
||||
@@ -218,7 +229,8 @@ pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
f.sync_all()?;
|
||||
private_path(&path, uid, false)?;
|
||||
let db = rusqlite::Connection::open(path)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=16384;
|
||||
bounded_journal_capacity(&db)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE;
|
||||
CREATE TABLE IF NOT EXISTS consumed(grant_id TEXT PRIMARY KEY, agent TEXT NOT NULL, generation INTEGER NOT NULL,
|
||||
job TEXT NOT NULL, attempt TEXT NOT NULL, result BLOB, UNIQUE(agent,generation,job,attempt));
|
||||
CREATE TABLE IF NOT EXISTS execution_evidence(grant_id TEXT PRIMARY KEY, start BLOB NOT NULL, outcome BLOB);")?;
|
||||
|
||||
@@ -472,7 +472,30 @@ fn handle(
|
||||
stream.write_all(&bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
fn safe_error_code(error: &anyhow::Error) -> String {
|
||||
if error.chain().any(|cause| {
|
||||
matches!(cause.downcast_ref::<rusqlite::Error>(),
|
||||
Some(rusqlite::Error::SqliteFailure(code, _)) if code.code == rusqlite::ErrorCode::DiskFull)
|
||||
}) {
|
||||
return "JOURNAL_STORAGE_CAPACITY_BLOCKED".into();
|
||||
}
|
||||
let text = error.to_string();
|
||||
if text.len() <= 96
|
||||
&& text
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
{
|
||||
text
|
||||
} else {
|
||||
"HELPER_VALIDATION_OR_IO_FAILURE".into()
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
run().map_err(|error| anyhow::anyhow!(safe_error_code(&error)))
|
||||
}
|
||||
|
||||
fn run() -> Result<()> {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
if args == [std::ffi::OsString::from("--help")] {
|
||||
println!(
|
||||
@@ -607,16 +630,7 @@ fn main() -> Result<()> {
|
||||
&mut sync,
|
||||
&sync_clock,
|
||||
) {
|
||||
let text = error.to_string();
|
||||
let code = if text.len() <= 96
|
||||
&& text
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
{
|
||||
text.as_str()
|
||||
} else {
|
||||
"HELPER_VALIDATION_OR_IO_FAILURE"
|
||||
};
|
||||
let code = safe_error_code(&error);
|
||||
eprintln!("helper request rejected: {code}");
|
||||
}
|
||||
}
|
||||
@@ -627,6 +641,41 @@ fn main() -> Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn sqlite_full_is_sanitized_and_failed_transaction_preserves_history() {
|
||||
let db = rusqlite::Connection::open_in_memory().unwrap();
|
||||
rmm_privileged_helper::bounded_journal_capacity(&db).unwrap();
|
||||
db.execute_batch("CREATE TABLE witness(value BLOB);INSERT INTO witness VALUES('retained');PRAGMA max_page_count=4;").unwrap();
|
||||
db.execute_batch("BEGIN;INSERT INTO witness VALUES('must roll back');")
|
||||
.unwrap();
|
||||
let error = db
|
||||
.execute("INSERT INTO witness VALUES(zeroblob(1048576))", [])
|
||||
.unwrap_err();
|
||||
let wrapped = anyhow::Error::new(error).context("private SQL context must never leak");
|
||||
assert_eq!(
|
||||
safe_error_code(&wrapped),
|
||||
"JOURNAL_STORAGE_CAPACITY_BLOCKED"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM witness", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA integrity_check", [], |r| r.get::<_, String>(0))
|
||||
.unwrap(),
|
||||
"ok"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4
|
||||
);
|
||||
assert_eq!(
|
||||
safe_error_code(&anyhow::anyhow!("private SQL text")),
|
||||
"HELPER_VALIDATION_OR_IO_FAILURE"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn apt_only_profile_denies_scripts_services_reboot_and_wrong_shared_state() {
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let mut config:Config=serde_json::from_value(serde_json::json!({"profile":"apt_only",
|
||||
|
||||
@@ -85,6 +85,57 @@ fn directory() -> std::path::PathBuf {
|
||||
path
|
||||
}
|
||||
#[test]
|
||||
fn journal_capacity_is_finite_and_retains_original_consumption_on_reopen() {
|
||||
let dir = directory();
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
let mut db = open_ledger(&dir, uid).unwrap();
|
||||
let (binding, request, _, now) = fixture();
|
||||
let (_, permit) = binding.verify(&request, now).unwrap();
|
||||
consume(&mut db, &permit).unwrap();
|
||||
save_start(&db, &permit, &request).unwrap();
|
||||
let original: Vec<u8> = db
|
||||
.query_row("SELECT start FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
|
||||
drop(db);
|
||||
let mut db = open_ledger(&dir, uid).unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4096
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
65536
|
||||
);
|
||||
let preserved: (String, Option<Vec<u8>>, Vec<u8>) = db.query_row("SELECT c.grant_id,c.result,e.start FROM consumed c JOIN execution_evidence e USING(grant_id)", [], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(preserved, (permit.grant_id.to_string(), None, original));
|
||||
assert!(consume(&mut db, &permit).is_err());
|
||||
drop(db);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_capacity_rejects_unexpected_page_size_without_rewriting_history() {
|
||||
let dir = directory();
|
||||
let db = rusqlite::Connection::open(dir.join("odd.sqlite")).unwrap();
|
||||
db.execute_batch("PRAGMA page_size=8192;CREATE TABLE witness(value TEXT);INSERT INTO witness VALUES('retained');").unwrap();
|
||||
assert!(bounded_journal_capacity(&db).is_err());
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
8192
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT value FROM witness", [], |r| r.get::<_, String>(0))
|
||||
.unwrap(),
|
||||
"retained"
|
||||
);
|
||||
drop(db);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn exact_signature_and_every_permit_binding() {
|
||||
let (binding, request, key, now) = fixture();
|
||||
assert!(binding.verify(&request, now).is_ok());
|
||||
|
||||
@@ -157,6 +157,7 @@ fn load() -> Result<Option<Loaded>> {
|
||||
}))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
super::dev_transport_faults::validate_environment()?;
|
||||
load().map(|_| ())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Development-only, exact-device WSS handshake rejection after normal mTLS
|
||||
//! binding. HTTPS and every execution/approval path remain untouched.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use rmm_domain::jobs::hash;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::PgPool;
|
||||
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Config {
|
||||
schema_version: u32,
|
||||
fault_id: Uuid,
|
||||
tenant_id: Uuid,
|
||||
agent_id: Uuid,
|
||||
issued_at: DateTime<Utc>,
|
||||
expires_at: DateTime<Utc>,
|
||||
}
|
||||
impl Config {
|
||||
fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == 1
|
||||
&& !self.fault_id.is_nil()
|
||||
&& !self.tenant_id.is_nil()
|
||||
&& !self.agent_id.is_nil()
|
||||
&& self.issued_at.timestamp() > 0
|
||||
&& self.expires_at > self.issued_at
|
||||
&& self.expires_at - self.issued_at <= Duration::minutes(15),
|
||||
"DEV_TRANSPORT_FAULT_CONFIG"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn active(&self, tenant: Uuid, agent: Uuid, now: DateTime<Utc>) -> bool {
|
||||
self.tenant_id == tenant
|
||||
&& self.agent_id == agent
|
||||
&& now >= self.issued_at
|
||||
&& now < self.expires_at
|
||||
}
|
||||
}
|
||||
fn configured_path(
|
||||
flag: Option<&str>,
|
||||
path: Option<&str>,
|
||||
profile: &str,
|
||||
dev: &str,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
match (flag, path) {
|
||||
(None | Some("false"), None) => Ok(None),
|
||||
(Some("true"), Some("/root-policy/transport-fault.json")) => {
|
||||
ensure!(
|
||||
profile == "development" && dev == "true",
|
||||
"DEV_TRANSPORT_FAULT_PROFILE"
|
||||
);
|
||||
Ok(Some(PathBuf::from("/root-policy/transport-fault.json")))
|
||||
}
|
||||
_ => anyhow::bail!("DEV_TRANSPORT_FAULT_EXPLICIT_OPT_IN"),
|
||||
}
|
||||
}
|
||||
fn load() -> Result<Option<(Config, String)>> {
|
||||
let flag = std::env::var("RMM_DEV_TRANSPORT_FAULTS").ok();
|
||||
let path = std::env::var("RMM_DEV_TRANSPORT_FAULT_FILE").ok();
|
||||
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
|
||||
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
|
||||
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = fs::symlink_metadata(path.parent().context("DEV_TRANSPORT_FAULT_PARENT")?)?;
|
||||
let metadata = fs::symlink_metadata(&path)?;
|
||||
ensure!(
|
||||
directory.is_dir()
|
||||
&& directory.uid() == 10001
|
||||
&& directory.mode() & 0o777 == 0o700
|
||||
&& metadata.is_file()
|
||||
&& metadata.uid() == 10001
|
||||
&& metadata.mode() & 0o777 == 0o600
|
||||
&& metadata.nlink() == 1
|
||||
&& metadata.len() <= 4096,
|
||||
"DEV_TRANSPORT_FAULT_PRIVATE_CONFIG"
|
||||
);
|
||||
let raw = fs::read(path)?;
|
||||
ensure!(raw.len() <= 4096, "DEV_TRANSPORT_FAULT_CONFIG_SIZE");
|
||||
let config: Config = serde_json::from_slice(&raw)?;
|
||||
config.validate()?;
|
||||
Ok(Some((config, hash(&raw))))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
load().map(|_| ())
|
||||
}
|
||||
fn slot_id(fault: Uuid, slot: u8) -> Uuid {
|
||||
use sha2::{Digest, Sha256};
|
||||
let digest = Sha256::digest(format!("rmm.dev-transport-fault.v1:{fault}:{slot}"));
|
||||
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
|
||||
}
|
||||
fn same_configuration(value: &Value, sha256: &str) -> Result<()> {
|
||||
ensure!(
|
||||
value.get("config_sha256") == Some(&json!(sha256)),
|
||||
"DEV_TRANSPORT_FAULT_ID_REUSED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Caller has already checked certificate expiry and authoritative device_binding.
|
||||
/// No upgrade/ClientHello/JobOffer occurs on this path. At most two narrow durable
|
||||
/// observations are inserted; every matching WSS request stays denied until expiry.
|
||||
pub(crate) async fn reject_wss(
|
||||
pool: &PgPool,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
certificate_sha256: &str,
|
||||
) -> Result<bool> {
|
||||
let Some((config, sha256)) = load()? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !config.active(tenant, agent, Utc::now()) {
|
||||
return Ok(false);
|
||||
}
|
||||
ensure!(
|
||||
certificate_sha256.len() == 64
|
||||
&& certificate_sha256
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
|
||||
"DEV_TRANSPORT_FAULT_PEER_FINGERPRINT"
|
||||
);
|
||||
let mut tx = rmm_persistence::tenant_tx(pool, tenant).await?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
||||
.bind(format!("dev-transport-fault:{}", config.fault_id))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
for slot in 1..=2 {
|
||||
let id = slot_id(config.fault_id, slot);
|
||||
let existing: Option<Value> = sqlx::query_scalar(
|
||||
"SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_transport_fault_adapter'",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some(value) = existing {
|
||||
same_configuration(&value, &sha256)?;
|
||||
} else {
|
||||
let value = json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
|
||||
"fault_id":config.fault_id,"agent_id":agent,"tenant_id":tenant,"slot":slot,
|
||||
"config_sha256":sha256,"certificate_sha256":certificate_sha256,
|
||||
"handshake_id":Uuid::new_v4(),"status":503,
|
||||
"detail":"authenticated_mtls_device_wss_rejected_before_upgrade"});
|
||||
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_transport_fault_adapter','dev_transport_fault.wss_rejected',$3) ON CONFLICT(tenant_id,event_id) DO NOTHING")
|
||||
.bind(tenant).bind(id).bind(value).execute(&mut *tx).await?;
|
||||
break;
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn opt_in_is_explicit_and_production_activation_is_rejected() {
|
||||
assert!(
|
||||
configured_path(None, None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
configured_path(Some("false"), None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for (flag, path, profile, dev) in [
|
||||
(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"production",
|
||||
"true",
|
||||
),
|
||||
(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"false",
|
||||
),
|
||||
(
|
||||
Some("false"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"true",
|
||||
),
|
||||
(Some("true"), None, "development", "true"),
|
||||
(Some("true"), Some("/etc/passwd"), "development", "true"),
|
||||
] {
|
||||
assert!(configured_path(flag, path, profile, dev).is_err());
|
||||
}
|
||||
assert!(
|
||||
configured_path(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"true"
|
||||
)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn exact_scope_and_bounded_window_never_affect_another_device() {
|
||||
let now = Utc::now();
|
||||
let mut config = Config {
|
||||
schema_version: 1,
|
||||
fault_id: Uuid::new_v4(),
|
||||
tenant_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
issued_at: now,
|
||||
expires_at: now + Duration::minutes(15),
|
||||
};
|
||||
config.validate().unwrap();
|
||||
assert!(config.active(config.tenant_id, config.agent_id, now));
|
||||
assert!(!config.active(Uuid::new_v4(), config.agent_id, now));
|
||||
assert!(!config.active(config.tenant_id, Uuid::new_v4(), now));
|
||||
assert!(!config.active(
|
||||
config.tenant_id,
|
||||
config.agent_id,
|
||||
now - Duration::seconds(1)
|
||||
));
|
||||
assert!(!config.active(config.tenant_id, config.agent_id, config.expires_at));
|
||||
config.expires_at += Duration::milliseconds(1);
|
||||
assert!(config.validate().is_err());
|
||||
config.expires_at = now;
|
||||
assert!(config.validate().is_err());
|
||||
config.expires_at = now + Duration::minutes(1);
|
||||
config.agent_id = Uuid::nil();
|
||||
assert!(config.validate().is_err());
|
||||
}
|
||||
#[test]
|
||||
fn slots_are_bounded_stable_and_cannot_change_configuration() {
|
||||
let fault = Uuid::new_v4();
|
||||
assert_eq!(slot_id(fault, 1), slot_id(fault, 1));
|
||||
assert_ne!(slot_id(fault, 1), slot_id(fault, 2));
|
||||
assert_ne!(slot_id(fault, 1), slot_id(Uuid::new_v4(), 1));
|
||||
let value = json!({"config_sha256":"a".repeat(64)});
|
||||
same_configuration(&value, &"a".repeat(64)).unwrap();
|
||||
assert!(same_configuration(&value, &"b".repeat(64)).is_err());
|
||||
assert!(same_configuration(&json!({}), &"a".repeat(64)).is_err());
|
||||
}
|
||||
}
|
||||
@@ -92,6 +92,12 @@ pub async fn connect(
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
|
||||
let (tenant, agent) = binding.ok_or(StatusCode::UNAUTHORIZED)?;
|
||||
if dev_transport_faults::reject_wss(&state.pool, tenant, agent, &fingerprint)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
{
|
||||
return Err(StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
Ok(ws
|
||||
.max_message_size(MAX_MESSAGE)
|
||||
.max_frame_size(MAX_MESSAGE)
|
||||
@@ -226,3 +232,6 @@ async fn channel(
|
||||
}
|
||||
#[path = "dev_delivery_faults.rs"]
|
||||
pub(crate) mod dev_delivery_faults;
|
||||
|
||||
#[path = "dev_transport_faults.rs"]
|
||||
pub(crate) mod dev_transport_faults;
|
||||
|
||||
Reference in New Issue
Block a user