fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:11:45 +01:00
parent da7eb8109c
commit c512bd6fb2
36 changed files with 2556 additions and 38 deletions
+86
View File
@@ -92,6 +92,92 @@ fn hex_hash(s: &str) -> bool {
s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
/// executor thread from the previous process can still deliver its outcome.
/// This does not execute, release consumption or invent a successful outcome.
pub fn interrupted(
db: &Connection,
ledger: &Connection,
binding: &Binding,
) -> Result<Vec<(JobOffer, String)>> {
let mut statement = db.prepare(
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
)?;
let rows = statement.query_map([], |r| {
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
})?;
let mut interrupted = Vec::new();
for row in rows {
let (bytes, grant) = row?;
let offer = JobOffer::decode(bytes.as_slice())?;
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"INTERRUPTED_OFFER_BINDING"
);
if !grant.is_empty() {
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
[&grant],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
)
.optional()?;
let (agent, generation, job_id, attempt, start) =
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
ensure!(
agent == binding.agent_id.to_string()
&& generation == binding.enrollment_generation
&& job_id == offer.job_id
&& attempt == offer.attempt_id,
"INTERRUPTED_CONSUMPTION_BINDING"
);
let start: serde_json::Value = serde_json::from_slice(&start)?;
let request: Request = serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
// Verify the historical signed request at its original valid time;
// this is evidence only, never permission to start it again.
let (_, verified) = binding.verify(&request, permit.not_before)?;
ensure!(
verified == permit
&& permit.grant_id.to_string() == grant
&& request.job_bytes == offer.payload_bytes
&& request.plan_hash == offer.plan_hash
&& request.authorization_id.to_string() == offer.authorization_id,
"INTERRUPTED_START_BINDING"
);
} else {
// A request may have committed remotely without its reply arriving.
// It can only become local unknown, never a fabricated grant/result.
let state: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
}
interrupted.push((offer, grant));
}
Ok(interrupted)
}
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
ensure!(
db.execute(
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
params![offer.job_id, offer.attempt_id],
)? == 1,
"INTERRUPTED_REQUEST_STATE"
);
Ok(())
}
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
+24
View File
@@ -209,6 +209,29 @@ impl<C: reconcile::SyncClock> Agent<C> {
tx.commit()?;
Ok(())
}
/// New process only. Persist honest uncertainty before any helper snapshot
/// can block; the ordinary first-channel replay publishes it before Sync.
fn recover_interrupted(&mut self) -> Result<()> {
let Some(binding) = &self.binding else {
return Ok(());
};
let interrupted = jobs::interrupted(&self.db, &self.ledger, binding)?;
for (offer, grant) in interrupted {
if grant.is_empty() {
jobs::interrupted_request(&self.db, &offer)?;
} else {
self.record(
0,
&offer,
&grant,
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
)?;
}
}
// No finish/unlock here. Only normal bound reconciliation can release
// a consumed attempt once independent definitive evidence is available.
Ok(())
}
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
// A large replay backlog must not monopolize heartbeat/cancel handling.
// On backpressure timeout, discard this channel rather than continuing
@@ -789,6 +812,7 @@ async fn main() -> Result<()> {
certificates,
rotation_policy,
};
agent.recover_interrupted()?;
let mut backoff = 1_u64;
loop {
let started = Instant::now();
+175
View File
@@ -18,6 +18,181 @@ use std::{
struct Clock(bool);
#[test]
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
use openssl::sign::Signer;
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
use rmm_privileged_helper::{Binding, Request};
use rusqlite::params;
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
storage::prepare_dir(&dir).unwrap();
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf 'fixture only; never executed\\n'".into(),
arguments: vec![],
expected_stdout: Some("fixture only; never executed\n".into()),
},
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(5),
max_runtime_seconds: 120,
max_output_bytes: 4096,
};
let permit = ExecutionPermitPayload {
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: Uuid::new_v4(),
tenant_id: job.tenant_id,
asset_id: job.asset_id,
agent_id: job.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: job.job_id,
attempt_id: job.attempt_id,
plan_hash: hash(b"fixed test plan; no native approval"),
job_payload_hash: hash(&encode(&job).unwrap()),
action: "run_script".into(),
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::seconds(59),
max_runtime_seconds: 120,
key_id: hash(&public),
};
let mut request = Request {
job_bytes: encode(&job).unwrap(),
permit_bytes: encode(&permit).unwrap(),
signature: vec![],
plan_hash: permit.plan_hash.clone(),
authorization_id: permit.authorization_id,
};
request.signature = Signer::new_without_digest(&key)
.unwrap()
.sign_oneshot_to_vec(&request.permit_bytes)
.unwrap();
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: permit.authorization_id.to_string(),
plan_hash: permit.plan_hash.clone(),
job_payload_hash: permit.job_payload_hash.clone(),
payload_bytes: request.job_bytes.clone(),
};
let db = storage::open(&dir).unwrap();
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
let mut queued_job = job.clone();
queued_job.job_id = Uuid::new_v4();
queued_job.attempt_id = Uuid::new_v4();
let mut queued = offer.clone();
queued.job_id = queued_job.job_id.to_string();
queued.attempt_id = queued_job.attempt_id.to_string();
queued.payload_bytes = encode(&queued_job).unwrap();
queued.job_payload_hash = hash(&queued.payload_bytes);
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
let (tx, rx) = tokio::sync::mpsc::channel(1);
let (itx, irx) = tokio::sync::mpsc::channel(1);
let certificates = certificate_rotation::RotationStore::open(
&dir.join("certificates"),
&[],
binding.agent_id,
1,
)
.unwrap();
let mut agent = Agent {
address: "127.0.0.1:1".parse().unwrap(),
ca: vec![],
id: binding.agent_id.to_string(),
boot: rmm_agent_linux::boot_id().unwrap(),
instance: Uuid::new_v4().to_string(),
db,
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding: Some(binding.clone()),
ledger,
completed_tx: tx,
completed_rx: rx,
inventory_tx: itx,
inventory_rx: irx,
inventory_inflight: false,
sync: Some(reconcile::ClientSync::new(binding.clone())),
sync_clock: Clock(true),
helpers: None,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode: wire_channel::Mode::Wss,
certificates,
rotation_policy: certificate_rotation::RotationPolicy::default(),
};
// A forged local consumption/start cannot become a purported grant result.
agent
.ledger
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
.unwrap();
assert!(agent.recover_interrupted().is_err());
assert!(jobs::events(&agent.db).unwrap().is_empty());
agent
.ledger
.execute(
"UPDATE consumed SET agent=?1",
[binding.agent_id.to_string()],
)
.unwrap();
agent.recover_interrupted().unwrap();
let original = jobs::events(&agent.db).unwrap();
assert_eq!(original.len(), 1);
let Some(Payload::JobResult(result)) = &original[0].payload else {
panic!("real durable result required")
};
assert_eq!(result.state, "unknown");
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
assert_eq!(result.grant_id, permit.grant_id.to_string());
assert_eq!(result.revision, 2);
assert!(result.stdout.is_empty() && result.exit_code.is_none());
assert!(!agent.fence.allows(1, true));
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
assert_eq!(jobs::events(&reopened).unwrap(), original);
let requesting: (String, u64) = reopened
.query_row(
"SELECT state,revision FROM job_attempts WHERE job=?1",
[&queued.job_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(requesting, ("unknown".into(), 1));
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert_eq!(retained, (1, 1, 1));
// The same process/restart cannot invent another revision or message ID.
agent.recover_interrupted().unwrap();
assert_eq!(jobs::events(&agent.db).unwrap(), original);
drop(reopened);
drop(agent);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
let gate = EffectFence::default();
assert!(!gate.allows(1, false));
+26 -2
View File
@@ -65,8 +65,9 @@ pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
let uid = unsafe { libc::geteuid() };
rmm_privileged_helper::private_path(&path, uid, false)?;
let db = rusqlite::Connection::open(path)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
db.execute_batch("PRAGMA max_page_count=16384;
rmm_privileged_helper::bounded_journal_capacity(&db)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
db.execute_batch("
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
@@ -106,6 +107,29 @@ pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
#[cfg(test)]
mod passive_pressure_tests {
#[test]
fn agent_journal_capacity_is_finite_and_preserves_sequence_on_reopen() {
let dir = std::env::temp_dir().join(format!("rmm-capacity-{}", uuid::Uuid::new_v4()));
super::prepare_dir(&dir).unwrap();
let mut db = super::open(&dir).unwrap();
assert_eq!(super::next_sequence(&mut db).unwrap(), 1);
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
drop(db);
let mut db = super::open(&dir).unwrap();
assert_eq!(
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
.unwrap(),
4096
);
assert_eq!(
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
.unwrap(),
65536
);
assert_eq!(super::next_sequence(&mut db).unwrap(), 2);
drop(db);
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn pending_durable_traffic_backpressures_collection_without_deletion() {
let db = rusqlite::Connection::open_in_memory().unwrap();