fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:11:45 +01:00
parent da7eb8109c
commit c512bd6fb2
36 changed files with 2556 additions and 38 deletions
+175
View File
@@ -18,6 +18,181 @@ use std::{
struct Clock(bool);
#[test]
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
use openssl::sign::Signer;
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
use rmm_privileged_helper::{Binding, Request};
use rusqlite::params;
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
storage::prepare_dir(&dir).unwrap();
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf 'fixture only; never executed\\n'".into(),
arguments: vec![],
expected_stdout: Some("fixture only; never executed\n".into()),
},
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(5),
max_runtime_seconds: 120,
max_output_bytes: 4096,
};
let permit = ExecutionPermitPayload {
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: Uuid::new_v4(),
tenant_id: job.tenant_id,
asset_id: job.asset_id,
agent_id: job.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: job.job_id,
attempt_id: job.attempt_id,
plan_hash: hash(b"fixed test plan; no native approval"),
job_payload_hash: hash(&encode(&job).unwrap()),
action: "run_script".into(),
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::seconds(59),
max_runtime_seconds: 120,
key_id: hash(&public),
};
let mut request = Request {
job_bytes: encode(&job).unwrap(),
permit_bytes: encode(&permit).unwrap(),
signature: vec![],
plan_hash: permit.plan_hash.clone(),
authorization_id: permit.authorization_id,
};
request.signature = Signer::new_without_digest(&key)
.unwrap()
.sign_oneshot_to_vec(&request.permit_bytes)
.unwrap();
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: permit.authorization_id.to_string(),
plan_hash: permit.plan_hash.clone(),
job_payload_hash: permit.job_payload_hash.clone(),
payload_bytes: request.job_bytes.clone(),
};
let db = storage::open(&dir).unwrap();
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
let mut queued_job = job.clone();
queued_job.job_id = Uuid::new_v4();
queued_job.attempt_id = Uuid::new_v4();
let mut queued = offer.clone();
queued.job_id = queued_job.job_id.to_string();
queued.attempt_id = queued_job.attempt_id.to_string();
queued.payload_bytes = encode(&queued_job).unwrap();
queued.job_payload_hash = hash(&queued.payload_bytes);
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
let (tx, rx) = tokio::sync::mpsc::channel(1);
let (itx, irx) = tokio::sync::mpsc::channel(1);
let certificates = certificate_rotation::RotationStore::open(
&dir.join("certificates"),
&[],
binding.agent_id,
1,
)
.unwrap();
let mut agent = Agent {
address: "127.0.0.1:1".parse().unwrap(),
ca: vec![],
id: binding.agent_id.to_string(),
boot: rmm_agent_linux::boot_id().unwrap(),
instance: Uuid::new_v4().to_string(),
db,
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding: Some(binding.clone()),
ledger,
completed_tx: tx,
completed_rx: rx,
inventory_tx: itx,
inventory_rx: irx,
inventory_inflight: false,
sync: Some(reconcile::ClientSync::new(binding.clone())),
sync_clock: Clock(true),
helpers: None,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode: wire_channel::Mode::Wss,
certificates,
rotation_policy: certificate_rotation::RotationPolicy::default(),
};
// A forged local consumption/start cannot become a purported grant result.
agent
.ledger
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
.unwrap();
assert!(agent.recover_interrupted().is_err());
assert!(jobs::events(&agent.db).unwrap().is_empty());
agent
.ledger
.execute(
"UPDATE consumed SET agent=?1",
[binding.agent_id.to_string()],
)
.unwrap();
agent.recover_interrupted().unwrap();
let original = jobs::events(&agent.db).unwrap();
assert_eq!(original.len(), 1);
let Some(Payload::JobResult(result)) = &original[0].payload else {
panic!("real durable result required")
};
assert_eq!(result.state, "unknown");
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
assert_eq!(result.grant_id, permit.grant_id.to_string());
assert_eq!(result.revision, 2);
assert!(result.stdout.is_empty() && result.exit_code.is_none());
assert!(!agent.fence.allows(1, true));
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
assert_eq!(jobs::events(&reopened).unwrap(), original);
let requesting: (String, u64) = reopened
.query_row(
"SELECT state,revision FROM job_attempts WHERE job=?1",
[&queued.job_id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(requesting, ("unknown".into(), 1));
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert_eq!(retained, (1, 1, 1));
// The same process/restart cannot invent another revision or message ID.
agent.recover_interrupted().unwrap();
assert_eq!(jobs::events(&agent.db).unwrap(), original);
drop(reopened);
drop(agent);
fs::remove_dir_all(dir).unwrap();
}
#[test]
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
let gate = EffectFence::default();
assert!(!gate.allows(1, false));