fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -18,6 +18,181 @@ use std::{
|
||||
|
||||
struct Clock(bool);
|
||||
#[test]
|
||||
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
|
||||
use openssl::sign::Signer;
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
|
||||
use rmm_privileged_helper::{Binding, Request};
|
||||
use rusqlite::params;
|
||||
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&dir).unwrap();
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let public = key.raw_public_key().unwrap();
|
||||
let binding = Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
};
|
||||
let now = Utc::now();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::RunScript {
|
||||
interpreter: "posix_sh".into(),
|
||||
script_utf8: "printf 'fixture only; never executed\\n'".into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: Some("fixture only; never executed\n".into()),
|
||||
},
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::minutes(5),
|
||||
max_runtime_seconds: 120,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let permit = ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: Uuid::new_v4(),
|
||||
authorization_id: Uuid::new_v4(),
|
||||
tenant_id: job.tenant_id,
|
||||
asset_id: job.asset_id,
|
||||
agent_id: job.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
plan_hash: hash(b"fixed test plan; no native approval"),
|
||||
job_payload_hash: hash(&encode(&job).unwrap()),
|
||||
action: "run_script".into(),
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(59),
|
||||
max_runtime_seconds: 120,
|
||||
key_id: hash(&public),
|
||||
};
|
||||
let mut request = Request {
|
||||
job_bytes: encode(&job).unwrap(),
|
||||
permit_bytes: encode(&permit).unwrap(),
|
||||
signature: vec![],
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
authorization_id: permit.authorization_id,
|
||||
};
|
||||
request.signature = Signer::new_without_digest(&key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&request.permit_bytes)
|
||||
.unwrap();
|
||||
let offer = JobOffer {
|
||||
job_id: job.job_id.to_string(),
|
||||
attempt_id: job.attempt_id.to_string(),
|
||||
authorization_id: permit.authorization_id.to_string(),
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
job_payload_hash: permit.job_payload_hash.clone(),
|
||||
payload_bytes: request.job_bytes.clone(),
|
||||
};
|
||||
let db = storage::open(&dir).unwrap();
|
||||
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
|
||||
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
|
||||
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
|
||||
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
|
||||
let mut queued_job = job.clone();
|
||||
queued_job.job_id = Uuid::new_v4();
|
||||
queued_job.attempt_id = Uuid::new_v4();
|
||||
let mut queued = offer.clone();
|
||||
queued.job_id = queued_job.job_id.to_string();
|
||||
queued.attempt_id = queued_job.attempt_id.to_string();
|
||||
queued.payload_bytes = encode(&queued_job).unwrap();
|
||||
queued.job_payload_hash = hash(&queued.payload_bytes);
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
|
||||
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
let (itx, irx) = tokio::sync::mpsc::channel(1);
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&dir.join("certificates"),
|
||||
&[],
|
||||
binding.agent_id,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
let mut agent = Agent {
|
||||
address: "127.0.0.1:1".parse().unwrap(),
|
||||
ca: vec![],
|
||||
id: binding.agent_id.to_string(),
|
||||
boot: rmm_agent_linux::boot_id().unwrap(),
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db,
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding: Some(binding.clone()),
|
||||
ledger,
|
||||
completed_tx: tx,
|
||||
completed_rx: rx,
|
||||
inventory_tx: itx,
|
||||
inventory_rx: irx,
|
||||
inventory_inflight: false,
|
||||
sync: Some(reconcile::ClientSync::new(binding.clone())),
|
||||
sync_clock: Clock(true),
|
||||
helpers: None,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode: wire_channel::Mode::Wss,
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
};
|
||||
// A forged local consumption/start cannot become a purported grant result.
|
||||
agent
|
||||
.ledger
|
||||
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
|
||||
.unwrap();
|
||||
assert!(agent.recover_interrupted().is_err());
|
||||
assert!(jobs::events(&agent.db).unwrap().is_empty());
|
||||
agent
|
||||
.ledger
|
||||
.execute(
|
||||
"UPDATE consumed SET agent=?1",
|
||||
[binding.agent_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
agent.recover_interrupted().unwrap();
|
||||
let original = jobs::events(&agent.db).unwrap();
|
||||
assert_eq!(original.len(), 1);
|
||||
let Some(Payload::JobResult(result)) = &original[0].payload else {
|
||||
panic!("real durable result required")
|
||||
};
|
||||
assert_eq!(result.state, "unknown");
|
||||
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
|
||||
assert_eq!(result.grant_id, permit.grant_id.to_string());
|
||||
assert_eq!(result.revision, 2);
|
||||
assert!(result.stdout.is_empty() && result.exit_code.is_none());
|
||||
assert!(!agent.fence.allows(1, true));
|
||||
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
|
||||
assert_eq!(jobs::events(&reopened).unwrap(), original);
|
||||
let requesting: (String, u64) = reopened
|
||||
.query_row(
|
||||
"SELECT state,revision FROM job_attempts WHERE job=?1",
|
||||
[&queued.job_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(requesting, ("unknown".into(), 1));
|
||||
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(retained, (1, 1, 1));
|
||||
// The same process/restart cannot invent another revision or message ID.
|
||||
agent.recover_interrupted().unwrap();
|
||||
assert_eq!(jobs::events(&agent.db).unwrap(), original);
|
||||
drop(reopened);
|
||||
drop(agent);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
|
||||
let gate = EffectFence::default();
|
||||
assert!(!gate.allows(1, false));
|
||||
|
||||
Reference in New Issue
Block a user