#[path = "../src/reconcile.rs"] mod reconcile; use chrono::{DateTime, Duration, Utc}; use openssl::{ pkey::{PKey, Private}, sign::Signer, }; use prost::Message; use reconcile::*; use rmm_domain::{jobs::*, reconciliation::*}; use rmm_privileged_helper::{Binding, Request}; use rmm_protocol::*; use rusqlite::{Connection, params}; use std::{fs, os::unix::fs::DirBuilderExt, path::PathBuf}; use uuid::Uuid; struct Clock { now: DateTime, trusted: bool, } impl SyncClock for Clock { fn now(&self) -> anyhow::Result> { anyhow::ensure!(self.trusted, "CLOCK_UNTRUSTED"); Ok(self.now) } } struct Fixture { dir: PathBuf, db: Connection, ledger: Connection, key: PKey, binding: Binding, channel: Channel, clock: Clock, } impl Drop for Fixture { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.dir); } } impl Fixture { fn new() -> Self { let dir = PathBuf::from("/tmp/opencode").join(format!("reconcile-{}", Uuid::new_v4())); fs::DirBuilder::new().mode(0o700).create(&dir).unwrap(); let db = Connection::open(dir.join("journal.sqlite")).unwrap(); db.execute_batch("PRAGMA synchronous=FULL;CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);INSERT INTO counter VALUES(1,10); CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT DEFAULT '',cancel_requested INTEGER DEFAULT 0,PRIMARY KEY(job,attempt)); CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE safety(key TEXT PRIMARY KEY,value TEXT);INSERT INTO safety VALUES('connected','1');").unwrap(); let ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(); let key = PKey::generate_ed25519().unwrap(); let public = key.raw_public_key().unwrap(); let binding = Binding { tenant_id: Uuid::new_v4(), asset_id: Uuid::new_v4(), agent_id: Uuid::new_v4(), enrollment_generation: 1, assignment_version: 3, public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(), }; let channel = Channel { connection_generation: 1, boot_id: Uuid::parse_str(&rmm_agent_linux::boot_id().unwrap()).unwrap(), }; Self { dir, db, ledger, key, binding, channel, clock: Clock { now: Utc::now(), trusted: true, }, } } fn client(&self) -> ClientSync { ClientSync::new(self.binding.clone()) } fn prepare(&mut self, client: &mut ClientSync) -> PreparedSync { client .prepare( &mut self.db, &self.ledger, self.channel.clone(), None, &self.clock, ) .unwrap() } fn decision(&self, prepared: &PreparedSync, ready: bool) -> ReconcileDecisionPayload { let request = &prepared.request; let attempts = request .attempts .iter() .map(|local| { let bytes: Vec = self .db .query_row( "SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2", params![local.job_id, local.attempt_id], |r| r.get(0), ) .unwrap(); let offer = JobOffer::decode(bytes.as_slice()).unwrap(); let job: JobPayload = serde_json::from_slice(&offer.payload_bytes).unwrap(); let grant = if local.grant_id.is_empty() { None } else { Some(Uuid::parse_str(&local.grant_id).unwrap()) }; let final_state = matches!( local.state.as_str(), "succeeded" | "failed" | "cancelled" | "expired" ); AttemptDecision { job_id: job.job_id, attempt_id: job.attempt_id, job_payload_hash: local.job_payload_hash.clone(), grant_id: grant, authorization_id: grant .map(|_| Uuid::parse_str(&offer.authorization_id).unwrap()), plan_hash: grant.map(|_| offer.plan_hash), action: job.action.name().into(), execution_identity: job.execution_identity, revision: local.revision, outcome_state: if local.state == "accepted" { "accepted".into() } else if final_state { local.state.clone() } else { "unknown".into() }, previous_outcome_sha256: (!local.outcome_sha256.is_empty()) .then(|| local.outcome_sha256.clone()), outcome_sha256: final_state .then(|| local.outcome_sha256.clone()) .filter(|s| !s.is_empty()), outcome_bytes: vec![], original_boot_id: (!local.original_boot_id.is_empty()) .then(|| Uuid::parse_str(&local.original_boot_id).unwrap()), unlock: final_state && local.consumption == "consumed", reason: if local.state == "accepted" { "ACCEPTED_NOT_STARTED" } else if final_state { "DEFINITIVE_OUTCOME_RECONCILED" } else { "EFFECT_UNRESOLVED" } .into(), } }) .collect(); ReconcileDecisionPayload { domain: "rmm.reconciliation-decision".into(), schema_version: 1, decision_id: Uuid::new_v4(), request_id: Uuid::parse_str(&request.request_id).unwrap(), request_hash: hash(&request.encode_to_vec()), tenant_id: self.binding.tenant_id, asset_id: self.binding.asset_id, agent_id: self.binding.agent_id, enrollment_generation: self.binding.enrollment_generation, assignment_version: self.binding.assignment_version, connection_generation: self.channel.connection_generation, boot_id: self.channel.boot_id, journal_id: Uuid::parse_str(&request.journal_id).unwrap(), checkpoint: request.checkpoint, helper_journal_id: (!request.helper_journal_id.is_empty()) .then(|| Uuid::parse_str(&request.helper_journal_id).unwrap()), helper_checkpoint: request.helper_checkpoint, issued_at: self.clock.now - Duration::milliseconds(1), valid_until: self.clock.now + Duration::seconds(59), ready, helper_ready: ready && !request.helper_journal_id.is_empty(), reason: if ready { "CONSISTENT" } else { "ATTEMPT_RECONCILIATION_REQUIRED" } .into(), attempts, acknowledged_message_ids: vec![], key_id: hash(&self.key.raw_public_key().unwrap()), } } fn sign(&self, decision: &ReconcileDecisionPayload) -> SyncResponse { let bytes = encode(decision).unwrap(); SyncResponse { signature: Signer::new_without_digest(&self.key) .unwrap() .sign_oneshot_to_vec(&bytes) .unwrap(), decision_bytes: bytes, } } fn apply( &mut self, client: &mut ClientSync, decision: &ReconcileDecisionPayload, ) -> anyhow::Result { let response = self.sign(decision); client.apply(&mut self.db, &response, &self.clock) } fn add( &mut self, state: &str, consume: bool, outcome: Option, action: Action, ) -> (JobOffer, Option, Option) { let execution_identity = if matches!(action, Action::SetServiceState { .. }) { "root" } else { "agent_service" }; let job = JobPayload { schema_version: 1, tenant_id: self.binding.tenant_id, asset_id: self.binding.asset_id, agent_id: self.binding.agent_id, enrollment_generation: 1, assignment_version: 3, job_id: Uuid::new_v4(), attempt_id: Uuid::new_v4(), action, execution_identity: execution_identity.into(), not_before: self.clock.now - Duration::seconds(5), start_before: self.clock.now + Duration::minutes(5), max_runtime_seconds: 120, max_output_bytes: 4096, }; let offer = JobOffer { job_id: job.job_id.to_string(), attempt_id: job.attempt_id.to_string(), authorization_id: Uuid::new_v4().to_string(), plan_hash: hash(b"approved immutable plan"), job_payload_hash: hash(&encode(&job).unwrap()), payload_bytes: encode(&job).unwrap(), }; let mut grant = None; if consume { let p = ExecutionPermitPayload { domain: "rmm.execution-permit".into(), schema_version: 1, grant_id: Uuid::new_v4(), authorization_id: Uuid::parse_str(&offer.authorization_id).unwrap(), tenant_id: job.tenant_id, asset_id: job.asset_id, agent_id: job.agent_id, enrollment_generation: 1, assignment_version: 3, job_id: job.job_id, attempt_id: job.attempt_id, plan_hash: offer.plan_hash.clone(), job_payload_hash: offer.job_payload_hash.clone(), action: job.action.name().into(), execution_identity: job.execution_identity.clone(), not_before: self.clock.now - Duration::milliseconds(1), start_before: self.clock.now + Duration::seconds(59), max_runtime_seconds: 120, key_id: hash(&self.key.raw_public_key().unwrap()), }; let bytes = encode(&p).unwrap(); let request = Request { job_bytes: offer.payload_bytes.clone(), permit_bytes: bytes.clone(), signature: Signer::new_without_digest(&self.key) .unwrap() .sign_oneshot_to_vec(&bytes) .unwrap(), plan_hash: offer.plan_hash.clone(), authorization_id: p.authorization_id, }; rmm_privileged_helper::consume(&mut self.ledger, &p).unwrap(); rmm_privileged_helper::save_start(&self.ledger, &p, &request).unwrap(); if let Some(o) = &outcome { self.ledger .execute( "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1", params![p.grant_id.to_string(), canonical_outcome_bytes(o).unwrap()], ) .unwrap(); } grant = Some(p); } self.db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,?4,1,?5)",params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),state,grant.as_ref().map(|p|p.grant_id.to_string()).unwrap_or_default()]).unwrap(); let envelope = outcome.map(|o| { let sequence: i64 = self .db .query_row("UPDATE counter SET seq=seq+1 RETURNING seq", [], |r| { r.get(0) }) .unwrap(); let r = JobResult { job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(), revision: 1, state: o.state, grant_id: grant .as_ref() .map(|p| p.grant_id.to_string()) .unwrap_or_default(), stdout: o.stdout, stderr: o.stderr, output_truncated: o.output_truncated, reboot_required: o.reboot_required, exit_code: o.exit_code, evidence_json: o.evidence.to_string(), error_code: o.error_code, }; let e = Envelope { protocol_major: 1, protocol_minor: 0, message_id: Uuid::new_v4().to_string(), agent_id: self.binding.agent_id.to_string(), enrollment_generation: 1, connection_generation: self.channel.connection_generation, boot_id: self.channel.boot_id.to_string(), sequence: sequence as u64, observed_at: self.clock.now.timestamp_millis(), payload: Some(Payload::JobResult(r)), }; self.db .execute( "INSERT INTO job_events VALUES(?1,?2)", params![e.message_id, e.encode_to_vec()], ) .unwrap(); e }); (offer, grant, envelope) } } #[test] fn authenticated_helper_outcome_is_imported_as_new_durable_evidence_not_reexecuted() { let mut f = Fixture::new(); let mut unknown = success(); unknown.state = "unknown".into(); unknown.error_code = "HELPER_TRANSPORT_EFFECT_UNKNOWN".into(); unknown.evidence = serde_json::json!({"postconditions_passed":false}); let (offer, grant, old) = f.add( "unknown", true, Some(unknown), Action::SetServiceState { unit: "rmm-acceptance.service".into(), state: "running".into(), }, ); let grant = grant.unwrap(); let mut definitive = success(); definitive.stdout.clear(); definitive.exit_code = None; let bytes = canonical_outcome_bytes(&definitive).unwrap(); let mut helper = HelperSnapshot { journal_id: Uuid::new_v4(), checkpoint: 3, previous_decision_id: None, attempts: vec![SyncAttempt { job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(), job_payload_hash: offer.job_payload_hash.clone(), grant_id: grant.grant_id.to_string(), consumption: "consumed".into(), state: "succeeded".into(), revision: 1, original_boot_id: f.channel.boot_id.to_string(), outcome_sha256: hash(&bytes), outcome_bytes: bytes, ..Default::default() }], }; let mut c = f.client(); let p = c .prepare( &mut f.db, &f.ledger, f.channel.clone(), Some(&helper), &f.clock, ) .unwrap(); let imported = p.request.attempts[0].result_envelope.as_ref().unwrap(); assert_ne!(imported.message_id, old.unwrap().message_id); assert_eq!(p.request.attempts[0].state, "succeeded"); assert_eq!(p.request.attempts[0].revision, 2); let d = f.decision(&p, true); let applied = f.apply(&mut c, &d).unwrap(); assert!(applied.helper_instruction.is_some()); c.apply_consumption(&mut f.ledger, &f.clock).unwrap(); assert!(!c.can_start(&f.channel, true, &f.clock)); helper.previous_decision_id = Some(d.decision_id); c.confirm_helper(&helper).unwrap(); assert!(c.can_start(&f.channel, true, &f.clock)); let retained: i64 = f.db.query_row("SELECT count(*) FROM reconcile_results", [], |r| r.get(0)) .unwrap(); assert_eq!(retained, 2); let tombstones: i64 = f .ledger .query_row("SELECT count(*) FROM consumed", [], |r| r.get(0)) .unwrap(); assert_eq!(tombstones, 1); } #[test] fn helper_omitting_a_consumed_root_attempt_is_a_sticky_restore_fault() { let mut f = Fixture::new(); f.add( "running", true, None, Action::SetServiceState { unit: "rmm-acceptance.service".into(), state: "running".into(), }, ); let helper = HelperSnapshot { journal_id: Uuid::new_v4(), checkpoint: 1, previous_decision_id: None, attempts: vec![], }; let mut c = f.client(); assert!( c.prepare( &mut f.db, &f.ledger, f.channel.clone(), Some(&helper), &f.clock ) .is_err() ); assert!( c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock) .is_err() ); assert!(!c.can_start(&f.channel, true, &f.clock)); } #[test] fn definite_nonstart_result_is_unconsumed_not_ambiguous() { let mut f = Fixture::new(); let mut cancelled = success(); cancelled.state = "cancelled".into(); cancelled.stdout.clear(); cancelled.exit_code = None; cancelled.evidence = serde_json::json!({"not_started":true,"postconditions_passed":false}); f.add("cancelled", false, Some(cancelled), script()); let mut c = f.client(); let p = f.prepare(&mut c); assert_eq!(p.request.attempts[0].consumption, "unconsumed"); let d = f.decision(&p, true); f.apply(&mut c, &d).unwrap(); assert!(c.can_start(&f.channel, false, &f.clock)); } #[test] fn signed_terminal_nonstart_stays_unconsumed_across_sync_and_pending_denial() { for terminal in ["expired", "cancelled"] { let mut f = Fixture::new(); let (offer, _, _) = f.add("accepted", false, None, script()); let mut client = f.client(); let first = f.prepare(&mut client); assert_eq!(first.request.attempts[0].consumption, "unconsumed"); let mut decision = f.decision(&first, true); decision.attempts[0].outcome_state = terminal.into(); decision.attempts[0].reason = "DEFINITIVE_NONSTART".into(); f.apply(&mut client, &decision).unwrap(); assert!(client.can_start(&f.channel, false, &f.clock)); let second = f.prepare(&mut client); let local = &second.request.attempts[0]; assert_eq!(local.state, terminal); assert_eq!(local.consumption, "unconsumed"); assert!(local.grant_id.is_empty() && local.outcome_bytes.is_empty()); let mut repeated = f.decision(&second, true); repeated.attempts[0].reason = "DEFINITIVE_NONSTART".into(); f.apply(&mut client, &repeated).unwrap(); assert!(client.can_start(&f.channel, false, &f.clock)); // Reproduce the old server's rejected terminal->unknown response. // Keep its pending request and the last applied signed anchor, then // recover through a fresh channel/request with the ordinary protocol. let pending = f.prepare(&mut client); let mut denied = f.decision(&pending, false); denied.attempts[0].outcome_state = "unknown".into(); denied.attempts[0].reason = "GRANT_OR_EFFECT_UNRESOLVED".into(); assert!(f.apply(&mut client, &denied).is_err()); assert!(!client.can_start(&f.channel, false, &f.clock)); client = f.client(); f.channel.connection_generation += 1; let fresh = f.prepare(&mut client); assert_eq!( fresh.request.previous_request_id, pending.request.request_id ); assert_eq!( fresh.request.previous_decision_id, repeated.decision_id.to_string() ); assert_eq!(fresh.request.attempts[0].consumption, "unconsumed"); let mut recovered = f.decision(&fresh, true); recovered.attempts[0].reason = "DEFINITIVE_NONSTART".into(); f.apply(&mut client, &recovered).unwrap(); assert!(client.can_start(&f.channel, false, &f.clock)); let stored: (String, String, String) = f.db.query_row("SELECT job,attempt,state FROM job_attempts", [], |r| { Ok((r.get(0)?, r.get(1)?, r.get(2)?)) }) .unwrap(); assert_eq!(stored, (offer.job_id, offer.attempt_id, terminal.into())); assert_eq!( f.ledger .query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0)) .unwrap(), 0 ); assert_eq!( f.db.query_row("SELECT count(*) FROM reconcile_outcomes", [], |r| r .get::<_, i64>(0)) .unwrap(), 0 ); } } #[test] fn terminal_labels_never_override_grant_consumption_or_unclear_attempts() { for state in ["requesting", "running", "unknown"] { let mut f = Fixture::new(); f.add(state, false, None, script()); let mut client = f.client(); let prepared = f.prepare(&mut client); assert_eq!(prepared.request.attempts[0].consumption, "uncertain"); let decision = f.decision(&prepared, true); f.apply(&mut client, &decision).unwrap(); assert!(!client.can_start(&f.channel, false, &f.clock)); } for terminal in ["expired", "cancelled"] { for consumed in [false, true] { let mut f = Fixture::new(); f.add(terminal, consumed, None, script()); // A surviving grant ID also forbids the nonstart shortcut when // no consumption row exists. A real row always remains consumed. if !consumed { f.db.execute( "UPDATE job_attempts SET grant_id=?1", [Uuid::new_v4().to_string()], ) .unwrap(); } let mut client = f.client(); let prepared = f.prepare(&mut client); assert_eq!( prepared.request.attempts[0].consumption, if consumed { "consumed" } else { "uncertain" } ); let mut decision = f.decision(&prepared, true); decision.attempts[0].unlock = false; assert!(f.apply(&mut client, &decision).is_err()); assert!(!client.can_start(&f.channel, false, &f.clock)); assert_eq!( f.ledger .query_row( "SELECT count(*) FROM consumed WHERE result IS NULL", [], |r| r.get::<_, i64>(0) ) .unwrap(), i64::from(consumed) ); } } } #[test] fn negative_server_revision_never_rolls_back_local_unknown_revision() { let mut f = Fixture::new(); f.add("running", true, None, script()); let mut c = f.client(); let p = f.prepare(&mut c); assert_eq!(p.request.attempts[0].revision, 2); let mut d = f.decision(&p, false); d.attempts[0].revision = 1; f.apply(&mut c, &d).unwrap(); let revision: u64 = f.db.query_row("SELECT revision FROM job_attempts", [], |r| r.get(0)) .unwrap(); assert_eq!(revision, 2); assert!(!c.can_start(&f.channel, false, &f.clock)); } fn script() -> Action { Action::RunScript { interpreter: "posix_sh".into(), script_utf8: "printf hello".into(), arguments: vec![], expected_stdout: Some("hello".into()), windows_execution_profile: None, privileged_script_profile: None, } } fn success() -> ReconciledOutcome { ReconciledOutcome { state: "succeeded".into(), stdout: b"hello".to_vec(), stderr: vec![], exit_code: Some(0), output_truncated: false, reboot_required: false, evidence: serde_json::json!({"postconditions_passed":true,"z":{"b":2,"a":1}}), error_code: String::new(), } } #[test] fn clean_reconnect_and_reopen_require_new_signed_challenge() { let mut f = Fixture::new(); let mut client = f.client(); assert!(!client.can_start(&f.channel, false, &f.clock)); let prepared = f.prepare(&mut client); let decision = f.decision(&prepared, true); let response = f.sign(&decision); client.apply(&mut f.db, &response, &f.clock).unwrap(); assert!(client.can_start(&f.channel, false, &f.clock)); assert_eq!(client.validity_deadline(), Some(decision.valid_until)); assert!(!client.can_start(&f.channel, true, &f.clock)); let mut reopened = Connection::open(f.dir.join("journal.sqlite")).unwrap(); let mut process = f.client(); assert!(!process.can_start(&f.channel, false, &f.clock)); f.channel.connection_generation += 1; let next = process .prepare(&mut reopened, &f.ledger, f.channel.clone(), None, &f.clock) .unwrap(); assert_eq!( next.request.previous_decision_id, decision.decision_id.to_string() ); assert_eq!(next.request.journal_id, prepared.request.journal_id); assert!(next.request.checkpoint > prepared.request.checkpoint); assert!(process.apply(&mut reopened, &response, &f.clock).is_err()); let next_decision = f.decision(&next, true); process .apply(&mut reopened, &f.sign(&next_decision), &f.clock) .unwrap(); assert!(process.can_start(&f.channel, false, &f.clock)); let safety: String = reopened .query_row("SELECT value FROM safety WHERE key='connected'", [], |r| { r.get(0) }) .unwrap(); assert_eq!(safety, "1"); } #[test] fn exact_signatures_all_identity_channel_checkpoint_and_request_bindings() { let mut f = Fixture::new(); let mut client = f.client(); let prepared = f.prepare(&mut client); let d = f.decision(&prepared, true); assert!( verify_response( &f.binding, &prepared.request, &f.channel, &f.sign(&d), f.clock.now ) .is_ok() ); let value = serde_json::to_value(&d).unwrap(); for (field, replacement) in [ ("domain", serde_json::json!("rmm.execution-permit")), ("schema_version", serde_json::json!(2)), ("tenant_id", serde_json::json!(Uuid::new_v4())), ("asset_id", serde_json::json!(Uuid::new_v4())), ("agent_id", serde_json::json!(Uuid::new_v4())), ("enrollment_generation", serde_json::json!(2)), ("assignment_version", serde_json::json!(4)), ("connection_generation", serde_json::json!(2)), ("boot_id", serde_json::json!(Uuid::new_v4())), ("journal_id", serde_json::json!(Uuid::new_v4())), ("checkpoint", serde_json::json!(0)), ("request_id", serde_json::json!(Uuid::new_v4())), ("request_hash", serde_json::json!(hash(b"different"))), ("helper_journal_id", serde_json::json!(Uuid::new_v4())), ("helper_checkpoint", serde_json::json!(1)), ("key_id", serde_json::json!(hash(b"wrongkey"))), ] { let mut changed = value.clone(); changed[field] = replacement; let changed: ReconcileDecisionPayload = serde_json::from_value(changed).unwrap(); assert!( verify_response( &f.binding, &prepared.request, &f.channel, &f.sign(&changed), f.clock.now ) .is_err(), "{field}" ); } let mut forged = f.sign(&d); forged.signature[0] ^= 1; assert!(client.apply(&mut f.db, &forged, &f.clock).is_err()); let fake = PKey::generate_ed25519().unwrap(); let mut forged = f.sign(&d); forged.signature = Signer::new_without_digest(&fake) .unwrap() .sign_oneshot_to_vec(&forged.decision_bytes) .unwrap(); assert!(client.apply(&mut f.db, &forged, &f.clock).is_err()); let mut changed = f.sign(&d); changed.decision_bytes.push(b' '); assert!(client.apply(&mut f.db, &changed, &f.clock).is_err()); let mut expired = d.clone(); expired.valid_until = f.clock.now; assert!(f.apply(&mut client, &expired).is_err()); let mut unbounded = d; unbounded.valid_until = unbounded.issued_at + Duration::seconds(60) + Duration::milliseconds(1); assert!(f.apply(&mut client, &unbounded).is_err()); assert!(!client.can_start(&f.channel, false, &f.clock)); } #[test] fn accepted_unstarted_keeps_ids_and_never_consumes_or_executes() { let mut f = Fixture::new(); let (offer, _, _) = f.add("accepted", false, None, script()); let mut c = f.client(); let p = f.prepare(&mut c); assert_eq!(p.request.attempts[0].consumption, "unconsumed"); let d = f.decision(&p, true); let r = f.sign(&d); c.apply(&mut f.db, &r, &f.clock).unwrap(); assert!(c.can_start(&f.channel, false, &f.clock)); let count: i64 = f .ledger .query_row("SELECT count(*) FROM consumed", [], |r| r.get(0)) .unwrap(); assert_eq!(count, 0); let ids: (String, String, u64) = f.db.query_row("SELECT job,attempt,revision FROM job_attempts", [], |r| { Ok((r.get(0)?, r.get(1)?, r.get(2)?)) }) .unwrap(); assert_eq!(ids, (offer.job_id, offer.attempt_id, 1)); c.apply(&mut f.db, &r, &f.clock).unwrap(); assert!(c.can_start(&f.channel, false, &f.clock)); let mut forged = r; forged.signature[0] ^= 1; assert!(c.apply(&mut f.db, &forged, &f.clock).is_err()); assert!(!c.can_start(&f.channel, false, &f.clock)); } #[test] fn requesting_running_and_omitted_attempts_cannot_regain_readiness() { for state in ["requesting", "running"] { let mut f = Fixture::new(); f.add(state, state == "running", None, script()); let mut c = f.client(); let p = f.prepare(&mut c); assert_eq!(p.request.attempts[0].state, "unknown"); let mut d = f.decision(&p, true); d.attempts.clear(); assert!(f.apply(&mut c, &d).is_err()); let d = f.decision(&p, true); f.apply(&mut c, &d).unwrap(); assert!(!c.can_start(&f.channel, false, &f.clock)); let state: String = f.db.query_row("SELECT state FROM job_attempts", [], |r| r.get(0)) .unwrap(); assert_eq!(state, "unknown"); if p.request.attempts[0].consumption == "consumed" { let locked: i64 = f .ledger .query_row( "SELECT count(*) FROM consumed WHERE result IS NULL", [], |r| r.get(0), ) .unwrap(); assert_eq!(locked, 1); } } } #[test] fn final_result_preserves_original_envelope_outputs_and_exact_acks() { let mut f = Fixture::new(); let (_, _, original) = f.add("succeeded", true, Some(success()), script()); let original = original.unwrap(); let mut c = f.client(); let p = f.prepare(&mut c); assert_eq!( p.request.attempts[0].result_envelope, Some(original.clone()) ); assert_eq!( p.request.attempts[0].outcome_bytes, canonical_outcome_bytes(&success()).unwrap() ); let unrelated = Envelope { message_id: Uuid::new_v4().to_string(), ..Default::default() }; f.db.execute( "INSERT INTO job_events VALUES(?1,?2)", params![unrelated.message_id, unrelated.encode_to_vec()], ) .unwrap(); let mut d = f.decision(&p, true); d.acknowledged_message_ids = vec![Uuid::parse_str(&original.message_id).unwrap()]; let r = f.sign(&d); c.apply(&mut f.db, &r, &f.clock).unwrap(); assert!(!c.can_start(&f.channel, false, &f.clock)); c.apply_consumption(&mut f.ledger, &f.clock).unwrap(); c.apply_consumption(&mut f.ledger, &f.clock).unwrap(); assert!(c.can_start(&f.channel, false, &f.clock)); let pending: Vec = f.db.prepare("SELECT id FROM job_events") .unwrap() .query_map([], |r| r.get(0)) .unwrap() .collect::>() .unwrap(); assert_eq!(pending, vec![unrelated.message_id]); let archived: Vec = f.db.query_row( "SELECT envelope FROM reconcile_results WHERE id=?1", [&original.message_id], |r| r.get(0), ) .unwrap(); assert_eq!(Envelope::decode(archived.as_slice()).unwrap(), original); assert_eq!( f.ledger .query_row( "SELECT count(*) FROM consumed WHERE result IS NOT NULL", [], |r| r.get::<_, i64>(0) ) .unwrap(), 1 ); } #[test] fn foreign_ack_and_noncanonical_outcome_are_transactionally_rejected() { let mut f = Fixture::new(); f.add("succeeded", true, Some(success()), script()); let mut c = f.client(); let p = f.prepare(&mut c); let mut d = f.decision(&p, true); d.acknowledged_message_ids = vec![Uuid::new_v4()]; assert!(f.apply(&mut c, &d).is_err()); let mut d = f.decision(&p, true); d.attempts[0].outcome_bytes = serde_json::to_vec_pretty(&success()).unwrap(); d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes)); assert!(f.apply(&mut c, &d).is_err()); assert_eq!( f.db.query_row("SELECT count(*) FROM job_events", [], |r| r .get::<_, i64>(0)) .unwrap(), 1 ); } #[test] fn missing_server_job_fault_is_sticky_even_if_attempt_disappears_later() { let mut f = Fixture::new(); f.add("accepted", false, None, script()); let mut c = f.client(); let p = f.prepare(&mut c); let mut d = f.decision(&p, false); d.reason = "MISSING_SERVER_JOB".into(); d.attempts[0].reason = "MISSING_SERVER_JOB".into(); d.attempts[0].action.clear(); d.attempts[0].execution_identity.clear(); f.apply(&mut c, &d).unwrap(); f.db.execute("DELETE FROM job_attempts", []).unwrap(); let mut fresh = f.client(); assert!( fresh .prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock) .is_err() ); assert!(!fresh.can_start(&f.channel, false, &f.clock)); } #[test] fn rollback_and_false_clock_close_mutations_but_do_not_destroy_journal() { let mut f = Fixture::new(); let mut c = f.client(); let p = f.prepare(&mut c); let d = f.decision(&p, true); f.apply(&mut c, &d).unwrap(); f.clock.trusted = false; assert!(!c.can_start(&f.channel, false, &f.clock)); assert!( c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock) .is_err() ); f.clock.trusted = true; f.db.execute("UPDATE counter SET seq=0", []).unwrap(); assert!( c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock) .is_err() ); f.db.execute("UPDATE counter SET seq=999", []).unwrap(); assert!( c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock) .is_err() ); let _native = NativeClock::default(); } #[test] fn uncertain_request_preserves_pending_request_and_last_applied_anchor() { let mut f = Fixture::new(); let mut c = f.client(); let first = f.prepare(&mut c); let d = f.decision(&first, true); f.apply(&mut c, &d).unwrap(); let lost = f.prepare(&mut c); let _server_decision = f.sign(&f.decision(&lost, true)); c.close(); f.channel.connection_generation += 1; let recovered = f.prepare(&mut c); assert_ne!(lost.request.request_id, recovered.request.request_id); assert_eq!( recovered.request.previous_request_id, lost.request.request_id ); assert_eq!( recovered.request.previous_decision_id, d.decision_id.to_string() ); } #[test] fn helper_gate_requires_exact_signed_instruction_and_applied_checkpoint() { let mut f = Fixture::new(); let mut c = f.client(); let mut helper = HelperSnapshot { journal_id: Uuid::new_v4(), checkpoint: 42, previous_decision_id: None, attempts: vec![], }; let p = c .prepare( &mut f.db, &f.ledger, f.channel.clone(), Some(&helper), &f.clock, ) .unwrap(); assert_eq!(p.envelope.sequence, p.request.checkpoint); assert_eq!( p.envelope.payload, Some(Payload::SyncRequest(p.request.clone())) ); let d = f.decision(&p, true); let response = f.sign(&d); let applied = c.apply(&mut f.db, &response, &f.clock).unwrap(); assert_eq!(applied.decision_id, d.decision_id); assert_eq!(applied.reason, "CONSISTENT"); assert!(applied.acknowledged_message_ids.is_empty()); let instruction = applied.helper_instruction.unwrap(); assert_eq!(instruction.response, response); assert_eq!(instruction.checkpoint, 42); assert_eq!(instruction.journal_id, helper.journal_id); assert!(c.can_start(&f.channel, false, &f.clock)); assert!(!c.can_start(&f.channel, true, &f.clock)); assert!(c.confirm_helper(&helper).is_err()); helper.previous_decision_id = Some(instruction.decision_id); c.confirm_helper(&helper).unwrap(); assert!(c.can_start(&f.channel, true, &f.clock)); f.clock.now += Duration::seconds(60); assert!(!c.can_start(&f.channel, false, &f.clock)); assert!(!c.can_start(&f.channel, true, &f.clock)); } #[test] fn independent_helper_checkpoint_regression_is_sticky() { let mut f = Fixture::new(); let mut c = f.client(); let mut helper = HelperSnapshot { journal_id: Uuid::new_v4(), checkpoint: 5, previous_decision_id: None, attempts: vec![], }; let p = c .prepare( &mut f.db, &f.ledger, f.channel.clone(), Some(&helper), &f.clock, ) .unwrap(); let d = f.decision(&p, true); f.apply(&mut c, &d).unwrap(); helper.previous_decision_id = Some(d.decision_id); helper.checkpoint = 4; f.channel.connection_generation += 1; assert!( c.prepare( &mut f.db, &f.ledger, f.channel.clone(), Some(&helper), &f.clock ) .is_err() ); helper.checkpoint = 5; assert!( c.prepare( &mut f.db, &f.ledger, f.channel.clone(), Some(&helper), &f.clock ) .is_err() ); } #[test] fn reboot_keeps_lock_until_signed_new_authenticated_boot_inventory_evidence() { let mut f = Fixture::new(); let previous_boot = f.channel.boot_id; let mut awaiting = success(); awaiting.state = "awaiting_reboot".into(); awaiting.stdout.clear(); awaiting.exit_code = None; awaiting.reboot_required = true; awaiting.evidence = serde_json::json!({"postconditions_passed":false,"previous_boot_id":previous_boot}); let (_, grant, _) = f.add( "awaiting_reboot", true, Some(awaiting), Action::Reboot { reason: "mock only".into(), reconnect_timeout_seconds: 120, dependency_group: "acceptance".into(), }, ); let mut c = f.client(); let p = f.prepare(&mut c); assert_eq!( p.request.attempts[0].original_boot_id, previous_boot.to_string() ); let d = f.decision(&p, false); f.apply(&mut c, &d).unwrap(); assert!(!c.can_start(&f.channel, false, &f.clock)); let preserved: String = f.db.query_row("SELECT state FROM job_attempts", [], |r| r.get(0)) .unwrap(); assert_eq!(preserved, "awaiting_reboot"); f.channel.connection_generation += 1; f.channel.boot_id = Uuid::new_v4(); let p = f.prepare(&mut c); let mut d = f.decision(&p, true); let mut definitive = success(); definitive.stdout.clear(); definitive.exit_code = None; definitive.evidence = serde_json::json!({"postconditions_passed":true,"new_boot_id":previous_boot,"reconciled_after_reboot":true,"inventory_message_id":Uuid::new_v4()}); d.attempts[0].outcome_state = "succeeded".into(); d.attempts[0].revision += 1; d.attempts[0].unlock = true; d.attempts[0].outcome_bytes = canonical_outcome_bytes(&definitive).unwrap(); d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes)); assert!(f.apply(&mut c, &d).is_err()); definitive.evidence["new_boot_id"] = serde_json::json!(f.channel.boot_id); d.attempts[0].outcome_bytes = canonical_outcome_bytes(&definitive).unwrap(); d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes)); f.apply(&mut c, &d).unwrap(); assert!(!c.can_start(&f.channel, false, &f.clock)); c.apply_consumption(&mut f.ledger, &f.clock).unwrap(); assert!(c.can_start(&f.channel, false, &f.clock)); let grant = grant.unwrap(); let result: Vec = f .ledger .query_row( "SELECT result FROM consumed WHERE grant_id=?1", [grant.grant_id.to_string()], |r| r.get(0), ) .unwrap(); assert_eq!(result, canonical_outcome_bytes(&definitive).unwrap()); assert_eq!( f.ledger .query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0)) .unwrap(), 1 ); } #[test] fn independent_local_outcome_mismatch_never_unlocks() { let mut f = Fixture::new(); let (_, grant, _) = f.add("succeeded", true, Some(success()), script()); let mut c = f.client(); let p = f.prepare(&mut c); let d = f.decision(&p, true); f.apply(&mut c, &d).unwrap(); let mut altered = success(); altered.stdout = b"forged result".to_vec(); f.ledger .execute( "UPDATE execution_evidence SET outcome=?1", [canonical_outcome_bytes(&altered).unwrap()], ) .unwrap(); assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err()); assert!(!c.can_start(&f.channel, false, &f.clock)); let unresolved: i64 = f .ledger .query_row( "SELECT count(*) FROM consumed WHERE grant_id=?1 AND result IS NULL", [grant.unwrap().grant_id.to_string()], |r| r.get(0), ) .unwrap(); assert_eq!(unresolved, 1); } #[test] fn signed_attempt_mutation_or_terminal_regression_is_rejected() { let mut f = Fixture::new(); f.add("succeeded", true, Some(success()), script()); let mut c = f.client(); let p = f.prepare(&mut c); let d = f.decision(&p, true); let mut changed = d.clone(); changed.attempts[0].outcome_state = "unknown".into(); changed.attempts[0].unlock = false; assert!(f.apply(&mut c, &changed).is_err()); let mut changed = d.clone(); changed.attempts[0].authorization_id = Some(Uuid::new_v4()); assert!(f.apply(&mut c, &changed).is_err()); let mut changed = d.clone(); changed.attempts[0].plan_hash = Some(hash(b"changed plan")); assert!(f.apply(&mut c, &changed).is_err()); let mut changed = d; changed.attempts[0].previous_outcome_sha256 = Some(hash(b"other outcome")); assert!(f.apply(&mut c, &changed).is_err()); }