Files
T
2026-11-14 16:35:11 +01:00

908 lines
32 KiB
Rust

#[path = "../src/reconcile.rs"]
mod reconcile;
use chrono::{DateTime, Duration, Utc};
use openssl::{
pkey::{PKey, Private},
sign::Signer,
};
use reconcile::*;
use rmm_domain::{jobs::*, reconciliation::*};
use rmm_privileged_helper::{Binding, Request};
use rusqlite::{Connection, params};
use std::{fs, os::unix::fs::DirBuilderExt, path::PathBuf};
use uuid::Uuid;
struct TestClock {
now: DateTime<Utc>,
trusted: bool,
}
impl Clock for TestClock {
fn now(&self) -> anyhow::Result<DateTime<Utc>> {
anyhow::ensure!(self.trusted, "CLOCK_UNTRUSTED");
Ok(self.now)
}
}
struct TestBoot(Uuid);
impl BootSource for TestBoot {
fn current_boot(&self) -> anyhow::Result<Uuid> {
Ok(self.0)
}
}
struct Fixture {
dir: PathBuf,
db: Connection,
guard: ExecutorGuard,
binding: Binding,
key: PKey<Private>,
clock: TestClock,
boot: TestBoot,
agent_journal: Uuid,
}
#[test]
fn checkpoint_repair_verifies_complete_signed_history_and_detects_removed_predecessor() {
let mut f = Fixture::new();
let mut sync = f.sync();
let first = f.snapshot(&mut sync, 7);
let request = f.request(&first);
let d = f.decision(&first, &request, 7, false);
let signed = f.sign(&d);
sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot)
.unwrap();
f.clock.now += Duration::seconds(1);
let second = f.snapshot(&mut sync, 7);
let request = f.request(&second);
let d = f.decision(&second, &request, 7, false);
let signed = f.sign(&d);
sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot)
.unwrap();
verify_checkpoint_repair_history(&f.db, &f.binding, None).unwrap();
f.db.execute(
"DELETE FROM helper_sync_decisions WHERE checkpoint=?1",
[first.checkpoint],
)
.unwrap();
assert!(verify_checkpoint_repair_history(&f.db, &f.binding, None).is_err());
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.dir);
}
}
impl Fixture {
fn new() -> Self {
let dir = std::env::temp_dir().join(format!("helper-sync-{}", Uuid::new_v4()));
fs::DirBuilder::new().mode(0o700).create(&dir).unwrap();
let uid = unsafe { libc::geteuid() };
let guard = ExecutorGuard::acquire(&dir, uid).unwrap();
let db = rmm_privileged_helper::open_ledger(&dir, uid).unwrap();
let key = PKey::generate_ed25519().unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 3,
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
};
let boot = TestBoot(NativeBoot.current_boot().unwrap());
Self {
dir,
db,
guard,
binding,
key,
clock: TestClock {
now: Utc::now(),
trusted: true,
},
boot,
agent_journal: Uuid::new_v4(),
}
}
fn sync(&self) -> HelperSync {
HelperSync::new(self.binding.clone(), &self.clock).unwrap()
}
fn snapshot(&mut self, sync: &mut HelperSync, generation: u64) -> Snapshot {
sync.snapshot(
&mut self.db,
&self.guard,
generation,
&self.clock,
&self.boot,
)
.unwrap()
}
fn request(&self, snapshot: &Snapshot) -> Vec<u8> {
request(
snapshot,
self.agent_journal,
100 + snapshot.checkpoint,
&snapshot.attempts,
)
}
fn decision(
&self,
snapshot: &Snapshot,
raw: &[u8],
generation: u64,
ready: bool,
) -> ReconcileDecisionPayload {
let mut d = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::nil(),
request_hash: hash(raw),
tenant_id: self.binding.tenant_id,
asset_id: self.binding.asset_id,
agent_id: self.binding.agent_id,
enrollment_generation: 1,
assignment_version: 3,
connection_generation: generation,
boot_id: self.boot.0,
journal_id: self.agent_journal,
checkpoint: 100 + snapshot.checkpoint,
helper_journal_id: Some(snapshot.journal_id),
helper_checkpoint: snapshot.checkpoint,
issued_at: self.clock.now,
valid_until: self.clock.now + Duration::seconds(60),
ready,
helper_ready: ready,
reason: if ready {
"CONSISTENT"
} else {
"ATTEMPT_RECONCILIATION_REQUIRED"
}
.into(),
attempts: snapshot
.attempts
.iter()
.map(|a| AttemptDecision {
job_id: a.job_id,
attempt_id: a.attempt_id,
job_payload_hash: a.job_payload_hash.clone(),
grant_id: Some(a.grant_id),
authorization_id: Some(a.authorization_id),
plan_hash: Some(a.plan_hash.clone()),
action: a.action.clone(),
execution_identity: "root".into(),
revision: a.revision + 1,
outcome_state: if matches!(
a.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
) {
a.state.clone()
} else {
"unknown".into()
},
previous_outcome_sha256: (!a.outcome_sha256.is_empty())
.then(|| a.outcome_sha256.clone()),
outcome_sha256: matches!(
a.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
)
.then(|| a.outcome_sha256.clone()),
outcome_bytes: vec![],
original_boot_id: Some(a.original_boot_id),
unlock: matches!(
a.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
),
reason: "TEST_SERVER_DECISION".into(),
})
.collect(),
acknowledged_message_ids: vec![],
key_id: hash(&self.key.raw_public_key().unwrap()),
};
// The fixture emits a canonical tag-1 request ID first, exactly as prost.
let len = usize::from(raw[1]);
d.request_id = Uuid::parse_str(std::str::from_utf8(&raw[2..2 + len]).unwrap()).unwrap();
d
}
fn sign(&self, d: &ReconcileDecisionPayload) -> SignedResponse {
let bytes = encode(d).unwrap();
SignedResponse {
signature: Signer::new_without_digest(&self.key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap(),
decision_bytes: bytes,
}
}
fn apply(
&mut self,
sync: &mut HelperSync,
d: &ReconcileDecisionPayload,
raw: &[u8],
) -> anyhow::Result<Applied> {
let r = self.sign(d);
sync.apply(&mut self.db, &self.guard, &r, raw, &self.clock, &self.boot)
}
fn add(
&mut self,
action: Action,
outcome: Option<ReconciledOutcome>,
) -> ExecutionPermitPayload {
let job = JobPayload {
schema_version: 1,
tenant_id: self.binding.tenant_id,
asset_id: self.binding.asset_id,
agent_id: self.binding.agent_id,
enrollment_generation: 1,
assignment_version: 3,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action,
execution_identity: "root".into(),
not_before: self.clock.now - Duration::seconds(10),
start_before: self.clock.now + Duration::minutes(5),
max_runtime_seconds: 180,
max_output_bytes: 4096,
};
let permit = ExecutionPermitPayload {
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: Uuid::new_v4(),
tenant_id: job.tenant_id,
asset_id: job.asset_id,
agent_id: job.agent_id,
enrollment_generation: 1,
assignment_version: 3,
job_id: job.job_id,
attempt_id: job.attempt_id,
plan_hash: hash(b"approved fixed plan"),
job_payload_hash: hash(&encode(&job).unwrap()),
action: job.action.name().into(),
execution_identity: "root".into(),
not_before: self.clock.now - Duration::seconds(1),
start_before: self.clock.now + Duration::seconds(59),
max_runtime_seconds: 180,
key_id: hash(&self.key.raw_public_key().unwrap()),
};
let bytes = encode(&permit).unwrap();
let original = Request {
job_bytes: encode(&job).unwrap(),
permit_bytes: bytes.clone(),
signature: Signer::new_without_digest(&self.key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap(),
plan_hash: permit.plan_hash.clone(),
authorization_id: permit.authorization_id,
};
rmm_privileged_helper::consume(&mut self.db, &permit).unwrap();
rmm_privileged_helper::save_start(&self.db, &permit, &original).unwrap();
if let Some(o) = outcome {
self.db
.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
params![
permit.grant_id.to_string(),
canonical_outcome_bytes(&o).unwrap()
],
)
.unwrap();
}
permit
}
}
#[test]
fn general_and_apt_instances_share_fencing_but_never_inherit_a_process_gate() {
let mut f = Fixture::new();
let mut general = f.sync();
let first = f.snapshot(&mut general, 1);
let raw = f.request(&first);
let d = f.decision(&first, &raw, 1, true);
f.apply(&mut general, &d, &raw).unwrap();
assert!(general.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let mut apt = f.sync();
assert!(!apt.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let second = f.snapshot(&mut apt, 1);
assert!(!general.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let raw = f.request(&second);
let d = f.decision(&second, &raw, 1, true);
f.apply(&mut apt, &d, &raw).unwrap();
assert!(apt.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
assert!(!general.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
}
#[test]
fn targeted_final_unlock_does_not_release_an_unrelated_unknown() {
let mut f = Fixture::new();
let known = f.add(script(), Some(success()));
let actual: rmm_privileged_helper::Outcome =
serde_json::from_slice(&canonical_outcome_bytes(&success()).unwrap()).unwrap();
rmm_privileged_helper::finish(&f.db, &known, &actual).unwrap();
let uncertain = f.add(script(), Some(unknown()));
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, false);
let applied = f.apply(&mut sync, &d, &raw).unwrap();
assert_eq!(applied.unlocked_grants, vec![known.grant_id]);
assert!(!applied.ready);
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let held: bool =
f.db.query_row(
"SELECT result IS NULL FROM consumed WHERE grant_id=?1",
[uncertain.grant_id.to_string()],
|r| r.get(0),
)
.unwrap();
assert!(held);
}
#[test]
fn independently_remembered_agent_journal_cannot_be_replaced_by_server_ready() {
let mut f = Fixture::new();
let mut sync = f.sync();
let first = f.snapshot(&mut sync, 1);
let raw = f.request(&first);
let d = f.decision(&first, &raw, 1, true);
f.apply(&mut sync, &d, &raw).unwrap();
let next = f.snapshot(&mut sync, 2);
f.agent_journal = Uuid::new_v4();
let raw = f.request(&next);
let d = f.decision(&next, &raw, 2, true);
assert!(f.apply(&mut sync, &d, &raw).is_err());
assert!(
sync.snapshot(&mut f.db, &f.guard, 2, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn missing_signed_checkpoint_history_closes_the_live_gate_stickily() {
let mut f = Fixture::new();
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
f.apply(&mut sync, &d, &raw).unwrap();
assert!(sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
f.db.execute("DELETE FROM helper_sync_decisions", [])
.unwrap();
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
assert!(
sync.snapshot(&mut f.db, &f.guard, 2, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn stale_process_connection_does_not_poison_a_new_fenced_process() {
let mut f = Fixture::new();
let mut general = f.sync();
let snapshot = f.snapshot(&mut general, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
f.apply(&mut general, &d, &raw).unwrap();
let mut apt = f.sync();
let snapshot = f.snapshot(&mut apt, 2);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 2, true);
f.apply(&mut apt, &d, &raw).unwrap();
assert!(
general
.snapshot(&mut f.db, &f.guard, 1, &f.clock, &f.boot)
.is_err()
);
assert!(apt.can_start(&f.db, &f.guard, 2, &f.clock, &f.boot));
}
#[test]
fn signed_noncanonical_outcome_bytes_do_not_unlock_root_consumption() {
let mut f = Fixture::new();
f.add(script(), Some(success()));
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let mut d = f.decision(&snapshot, &raw, 1, true);
d.attempts[0].outcome_bytes = serde_json::to_vec_pretty(&success()).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
assert!(f.apply(&mut sync, &d, &raw).is_err());
assert_eq!(
f.db.query_row(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get::<_, i64>(0)
)
.unwrap(),
1
);
}
#[test]
fn both_live_profiles_apply_one_signature_and_checkpoint_without_inheriting_readiness() {
let mut f = Fixture::new();
let mut general = f.sync();
let mut apt = f.sync();
let first = f.snapshot(&mut general, 1);
let second = f.snapshot(&mut apt, 1);
assert_eq!(first.checkpoint, second.checkpoint);
assert!(!general.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
assert!(!apt.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let raw = f.request(&first);
let d = f.decision(&first, &raw, 1, true);
f.apply(&mut general, &d, &raw).unwrap();
f.apply(&mut apt, &d, &raw).unwrap();
assert!(general.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
assert!(apt.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
}
fn script() -> Action {
Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf hello".into(),
arguments: vec![],
expected_stdout: Some("hello".into()),
windows_execution_profile: None,
privileged_script_profile: None,
}
}
fn success() -> ReconciledOutcome {
ReconciledOutcome {
state: "succeeded".into(),
stdout: b"hello".to_vec(),
stderr: vec![],
exit_code: Some(0),
output_truncated: false,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":true}),
error_code: String::new(),
}
}
fn unknown() -> ReconciledOutcome {
ReconciledOutcome {
state: "unknown".into(),
stdout: vec![],
stderr: vec![],
exit_code: None,
output_truncated: false,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":false}),
error_code: "EFFECT_UNKNOWN".into(),
}
}
// Test-only encoder for the actual existing prost field numbers. No substitute
// protocol dependency is added to the independently executable helper crate.
fn vi(mut n: u64, out: &mut Vec<u8>) {
loop {
let mut byte = (n & 127) as u8;
n >>= 7;
if n != 0 {
byte |= 128;
}
out.push(byte);
if n == 0 {
break;
}
}
}
fn num(tag: u64, n: u64, out: &mut Vec<u8>) {
if n != 0 {
vi(tag << 3, out);
vi(n, out);
}
}
fn blob(tag: u64, b: &[u8], out: &mut Vec<u8>) {
if !b.is_empty() {
vi((tag << 3) | 2, out);
vi(b.len() as u64, out);
out.extend_from_slice(b);
}
}
fn text(tag: u64, s: &str, out: &mut Vec<u8>) {
blob(tag, s.as_bytes(), out)
}
fn request(
snapshot: &Snapshot,
journal: Uuid,
checkpoint: u64,
attempts: &[LocalAttempt],
) -> Vec<u8> {
let mut out = vec![];
text(1, &Uuid::new_v4().to_string(), &mut out);
text(2, &journal.to_string(), &mut out);
num(3, checkpoint, &mut out);
text(
4,
&snapshot
.previous_decision_id
.map(|id| id.to_string())
.unwrap_or_default(),
&mut out,
);
num(5, 3, &mut out);
for a in attempts {
let mut row = vec![];
text(1, &a.job_id.to_string(), &mut row);
text(2, &a.attempt_id.to_string(), &mut row);
text(3, &a.job_payload_hash, &mut row);
text(4, &a.grant_id.to_string(), &mut row);
text(5, &a.consumption, &mut row);
text(6, &a.state, &mut row);
num(7, a.revision + 1, &mut row);
text(8, &a.original_boot_id.to_string(), &mut row);
text(11, &a.outcome_sha256, &mut row);
blob(12, &a.outcome_bytes, &mut row);
blob(6, &row, &mut out);
}
num(8, 1, &mut out);
text(9, &snapshot.journal_id.to_string(), &mut out);
num(10, snapshot.checkpoint, &mut out);
text(
11,
&snapshot
.previous_decision_id
.map(|id| id.to_string())
.unwrap_or_default(),
&mut out,
);
out
}
#[test]
fn clean_ready_shared_lock_and_reopen_start_closed() {
let mut f = Fixture::new();
let mut sync = f.sync();
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
assert!(ExecutorGuard::acquire(&f.dir, unsafe { libc::geteuid() }).is_err());
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let applied = f.apply(&mut sync, &d, &raw).unwrap();
assert!(applied.ready);
assert_eq!(sync.active_generation(), Some(1));
assert_eq!(applied.snapshot.previous_decision_id, Some(d.decision_id));
assert!(applied.unlocked_grants.is_empty());
assert!(sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
assert!(!sync.can_start(&f.db, &f.guard, 2, &f.clock, &f.boot));
let response = f.sign(&d);
let mut new_process = f.sync();
let mut reopened = Connection::open(f.dir.join("consumption.sqlite")).unwrap();
assert!(!new_process.can_start(&reopened, &f.guard, 1, &f.clock, &f.boot));
assert!(
new_process
.apply(&mut reopened, &f.guard, &response, &raw, &f.clock, &f.boot)
.is_err()
);
sync.close();
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let _native = NativeClock::default();
}
#[test]
fn exact_signatures_fake_key_wrong_domain_and_all_identity_bindings() {
let mut f = Fixture::new();
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let base = serde_json::to_value(&d).unwrap();
for (field, value) in [
("domain", serde_json::json!("rmm.execution-permit")),
("schema_version", serde_json::json!(2)),
("tenant_id", serde_json::json!(Uuid::new_v4())),
("asset_id", serde_json::json!(Uuid::new_v4())),
("agent_id", serde_json::json!(Uuid::new_v4())),
("enrollment_generation", serde_json::json!(2)),
("assignment_version", serde_json::json!(4)),
("connection_generation", serde_json::json!(2)),
("boot_id", serde_json::json!(Uuid::new_v4())),
("journal_id", serde_json::json!(Uuid::new_v4())),
("checkpoint", serde_json::json!(1)),
("helper_journal_id", serde_json::json!(Uuid::new_v4())),
(
"helper_checkpoint",
serde_json::json!(snapshot.checkpoint + 1),
),
("request_id", serde_json::json!(Uuid::new_v4())),
("request_hash", serde_json::json!(hash(b"changed"))),
("key_id", serde_json::json!(hash(b"wrongkey"))),
] {
let mut changed = base.clone();
changed[field] = value;
let changed: ReconcileDecisionPayload = serde_json::from_value(changed).unwrap();
assert!(f.apply(&mut sync, &changed, &raw).is_err(), "{field}");
}
let mut response = f.sign(&d);
response.signature[0] ^= 1;
assert!(
sync.apply(&mut f.db, &f.guard, &response, &raw, &f.clock, &f.boot)
.is_err()
);
let mut response = f.sign(&d);
response.decision_bytes.push(b' ');
assert!(
sync.apply(&mut f.db, &f.guard, &response, &raw, &f.clock, &f.boot)
.is_err()
);
let fake = PKey::generate_ed25519().unwrap();
let mut response = f.sign(&d);
response.signature = Signer::new_without_digest(&fake)
.unwrap()
.sign_oneshot_to_vec(&response.decision_bytes)
.unwrap();
assert!(
sync.apply(&mut f.db, &f.guard, &response, &raw, &f.clock, &f.boot)
.is_err()
);
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
}
#[test]
fn definitive_unlock_is_transactional_idempotent_and_keeps_tombstones() {
let mut f = Fixture::new();
let grant = f.add(script(), Some(success()));
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
assert_eq!(
snapshot.attempts[0].original_boot_id,
NativeBoot.current_boot().unwrap()
);
assert_eq!(
snapshot.attempts[0].job_payload_hash,
grant.job_payload_hash
);
assert_eq!(snapshot.attempts[0].original_request_sha256.len(), 64);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let applied = f.apply(&mut sync, &d, &raw).unwrap();
assert_eq!(applied.unlocked_grants, vec![grant.grant_id]);
assert!(applied.ready);
let again = f.apply(&mut sync, &d, &raw).unwrap();
assert!(again.ready);
assert!(again.unlocked_grants.is_empty());
assert_eq!(again.snapshot.checkpoint, snapshot.checkpoint);
let body: Vec<u8> =
f.db.query_row(
"SELECT result FROM consumed WHERE grant_id=?1",
[grant.grant_id.to_string()],
|r| r.get(0),
)
.unwrap();
assert_eq!(body, canonical_outcome_bytes(&success()).unwrap());
assert!(rmm_privileged_helper::consume(&mut f.db, &grant).is_err());
assert_eq!(
f.db.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
.unwrap(),
1
);
}
#[test]
fn omitted_unknown_root_row_is_sticky_despite_signed_ready() {
let mut f = Fixture::new();
f.add(script(), Some(unknown()));
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = request(&snapshot, f.agent_journal, 100 + snapshot.checkpoint, &[]);
let d = f.decision(&snapshot, &raw, 1, true);
assert!(f.apply(&mut sync, &d, &raw).is_err());
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let mut restarted = f.sync();
assert!(
restarted
.snapshot(&mut f.db, &f.guard, 2, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn unknown_code_or_no_result_never_unlocks_even_signed_ready() {
for outcome in [None, Some(unknown())] {
let mut f = Fixture::new();
f.add(script(), outcome);
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
assert_eq!(snapshot.attempts[0].state, "unknown");
let raw = f.request(&snapshot);
let mut d = f.decision(&snapshot, &raw, 1, true);
let applied = f.apply(&mut sync, &d, &raw).unwrap();
assert!(!applied.ready);
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
d = f.decision(&snapshot, &raw, 1, true);
d.attempts[0].unlock = true;
d.attempts[0].outcome_state = "succeeded".into();
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&success()).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
assert!(f.apply(&mut sync, &d, &raw).is_err());
assert_eq!(
f.db.query_row(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get::<_, i64>(0)
)
.unwrap(),
1
);
}
}
#[test]
fn old_process_response_fails_after_fresh_capture_even_inside_ttl() {
let mut f = Fixture::new();
let mut first = f.sync();
let snapshot = f.snapshot(&mut first, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let response = f.sign(&d);
f.clock.now += Duration::seconds(5);
let mut second = f.sync();
let newer = f.snapshot(&mut second, 1);
assert!(newer.checkpoint > snapshot.checkpoint);
assert!(
second
.apply(&mut f.db, &f.guard, &response, &raw, &f.clock, &f.boot)
.is_err()
);
assert!(!second.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
}
#[test]
fn checkpoint_rollback_is_sticky_and_cannot_be_fixed_by_omission() {
let mut f = Fixture::new();
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
f.db.execute("UPDATE helper_sync_meta SET checkpoint=0", [])
.unwrap();
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
assert!(f.apply(&mut sync, &d, &raw).is_err());
f.db.execute(
"UPDATE helper_sync_meta SET checkpoint=?1",
[snapshot.checkpoint],
)
.unwrap();
assert!(
sync.snapshot(&mut f.db, &f.guard, 1, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn mutated_root_ledger_after_capture_is_not_the_signed_snapshot() {
let mut f = Fixture::new();
f.add(script(), Some(success()));
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let mut changed = success();
changed.stdout = b"different".to_vec();
f.db.execute(
"UPDATE execution_evidence SET outcome=?1",
[canonical_outcome_bytes(&changed).unwrap()],
)
.unwrap();
assert!(f.apply(&mut sync, &d, &raw).is_err());
assert!(
sync.snapshot(&mut f.db, &f.guard, 1, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn unrelated_action_plan_and_previous_outcome_cannot_unlock() {
let mut f = Fixture::new();
f.add(script(), Some(success()));
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let mut changed = d.clone();
changed.attempts[0].action = "reboot".into();
assert!(f.apply(&mut sync, &changed, &raw).is_err());
let mut changed = d.clone();
changed.attempts[0].plan_hash = Some(hash(b"changed plan"));
assert!(f.apply(&mut sync, &changed, &raw).is_err());
let mut changed = d;
changed.attempts[0].previous_outcome_sha256 = Some(hash(b"different outcome"));
assert!(f.apply(&mut sync, &changed, &raw).is_err());
}
#[test]
fn false_clock_expiry_and_invalid_independent_key_stay_closed() {
let mut f = Fixture::new();
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
f.clock.trusted = false;
assert!(f.apply(&mut sync, &d, &raw).is_err());
assert!(!sync.can_start(&f.db, &f.guard, 1, &f.clock, &f.boot));
f.clock.trusted = true;
f.clock.now += Duration::seconds(61);
assert!(f.apply(&mut sync, &d, &raw).is_err());
let mut invalid = f.binding.clone();
invalid.public_key_hex = "not a key".into();
assert!(HelperSync::new(invalid, &f.clock).is_err());
let mut changed = f.binding.clone();
changed.public_key_hex =
hex::encode(PKey::generate_ed25519().unwrap().raw_public_key().unwrap());
let mut different = HelperSync::new(changed, &f.clock).unwrap();
assert!(
different
.snapshot(&mut f.db, &f.guard, 2, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn shared_database_only_and_missing_local_start_fail_closed() {
let mut f = Fixture::new();
let mut sync = f.sync();
let mut copy = Connection::open_in_memory().unwrap();
assert!(
sync.snapshot(&mut copy, &f.guard, 1, &f.clock, &f.boot)
.is_err()
);
let grant = f.add(script(), None);
f.db.execute(
"DELETE FROM execution_evidence WHERE grant_id=?1",
[grant.grant_id.to_string()],
)
.unwrap();
assert!(
sync.snapshot(&mut f.db, &f.guard, 1, &f.clock, &f.boot)
.is_err()
);
}
#[test]
fn real_local_boot_must_match_signed_reboot_completion() {
let mut f = Fixture::new();
let old = f.boot.0;
let mut awaiting = unknown();
awaiting.state = "awaiting_reboot".into();
awaiting.reboot_required = true;
f.add(
Action::Reboot {
reason: "mock reboot no native effect".into(),
reconnect_timeout_seconds: 120,
dependency_group: "acceptance".into(),
},
Some(awaiting),
);
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let mut d = f.decision(&snapshot, &raw, 1, true);
let mut done = success();
done.stdout.clear();
done.exit_code = None;
done.evidence = serde_json::json!({"postconditions_passed":true,"new_boot_id":Uuid::new_v4(),"reconciled_after_reboot":true,"inventory_message_id":Uuid::new_v4()});
d.attempts[0].unlock = true;
d.attempts[0].outcome_state = "succeeded".into();
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&done).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
assert!(f.apply(&mut sync, &d, &raw).is_err()); // native current boot is still old
f.boot.0 = Uuid::new_v4();
f.clock.now += Duration::seconds(1);
let snapshot = f.snapshot(&mut sync, 2);
let raw = f.request(&snapshot);
let mut d = f.decision(&snapshot, &raw, 2, true);
assert_eq!(snapshot.attempts[0].original_boot_id, old);
assert_ne!(f.boot.0, old);
done.evidence["new_boot_id"] = serde_json::json!(f.boot.0);
d.attempts[0].unlock = true;
d.attempts[0].outcome_state = "succeeded".into();
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&done).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
let applied = f.apply(&mut sync, &d, &raw).unwrap();
assert!(applied.ready);
assert_eq!(applied.snapshot.attempts[0].state, "succeeded");
assert!(sync.can_start(&f.db, &f.guard, 2, &f.clock, &f.boot));
}
#[test]
fn malformed_original_protobuf_or_digest_does_not_open_gate() {
let mut f = Fixture::new();
let mut sync = f.sync();
let snapshot = f.snapshot(&mut sync, 1);
let raw = f.request(&snapshot);
let d = f.decision(&snapshot, &raw, 1, true);
let mut malformed = raw.clone();
malformed.push(0);
assert!(f.apply(&mut sync, &d, &malformed).is_err());
let mut malformed = raw.clone();
text(1, &Uuid::new_v4().to_string(), &mut malformed);
let mut signed = d.clone();
signed.request_hash = hash(&malformed);
assert!(f.apply(&mut sync, &signed, &malformed).is_err());
let mut signed = d;
signed.request_hash.clear();
assert!(f.apply(&mut sync, &signed, &raw).is_err());
assert!(
sync.snapshot(&mut f.db, &f.guard, 1, &f.clock, &f.boot)
.is_err()
);
}