Files
rust-discovery/crates/agent/tests/reconcile.rs
T
2026-11-14 16:35:11 +01:00

1143 lines
42 KiB
Rust

#[path = "../src/reconcile.rs"]
mod reconcile;
use chrono::{DateTime, Duration, Utc};
use openssl::{
pkey::{PKey, Private},
sign::Signer,
};
use prost::Message;
use reconcile::*;
use rmm_domain::{jobs::*, reconciliation::*};
use rmm_privileged_helper::{Binding, Request};
use rmm_protocol::*;
use rusqlite::{Connection, params};
use std::{fs, os::unix::fs::DirBuilderExt, path::PathBuf};
use uuid::Uuid;
struct Clock {
now: DateTime<Utc>,
trusted: bool,
}
impl SyncClock for Clock {
fn now(&self) -> anyhow::Result<DateTime<Utc>> {
anyhow::ensure!(self.trusted, "CLOCK_UNTRUSTED");
Ok(self.now)
}
}
struct Fixture {
dir: PathBuf,
db: Connection,
ledger: Connection,
key: PKey<Private>,
binding: Binding,
channel: Channel,
clock: Clock,
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.dir);
}
}
impl Fixture {
fn new() -> Self {
let dir = PathBuf::from("/tmp/opencode").join(format!("reconcile-{}", Uuid::new_v4()));
fs::DirBuilder::new().mode(0o700).create(&dir).unwrap();
let db = Connection::open(dir.join("journal.sqlite")).unwrap();
db.execute_batch("PRAGMA synchronous=FULL;CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);INSERT INTO counter VALUES(1,10);
CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT DEFAULT '',cancel_requested INTEGER DEFAULT 0,PRIMARY KEY(job,attempt));
CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE safety(key TEXT PRIMARY KEY,value TEXT);INSERT INTO safety VALUES('connected','1');").unwrap();
let ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 3,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let channel = Channel {
connection_generation: 1,
boot_id: Uuid::parse_str(&rmm_agent_linux::boot_id().unwrap()).unwrap(),
};
Self {
dir,
db,
ledger,
key,
binding,
channel,
clock: Clock {
now: Utc::now(),
trusted: true,
},
}
}
fn client(&self) -> ClientSync {
ClientSync::new(self.binding.clone())
}
fn prepare(&mut self, client: &mut ClientSync) -> PreparedSync {
client
.prepare(
&mut self.db,
&self.ledger,
self.channel.clone(),
None,
&self.clock,
)
.unwrap()
}
fn decision(&self, prepared: &PreparedSync, ready: bool) -> ReconcileDecisionPayload {
let request = &prepared.request;
let attempts = request
.attempts
.iter()
.map(|local| {
let bytes: Vec<u8> = self
.db
.query_row(
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
params![local.job_id, local.attempt_id],
|r| r.get(0),
)
.unwrap();
let offer = JobOffer::decode(bytes.as_slice()).unwrap();
let job: JobPayload = serde_json::from_slice(&offer.payload_bytes).unwrap();
let grant = if local.grant_id.is_empty() {
None
} else {
Some(Uuid::parse_str(&local.grant_id).unwrap())
};
let final_state = matches!(
local.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
);
AttemptDecision {
job_id: job.job_id,
attempt_id: job.attempt_id,
job_payload_hash: local.job_payload_hash.clone(),
grant_id: grant,
authorization_id: grant
.map(|_| Uuid::parse_str(&offer.authorization_id).unwrap()),
plan_hash: grant.map(|_| offer.plan_hash),
action: job.action.name().into(),
execution_identity: job.execution_identity,
revision: local.revision,
outcome_state: if local.state == "accepted" {
"accepted".into()
} else if final_state {
local.state.clone()
} else {
"unknown".into()
},
previous_outcome_sha256: (!local.outcome_sha256.is_empty())
.then(|| local.outcome_sha256.clone()),
outcome_sha256: final_state
.then(|| local.outcome_sha256.clone())
.filter(|s| !s.is_empty()),
outcome_bytes: vec![],
original_boot_id: (!local.original_boot_id.is_empty())
.then(|| Uuid::parse_str(&local.original_boot_id).unwrap()),
unlock: final_state && local.consumption == "consumed",
reason: if local.state == "accepted" {
"ACCEPTED_NOT_STARTED"
} else if final_state {
"DEFINITIVE_OUTCOME_RECONCILED"
} else {
"EFFECT_UNRESOLVED"
}
.into(),
}
})
.collect();
ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::parse_str(&request.request_id).unwrap(),
request_hash: hash(&request.encode_to_vec()),
tenant_id: self.binding.tenant_id,
asset_id: self.binding.asset_id,
agent_id: self.binding.agent_id,
enrollment_generation: self.binding.enrollment_generation,
assignment_version: self.binding.assignment_version,
connection_generation: self.channel.connection_generation,
boot_id: self.channel.boot_id,
journal_id: Uuid::parse_str(&request.journal_id).unwrap(),
checkpoint: request.checkpoint,
helper_journal_id: (!request.helper_journal_id.is_empty())
.then(|| Uuid::parse_str(&request.helper_journal_id).unwrap()),
helper_checkpoint: request.helper_checkpoint,
issued_at: self.clock.now - Duration::milliseconds(1),
valid_until: self.clock.now + Duration::seconds(59),
ready,
helper_ready: ready && !request.helper_journal_id.is_empty(),
reason: if ready {
"CONSISTENT"
} else {
"ATTEMPT_RECONCILIATION_REQUIRED"
}
.into(),
attempts,
acknowledged_message_ids: vec![],
key_id: hash(&self.key.raw_public_key().unwrap()),
}
}
fn sign(&self, decision: &ReconcileDecisionPayload) -> SyncResponse {
let bytes = encode(decision).unwrap();
SyncResponse {
signature: Signer::new_without_digest(&self.key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap(),
decision_bytes: bytes,
}
}
fn apply(
&mut self,
client: &mut ClientSync,
decision: &ReconcileDecisionPayload,
) -> anyhow::Result<Applied> {
let response = self.sign(decision);
client.apply(&mut self.db, &response, &self.clock)
}
fn add(
&mut self,
state: &str,
consume: bool,
outcome: Option<ReconciledOutcome>,
action: Action,
) -> (JobOffer, Option<ExecutionPermitPayload>, Option<Envelope>) {
let execution_identity = if matches!(action, Action::SetServiceState { .. }) {
"root"
} else {
"agent_service"
};
let job = JobPayload {
schema_version: 1,
tenant_id: self.binding.tenant_id,
asset_id: self.binding.asset_id,
agent_id: self.binding.agent_id,
enrollment_generation: 1,
assignment_version: 3,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action,
execution_identity: execution_identity.into(),
not_before: self.clock.now - Duration::seconds(5),
start_before: self.clock.now + Duration::minutes(5),
max_runtime_seconds: 120,
max_output_bytes: 4096,
};
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: Uuid::new_v4().to_string(),
plan_hash: hash(b"approved immutable plan"),
job_payload_hash: hash(&encode(&job).unwrap()),
payload_bytes: encode(&job).unwrap(),
};
let mut grant = None;
if consume {
let p = ExecutionPermitPayload {
domain: "rmm.execution-permit".into(),
schema_version: 1,
grant_id: Uuid::new_v4(),
authorization_id: Uuid::parse_str(&offer.authorization_id).unwrap(),
tenant_id: job.tenant_id,
asset_id: job.asset_id,
agent_id: job.agent_id,
enrollment_generation: 1,
assignment_version: 3,
job_id: job.job_id,
attempt_id: job.attempt_id,
plan_hash: offer.plan_hash.clone(),
job_payload_hash: offer.job_payload_hash.clone(),
action: job.action.name().into(),
execution_identity: job.execution_identity.clone(),
not_before: self.clock.now - Duration::milliseconds(1),
start_before: self.clock.now + Duration::seconds(59),
max_runtime_seconds: 120,
key_id: hash(&self.key.raw_public_key().unwrap()),
};
let bytes = encode(&p).unwrap();
let request = Request {
job_bytes: offer.payload_bytes.clone(),
permit_bytes: bytes.clone(),
signature: Signer::new_without_digest(&self.key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap(),
plan_hash: offer.plan_hash.clone(),
authorization_id: p.authorization_id,
};
rmm_privileged_helper::consume(&mut self.ledger, &p).unwrap();
rmm_privileged_helper::save_start(&self.ledger, &p, &request).unwrap();
if let Some(o) = &outcome {
self.ledger
.execute(
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
params![p.grant_id.to_string(), canonical_outcome_bytes(o).unwrap()],
)
.unwrap();
}
grant = Some(p);
}
self.db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,?4,1,?5)",params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),state,grant.as_ref().map(|p|p.grant_id.to_string()).unwrap_or_default()]).unwrap();
let envelope = outcome.map(|o| {
let sequence: i64 = self
.db
.query_row("UPDATE counter SET seq=seq+1 RETURNING seq", [], |r| {
r.get(0)
})
.unwrap();
let r = JobResult {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
revision: 1,
state: o.state,
grant_id: grant
.as_ref()
.map(|p| p.grant_id.to_string())
.unwrap_or_default(),
stdout: o.stdout,
stderr: o.stderr,
output_truncated: o.output_truncated,
reboot_required: o.reboot_required,
exit_code: o.exit_code,
evidence_json: o.evidence.to_string(),
error_code: o.error_code,
};
let e = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: self.binding.agent_id.to_string(),
enrollment_generation: 1,
connection_generation: self.channel.connection_generation,
boot_id: self.channel.boot_id.to_string(),
sequence: sequence as u64,
observed_at: self.clock.now.timestamp_millis(),
payload: Some(Payload::JobResult(r)),
};
self.db
.execute(
"INSERT INTO job_events VALUES(?1,?2)",
params![e.message_id, e.encode_to_vec()],
)
.unwrap();
e
});
(offer, grant, envelope)
}
}
#[test]
fn authenticated_helper_outcome_is_imported_as_new_durable_evidence_not_reexecuted() {
let mut f = Fixture::new();
let mut unknown = success();
unknown.state = "unknown".into();
unknown.error_code = "HELPER_TRANSPORT_EFFECT_UNKNOWN".into();
unknown.evidence = serde_json::json!({"postconditions_passed":false});
let (offer, grant, old) = f.add(
"unknown",
true,
Some(unknown),
Action::SetServiceState {
unit: "rmm-acceptance.service".into(),
state: "running".into(),
},
);
let grant = grant.unwrap();
let mut definitive = success();
definitive.stdout.clear();
definitive.exit_code = None;
let bytes = canonical_outcome_bytes(&definitive).unwrap();
let mut helper = HelperSnapshot {
journal_id: Uuid::new_v4(),
checkpoint: 3,
previous_decision_id: None,
attempts: vec![SyncAttempt {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
job_payload_hash: offer.job_payload_hash.clone(),
grant_id: grant.grant_id.to_string(),
consumption: "consumed".into(),
state: "succeeded".into(),
revision: 1,
original_boot_id: f.channel.boot_id.to_string(),
outcome_sha256: hash(&bytes),
outcome_bytes: bytes,
..Default::default()
}],
};
let mut c = f.client();
let p = c
.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock,
)
.unwrap();
let imported = p.request.attempts[0].result_envelope.as_ref().unwrap();
assert_ne!(imported.message_id, old.unwrap().message_id);
assert_eq!(p.request.attempts[0].state, "succeeded");
assert_eq!(p.request.attempts[0].revision, 2);
let d = f.decision(&p, true);
let applied = f.apply(&mut c, &d).unwrap();
assert!(applied.helper_instruction.is_some());
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
assert!(!c.can_start(&f.channel, true, &f.clock));
helper.previous_decision_id = Some(d.decision_id);
c.confirm_helper(&helper).unwrap();
assert!(c.can_start(&f.channel, true, &f.clock));
let retained: i64 =
f.db.query_row("SELECT count(*) FROM reconcile_results", [], |r| r.get(0))
.unwrap();
assert_eq!(retained, 2);
let tombstones: i64 = f
.ledger
.query_row("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap();
assert_eq!(tombstones, 1);
}
#[test]
fn helper_omitting_a_consumed_root_attempt_is_a_sticky_restore_fault() {
let mut f = Fixture::new();
f.add(
"running",
true,
None,
Action::SetServiceState {
unit: "rmm-acceptance.service".into(),
state: "running".into(),
},
);
let helper = HelperSnapshot {
journal_id: Uuid::new_v4(),
checkpoint: 1,
previous_decision_id: None,
attempts: vec![],
};
let mut c = f.client();
assert!(
c.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock
)
.is_err()
);
assert!(
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
.is_err()
);
assert!(!c.can_start(&f.channel, true, &f.clock));
}
#[test]
fn definite_nonstart_result_is_unconsumed_not_ambiguous() {
let mut f = Fixture::new();
let mut cancelled = success();
cancelled.state = "cancelled".into();
cancelled.stdout.clear();
cancelled.exit_code = None;
cancelled.evidence = serde_json::json!({"not_started":true,"postconditions_passed":false});
f.add("cancelled", false, Some(cancelled), script());
let mut c = f.client();
let p = f.prepare(&mut c);
assert_eq!(p.request.attempts[0].consumption, "unconsumed");
let d = f.decision(&p, true);
f.apply(&mut c, &d).unwrap();
assert!(c.can_start(&f.channel, false, &f.clock));
}
#[test]
fn signed_terminal_nonstart_stays_unconsumed_across_sync_and_pending_denial() {
for terminal in ["expired", "cancelled"] {
let mut f = Fixture::new();
let (offer, _, _) = f.add("accepted", false, None, script());
let mut client = f.client();
let first = f.prepare(&mut client);
assert_eq!(first.request.attempts[0].consumption, "unconsumed");
let mut decision = f.decision(&first, true);
decision.attempts[0].outcome_state = terminal.into();
decision.attempts[0].reason = "DEFINITIVE_NONSTART".into();
f.apply(&mut client, &decision).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
let second = f.prepare(&mut client);
let local = &second.request.attempts[0];
assert_eq!(local.state, terminal);
assert_eq!(local.consumption, "unconsumed");
assert!(local.grant_id.is_empty() && local.outcome_bytes.is_empty());
let mut repeated = f.decision(&second, true);
repeated.attempts[0].reason = "DEFINITIVE_NONSTART".into();
f.apply(&mut client, &repeated).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
// Reproduce the old server's rejected terminal->unknown response.
// Keep its pending request and the last applied signed anchor, then
// recover through a fresh channel/request with the ordinary protocol.
let pending = f.prepare(&mut client);
let mut denied = f.decision(&pending, false);
denied.attempts[0].outcome_state = "unknown".into();
denied.attempts[0].reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
assert!(f.apply(&mut client, &denied).is_err());
assert!(!client.can_start(&f.channel, false, &f.clock));
client = f.client();
f.channel.connection_generation += 1;
let fresh = f.prepare(&mut client);
assert_eq!(
fresh.request.previous_request_id,
pending.request.request_id
);
assert_eq!(
fresh.request.previous_decision_id,
repeated.decision_id.to_string()
);
assert_eq!(fresh.request.attempts[0].consumption, "unconsumed");
let mut recovered = f.decision(&fresh, true);
recovered.attempts[0].reason = "DEFINITIVE_NONSTART".into();
f.apply(&mut client, &recovered).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
let stored: (String, String, String) =
f.db.query_row("SELECT job,attempt,state FROM job_attempts", [], |r| {
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
})
.unwrap();
assert_eq!(stored, (offer.job_id, offer.attempt_id, terminal.into()));
assert_eq!(
f.ledger
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
.unwrap(),
0
);
assert_eq!(
f.db.query_row("SELECT count(*) FROM reconcile_outcomes", [], |r| r
.get::<_, i64>(0))
.unwrap(),
0
);
}
}
#[test]
fn terminal_labels_never_override_grant_consumption_or_unclear_attempts() {
for state in ["requesting", "running", "unknown"] {
let mut f = Fixture::new();
f.add(state, false, None, script());
let mut client = f.client();
let prepared = f.prepare(&mut client);
assert_eq!(prepared.request.attempts[0].consumption, "uncertain");
let decision = f.decision(&prepared, true);
f.apply(&mut client, &decision).unwrap();
assert!(!client.can_start(&f.channel, false, &f.clock));
}
for terminal in ["expired", "cancelled"] {
for consumed in [false, true] {
let mut f = Fixture::new();
f.add(terminal, consumed, None, script());
// A surviving grant ID also forbids the nonstart shortcut when
// no consumption row exists. A real row always remains consumed.
if !consumed {
f.db.execute(
"UPDATE job_attempts SET grant_id=?1",
[Uuid::new_v4().to_string()],
)
.unwrap();
}
let mut client = f.client();
let prepared = f.prepare(&mut client);
assert_eq!(
prepared.request.attempts[0].consumption,
if consumed { "consumed" } else { "uncertain" }
);
let mut decision = f.decision(&prepared, true);
decision.attempts[0].unlock = false;
assert!(f.apply(&mut client, &decision).is_err());
assert!(!client.can_start(&f.channel, false, &f.clock));
assert_eq!(
f.ledger
.query_row(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get::<_, i64>(0)
)
.unwrap(),
i64::from(consumed)
);
}
}
}
#[test]
fn negative_server_revision_never_rolls_back_local_unknown_revision() {
let mut f = Fixture::new();
f.add("running", true, None, script());
let mut c = f.client();
let p = f.prepare(&mut c);
assert_eq!(p.request.attempts[0].revision, 2);
let mut d = f.decision(&p, false);
d.attempts[0].revision = 1;
f.apply(&mut c, &d).unwrap();
let revision: u64 =
f.db.query_row("SELECT revision FROM job_attempts", [], |r| r.get(0))
.unwrap();
assert_eq!(revision, 2);
assert!(!c.can_start(&f.channel, false, &f.clock));
}
fn script() -> Action {
Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf hello".into(),
arguments: vec![],
expected_stdout: Some("hello".into()),
windows_execution_profile: None,
privileged_script_profile: None,
}
}
fn success() -> ReconciledOutcome {
ReconciledOutcome {
state: "succeeded".into(),
stdout: b"hello".to_vec(),
stderr: vec![],
exit_code: Some(0),
output_truncated: false,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":true,"z":{"b":2,"a":1}}),
error_code: String::new(),
}
}
#[test]
fn clean_reconnect_and_reopen_require_new_signed_challenge() {
let mut f = Fixture::new();
let mut client = f.client();
assert!(!client.can_start(&f.channel, false, &f.clock));
let prepared = f.prepare(&mut client);
let decision = f.decision(&prepared, true);
let response = f.sign(&decision);
client.apply(&mut f.db, &response, &f.clock).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
assert_eq!(client.validity_deadline(), Some(decision.valid_until));
assert!(!client.can_start(&f.channel, true, &f.clock));
let mut reopened = Connection::open(f.dir.join("journal.sqlite")).unwrap();
let mut process = f.client();
assert!(!process.can_start(&f.channel, false, &f.clock));
f.channel.connection_generation += 1;
let next = process
.prepare(&mut reopened, &f.ledger, f.channel.clone(), None, &f.clock)
.unwrap();
assert_eq!(
next.request.previous_decision_id,
decision.decision_id.to_string()
);
assert_eq!(next.request.journal_id, prepared.request.journal_id);
assert!(next.request.checkpoint > prepared.request.checkpoint);
assert!(process.apply(&mut reopened, &response, &f.clock).is_err());
let next_decision = f.decision(&next, true);
process
.apply(&mut reopened, &f.sign(&next_decision), &f.clock)
.unwrap();
assert!(process.can_start(&f.channel, false, &f.clock));
let safety: String = reopened
.query_row("SELECT value FROM safety WHERE key='connected'", [], |r| {
r.get(0)
})
.unwrap();
assert_eq!(safety, "1");
}
#[test]
fn exact_signatures_all_identity_channel_checkpoint_and_request_bindings() {
let mut f = Fixture::new();
let mut client = f.client();
let prepared = f.prepare(&mut client);
let d = f.decision(&prepared, true);
assert!(
verify_response(
&f.binding,
&prepared.request,
&f.channel,
&f.sign(&d),
f.clock.now
)
.is_ok()
);
let value = serde_json::to_value(&d).unwrap();
for (field, replacement) in [
("domain", serde_json::json!("rmm.execution-permit")),
("schema_version", serde_json::json!(2)),
("tenant_id", serde_json::json!(Uuid::new_v4())),
("asset_id", serde_json::json!(Uuid::new_v4())),
("agent_id", serde_json::json!(Uuid::new_v4())),
("enrollment_generation", serde_json::json!(2)),
("assignment_version", serde_json::json!(4)),
("connection_generation", serde_json::json!(2)),
("boot_id", serde_json::json!(Uuid::new_v4())),
("journal_id", serde_json::json!(Uuid::new_v4())),
("checkpoint", serde_json::json!(0)),
("request_id", serde_json::json!(Uuid::new_v4())),
("request_hash", serde_json::json!(hash(b"different"))),
("helper_journal_id", serde_json::json!(Uuid::new_v4())),
("helper_checkpoint", serde_json::json!(1)),
("key_id", serde_json::json!(hash(b"wrongkey"))),
] {
let mut changed = value.clone();
changed[field] = replacement;
let changed: ReconcileDecisionPayload = serde_json::from_value(changed).unwrap();
assert!(
verify_response(
&f.binding,
&prepared.request,
&f.channel,
&f.sign(&changed),
f.clock.now
)
.is_err(),
"{field}"
);
}
let mut forged = f.sign(&d);
forged.signature[0] ^= 1;
assert!(client.apply(&mut f.db, &forged, &f.clock).is_err());
let fake = PKey::generate_ed25519().unwrap();
let mut forged = f.sign(&d);
forged.signature = Signer::new_without_digest(&fake)
.unwrap()
.sign_oneshot_to_vec(&forged.decision_bytes)
.unwrap();
assert!(client.apply(&mut f.db, &forged, &f.clock).is_err());
let mut changed = f.sign(&d);
changed.decision_bytes.push(b' ');
assert!(client.apply(&mut f.db, &changed, &f.clock).is_err());
let mut expired = d.clone();
expired.valid_until = f.clock.now;
assert!(f.apply(&mut client, &expired).is_err());
let mut unbounded = d;
unbounded.valid_until = unbounded.issued_at + Duration::seconds(60) + Duration::milliseconds(1);
assert!(f.apply(&mut client, &unbounded).is_err());
assert!(!client.can_start(&f.channel, false, &f.clock));
}
#[test]
fn accepted_unstarted_keeps_ids_and_never_consumes_or_executes() {
let mut f = Fixture::new();
let (offer, _, _) = f.add("accepted", false, None, script());
let mut c = f.client();
let p = f.prepare(&mut c);
assert_eq!(p.request.attempts[0].consumption, "unconsumed");
let d = f.decision(&p, true);
let r = f.sign(&d);
c.apply(&mut f.db, &r, &f.clock).unwrap();
assert!(c.can_start(&f.channel, false, &f.clock));
let count: i64 = f
.ledger
.query_row("SELECT count(*) FROM consumed", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
let ids: (String, String, u64) =
f.db.query_row("SELECT job,attempt,revision FROM job_attempts", [], |r| {
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
})
.unwrap();
assert_eq!(ids, (offer.job_id, offer.attempt_id, 1));
c.apply(&mut f.db, &r, &f.clock).unwrap();
assert!(c.can_start(&f.channel, false, &f.clock));
let mut forged = r;
forged.signature[0] ^= 1;
assert!(c.apply(&mut f.db, &forged, &f.clock).is_err());
assert!(!c.can_start(&f.channel, false, &f.clock));
}
#[test]
fn requesting_running_and_omitted_attempts_cannot_regain_readiness() {
for state in ["requesting", "running"] {
let mut f = Fixture::new();
f.add(state, state == "running", None, script());
let mut c = f.client();
let p = f.prepare(&mut c);
assert_eq!(p.request.attempts[0].state, "unknown");
let mut d = f.decision(&p, true);
d.attempts.clear();
assert!(f.apply(&mut c, &d).is_err());
let d = f.decision(&p, true);
f.apply(&mut c, &d).unwrap();
assert!(!c.can_start(&f.channel, false, &f.clock));
let state: String =
f.db.query_row("SELECT state FROM job_attempts", [], |r| r.get(0))
.unwrap();
assert_eq!(state, "unknown");
if p.request.attempts[0].consumption == "consumed" {
let locked: i64 = f
.ledger
.query_row(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(locked, 1);
}
}
}
#[test]
fn final_result_preserves_original_envelope_outputs_and_exact_acks() {
let mut f = Fixture::new();
let (_, _, original) = f.add("succeeded", true, Some(success()), script());
let original = original.unwrap();
let mut c = f.client();
let p = f.prepare(&mut c);
assert_eq!(
p.request.attempts[0].result_envelope,
Some(original.clone())
);
assert_eq!(
p.request.attempts[0].outcome_bytes,
canonical_outcome_bytes(&success()).unwrap()
);
let unrelated = Envelope {
message_id: Uuid::new_v4().to_string(),
..Default::default()
};
f.db.execute(
"INSERT INTO job_events VALUES(?1,?2)",
params![unrelated.message_id, unrelated.encode_to_vec()],
)
.unwrap();
let mut d = f.decision(&p, true);
d.acknowledged_message_ids = vec![Uuid::parse_str(&original.message_id).unwrap()];
let r = f.sign(&d);
c.apply(&mut f.db, &r, &f.clock).unwrap();
assert!(!c.can_start(&f.channel, false, &f.clock));
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
assert!(c.can_start(&f.channel, false, &f.clock));
let pending: Vec<String> =
f.db.prepare("SELECT id FROM job_events")
.unwrap()
.query_map([], |r| r.get(0))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap();
assert_eq!(pending, vec![unrelated.message_id]);
let archived: Vec<u8> =
f.db.query_row(
"SELECT envelope FROM reconcile_results WHERE id=?1",
[&original.message_id],
|r| r.get(0),
)
.unwrap();
assert_eq!(Envelope::decode(archived.as_slice()).unwrap(), original);
assert_eq!(
f.ledger
.query_row(
"SELECT count(*) FROM consumed WHERE result IS NOT NULL",
[],
|r| r.get::<_, i64>(0)
)
.unwrap(),
1
);
}
#[test]
fn foreign_ack_and_noncanonical_outcome_are_transactionally_rejected() {
let mut f = Fixture::new();
f.add("succeeded", true, Some(success()), script());
let mut c = f.client();
let p = f.prepare(&mut c);
let mut d = f.decision(&p, true);
d.acknowledged_message_ids = vec![Uuid::new_v4()];
assert!(f.apply(&mut c, &d).is_err());
let mut d = f.decision(&p, true);
d.attempts[0].outcome_bytes = serde_json::to_vec_pretty(&success()).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
assert!(f.apply(&mut c, &d).is_err());
assert_eq!(
f.db.query_row("SELECT count(*) FROM job_events", [], |r| r
.get::<_, i64>(0))
.unwrap(),
1
);
}
#[test]
fn missing_server_job_fault_is_sticky_even_if_attempt_disappears_later() {
let mut f = Fixture::new();
f.add("accepted", false, None, script());
let mut c = f.client();
let p = f.prepare(&mut c);
let mut d = f.decision(&p, false);
d.reason = "MISSING_SERVER_JOB".into();
d.attempts[0].reason = "MISSING_SERVER_JOB".into();
d.attempts[0].action.clear();
d.attempts[0].execution_identity.clear();
f.apply(&mut c, &d).unwrap();
f.db.execute("DELETE FROM job_attempts", []).unwrap();
let mut fresh = f.client();
assert!(
fresh
.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
.is_err()
);
assert!(!fresh.can_start(&f.channel, false, &f.clock));
}
#[test]
fn rollback_and_false_clock_close_mutations_but_do_not_destroy_journal() {
let mut f = Fixture::new();
let mut c = f.client();
let p = f.prepare(&mut c);
let d = f.decision(&p, true);
f.apply(&mut c, &d).unwrap();
f.clock.trusted = false;
assert!(!c.can_start(&f.channel, false, &f.clock));
assert!(
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
.is_err()
);
f.clock.trusted = true;
f.db.execute("UPDATE counter SET seq=0", []).unwrap();
assert!(
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
.is_err()
);
f.db.execute("UPDATE counter SET seq=999", []).unwrap();
assert!(
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
.is_err()
);
let _native = NativeClock::default();
}
#[test]
fn uncertain_request_preserves_pending_request_and_last_applied_anchor() {
let mut f = Fixture::new();
let mut c = f.client();
let first = f.prepare(&mut c);
let d = f.decision(&first, true);
f.apply(&mut c, &d).unwrap();
let lost = f.prepare(&mut c);
let _server_decision = f.sign(&f.decision(&lost, true));
c.close();
f.channel.connection_generation += 1;
let recovered = f.prepare(&mut c);
assert_ne!(lost.request.request_id, recovered.request.request_id);
assert_eq!(
recovered.request.previous_request_id,
lost.request.request_id
);
assert_eq!(
recovered.request.previous_decision_id,
d.decision_id.to_string()
);
}
#[test]
fn helper_gate_requires_exact_signed_instruction_and_applied_checkpoint() {
let mut f = Fixture::new();
let mut c = f.client();
let mut helper = HelperSnapshot {
journal_id: Uuid::new_v4(),
checkpoint: 42,
previous_decision_id: None,
attempts: vec![],
};
let p = c
.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock,
)
.unwrap();
assert_eq!(p.envelope.sequence, p.request.checkpoint);
assert_eq!(
p.envelope.payload,
Some(Payload::SyncRequest(p.request.clone()))
);
let d = f.decision(&p, true);
let response = f.sign(&d);
let applied = c.apply(&mut f.db, &response, &f.clock).unwrap();
assert_eq!(applied.decision_id, d.decision_id);
assert_eq!(applied.reason, "CONSISTENT");
assert!(applied.acknowledged_message_ids.is_empty());
let instruction = applied.helper_instruction.unwrap();
assert_eq!(instruction.response, response);
assert_eq!(instruction.checkpoint, 42);
assert_eq!(instruction.journal_id, helper.journal_id);
assert!(c.can_start(&f.channel, false, &f.clock));
assert!(!c.can_start(&f.channel, true, &f.clock));
assert!(c.confirm_helper(&helper).is_err());
helper.previous_decision_id = Some(instruction.decision_id);
c.confirm_helper(&helper).unwrap();
assert!(c.can_start(&f.channel, true, &f.clock));
f.clock.now += Duration::seconds(60);
assert!(!c.can_start(&f.channel, false, &f.clock));
assert!(!c.can_start(&f.channel, true, &f.clock));
}
#[test]
fn independent_helper_checkpoint_regression_is_sticky() {
let mut f = Fixture::new();
let mut c = f.client();
let mut helper = HelperSnapshot {
journal_id: Uuid::new_v4(),
checkpoint: 5,
previous_decision_id: None,
attempts: vec![],
};
let p = c
.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock,
)
.unwrap();
let d = f.decision(&p, true);
f.apply(&mut c, &d).unwrap();
helper.previous_decision_id = Some(d.decision_id);
helper.checkpoint = 4;
f.channel.connection_generation += 1;
assert!(
c.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock
)
.is_err()
);
helper.checkpoint = 5;
assert!(
c.prepare(
&mut f.db,
&f.ledger,
f.channel.clone(),
Some(&helper),
&f.clock
)
.is_err()
);
}
#[test]
fn reboot_keeps_lock_until_signed_new_authenticated_boot_inventory_evidence() {
let mut f = Fixture::new();
let previous_boot = f.channel.boot_id;
let mut awaiting = success();
awaiting.state = "awaiting_reboot".into();
awaiting.stdout.clear();
awaiting.exit_code = None;
awaiting.reboot_required = true;
awaiting.evidence =
serde_json::json!({"postconditions_passed":false,"previous_boot_id":previous_boot});
let (_, grant, _) = f.add(
"awaiting_reboot",
true,
Some(awaiting),
Action::Reboot {
reason: "mock only".into(),
reconnect_timeout_seconds: 120,
dependency_group: "acceptance".into(),
},
);
let mut c = f.client();
let p = f.prepare(&mut c);
assert_eq!(
p.request.attempts[0].original_boot_id,
previous_boot.to_string()
);
let d = f.decision(&p, false);
f.apply(&mut c, &d).unwrap();
assert!(!c.can_start(&f.channel, false, &f.clock));
let preserved: String =
f.db.query_row("SELECT state FROM job_attempts", [], |r| r.get(0))
.unwrap();
assert_eq!(preserved, "awaiting_reboot");
f.channel.connection_generation += 1;
f.channel.boot_id = Uuid::new_v4();
let p = f.prepare(&mut c);
let mut d = f.decision(&p, true);
let mut definitive = success();
definitive.stdout.clear();
definitive.exit_code = None;
definitive.evidence = serde_json::json!({"postconditions_passed":true,"new_boot_id":previous_boot,"reconciled_after_reboot":true,"inventory_message_id":Uuid::new_v4()});
d.attempts[0].outcome_state = "succeeded".into();
d.attempts[0].revision += 1;
d.attempts[0].unlock = true;
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&definitive).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
assert!(f.apply(&mut c, &d).is_err());
definitive.evidence["new_boot_id"] = serde_json::json!(f.channel.boot_id);
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&definitive).unwrap();
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
f.apply(&mut c, &d).unwrap();
assert!(!c.can_start(&f.channel, false, &f.clock));
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
assert!(c.can_start(&f.channel, false, &f.clock));
let grant = grant.unwrap();
let result: Vec<u8> = f
.ledger
.query_row(
"SELECT result FROM consumed WHERE grant_id=?1",
[grant.grant_id.to_string()],
|r| r.get(0),
)
.unwrap();
assert_eq!(result, canonical_outcome_bytes(&definitive).unwrap());
assert_eq!(
f.ledger
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
.unwrap(),
1
);
}
#[test]
fn independent_local_outcome_mismatch_never_unlocks() {
let mut f = Fixture::new();
let (_, grant, _) = f.add("succeeded", true, Some(success()), script());
let mut c = f.client();
let p = f.prepare(&mut c);
let d = f.decision(&p, true);
f.apply(&mut c, &d).unwrap();
let mut altered = success();
altered.stdout = b"forged result".to_vec();
f.ledger
.execute(
"UPDATE execution_evidence SET outcome=?1",
[canonical_outcome_bytes(&altered).unwrap()],
)
.unwrap();
assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err());
assert!(!c.can_start(&f.channel, false, &f.clock));
let unresolved: i64 = f
.ledger
.query_row(
"SELECT count(*) FROM consumed WHERE grant_id=?1 AND result IS NULL",
[grant.unwrap().grant_id.to_string()],
|r| r.get(0),
)
.unwrap();
assert_eq!(unresolved, 1);
}
#[test]
fn signed_attempt_mutation_or_terminal_regression_is_rejected() {
let mut f = Fixture::new();
f.add("succeeded", true, Some(success()), script());
let mut c = f.client();
let p = f.prepare(&mut c);
let d = f.decision(&p, true);
let mut changed = d.clone();
changed.attempts[0].outcome_state = "unknown".into();
changed.attempts[0].unlock = false;
assert!(f.apply(&mut c, &changed).is_err());
let mut changed = d.clone();
changed.attempts[0].authorization_id = Some(Uuid::new_v4());
assert!(f.apply(&mut c, &changed).is_err());
let mut changed = d.clone();
changed.attempts[0].plan_hash = Some(hash(b"changed plan"));
assert!(f.apply(&mut c, &changed).is_err());
let mut changed = d;
changed.attempts[0].previous_outcome_sha256 = Some(hash(b"other outcome"));
assert!(f.apply(&mut c, &changed).is_err());
}