1466 lines
54 KiB
Rust
1466 lines
54 KiB
Rust
//! Existing Linux Sync semantic cases, run through the shared implementation
|
|
//! with only OS fixtures/signing adapters changed; no OS effects are performed.
|
|
use rmm_agent_core::reconcile;
|
|
|
|
use chrono::{DateTime, Duration, Utc};
|
|
use ed25519_dalek::{Signer, SigningKey};
|
|
use prost::Message;
|
|
use reconcile::*;
|
|
use rmm_agent_core::auth::{Binding, Request};
|
|
use rmm_domain::{jobs::*, reconciliation::*};
|
|
use rmm_protocol::*;
|
|
use rusqlite::{Connection, params};
|
|
use std::{fs, path::PathBuf};
|
|
use uuid::Uuid;
|
|
|
|
struct Clock {
|
|
now: DateTime<Utc>,
|
|
trusted: bool,
|
|
}
|
|
impl SyncClock for Clock {
|
|
fn now(&self) -> anyhow::Result<DateTime<Utc>> {
|
|
anyhow::ensure!(self.trusted, "CLOCK_UNTRUSTED");
|
|
Ok(self.now)
|
|
}
|
|
}
|
|
struct Fixture {
|
|
dir: PathBuf,
|
|
db: Connection,
|
|
ledger: Connection,
|
|
key: SigningKey,
|
|
binding: Binding,
|
|
channel: Channel,
|
|
clock: Clock,
|
|
helper_identity: HelperIdentity,
|
|
}
|
|
impl Drop for Fixture {
|
|
fn drop(&mut self) {
|
|
// Windows refuses removal of open SQLite files; close only our fixtures.
|
|
drop(std::mem::replace(
|
|
&mut self.db,
|
|
Connection::open_in_memory().unwrap(),
|
|
));
|
|
drop(std::mem::replace(
|
|
&mut self.ledger,
|
|
Connection::open_in_memory().unwrap(),
|
|
));
|
|
let _ = fs::remove_dir_all(&self.dir);
|
|
}
|
|
}
|
|
impl Fixture {
|
|
fn new() -> Self {
|
|
let dir = std::env::temp_dir().join(format!("reconcile-{}", Uuid::new_v4()));
|
|
fs::create_dir(&dir).unwrap();
|
|
let db = Connection::open(dir.join("journal.sqlite")).unwrap();
|
|
db.execute_batch("PRAGMA synchronous=FULL;CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);INSERT INTO counter VALUES(1,10);
|
|
CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT DEFAULT '',cancel_requested INTEGER DEFAULT 0,PRIMARY KEY(job,attempt));
|
|
CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE safety(key TEXT PRIMARY KEY,value TEXT);INSERT INTO safety VALUES('connected','1');").unwrap();
|
|
let ledger = Connection::open(dir.join("consumption.sqlite")).unwrap();
|
|
rmm_agent_core::ledger::initialize_ledger(&ledger).unwrap();
|
|
let key = SigningKey::from_bytes(&[42; 32]);
|
|
let public = key.verifying_key().to_bytes();
|
|
let binding = Binding {
|
|
tenant_id: Uuid::new_v4(),
|
|
asset_id: Uuid::new_v4(),
|
|
agent_id: Uuid::new_v4(),
|
|
enrollment_generation: 1,
|
|
assignment_version: 3,
|
|
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
|
};
|
|
let channel = Channel {
|
|
connection_generation: 1,
|
|
boot_id: Uuid::new_v4(),
|
|
};
|
|
Self {
|
|
dir,
|
|
db,
|
|
ledger,
|
|
key,
|
|
binding,
|
|
channel,
|
|
helper_identity: HelperIdentity::Root,
|
|
clock: Clock {
|
|
now: Utc::now(),
|
|
trusted: true,
|
|
},
|
|
}
|
|
}
|
|
fn client(&self) -> ClientSync {
|
|
ClientSync::new(self.binding.clone()).with_helper_identity(self.helper_identity)
|
|
}
|
|
fn prepare(&mut self, client: &mut ClientSync) -> PreparedSync {
|
|
client
|
|
.prepare(
|
|
&mut self.db,
|
|
&self.ledger,
|
|
self.channel.clone(),
|
|
None,
|
|
&self.clock,
|
|
)
|
|
.unwrap()
|
|
}
|
|
fn decision(&self, prepared: &PreparedSync, ready: bool) -> ReconcileDecisionPayload {
|
|
let request = &prepared.request;
|
|
let attempts = request
|
|
.attempts
|
|
.iter()
|
|
.map(|local| {
|
|
let bytes: Vec<u8> = self
|
|
.db
|
|
.query_row(
|
|
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
params![local.job_id, local.attempt_id],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
let offer = JobOffer::decode(bytes.as_slice()).unwrap();
|
|
let job: JobPayload = serde_json::from_slice(&offer.payload_bytes).unwrap();
|
|
let grant = if local.grant_id.is_empty() {
|
|
None
|
|
} else {
|
|
Some(Uuid::parse_str(&local.grant_id).unwrap())
|
|
};
|
|
let final_state = matches!(
|
|
local.state.as_str(),
|
|
"succeeded" | "failed" | "cancelled" | "expired"
|
|
);
|
|
AttemptDecision {
|
|
job_id: job.job_id,
|
|
attempt_id: job.attempt_id,
|
|
job_payload_hash: local.job_payload_hash.clone(),
|
|
grant_id: grant,
|
|
authorization_id: grant
|
|
.map(|_| Uuid::parse_str(&offer.authorization_id).unwrap()),
|
|
plan_hash: grant.map(|_| offer.plan_hash),
|
|
action: job.action.name().into(),
|
|
execution_identity: job.execution_identity,
|
|
revision: local.revision,
|
|
outcome_state: if local.state == "accepted" {
|
|
"accepted".into()
|
|
} else if final_state {
|
|
local.state.clone()
|
|
} else {
|
|
"unknown".into()
|
|
},
|
|
previous_outcome_sha256: (!local.outcome_sha256.is_empty())
|
|
.then(|| local.outcome_sha256.clone()),
|
|
outcome_sha256: final_state
|
|
.then(|| local.outcome_sha256.clone())
|
|
.filter(|s| !s.is_empty()),
|
|
outcome_bytes: vec![],
|
|
original_boot_id: (!local.original_boot_id.is_empty())
|
|
.then(|| Uuid::parse_str(&local.original_boot_id).unwrap()),
|
|
unlock: final_state && local.consumption == "consumed",
|
|
reason: if local.state == "accepted" {
|
|
"ACCEPTED_NOT_STARTED"
|
|
} else if final_state {
|
|
"DEFINITIVE_OUTCOME_RECONCILED"
|
|
} else {
|
|
"EFFECT_UNRESOLVED"
|
|
}
|
|
.into(),
|
|
}
|
|
})
|
|
.collect();
|
|
ReconcileDecisionPayload {
|
|
domain: "rmm.reconciliation-decision".into(),
|
|
schema_version: 1,
|
|
decision_id: Uuid::new_v4(),
|
|
request_id: Uuid::parse_str(&request.request_id).unwrap(),
|
|
request_hash: hash(&request.encode_to_vec()),
|
|
tenant_id: self.binding.tenant_id,
|
|
asset_id: self.binding.asset_id,
|
|
agent_id: self.binding.agent_id,
|
|
enrollment_generation: self.binding.enrollment_generation,
|
|
assignment_version: self.binding.assignment_version,
|
|
connection_generation: self.channel.connection_generation,
|
|
boot_id: self.channel.boot_id,
|
|
journal_id: Uuid::parse_str(&request.journal_id).unwrap(),
|
|
checkpoint: request.checkpoint,
|
|
helper_journal_id: (!request.helper_journal_id.is_empty())
|
|
.then(|| Uuid::parse_str(&request.helper_journal_id).unwrap()),
|
|
helper_checkpoint: request.helper_checkpoint,
|
|
issued_at: self.clock.now - Duration::milliseconds(1),
|
|
valid_until: self.clock.now + Duration::seconds(59),
|
|
ready,
|
|
helper_ready: ready && !request.helper_journal_id.is_empty(),
|
|
reason: if ready {
|
|
"CONSISTENT"
|
|
} else {
|
|
"ATTEMPT_RECONCILIATION_REQUIRED"
|
|
}
|
|
.into(),
|
|
attempts,
|
|
acknowledged_message_ids: vec![],
|
|
key_id: hash(&self.key.verifying_key().to_bytes()),
|
|
}
|
|
}
|
|
fn sign(&self, decision: &ReconcileDecisionPayload) -> SyncResponse {
|
|
let bytes = encode(decision).unwrap();
|
|
SyncResponse {
|
|
signature: self.key.sign(&bytes).to_bytes().to_vec(),
|
|
decision_bytes: bytes,
|
|
}
|
|
}
|
|
fn apply(
|
|
&mut self,
|
|
client: &mut ClientSync,
|
|
decision: &ReconcileDecisionPayload,
|
|
) -> anyhow::Result<Applied> {
|
|
let response = self.sign(decision);
|
|
client.apply(&mut self.db, &response, &self.clock)
|
|
}
|
|
fn add(
|
|
&mut self,
|
|
state: &str,
|
|
consume: bool,
|
|
outcome: Option<ReconciledOutcome>,
|
|
action: Action,
|
|
) -> (JobOffer, Option<ExecutionPermitPayload>, Option<Envelope>) {
|
|
let execution_identity = if matches!(action, Action::SetServiceState { .. }) {
|
|
self.helper_identity.as_str()
|
|
} else {
|
|
"agent_service"
|
|
};
|
|
let job = JobPayload {
|
|
schema_version: 1,
|
|
tenant_id: self.binding.tenant_id,
|
|
asset_id: self.binding.asset_id,
|
|
agent_id: self.binding.agent_id,
|
|
enrollment_generation: 1,
|
|
assignment_version: 3,
|
|
job_id: Uuid::new_v4(),
|
|
attempt_id: Uuid::new_v4(),
|
|
action,
|
|
execution_identity: execution_identity.into(),
|
|
not_before: self.clock.now - Duration::seconds(5),
|
|
start_before: self.clock.now + Duration::minutes(5),
|
|
max_runtime_seconds: 120,
|
|
max_output_bytes: 4096,
|
|
};
|
|
let offer = JobOffer {
|
|
job_id: job.job_id.to_string(),
|
|
attempt_id: job.attempt_id.to_string(),
|
|
authorization_id: Uuid::new_v4().to_string(),
|
|
plan_hash: hash(b"approved immutable plan"),
|
|
job_payload_hash: hash(&encode(&job).unwrap()),
|
|
payload_bytes: encode(&job).unwrap(),
|
|
};
|
|
let mut grant = None;
|
|
if consume {
|
|
let p = ExecutionPermitPayload {
|
|
domain: "rmm.execution-permit".into(),
|
|
schema_version: 1,
|
|
grant_id: Uuid::new_v4(),
|
|
authorization_id: Uuid::parse_str(&offer.authorization_id).unwrap(),
|
|
tenant_id: job.tenant_id,
|
|
asset_id: job.asset_id,
|
|
agent_id: job.agent_id,
|
|
enrollment_generation: 1,
|
|
assignment_version: 3,
|
|
job_id: job.job_id,
|
|
attempt_id: job.attempt_id,
|
|
plan_hash: offer.plan_hash.clone(),
|
|
job_payload_hash: offer.job_payload_hash.clone(),
|
|
action: job.action.name().into(),
|
|
execution_identity: job.execution_identity.clone(),
|
|
not_before: self.clock.now - Duration::milliseconds(1),
|
|
start_before: self.clock.now + Duration::seconds(59),
|
|
max_runtime_seconds: 120,
|
|
key_id: hash(&self.key.verifying_key().to_bytes()),
|
|
};
|
|
let bytes = encode(&p).unwrap();
|
|
let request = Request {
|
|
job_bytes: offer.payload_bytes.clone(),
|
|
permit_bytes: bytes.clone(),
|
|
signature: self.key.sign(&bytes).to_bytes().to_vec(),
|
|
plan_hash: offer.plan_hash.clone(),
|
|
authorization_id: p.authorization_id,
|
|
};
|
|
rmm_agent_core::ledger::consume(&mut self.ledger, &p).unwrap();
|
|
rmm_agent_core::ledger::save_start(
|
|
&self.ledger,
|
|
&p,
|
|
&request,
|
|
self.channel.boot_id,
|
|
self.clock.now,
|
|
)
|
|
.unwrap();
|
|
if let Some(o) = &outcome {
|
|
self.ledger
|
|
.execute(
|
|
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
|
|
params![p.grant_id.to_string(), canonical_outcome_bytes(o).unwrap()],
|
|
)
|
|
.unwrap();
|
|
}
|
|
grant = Some(p);
|
|
}
|
|
self.db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,?4,1,?5)",params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),state,grant.as_ref().map(|p|p.grant_id.to_string()).unwrap_or_default()]).unwrap();
|
|
let envelope = outcome.map(|o| {
|
|
let sequence: i64 = self
|
|
.db
|
|
.query_row("UPDATE counter SET seq=seq+1 RETURNING seq", [], |r| {
|
|
r.get(0)
|
|
})
|
|
.unwrap();
|
|
let r = JobResult {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
revision: 1,
|
|
state: o.state,
|
|
grant_id: grant
|
|
.as_ref()
|
|
.map(|p| p.grant_id.to_string())
|
|
.unwrap_or_default(),
|
|
stdout: o.stdout,
|
|
stderr: o.stderr,
|
|
output_truncated: o.output_truncated,
|
|
reboot_required: o.reboot_required,
|
|
exit_code: o.exit_code,
|
|
evidence_json: o.evidence.to_string(),
|
|
error_code: o.error_code,
|
|
};
|
|
let e = Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: self.binding.agent_id.to_string(),
|
|
enrollment_generation: 1,
|
|
connection_generation: self.channel.connection_generation,
|
|
boot_id: self.channel.boot_id.to_string(),
|
|
sequence: sequence as u64,
|
|
observed_at: self.clock.now.timestamp_millis(),
|
|
payload: Some(Payload::JobResult(r)),
|
|
};
|
|
self.db
|
|
.execute(
|
|
"INSERT INTO job_events VALUES(?1,?2)",
|
|
params![e.message_id, e.encode_to_vec()],
|
|
)
|
|
.unwrap();
|
|
e
|
|
});
|
|
(offer, grant, envelope)
|
|
}
|
|
}
|
|
#[test]
|
|
fn authenticated_helper_outcome_is_imported_as_new_durable_evidence_not_reexecuted() {
|
|
let mut f = Fixture::new();
|
|
let mut unknown = success();
|
|
unknown.state = "unknown".into();
|
|
unknown.error_code = "HELPER_TRANSPORT_EFFECT_UNKNOWN".into();
|
|
unknown.evidence = serde_json::json!({"postconditions_passed":false});
|
|
let (offer, grant, old) = f.add(
|
|
"unknown",
|
|
true,
|
|
Some(unknown),
|
|
Action::SetServiceState {
|
|
unit: "rmm-acceptance.service".into(),
|
|
state: "running".into(),
|
|
},
|
|
);
|
|
let grant = grant.unwrap();
|
|
let mut definitive = success();
|
|
definitive.stdout.clear();
|
|
definitive.exit_code = None;
|
|
let bytes = canonical_outcome_bytes(&definitive).unwrap();
|
|
let mut helper = HelperSnapshot {
|
|
journal_id: Uuid::new_v4(),
|
|
checkpoint: 3,
|
|
previous_decision_id: None,
|
|
attempts: vec![SyncAttempt {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
job_payload_hash: offer.job_payload_hash.clone(),
|
|
grant_id: grant.grant_id.to_string(),
|
|
consumption: "consumed".into(),
|
|
state: "succeeded".into(),
|
|
revision: 1,
|
|
original_boot_id: f.channel.boot_id.to_string(),
|
|
outcome_sha256: hash(&bytes),
|
|
outcome_bytes: bytes,
|
|
..Default::default()
|
|
}],
|
|
};
|
|
let mut c = f.client();
|
|
let p = c
|
|
.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock,
|
|
)
|
|
.unwrap();
|
|
let imported = p.request.attempts[0].result_envelope.as_ref().unwrap();
|
|
assert_ne!(imported.message_id, old.unwrap().message_id);
|
|
assert_eq!(p.request.attempts[0].state, "succeeded");
|
|
assert_eq!(p.request.attempts[0].revision, 2);
|
|
let d = f.decision(&p, true);
|
|
let applied = f.apply(&mut c, &d).unwrap();
|
|
assert!(applied.helper_instruction.is_some());
|
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(!c.can_start(&f.channel, true, &f.clock));
|
|
helper.previous_decision_id = Some(d.decision_id);
|
|
c.confirm_helper(&helper).unwrap();
|
|
assert!(c.can_start(&f.channel, true, &f.clock));
|
|
let retained: i64 =
|
|
f.db.query_row("SELECT count(*) FROM reconcile_results", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(retained, 2);
|
|
let tombstones: i64 = f
|
|
.ledger
|
|
.query_row("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(tombstones, 1);
|
|
}
|
|
#[test]
|
|
fn helper_omitting_a_consumed_root_attempt_is_a_sticky_restore_fault() {
|
|
let mut f = Fixture::new();
|
|
f.add(
|
|
"running",
|
|
true,
|
|
None,
|
|
Action::SetServiceState {
|
|
unit: "rmm-acceptance.service".into(),
|
|
state: "running".into(),
|
|
},
|
|
);
|
|
let helper = HelperSnapshot {
|
|
journal_id: Uuid::new_v4(),
|
|
checkpoint: 1,
|
|
previous_decision_id: None,
|
|
attempts: vec![],
|
|
};
|
|
let mut c = f.client();
|
|
assert!(
|
|
c.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock
|
|
)
|
|
.is_err()
|
|
);
|
|
assert!(
|
|
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
|
|
.is_err()
|
|
);
|
|
assert!(!c.can_start(&f.channel, true, &f.clock));
|
|
}
|
|
#[test]
|
|
fn definite_nonstart_result_is_unconsumed_not_ambiguous() {
|
|
let mut f = Fixture::new();
|
|
let mut cancelled = success();
|
|
cancelled.state = "cancelled".into();
|
|
cancelled.stdout.clear();
|
|
cancelled.exit_code = None;
|
|
cancelled.evidence = serde_json::json!({"not_started":true,"postconditions_passed":false});
|
|
f.add("cancelled", false, Some(cancelled), script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
assert_eq!(p.request.attempts[0].consumption, "unconsumed");
|
|
let d = f.decision(&p, true);
|
|
f.apply(&mut c, &d).unwrap();
|
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
|
}
|
|
#[test]
|
|
fn signed_terminal_nonstart_stays_unconsumed_across_sync_and_pending_denial() {
|
|
for terminal in ["expired", "cancelled"] {
|
|
let mut f = Fixture::new();
|
|
let (offer, _, _) = f.add("accepted", false, None, script());
|
|
let mut client = f.client();
|
|
let first = f.prepare(&mut client);
|
|
assert_eq!(first.request.attempts[0].consumption, "unconsumed");
|
|
let mut decision = f.decision(&first, true);
|
|
decision.attempts[0].outcome_state = terminal.into();
|
|
decision.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
|
f.apply(&mut client, &decision).unwrap();
|
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
|
|
|
let second = f.prepare(&mut client);
|
|
let local = &second.request.attempts[0];
|
|
assert_eq!(local.state, terminal);
|
|
assert_eq!(local.consumption, "unconsumed");
|
|
assert!(local.grant_id.is_empty() && local.outcome_bytes.is_empty());
|
|
let mut repeated = f.decision(&second, true);
|
|
repeated.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
|
f.apply(&mut client, &repeated).unwrap();
|
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
|
|
|
// Reproduce the old server's rejected terminal->unknown response.
|
|
// Keep its pending request and the last applied signed anchor, then
|
|
// recover through a fresh channel/request with the ordinary protocol.
|
|
let pending = f.prepare(&mut client);
|
|
let mut denied = f.decision(&pending, false);
|
|
denied.attempts[0].outcome_state = "unknown".into();
|
|
denied.attempts[0].reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
|
|
assert!(f.apply(&mut client, &denied).is_err());
|
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
|
client = f.client();
|
|
f.channel.connection_generation += 1;
|
|
let fresh = f.prepare(&mut client);
|
|
assert_eq!(
|
|
fresh.request.previous_request_id,
|
|
pending.request.request_id
|
|
);
|
|
assert_eq!(
|
|
fresh.request.previous_decision_id,
|
|
repeated.decision_id.to_string()
|
|
);
|
|
assert_eq!(fresh.request.attempts[0].consumption, "unconsumed");
|
|
let mut recovered = f.decision(&fresh, true);
|
|
recovered.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
|
f.apply(&mut client, &recovered).unwrap();
|
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
|
let stored: (String, String, String) =
|
|
f.db.query_row("SELECT job,attempt,state FROM job_attempts", [], |r| {
|
|
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
|
|
})
|
|
.unwrap();
|
|
assert_eq!(stored, (offer.job_id, offer.attempt_id, terminal.into()));
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
|
|
.unwrap(),
|
|
0
|
|
);
|
|
assert_eq!(
|
|
f.db.query_row("SELECT count(*) FROM reconcile_outcomes", [], |r| r
|
|
.get::<_, i64>(0))
|
|
.unwrap(),
|
|
0
|
|
);
|
|
}
|
|
}
|
|
#[test]
|
|
fn terminal_labels_never_override_grant_consumption_or_unclear_attempts() {
|
|
for state in ["requesting", "running", "unknown"] {
|
|
let mut f = Fixture::new();
|
|
f.add(state, false, None, script());
|
|
let mut client = f.client();
|
|
let prepared = f.prepare(&mut client);
|
|
assert_eq!(prepared.request.attempts[0].consumption, "uncertain");
|
|
let decision = f.decision(&prepared, true);
|
|
f.apply(&mut client, &decision).unwrap();
|
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
|
}
|
|
for terminal in ["expired", "cancelled"] {
|
|
for consumed in [false, true] {
|
|
let mut f = Fixture::new();
|
|
f.add(terminal, consumed, None, script());
|
|
// A surviving grant ID also forbids the nonstart shortcut when
|
|
// no consumption row exists. A real row always remains consumed.
|
|
if !consumed {
|
|
f.db.execute(
|
|
"UPDATE job_attempts SET grant_id=?1",
|
|
[Uuid::new_v4().to_string()],
|
|
)
|
|
.unwrap();
|
|
}
|
|
let mut client = f.client();
|
|
let prepared = f.prepare(&mut client);
|
|
assert_eq!(
|
|
prepared.request.attempts[0].consumption,
|
|
if consumed { "consumed" } else { "uncertain" }
|
|
);
|
|
let mut decision = f.decision(&prepared, true);
|
|
decision.attempts[0].unlock = false;
|
|
assert!(f.apply(&mut client, &decision).is_err());
|
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row(
|
|
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
|
[],
|
|
|r| r.get::<_, i64>(0)
|
|
)
|
|
.unwrap(),
|
|
i64::from(consumed)
|
|
);
|
|
}
|
|
}
|
|
}
|
|
#[test]
|
|
fn negative_server_revision_never_rolls_back_local_unknown_revision() {
|
|
let mut f = Fixture::new();
|
|
f.add("running", true, None, script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
assert_eq!(p.request.attempts[0].revision, 2);
|
|
let mut d = f.decision(&p, false);
|
|
d.attempts[0].revision = 1;
|
|
f.apply(&mut c, &d).unwrap();
|
|
let revision: u64 =
|
|
f.db.query_row("SELECT revision FROM job_attempts", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(revision, 2);
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
}
|
|
fn script() -> Action {
|
|
Action::RunScript {
|
|
interpreter: "posix_sh".into(),
|
|
script_utf8: "printf hello".into(),
|
|
arguments: vec![],
|
|
expected_stdout: Some("hello".into()),
|
|
}
|
|
}
|
|
fn success() -> ReconciledOutcome {
|
|
ReconciledOutcome {
|
|
state: "succeeded".into(),
|
|
stdout: b"hello".to_vec(),
|
|
stderr: vec![],
|
|
exit_code: Some(0),
|
|
output_truncated: false,
|
|
reboot_required: false,
|
|
evidence: serde_json::json!({"postconditions_passed":true,"z":{"b":2,"a":1}}),
|
|
error_code: String::new(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn clean_reconnect_and_reopen_require_new_signed_challenge() {
|
|
let mut f = Fixture::new();
|
|
let mut client = f.client();
|
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
|
let prepared = f.prepare(&mut client);
|
|
let decision = f.decision(&prepared, true);
|
|
let response = f.sign(&decision);
|
|
client.apply(&mut f.db, &response, &f.clock).unwrap();
|
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
|
assert_eq!(client.validity_deadline(), Some(decision.valid_until));
|
|
assert!(!client.can_start(&f.channel, true, &f.clock));
|
|
let mut reopened = Connection::open(f.dir.join("journal.sqlite")).unwrap();
|
|
let mut process = f.client();
|
|
assert!(!process.can_start(&f.channel, false, &f.clock));
|
|
f.channel.connection_generation += 1;
|
|
let next = process
|
|
.prepare(&mut reopened, &f.ledger, f.channel.clone(), None, &f.clock)
|
|
.unwrap();
|
|
assert_eq!(
|
|
next.request.previous_decision_id,
|
|
decision.decision_id.to_string()
|
|
);
|
|
assert_eq!(next.request.journal_id, prepared.request.journal_id);
|
|
assert!(next.request.checkpoint > prepared.request.checkpoint);
|
|
assert!(process.apply(&mut reopened, &response, &f.clock).is_err());
|
|
let next_decision = f.decision(&next, true);
|
|
process
|
|
.apply(&mut reopened, &f.sign(&next_decision), &f.clock)
|
|
.unwrap();
|
|
assert!(process.can_start(&f.channel, false, &f.clock));
|
|
let safety: String = reopened
|
|
.query_row("SELECT value FROM safety WHERE key='connected'", [], |r| {
|
|
r.get(0)
|
|
})
|
|
.unwrap();
|
|
assert_eq!(safety, "1");
|
|
}
|
|
#[test]
|
|
fn exact_signatures_all_identity_channel_checkpoint_and_request_bindings() {
|
|
let mut f = Fixture::new();
|
|
let mut client = f.client();
|
|
let prepared = f.prepare(&mut client);
|
|
let d = f.decision(&prepared, true);
|
|
assert!(
|
|
verify_response(
|
|
&f.binding,
|
|
&prepared.request,
|
|
&f.channel,
|
|
&f.sign(&d),
|
|
f.clock.now
|
|
)
|
|
.is_ok()
|
|
);
|
|
let value = serde_json::to_value(&d).unwrap();
|
|
for (field, replacement) in [
|
|
("domain", serde_json::json!("rmm.execution-permit")),
|
|
("schema_version", serde_json::json!(2)),
|
|
("tenant_id", serde_json::json!(Uuid::new_v4())),
|
|
("asset_id", serde_json::json!(Uuid::new_v4())),
|
|
("agent_id", serde_json::json!(Uuid::new_v4())),
|
|
("enrollment_generation", serde_json::json!(2)),
|
|
("assignment_version", serde_json::json!(4)),
|
|
("connection_generation", serde_json::json!(2)),
|
|
("boot_id", serde_json::json!(Uuid::new_v4())),
|
|
("journal_id", serde_json::json!(Uuid::new_v4())),
|
|
("checkpoint", serde_json::json!(0)),
|
|
("request_id", serde_json::json!(Uuid::new_v4())),
|
|
("request_hash", serde_json::json!(hash(b"different"))),
|
|
("helper_journal_id", serde_json::json!(Uuid::new_v4())),
|
|
("helper_checkpoint", serde_json::json!(1)),
|
|
("key_id", serde_json::json!(hash(b"wrongkey"))),
|
|
] {
|
|
let mut changed = value.clone();
|
|
changed[field] = replacement;
|
|
let changed: ReconcileDecisionPayload = serde_json::from_value(changed).unwrap();
|
|
assert!(
|
|
verify_response(
|
|
&f.binding,
|
|
&prepared.request,
|
|
&f.channel,
|
|
&f.sign(&changed),
|
|
f.clock.now
|
|
)
|
|
.is_err(),
|
|
"{field}"
|
|
);
|
|
}
|
|
let mut forged = f.sign(&d);
|
|
forged.signature[0] ^= 1;
|
|
assert!(client.apply(&mut f.db, &forged, &f.clock).is_err());
|
|
let fake = SigningKey::from_bytes(&[43; 32]);
|
|
let mut forged = f.sign(&d);
|
|
forged.signature = fake.sign(&forged.decision_bytes).to_bytes().to_vec();
|
|
assert!(client.apply(&mut f.db, &forged, &f.clock).is_err());
|
|
let mut changed = f.sign(&d);
|
|
changed.decision_bytes.push(b' ');
|
|
assert!(client.apply(&mut f.db, &changed, &f.clock).is_err());
|
|
let mut expired = d.clone();
|
|
expired.valid_until = f.clock.now;
|
|
assert!(f.apply(&mut client, &expired).is_err());
|
|
let mut unbounded = d;
|
|
unbounded.valid_until = unbounded.issued_at + Duration::seconds(60) + Duration::milliseconds(1);
|
|
assert!(f.apply(&mut client, &unbounded).is_err());
|
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
|
}
|
|
#[test]
|
|
fn accepted_unstarted_keeps_ids_and_never_consumes_or_executes() {
|
|
let mut f = Fixture::new();
|
|
let (offer, _, _) = f.add("accepted", false, None, script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
assert_eq!(p.request.attempts[0].consumption, "unconsumed");
|
|
let d = f.decision(&p, true);
|
|
let r = f.sign(&d);
|
|
c.apply(&mut f.db, &r, &f.clock).unwrap();
|
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
|
let count: i64 = f
|
|
.ledger
|
|
.query_row("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(count, 0);
|
|
let ids: (String, String, u64) =
|
|
f.db.query_row("SELECT job,attempt,revision FROM job_attempts", [], |r| {
|
|
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
|
|
})
|
|
.unwrap();
|
|
assert_eq!(ids, (offer.job_id, offer.attempt_id, 1));
|
|
c.apply(&mut f.db, &r, &f.clock).unwrap();
|
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
|
let mut forged = r;
|
|
forged.signature[0] ^= 1;
|
|
assert!(c.apply(&mut f.db, &forged, &f.clock).is_err());
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
}
|
|
#[test]
|
|
fn requesting_running_and_omitted_attempts_cannot_regain_readiness() {
|
|
for state in ["requesting", "running"] {
|
|
let mut f = Fixture::new();
|
|
f.add(state, state == "running", None, script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
assert_eq!(p.request.attempts[0].state, "unknown");
|
|
let mut d = f.decision(&p, true);
|
|
d.attempts.clear();
|
|
assert!(f.apply(&mut c, &d).is_err());
|
|
let d = f.decision(&p, true);
|
|
f.apply(&mut c, &d).unwrap();
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
let state: String =
|
|
f.db.query_row("SELECT state FROM job_attempts", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(state, "unknown");
|
|
if p.request.attempts[0].consumption == "consumed" {
|
|
let locked: i64 = f
|
|
.ledger
|
|
.query_row(
|
|
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
|
[],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(locked, 1);
|
|
}
|
|
}
|
|
}
|
|
#[test]
|
|
fn final_result_preserves_original_envelope_outputs_and_exact_acks() {
|
|
let mut f = Fixture::new();
|
|
let (_, _, original) = f.add("succeeded", true, Some(success()), script());
|
|
let original = original.unwrap();
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
assert_eq!(
|
|
p.request.attempts[0].result_envelope,
|
|
Some(original.clone())
|
|
);
|
|
assert_eq!(
|
|
p.request.attempts[0].outcome_bytes,
|
|
canonical_outcome_bytes(&success()).unwrap()
|
|
);
|
|
let unrelated = Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
..Default::default()
|
|
};
|
|
f.db.execute(
|
|
"INSERT INTO job_events VALUES(?1,?2)",
|
|
params![unrelated.message_id, unrelated.encode_to_vec()],
|
|
)
|
|
.unwrap();
|
|
let mut d = f.decision(&p, true);
|
|
d.acknowledged_message_ids = vec![Uuid::parse_str(&original.message_id).unwrap()];
|
|
let r = f.sign(&d);
|
|
c.apply(&mut f.db, &r, &f.clock).unwrap();
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
|
let pending: Vec<String> =
|
|
f.db.prepare("SELECT id FROM job_events")
|
|
.unwrap()
|
|
.query_map([], |r| r.get(0))
|
|
.unwrap()
|
|
.collect::<rusqlite::Result<_>>()
|
|
.unwrap();
|
|
assert_eq!(pending, vec![unrelated.message_id]);
|
|
let archived: Vec<u8> =
|
|
f.db.query_row(
|
|
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
|
[&original.message_id],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(Envelope::decode(archived.as_slice()).unwrap(), original);
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row(
|
|
"SELECT count(*) FROM consumed WHERE result IS NOT NULL",
|
|
[],
|
|
|r| r.get::<_, i64>(0)
|
|
)
|
|
.unwrap(),
|
|
1
|
|
);
|
|
}
|
|
#[test]
|
|
fn foreign_ack_and_noncanonical_outcome_are_transactionally_rejected() {
|
|
let mut f = Fixture::new();
|
|
f.add("succeeded", true, Some(success()), script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let mut d = f.decision(&p, true);
|
|
d.acknowledged_message_ids = vec![Uuid::new_v4()];
|
|
assert!(f.apply(&mut c, &d).is_err());
|
|
let mut d = f.decision(&p, true);
|
|
d.attempts[0].outcome_bytes = serde_json::to_vec_pretty(&success()).unwrap();
|
|
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
|
|
assert!(f.apply(&mut c, &d).is_err());
|
|
assert_eq!(
|
|
f.db.query_row("SELECT count(*) FROM job_events", [], |r| r
|
|
.get::<_, i64>(0))
|
|
.unwrap(),
|
|
1
|
|
);
|
|
}
|
|
#[test]
|
|
fn missing_server_job_fault_is_sticky_even_if_attempt_disappears_later() {
|
|
let mut f = Fixture::new();
|
|
f.add("accepted", false, None, script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let mut d = f.decision(&p, false);
|
|
d.reason = "MISSING_SERVER_JOB".into();
|
|
d.attempts[0].reason = "MISSING_SERVER_JOB".into();
|
|
d.attempts[0].action.clear();
|
|
d.attempts[0].execution_identity.clear();
|
|
f.apply(&mut c, &d).unwrap();
|
|
f.db.execute("DELETE FROM job_attempts", []).unwrap();
|
|
let mut fresh = f.client();
|
|
assert!(
|
|
fresh
|
|
.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
|
|
.is_err()
|
|
);
|
|
assert!(!fresh.can_start(&f.channel, false, &f.clock));
|
|
}
|
|
#[test]
|
|
fn rollback_and_false_clock_close_mutations_but_do_not_destroy_journal() {
|
|
let mut f = Fixture::new();
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let d = f.decision(&p, true);
|
|
f.apply(&mut c, &d).unwrap();
|
|
f.clock.trusted = false;
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
assert!(
|
|
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
|
|
.is_err()
|
|
);
|
|
f.clock.trusted = true;
|
|
f.db.execute("UPDATE counter SET seq=0", []).unwrap();
|
|
assert!(
|
|
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
|
|
.is_err()
|
|
);
|
|
f.db.execute("UPDATE counter SET seq=999", []).unwrap();
|
|
assert!(
|
|
c.prepare(&mut f.db, &f.ledger, f.channel.clone(), None, &f.clock)
|
|
.is_err()
|
|
);
|
|
}
|
|
#[test]
|
|
fn uncertain_request_preserves_pending_request_and_last_applied_anchor() {
|
|
let mut f = Fixture::new();
|
|
let mut c = f.client();
|
|
let first = f.prepare(&mut c);
|
|
let d = f.decision(&first, true);
|
|
f.apply(&mut c, &d).unwrap();
|
|
let lost = f.prepare(&mut c);
|
|
let _server_decision = f.sign(&f.decision(&lost, true));
|
|
c.close();
|
|
f.channel.connection_generation += 1;
|
|
let recovered = f.prepare(&mut c);
|
|
assert_ne!(lost.request.request_id, recovered.request.request_id);
|
|
assert_eq!(
|
|
recovered.request.previous_request_id,
|
|
lost.request.request_id
|
|
);
|
|
assert_eq!(
|
|
recovered.request.previous_decision_id,
|
|
d.decision_id.to_string()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn helper_gate_requires_exact_signed_instruction_and_applied_checkpoint() {
|
|
let mut f = Fixture::new();
|
|
let mut c = f.client();
|
|
let mut helper = HelperSnapshot {
|
|
journal_id: Uuid::new_v4(),
|
|
checkpoint: 42,
|
|
previous_decision_id: None,
|
|
attempts: vec![],
|
|
};
|
|
let p = c
|
|
.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock,
|
|
)
|
|
.unwrap();
|
|
assert_eq!(p.envelope.sequence, p.request.checkpoint);
|
|
assert_eq!(
|
|
p.envelope.payload,
|
|
Some(Payload::SyncRequest(p.request.clone()))
|
|
);
|
|
let d = f.decision(&p, true);
|
|
let response = f.sign(&d);
|
|
let applied = c.apply(&mut f.db, &response, &f.clock).unwrap();
|
|
assert_eq!(applied.decision_id, d.decision_id);
|
|
assert_eq!(applied.reason, "CONSISTENT");
|
|
assert!(applied.acknowledged_message_ids.is_empty());
|
|
let instruction = applied.helper_instruction.unwrap();
|
|
assert_eq!(instruction.response, response);
|
|
assert_eq!(instruction.checkpoint, 42);
|
|
assert_eq!(instruction.journal_id, helper.journal_id);
|
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
|
assert!(!c.can_start(&f.channel, true, &f.clock));
|
|
assert!(c.confirm_helper(&helper).is_err());
|
|
helper.previous_decision_id = Some(instruction.decision_id);
|
|
c.confirm_helper(&helper).unwrap();
|
|
assert!(c.can_start(&f.channel, true, &f.clock));
|
|
f.clock.now += Duration::seconds(60);
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
assert!(!c.can_start(&f.channel, true, &f.clock));
|
|
}
|
|
#[test]
|
|
fn independent_helper_checkpoint_regression_is_sticky() {
|
|
let mut f = Fixture::new();
|
|
let mut c = f.client();
|
|
let mut helper = HelperSnapshot {
|
|
journal_id: Uuid::new_v4(),
|
|
checkpoint: 5,
|
|
previous_decision_id: None,
|
|
attempts: vec![],
|
|
};
|
|
let p = c
|
|
.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock,
|
|
)
|
|
.unwrap();
|
|
let d = f.decision(&p, true);
|
|
f.apply(&mut c, &d).unwrap();
|
|
helper.previous_decision_id = Some(d.decision_id);
|
|
helper.checkpoint = 4;
|
|
f.channel.connection_generation += 1;
|
|
assert!(
|
|
c.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock
|
|
)
|
|
.is_err()
|
|
);
|
|
helper.checkpoint = 5;
|
|
assert!(
|
|
c.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock
|
|
)
|
|
.is_err()
|
|
);
|
|
}
|
|
#[test]
|
|
fn reboot_keeps_lock_until_signed_new_authenticated_boot_inventory_evidence() {
|
|
let mut f = Fixture::new();
|
|
let previous_boot = f.channel.boot_id;
|
|
let mut awaiting = success();
|
|
awaiting.state = "awaiting_reboot".into();
|
|
awaiting.stdout.clear();
|
|
awaiting.exit_code = None;
|
|
awaiting.reboot_required = true;
|
|
awaiting.evidence =
|
|
serde_json::json!({"postconditions_passed":false,"previous_boot_id":previous_boot});
|
|
let (_, grant, _) = f.add(
|
|
"awaiting_reboot",
|
|
true,
|
|
Some(awaiting),
|
|
Action::Reboot {
|
|
reason: "mock only".into(),
|
|
reconnect_timeout_seconds: 120,
|
|
dependency_group: "acceptance".into(),
|
|
},
|
|
);
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
assert_eq!(
|
|
p.request.attempts[0].original_boot_id,
|
|
previous_boot.to_string()
|
|
);
|
|
let d = f.decision(&p, false);
|
|
f.apply(&mut c, &d).unwrap();
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
let preserved: String =
|
|
f.db.query_row("SELECT state FROM job_attempts", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(preserved, "awaiting_reboot");
|
|
f.channel.connection_generation += 1;
|
|
f.channel.boot_id = Uuid::new_v4();
|
|
let p = f.prepare(&mut c);
|
|
let mut d = f.decision(&p, true);
|
|
let mut definitive = success();
|
|
definitive.stdout.clear();
|
|
definitive.exit_code = None;
|
|
definitive.evidence = serde_json::json!({"postconditions_passed":true,"new_boot_id":previous_boot,"reconciled_after_reboot":true,"inventory_message_id":Uuid::new_v4()});
|
|
d.attempts[0].outcome_state = "succeeded".into();
|
|
d.attempts[0].revision += 1;
|
|
d.attempts[0].unlock = true;
|
|
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&definitive).unwrap();
|
|
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
|
|
assert!(f.apply(&mut c, &d).is_err());
|
|
definitive.evidence["new_boot_id"] = serde_json::json!(f.channel.boot_id);
|
|
d.attempts[0].outcome_bytes = canonical_outcome_bytes(&definitive).unwrap();
|
|
d.attempts[0].outcome_sha256 = Some(hash(&d.attempts[0].outcome_bytes));
|
|
f.apply(&mut c, &d).unwrap();
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
|
let grant = grant.unwrap();
|
|
let result: Vec<u8> = f
|
|
.ledger
|
|
.query_row(
|
|
"SELECT result FROM consumed WHERE grant_id=?1",
|
|
[grant.grant_id.to_string()],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(result, canonical_outcome_bytes(&definitive).unwrap());
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
|
|
.unwrap(),
|
|
1
|
|
);
|
|
}
|
|
#[test]
|
|
fn independent_local_outcome_mismatch_never_unlocks() {
|
|
let mut f = Fixture::new();
|
|
let (_, grant, _) = f.add("succeeded", true, Some(success()), script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let d = f.decision(&p, true);
|
|
f.apply(&mut c, &d).unwrap();
|
|
let mut altered = success();
|
|
altered.stdout = b"forged result".to_vec();
|
|
f.ledger
|
|
.execute(
|
|
"UPDATE execution_evidence SET outcome=?1",
|
|
[canonical_outcome_bytes(&altered).unwrap()],
|
|
)
|
|
.unwrap();
|
|
assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err());
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
let unresolved: i64 = f
|
|
.ledger
|
|
.query_row(
|
|
"SELECT count(*) FROM consumed WHERE grant_id=?1 AND result IS NULL",
|
|
[grant.unwrap().grant_id.to_string()],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(unresolved, 1);
|
|
}
|
|
#[test]
|
|
fn signed_attempt_mutation_or_terminal_regression_is_rejected() {
|
|
let mut f = Fixture::new();
|
|
f.add("succeeded", true, Some(success()), script());
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let d = f.decision(&p, true);
|
|
let mut changed = d.clone();
|
|
changed.attempts[0].outcome_state = "unknown".into();
|
|
changed.attempts[0].unlock = false;
|
|
assert!(f.apply(&mut c, &changed).is_err());
|
|
let mut changed = d.clone();
|
|
changed.attempts[0].authorization_id = Some(Uuid::new_v4());
|
|
assert!(f.apply(&mut c, &changed).is_err());
|
|
let mut changed = d.clone();
|
|
changed.attempts[0].plan_hash = Some(hash(b"changed plan"));
|
|
assert!(f.apply(&mut c, &changed).is_err());
|
|
let mut changed = d;
|
|
changed.attempts[0].previous_outcome_sha256 = Some(hash(b"other outcome"));
|
|
assert!(f.apply(&mut c, &changed).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn system_helper_import_requires_explicit_system_identity_and_never_root_alias() {
|
|
let mut f = Fixture::new();
|
|
f.helper_identity = HelperIdentity::System;
|
|
let mut unknown = success();
|
|
unknown.state = "unknown".into();
|
|
unknown.error_code = "HELPER_TRANSPORT_EFFECT_UNKNOWN".into();
|
|
unknown.evidence = serde_json::json!({"postconditions_passed":false});
|
|
let (offer, grant, old) = f.add(
|
|
"unknown",
|
|
true,
|
|
Some(unknown),
|
|
Action::SetServiceState {
|
|
unit: "rmm-acceptance.service".into(),
|
|
state: "running".into(),
|
|
},
|
|
);
|
|
let grant = grant.unwrap();
|
|
let mut definitive = success();
|
|
definitive.stdout.clear();
|
|
definitive.exit_code = None;
|
|
let bytes = canonical_outcome_bytes(&definitive).unwrap();
|
|
let mut helper = HelperSnapshot {
|
|
journal_id: Uuid::new_v4(),
|
|
checkpoint: 3,
|
|
previous_decision_id: None,
|
|
attempts: vec![SyncAttempt {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
job_payload_hash: offer.job_payload_hash.clone(),
|
|
grant_id: grant.grant_id.to_string(),
|
|
consumption: "consumed".into(),
|
|
state: "succeeded".into(),
|
|
revision: 1,
|
|
original_boot_id: f.channel.boot_id.to_string(),
|
|
outcome_sha256: hash(&bytes),
|
|
outcome_bytes: bytes,
|
|
..Default::default()
|
|
}],
|
|
};
|
|
let mut wrong = ClientSync::new(f.binding.clone());
|
|
assert!(
|
|
wrong
|
|
.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock
|
|
)
|
|
.is_err()
|
|
);
|
|
let mut c = f.client();
|
|
let p = c
|
|
.prepare(
|
|
&mut f.db,
|
|
&f.ledger,
|
|
f.channel.clone(),
|
|
Some(&helper),
|
|
&f.clock,
|
|
)
|
|
.unwrap();
|
|
let imported = p.request.attempts[0].result_envelope.as_ref().unwrap();
|
|
assert_ne!(imported.message_id, old.unwrap().message_id);
|
|
assert_eq!(p.request.attempts[0].state, "succeeded");
|
|
assert_eq!(p.request.attempts[0].revision, 2);
|
|
let d = f.decision(&p, true);
|
|
let applied = f.apply(&mut c, &d).unwrap();
|
|
assert!(applied.helper_instruction.is_some());
|
|
c.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(!c.can_start(&f.channel, true, &f.clock));
|
|
helper.previous_decision_id = Some(d.decision_id);
|
|
c.confirm_helper(&helper).unwrap();
|
|
assert!(c.can_start(&f.channel, true, &f.clock));
|
|
let retained: i64 =
|
|
f.db.query_row("SELECT count(*) FROM reconcile_results", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(retained, 2);
|
|
let tombstones: i64 = f
|
|
.ledger
|
|
.query_row("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert_eq!(tombstones, 1);
|
|
let changed = c.with_helper_identity(HelperIdentity::Root);
|
|
assert!(!changed.can_start(&f.channel, true, &f.clock));
|
|
}
|
|
|
|
fn persist_original_fixture_result(
|
|
f: &mut Fixture,
|
|
offer: &JobOffer,
|
|
grant: &ExecutionPermitPayload,
|
|
outcome: rmm_agent_core::auth::Outcome,
|
|
) -> Envelope {
|
|
let tx = f.db.transaction().unwrap();
|
|
let revision =
|
|
rmm_agent_core::jobs::transition(&tx, offer, &outcome.state, &grant.grant_id.to_string())
|
|
.unwrap();
|
|
let sequence: i64 = tx
|
|
.query_row(
|
|
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
|
[],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
let envelope = Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: f.binding.agent_id.to_string(),
|
|
enrollment_generation: f.binding.enrollment_generation,
|
|
connection_generation: f.channel.connection_generation,
|
|
boot_id: f.channel.boot_id.to_string(),
|
|
sequence: sequence.try_into().unwrap(),
|
|
observed_at: f.clock.now.timestamp_millis(),
|
|
payload: Some(Payload::JobResult(rmm_agent_core::jobs::result(
|
|
offer,
|
|
revision,
|
|
&grant.grant_id.to_string(),
|
|
outcome,
|
|
))),
|
|
};
|
|
rmm_agent_core::jobs::event(&tx, &envelope).unwrap();
|
|
tx.commit().unwrap();
|
|
envelope
|
|
}
|
|
fn completion_outcome() -> rmm_agent_core::auth::Outcome {
|
|
serde_json::from_slice(&canonical_outcome_bytes(&success()).unwrap()).unwrap()
|
|
}
|
|
#[test]
|
|
fn crash_after_durable_outcome_before_result_recovers_original_then_signed_sync_closes() {
|
|
let mut f = Fixture::new();
|
|
let (offer, grant, _) = f.add("running", true, None, script());
|
|
let grant = grant.unwrap();
|
|
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
|
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
|
assert_eq!(interrupted.len(), 1);
|
|
let original = rmm_agent_core::jobs::recover_interrupted_outcome(
|
|
&f.ledger,
|
|
&f.binding,
|
|
&offer,
|
|
&interrupted[0].1,
|
|
)
|
|
.unwrap()
|
|
.unwrap();
|
|
let envelope = persist_original_fixture_result(&mut f, &offer, &grant, original);
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row::<u32, _, _>(
|
|
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
|
[],
|
|
|r| r.get(0)
|
|
)
|
|
.unwrap(),
|
|
1
|
|
);
|
|
let mut client = f.client();
|
|
let prepared = f.prepare(&mut client);
|
|
let decision = f.decision(&prepared, true);
|
|
assert!(decision.attempts[0].unlock);
|
|
assert_eq!(
|
|
prepared.request.attempts[0].result_message_id,
|
|
envelope.message_id
|
|
);
|
|
f.apply(&mut client, &decision).unwrap();
|
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
|
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
|
.unwrap(),
|
|
1
|
|
);
|
|
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
|
|
}
|
|
#[test]
|
|
fn crash_after_result_commit_before_finish_retains_exact_result_and_local_prior() {
|
|
let mut f = Fixture::new();
|
|
let (offer, grant, _) = f.add("running", true, None, script());
|
|
let grant = grant.unwrap();
|
|
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
|
let original = persist_original_fixture_result(&mut f, &offer, &grant, completion_outcome());
|
|
assert!(
|
|
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
|
.unwrap()
|
|
.is_empty()
|
|
);
|
|
assert_eq!(
|
|
rmm_agent_core::jobs::events(&f.db).unwrap(),
|
|
vec![original.clone()]
|
|
);
|
|
let mut client = f.client();
|
|
let p = f.prepare(&mut client);
|
|
let d = f.decision(&p, true);
|
|
f.apply(&mut client, &d).unwrap();
|
|
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
|
let archived: Vec<u8> =
|
|
f.db.query_row(
|
|
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
|
[&original.message_id],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(archived, original.encode_to_vec());
|
|
}
|
|
#[test]
|
|
fn crash_before_running_event_recovers_real_consumption_without_inventing_completion() {
|
|
let mut f = Fixture::new();
|
|
let (offer, grant, _) = f.add("running", true, None, script());
|
|
let grant = grant.unwrap();
|
|
f.db.execute("UPDATE job_attempts SET state='requesting',grant_id=''", [])
|
|
.unwrap();
|
|
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
|
assert_eq!(interrupted[0].1, grant.grant_id.to_string());
|
|
assert!(
|
|
rmm_agent_core::jobs::recover_interrupted_outcome(
|
|
&f.ledger,
|
|
&f.binding,
|
|
&offer,
|
|
&interrupted[0].1
|
|
)
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
persist_original_fixture_result(
|
|
&mut f,
|
|
&offer,
|
|
&grant,
|
|
rmm_agent_core::auth::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
|
);
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let d = f.decision(&p, false);
|
|
assert!(!d.attempts[0].unlock);
|
|
f.apply(&mut c, &d).unwrap();
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row::<u32, _, _>(
|
|
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
|
[],
|
|
|r| r.get(0)
|
|
)
|
|
.unwrap(),
|
|
1
|
|
);
|
|
let missing: Option<Vec<u8>> = f
|
|
.ledger
|
|
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
|
|
.unwrap();
|
|
assert!(missing.is_none());
|
|
}
|
|
#[test]
|
|
fn signed_unlock_still_rejects_a_missing_local_outcome_in_legacy_crash_window() {
|
|
let mut f = Fixture::new();
|
|
let (_, grant, _) = f.add("succeeded", true, Some(success()), script());
|
|
let grant = grant.unwrap();
|
|
f.ledger
|
|
.execute("UPDATE execution_evidence SET outcome=NULL", [])
|
|
.unwrap();
|
|
let mut c = f.client();
|
|
let p = f.prepare(&mut c);
|
|
let d = f.decision(&p, true);
|
|
assert!(d.attempts[0].unlock);
|
|
f.apply(&mut c, &d).unwrap();
|
|
assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err());
|
|
assert!(!c.can_start(&f.channel, false, &f.clock));
|
|
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn original_completion_after_reconnect_unknown_is_recovered_without_reexecuting() {
|
|
let mut f = Fixture::new();
|
|
let (offer, grant, _) = f.add("running", true, None, script());
|
|
let grant = grant.unwrap();
|
|
let mut previous = f.client();
|
|
let previous_request = f.prepare(&mut previous);
|
|
assert_eq!(previous_request.request.attempts[0].state, "unknown");
|
|
assert!(
|
|
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
|
.unwrap()
|
|
.is_empty()
|
|
);
|
|
// The already-running executor completes after the transport fenced it.
|
|
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
|
drop(previous);
|
|
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
|
assert_eq!(interrupted.len(), 1);
|
|
let saved = rmm_agent_core::jobs::recover_interrupted_outcome(
|
|
&f.ledger,
|
|
&f.binding,
|
|
&offer,
|
|
&interrupted[0].1,
|
|
)
|
|
.unwrap()
|
|
.unwrap();
|
|
let original = persist_original_fixture_result(&mut f, &offer, &grant, saved);
|
|
let mut process = f.client();
|
|
let p = f.prepare(&mut process);
|
|
let d = f.decision(&p, true);
|
|
assert_eq!(p.request.attempts[0].result_message_id, original.message_id);
|
|
f.apply(&mut process, &d).unwrap();
|
|
process.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
|
assert!(process.can_start(&f.channel, false, &f.clock));
|
|
assert_eq!(
|
|
f.ledger
|
|
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
|
.unwrap(),
|
|
1
|
|
);
|
|
assert!(
|
|
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
|
.unwrap()
|
|
.is_empty()
|
|
);
|
|
}
|