Qualify atomic deny-only helper abandonment on actual Linux VM
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -47,6 +47,412 @@ impl BootSource for TestBoot {
|
||||
Ok(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only protected-store/peer adapter. These unit checks do not qualify
|
||||
/// Windows ACLs, a live SCM peer, file identities or the real intent CAS.
|
||||
struct TestNoStartGuard {
|
||||
ledger: ExecutorGuard,
|
||||
instance: String,
|
||||
request_sha256: String,
|
||||
anchor_sha256: String,
|
||||
intent_recorded_at: DateTime<Utc>,
|
||||
original_boot_id: Uuid,
|
||||
dispatch_possible: bool,
|
||||
}
|
||||
impl ExecutorLedgerGuard for TestNoStartGuard {
|
||||
fn check(&self, db: &Connection) -> anyhow::Result<()> {
|
||||
self.ledger.check(db)
|
||||
}
|
||||
}
|
||||
impl NoStartGuard for TestNoStartGuard {
|
||||
fn live_instance_sha256(&self) -> anyhow::Result<String> {
|
||||
Ok(self.instance.clone())
|
||||
}
|
||||
fn verify_no_dispatch(
|
||||
&self,
|
||||
original: &Request,
|
||||
anchor: &NoStartAnchor,
|
||||
) -> anyhow::Result<NoDispatchObservation> {
|
||||
anyhow::ensure!(
|
||||
!self.dispatch_possible
|
||||
&& self.request_sha256 == hash(&serde_json::to_vec(original)?)
|
||||
&& self.anchor_sha256 == hash(&serde_json::to_vec(anchor)?),
|
||||
"TEST_INTENT_NOT_PROVEN"
|
||||
);
|
||||
Ok(NoDispatchObservation {
|
||||
intent_id: Uuid::from_u128(900),
|
||||
intent_sha256: hash(b"immutable test-only intent bytes"),
|
||||
intent_recorded_at: self.intent_recorded_at,
|
||||
original_boot_id: self.original_boot_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn denial_case() -> (
|
||||
Fixture,
|
||||
HelperSync,
|
||||
NoStartAnchor,
|
||||
Request,
|
||||
TestNoStartGuard,
|
||||
) {
|
||||
let mut f = Fixture::new();
|
||||
f.identity = HelperIdentity::System;
|
||||
let mut sync = f.sync();
|
||||
let capture = f.snapshot(&mut sync, 7);
|
||||
let raw = f.request(&capture);
|
||||
let ready = f.decision(&capture, &raw, 7, true);
|
||||
assert!(f.apply(&mut sync, &ready, &raw).unwrap().ready);
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: f.binding.tenant_id,
|
||||
asset_id: f.binding.asset_id,
|
||||
agent_id: f.binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 3,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::SetServiceState {
|
||||
unit: "RustDiscoveryMvpFixture".into(),
|
||||
state: "running".into(),
|
||||
},
|
||||
execution_identity: "system".into(),
|
||||
not_before: f.clock.now - Duration::seconds(10),
|
||||
start_before: f.clock.now + Duration::minutes(5),
|
||||
max_runtime_seconds: 30,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let permit = ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: Uuid::new_v4(),
|
||||
authorization_id: Uuid::new_v4(),
|
||||
tenant_id: job.tenant_id,
|
||||
asset_id: job.asset_id,
|
||||
agent_id: job.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 3,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
plan_hash: hash(b"approved exact service plan"),
|
||||
job_payload_hash: hash(&encode(&job).unwrap()),
|
||||
action: job.action.name().into(),
|
||||
execution_identity: job.execution_identity.clone(),
|
||||
max_runtime_seconds: 30,
|
||||
not_before: f.clock.now - Duration::seconds(1),
|
||||
start_before: f.clock.now + Duration::seconds(59),
|
||||
key_id: hash(&f.key.verifying_key().to_bytes()),
|
||||
};
|
||||
let bytes = encode(&permit).unwrap();
|
||||
let request = Request {
|
||||
job_bytes: encode(&job).unwrap(),
|
||||
signature: f.key.sign(&bytes).to_bytes().to_vec(),
|
||||
permit_bytes: bytes,
|
||||
plan_hash: permit.plan_hash,
|
||||
authorization_id: permit.authorization_id,
|
||||
};
|
||||
let mut guard = TestNoStartGuard {
|
||||
ledger: ExecutorGuard {
|
||||
directory: f.dir.clone(),
|
||||
},
|
||||
instance: hash(b"same live helper PID/creation/ledger/lock fixture"),
|
||||
request_sha256: hash(&serde_json::to_vec(&request).unwrap()),
|
||||
anchor_sha256: String::new(),
|
||||
intent_recorded_at: f.clock.now,
|
||||
original_boot_id: f.boot.0,
|
||||
dispatch_possible: false,
|
||||
};
|
||||
let anchor = sync
|
||||
.no_start_anchor(&f.db, &guard, 7, &f.clock, &f.boot)
|
||||
.unwrap();
|
||||
guard.anchor_sha256 = hash(&serde_json::to_vec(&anchor).unwrap());
|
||||
(f, sync, anchor, request, guard)
|
||||
}
|
||||
fn denial_ledger_bytes(db: &Connection) -> Vec<u8> {
|
||||
let mut statement=db.prepare("SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id").unwrap();
|
||||
type Row = (
|
||||
String,
|
||||
String,
|
||||
u64,
|
||||
String,
|
||||
String,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
);
|
||||
let rows = statement
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get(0)?,
|
||||
r.get(1)?,
|
||||
r.get(2)?,
|
||||
r.get(3)?,
|
||||
r.get(4)?,
|
||||
r.get(5)?,
|
||||
r.get(6)?,
|
||||
r.get(7)?,
|
||||
))
|
||||
})
|
||||
.unwrap()
|
||||
.collect::<rusqlite::Result<Vec<Row>>>()
|
||||
.unwrap();
|
||||
serde_json::to_vec(&rows).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandon_after_expiry_records_deny_only_evidence_and_requires_normal_signed_unlock() {
|
||||
let (mut f, mut sync, anchor, request, mut guard) = denial_case();
|
||||
let permit: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes).unwrap();
|
||||
f.clock.now += Duration::seconds(120);
|
||||
assert!(f.binding.verify(&request, f.clock.now).is_err());
|
||||
let outcome = sync
|
||||
.abandon_no_start(&mut f.db, &guard, &anchor, &request, &f.clock, &f.boot)
|
||||
.unwrap();
|
||||
assert_eq!(outcome.state, "failed");
|
||||
assert_eq!(outcome.error_code, "HELPER_ABANDONED_BEFORE_DISPATCH");
|
||||
assert_eq!(outcome.evidence["not_started"], true);
|
||||
assert_eq!(outcome.evidence["postconditions_passed"], false);
|
||||
assert_eq!(
|
||||
outcome.evidence["observed_at"],
|
||||
serde_json::to_value(f.clock.now).unwrap()
|
||||
);
|
||||
assert!(
|
||||
outcome.stdout.is_empty()
|
||||
&& outcome.stderr.is_empty()
|
||||
&& outcome.exit_code.is_none()
|
||||
&& !outcome.reboot_required
|
||||
);
|
||||
assert_eq!(
|
||||
f.db.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let before = denial_ledger_bytes(&f.db);
|
||||
guard.dispatch_possible = true; // An existing outcome is returned, never reclassified.
|
||||
let repeated = sync
|
||||
.abandon_no_start(&mut f.db, &guard, &anchor, &request, &f.clock, &f.boot)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_vec(&outcome).unwrap(),
|
||||
serde_json::to_vec(&repeated).unwrap()
|
||||
);
|
||||
assert_eq!(denial_ledger_bytes(&f.db), before);
|
||||
assert!(
|
||||
rmm_agent_core::ledger::begin_execution(
|
||||
&mut f.db,
|
||||
&permit,
|
||||
&request,
|
||||
f.boot.0,
|
||||
f.clock.now
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!sync.can_start(&f.db, &f.guard, 7, &f.clock, &f.boot));
|
||||
let capture = f.snapshot(&mut sync, 7);
|
||||
assert_eq!(
|
||||
capture.attempts[0].original_request_sha256,
|
||||
hash(&serde_json::to_vec(&request).unwrap())
|
||||
);
|
||||
let raw = f.request(&capture);
|
||||
let decision = f.decision(&capture, &raw, 7, true);
|
||||
assert!(f.apply(&mut sync, &decision, &raw).unwrap().ready);
|
||||
assert_eq!(
|
||||
f.db.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NOT NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
rmm_agent_core::ledger::begin_execution(
|
||||
&mut f.db,
|
||||
&permit,
|
||||
&request,
|
||||
f.boot.0,
|
||||
f.clock.now
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandon_storage_failure_rolls_back_consumption_start_and_outcome_together() {
|
||||
for trigger in [
|
||||
"CREATE TRIGGER refuse_start BEFORE INSERT ON execution_evidence BEGIN SELECT RAISE(ABORT,'fixture failure'); END;",
|
||||
"CREATE TRIGGER refuse_outcome BEFORE UPDATE OF outcome ON execution_evidence BEGIN SELECT RAISE(ABORT,'fixture failure'); END;",
|
||||
] {
|
||||
let (mut f, mut sync, anchor, request, guard) = denial_case();
|
||||
f.db.execute_batch(trigger).unwrap();
|
||||
assert!(
|
||||
sync.abandon_no_start(&mut f.db, &guard, &anchor, &request, &f.clock, &f.boot)
|
||||
.is_err()
|
||||
);
|
||||
for table in ["consumed", "execution_evidence"] {
|
||||
assert_eq!(
|
||||
f.db.query_row::<u32, _, _>(&format!("SELECT count(*) FROM {table}"), [], |r| r
|
||||
.get(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
f.db.execute_batch(
|
||||
"DROP TRIGGER IF EXISTS refuse_start;DROP TRIGGER IF EXISTS refuse_outcome",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
sync.abandon_no_start(&mut f.db, &guard, &anchor, &request, &f.clock, &f.boot)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandon_preserves_existing_known_unknown_missing_start_and_foreign_request_rows() {
|
||||
use rmm_agent_core::{auth::Outcome, ledger};
|
||||
for variant in [
|
||||
"known",
|
||||
"unknown",
|
||||
"missing_start",
|
||||
"missing_outcome",
|
||||
"foreign_grant",
|
||||
"changed_request",
|
||||
] {
|
||||
let (mut f, mut sync, anchor, request, guard) = denial_case();
|
||||
let mut stored = request.clone();
|
||||
let mut permit: ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes).unwrap();
|
||||
if variant == "foreign_grant" {
|
||||
permit.grant_id = Uuid::new_v4();
|
||||
}
|
||||
if variant == "changed_request" {
|
||||
permit.plan_hash = hash(b"other signed plan");
|
||||
stored.plan_hash = permit.plan_hash.clone();
|
||||
}
|
||||
stored.permit_bytes = encode(&permit).unwrap();
|
||||
stored.signature = f.key.sign(&stored.permit_bytes).to_bytes().to_vec();
|
||||
if variant == "missing_start" {
|
||||
ledger::consume(&mut f.db, &permit).unwrap();
|
||||
} else {
|
||||
ledger::begin_execution(&mut f.db, &permit, &stored, f.boot.0, f.clock.now).unwrap();
|
||||
}
|
||||
if variant == "known" || variant == "unknown" {
|
||||
let mut outcome = Outcome::unknown("ORIGINAL_OUTCOME");
|
||||
if variant == "known" {
|
||||
outcome.state = "failed".into();
|
||||
}
|
||||
ledger::persist_outcome(&f.db, &permit, &outcome).unwrap();
|
||||
if variant == "known" {
|
||||
ledger::finish(&f.db, &permit, &outcome).unwrap();
|
||||
}
|
||||
}
|
||||
let before = denial_ledger_bytes(&f.db);
|
||||
let response =
|
||||
sync.abandon_no_start(&mut f.db, &guard, &anchor, &request, &f.clock, &f.boot);
|
||||
if matches!(variant, "foreign_grant" | "changed_request") {
|
||||
assert!(response.is_err(), "{variant}");
|
||||
} else {
|
||||
let outcome = response.unwrap();
|
||||
assert_eq!(
|
||||
outcome.state,
|
||||
if variant == "known" {
|
||||
"failed"
|
||||
} else {
|
||||
"unknown"
|
||||
},
|
||||
"{variant}"
|
||||
);
|
||||
assert_ne!(outcome.error_code, "HELPER_ABANDONED_BEFORE_DISPATCH");
|
||||
}
|
||||
assert_eq!(denial_ledger_bytes(&f.db), before, "{variant}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandon_rejects_wrong_signature_scope_clock_boot_lineage_instance_and_dispatch_proof() {
|
||||
for variant in [
|
||||
"signature",
|
||||
"tenant",
|
||||
"root_identity",
|
||||
"script",
|
||||
"restarted",
|
||||
"clock",
|
||||
"boot",
|
||||
"instance",
|
||||
"history",
|
||||
"checkpoint",
|
||||
"dispatch",
|
||||
"future_intent",
|
||||
] {
|
||||
let (mut f, mut sync, anchor, mut request, mut guard) = denial_case();
|
||||
match variant {
|
||||
"signature" => request.signature[0] ^= 1,
|
||||
"tenant" | "root_identity" | "script" | "restarted" => {
|
||||
let mut job: JobPayload = serde_json::from_slice(&request.job_bytes).unwrap();
|
||||
let mut p: ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes).unwrap();
|
||||
if variant == "tenant" {
|
||||
job.tenant_id = Uuid::new_v4();
|
||||
p.tenant_id = job.tenant_id;
|
||||
}
|
||||
if variant == "root_identity" {
|
||||
job.execution_identity = "root".into();
|
||||
p.execution_identity = "root".into();
|
||||
}
|
||||
if variant == "script" {
|
||||
job.action = script();
|
||||
p.action = job.action.name().into();
|
||||
}
|
||||
if variant == "restarted" {
|
||||
job.action = Action::SetServiceState {
|
||||
unit: "RustDiscoveryMvpFixture".into(),
|
||||
state: "restarted".into(),
|
||||
};
|
||||
}
|
||||
request.job_bytes = encode(&job).unwrap();
|
||||
p.job_payload_hash = hash(&request.job_bytes);
|
||||
request.permit_bytes = encode(&p).unwrap();
|
||||
request.signature = f.key.sign(&request.permit_bytes).to_bytes().to_vec();
|
||||
}
|
||||
"clock" => f.clock.trusted = false,
|
||||
"boot" => f.boot.0 = Uuid::new_v4(),
|
||||
"instance" => guard.instance = hash(b"different helper PID or ledger identity"),
|
||||
"history" => {
|
||||
f.db.execute("DELETE FROM helper_sync_decisions", [])
|
||||
.unwrap();
|
||||
}
|
||||
"checkpoint" => {
|
||||
f.db.execute("UPDATE helper_sync_meta SET checkpoint=checkpoint+1", [])
|
||||
.unwrap();
|
||||
}
|
||||
"dispatch" => guard.dispatch_possible = true,
|
||||
"future_intent" => guard.intent_recorded_at = f.clock.now + Duration::seconds(1),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
sync.abandon_no_start(&mut f.db, &guard, &anchor, &request, &f.clock, &f.boot)
|
||||
.is_err(),
|
||||
"{variant}"
|
||||
);
|
||||
assert_eq!(
|
||||
f.db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
0,
|
||||
"{variant}"
|
||||
);
|
||||
assert_eq!(
|
||||
f.db.query_row::<u32, _, _>("SELECT count(*) FROM execution_evidence", [], |r| r
|
||||
.get(0))
|
||||
.unwrap(),
|
||||
0,
|
||||
"{variant}"
|
||||
);
|
||||
}
|
||||
}
|
||||
struct Fixture {
|
||||
dir: PathBuf,
|
||||
db: Connection,
|
||||
|
||||
Reference in New Issue
Block a user