Make start evidence atomic and recover only bound durable outcomes
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
+167
-45
@@ -72,60 +72,56 @@ pub fn interrupted_with(
|
||||
verifier: &impl crate::crypto::Ed25519Verifier,
|
||||
) -> Result<Vec<(JobOffer, String)>> {
|
||||
let mut statement = db.prepare(
|
||||
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
|
||||
"SELECT offer,grant_id,state FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown') ORDER BY job,attempt",
|
||||
)?;
|
||||
let rows = statement.query_map([], |r| {
|
||||
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
|
||||
Ok((
|
||||
r.get::<_, Vec<u8>>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut interrupted = Vec::new();
|
||||
for row in rows {
|
||||
let (bytes, grant) = row?;
|
||||
let (bytes, mut grant, state) = row?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
validate_interrupted_offer(&offer, &job)?;
|
||||
// begin_execution may have committed before the running journal event.
|
||||
// Recover its actual grant, never invent one from an accepted offer.
|
||||
let (count, stored): (u64, Option<String>) = ledger.query_row(
|
||||
"SELECT count(*),MIN(grant_id) FROM consumed WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
ensure!(count <= 1, "INTERRUPTED_MULTIPLE_CONSUMPTIONS");
|
||||
if let Some(stored) = stored {
|
||||
ensure!(
|
||||
grant.is_empty() || grant == stored,
|
||||
"INTERRUPTED_GRANT_CONFLICT"
|
||||
);
|
||||
grant = stored;
|
||||
}
|
||||
if state == "unknown" {
|
||||
// A reconnect may have closed the running journal while the same
|
||||
// original executor was still finishing. Only its already-durable
|
||||
// terminal completion can repair this event gap; uncertainty stays.
|
||||
if !grant.is_empty()
|
||||
&& recover_interrupted_outcome_with(ledger, binding, &offer, &grant, verifier)?
|
||||
.is_some_and(|saved| {
|
||||
matches!(
|
||||
saved.state.as_str(),
|
||||
"succeeded" | "failed" | "cancelled" | "expired"
|
||||
)
|
||||
})
|
||||
{
|
||||
interrupted.push((offer, grant));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !grant.is_empty() {
|
||||
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
|
||||
.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[&grant],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (agent, generation, job_id, attempt, start) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Verify the historical signed request at its original valid time;
|
||||
// this is evidence only, never permission to start it again.
|
||||
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
verified_interrupted_start(ledger, binding, &offer, &grant, verifier)?;
|
||||
} else {
|
||||
// A request may have committed remotely without its reply arriving.
|
||||
// It can only become local unknown, never a fabricated grant/result.
|
||||
let state: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
@@ -138,6 +134,132 @@ pub fn interrupted_with(
|
||||
Ok(interrupted)
|
||||
}
|
||||
|
||||
fn validate_interrupted_offer(offer: &JobOffer, job: &JobPayload) -> Result<()> {
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
type InterruptedStartRow = (
|
||||
String,
|
||||
u64,
|
||||
String,
|
||||
String,
|
||||
Vec<u8>,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
);
|
||||
type InterruptedOutcomes = (Option<Vec<u8>>, Option<Vec<u8>>);
|
||||
fn verified_interrupted_start(
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
verifier: &impl crate::crypto::Ed25519Verifier,
|
||||
) -> Result<InterruptedOutcomes> {
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
validate_interrupted_offer(offer, &job)?;
|
||||
ensure!(
|
||||
Uuid::parse_str(grant).is_ok_and(|id| !id.is_nil()),
|
||||
"INTERRUPTED_GRANT_ID"
|
||||
);
|
||||
let row:Option<InterruptedStartRow>=ledger.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[grant],
|
||||
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
||||
let (agent, generation, job_id, attempt, start, outcome, result) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let boot = start["previous_boot_id"]
|
||||
.as_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_ORIGINAL_BOOT_MISSING"))?;
|
||||
ensure!(
|
||||
Uuid::parse_str(boot).is_ok_and(|id| !id.is_nil()),
|
||||
"INTERRUPTED_ORIGINAL_BOOT_MISSING"
|
||||
);
|
||||
let _: chrono::DateTime<chrono::Utc> = serde_json::from_value(start["reserved_at"].clone())?;
|
||||
ensure!(
|
||||
start["reauthentication_required"] == true && start["postconditions_passed"] == false,
|
||||
"INTERRUPTED_START_WITNESS_PROFILE"
|
||||
);
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Historical verification is evidence only; it can NEVER start another job.
|
||||
let (_, verified) = binding.verify_with(&request, permit.not_before, verifier)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
Ok((outcome, result))
|
||||
}
|
||||
/// Recover only a completion already durably recorded by the original executor.
|
||||
/// No outcome means uncertainty, not nonstart or a permission to execute again.
|
||||
/// Caller re-journalizes this evidence, never calls finish/unlocks at startup.
|
||||
pub fn recover_interrupted_outcome(
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
) -> Result<Option<Outcome>> {
|
||||
recover_interrupted_outcome_with(
|
||||
ledger,
|
||||
binding,
|
||||
offer,
|
||||
grant,
|
||||
&crate::crypto::DefaultVerifier::default(),
|
||||
)
|
||||
}
|
||||
pub fn recover_interrupted_outcome_with(
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
verifier: &impl crate::crypto::Ed25519Verifier,
|
||||
) -> Result<Option<Outcome>> {
|
||||
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
|
||||
let (outcome, result) = verified_interrupted_start(ledger, binding, offer, grant, verifier)?;
|
||||
let canonical = |bytes: &[u8]| -> Result<Vec<u8>> {
|
||||
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
|
||||
ReconciledOutcome,
|
||||
>(bytes)?)?)
|
||||
};
|
||||
if let (Some(a), Some(b)) = (&outcome, &result) {
|
||||
ensure!(
|
||||
canonical(a)? == canonical(b)?,
|
||||
"INTERRUPTED_OUTCOME_CONFLICT"
|
||||
);
|
||||
}
|
||||
let Some(bytes) = outcome.or(result) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let saved: Outcome = serde_json::from_slice(&canonical(&bytes)?)?;
|
||||
ensure!(
|
||||
matches!(
|
||||
saved.state.as_str(),
|
||||
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
|
||||
),
|
||||
"INTERRUPTED_OUTCOME_STATE"
|
||||
);
|
||||
Ok(Some(saved))
|
||||
}
|
||||
|
||||
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
||||
ensure!(
|
||||
db.execute(
|
||||
|
||||
+141
-19
@@ -1,8 +1,13 @@
|
||||
//! Connection-level ledger rules, with OS path/lock/ACL handling left to adapters.
|
||||
use crate::auth::{Outcome, Request};
|
||||
use anyhow::{Result, ensure};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rmm_domain::jobs::{ExecutionPermitPayload, JobPayload};
|
||||
use rmm_domain::{
|
||||
jobs::hash,
|
||||
reconciliation::{ReconciledOutcome, canonical_outcome_bytes},
|
||||
};
|
||||
use rusqlite::{OptionalExtension, TransactionBehavior, params};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Finite 1 GiB bound for durable agent/helper history. Unexpected page sizes
|
||||
@@ -29,18 +34,24 @@ pub fn initialize_ledger(db: &rusqlite::Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Legacy split reservation API retained for old fixtures/readers. Production
|
||||
/// callers use begin_execution so a crash cannot lose the original start witness.
|
||||
pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
|
||||
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
// Unknown effects retain the global device lock, including across restart.
|
||||
let unresolved: i64 = tx.query_row(
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
consume_in(&tx, p)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn consume_in(db: &rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
|
||||
let unresolved: i64 = db.query_row(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(unresolved == 0, "UNRESOLVED_EFFECT_RESOURCE_LOCKED");
|
||||
tx.execute(
|
||||
db.execute(
|
||||
"INSERT INTO consumed(grant_id,agent,generation,job,attempt) VALUES(?1,?2,?3,?4,?5)",
|
||||
rusqlite::params![
|
||||
params![
|
||||
p.grant_id.to_string(),
|
||||
p.agent_id.to_string(),
|
||||
p.enrollment_generation,
|
||||
@@ -48,29 +59,140 @@ pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Res
|
||||
p.attempt_id.to_string()
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
/// Caller has already checked the signature, qualified clock, fresh Sync and
|
||||
/// policy. Persist consumption and the exact legacy start witness atomically.
|
||||
pub fn begin_execution(
|
||||
db: &mut rusqlite::Connection,
|
||||
permit: &ExecutionPermitPayload,
|
||||
request: &Request,
|
||||
previous_boot_id: Uuid,
|
||||
recorded_at: DateTime<Utc>,
|
||||
) -> Result<()> {
|
||||
validate_start_request(permit, request)?;
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
consume_in(&tx, permit)?;
|
||||
save_start(&tx, permit, request, previous_boot_id, recorded_at)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn validate_start_request(p: &ExecutionPermitPayload, request: &Request) -> Result<()> {
|
||||
ensure!(
|
||||
request.job_bytes.len() <= 65536
|
||||
&& request.permit_bytes.len() <= 8192
|
||||
&& request.signature.len() == 64,
|
||||
"START_REQUEST_SIZE"
|
||||
);
|
||||
let saved: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?;
|
||||
let j: JobPayload = serde_json::from_slice(&request.job_bytes)?;
|
||||
ensure!(
|
||||
saved == *p
|
||||
&& request.authorization_id == p.authorization_id
|
||||
&& request.plan_hash == p.plan_hash
|
||||
&& p.job_payload_hash == hash(&request.job_bytes)
|
||||
&& p.tenant_id == j.tenant_id
|
||||
&& p.asset_id == j.asset_id
|
||||
&& p.agent_id == j.agent_id
|
||||
&& p.enrollment_generation == j.enrollment_generation
|
||||
&& p.assignment_version == j.assignment_version
|
||||
&& p.job_id == j.job_id
|
||||
&& p.attempt_id == j.attempt_id
|
||||
&& p.action == j.action.name()
|
||||
&& p.execution_identity == j.execution_identity
|
||||
&& p.max_runtime_seconds == j.max_runtime_seconds,
|
||||
"STORED_START_REQUEST_BINDING"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn canonical(bytes: &[u8]) -> Result<Vec<u8>> {
|
||||
Ok(canonical_outcome_bytes(&serde_json::from_slice::<
|
||||
ReconciledOutcome,
|
||||
>(bytes)?)?)
|
||||
}
|
||||
/// Compare-and-set the original completion evidence WITHOUT releasing a lock.
|
||||
/// Existing identical bytes are preserved; contradictory completions fail closed.
|
||||
pub fn persist_outcome(
|
||||
db: &rusqlite::Connection,
|
||||
p: &ExecutionPermitPayload,
|
||||
outcome: &Outcome,
|
||||
) -> Result<()> {
|
||||
let tx = db.unchecked_transaction()?;
|
||||
persist_outcome_in(&tx, p, outcome)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn persist_outcome_in(
|
||||
db: &rusqlite::Connection,
|
||||
p: &ExecutionPermitPayload,
|
||||
outcome: &Outcome,
|
||||
) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
matches!(
|
||||
outcome.state.as_str(),
|
||||
"unknown" | "awaiting_reboot" | "succeeded" | "failed" | "cancelled" | "expired"
|
||||
),
|
||||
"INVALID_STORED_OUTCOME_STATE"
|
||||
);
|
||||
type CompletionRow = (
|
||||
String,
|
||||
u64,
|
||||
String,
|
||||
String,
|
||||
Vec<u8>,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
);
|
||||
let row:Option<CompletionRow>=db.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[p.grant_id.to_string()],
|
||||
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
||||
let (agent, generation, job, attempt, start, prior, finished) =
|
||||
row.context("STORED_COMPLETION_START_MISSING")?;
|
||||
ensure!(
|
||||
agent == p.agent_id.to_string()
|
||||
&& generation == p.enrollment_generation
|
||||
&& job == p.job_id.to_string()
|
||||
&& attempt == p.attempt_id.to_string(),
|
||||
"STORED_COMPLETION_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
validate_start_request(p, &request)?;
|
||||
let bytes = serde_json::to_vec(outcome)?;
|
||||
let expected = canonical(&bytes)?;
|
||||
for previous in [&prior, &finished].into_iter().flatten() {
|
||||
ensure!(
|
||||
canonical(previous)? == expected,
|
||||
"STORED_COMPLETION_CONFLICT"
|
||||
);
|
||||
}
|
||||
if prior.is_none() {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1 AND outcome IS NULL",
|
||||
params![p.grant_id.to_string(), bytes]
|
||||
)? == 1,
|
||||
"STORED_COMPLETION_CAS_CONFLICT"
|
||||
);
|
||||
}
|
||||
Ok(prior.unwrap_or(bytes))
|
||||
}
|
||||
/// Called only AFTER the corresponding journal result committed. A crash before
|
||||
/// this call retains the original outcome for the normal signed reconciliation.
|
||||
pub fn finish(
|
||||
db: &rusqlite::Connection,
|
||||
p: &ExecutionPermitPayload,
|
||||
outcome: &Outcome,
|
||||
) -> Result<()> {
|
||||
db.execute(
|
||||
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1",
|
||||
rusqlite::params![p.grant_id.to_string(), serde_json::to_vec(outcome)?],
|
||||
)?;
|
||||
// Unknown / awaiting reboot keep the resource reserved until a real sync
|
||||
// protocol exists. Never unlock on a process exit or a network reconnect.
|
||||
let tx = db.unchecked_transaction()?;
|
||||
let bytes = persist_outcome_in(&tx, p, outcome)?;
|
||||
if outcome.state != "unknown" && outcome.state != "awaiting_reboot" {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE consumed SET result=?2 WHERE grant_id=?1 AND result IS NULL",
|
||||
rusqlite::params![p.grant_id.to_string(), serde_json::to_vec(outcome)?]
|
||||
)? == 1,
|
||||
"CONSUMPTION_NOT_FOUND"
|
||||
);
|
||||
tx.execute(
|
||||
"UPDATE consumed SET result=?2 WHERE grant_id=?1 AND result IS NULL",
|
||||
params![p.grant_id.to_string(), bytes],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -290,3 +290,129 @@ fn portable_acceptance_requires_platform_capability_and_exact_redelivery() {
|
||||
assert!(jobs::next(&db).unwrap().is_none());
|
||||
assert!(jobs::transition(&db, &offer, "running", "").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_begin_execution_rolls_back_both_rows_on_missing_witness_or_start_insert_failure() {
|
||||
let (_, request, permit, now) = fixture();
|
||||
let mut db = Connection::open_in_memory().unwrap();
|
||||
ledger::initialize_ledger(&db).unwrap();
|
||||
assert!(ledger::begin_execution(&mut db, &permit, &request, Uuid::nil(), now).is_err());
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
db.execute_batch("CREATE TRIGGER refuse_start BEFORE INSERT ON execution_evidence BEGIN SELECT RAISE(ABORT,'fixture storage failure'); END;").unwrap();
|
||||
assert!(ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).is_err());
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>("SELECT count(*) FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
db.execute_batch("DROP TRIGGER refuse_start").unwrap();
|
||||
ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).unwrap();
|
||||
assert!(ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).is_err());
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_cas_preserves_original_bytes_and_recovers_only_verified_bound_evidence() {
|
||||
let (binding, request, permit, now) = fixture();
|
||||
let mut db = Connection::open_in_memory().unwrap();
|
||||
ledger::initialize_ledger(&db).unwrap();
|
||||
ledger::begin_execution(&mut db, &permit, &request, Uuid::from_u128(9), now).unwrap();
|
||||
let offer = JobOffer {
|
||||
job_id: permit.job_id.to_string(),
|
||||
attempt_id: permit.attempt_id.to_string(),
|
||||
payload_bytes: request.job_bytes.clone(),
|
||||
job_payload_hash: permit.job_payload_hash.clone(),
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
authorization_id: permit.authorization_id.to_string(),
|
||||
};
|
||||
assert!(
|
||||
jobs::recover_interrupted_outcome(&db, &binding, &offer, &permit.grant_id.to_string())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let mut original = Outcome::unknown("");
|
||||
original.state = "succeeded".into();
|
||||
original.stdout = b"Hello\n".to_vec();
|
||||
original.exit_code = Some(0);
|
||||
original.evidence = serde_json::json!({"postconditions_passed":true});
|
||||
ledger::persist_outcome(&db, &permit, &original).unwrap();
|
||||
let saved: Vec<u8> = db
|
||||
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
ledger::persist_outcome(&db, &permit, &original).unwrap();
|
||||
assert_eq!(
|
||||
saved,
|
||||
db.query_row::<Vec<u8>, _, _>("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(ledger::persist_outcome(&db, &permit, &Outcome::unknown("late uncertainty")).is_err());
|
||||
assert!(ledger::finish(&db, &permit, &Outcome::unknown("late uncertainty")).is_err());
|
||||
let recovered =
|
||||
jobs::recover_interrupted_outcome(&db, &binding, &offer, &permit.grant_id.to_string())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(serde_json::to_vec(&recovered).unwrap(), saved);
|
||||
let mut forged = binding.clone();
|
||||
forged.public_key_hex =
|
||||
hex::encode(SigningKey::from_bytes(&[31; 32]).verifying_key().as_bytes());
|
||||
assert!(
|
||||
jobs::recover_interrupted_outcome(&db, &forged, &offer, &permit.grant_id.to_string())
|
||||
.is_err()
|
||||
);
|
||||
let mut changed = offer.clone();
|
||||
changed.authorization_id = Uuid::new_v4().to_string();
|
||||
assert!(
|
||||
jobs::recover_interrupted_outcome(&db, &binding, &changed, &permit.grant_id.to_string())
|
||||
.is_err()
|
||||
);
|
||||
let mut changed = permit.clone();
|
||||
changed.assignment_version += 1;
|
||||
assert!(ledger::persist_outcome(&db, &changed, &original).is_err());
|
||||
ledger::finish(&db, &permit, &original).unwrap();
|
||||
ledger::finish(&db, &permit, &original).unwrap();
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(ledger::consume(&mut db, &permit).is_err());
|
||||
let mut corrupted: serde_json::Value = db
|
||||
.query_row("SELECT start FROM execution_evidence", [], |r| {
|
||||
r.get::<_, Vec<u8>>(0)
|
||||
})
|
||||
.map(|b| serde_json::from_slice(&b).unwrap())
|
||||
.unwrap();
|
||||
let changed = corrupted["request"]["signature"][0].as_u64().unwrap() ^ 1;
|
||||
corrupted["request"]["signature"][0] = serde_json::json!(changed);
|
||||
db.execute(
|
||||
"UPDATE execution_evidence SET start=?1",
|
||||
[serde_json::to_vec(&corrupted).unwrap()],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
jobs::recover_interrupted_outcome(&db, &binding, &offer, &permit.grant_id.to_string())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1232,3 +1232,234 @@ fn system_helper_import_requires_explicit_system_identity_and_never_root_alias()
|
||||
let changed = c.with_helper_identity(HelperIdentity::Root);
|
||||
assert!(!changed.can_start(&f.channel, true, &f.clock));
|
||||
}
|
||||
|
||||
fn persist_original_fixture_result(
|
||||
f: &mut Fixture,
|
||||
offer: &JobOffer,
|
||||
grant: &ExecutionPermitPayload,
|
||||
outcome: rmm_agent_core::auth::Outcome,
|
||||
) -> Envelope {
|
||||
let tx = f.db.transaction().unwrap();
|
||||
let revision =
|
||||
rmm_agent_core::jobs::transition(&tx, offer, &outcome.state, &grant.grant_id.to_string())
|
||||
.unwrap();
|
||||
let sequence: i64 = tx
|
||||
.query_row(
|
||||
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
let envelope = Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: f.binding.agent_id.to_string(),
|
||||
enrollment_generation: f.binding.enrollment_generation,
|
||||
connection_generation: f.channel.connection_generation,
|
||||
boot_id: f.channel.boot_id.to_string(),
|
||||
sequence: sequence.try_into().unwrap(),
|
||||
observed_at: f.clock.now.timestamp_millis(),
|
||||
payload: Some(Payload::JobResult(rmm_agent_core::jobs::result(
|
||||
offer,
|
||||
revision,
|
||||
&grant.grant_id.to_string(),
|
||||
outcome,
|
||||
))),
|
||||
};
|
||||
rmm_agent_core::jobs::event(&tx, &envelope).unwrap();
|
||||
tx.commit().unwrap();
|
||||
envelope
|
||||
}
|
||||
fn completion_outcome() -> rmm_agent_core::auth::Outcome {
|
||||
serde_json::from_slice(&canonical_outcome_bytes(&success()).unwrap()).unwrap()
|
||||
}
|
||||
#[test]
|
||||
fn crash_after_durable_outcome_before_result_recovers_original_then_signed_sync_closes() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
||||
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
||||
assert_eq!(interrupted.len(), 1);
|
||||
let original = rmm_agent_core::jobs::recover_interrupted_outcome(
|
||||
&f.ledger,
|
||||
&f.binding,
|
||||
&offer,
|
||||
&interrupted[0].1,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let envelope = persist_original_fixture_result(&mut f, &offer, &grant, original);
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let mut client = f.client();
|
||||
let prepared = f.prepare(&mut client);
|
||||
let decision = f.decision(&prepared, true);
|
||||
assert!(decision.attempts[0].unlock);
|
||||
assert_eq!(
|
||||
prepared.request.attempts[0].result_message_id,
|
||||
envelope.message_id
|
||||
);
|
||||
f.apply(&mut client, &decision).unwrap();
|
||||
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn crash_after_result_commit_before_finish_retains_exact_result_and_local_prior() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
||||
let original = persist_original_fixture_result(&mut f, &offer, &grant, completion_outcome());
|
||||
assert!(
|
||||
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
rmm_agent_core::jobs::events(&f.db).unwrap(),
|
||||
vec![original.clone()]
|
||||
);
|
||||
let mut client = f.client();
|
||||
let p = f.prepare(&mut client);
|
||||
let d = f.decision(&p, true);
|
||||
f.apply(&mut client, &d).unwrap();
|
||||
client.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
let archived: Vec<u8> =
|
||||
f.db.query_row(
|
||||
"SELECT envelope FROM reconcile_results WHERE id=?1",
|
||||
[&original.message_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(archived, original.encode_to_vec());
|
||||
}
|
||||
#[test]
|
||||
fn crash_before_running_event_recovers_real_consumption_without_inventing_completion() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
f.db.execute("UPDATE job_attempts SET state='requesting',grant_id=''", [])
|
||||
.unwrap();
|
||||
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
||||
assert_eq!(interrupted[0].1, grant.grant_id.to_string());
|
||||
assert!(
|
||||
rmm_agent_core::jobs::recover_interrupted_outcome(
|
||||
&f.ledger,
|
||||
&f.binding,
|
||||
&offer,
|
||||
&interrupted[0].1
|
||||
)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
persist_original_fixture_result(
|
||||
&mut f,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_agent_core::auth::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
);
|
||||
let mut c = f.client();
|
||||
let p = f.prepare(&mut c);
|
||||
let d = f.decision(&p, false);
|
||||
assert!(!d.attempts[0].unlock);
|
||||
f.apply(&mut c, &d).unwrap();
|
||||
assert!(!c.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let missing: Option<Vec<u8>> = f
|
||||
.ledger
|
||||
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert!(missing.is_none());
|
||||
}
|
||||
#[test]
|
||||
fn signed_unlock_still_rejects_a_missing_local_outcome_in_legacy_crash_window() {
|
||||
let mut f = Fixture::new();
|
||||
let (_, grant, _) = f.add("succeeded", true, Some(success()), script());
|
||||
let grant = grant.unwrap();
|
||||
f.ledger
|
||||
.execute("UPDATE execution_evidence SET outcome=NULL", [])
|
||||
.unwrap();
|
||||
let mut c = f.client();
|
||||
let p = f.prepare(&mut c);
|
||||
let d = f.decision(&p, true);
|
||||
assert!(d.attempts[0].unlock);
|
||||
f.apply(&mut c, &d).unwrap();
|
||||
assert!(c.apply_consumption(&mut f.ledger, &f.clock).is_err());
|
||||
assert!(!c.can_start(&f.channel, false, &f.clock));
|
||||
assert!(rmm_agent_core::ledger::consume(&mut f.ledger, &grant).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn original_completion_after_reconnect_unknown_is_recovered_without_reexecuting() {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, grant, _) = f.add("running", true, None, script());
|
||||
let grant = grant.unwrap();
|
||||
let mut previous = f.client();
|
||||
let previous_request = f.prepare(&mut previous);
|
||||
assert_eq!(previous_request.request.attempts[0].state, "unknown");
|
||||
assert!(
|
||||
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
// The already-running executor completes after the transport fenced it.
|
||||
rmm_agent_core::ledger::persist_outcome(&f.ledger, &grant, &completion_outcome()).unwrap();
|
||||
drop(previous);
|
||||
let interrupted = rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding).unwrap();
|
||||
assert_eq!(interrupted.len(), 1);
|
||||
let saved = rmm_agent_core::jobs::recover_interrupted_outcome(
|
||||
&f.ledger,
|
||||
&f.binding,
|
||||
&offer,
|
||||
&interrupted[0].1,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let original = persist_original_fixture_result(&mut f, &offer, &grant, saved);
|
||||
let mut process = f.client();
|
||||
let p = f.prepare(&mut process);
|
||||
let d = f.decision(&p, true);
|
||||
assert_eq!(p.request.attempts[0].result_message_id, original.message_id);
|
||||
f.apply(&mut process, &d).unwrap();
|
||||
process.apply_consumption(&mut f.ledger, &f.clock).unwrap();
|
||||
assert!(process.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row::<u32, _, _>("SELECT count(*) FROM consumed", [], |r| r.get(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
rmm_agent_core::jobs::interrupted(&f.db, &f.ledger, &f.binding)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ fn supported_as(job: &JobPayload, uid: u32, helper_configured: bool) -> bool {
|
||||
}
|
||||
pub use rmm_agent_core::jobs::{
|
||||
ack, event, events, interrupted, interrupted_request, next, no_start, permit_request,
|
||||
requesting, reserve_request, result, transition,
|
||||
recover_interrupted_outcome, requesting, reserve_request, result, transition,
|
||||
};
|
||||
pub fn accept(db: &Connection, binding: &Binding, offer: &JobOffer) -> Result<bool> {
|
||||
rmm_agent_core::jobs::accept(db, binding, offer, |job| {
|
||||
|
||||
@@ -212,10 +212,10 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
/// New process only. Persist honest uncertainty before any helper snapshot
|
||||
/// can block; the ordinary first-channel replay publishes it before Sync.
|
||||
fn recover_interrupted(&mut self) -> Result<()> {
|
||||
let Some(binding) = &self.binding else {
|
||||
let Some(binding) = self.binding.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
let interrupted = jobs::interrupted(&self.db, &self.ledger, binding)?;
|
||||
let interrupted = jobs::interrupted(&self.db, &self.ledger, &binding)?;
|
||||
for (offer, grant) in interrupted {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(&self.db, &offer)?;
|
||||
@@ -224,7 +224,10 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
0,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?
|
||||
.unwrap_or_else(|| {
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
|
||||
}),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
@@ -286,8 +289,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
jobs::supported_with_binding(&job, Some(&binding)),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
);
|
||||
rmm_privileged_helper::consume(&mut self.ledger, &grant)?;
|
||||
rmm_privileged_helper::save_start(&self.ledger, &grant, &request)?;
|
||||
rmm_privileged_helper::begin_execution(&mut self.ledger, &grant, &request)?;
|
||||
let cancelled: bool = self.db.query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
@@ -295,6 +297,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
)?;
|
||||
if cancelled {
|
||||
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
||||
rmm_privileged_helper::persist_outcome(&self.ledger, &grant, &outcome)?;
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
@@ -546,10 +549,11 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
}
|
||||
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
|
||||
// Persist the wire result before releasing a consumption lock.
|
||||
// A crash in between is fail-closed, not a second execution.
|
||||
// First preserve the original completion without unlocking.
|
||||
// A startup can replay it if the journal transaction is lost.
|
||||
let state = outcome.state.clone();
|
||||
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
|
||||
rmm_privileged_helper::persist_outcome(&self.ledger,&grant,&outcome)?;
|
||||
let serialized = serde_json::to_vec(&outcome)?;
|
||||
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
|
||||
let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?;
|
||||
|
||||
@@ -78,7 +78,7 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
f.try_lock_exclusive()?;
|
||||
Ok(f)
|
||||
}
|
||||
pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish};
|
||||
pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish, persist_outcome};
|
||||
|
||||
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
@@ -98,6 +98,16 @@ pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
fs::File::open(dir)?.sync_all()?;
|
||||
Ok(db)
|
||||
}
|
||||
/// Atomically reserve and persist the exact original signed request before effect.
|
||||
pub fn begin_execution(
|
||||
db: &mut rusqlite::Connection,
|
||||
permit: &ExecutionPermitPayload,
|
||||
request: &Request,
|
||||
) -> Result<()> {
|
||||
let recorded_at = Utc::now();
|
||||
let boot = Uuid::parse_str(&rmm_agent_linux::boot_id()?)?;
|
||||
rmm_agent_core::ledger::begin_execution(db, permit, request, boot, recorded_at)
|
||||
}
|
||||
/// Persist the same exact request/boot/time evidence through the shared ledger.
|
||||
pub fn save_start(
|
||||
db: &rusqlite::Connection,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
use anyhow::{Result, ensure};
|
||||
use rmm_domain::jobs::Action;
|
||||
use rmm_privileged_helper::{
|
||||
Binding, Outcome, Request, consume, control, finish, lock_state, open_ledger, private_path,
|
||||
reconcile,
|
||||
Binding, Outcome, Request, control, finish, lock_state, open_ledger, private_path, reconcile,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::{
|
||||
@@ -442,8 +441,7 @@ fn handle(
|
||||
if let Some(apt) = &apt {
|
||||
apt.verify_retry_history(db)?;
|
||||
}
|
||||
consume(db, &permit)?;
|
||||
rmm_privileged_helper::save_start(db, &permit, &request)?;
|
||||
rmm_privileged_helper::begin_execution(db, &permit, &request)?;
|
||||
let mut outcome =
|
||||
if !clock.trusted() || config.binding.verify(&request, chrono::Utc::now()).is_err() {
|
||||
Outcome {
|
||||
|
||||
@@ -237,7 +237,7 @@ fn reopen_consumption_duplicate_grant_attempt_unknown_and_resource_lock() {
|
||||
let lock = lock_state(&path, uid).unwrap();
|
||||
assert!(lock_state(&path, uid).is_err());
|
||||
let mut db = open_ledger(&path, uid).unwrap();
|
||||
consume(&mut db, &permit).unwrap();
|
||||
begin_execution(&mut db, &permit, &request).unwrap();
|
||||
drop(db);
|
||||
let mut db = open_ledger(&path, uid).unwrap();
|
||||
assert!(consume(&mut db, &permit).is_err());
|
||||
@@ -248,17 +248,26 @@ fn reopen_consumption_duplicate_grant_attempt_unknown_and_resource_lock() {
|
||||
replacement.job_id = Uuid::new_v4();
|
||||
replacement.attempt_id = Uuid::new_v4();
|
||||
assert!(consume(&mut db, &replacement).is_err());
|
||||
// Only definitive evidence releases the resource, not an ACK or new grant.
|
||||
// A contradictory completion cannot overwrite original unknown evidence.
|
||||
let mut definitive = Outcome::unknown("");
|
||||
definitive.state = "succeeded".into();
|
||||
finish(&db, &permit, &definitive).unwrap();
|
||||
assert!(finish(&db, &permit, &definitive).is_err());
|
||||
assert!(consume(&mut db, &permit).is_err());
|
||||
replacement.job_id = permit.job_id;
|
||||
replacement.attempt_id = permit.attempt_id;
|
||||
assert!(consume(&mut db, &replacement).is_err());
|
||||
replacement.job_id = Uuid::new_v4();
|
||||
replacement.attempt_id = Uuid::new_v4();
|
||||
consume(&mut db, &replacement).unwrap();
|
||||
assert!(consume(&mut db, &replacement).is_err());
|
||||
assert_eq!(
|
||||
db.query_row::<u32, _, _>(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
drop(db);
|
||||
drop(lock);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
@@ -327,7 +336,7 @@ fn two_helper_connections_share_exact_lock_and_sqlite_consumption() {
|
||||
);
|
||||
let mut general = open_ledger(&path, uid).unwrap();
|
||||
let mut apt = open_ledger(&path, uid).unwrap();
|
||||
consume(&mut general, &permit).unwrap();
|
||||
begin_execution(&mut general, &permit, &request).unwrap();
|
||||
assert!(consume(&mut apt, &permit).is_err());
|
||||
let mut other = permit.clone();
|
||||
other.grant_id = Uuid::new_v4();
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
# Native gemeinsame Core-Unitabnahme
|
||||
|
||||
**Persistenzprofil tatsächlich ausgeführt, 2026-11-14:**
|
||||
Fall `08a9be72-dd7c-46b7-a2c6-7d17424533c2`, Linux76 vor Windows54, ohne Skip.
|
||||
Getrennte eigene SQLite-/Signaturfixturen, keine native Job-/Dienstabnahme.
|
||||
|
||||
**Helper-Sync-Profil tatsächlich ausgeführt, 2026-11-14:**
|
||||
Fall `5704073c-a135-431c-aa84-fedc68a3de0b`, Linux 69/69 vor Windows 47/47,
|
||||
keine Skips oder RMM-Wirkung. Auch dies ersetzt keine Windows-SYSTEM-Dienstprobe.
|
||||
@@ -72,3 +76,25 @@ python3 deploy/shared_core_acceptance.py finish-windows --profile helper-sync-v1
|
||||
|
||||
Profil, Quellen und Artefakte sind an den privaten create-only Intent gebunden.
|
||||
Es gibt weiterhin keinen frei wählbaren Programmpfad oder Argumentvektor.
|
||||
|
||||
## Persistenzkorrektur `durable-outcome-v1`
|
||||
|
||||
Neue unveränderliche Qualifikation unter `devstate/*core/durable-outcome-v1/`.
|
||||
Das feste Profil erwartet Linux **7+26+22+21=76**, danach Windows **6+26+22=54**.
|
||||
Lokale Regressionen und Cross-Builds sind im Manifest separat von der noch
|
||||
geplanten tatsächlichen VM-Ausführung ausgewiesen. Kein früherer Nachweis wird
|
||||
umgeschrieben. Vorläufige verworfene Builds sind ausdrücklich unqualifiziert.
|
||||
|
||||
```sh
|
||||
# GEPLANT, NICHT AUSGEFÜHRT: tatsächliche VM-Ausführung durch Parent.
|
||||
python3 deploy/shared_core_acceptance.py linux --profile durable-outcome-v1
|
||||
python3 deploy/shared_core_acceptance.py windows --profile durable-outcome-v1 --case-id <UUID>
|
||||
```
|
||||
|
||||
Die gezielten Fälle prüfen atomaren Verbrauch+Startzeugen, Original-Outcome-CAS,
|
||||
Crash vor/nach Resultcommit, Reconnect-Unknown mit späterer Originalcompletion
|
||||
und die weiterhin verweigerte Freigabe bei fehlendem lokalen Outcome.
|
||||
Startup journalisiert ausschließlich bereits gebundene gespeicherte Belege neu;
|
||||
Unknown ohne solchen Beleg bleibt gesperrt. Server-/Unlockregeln bleiben strikt.
|
||||
Dies simuliert Commitgrenzen mit eigenen Fixtures; es ersetzt keinen späteren
|
||||
wirklichen Agentprozessabsturz oder unabhängigen SYSTEM-Helperabschlussnachweis.
|
||||
|
||||
@@ -26,6 +26,7 @@ STATE = ROOT / 'devstate/shared-core/acceptance'
|
||||
PROFILES = {
|
||||
'original': {'linux': {'contracts':5, 'reconcile':20}, 'windows': {'contracts':4, 'reconcile':20}},
|
||||
'helper-sync-v1': {'linux': {'contracts':5, 'reconcile':21, 'helper-reconcile':22, 'linux-helper':21}, 'windows': {'contracts':4, 'reconcile':21, 'helper-reconcile':22}},
|
||||
'durable-outcome-v1': {'linux': {'contracts':7, 'reconcile':26, 'helper-reconcile':22, 'linux-helper':21}, 'windows': {'contracts':6, 'reconcile':26, 'helper-reconcile':22}},
|
||||
}
|
||||
KINDS = ('contracts', 'reconcile') # Historical original qualification only.
|
||||
REMOTE_LINUX = '/var/lib/rmm-shared-core-acceptance'
|
||||
@@ -44,11 +45,11 @@ def source_paths(profile):
|
||||
contract.require(profile in PROFILES, 'fixed qualification profile')
|
||||
sources = [ROOT/'Cargo.toml', ROOT/'Cargo.lock', ROOT/'rust-toolchain.toml']
|
||||
crates = ['agent-core', 'domain', 'protocol']
|
||||
if profile == 'helper-sync-v1': crates.extend(('agent', 'agent-linux', 'privileged-helper'))
|
||||
if profile in ('helper-sync-v1','durable-outcome-v1'): crates.extend(('agent', 'agent-linux', 'privileged-helper'))
|
||||
for name in crates:
|
||||
sources.extend(p for p in (ROOT/'crates'/name).rglob('*') if p.is_file() and (p.suffix == '.rs' or p.name == 'Cargo.toml'))
|
||||
sources.extend((ROOT/'proto').glob('*.proto'))
|
||||
if profile == 'helper-sync-v1':
|
||||
if profile in ('helper-sync-v1','durable-outcome-v1'):
|
||||
sources.extend(ROOT/p for p in ('tools/apt-transaction/apt_transaction.py', 'tools/apt-transaction/hook', 'deploy/linux/rmm-apt-helper.service', 'tests/apt_transaction/fixtures/planning-canonical.json'))
|
||||
return sources
|
||||
|
||||
@@ -83,7 +84,7 @@ def linux_case(payload):
|
||||
require(ident == payload['id'] and payload['action'] in ('prepare', 'run', 'read'), 'fixed case')
|
||||
entries = payload['artifacts']
|
||||
profile=payload.get('profile','original')
|
||||
profiles={'original':{'contracts':5,'reconcile':20},'helper-sync-v1':{'contracts':5,'reconcile':21,'helper-reconcile':22,'linux-helper':21}}
|
||||
profiles={'original':{'contracts':5,'reconcile':20},'helper-sync-v1':{'contracts':5,'reconcile':21,'helper-reconcile':22,'linux-helper':21},'durable-outcome-v1':{'contracts':7,'reconcile':26,'helper-reconcile':22,'linux-helper':21}}
|
||||
require(profile in profiles and set(entries)==set(profiles[profile]), 'fixed qualification artifacts')
|
||||
for kind, item in entries.items():
|
||||
require(len(item['binary_sha256']) == 64 and all(c in '0123456789abcdef' for c in item['binary_sha256']), 'fixed hash')
|
||||
@@ -141,8 +142,8 @@ WINDOWS = r'''
|
||||
$ErrorActionPreference='Stop'
|
||||
$p=([Console]::In.ReadLine() | ConvertFrom-Json)
|
||||
if (!$p.profile) {$p | Add-Member -NotePropertyName profile -NotePropertyValue 'original'}
|
||||
$kinds=@('contracts','reconcile');if ($p.profile -eq 'helper-sync-v1') {$kinds+=@('helper-reconcile')}
|
||||
if ($p.profile -notin @('original','helper-sync-v1')) {throw 'Fixed qualification profile'}
|
||||
$kinds=@('contracts','reconcile');if ($p.profile -in @('helper-sync-v1','durable-outcome-v1')) {$kinds+=@('helper-reconcile')}
|
||||
if ($p.profile -notin @('original','helper-sync-v1','durable-outcome-v1')) {throw 'Fixed qualification profile'}
|
||||
if ([string]([guid]$p.id) -cne $p.id -or $p.kind -notin $kinds -or $p.action -notin @('run','read') -or $p.sha256 -notmatch '^[0-9a-f]{64}$' -or $p.linux_evidence_sha256 -notmatch '^[0-9a-f]{64}$') {throw 'Fixed native core case'}
|
||||
$root='C:\ProgramData\RustDiscoveryBootstrap'
|
||||
$case=Join-Path $root ('shared-core-'+$p.id)
|
||||
|
||||
@@ -76,6 +76,13 @@ Rootpolicy und deaktivierte Faults erhalten. Passiver Windowsclient hat nach
|
||||
Hello ein Legacy-Readyflag, weiterhin keinen signierten Sync-/Executornachweis.
|
||||
Windows-PowerShell-Plan ohne Fähigkeiten tatsächlich verweigert.
|
||||
|
||||
Gemeinsamer Persistenzfix anschließend nativ qualifiziert: atomarer Start,
|
||||
Outcome-CAS vor Result und Startup-Recovery nur aus vollständig gebundenen
|
||||
Originalbelegen. Fall `08a9be72-dd7c-46b7-a2c6-7d17424533c2`, zuerst Linux76,
|
||||
danach Windows54, keine Skips oder RMM-Wirkung. Fehlender Outcome bleibt Unknown;
|
||||
keine abgeschwächte signierte Freigabe. Die noch laufenden Dienste sind dadurch
|
||||
nicht automatisch als aktualisiert nachgewiesen.
|
||||
|
||||
Der bestehende Windows-11-LocalService bleibt passiv: gleiche Geräteidentität,
|
||||
Telemetriehistorie und Schlüssel; keine Jobs, Outbox oder Startgrants. Native
|
||||
WSS-/Inventar-/Tenantnachweise und sieben Rusttests liegen vor. Explizites HTTPS
|
||||
|
||||
@@ -1,5 +1,26 @@
|
||||
# Tatsächlicher Implementierungsstand
|
||||
|
||||
## Dauerhafte Outcomes und atomarer Start nativ geprüft — 2026-11-14
|
||||
|
||||
Beim Portieren wurde eine bestehende Verfügbarkeitslücke gefunden: Ein Crash
|
||||
zwischen dauerhaftem JobResult und Ledgerabschluss konnte den Verbrauch ohne
|
||||
lokalen Outcome gesperrt lassen. Jetzt werden Verbrauch und Startzeuge atomar
|
||||
gespeichert; der gebundene Outcome wird ohne Unlock vor der Result-Transaktion
|
||||
persistiert. Erst danach darf der Verbrauch abgeschlossen werden. Startup
|
||||
rekonstruiert ausschließlich vollständig historisch verifizierte Original-
|
||||
Outcomes; ohne solchen Beleg bleibt `unknown` gesperrt. Konflikte überschreiben
|
||||
keine vorhandenen Ergebnisse, und Server-/Sync-Unlockprüfungen bleiben streng.
|
||||
|
||||
Lokal 181 betroffene Tests sowie Linux-/Windows-Clippy bestanden. Im eigenen
|
||||
Fall `08a9be72-dd7c-46b7-a2c6-7d17424533c2` danach zuerst 76 Linux- und
|
||||
anschließend 54 Windows-Fixturen nativ ohne Skip bestanden. Geprüft wurden
|
||||
beide Crashfenster, ein noch nicht journalisierter Grant, später gespeicherter
|
||||
Outcome nach Unknown sowie verweigerte Manipulationen und atomarer Rollback.
|
||||
Eigene Testdaten, kein tatsächlicher Jobabbruch oder neuer RMM-Job.
|
||||
Private Profile/Artefakte: `devstate/{shared-core,windows-core}/durable-outcome-v1/`;
|
||||
Rohbelege unter `devstate/shared-core/acceptance/<Fall>/`. Native Dienstintegration
|
||||
und Windows-Ausführung bleiben separat offen.
|
||||
|
||||
## Windows-Plattformverträge auf endgültigem Server bereitgestellt — 2026-11-14
|
||||
|
||||
Servercandidate `9058f6a3-0832-400f-ba93-b3fb94faaba0` wurde ausschließlich aus
|
||||
|
||||
Reference in New Issue
Block a user