Reconcile a human closure when the helper has no result
Rust development slice / linux (push) Failing after 9s

This commit is contained in:
2026-11-18 19:03:02 +01:00
parent abad2022e2
commit 915e126f24
5 changed files with 362 additions and 29 deletions
+39 -15
View File
@@ -782,10 +782,10 @@ impl<V: Ed25519Verifier> HelperSync<V> {
let d = decisions
.get(&id)
.context("ROOT_ATTEMPT_OMITTED_FROM_DECISION")?;
validate_summary(row, local)?;
validate_decision(row, local, d)?;
let unreported = validate_summary(row, local)?;
validate_decision(row, local, d, unreported)?;
if d.unlock {
let bytes = unlock_outcome(row, d, current_boot)?;
let bytes = unlock_outcome(row, d, current_boot, unreported.then_some(local))?;
if d.outcome_state == "closed_unknown" {
crate::ledger::retain_closure_original(
&tx,
@@ -1636,22 +1636,33 @@ fn definitive(row: &LocalAttempt) -> bool {
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
) && !row.outcome_bytes.is_empty()
}
fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<()> {
fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<bool> {
let unreported = row.state == "unknown"
&& row.outcome_sha256.is_empty()
&& row.outcome_bytes.is_empty()
&& row.update_handoff_bytes.is_empty()
&& crate::reconcile::uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
ensure!(
a.job_payload_hash == row.job_payload_hash
&& a.grant_id == row.grant_id.to_string()
&& a.consumption == "consumed"
&& a.state == row.state
&& a.revision >= row.revision
&& a.original_boot_id == row.original_boot_id.to_string()
&& a.outcome_sha256 == row.outcome_sha256
&& a.update_handoff_bytes == row.update_handoff_bytes
&& (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes),
&& (unreported
|| (a.state == row.state
&& a.outcome_sha256 == row.outcome_sha256
&& (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes))),
"ROOT_SNAPSHOT_REQUEST_MISMATCH"
);
Ok(())
Ok(unreported)
}
fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -> Result<()> {
fn validate_decision(
row: &LocalAttempt,
a: &AttemptView,
d: &AttemptDecision,
unreported: bool,
) -> Result<()> {
if !row.update_handoff_bytes.is_empty() {
let proof =
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&row.update_handoff_bytes)
@@ -1686,7 +1697,7 @@ fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -
&& d.original_boot_id == Some(row.original_boot_id)
&& (d.revision >= a.revision || (!d.unlock && d.outcome_state == "unknown"))
&& d.previous_outcome_sha256.as_deref()
== (!row.outcome_sha256.is_empty()).then_some(row.outcome_sha256.as_str()),
== (!a.outcome_sha256.is_empty()).then_some(a.outcome_sha256.as_str()),
"ROOT_SIGNED_ATTEMPT_BINDING"
);
ensure!(
@@ -1697,11 +1708,26 @@ fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -
),
"UNKNOWN_CANNOT_UNLOCK"
);
ensure!(
!unreported
|| (!d.unlock && d.outcome_state == "unknown")
|| (d.unlock
&& d.outcome_state == "closed_unknown"
&& d.reason == "OPERATOR_CLOSED_UNKNOWN"),
"ROOT_UNREPORTED_OUTCOME_CLOSURE_REQUIRED"
);
Ok(())
}
fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -> Result<Vec<u8>> {
fn unlock_outcome(
row: &LocalAttempt,
d: &AttemptDecision,
current_boot: Uuid,
unreported: Option<&AttemptView>,
) -> Result<Vec<u8>> {
let bytes = if d.outcome_bytes.is_empty() {
row.outcome_bytes.clone()
unreported
.map_or(&row.outcome_bytes, |a| &a.outcome_bytes)
.clone()
} else {
d.outcome_bytes.clone()
};
@@ -1712,9 +1738,7 @@ fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -
&& o.state == d.outcome_state,
"ROOT_CANONICAL_UNLOCK_OUTCOME"
);
if bytes != row.outcome_bytes
&& crate::reconcile::operator_closure(&d.reason, &row.state, &o)
{
if bytes != row.outcome_bytes && crate::reconcile::operator_closure(&d.reason, &row.state, &o) {
return Ok(bytes);
}
if bytes != row.outcome_bytes {
+120 -13
View File
@@ -1019,9 +1019,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
} else {
for d in decision.attempts.iter().filter(|d| d.unlock) {
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
let (agent,generation,job,attempt,previous):(String,u64,String,String,Option<Vec<u8>>)=tx.query_row(
"SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1",
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?;
let (agent,generation,job,attempt,previous,start):(String,u64,String,String,Option<Vec<u8>>,Option<Vec<u8>>)=tx.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,COALESCE(e.outcome,c.result),e.start FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?;
ensure!(
agent == self.binding.agent_id.to_string()
&& generation == self.binding.enrollment_generation
@@ -1034,13 +1034,61 @@ impl<V: Ed25519Verifier> ClientSync<V> {
Some(&self.binding),
grant,
)?;
let previous = resolved
.or(previous)
.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
let original = previous.clone();
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&previous,
)?)?;
let previous = resolved.or(previous);
let summary = gate
.request
.attempts
.iter()
.find(|a| a.job_id == job && a.attempt_id == attempt)
.context("LOCAL_UNLOCK_ATTEMPT_MISSING")?;
let closure = d.reason == "OPERATOR_CLOSED_UNKNOWN"
&& d.outcome_state == "closed_unknown"
&& summary.consumption == "consumed"
&& uncertain_agent_outcome(
&summary.state,
&summary.outcome_bytes,
&summary.outcome_sha256,
);
let prior = match &previous {
Some(bytes) => {
let canonical = canonical_outcome_bytes(&serde_json::from_slice(bytes)?)?;
if closure
&& summary.state == "closed_unknown"
&& uncertain_agent_outcome("unknown", &canonical, &hash(&canonical))
{
summary.outcome_bytes.clone()
} else {
canonical
}
}
None => {
ensure!(closure, "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING");
let start: serde_json::Value = serde_json::from_slice(
&start.context("LOCAL_UNLOCK_ORIGINAL_START_MISSING")?,
)?;
let request: crate::auth::Request =
serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
let (original, permit) = self.binding.verify_with(
&request,
permit.not_before,
&self.verifier,
)?;
ensure!(
permit.grant_id == grant
&& original.job_id == d.job_id
&& original.attempt_id == d.attempt_id
&& hash(&request.job_bytes) == d.job_payload_hash
&& Some(permit.authorization_id) == d.authorization_id
&& Some(permit.plan_hash.as_str()) == d.plan_hash.as_deref()
&& start["previous_boot_id"].as_str()
== d.original_boot_id.map(|id| id.to_string()).as_deref(),
"LOCAL_UNLOCK_ORIGINAL_START_BINDING"
);
summary.outcome_bytes.clone()
}
};
ensure!(
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
@@ -1055,7 +1103,8 @@ impl<V: Ed25519Verifier> ClientSync<V> {
terminal(&outcome.state)
&& outcome.state == d.outcome_state
&& canonical_outcome_bytes(&outcome)? == bytes
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()),
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
&& (previous.is_some() || operator_closure(&d.reason, "unknown", &outcome)),
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
);
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
@@ -1065,7 +1114,12 @@ impl<V: Ed25519Verifier> ClientSync<V> {
&bytes,
)?;
if outcome.state == "closed_unknown" {
crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?;
crate::ledger::retain_closure_original(
&tx,
grant,
previous.as_deref().unwrap_or_default(),
&bytes,
)?;
}
tx.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
@@ -1304,6 +1358,32 @@ pub(crate) fn operator_closure(
&& outcome.evidence["closed_by_human"] == true
&& outcome.evidence["retry"] == false
}
/// An interrupted agent can record its own uncertainty while the independently
/// consumed helper grant has no result. Neither witness establishes an effect.
pub(crate) fn uncertain_agent_outcome(state: &str, bytes: &[u8], digest: &str) -> bool {
if bytes.is_empty() || hash(bytes) != digest {
return false;
}
let Ok(outcome) = serde_json::from_slice::<ReconciledOutcome>(bytes) else {
return false;
};
canonical_outcome_bytes(&outcome).is_ok_and(|canonical| canonical == bytes)
&& outcome.state == state
&& !outcome.output_truncated
&& !outcome.reboot_required
&& (operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &outcome)
|| (state == "unknown"
&& matches!(
outcome.error_code.as_str(),
"AGENT_PROCESS_INTERRUPTED" | "HELPER_TRANSPORT_EFFECT_UNKNOWN"
)
&& outcome.exit_code.is_none()
&& outcome.stdout.is_empty()
&& outcome.stderr.is_empty()
&& outcome.evidence["postconditions_passed"] == false))
}
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
if s.is_empty() {
Ok(None)
@@ -1452,6 +1532,24 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
result_envelope = Some(envelope);
}
}
// The signed closure projection can commit before either consumption
// ledger. A fresh signed check may then finish the remaining writes.
if state == "closed_unknown"
&& consumption == "consumed"
&& (canonical.is_empty()
|| uncertain_agent_outcome("unknown", &canonical, &hash(&canonical)))
{
let closed: Vec<u8> = db.query_row(
"SELECT outcome FROM reconcile_outcomes WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
ensure!(
uncertain_agent_outcome(&state, &closed, &hash(&closed)),
"LOCAL_CLOSURE_OUTCOME_MISMATCH"
);
canonical = closed;
}
if !canonical.is_empty() {
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
@@ -1746,6 +1844,15 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
"HELPER_ATTEMPT_BINDING"
);
if h.consumption != "unconsumed" {
let unreported = h.state == "unknown"
&& h.consumption == "consumed"
&& h.outcome_bytes.is_empty()
&& h.outcome_sha256.is_empty()
&& h.update_handoff_bytes.is_empty()
&& a.consumption == "consumed"
&& a.grant_id == h.grant_id
&& a.original_boot_id == h.original_boot_id
&& uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
if !h.update_handoff_bytes.is_empty() {
ensure!(
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
@@ -1756,7 +1863,7 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
a.grant_id = h.grant_id.clone();
a.consumption = h.consumption.clone();
ensure!(
a.outcome_sha256 == h.outcome_sha256,
a.outcome_sha256 == h.outcome_sha256 || unreported,
"HELPER_OUTCOME_MISMATCH"
);
if a.original_boot_id.is_empty() {