Reconcile a human closure when the helper has no result
Rust development slice / linux (push) Failing after 9s

This commit is contained in:
2026-11-18 19:03:02 +01:00
parent abad2022e2
commit 915e126f24
5 changed files with 362 additions and 29 deletions
+120 -13
View File
@@ -1019,9 +1019,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
} else {
for d in decision.attempts.iter().filter(|d| d.unlock) {
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
let (agent,generation,job,attempt,previous):(String,u64,String,String,Option<Vec<u8>>)=tx.query_row(
"SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1",
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?;
let (agent,generation,job,attempt,previous,start):(String,u64,String,String,Option<Vec<u8>>,Option<Vec<u8>>)=tx.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,COALESCE(e.outcome,c.result),e.start FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?;
ensure!(
agent == self.binding.agent_id.to_string()
&& generation == self.binding.enrollment_generation
@@ -1034,13 +1034,61 @@ impl<V: Ed25519Verifier> ClientSync<V> {
Some(&self.binding),
grant,
)?;
let previous = resolved
.or(previous)
.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
let original = previous.clone();
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&previous,
)?)?;
let previous = resolved.or(previous);
let summary = gate
.request
.attempts
.iter()
.find(|a| a.job_id == job && a.attempt_id == attempt)
.context("LOCAL_UNLOCK_ATTEMPT_MISSING")?;
let closure = d.reason == "OPERATOR_CLOSED_UNKNOWN"
&& d.outcome_state == "closed_unknown"
&& summary.consumption == "consumed"
&& uncertain_agent_outcome(
&summary.state,
&summary.outcome_bytes,
&summary.outcome_sha256,
);
let prior = match &previous {
Some(bytes) => {
let canonical = canonical_outcome_bytes(&serde_json::from_slice(bytes)?)?;
if closure
&& summary.state == "closed_unknown"
&& uncertain_agent_outcome("unknown", &canonical, &hash(&canonical))
{
summary.outcome_bytes.clone()
} else {
canonical
}
}
None => {
ensure!(closure, "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING");
let start: serde_json::Value = serde_json::from_slice(
&start.context("LOCAL_UNLOCK_ORIGINAL_START_MISSING")?,
)?;
let request: crate::auth::Request =
serde_json::from_value(start["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
let (original, permit) = self.binding.verify_with(
&request,
permit.not_before,
&self.verifier,
)?;
ensure!(
permit.grant_id == grant
&& original.job_id == d.job_id
&& original.attempt_id == d.attempt_id
&& hash(&request.job_bytes) == d.job_payload_hash
&& Some(permit.authorization_id) == d.authorization_id
&& Some(permit.plan_hash.as_str()) == d.plan_hash.as_deref()
&& start["previous_boot_id"].as_str()
== d.original_boot_id.map(|id| id.to_string()).as_deref(),
"LOCAL_UNLOCK_ORIGINAL_START_BINDING"
);
summary.outcome_bytes.clone()
}
};
ensure!(
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
@@ -1055,7 +1103,8 @@ impl<V: Ed25519Verifier> ClientSync<V> {
terminal(&outcome.state)
&& outcome.state == d.outcome_state
&& canonical_outcome_bytes(&outcome)? == bytes
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()),
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
&& (previous.is_some() || operator_closure(&d.reason, "unknown", &outcome)),
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
);
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
@@ -1065,7 +1114,12 @@ impl<V: Ed25519Verifier> ClientSync<V> {
&bytes,
)?;
if outcome.state == "closed_unknown" {
crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?;
crate::ledger::retain_closure_original(
&tx,
grant,
previous.as_deref().unwrap_or_default(),
&bytes,
)?;
}
tx.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
@@ -1304,6 +1358,32 @@ pub(crate) fn operator_closure(
&& outcome.evidence["closed_by_human"] == true
&& outcome.evidence["retry"] == false
}
/// An interrupted agent can record its own uncertainty while the independently
/// consumed helper grant has no result. Neither witness establishes an effect.
pub(crate) fn uncertain_agent_outcome(state: &str, bytes: &[u8], digest: &str) -> bool {
if bytes.is_empty() || hash(bytes) != digest {
return false;
}
let Ok(outcome) = serde_json::from_slice::<ReconciledOutcome>(bytes) else {
return false;
};
canonical_outcome_bytes(&outcome).is_ok_and(|canonical| canonical == bytes)
&& outcome.state == state
&& !outcome.output_truncated
&& !outcome.reboot_required
&& (operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &outcome)
|| (state == "unknown"
&& matches!(
outcome.error_code.as_str(),
"AGENT_PROCESS_INTERRUPTED" | "HELPER_TRANSPORT_EFFECT_UNKNOWN"
)
&& outcome.exit_code.is_none()
&& outcome.stdout.is_empty()
&& outcome.stderr.is_empty()
&& outcome.evidence["postconditions_passed"] == false))
}
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
if s.is_empty() {
Ok(None)
@@ -1452,6 +1532,24 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
result_envelope = Some(envelope);
}
}
// The signed closure projection can commit before either consumption
// ledger. A fresh signed check may then finish the remaining writes.
if state == "closed_unknown"
&& consumption == "consumed"
&& (canonical.is_empty()
|| uncertain_agent_outcome("unknown", &canonical, &hash(&canonical)))
{
let closed: Vec<u8> = db.query_row(
"SELECT outcome FROM reconcile_outcomes WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
ensure!(
uncertain_agent_outcome(&state, &closed, &hash(&closed)),
"LOCAL_CLOSURE_OUTCOME_MISMATCH"
);
canonical = closed;
}
if !canonical.is_empty() {
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
@@ -1746,6 +1844,15 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
"HELPER_ATTEMPT_BINDING"
);
if h.consumption != "unconsumed" {
let unreported = h.state == "unknown"
&& h.consumption == "consumed"
&& h.outcome_bytes.is_empty()
&& h.outcome_sha256.is_empty()
&& h.update_handoff_bytes.is_empty()
&& a.consumption == "consumed"
&& a.grant_id == h.grant_id
&& a.original_boot_id == h.original_boot_id
&& uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
if !h.update_handoff_bytes.is_empty() {
ensure!(
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
@@ -1756,7 +1863,7 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
a.grant_id = h.grant_id.clone();
a.consumption = h.consumption.clone();
ensure!(
a.outcome_sha256 == h.outcome_sha256,
a.outcome_sha256 == h.outcome_sha256 || unreported,
"HELPER_OUTCOME_MISMATCH"
);
if a.original_boot_id.is_empty() {