Reconcile a human closure when the helper has no result
Rust development slice / linux (push) Failing after 9s
Rust development slice / linux (push) Failing after 9s
This commit is contained in:
@@ -1019,9 +1019,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
} else {
|
||||
for d in decision.attempts.iter().filter(|d| d.unlock) {
|
||||
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
|
||||
let (agent,generation,job,attempt,previous):(String,u64,String,String,Option<Vec<u8>>)=tx.query_row(
|
||||
"SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1",
|
||||
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?;
|
||||
let (agent,generation,job,attempt,previous,start):(String,u64,String,String,Option<Vec<u8>>,Option<Vec<u8>>)=tx.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,COALESCE(e.outcome,c.result),e.start FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?;
|
||||
ensure!(
|
||||
agent == self.binding.agent_id.to_string()
|
||||
&& generation == self.binding.enrollment_generation
|
||||
@@ -1034,13 +1034,61 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
Some(&self.binding),
|
||||
grant,
|
||||
)?;
|
||||
let previous = resolved
|
||||
.or(previous)
|
||||
.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
|
||||
let original = previous.clone();
|
||||
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
|
||||
&previous,
|
||||
)?)?;
|
||||
let previous = resolved.or(previous);
|
||||
let summary = gate
|
||||
.request
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|a| a.job_id == job && a.attempt_id == attempt)
|
||||
.context("LOCAL_UNLOCK_ATTEMPT_MISSING")?;
|
||||
let closure = d.reason == "OPERATOR_CLOSED_UNKNOWN"
|
||||
&& d.outcome_state == "closed_unknown"
|
||||
&& summary.consumption == "consumed"
|
||||
&& uncertain_agent_outcome(
|
||||
&summary.state,
|
||||
&summary.outcome_bytes,
|
||||
&summary.outcome_sha256,
|
||||
);
|
||||
let prior = match &previous {
|
||||
Some(bytes) => {
|
||||
let canonical = canonical_outcome_bytes(&serde_json::from_slice(bytes)?)?;
|
||||
if closure
|
||||
&& summary.state == "closed_unknown"
|
||||
&& uncertain_agent_outcome("unknown", &canonical, &hash(&canonical))
|
||||
{
|
||||
summary.outcome_bytes.clone()
|
||||
} else {
|
||||
canonical
|
||||
}
|
||||
}
|
||||
None => {
|
||||
ensure!(closure, "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING");
|
||||
let start: serde_json::Value = serde_json::from_slice(
|
||||
&start.context("LOCAL_UNLOCK_ORIGINAL_START_MISSING")?,
|
||||
)?;
|
||||
let request: crate::auth::Request =
|
||||
serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
let (original, permit) = self.binding.verify_with(
|
||||
&request,
|
||||
permit.not_before,
|
||||
&self.verifier,
|
||||
)?;
|
||||
ensure!(
|
||||
permit.grant_id == grant
|
||||
&& original.job_id == d.job_id
|
||||
&& original.attempt_id == d.attempt_id
|
||||
&& hash(&request.job_bytes) == d.job_payload_hash
|
||||
&& Some(permit.authorization_id) == d.authorization_id
|
||||
&& Some(permit.plan_hash.as_str()) == d.plan_hash.as_deref()
|
||||
&& start["previous_boot_id"].as_str()
|
||||
== d.original_boot_id.map(|id| id.to_string()).as_deref(),
|
||||
"LOCAL_UNLOCK_ORIGINAL_START_BINDING"
|
||||
);
|
||||
summary.outcome_bytes.clone()
|
||||
}
|
||||
};
|
||||
ensure!(
|
||||
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
|
||||
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
|
||||
@@ -1055,7 +1103,8 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
terminal(&outcome.state)
|
||||
&& outcome.state == d.outcome_state
|
||||
&& canonical_outcome_bytes(&outcome)? == bytes
|
||||
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()),
|
||||
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
|
||||
&& (previous.is_some() || operator_closure(&d.reason, "unknown", &outcome)),
|
||||
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
|
||||
);
|
||||
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
|
||||
@@ -1065,7 +1114,12 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
&bytes,
|
||||
)?;
|
||||
if outcome.state == "closed_unknown" {
|
||||
crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?;
|
||||
crate::ledger::retain_closure_original(
|
||||
&tx,
|
||||
grant,
|
||||
previous.as_deref().unwrap_or_default(),
|
||||
&bytes,
|
||||
)?;
|
||||
}
|
||||
tx.execute(
|
||||
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
|
||||
@@ -1304,6 +1358,32 @@ pub(crate) fn operator_closure(
|
||||
&& outcome.evidence["closed_by_human"] == true
|
||||
&& outcome.evidence["retry"] == false
|
||||
}
|
||||
|
||||
/// An interrupted agent can record its own uncertainty while the independently
|
||||
/// consumed helper grant has no result. Neither witness establishes an effect.
|
||||
pub(crate) fn uncertain_agent_outcome(state: &str, bytes: &[u8], digest: &str) -> bool {
|
||||
if bytes.is_empty() || hash(bytes) != digest {
|
||||
return false;
|
||||
}
|
||||
let Ok(outcome) = serde_json::from_slice::<ReconciledOutcome>(bytes) else {
|
||||
return false;
|
||||
};
|
||||
canonical_outcome_bytes(&outcome).is_ok_and(|canonical| canonical == bytes)
|
||||
&& outcome.state == state
|
||||
&& !outcome.output_truncated
|
||||
&& !outcome.reboot_required
|
||||
&& (operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &outcome)
|
||||
|| (state == "unknown"
|
||||
&& matches!(
|
||||
outcome.error_code.as_str(),
|
||||
"AGENT_PROCESS_INTERRUPTED" | "HELPER_TRANSPORT_EFFECT_UNKNOWN"
|
||||
)
|
||||
&& outcome.exit_code.is_none()
|
||||
&& outcome.stdout.is_empty()
|
||||
&& outcome.stderr.is_empty()
|
||||
&& outcome.evidence["postconditions_passed"] == false))
|
||||
}
|
||||
|
||||
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
|
||||
if s.is_empty() {
|
||||
Ok(None)
|
||||
@@ -1452,6 +1532,24 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|
||||
result_envelope = Some(envelope);
|
||||
}
|
||||
}
|
||||
// The signed closure projection can commit before either consumption
|
||||
// ledger. A fresh signed check may then finish the remaining writes.
|
||||
if state == "closed_unknown"
|
||||
&& consumption == "consumed"
|
||||
&& (canonical.is_empty()
|
||||
|| uncertain_agent_outcome("unknown", &canonical, &hash(&canonical)))
|
||||
{
|
||||
let closed: Vec<u8> = db.query_row(
|
||||
"SELECT outcome FROM reconcile_outcomes WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
uncertain_agent_outcome(&state, &closed, &hash(&closed)),
|
||||
"LOCAL_CLOSURE_OUTCOME_MISMATCH"
|
||||
);
|
||||
canonical = closed;
|
||||
}
|
||||
if !canonical.is_empty() {
|
||||
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
|
||||
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
|
||||
@@ -1746,6 +1844,15 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
|
||||
"HELPER_ATTEMPT_BINDING"
|
||||
);
|
||||
if h.consumption != "unconsumed" {
|
||||
let unreported = h.state == "unknown"
|
||||
&& h.consumption == "consumed"
|
||||
&& h.outcome_bytes.is_empty()
|
||||
&& h.outcome_sha256.is_empty()
|
||||
&& h.update_handoff_bytes.is_empty()
|
||||
&& a.consumption == "consumed"
|
||||
&& a.grant_id == h.grant_id
|
||||
&& a.original_boot_id == h.original_boot_id
|
||||
&& uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
|
||||
if !h.update_handoff_bytes.is_empty() {
|
||||
ensure!(
|
||||
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
|
||||
@@ -1756,7 +1863,7 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
|
||||
a.grant_id = h.grant_id.clone();
|
||||
a.consumption = h.consumption.clone();
|
||||
ensure!(
|
||||
a.outcome_sha256 == h.outcome_sha256,
|
||||
a.outcome_sha256 == h.outcome_sha256 || unreported,
|
||||
"HELPER_OUTCOME_MISMATCH"
|
||||
);
|
||||
if a.original_boot_id.is_empty() {
|
||||
|
||||
Reference in New Issue
Block a user