Reconcile a human closure when the helper has no result
Rust development slice / linux (push) Failing after 9s
Rust development slice / linux (push) Failing after 9s
This commit is contained in:
@@ -782,10 +782,10 @@ impl<V: Ed25519Verifier> HelperSync<V> {
|
||||
let d = decisions
|
||||
.get(&id)
|
||||
.context("ROOT_ATTEMPT_OMITTED_FROM_DECISION")?;
|
||||
validate_summary(row, local)?;
|
||||
validate_decision(row, local, d)?;
|
||||
let unreported = validate_summary(row, local)?;
|
||||
validate_decision(row, local, d, unreported)?;
|
||||
if d.unlock {
|
||||
let bytes = unlock_outcome(row, d, current_boot)?;
|
||||
let bytes = unlock_outcome(row, d, current_boot, unreported.then_some(local))?;
|
||||
if d.outcome_state == "closed_unknown" {
|
||||
crate::ledger::retain_closure_original(
|
||||
&tx,
|
||||
@@ -1636,22 +1636,33 @@ fn definitive(row: &LocalAttempt) -> bool {
|
||||
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
|
||||
) && !row.outcome_bytes.is_empty()
|
||||
}
|
||||
fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<()> {
|
||||
fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<bool> {
|
||||
let unreported = row.state == "unknown"
|
||||
&& row.outcome_sha256.is_empty()
|
||||
&& row.outcome_bytes.is_empty()
|
||||
&& row.update_handoff_bytes.is_empty()
|
||||
&& crate::reconcile::uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
|
||||
ensure!(
|
||||
a.job_payload_hash == row.job_payload_hash
|
||||
&& a.grant_id == row.grant_id.to_string()
|
||||
&& a.consumption == "consumed"
|
||||
&& a.state == row.state
|
||||
&& a.revision >= row.revision
|
||||
&& a.original_boot_id == row.original_boot_id.to_string()
|
||||
&& a.outcome_sha256 == row.outcome_sha256
|
||||
&& a.update_handoff_bytes == row.update_handoff_bytes
|
||||
&& (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes),
|
||||
&& (unreported
|
||||
|| (a.state == row.state
|
||||
&& a.outcome_sha256 == row.outcome_sha256
|
||||
&& (a.outcome_bytes.is_empty() || a.outcome_bytes == row.outcome_bytes))),
|
||||
"ROOT_SNAPSHOT_REQUEST_MISMATCH"
|
||||
);
|
||||
Ok(())
|
||||
Ok(unreported)
|
||||
}
|
||||
fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -> Result<()> {
|
||||
fn validate_decision(
|
||||
row: &LocalAttempt,
|
||||
a: &AttemptView,
|
||||
d: &AttemptDecision,
|
||||
unreported: bool,
|
||||
) -> Result<()> {
|
||||
if !row.update_handoff_bytes.is_empty() {
|
||||
let proof =
|
||||
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&row.update_handoff_bytes)
|
||||
@@ -1686,7 +1697,7 @@ fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -
|
||||
&& d.original_boot_id == Some(row.original_boot_id)
|
||||
&& (d.revision >= a.revision || (!d.unlock && d.outcome_state == "unknown"))
|
||||
&& d.previous_outcome_sha256.as_deref()
|
||||
== (!row.outcome_sha256.is_empty()).then_some(row.outcome_sha256.as_str()),
|
||||
== (!a.outcome_sha256.is_empty()).then_some(a.outcome_sha256.as_str()),
|
||||
"ROOT_SIGNED_ATTEMPT_BINDING"
|
||||
);
|
||||
ensure!(
|
||||
@@ -1697,11 +1708,26 @@ fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -
|
||||
),
|
||||
"UNKNOWN_CANNOT_UNLOCK"
|
||||
);
|
||||
ensure!(
|
||||
!unreported
|
||||
|| (!d.unlock && d.outcome_state == "unknown")
|
||||
|| (d.unlock
|
||||
&& d.outcome_state == "closed_unknown"
|
||||
&& d.reason == "OPERATOR_CLOSED_UNKNOWN"),
|
||||
"ROOT_UNREPORTED_OUTCOME_CLOSURE_REQUIRED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -> Result<Vec<u8>> {
|
||||
fn unlock_outcome(
|
||||
row: &LocalAttempt,
|
||||
d: &AttemptDecision,
|
||||
current_boot: Uuid,
|
||||
unreported: Option<&AttemptView>,
|
||||
) -> Result<Vec<u8>> {
|
||||
let bytes = if d.outcome_bytes.is_empty() {
|
||||
row.outcome_bytes.clone()
|
||||
unreported
|
||||
.map_or(&row.outcome_bytes, |a| &a.outcome_bytes)
|
||||
.clone()
|
||||
} else {
|
||||
d.outcome_bytes.clone()
|
||||
};
|
||||
@@ -1712,9 +1738,7 @@ fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -
|
||||
&& o.state == d.outcome_state,
|
||||
"ROOT_CANONICAL_UNLOCK_OUTCOME"
|
||||
);
|
||||
if bytes != row.outcome_bytes
|
||||
&& crate::reconcile::operator_closure(&d.reason, &row.state, &o)
|
||||
{
|
||||
if bytes != row.outcome_bytes && crate::reconcile::operator_closure(&d.reason, &row.state, &o) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
if bytes != row.outcome_bytes {
|
||||
|
||||
@@ -1019,9 +1019,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
} else {
|
||||
for d in decision.attempts.iter().filter(|d| d.unlock) {
|
||||
let grant = d.grant_id.context("UNLOCK_GRANT_MISSING")?;
|
||||
let (agent,generation,job,attempt,previous):(String,u64,String,String,Option<Vec<u8>>)=tx.query_row(
|
||||
"SELECT agent,generation,job,attempt,COALESCE((SELECT outcome FROM execution_evidence WHERE execution_evidence.grant_id=consumed.grant_id),result) FROM consumed WHERE grant_id=?1",
|
||||
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)))?;
|
||||
let (agent,generation,job,attempt,previous,start):(String,u64,String,String,Option<Vec<u8>>,Option<Vec<u8>>)=tx.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,COALESCE(e.outcome,c.result),e.start FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[grant.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?;
|
||||
ensure!(
|
||||
agent == self.binding.agent_id.to_string()
|
||||
&& generation == self.binding.enrollment_generation
|
||||
@@ -1034,13 +1034,61 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
Some(&self.binding),
|
||||
grant,
|
||||
)?;
|
||||
let previous = resolved
|
||||
.or(previous)
|
||||
.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
|
||||
let original = previous.clone();
|
||||
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
|
||||
&previous,
|
||||
)?)?;
|
||||
let previous = resolved.or(previous);
|
||||
let summary = gate
|
||||
.request
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|a| a.job_id == job && a.attempt_id == attempt)
|
||||
.context("LOCAL_UNLOCK_ATTEMPT_MISSING")?;
|
||||
let closure = d.reason == "OPERATOR_CLOSED_UNKNOWN"
|
||||
&& d.outcome_state == "closed_unknown"
|
||||
&& summary.consumption == "consumed"
|
||||
&& uncertain_agent_outcome(
|
||||
&summary.state,
|
||||
&summary.outcome_bytes,
|
||||
&summary.outcome_sha256,
|
||||
);
|
||||
let prior = match &previous {
|
||||
Some(bytes) => {
|
||||
let canonical = canonical_outcome_bytes(&serde_json::from_slice(bytes)?)?;
|
||||
if closure
|
||||
&& summary.state == "closed_unknown"
|
||||
&& uncertain_agent_outcome("unknown", &canonical, &hash(&canonical))
|
||||
{
|
||||
summary.outcome_bytes.clone()
|
||||
} else {
|
||||
canonical
|
||||
}
|
||||
}
|
||||
None => {
|
||||
ensure!(closure, "LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING");
|
||||
let start: serde_json::Value = serde_json::from_slice(
|
||||
&start.context("LOCAL_UNLOCK_ORIGINAL_START_MISSING")?,
|
||||
)?;
|
||||
let request: crate::auth::Request =
|
||||
serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
let (original, permit) = self.binding.verify_with(
|
||||
&request,
|
||||
permit.not_before,
|
||||
&self.verifier,
|
||||
)?;
|
||||
ensure!(
|
||||
permit.grant_id == grant
|
||||
&& original.job_id == d.job_id
|
||||
&& original.attempt_id == d.attempt_id
|
||||
&& hash(&request.job_bytes) == d.job_payload_hash
|
||||
&& Some(permit.authorization_id) == d.authorization_id
|
||||
&& Some(permit.plan_hash.as_str()) == d.plan_hash.as_deref()
|
||||
&& start["previous_boot_id"].as_str()
|
||||
== d.original_boot_id.map(|id| id.to_string()).as_deref(),
|
||||
"LOCAL_UNLOCK_ORIGINAL_START_BINDING"
|
||||
);
|
||||
summary.outcome_bytes.clone()
|
||||
}
|
||||
};
|
||||
ensure!(
|
||||
d.previous_outcome_sha256.as_deref() == Some(hash(&prior).as_str()),
|
||||
"LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISMATCH"
|
||||
@@ -1055,7 +1103,8 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
terminal(&outcome.state)
|
||||
&& outcome.state == d.outcome_state
|
||||
&& canonical_outcome_bytes(&outcome)? == bytes
|
||||
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str()),
|
||||
&& d.outcome_sha256.as_deref() == Some(hash(&bytes).as_str())
|
||||
&& (previous.is_some() || operator_closure(&d.reason, "unknown", &outcome)),
|
||||
"LOCAL_UNLOCK_CANONICAL_OUTCOME"
|
||||
);
|
||||
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
|
||||
@@ -1065,7 +1114,12 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
&bytes,
|
||||
)?;
|
||||
if outcome.state == "closed_unknown" {
|
||||
crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?;
|
||||
crate::ledger::retain_closure_original(
|
||||
&tx,
|
||||
grant,
|
||||
previous.as_deref().unwrap_or_default(),
|
||||
&bytes,
|
||||
)?;
|
||||
}
|
||||
tx.execute(
|
||||
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
|
||||
@@ -1304,6 +1358,32 @@ pub(crate) fn operator_closure(
|
||||
&& outcome.evidence["closed_by_human"] == true
|
||||
&& outcome.evidence["retry"] == false
|
||||
}
|
||||
|
||||
/// An interrupted agent can record its own uncertainty while the independently
|
||||
/// consumed helper grant has no result. Neither witness establishes an effect.
|
||||
pub(crate) fn uncertain_agent_outcome(state: &str, bytes: &[u8], digest: &str) -> bool {
|
||||
if bytes.is_empty() || hash(bytes) != digest {
|
||||
return false;
|
||||
}
|
||||
let Ok(outcome) = serde_json::from_slice::<ReconciledOutcome>(bytes) else {
|
||||
return false;
|
||||
};
|
||||
canonical_outcome_bytes(&outcome).is_ok_and(|canonical| canonical == bytes)
|
||||
&& outcome.state == state
|
||||
&& !outcome.output_truncated
|
||||
&& !outcome.reboot_required
|
||||
&& (operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &outcome)
|
||||
|| (state == "unknown"
|
||||
&& matches!(
|
||||
outcome.error_code.as_str(),
|
||||
"AGENT_PROCESS_INTERRUPTED" | "HELPER_TRANSPORT_EFFECT_UNKNOWN"
|
||||
)
|
||||
&& outcome.exit_code.is_none()
|
||||
&& outcome.stdout.is_empty()
|
||||
&& outcome.stderr.is_empty()
|
||||
&& outcome.evidence["postconditions_passed"] == false))
|
||||
}
|
||||
|
||||
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
|
||||
if s.is_empty() {
|
||||
Ok(None)
|
||||
@@ -1452,6 +1532,24 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|
||||
result_envelope = Some(envelope);
|
||||
}
|
||||
}
|
||||
// The signed closure projection can commit before either consumption
|
||||
// ledger. A fresh signed check may then finish the remaining writes.
|
||||
if state == "closed_unknown"
|
||||
&& consumption == "consumed"
|
||||
&& (canonical.is_empty()
|
||||
|| uncertain_agent_outcome("unknown", &canonical, &hash(&canonical)))
|
||||
{
|
||||
let closed: Vec<u8> = db.query_row(
|
||||
"SELECT outcome FROM reconcile_outcomes WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
uncertain_agent_outcome(&state, &closed, &hash(&closed)),
|
||||
"LOCAL_CLOSURE_OUTCOME_MISMATCH"
|
||||
);
|
||||
canonical = closed;
|
||||
}
|
||||
if !canonical.is_empty() {
|
||||
let o: ReconciledOutcome = serde_json::from_slice(&canonical)?;
|
||||
ensure!(o.state == state, "LOCAL_OUTCOME_STATE_MISMATCH");
|
||||
@@ -1746,6 +1844,15 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
|
||||
"HELPER_ATTEMPT_BINDING"
|
||||
);
|
||||
if h.consumption != "unconsumed" {
|
||||
let unreported = h.state == "unknown"
|
||||
&& h.consumption == "consumed"
|
||||
&& h.outcome_bytes.is_empty()
|
||||
&& h.outcome_sha256.is_empty()
|
||||
&& h.update_handoff_bytes.is_empty()
|
||||
&& a.consumption == "consumed"
|
||||
&& a.grant_id == h.grant_id
|
||||
&& a.original_boot_id == h.original_boot_id
|
||||
&& uncertain_agent_outcome(&a.state, &a.outcome_bytes, &a.outcome_sha256);
|
||||
if !h.update_handoff_bytes.is_empty() {
|
||||
ensure!(
|
||||
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
|
||||
@@ -1756,7 +1863,7 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
|
||||
a.grant_id = h.grant_id.clone();
|
||||
a.consumption = h.consumption.clone();
|
||||
ensure!(
|
||||
a.outcome_sha256 == h.outcome_sha256,
|
||||
a.outcome_sha256 == h.outcome_sha256 || unreported,
|
||||
"HELPER_OUTCOME_MISMATCH"
|
||||
);
|
||||
if a.original_boot_id.is_empty() {
|
||||
|
||||
Reference in New Issue
Block a user