Add human WebUI closure of unknown jobs as final closed_unknown state (spec 1.31); bound agent sync restart interval
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-17 11:13:36 +01:00
parent 2660f6a88d
commit d6dfd01a9c
11 changed files with 363 additions and 29 deletions
+16 -2
View File
@@ -786,6 +786,14 @@ impl<V: Ed25519Verifier> HelperSync<V> {
validate_decision(row, local, d)?;
if d.unlock {
let bytes = unlock_outcome(row, d, current_boot)?;
if d.outcome_state == "closed_unknown" {
crate::ledger::retain_closure_original(
&tx,
row.grant_id,
&row.outcome_bytes,
&bytes,
)?;
}
let preserve = crate::windows_system_nonstart::preserve_original_unlock(
&tx,
&self.binding,
@@ -1523,6 +1531,7 @@ fn load_rows(
| "failed"
| "cancelled"
| "expired"
| "closed_unknown"
| "running"
),
"UNKNOWN_LOCAL_STATE"
@@ -1610,7 +1619,7 @@ fn load_rows(
fn definitive(row: &LocalAttempt) -> bool {
matches!(
row.state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
) && !row.outcome_bytes.is_empty()
}
fn validate_summary(row: &LocalAttempt, a: &AttemptView) -> Result<()> {
@@ -1670,7 +1679,7 @@ fn validate_decision(row: &LocalAttempt, a: &AttemptView, d: &AttemptDecision) -
!d.unlock
|| matches!(
d.outcome_state.as_str(),
"succeeded" | "failed" | "cancelled" | "expired"
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
),
"UNKNOWN_CANNOT_UNLOCK"
);
@@ -1689,6 +1698,11 @@ fn unlock_outcome(row: &LocalAttempt, d: &AttemptDecision, current_boot: Uuid) -
&& o.state == d.outcome_state,
"ROOT_CANONICAL_UNLOCK_OUTCOME"
);
if bytes != row.outcome_bytes
&& crate::reconcile::operator_closure(&d.reason, &row.state, &o)
{
return Ok(bytes);
}
if bytes != row.outcome_bytes {
ensure!(
row.action == "reboot"
+31
View File
@@ -34,6 +34,37 @@ pub fn initialize_ledger(db: &rusqlite::Connection) -> Result<()> {
Ok(())
}
/// Before a human WebUI closure replaces the unresolved local outcome, keep the
/// exact original outcome bytes once. A differing second copy fails closed.
pub(crate) fn retain_closure_original(
db: &rusqlite::Connection,
grant: Uuid,
original: &[u8],
closure: &[u8],
) -> Result<()> {
db.execute_batch("CREATE TABLE IF NOT EXISTS unknown_closure_originals(grant_id TEXT PRIMARY KEY, original_outcome BLOB NOT NULL, closure_outcome BLOB NOT NULL);")?;
let existing: Option<(Vec<u8>, Vec<u8>)> = db
.query_row(
"SELECT original_outcome,closure_outcome FROM unknown_closure_originals WHERE grant_id=?1",
[grant.to_string()],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()?;
match existing {
Some((old, done)) => ensure!(
done == closure && (old == original || original == closure),
"CLOSURE_ORIGINAL_CHANGED"
),
None => {
db.execute(
"INSERT INTO unknown_closure_originals VALUES(?1,?2,?3)",
params![grant.to_string(), original, closure],
)?;
}
}
Ok(())
}
/// Legacy split reservation API retained for old fixtures/readers. Production
/// callers use begin_execution so a crash cannot lose the original start witness.
pub fn consume(db: &mut rusqlite::Connection, p: &ExecutionPermitPayload) -> Result<()> {
+64 -2
View File
@@ -1036,6 +1036,7 @@ impl<V: Ed25519Verifier> ClientSync<V> {
let previous = resolved
.or(previous)
.context("LOCAL_UNLOCK_PREVIOUS_OUTCOME_MISSING")?;
let original = previous.clone();
let prior = canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&previous,
)?)?;
@@ -1062,6 +1063,9 @@ impl<V: Ed25519Verifier> ClientSync<V> {
grant,
&bytes,
)?;
if outcome.state == "closed_unknown" {
crate::ledger::retain_closure_original(&tx, grant, &original, &bytes)?;
}
tx.execute(
"UPDATE consumed SET result=?2 WHERE grant_id=?1",
params![grant.to_string(), bytes],
@@ -1200,7 +1204,29 @@ pub(crate) fn outcome(r: &JobResult) -> Result<ReconciledOutcome> {
})
}
fn terminal(state: &str) -> bool {
matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
matches!(
state,
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
)
}
/// Server-signed final bytes of a human WebUI closure for an attempt whose
/// effect stayed unknown. They never claim success, failure or nonstart.
pub(crate) fn operator_closure(
reason: &str,
local_state: &str,
outcome: &ReconciledOutcome,
) -> bool {
reason == "OPERATOR_CLOSED_UNKNOWN"
&& local_state == "unknown"
&& outcome.state == "closed_unknown"
&& outcome.error_code == "OPERATOR_CLOSED_UNKNOWN"
&& outcome.exit_code.is_none()
&& outcome.stdout.is_empty()
&& outcome.stderr.is_empty()
&& outcome.evidence["postconditions_passed"] == false
&& outcome.evidence["effect"] == "unknown"
&& outcome.evidence["closed_by_human"] == true
&& outcome.evidence["retry"] == false
}
fn optional_uuid(s: &str) -> Result<Option<Uuid>> {
if s.is_empty() {
@@ -1861,7 +1887,7 @@ fn validate_attempt(
&& o.state == d.outcome_state,
"SYNC_CANONICAL_OUTCOME"
);
if bytes != local.outcome_bytes {
if bytes != local.outcome_bytes && !operator_closure(&d.reason, &local.state, &o) {
ensure!(
matches!(job.action, rmm_domain::jobs::Action::Reboot { .. })
&& d.outcome_state == "succeeded"
@@ -2166,3 +2192,39 @@ pub fn closed_control_contract(raw: &[u8]) -> Result<serde_json::Value> {
json!({"actual_closed_partial_edge_accepted":true,"mutated_agent_request_rejected":true,"mutated_server_signature_and_ack_rejected":true,"admission_read_only":true,"hidden_open_grant_rejected":true,"explicit_idle_attestation_required":true,"typed_precommit_clock_keeps_capture_without_writes":true,"fresh_request_keeps_both_heads":true,"old_authority_reused":false,"new_ready_or_unlock":false,"product_databases_opened":false,"effects":false}),
)
}
#[cfg(test)]
mod closure_tests {
use super::*;
#[test]
fn only_a_human_closure_of_a_locally_unknown_attempt_is_accepted() {
let closed = ReconciledOutcome {
state: "closed_unknown".into(),
stdout: vec![],
stderr: vec![],
exit_code: None,
output_truncated: false,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"retry":false}),
error_code: "OPERATOR_CLOSED_UNKNOWN".into(),
};
assert!(operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &closed));
assert!(!operator_closure("DEFINITIVE_OUTCOME_RECONCILED", "unknown", &closed));
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "running", &closed));
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "succeeded", &closed));
for case in 0..6 {
let mut other = closed.clone();
match case {
0 => other.state = "failed".into(),
1 => other.error_code = "SCRIPT_FAILED".into(),
2 => other.exit_code = Some(0),
3 => other.stdout = b"x".to_vec(),
4 => other.evidence["postconditions_passed"] = true.into(),
_ => other.evidence["retry"] = true.into(),
}
assert!(!operator_closure("OPERATOR_CLOSED_UNKNOWN", "unknown", &other));
}
assert!(terminal("closed_unknown"));
}
}
+18 -11
View File
@@ -62,18 +62,22 @@ pub fn clock_code(error: &anyhow::Error) -> &'static str {
/// Refresh is control work, not permission. A temporary loss of the100ms clock
/// evidence never discards an otherwise retained signed decision or extends it.
/// Fresh clock/readiness checks remain mandatory at each actual job gate.
/// A requested refresh waits at least `MIN_SYNC_INTERVAL` after the previous
/// start, so a persistently not-ready decision cannot become a tight loop.
pub const MIN_SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3);
pub fn refresh_due(
needs_sync: bool,
elapsed: std::time::Duration,
valid_until: Option<chrono::DateTime<chrono::Utc>>,
clock_upper: Option<chrono::DateTime<chrono::Utc>>,
) -> bool {
needs_sync
|| elapsed >= std::time::Duration::from_secs(40)
|| valid_until.is_none()
|| valid_until
.zip(clock_upper)
.is_some_and(|(until, upper)| upper >= until)
elapsed >= MIN_SYNC_INTERVAL
&& (needs_sync
|| elapsed >= std::time::Duration::from_secs(40)
|| valid_until.is_none()
|| valid_until
.zip(clock_upper)
.is_some_and(|(until, upper)| upper >= until))
}
#[cfg(windows)]
pub fn persist(state: &Path, value: &Observation) -> Result<()> {
@@ -143,11 +147,14 @@ mod tests {
let now = chrono::DateTime::from_timestamp(100, 0).unwrap();
let until = now + chrono::Duration::seconds(30);
let one = std::time::Duration::from_secs(1);
assert!(!refresh_due(false, one, Some(until), None));
assert!(!refresh_due(false, one, Some(until), Some(now)));
assert!(refresh_due(false, one, Some(until), Some(until)));
assert!(refresh_due(false, one, None, None));
assert!(refresh_due(true, one, Some(until), None));
let min = MIN_SYNC_INTERVAL;
assert!(!refresh_due(false, min, Some(until), None));
assert!(!refresh_due(false, min, Some(until), Some(now)));
assert!(refresh_due(false, min, Some(until), Some(until)));
assert!(refresh_due(false, min, None, None));
assert!(refresh_due(true, min, Some(until), None));
// No trigger may restart Sync sooner than the minimum interval.
assert!(!refresh_due(true, one, None, Some(until)));
assert!(refresh_due(
false,
std::time::Duration::from_secs(40),
+4 -1
View File
@@ -738,12 +738,15 @@ pub enum JobState {
Cancelled,
Expired,
Unknown,
/// A human closed an unknown attempt in the WebUI. The effect stays
/// unknown; the attempt is final and is never retried or reinterpreted.
ClosedUnknown,
}
impl JobState {
pub fn terminal(self) -> bool {
matches!(
self,
Self::Succeeded | Self::Failed | Self::Cancelled | Self::Expired
Self::Succeeded | Self::Failed | Self::Cancelled | Self::Expired | Self::ClosedUnknown
)
}
pub fn allows(self, next: Self) -> bool {
+100 -1
View File
@@ -84,6 +84,10 @@ pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
"/api/v1/tenants/{tenant}/jobs/{job}/cancel",
post(cancel_http),
)
.route(
"/api/v1/tenants/{tenant}/jobs/{job}/unknown-closure",
post(close_unknown_http),
)
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
@@ -1081,7 +1085,13 @@ pub async fn operation(
"needs_attention"
} else if c("succeeded") == n {
"succeeded"
} else if c("succeeded") + c("failed") + c("cancelled") + c("expired") == n {
} else if c("succeeded")
+ c("failed")
+ c("cancelled")
+ c("expired")
+ c("closed_unknown")
== n
{
if c("succeeded") > 0 {
"partially_succeeded"
} else if c("cancelled") == n {
@@ -1204,6 +1214,95 @@ async fn job_http(
let p = principal(&s, &h, false).await?;
Ok(Json(job(&s, &p, t, id).await?))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CloseUnknown {
pub attempt_id: Uuid,
pub revision: i64,
pub reason: String,
}
/// Human WebUI only: accept an unknown attempt as unresolved and final. The
/// effect stays unknown; there is no retry, new grant or reinterpretation.
pub async fn close_unknown(
state: &crate::State,
p: &Principal,
tenant: Uuid,
id: Uuid,
input: CloseUnknown,
) -> Result<Value> {
scope(p, tenant, true)?;
let session = p
.session_hash
.clone()
.ok_or(Error(StatusCode::FORBIDDEN, "INTERACTIVE_AUTH_REQUIRED"))?;
if input.reason.trim().is_empty() || input.reason.len() > 1024 {
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
crate::permissions::require_principal(&mut tx, p, "plans:approve").await?;
let row = sqlx::query(
"SELECT attempt_id,state,revision,result FROM jobs WHERE job_id=$1 FOR UPDATE",
)
.bind(id)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(missing)?;
if row.get::<Uuid, _>("attempt_id") != input.attempt_id
|| row.get::<i64, _>("revision") != input.revision
{
return Err(fail("JOB_CHANGED"));
}
if row.get::<String, _>("state") != "unknown" {
return Err(fail("JOB_NOT_UNKNOWN"));
}
let original: Option<Value> = row.get("result");
let grant: Option<Uuid> = sqlx::query_scalar(
"SELECT grant_id FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
)
.bind(id)
.bind(input.attempt_id)
.fetch_optional(&mut *tx)
.await?;
let closure = Uuid::new_v4();
sqlx::query("INSERT INTO job_unknown_closures(tenant_id,closure_id,job_id,attempt_id,grant_id,original_state,original_revision,original_result,principal,session_hash,approval_origin,testsimulation,reason) VALUES($1,$2,$3,$4,$5,'unknown',$6,$7,$8,$9,$10,$11,$12)")
.bind(tenant).bind(closure).bind(id).bind(input.attempt_id).bind(grant)
.bind(input.revision).bind(&original).bind(&p.name).bind(&session)
.bind(p.approval_origin()).bind(p.testsimulation).bind(&input.reason)
.execute(&mut *tx).await?;
let original_sha256 = original
.as_ref()
.map(|v| rmm_domain::jobs::hash(&serde_json::to_vec(v).unwrap_or_default()));
let result = json!({"stdout":[],"stderr":[],"exit_code":null,"output_truncated":false,"reboot_required":false,
"error_code":"OPERATOR_CLOSED_UNKNOWN","grant_id":grant.map(|g|g.to_string()).unwrap_or_default(),
"evidence":{"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"closure_id":closure,
"original_state":"unknown","original_revision":input.revision,"original_result_sha256":original_sha256,
"retry":false,"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation}});
sqlx::query("UPDATE jobs SET state='closed_unknown',revision=revision+1,result=$2,pending_reason=NULL WHERE job_id=$1 AND state='unknown'")
.bind(id)
.bind(&result)
.execute(&mut *tx)
.await?;
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'job_unknown_closed',$4,now())")
.bind(tenant)
.bind(Uuid::new_v4())
.bind(&p.name)
.bind(json!({"job_id":id,"attempt_id":input.attempt_id,"grant_id":grant,"closure_id":closure,
"original_revision":input.revision,"reason":input.reason,"session_hash":session,
"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation}))
.execute(&mut *tx)
.await?;
tx.commit().await?;
job(state, p, tenant, id).await
}
async fn close_unknown_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
Json(i): Json<CloseUnknown>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, true).await?;
Ok(Json(close_unknown(&s, &p, t, id, i).await?))
}
async fn cancel_http(
State(s): State<crate::State>,
h: HeaderMap,
+23 -8
View File
@@ -44,7 +44,10 @@ fn outcome(state: &str, value: &Value) -> Result<ReconciledOutcome> {
})
}
fn unsettled(state: &str) -> bool {
!matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
!matches!(
state,
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
)
}
fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool {
!reserved
@@ -420,6 +423,7 @@ pub async fn reconcile(
| "verifying"
| "awaiting_reboot"
| "unknown"
| "closed_unknown"
| "succeeded"
| "failed"
| "cancelled"
@@ -444,7 +448,7 @@ pub async fn reconcile(
ensure!(parsed.state == local.state, "local outcome state mismatch");
}
}
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
if rows.len() > 256 {
reason = "SYNC_SERVER_JOB_LIMIT".into();
chain_valid = false;
@@ -743,7 +747,9 @@ pub async fn reconcile(
&& result.grant_id == local.grant_id,
"sync result binding"
);
if !manual {
// A human-closed attempt is final; late agent evidence is neither
// replayed into nor allowed to reinterpret the closed ledger row.
if !manual && server_state != "closed_unknown" {
let evidence = local
.result_envelope
.as_ref()
@@ -857,13 +863,22 @@ pub async fn reconcile(
if let Some(value) = final_row.get::<Option<Value>, _>("result") {
let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?;
let hash = hash(&bytes);
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled {
decision.outcome_sha256 = Some(hash);
// Human WebUI closure: only an attempt still locally unknown (or
// already closed) receives the server-derived final bytes.
let closed = terminal == "closed_unknown"
&& matches!(local.state.as_str(), "unknown" | "closed_unknown");
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled || closed {
decision.unlock = local.consumption == "consumed" && grant_matches;
if reboot_reconciled {
if reboot_reconciled || (closed && prior_hash.as_deref() != Some(&hash)) {
decision.outcome_bytes = bytes;
}
decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into();
decision.outcome_sha256 = Some(hash);
decision.reason = if closed {
"OPERATOR_CLOSED_UNKNOWN"
} else {
"DEFINITIVE_OUTCOME_RECONCILED"
}
.into();
} else {
ready = false;
decision.reason = "LOCAL_OUTCOME_MISMATCH".into();
@@ -968,7 +983,7 @@ pub async fn reconcile(
})
}
async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> {
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?;
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").bind(job).bind(reason).execute(&mut **tx).await?;
Ok(())
}
#[cfg(test)]