Add human WebUI closure of unknown jobs as final closed_unknown state (spec 1.31); bound agent sync restart interval
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-17 11:13:36 +01:00
parent 2660f6a88d
commit d6dfd01a9c
11 changed files with 363 additions and 29 deletions
+100 -1
View File
@@ -84,6 +84,10 @@ pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
"/api/v1/tenants/{tenant}/jobs/{job}/cancel",
post(cancel_http),
)
.route(
"/api/v1/tenants/{tenant}/jobs/{job}/unknown-closure",
post(close_unknown_http),
)
}
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
#[serde(deny_unknown_fields)]
@@ -1081,7 +1085,13 @@ pub async fn operation(
"needs_attention"
} else if c("succeeded") == n {
"succeeded"
} else if c("succeeded") + c("failed") + c("cancelled") + c("expired") == n {
} else if c("succeeded")
+ c("failed")
+ c("cancelled")
+ c("expired")
+ c("closed_unknown")
== n
{
if c("succeeded") > 0 {
"partially_succeeded"
} else if c("cancelled") == n {
@@ -1204,6 +1214,95 @@ async fn job_http(
let p = principal(&s, &h, false).await?;
Ok(Json(job(&s, &p, t, id).await?))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CloseUnknown {
pub attempt_id: Uuid,
pub revision: i64,
pub reason: String,
}
/// Human WebUI only: accept an unknown attempt as unresolved and final. The
/// effect stays unknown; there is no retry, new grant or reinterpretation.
pub async fn close_unknown(
state: &crate::State,
p: &Principal,
tenant: Uuid,
id: Uuid,
input: CloseUnknown,
) -> Result<Value> {
scope(p, tenant, true)?;
let session = p
.session_hash
.clone()
.ok_or(Error(StatusCode::FORBIDDEN, "INTERACTIVE_AUTH_REQUIRED"))?;
if input.reason.trim().is_empty() || input.reason.len() > 1024 {
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
}
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
crate::permissions::require_principal(&mut tx, p, "plans:approve").await?;
let row = sqlx::query(
"SELECT attempt_id,state,revision,result FROM jobs WHERE job_id=$1 FOR UPDATE",
)
.bind(id)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(missing)?;
if row.get::<Uuid, _>("attempt_id") != input.attempt_id
|| row.get::<i64, _>("revision") != input.revision
{
return Err(fail("JOB_CHANGED"));
}
if row.get::<String, _>("state") != "unknown" {
return Err(fail("JOB_NOT_UNKNOWN"));
}
let original: Option<Value> = row.get("result");
let grant: Option<Uuid> = sqlx::query_scalar(
"SELECT grant_id FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
)
.bind(id)
.bind(input.attempt_id)
.fetch_optional(&mut *tx)
.await?;
let closure = Uuid::new_v4();
sqlx::query("INSERT INTO job_unknown_closures(tenant_id,closure_id,job_id,attempt_id,grant_id,original_state,original_revision,original_result,principal,session_hash,approval_origin,testsimulation,reason) VALUES($1,$2,$3,$4,$5,'unknown',$6,$7,$8,$9,$10,$11,$12)")
.bind(tenant).bind(closure).bind(id).bind(input.attempt_id).bind(grant)
.bind(input.revision).bind(&original).bind(&p.name).bind(&session)
.bind(p.approval_origin()).bind(p.testsimulation).bind(&input.reason)
.execute(&mut *tx).await?;
let original_sha256 = original
.as_ref()
.map(|v| rmm_domain::jobs::hash(&serde_json::to_vec(v).unwrap_or_default()));
let result = json!({"stdout":[],"stderr":[],"exit_code":null,"output_truncated":false,"reboot_required":false,
"error_code":"OPERATOR_CLOSED_UNKNOWN","grant_id":grant.map(|g|g.to_string()).unwrap_or_default(),
"evidence":{"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"closure_id":closure,
"original_state":"unknown","original_revision":input.revision,"original_result_sha256":original_sha256,
"retry":false,"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation}});
sqlx::query("UPDATE jobs SET state='closed_unknown',revision=revision+1,result=$2,pending_reason=NULL WHERE job_id=$1 AND state='unknown'")
.bind(id)
.bind(&result)
.execute(&mut *tx)
.await?;
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'job_unknown_closed',$4,now())")
.bind(tenant)
.bind(Uuid::new_v4())
.bind(&p.name)
.bind(json!({"job_id":id,"attempt_id":input.attempt_id,"grant_id":grant,"closure_id":closure,
"original_revision":input.revision,"reason":input.reason,"session_hash":session,
"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation}))
.execute(&mut *tx)
.await?;
tx.commit().await?;
job(state, p, tenant, id).await
}
async fn close_unknown_http(
State(s): State<crate::State>,
h: HeaderMap,
Path((t, id)): Path<(Uuid, Uuid)>,
Json(i): Json<CloseUnknown>,
) -> Result<Json<Value>> {
let p = principal(&s, &h, true).await?;
Ok(Json(close_unknown(&s, &p, t, id, i).await?))
}
async fn cancel_http(
State(s): State<crate::State>,
h: HeaderMap,
+23 -8
View File
@@ -44,7 +44,10 @@ fn outcome(state: &str, value: &Value) -> Result<ReconciledOutcome> {
})
}
fn unsettled(state: &str) -> bool {
!matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
!matches!(
state,
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
)
}
fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool {
!reserved
@@ -420,6 +423,7 @@ pub async fn reconcile(
| "verifying"
| "awaiting_reboot"
| "unknown"
| "closed_unknown"
| "succeeded"
| "failed"
| "cancelled"
@@ -444,7 +448,7 @@ pub async fn reconcile(
ensure!(parsed.state == local.state, "local outcome state mismatch");
}
}
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
if rows.len() > 256 {
reason = "SYNC_SERVER_JOB_LIMIT".into();
chain_valid = false;
@@ -743,7 +747,9 @@ pub async fn reconcile(
&& result.grant_id == local.grant_id,
"sync result binding"
);
if !manual {
// A human-closed attempt is final; late agent evidence is neither
// replayed into nor allowed to reinterpret the closed ledger row.
if !manual && server_state != "closed_unknown" {
let evidence = local
.result_envelope
.as_ref()
@@ -857,13 +863,22 @@ pub async fn reconcile(
if let Some(value) = final_row.get::<Option<Value>, _>("result") {
let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?;
let hash = hash(&bytes);
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled {
decision.outcome_sha256 = Some(hash);
// Human WebUI closure: only an attempt still locally unknown (or
// already closed) receives the server-derived final bytes.
let closed = terminal == "closed_unknown"
&& matches!(local.state.as_str(), "unknown" | "closed_unknown");
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled || closed {
decision.unlock = local.consumption == "consumed" && grant_matches;
if reboot_reconciled {
if reboot_reconciled || (closed && prior_hash.as_deref() != Some(&hash)) {
decision.outcome_bytes = bytes;
}
decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into();
decision.outcome_sha256 = Some(hash);
decision.reason = if closed {
"OPERATOR_CLOSED_UNKNOWN"
} else {
"DEFINITIVE_OUTCOME_RECONCILED"
}
.into();
} else {
ready = false;
decision.reason = "LOCAL_OUTCOME_MISMATCH".into();
@@ -968,7 +983,7 @@ pub async fn reconcile(
})
}
async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> {
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?;
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").bind(job).bind(reason).execute(&mut **tx).await?;
Ok(())
}
#[cfg(test)]