Add human WebUI closure of unknown jobs as final closed_unknown state (spec 1.31); bound agent sync restart interval
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
+100
-1
@@ -84,6 +84,10 @@ pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
|
||||
"/api/v1/tenants/{tenant}/jobs/{job}/cancel",
|
||||
post(cancel_http),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/jobs/{job}/unknown-closure",
|
||||
post(close_unknown_http),
|
||||
)
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize, schemars::JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -1081,7 +1085,13 @@ pub async fn operation(
|
||||
"needs_attention"
|
||||
} else if c("succeeded") == n {
|
||||
"succeeded"
|
||||
} else if c("succeeded") + c("failed") + c("cancelled") + c("expired") == n {
|
||||
} else if c("succeeded")
|
||||
+ c("failed")
|
||||
+ c("cancelled")
|
||||
+ c("expired")
|
||||
+ c("closed_unknown")
|
||||
== n
|
||||
{
|
||||
if c("succeeded") > 0 {
|
||||
"partially_succeeded"
|
||||
} else if c("cancelled") == n {
|
||||
@@ -1204,6 +1214,95 @@ async fn job_http(
|
||||
let p = principal(&s, &h, false).await?;
|
||||
Ok(Json(job(&s, &p, t, id).await?))
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CloseUnknown {
|
||||
pub attempt_id: Uuid,
|
||||
pub revision: i64,
|
||||
pub reason: String,
|
||||
}
|
||||
/// Human WebUI only: accept an unknown attempt as unresolved and final. The
|
||||
/// effect stays unknown; there is no retry, new grant or reinterpretation.
|
||||
pub async fn close_unknown(
|
||||
state: &crate::State,
|
||||
p: &Principal,
|
||||
tenant: Uuid,
|
||||
id: Uuid,
|
||||
input: CloseUnknown,
|
||||
) -> Result<Value> {
|
||||
scope(p, tenant, true)?;
|
||||
let session = p
|
||||
.session_hash
|
||||
.clone()
|
||||
.ok_or(Error(StatusCode::FORBIDDEN, "INTERACTIVE_AUTH_REQUIRED"))?;
|
||||
if input.reason.trim().is_empty() || input.reason.len() > 1024 {
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "VALIDATION_ERROR"));
|
||||
}
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
||||
crate::permissions::require_principal(&mut tx, p, "plans:approve").await?;
|
||||
let row = sqlx::query(
|
||||
"SELECT attempt_id,state,revision,result FROM jobs WHERE job_id=$1 FOR UPDATE",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?
|
||||
.ok_or_else(missing)?;
|
||||
if row.get::<Uuid, _>("attempt_id") != input.attempt_id
|
||||
|| row.get::<i64, _>("revision") != input.revision
|
||||
{
|
||||
return Err(fail("JOB_CHANGED"));
|
||||
}
|
||||
if row.get::<String, _>("state") != "unknown" {
|
||||
return Err(fail("JOB_NOT_UNKNOWN"));
|
||||
}
|
||||
let original: Option<Value> = row.get("result");
|
||||
let grant: Option<Uuid> = sqlx::query_scalar(
|
||||
"SELECT grant_id FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(input.attempt_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let closure = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO job_unknown_closures(tenant_id,closure_id,job_id,attempt_id,grant_id,original_state,original_revision,original_result,principal,session_hash,approval_origin,testsimulation,reason) VALUES($1,$2,$3,$4,$5,'unknown',$6,$7,$8,$9,$10,$11,$12)")
|
||||
.bind(tenant).bind(closure).bind(id).bind(input.attempt_id).bind(grant)
|
||||
.bind(input.revision).bind(&original).bind(&p.name).bind(&session)
|
||||
.bind(p.approval_origin()).bind(p.testsimulation).bind(&input.reason)
|
||||
.execute(&mut *tx).await?;
|
||||
let original_sha256 = original
|
||||
.as_ref()
|
||||
.map(|v| rmm_domain::jobs::hash(&serde_json::to_vec(v).unwrap_or_default()));
|
||||
let result = json!({"stdout":[],"stderr":[],"exit_code":null,"output_truncated":false,"reboot_required":false,
|
||||
"error_code":"OPERATOR_CLOSED_UNKNOWN","grant_id":grant.map(|g|g.to_string()).unwrap_or_default(),
|
||||
"evidence":{"postconditions_passed":false,"effect":"unknown","closed_by_human":true,"closure_id":closure,
|
||||
"original_state":"unknown","original_revision":input.revision,"original_result_sha256":original_sha256,
|
||||
"retry":false,"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation}});
|
||||
sqlx::query("UPDATE jobs SET state='closed_unknown',revision=revision+1,result=$2,pending_reason=NULL WHERE job_id=$1 AND state='unknown'")
|
||||
.bind(id)
|
||||
.bind(&result)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("INSERT INTO audit_events VALUES($1,$2,$3,'job_unknown_closed',$4,now())")
|
||||
.bind(tenant)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(&p.name)
|
||||
.bind(json!({"job_id":id,"attempt_id":input.attempt_id,"grant_id":grant,"closure_id":closure,
|
||||
"original_revision":input.revision,"reason":input.reason,"session_hash":session,
|
||||
"approval_origin":p.approval_origin(),"testsimulation":p.testsimulation}))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
job(state, p, tenant, id).await
|
||||
}
|
||||
async fn close_unknown_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
Path((t, id)): Path<(Uuid, Uuid)>,
|
||||
Json(i): Json<CloseUnknown>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = principal(&s, &h, true).await?;
|
||||
Ok(Json(close_unknown(&s, &p, t, id, i).await?))
|
||||
}
|
||||
async fn cancel_http(
|
||||
State(s): State<crate::State>,
|
||||
h: HeaderMap,
|
||||
|
||||
@@ -44,7 +44,10 @@ fn outcome(state: &str, value: &Value) -> Result<ReconciledOutcome> {
|
||||
})
|
||||
}
|
||||
fn unsettled(state: &str) -> bool {
|
||||
!matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
|
||||
!matches!(
|
||||
state,
|
||||
"succeeded" | "failed" | "cancelled" | "expired" | "closed_unknown"
|
||||
)
|
||||
}
|
||||
fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool {
|
||||
!reserved
|
||||
@@ -420,6 +423,7 @@ pub async fn reconcile(
|
||||
| "verifying"
|
||||
| "awaiting_reboot"
|
||||
| "unknown"
|
||||
| "closed_unknown"
|
||||
| "succeeded"
|
||||
| "failed"
|
||||
| "cancelled"
|
||||
@@ -444,7 +448,7 @@ pub async fn reconcile(
|
||||
ensure!(parsed.state == local.state, "local outcome state mismatch");
|
||||
}
|
||||
}
|
||||
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
|
||||
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
|
||||
if rows.len() > 256 {
|
||||
reason = "SYNC_SERVER_JOB_LIMIT".into();
|
||||
chain_valid = false;
|
||||
@@ -743,7 +747,9 @@ pub async fn reconcile(
|
||||
&& result.grant_id == local.grant_id,
|
||||
"sync result binding"
|
||||
);
|
||||
if !manual {
|
||||
// A human-closed attempt is final; late agent evidence is neither
|
||||
// replayed into nor allowed to reinterpret the closed ledger row.
|
||||
if !manual && server_state != "closed_unknown" {
|
||||
let evidence = local
|
||||
.result_envelope
|
||||
.as_ref()
|
||||
@@ -857,13 +863,22 @@ pub async fn reconcile(
|
||||
if let Some(value) = final_row.get::<Option<Value>, _>("result") {
|
||||
let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?;
|
||||
let hash = hash(&bytes);
|
||||
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled {
|
||||
decision.outcome_sha256 = Some(hash);
|
||||
// Human WebUI closure: only an attempt still locally unknown (or
|
||||
// already closed) receives the server-derived final bytes.
|
||||
let closed = terminal == "closed_unknown"
|
||||
&& matches!(local.state.as_str(), "unknown" | "closed_unknown");
|
||||
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled || closed {
|
||||
decision.unlock = local.consumption == "consumed" && grant_matches;
|
||||
if reboot_reconciled {
|
||||
if reboot_reconciled || (closed && prior_hash.as_deref() != Some(&hash)) {
|
||||
decision.outcome_bytes = bytes;
|
||||
}
|
||||
decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into();
|
||||
decision.outcome_sha256 = Some(hash);
|
||||
decision.reason = if closed {
|
||||
"OPERATOR_CLOSED_UNKNOWN"
|
||||
} else {
|
||||
"DEFINITIVE_OUTCOME_RECONCILED"
|
||||
}
|
||||
.into();
|
||||
} else {
|
||||
ready = false;
|
||||
decision.reason = "LOCAL_OUTCOME_MISMATCH".into();
|
||||
@@ -968,7 +983,7 @@ pub async fn reconcile(
|
||||
})
|
||||
}
|
||||
async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> {
|
||||
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?;
|
||||
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')").bind(job).bind(reason).execute(&mut **tx).await?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user