42 lines
4.1 KiB
Rust
42 lines
4.1 KiB
Rust
//! Durable housekeeping does not depend on device traffic or an in-memory lease.
|
|
use anyhow::Result;
|
|
use sqlx::{Postgres, Transaction};
|
|
use uuid::Uuid;
|
|
pub async fn maintain(tx: &mut Transaction<'_, Postgres>, agent: Option<Uuid>) -> Result<()> {
|
|
sqlx::query("UPDATE approval_requests r SET approval_state='expired' FROM operation_plans p WHERE p.tenant_id=r.tenant_id AND p.plan_id=r.plan_id AND r.approval_state='pending_approval' AND p.expires_at<=now()").execute(&mut **tx).await?;
|
|
sqlx::query("UPDATE jobs j SET state='expired',pending_reason=NULL FROM operations o,approval_requests r,operation_plans p WHERE j.tenant_id=o.tenant_id AND j.operation_id=o.operation_id AND r.tenant_id=o.tenant_id AND r.operation_id=o.operation_id AND p.tenant_id=r.tenant_id AND p.plan_id=r.plan_id AND ($1::uuid IS NULL OR j.agent_id=$1) AND j.state IN ('queued','accepted') AND NOT j.start_reserved AND p.expires_at<=now()").bind(agent).execute(&mut **tx).await?;
|
|
sqlx::query("UPDATE jobs j SET state='unknown',pending_reason='reconciliation_required',result=COALESCE(j.result,'{}'::jsonb)||jsonb_build_object('error_code','START_OR_RESULT_UNCONFIRMED','source_id','server_grant_ledger') FROM execution_permits e WHERE e.tenant_id=j.tenant_id AND e.job_id=j.job_id AND ($1::uuid IS NULL OR j.agent_id=$1) AND j.start_reserved AND ((j.state='accepted' AND (convert_from(e.payload,'UTF8')::jsonb->>'start_before')::timestamptz<now()) OR (j.state IN ('running','verifying') AND (convert_from(e.payload,'UTF8')::jsonb->>'start_before')::timestamptz+make_interval(secs=>(convert_from(e.payload,'UTF8')::jsonb->>'max_runtime_seconds')::integer)<now()))").bind(agent).execute(&mut **tx).await?;
|
|
sqlx::query("UPDATE jobs j SET pending_reason=CASE WHEN t.changes_stopped THEN 'tenant_stopped' WHEN r.approval_state<>'approved' THEN 'authorization_revoked' WHEN (convert_from(j.payload,'UTF8')::jsonb->>'not_before')::timestamptz>now() THEN 'maintenance_window' WHEN i.last_contact_at IS NULL OR i.last_contact_at<now()-interval '180 seconds' THEN 'device_offline' ELSE NULL END FROM tenants t,approval_requests r,agent_installations i WHERE j.tenant_id=t.tenant_id AND r.tenant_id=j.tenant_id AND r.operation_id=j.operation_id AND i.tenant_id=j.tenant_id AND i.agent_id=j.agent_id AND j.state IN ('queued','accepted') AND NOT j.start_reserved AND ($1::uuid IS NULL OR j.agent_id=$1)").bind(agent).execute(&mut **tx).await?;
|
|
sqlx::query("UPDATE jobs j SET state='unknown',pending_reason='reconciliation_required',result=COALESCE(j.result,'{}'::jsonb)||jsonb_build_object('error_code','REBOOT_REJOIN_UNCONFIRMED','source_id','server_grant_ledger') FROM execution_permits e WHERE e.tenant_id=j.tenant_id AND e.job_id=j.job_id AND ($1::uuid IS NULL OR j.agent_id=$1) AND j.state='awaiting_reboot' AND (convert_from(e.payload,'UTF8')::jsonb->>'start_before')::timestamptz+make_interval(secs=>COALESCE((convert_from(j.payload,'UTF8')::jsonb->'action'->>'reconnect_timeout_seconds')::integer,3600))<now()").bind(agent).execute(&mut **tx).await?;
|
|
Ok(())
|
|
}
|
|
pub fn start(state: crate::State) {
|
|
tokio::spawn(async move {
|
|
let mut timer = tokio::time::interval(std::time::Duration::from_secs(1));
|
|
loop {
|
|
timer.tick().await;
|
|
for tenant in [rmm_domain::TENANT_A, rmm_domain::TENANT_B] {
|
|
let result = async {
|
|
let tenant = Uuid::parse_str(tenant)?;
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, tenant).await?;
|
|
let acquired: bool = sqlx::query_scalar(
|
|
"SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))",
|
|
)
|
|
.bind(format!("maintenance:{tenant}"))
|
|
.fetch_one(&mut *tx)
|
|
.await?;
|
|
if acquired {
|
|
maintain(&mut tx, None).await?;
|
|
}
|
|
tx.commit().await?;
|
|
Ok::<(), anyhow::Error>(())
|
|
}
|
|
.await;
|
|
if result.is_err() {
|
|
eprintln!("tenant housekeeping failed; no starts authorized by scheduler");
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|