Files
rust-discovery/crates/server/src/synchronization.rs
T
kevin_janssen cafd13bca9
Rust development slice / linux (push) Failing after 7s
Deploy reviewed Windows sync alignment with schema 20 server
Bind the administrative connection-205 review to original signed history and preserve closed readiness. Include the qualified server dependency closure, UI review, native ARM64 build provenance, and guarded deployment evidence. Migration, API/gateway replacement and TLS checks passed with unchanged job authority, protected mounts and time/database services; native Windows receipt application remains separate.
2026-11-16 13:30:31 +01:00

1461 lines
66 KiB
Rust

//! Authoritative reconciliation only: this module never inserts jobs, attempts,
//! outbox entries or execution permits. A signed decision is not start authority.
use anyhow::{Context, Result, ensure};
use chrono::{Duration, Utc};
use prost::Message;
use rmm_domain::{jobs::*, reconciliation::*};
use rmm_protocol::*;
use serde_json::{Value, json};
use sqlx::{Postgres, Row, Transaction};
use std::collections::{BTreeMap, BTreeSet};
use uuid::Uuid;
fn optional_id(raw: &str) -> Result<Option<Uuid>> {
if raw.is_empty() {
Ok(None)
} else {
let id = Uuid::parse_str(raw)?;
ensure!(!id.is_nil(), "nil sync identity");
Ok(Some(id))
}
}
fn digest(raw: &str) -> bool {
raw.len() == 64
&& raw
.bytes()
.all(|c| c.is_ascii_digit() || b"abcdef".contains(&c))
}
fn outcome(state: &str, value: &Value) -> Result<ReconciledOutcome> {
Ok(ReconciledOutcome {
state: state.into(),
stdout: serde_json::from_value(value["stdout"].clone())?,
stderr: serde_json::from_value(value["stderr"].clone())?,
exit_code: serde_json::from_value(value["exit_code"].clone())?,
output_truncated: value["output_truncated"]
.as_bool()
.context("output bound evidence missing")?,
reboot_required: value["reboot_required"]
.as_bool()
.context("reboot evidence missing")?,
evidence: value["evidence"].clone(),
error_code: value["error_code"]
.as_str()
.context("error evidence missing")?
.into(),
})
}
fn unsettled(state: &str) -> bool {
!matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
}
fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool {
!reserved
&& local.consumption == "unconsumed"
&& local.grant_id.is_empty()
&& matches!(local.state.as_str(), "queued" | "accepted")
}
/// A missing reply and a request that never committed are different cases.
/// A committed pending request is the newer anchor. Otherwise only the last
/// applied decision can anchor the chain; the caller still compares it with
/// the authoritative head and performs all journal/generation checks.
fn previous_anchor(
last_applied: Option<Uuid>,
pending_request: Option<Uuid>,
committed_pending: Option<Uuid>,
) -> Option<Uuid> {
if pending_request.is_some() {
committed_pending.or(last_applied)
} else {
last_applied
}
}
/// The server's durable grant reservation precedes the helper's saved start.
/// Using that trusted timestamp is conservative: no device clock or late
/// reconnect may extend the fixed, approved reconnect window.
fn reboot_deadline(
action: &Action,
reserved_at: Option<chrono::DateTime<Utc>>,
) -> Option<chrono::DateTime<Utc>> {
let Action::Reboot {
reconnect_timeout_seconds,
..
} = action
else {
return None;
};
if !action.validate() {
return None;
}
reserved_at?.checked_add_signed(Duration::seconds(i64::from(*reconnect_timeout_seconds)))
}
struct RebootWindow {
reserved_at: chrono::DateTime<Utc>,
deadline: chrono::DateTime<Utc>,
now: chrono::DateTime<Utc>,
require_fresh: bool,
}
async fn reboot_fresh_inventory(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
boot: Uuid,
now: chrono::DateTime<Utc>,
receipt: Option<Uuid>,
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt)
.fetch_optional(&mut **tx).await?)
}
async fn reboot_inventory(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
boot: Uuid,
window: &RebootWindow,
existing_receipt: Option<Uuid>,
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
// Already-persisted definitive success keeps its original receipt. It need
// not be younger than five minutes again, but its original server receipt
// time must still prove the approved reconnect window and current boot.
let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt)
.fetch_optional(&mut **tx).await?;
// Verification may be delayed by local bootstrap readiness. Never move
// the original reconnect cutoff: retain the actual timely receipt and
// independently demand fresh evidence of the SAME current boot.
if timely.is_some()
&& window.require_fresh
&& reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None)
.await?
.is_none()
{
return Ok(None);
}
Ok(timely)
}
/// First ingress of success needs the same independent, timely receipt as an
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
pub(crate) async fn verify_reboot_success(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
envelope: &Envelope,
job: &JobPayload,
result: &JobResult,
evidence: &Value,
) -> Result<()> {
let boot = Uuid::parse_str(&envelope.boot_id)?;
let grant = Uuid::parse_str(&result.grant_id)?;
let now = Utc::now();
ensure!(
job.tenant_id == tenant
&& job.agent_id == agent
&& job.job_id.to_string() == result.job_id
&& job.attempt_id.to_string() == result.attempt_id
&& evidence["postconditions_passed"] == true
&& evidence["reconciled_after_reboot"] == true
&& evidence["new_boot_id"] == envelope.boot_id,
"REBOOT_RESULT_BINDING"
);
let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)")
.bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?;
ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED");
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
let deadline =
reboot_deadline(&job.action, Some(reserved)).context("REBOOT_FIXED_DEADLINE_REQUIRED")?;
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1")
.bind(tenant).bind(job.job_id.to_string()).bind(grant.to_string()).fetch_optional(&mut **tx).await?;
ensure!(
original
.as_deref()
.and_then(|v| Uuid::parse_str(v).ok())
.is_some_and(|old| !old.is_nil() && old != boot),
"REBOOT_ORIGINAL_BOOT_REQUIRED"
);
let receipt = evidence["inventory_message_id"]
.as_str()
.and_then(|v| Uuid::parse_str(v).ok())
.filter(|id| !id.is_nil())
.context("REBOOT_PERSISTED_INVENTORY_REQUIRED")?;
let (_, received) = reboot_inventory(
tx,
tenant,
agent,
boot,
&RebootWindow {
reserved_at: reserved,
deadline,
now,
require_fresh: true,
},
Some(receipt),
)
.await?
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
let verification_id = evidence
.get("verification_inventory_message_id")
.and_then(Value::as_str)
.map(Uuid::parse_str)
.transpose()?;
let verification_time = evidence.get("verification_inventory_received_at");
let verification = if verification_id.is_some()
|| verification_time.is_some()
|| received <= now - Duration::minutes(5)
{
let id = verification_id
.filter(|id| !id.is_nil())
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id))
.await?
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
ensure!(
serde_json::from_value::<chrono::DateTime<Utc>>(
verification_time.cloned().unwrap_or(Value::Null)
)
.ok()
== Some(time),
"REBOOT_VERIFICATION_TIME_BINDING"
);
(id, time)
} else {
(receipt, received)
};
for (key, actual) in [
("reboot_reserved_at", reserved),
("reconnect_deadline", deadline),
("inventory_received_at", received),
] {
ensure!(
serde_json::from_value::<chrono::DateTime<Utc>>(evidence[key].clone()).ok()
== Some(actual),
"REBOOT_SERVER_TIME_BINDING"
);
}
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())")
.bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot,
"connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline,
"inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1}))
.execute(&mut **tx).await?;
Ok(())
}
pub async fn reconcile(
state: &crate::State,
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
r: &SyncRequest,
) -> Result<SyncResponse> {
ensure!(
r.attempts.len() <= 256 && r.durable_message_ids.len() <= 256,
"sync count bound"
);
let request_id = optional_id(&r.request_id)?.context("sync request id required")?;
let journal_id = optional_id(&r.journal_id)?.context("journal identity required")?;
let helper_journal = optional_id(&r.helper_journal_id)?;
let boot = Uuid::parse_str(&e.boot_id)?;
let checkpoint = i64::try_from(r.checkpoint)?;
let helper_checkpoint = i64::try_from(r.helper_checkpoint)?;
let request_hash = hash(&r.encode_to_vec());
if let Some(cached)=sqlx::query("SELECT request_hash,connection_generation,payload,signature FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2").bind(agent).bind(request_id).fetch_optional(&mut **tx).await?{
ensure!(cached.get::<String,_>("request_hash")==request_hash,"sync request id conflict");
ensure!(cached.get::<i64,_>("connection_generation")==i64::try_from(e.connection_generation)?,"old sync request: recover via previous_request_id in a fresh request");
return Ok(SyncResponse{decision_bytes:cached.get("payload"),signature:cached.get("signature")});
}
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,sequence_regressed,sync_recovery_required,revoked_at FROM agent_installations WHERE agent_id=$1 FOR UPDATE").bind(agent).fetch_one(&mut **tx).await?;
ensure!(
device
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
.is_none()
&& device.get::<i64, _>("connection_generation")
== i64::try_from(e.connection_generation)?
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(boot),
"sync channel fenced"
);
let previous=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE agent_id=$1").bind(agent).fetch_optional(&mut **tx).await?;
let mut reason = if r.journal_complete {
"CONSISTENT"
} else {
"JOURNAL_INCOMPLETE"
}
.to_string();
let mut chain_valid = true;
if device.get::<bool, _>("sync_recovery_required") {
reason = "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED".into();
chain_valid = false;
}
if previous.is_none() {
let history: bool =
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE agent_id=$1)")
.bind(agent)
.fetch_one(&mut **tx)
.await?;
if history {
reason = "SERVER_STATE_ROLLBACK".into();
chain_valid = false;
}
}
if device.get::<i64, _>("assignment_version") != i64::try_from(r.assignment_version)? {
reason = "ASSIGNMENT_MISMATCH".into();
chain_valid = false;
}
if device.get::<bool, _>("sequence_regressed") {
reason = "JOURNAL_SEQUENCE_REGRESSION".into();
chain_valid = false;
}
let last_applied = optional_id(&r.previous_decision_id)?;
let pending_request = optional_id(&r.previous_request_id)?;
let committed_pending = if let Some(pending) = pending_request {
sqlx::query_scalar("SELECT decision_id FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2 AND journal_id=$3").bind(agent).bind(pending).bind(journal_id).fetch_optional(&mut **tx).await?
} else {
None
};
let previous_decision = previous_anchor(last_applied, pending_request, committed_pending);
if pending_request.is_some() && previous_decision.is_none() {
reason = "SERVER_CHECKPOINT_MISSING".into();
chain_valid = false;
}
if let Some(old) = &previous {
if old.get::<Uuid, _>("journal_id") != journal_id
|| checkpoint < old.get::<i64, _>("checkpoint")
{
reason = "JOURNAL_ROLLBACK_OR_REPLACEMENT".into();
chain_valid = false;
}
if previous_decision != Some(old.get::<Uuid, _>("decision_id"))
&& reason != "SERVER_CHECKPOINT_MISSING"
{
reason = "CHECKPOINT_CHAIN_MISMATCH".into();
chain_valid = false;
}
if let Some(helper) = helper_journal
&& (old
.get::<Option<Uuid>, _>("helper_journal_id")
.is_some_and(|id| id != helper)
|| helper_checkpoint < old.get::<i64, _>("helper_checkpoint"))
{
reason = "HELPER_JOURNAL_ROLLBACK".into();
chain_valid = false;
}
} else if previous_decision.is_some() || !r.helper_previous_decision_id.is_empty() {
reason = "SERVER_CHECKPOINT_MISSING".into();
chain_valid = false;
}
let mut helper_ready = helper_journal.is_some();
if helper_ready && !r.helper_previous_decision_id.is_empty() {
let previous_helper = sqlx::query_scalar::<_, Vec<u8>>(
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
)
.bind(agent)
.bind(Uuid::parse_str(&r.helper_previous_decision_id)?)
.fetch_optional(&mut **tx)
.await?;
if let Some(bytes) = previous_helper {
let old: ReconcileDecisionPayload = serde_json::from_slice(&bytes)?;
if old.helper_journal_id != helper_journal
|| old.helper_checkpoint > r.helper_checkpoint
{
helper_ready = false;
reason = "HELPER_CHECKPOINT_MISMATCH".into();
chain_valid = false;
}
} else {
helper_ready = false;
reason = "HELPER_CHECKPOINT_MISSING".into();
chain_valid = false;
}
} else if helper_ready
&& previous
.as_ref()
.is_some_and(|old| old.get::<Option<Uuid>, _>("helper_journal_id").is_some())
{
helper_ready = false;
reason = "HELPER_CHECKPOINT_REQUIRED".into();
chain_valid = false;
}
let previous_payload: Option<Vec<u8>> = if let Some(old) = &previous {
sqlx::query_scalar(
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
)
.bind(agent)
.bind(old.get::<Uuid, _>("decision_id"))
.fetch_optional(&mut **tx)
.await?
} else {
None
};
let previous_update_decisions: Vec<AttemptDecision> = previous_payload
.as_deref()
.map(serde_json::from_slice::<ReconcileDecisionPayload>)
.transpose()?
.map(|p| p.attempts)
.unwrap_or_default();
let mut locals = BTreeMap::new();
for local in &r.attempts {
let id = Uuid::parse_str(&local.job_id)?;
ensure!(
!id.is_nil() && locals.insert(id, local).is_none(),
"duplicate local attempt"
);
Uuid::parse_str(&local.attempt_id)?;
ensure!(
digest(&local.job_payload_hash)
&& matches!(
local.consumption.as_str(),
"unconsumed" | "consumed" | "uncertain"
),
"invalid attempt summary"
);
i64::try_from(local.revision)?;
if !local.update_handoff_bytes.is_empty() {
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
}
ensure!(
matches!(
local.state.as_str(),
"queued"
| "accepted"
| "requesting"
| "running"
| "verifying"
| "awaiting_reboot"
| "unknown"
| "succeeded"
| "failed"
| "cancelled"
| "expired"
),
"invalid local state"
);
ensure!(
local.outcome_bytes.len() <= 512 * 1024,
"local outcome bound"
);
if !local.outcome_bytes.is_empty() {
ensure!(
digest(&local.outcome_sha256) && hash(&local.outcome_bytes) == local.outcome_sha256,
"local outcome hash"
);
let parsed: ReconciledOutcome = serde_json::from_slice(&local.outcome_bytes)?;
ensure!(
canonical_outcome_bytes(&parsed)? == local.outcome_bytes,
"noncanonical local outcome bytes"
);
ensure!(parsed.state == local.state, "local outcome state mismatch");
}
}
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
if rows.len() > 256 {
reason = "SYNC_SERVER_JOB_LIMIT".into();
chain_valid = false;
}
let mut server = BTreeMap::new();
for row in rows {
server.insert(row.get::<Uuid, _>("job_id"), row);
}
if previous.is_none() {
let missing:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND NOT(job_id=ANY($2)))").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_one(&mut **tx).await?;
if missing {
reason = "JOURNAL_HISTORY_MISSING".into();
chain_valid = false;
}
}
let mut ready = chain_valid && r.journal_complete;
let mut decisions = Vec::new();
let mut seen = BTreeSet::new();
for (job_id, local) in locals {
let attempt = Uuid::parse_str(&local.attempt_id)?;
let prior_hash = if local.outcome_sha256.is_empty() {
None
} else {
ensure!(digest(&local.outcome_sha256), "outcome digest");
Some(local.outcome_sha256.clone())
};
let mut decision = AttemptDecision {
update_inflight: None,
job_id,
attempt_id: attempt,
job_payload_hash: local.job_payload_hash.clone(),
grant_id: optional_id(&local.grant_id)?,
authorization_id: None,
plan_hash: None,
action: String::new(),
execution_identity: String::new(),
revision: local.revision,
outcome_state: "unknown".into(),
previous_outcome_sha256: prior_hash.clone(),
outcome_sha256: None,
outcome_bytes: vec![],
original_boot_id: optional_id(&local.original_boot_id)?,
unlock: false,
reason: "MISSING_SERVER_JOB".into(),
};
let Some(row) = server.get(&job_id) else {
sqlx::query(
"UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1",
)
.bind(agent)
.execute(&mut **tx)
.await?;
ready = false;
reason = "MISSING_SERVER_JOB".into();
decisions.push(decision);
continue;
};
seen.insert(job_id);
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
decision.action = payload.action.name().into();
decision.execution_identity = payload.execution_identity.clone();
if row.get::<Uuid, _>("attempt_id") != attempt
|| row.get::<String, _>("payload_hash") != local.job_payload_hash
|| hash(&row.get::<Vec<u8>, _>("payload")) != local.job_payload_hash
|| payload.tenant_id != tenant
|| payload.agent_id != agent
|| payload.asset_id != device.get::<Uuid, _>("asset_id")
|| payload.enrollment_generation != device.get::<i64, _>("enrollment_generation") as u64
|| payload.assignment_version != device.get::<i64, _>("assignment_version") as u64
{
sqlx::query(
"UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1",
)
.bind(agent)
.execute(&mut **tx)
.await?;
ready = false;
decision.reason = "ATTEMPT_BINDING_MISMATCH".into();
decisions.push(decision);
continue;
}
let missing_helper =
crate::root_policy::needs_helper(&payload.execution_identity) && !helper_ready;
if missing_helper {
ready = false;
}
let grant = sqlx::query(
"SELECT grant_id,payload,signature,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
)
.bind(job_id)
.bind(attempt)
.fetch_optional(&mut **tx)
.await?;
if let Some(grant) = &grant {
let p: ExecutionPermitPayload =
serde_json::from_slice(&grant.get::<Vec<u8>, _>("payload"))?;
decision.authorization_id = Some(p.authorization_id);
decision.plan_hash = Some(p.plan_hash);
}
let reserved = row.get::<bool, _>("start_reserved");
let server_state: String = row.get("state");
if clean_unstarted(local, reserved)
&& matches!(server_state.as_str(), "queued" | "accepted")
{
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=COALESCE(journal_proof,$2),revision=GREATEST(revision,$3) WHERE job_id=$1").bind(job_id).bind(format!("sync:{journal_id}:{}",r.checkpoint)).bind(i64::try_from(local.revision)?).execute(&mut **tx).await?;
sqlx::query(
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
)
.bind(job_id)
.execute(&mut **tx)
.await?;
decision.outcome_state = "accepted".into();
decision.reason = "ACCEPTED_NOT_STARTED".into();
decision.revision = (row.get::<i64, _>("revision") as u64).max(local.revision);
decisions.push(decision);
continue;
}
let grant_matches = grant
.as_ref()
.is_some_and(|g| Some(g.get::<Uuid, _>("grant_id")) == decision.grant_id);
let unsafe_claim =
local.state == "requesting" || (!grant_matches && local.consumption != "unconsumed");
if unsafe_claim {
if unsettled(&server_state) {
mark_unknown(tx, job_id, "SYNC_GRANT_OR_EFFECT_UNRESOLVED").await?;
}
ready = false;
decision.reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
decisions.push(decision);
continue;
}
if !local.update_handoff_bytes.is_empty() {
use rmm_domain::agent_update as update;
ensure!(
chain_valid
&& r.journal_complete
&& reserved
&& grant_matches
&& helper_journal.is_some()
&& matches!(
server_state.as_str(),
"queued" | "accepted" | "running" | "verifying"
)
&& row.get::<Option<Value>, _>("result").is_none()
&& local.state == "running"
&& local.consumption == "consumed"
&& local.outcome_bytes.is_empty()
&& local.outcome_sha256.is_empty()
&& local.result.is_none()
&& local.result_envelope.is_none(),
"UPDATE_INFLIGHT_NOT_ORIGINAL_ACTIVE_ATTEMPT"
);
let proof = update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let key = update::job_health_public_key(&payload).map_err(anyhow::Error::msg)?;
let (header, _) =
update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
let grant = grant.as_ref().context("UPDATE_ORIGINAL_GRANT_REQUIRED")?;
let permit_bytes: Vec<u8> = grant.get("payload");
let signature: Vec<u8> = grant.get("signature");
let permit: ExecutionPermitPayload = serde_json::from_slice(&permit_bytes)?;
let job_bytes: Vec<u8> = row.get("payload");
let original = update::OriginalHelperRequest {
job_bytes: &job_bytes,
permit_bytes: &permit_bytes,
signature: &signature,
plan_hash: &permit.plan_hash,
authorization_id: permit.authorization_id,
};
let authorized_plan:Option<Uuid>=sqlx::query_scalar("SELECT plan_id FROM authorizations WHERE tenant_id=$1 AND authorization_id=$2 AND plan_hash=$3")
.bind(tenant).bind(permit.authorization_id).bind(&permit.plan_hash).fetch_optional(&mut **tx).await?;
let original_sync: Vec<u8> = sqlx::query_scalar(
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
)
.bind(agent)
.bind(header.previous_decision_id)
.fetch_optional(&mut **tx)
.await?
.context("UPDATE_ORIGINAL_READY_SYNC_REQUIRED")?;
let original_sync: ReconcileDecisionPayload = serde_json::from_slice(&original_sync)?;
ensure!(
original_sync.ready
&& original_sync.helper_ready
&& original_sync.agent_id == agent
&& original_sync.tenant_id == tenant
&& original_sync.journal_id == header.agent_journal_id
&& original_sync.helper_journal_id == Some(header.helper_journal_id)
&& original_sync.helper_checkpoint == header.helper_checkpoint
&& original_sync.boot_id == header.boot_id
&& original_sync.connection_generation == header.connection_generation
&& original_sync.issued_at <= proof.delegation.prepared_at
&& proof.delegation.prepared_at < original_sync.valid_until,
"UPDATE_ORIGINAL_READY_SYNC_BINDING"
);
let seconds = u64::try_from(Utc::now().timestamp())?;
ensure!(
update::handoff_matches_job(&header, &payload)
&& header.boot_id == boot
&& header.attempt.plan_id
== authorized_plan.context("UPDATE_ORIGINAL_PLAN_REQUIRED")?
&& permit.agent_update_plan_id == Some(header.attempt.plan_id)
&& header.agent_journal_id == journal_id
&& Some(header.helper_journal_id) == helper_journal
&& header.helper_checkpoint <= r.helper_checkpoint
&& header.connection_generation <= e.connection_generation
&& header.attempt.grant_id == permit.grant_id
&& header.attempt.authorization_id == permit.authorization_id
&& header.attempt.plan_sha256 == permit.plan_hash
&& header.attempt.job_payload_sha256 == permit.job_payload_hash
&& header.attempt.original_request_sha256
== hash(&serde_json::to_vec(&original)?)
&& permit.action == "update_agent"
&& permit.max_runtime_seconds == 600
&& permit.key_id == state.job_signer.key_id
&& header.finish_before
== header
.consumed_at
.checked_add(600)
.context("UPDATE_DEADLINE_OVERFLOW")?
&& header.consumed_at <= seconds
&& seconds < header.finish_before
&& proof.delegation.prepared_at >= permit.not_before
&& proof.delegation.prepared_at < permit.start_before
&& local.original_boot_id == boot.to_string(),
"UPDATE_INFLIGHT_ORIGINAL_BINDING"
);
let ack = update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
if let Some(old) = previous_update_decisions
.iter()
.find(|d| d.job_id == job_id && d.attempt_id == attempt)
.and_then(|d| d.update_inflight.as_ref())
{
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
ensure!(
old.handoff_sha256 == ack.handoff_sha256
&& old.delegation_sha256 == ack.delegation_sha256
&& old.owner_nonce == ack.owner_nonce
&& old.finish_before == ack.finish_before
&& (hash(&decoded.handoff_record) == old.head_sha256
|| decoded
.events
.iter()
.any(|bytes| hash(bytes) == old.head_sha256)),
"UPDATE_INFLIGHT_PREFIX_REGRESSION"
);
}
// Retain the already fully validated original public context once
// in this same control-lock transaction. The clock bridge never
// infers the original generation/deadline from a historical grant.
crate::agent_time::retain_validated_updater_context(tx, &header, &proof.delegation)
.await?;
let revision = local.revision.max(row.get::<i64, _>("revision") as u64);
sqlx::query(
"UPDATE jobs SET state='running',revision=GREATEST(revision,$2) WHERE job_id=$1",
)
.bind(job_id)
.bind(i64::try_from(revision)?)
.execute(&mut **tx)
.await?;
decision.revision = revision;
decision.outcome_state = "running".into();
decision.update_inflight = Some(ack);
decision.reason = "AGENT_UPDATE_INFLIGHT".into();
decision.unlock = false;
ready = false;
decisions.push(decision);
continue;
}
// A free script with no bound postcondition never becomes successful by
// merely asserting a new state after unknown effect.
let manual = server_state == "unknown"
&& matches!(
payload.action,
Action::RunScript {
expected_stdout: None,
..
}
);
let embedded =
local
.result_envelope
.as_ref()
.and_then(|envelope| match &envelope.payload {
Some(Payload::JobResult(result)) => Some(result),
_ => None,
});
if let Some(result) = local.result.as_ref().or(embedded) {
ensure!(
result.job_id == local.job_id
&& result.attempt_id == local.attempt_id
&& result.revision == local.revision
&& result.state == local.state
&& result.grant_id == local.grant_id,
"sync result binding"
);
if !manual {
let evidence = local
.result_envelope
.as_ref()
.context("original durable result envelope required")?;
ensure!(
valid_envelope(evidence, &agent.to_string(), evidence.connection_generation)
&& evidence.message_id == local.result_message_id
&& embedded == Some(result),
"original result envelope binding"
);
let observed = chrono::DateTime::from_timestamp_millis(evidence.observed_at)
.context("result timestamp")?;
ensure!(
observed <= Utc::now() + Duration::seconds(45),
"future result timestamp"
);
crate::job_gateway::event(tx, tenant, agent, evidence).await?;
}
}
let mut final_row = sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1")
.bind(job_id)
.fetch_one(&mut **tx)
.await?;
// A reboot's postcondition is independently tied to persisted current-
// boot inventory and the original boot in the durable reservation audit.
let mut reboot_reconciled = false;
let mut reboot_witness = None;
if matches!(payload.action, Action::Reboot { .. })
&& grant_matches
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
{
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1").bind(tenant).bind(job_id.to_string()).bind(&local.grant_id).fetch_optional(&mut **tx).await?;
let reserved_at = grant
.as_ref()
.map(|g| g.get::<chrono::DateTime<Utc>, _>("created_at"));
let deadline = reboot_deadline(&payload.action, reserved_at);
let inventory = if let (Some(reserved), Some(until)) = (reserved_at, deadline) {
let existing_receipt = if final_row.get::<String, _>("state") == "succeeded" {
final_row
.get::<Option<Value>, _>("result")
.and_then(|value| {
value["evidence"]["inventory_message_id"]
.as_str()
.and_then(|v| Uuid::parse_str(v).ok())
})
} else {
None
};
reboot_inventory(
tx,
tenant,
agent,
boot,
&RebootWindow {
reserved_at: reserved,
deadline: until,
now: Utc::now(),
require_fresh: existing_receipt.is_none(),
},
existing_receipt,
)
.await?
} else {
None
};
if let (Some(original), Some((inventory, received_at)), Some(reserved), Some(until)) =
(original, inventory, reserved_at, deadline)
&& original != e.boot_id
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
{
reboot_witness = Some((original, inventory));
let verification =
reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?;
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()};
let mut evidence = e.clone();
evidence.message_id = Uuid::new_v4().to_string();
evidence.payload = Some(Payload::JobResult(result));
if unsettled(&final_row.get::<String, _>("state")) {
crate::job_gateway::event(tx, tenant, agent, &evidence).await?;
reboot_reconciled = true;
final_row =
sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1")
.bind(job_id)
.fetch_one(&mut **tx)
.await?;
}
}
}
let terminal: String = final_row.get("state");
if matches!(payload.action, Action::Reboot { .. })
&& grant_matches
&& terminal == "succeeded"
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
&& let Some(value) = final_row.get::<Option<Value>, _>("result")
&& let Some((original, inventory)) = reboot_witness
&& Uuid::parse_str(&original).ok() == decision.original_boot_id
&& decision.original_boot_id.is_some()
&& value["evidence"]["postconditions_passed"] == json!(true)
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
&& value["evidence"]["new_boot_id"]
.as_str()
.is_some_and(|id| id == e.boot_id)
&& value["evidence"]["inventory_message_id"] == json!(inventory)
{
reboot_reconciled = true;
}
decision.revision = final_row.get::<i64, _>("revision") as u64;
decision.outcome_state = terminal.clone();
if !unsettled(&terminal) && !manual {
if let Some(value) = final_row.get::<Option<Value>, _>("result") {
let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?;
let hash = hash(&bytes);
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled {
decision.outcome_sha256 = Some(hash);
decision.unlock = local.consumption == "consumed" && grant_matches;
if reboot_reconciled {
decision.outcome_bytes = bytes;
}
decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into();
} else {
ready = false;
decision.reason = "LOCAL_OUTCOME_MISMATCH".into();
}
} else if local.consumption == "unconsumed" && !reserved {
decision.reason = "DEFINITIVE_NONSTART".into();
} else {
ready = false;
decision.reason = "DEFINITIVE_OUTCOME_MISSING".into();
}
} else {
if unsettled(&terminal) {
mark_unknown(tx, job_id, "SYNC_EFFECT_UNRESOLVED").await?;
}
ready = false;
decision.outcome_state = "unknown".into();
decision.reason = if manual {
"MANUAL_RECONCILIATION_REQUIRED"
} else {
"EFFECT_UNRESOLVED"
}
.into();
}
if missing_helper {
decision.unlock = false;
decision.reason = "HELPER_JOURNAL_REQUIRED".into();
}
decisions.push(decision);
}
for (id, row) in server {
if !seen.contains(&id)
&& (row.get::<bool, _>("start_reserved")
|| row.get::<String, _>("state") != "queued"
|| row
.get::<Option<chrono::DateTime<Utc>>, _>("acknowledged_at")
.is_some())
{
if unsettled(&row.get::<String, _>("state")) {
mark_unknown(tx, id, "SYNC_LOCAL_ATTEMPT_MISSING").await?;
}
ready = false;
reason = "LOCAL_ATTEMPT_MISSING".into();
}
}
if !ready && reason == "CONSISTENT" {
reason = "ATTEMPT_RECONCILIATION_REQUIRED".into();
}
helper_ready &= ready;
if !chain_valid {
for decision in &mut decisions {
decision.unlock = false;
}
}
let mut receipts = Vec::new();
for raw in &r.durable_message_ids {
let id = Uuid::parse_str(raw)?;
let persisted:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM job_receipts WHERE agent_id=$1 AND message_id=$2 UNION ALL SELECT 1 FROM observations WHERE agent_id=$1 AND message_id=$2)").bind(agent).bind(id).fetch_one(&mut **tx).await?;
if persisted {
receipts.push(id)
}
}
receipts.sort();
receipts.dedup();
let now = Utc::now();
let decision = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id,
request_hash: request_hash.clone(),
tenant_id: tenant,
asset_id: device.get("asset_id"),
agent_id: agent,
enrollment_generation: device.get::<i64, _>("enrollment_generation") as u64,
assignment_version: device.get::<i64, _>("assignment_version") as u64,
connection_generation: e.connection_generation,
boot_id: boot,
journal_id,
checkpoint: r.checkpoint,
helper_journal_id: helper_journal,
helper_checkpoint: r.helper_checkpoint,
issued_at: now,
valid_until: now + Duration::seconds(60),
ready,
helper_ready,
reason,
attempts: decisions,
acknowledged_message_ids: receipts,
key_id: state.job_signer.key_id.clone(),
};
let bytes = encode(&decision)?;
ensure!(bytes.len() <= 512 * 1024, "sync decision bound");
let signature = state.job_signer.sign(&bytes)?;
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)").bind(tenant).bind(agent).bind(request_id).bind(decision.decision_id).bind(request_hash).bind(journal_id).bind(checkpoint).bind(i64::try_from(e.connection_generation)?).bind(&bytes).bind(&signature).execute(&mut **tx).await?;
if chain_valid {
sqlx::query("INSERT INTO agent_sync_state(tenant_id,agent_id,journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint) VALUES($1,$2,$3,$4,$5,$6,$7) ON CONFLICT(tenant_id,agent_id) DO UPDATE SET checkpoint=GREATEST(agent_sync_state.checkpoint,EXCLUDED.checkpoint),decision_id=EXCLUDED.decision_id,helper_journal_id=COALESCE(agent_sync_state.helper_journal_id,EXCLUDED.helper_journal_id),helper_checkpoint=GREATEST(agent_sync_state.helper_checkpoint,EXCLUDED.helper_checkpoint)").bind(tenant).bind(agent).bind(journal_id).bind(checkpoint).bind(decision.decision_id).bind(helper_journal).bind(helper_checkpoint).execute(&mut **tx).await?;
}
sqlx::query("UPDATE agent_installations SET sync_ready=$2,helper_sync_ready=$3,restore_blocked=NOT $2 WHERE agent_id=$1").bind(agent).bind(ready).bind(helper_ready).execute(&mut **tx).await?;
Ok(SyncResponse {
decision_bytes: bytes,
signature,
})
}
async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> {
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reboot_deadline_uses_only_the_fixed_plan_and_trusted_reservation() {
let reserved = Utc::now();
let action = Action::Reboot {
reason: "Explicit final Cloud VM acceptance".into(),
reconnect_timeout_seconds: 180,
dependency_group: "single-test-vm".into(),
};
assert_eq!(
reboot_deadline(&action, Some(reserved)),
Some(reserved + Duration::seconds(180))
);
assert_eq!(reboot_deadline(&action, None), None);
assert_eq!(
reboot_deadline(
&Action::Reboot {
reason: "test".into(),
reconnect_timeout_seconds: 0,
dependency_group: "single-test-vm".into()
},
Some(reserved)
),
None
);
assert_eq!(
reboot_deadline(&action, Some(chrono::DateTime::<Utc>::MAX_UTC)),
None
);
}
#[tokio::test]
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
async fn reboot_inventory_requires_trusted_timely_same_identity_and_boot() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await
.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await
.unwrap();
rmm_persistence::verify_application_role(&app)
.await
.unwrap();
let tenant = Uuid::new_v4();
let asset = Uuid::new_v4();
let agent = Uuid::new_v4();
let boot = Uuid::new_v4();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot deadline test')")
.bind(tenant)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets VALUES($1,$2)")
.bind(tenant)
.bind(asset)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)")
.bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
let now = Utc::now();
let reserved = now - Duration::seconds(200);
let deadline = reserved + Duration::seconds(180);
let window = RebootWindow {
reserved_at: reserved,
deadline,
now,
require_fresh: true,
};
let timely = Uuid::new_v4();
let late = Uuid::new_v4();
for (sequence, message, received) in [
(1, timely, deadline),
(2, late, deadline + Duration::microseconds(1)),
(3, Uuid::new_v4(), reserved),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&admin).await.unwrap();
}
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &window, None)
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &window, Some(late))
.await
.unwrap()
.is_none()
);
assert!(
reboot_inventory(&mut tx, tenant, Uuid::new_v4(), boot, &window, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_inventory(&mut tx, Uuid::new_v4(), agent, boot, &window, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_inventory(&mut tx, tenant, agent, Uuid::new_v4(), &window, None)
.await
.unwrap()
.is_none()
);
let later = RebootWindow {
now: now + Duration::minutes(6),
..window
};
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
.await
.unwrap()
.is_none()
);
// A terminal result reuses only its exact original timely receipt;
// later metadata cannot replace or extend that immutable proof.
let historical = RebootWindow {
require_fresh: false,
..later
};
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &historical, Some(timely))
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
let fresh = Uuid::new_v4();
let future = Uuid::new_v4();
for (sequence, message, received) in [
(4, fresh, later.now - Duration::seconds(1)),
(5, future, later.now + Duration::days(1)),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap();
}
// Delayed first verification keeps the original cutoff-bound ID and
// uses a DIFFERENT fresh same-boot receipt; late-only never suffices.
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
.await
.unwrap()
.is_none()
);
assert_eq!(
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None)
.await
.unwrap()
.map(|v| v.0),
Some(fresh)
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future))
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None)
.await
.unwrap()
.is_none()
);
let before_return = RebootWindow {
now: deadline - Duration::microseconds(1),
..window
};
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &before_return, None)
.await
.unwrap()
.is_none()
);
tx.rollback().await.unwrap();
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs WHERE tenant_id=$1)+(SELECT count(*) FROM outbox WHERE tenant_id=$1)+(SELECT count(*) FROM execution_permits WHERE tenant_id=$1)").bind(tenant).fetch_one(&admin).await.unwrap();
assert_eq!(effects, 0);
}
#[tokio::test]
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
async fn reboot_success_ingress_rejects_forged_late_proof_and_preserves_exact_replay() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await
.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await
.unwrap();
rmm_persistence::verify_application_role(&app)
.await
.unwrap();
let tenant = Uuid::new_v4();
let asset = Uuid::new_v4();
let agent = Uuid::new_v4();
let boot = Uuid::new_v4();
let old = Uuid::new_v4();
let job = Uuid::new_v4();
let attempt = Uuid::new_v4();
let operation = Uuid::new_v4();
let plan = Uuid::new_v4();
let authorization = Uuid::new_v4();
let grant = Uuid::new_v4();
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
let reserved = now - Duration::seconds(600);
let deadline = reserved + Duration::seconds(180);
let payload = JobPayload {
schema_version: 1,
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
job_id: job,
attempt_id: attempt,
action: Action::Reboot {
reason: "Isolated postcondition test; no actual reboot".into(),
reconnect_timeout_seconds: 180,
dependency_group: "single-test-vm".into(),
},
execution_identity: "root".into(),
not_before: reserved,
start_before: now + Duration::minutes(5),
max_runtime_seconds: 240,
max_output_bytes: 65536,
};
let bytes = serde_json::to_vec(&payload).unwrap();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot ingress test')")
.bind(tenant)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets VALUES($1,$2)")
.bind(tenant)
.bind(asset)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
.bind(tenant)
.bind(operation)
.bind(authorization)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)")
.bind(tenant)
.bind(grant)
.bind(job)
.bind(attempt)
.bind(b"{}".as_slice())
.bind(vec![0u8; 64])
.bind(reserved)
.execute(&admin)
.await
.unwrap();
sqlx::query(
"INSERT INTO audit_events VALUES($1,$2,'isolated-test','start_reserved',$3,$4)",
)
.bind(tenant)
.bind(Uuid::new_v4())
.bind(json!({"job_id":job,"grant_id":grant,"boot_id":old}))
.bind(reserved)
.execute(&admin)
.await
.unwrap();
let timely = Uuid::new_v4();
let late = Uuid::new_v4();
let fresh = Uuid::new_v4();
let wrong_boot = Uuid::new_v4();
let future = Uuid::new_v4();
for (sequence, message, received) in [
(1, timely, deadline),
(2, late, deadline + Duration::microseconds(1)),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
}
for (sequence, message, received, observed_boot) in [
(3, fresh, now - Duration::seconds(1), boot),
(4, wrong_boot, now - Duration::seconds(1), old),
(5, future, now + Duration::days(1), boot),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
}
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline,
"verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)});
let mut result = JobResult {
job_id: job.to_string(),
attempt_id: attempt.to_string(),
grant_id: grant.to_string(),
revision: 1,
state: "succeeded".into(),
evidence_json: evidence.to_string(),
..Default::default()
};
let mut envelope = Envelope {
message_id: Uuid::new_v4().to_string(),
agent_id: agent.to_string(),
connection_generation: 1,
boot_id: boot.to_string(),
payload: Some(Payload::JobResult(result.clone())),
..Default::default()
};
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
assert_eq!(
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
.bind(job)
.fetch_one(&mut *tx)
.await
.unwrap(),
"unknown"
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
.fetch_one(&mut *tx)
.await
.unwrap(),
0
);
let mut good = evidence.clone();
good["inventory_message_id"] = json!(timely);
for (id, time) in [
(Some(wrong_boot), Some(now - Duration::seconds(1))),
(Some(future), Some(now + Duration::days(1))),
(None, None),
(Some(fresh), Some(now)),
] {
let mut bad = good.clone();
bad["verification_inventory_message_id"] = json!(id);
bad["verification_inventory_received_at"] = json!(time);
result.evidence_json = bad.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
}
envelope.connection_generation = 2;
result.evidence_json = good.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
envelope.connection_generation = 1;
let mut false_clock = good.clone();
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
result.evidence_json = false_clock.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
result.evidence_json = good.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
.bind(job)
.fetch_one(&mut *tx)
.await
.unwrap(),
"succeeded"
);
// The original timely receipt survives later inventories and wall time.
let later = RebootWindow {
reserved_at: reserved,
deadline,
now: now + Duration::minutes(10),
require_fresh: false,
};
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(timely))
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
.await
.unwrap()
.is_none()
);
// Exact durable-message replay ACKs the existing receipt, without
// accepting a new result or extending its original evidence window.
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
.fetch_one(&mut *tx)
.await
.unwrap(),
1
);
let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'")
.bind(tenant).fetch_one(&mut *tx).await.unwrap();
assert_eq!(audit["inventory_message_id"], json!(timely));
assert_eq!(audit["inventory_received_at"], json!(deadline));
assert_eq!(audit["verification_inventory_message_id"], json!(fresh));
assert_eq!(
audit["verification_inventory_received_at"],
json!(now - Duration::seconds(1))
);
assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1);
tx.rollback().await.unwrap();
}
#[test]
fn uncertain_request_resolves_only_to_a_known_checkpoint_anchor() {
let applied = Uuid::new_v4();
let pending = Uuid::new_v4();
let committed = Uuid::new_v4();
// Transport cut before server commit: retain the applied anchor.
assert_eq!(
previous_anchor(Some(applied), Some(pending), None),
Some(applied)
);
// Server committed but its reply was lost: use the newer checkpoint,
// even while the agent also retains its older applied witness.
assert_eq!(
previous_anchor(Some(applied), Some(pending), Some(committed)),
Some(committed)
);
// An absent pending request cannot invent an anchor for a legacy
// request that discarded its last applied decision.
assert_eq!(previous_anchor(None, Some(pending), None), None);
assert_eq!(
previous_anchor(None, Some(pending), Some(committed)),
Some(committed)
);
assert_eq!(previous_anchor(Some(applied), None, None), Some(applied));
}
#[test]
fn accepted_is_not_a_new_attempt_or_grant() {
let mut local = SyncAttempt {
update_handoff_bytes: Vec::new(),
state: "accepted".into(),
consumption: "unconsumed".into(),
..Default::default()
};
assert!(clean_unstarted(&local, false));
assert!(!clean_unstarted(&local, true));
local.state = "requesting".into();
assert!(!clean_unstarted(&local, false));
local.state = "running".into();
assert!(!clean_unstarted(&local, false));
}
}