1301 lines
58 KiB
Rust
1301 lines
58 KiB
Rust
//! Authoritative reconciliation only: this module never inserts jobs, attempts,
|
|
//! outbox entries or execution permits. A signed decision is not start authority.
|
|
use anyhow::{Context, Result, ensure};
|
|
use chrono::{Duration, Utc};
|
|
use prost::Message;
|
|
use rmm_domain::{jobs::*, reconciliation::*};
|
|
use rmm_protocol::*;
|
|
use serde_json::{Value, json};
|
|
use sqlx::{Postgres, Row, Transaction};
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use uuid::Uuid;
|
|
fn optional_id(raw: &str) -> Result<Option<Uuid>> {
|
|
if raw.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
let id = Uuid::parse_str(raw)?;
|
|
ensure!(!id.is_nil(), "nil sync identity");
|
|
Ok(Some(id))
|
|
}
|
|
}
|
|
fn digest(raw: &str) -> bool {
|
|
raw.len() == 64
|
|
&& raw
|
|
.bytes()
|
|
.all(|c| c.is_ascii_digit() || b"abcdef".contains(&c))
|
|
}
|
|
fn outcome(state: &str, value: &Value) -> Result<ReconciledOutcome> {
|
|
Ok(ReconciledOutcome {
|
|
state: state.into(),
|
|
stdout: serde_json::from_value(value["stdout"].clone())?,
|
|
stderr: serde_json::from_value(value["stderr"].clone())?,
|
|
exit_code: serde_json::from_value(value["exit_code"].clone())?,
|
|
output_truncated: value["output_truncated"]
|
|
.as_bool()
|
|
.context("output bound evidence missing")?,
|
|
reboot_required: value["reboot_required"]
|
|
.as_bool()
|
|
.context("reboot evidence missing")?,
|
|
evidence: value["evidence"].clone(),
|
|
error_code: value["error_code"]
|
|
.as_str()
|
|
.context("error evidence missing")?
|
|
.into(),
|
|
})
|
|
}
|
|
fn unsettled(state: &str) -> bool {
|
|
!matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
|
|
}
|
|
fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool {
|
|
!reserved
|
|
&& local.consumption == "unconsumed"
|
|
&& local.grant_id.is_empty()
|
|
&& matches!(local.state.as_str(), "queued" | "accepted")
|
|
}
|
|
/// A missing reply and a request that never committed are different cases.
|
|
/// A committed pending request is the newer anchor. Otherwise only the last
|
|
/// applied decision can anchor the chain; the caller still compares it with
|
|
/// the authoritative head and performs all journal/generation checks.
|
|
fn previous_anchor(
|
|
last_applied: Option<Uuid>,
|
|
pending_request: Option<Uuid>,
|
|
committed_pending: Option<Uuid>,
|
|
) -> Option<Uuid> {
|
|
if pending_request.is_some() {
|
|
committed_pending.or(last_applied)
|
|
} else {
|
|
last_applied
|
|
}
|
|
}
|
|
/// The server's durable grant reservation precedes the helper's saved start.
|
|
/// Using that trusted timestamp is conservative: no device clock or late
|
|
/// reconnect may extend the fixed, approved reconnect window.
|
|
fn reboot_deadline(
|
|
action: &Action,
|
|
reserved_at: Option<chrono::DateTime<Utc>>,
|
|
) -> Option<chrono::DateTime<Utc>> {
|
|
let Action::Reboot {
|
|
reconnect_timeout_seconds,
|
|
..
|
|
} = action
|
|
else {
|
|
return None;
|
|
};
|
|
if !action.validate() {
|
|
return None;
|
|
}
|
|
reserved_at?.checked_add_signed(Duration::seconds(i64::from(*reconnect_timeout_seconds)))
|
|
}
|
|
struct RebootWindow {
|
|
reserved_at: chrono::DateTime<Utc>,
|
|
deadline: chrono::DateTime<Utc>,
|
|
now: chrono::DateTime<Utc>,
|
|
require_fresh: bool,
|
|
}
|
|
async fn reboot_fresh_inventory(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
boot: Uuid,
|
|
now: chrono::DateTime<Utc>,
|
|
receipt: Option<Uuid>,
|
|
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
|
|
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1")
|
|
.bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt)
|
|
.fetch_optional(&mut **tx).await?)
|
|
}
|
|
async fn reboot_inventory(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
boot: Uuid,
|
|
window: &RebootWindow,
|
|
existing_receipt: Option<Uuid>,
|
|
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
|
|
// Already-persisted definitive success keeps its original receipt. It need
|
|
// not be younger than five minutes again, but its original server receipt
|
|
// time must still prove the approved reconnect window and current boot.
|
|
let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1")
|
|
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt)
|
|
.fetch_optional(&mut **tx).await?;
|
|
// Verification may be delayed by local bootstrap readiness. Never move
|
|
// the original reconnect cutoff: retain the actual timely receipt and
|
|
// independently demand fresh evidence of the SAME current boot.
|
|
if timely.is_some()
|
|
&& window.require_fresh
|
|
&& reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None)
|
|
.await?
|
|
.is_none()
|
|
{
|
|
return Ok(None);
|
|
}
|
|
Ok(timely)
|
|
}
|
|
/// First ingress of success needs the same independent, timely receipt as an
|
|
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
|
|
pub(crate) async fn verify_reboot_success(
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
envelope: &Envelope,
|
|
job: &JobPayload,
|
|
result: &JobResult,
|
|
evidence: &Value,
|
|
) -> Result<()> {
|
|
let boot = Uuid::parse_str(&envelope.boot_id)?;
|
|
let grant = Uuid::parse_str(&result.grant_id)?;
|
|
let now = Utc::now();
|
|
ensure!(
|
|
job.tenant_id == tenant
|
|
&& job.agent_id == agent
|
|
&& job.job_id.to_string() == result.job_id
|
|
&& job.attempt_id.to_string() == result.attempt_id
|
|
&& evidence["postconditions_passed"] == true
|
|
&& evidence["reconciled_after_reboot"] == true
|
|
&& evidence["new_boot_id"] == envelope.boot_id,
|
|
"REBOOT_RESULT_BINDING"
|
|
);
|
|
let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)")
|
|
.bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?;
|
|
ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED");
|
|
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
|
|
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
|
|
let deadline =
|
|
reboot_deadline(&job.action, Some(reserved)).context("REBOOT_FIXED_DEADLINE_REQUIRED")?;
|
|
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1")
|
|
.bind(tenant).bind(job.job_id.to_string()).bind(grant.to_string()).fetch_optional(&mut **tx).await?;
|
|
ensure!(
|
|
original
|
|
.as_deref()
|
|
.and_then(|v| Uuid::parse_str(v).ok())
|
|
.is_some_and(|old| !old.is_nil() && old != boot),
|
|
"REBOOT_ORIGINAL_BOOT_REQUIRED"
|
|
);
|
|
let receipt = evidence["inventory_message_id"]
|
|
.as_str()
|
|
.and_then(|v| Uuid::parse_str(v).ok())
|
|
.filter(|id| !id.is_nil())
|
|
.context("REBOOT_PERSISTED_INVENTORY_REQUIRED")?;
|
|
let (_, received) = reboot_inventory(
|
|
tx,
|
|
tenant,
|
|
agent,
|
|
boot,
|
|
&RebootWindow {
|
|
reserved_at: reserved,
|
|
deadline,
|
|
now,
|
|
require_fresh: true,
|
|
},
|
|
Some(receipt),
|
|
)
|
|
.await?
|
|
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
|
|
let verification_id = evidence
|
|
.get("verification_inventory_message_id")
|
|
.and_then(Value::as_str)
|
|
.map(Uuid::parse_str)
|
|
.transpose()?;
|
|
let verification_time = evidence.get("verification_inventory_received_at");
|
|
let verification = if verification_id.is_some()
|
|
|| verification_time.is_some()
|
|
|| received <= now - Duration::minutes(5)
|
|
{
|
|
let id = verification_id
|
|
.filter(|id| !id.is_nil())
|
|
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
|
|
let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id))
|
|
.await?
|
|
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
|
|
ensure!(
|
|
serde_json::from_value::<chrono::DateTime<Utc>>(
|
|
verification_time.cloned().unwrap_or(Value::Null)
|
|
)
|
|
.ok()
|
|
== Some(time),
|
|
"REBOOT_VERIFICATION_TIME_BINDING"
|
|
);
|
|
(id, time)
|
|
} else {
|
|
(receipt, received)
|
|
};
|
|
for (key, actual) in [
|
|
("reboot_reserved_at", reserved),
|
|
("reconnect_deadline", deadline),
|
|
("inventory_received_at", received),
|
|
] {
|
|
ensure!(
|
|
serde_json::from_value::<chrono::DateTime<Utc>>(evidence[key].clone()).ok()
|
|
== Some(actual),
|
|
"REBOOT_SERVER_TIME_BINDING"
|
|
);
|
|
}
|
|
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())")
|
|
.bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot,
|
|
"connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline,
|
|
"inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1}))
|
|
.execute(&mut **tx).await?;
|
|
Ok(())
|
|
}
|
|
pub async fn reconcile(
|
|
state: &crate::State,
|
|
tx: &mut Transaction<'_, Postgres>,
|
|
tenant: Uuid,
|
|
agent: Uuid,
|
|
e: &Envelope,
|
|
r: &SyncRequest,
|
|
) -> Result<SyncResponse> {
|
|
ensure!(
|
|
r.attempts.len() <= 256 && r.durable_message_ids.len() <= 256,
|
|
"sync count bound"
|
|
);
|
|
let request_id = optional_id(&r.request_id)?.context("sync request id required")?;
|
|
let journal_id = optional_id(&r.journal_id)?.context("journal identity required")?;
|
|
let helper_journal = optional_id(&r.helper_journal_id)?;
|
|
let boot = Uuid::parse_str(&e.boot_id)?;
|
|
let checkpoint = i64::try_from(r.checkpoint)?;
|
|
let helper_checkpoint = i64::try_from(r.helper_checkpoint)?;
|
|
let request_hash = hash(&r.encode_to_vec());
|
|
if let Some(cached)=sqlx::query("SELECT request_hash,connection_generation,payload,signature FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2").bind(agent).bind(request_id).fetch_optional(&mut **tx).await?{
|
|
ensure!(cached.get::<String,_>("request_hash")==request_hash,"sync request id conflict");
|
|
ensure!(cached.get::<i64,_>("connection_generation")==i64::try_from(e.connection_generation)?,"old sync request: recover via previous_request_id in a fresh request");
|
|
return Ok(SyncResponse{decision_bytes:cached.get("payload"),signature:cached.get("signature")});
|
|
}
|
|
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,sequence_regressed,sync_recovery_required,revoked_at FROM agent_installations WHERE agent_id=$1 FOR UPDATE").bind(agent).fetch_one(&mut **tx).await?;
|
|
ensure!(
|
|
device
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
|
|
.is_none()
|
|
&& device.get::<i64, _>("connection_generation")
|
|
== i64::try_from(e.connection_generation)?
|
|
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(boot),
|
|
"sync channel fenced"
|
|
);
|
|
let previous=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE agent_id=$1").bind(agent).fetch_optional(&mut **tx).await?;
|
|
let mut reason = if r.journal_complete {
|
|
"CONSISTENT"
|
|
} else {
|
|
"JOURNAL_INCOMPLETE"
|
|
}
|
|
.to_string();
|
|
let mut chain_valid = true;
|
|
if device.get::<bool, _>("sync_recovery_required") {
|
|
reason = "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED".into();
|
|
chain_valid = false;
|
|
}
|
|
if previous.is_none() {
|
|
let history: bool =
|
|
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE agent_id=$1)")
|
|
.bind(agent)
|
|
.fetch_one(&mut **tx)
|
|
.await?;
|
|
if history {
|
|
reason = "SERVER_STATE_ROLLBACK".into();
|
|
chain_valid = false;
|
|
}
|
|
}
|
|
if device.get::<i64, _>("assignment_version") != i64::try_from(r.assignment_version)? {
|
|
reason = "ASSIGNMENT_MISMATCH".into();
|
|
chain_valid = false;
|
|
}
|
|
if device.get::<bool, _>("sequence_regressed") {
|
|
reason = "JOURNAL_SEQUENCE_REGRESSION".into();
|
|
chain_valid = false;
|
|
}
|
|
let last_applied = optional_id(&r.previous_decision_id)?;
|
|
let pending_request = optional_id(&r.previous_request_id)?;
|
|
let committed_pending = if let Some(pending) = pending_request {
|
|
sqlx::query_scalar("SELECT decision_id FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2 AND journal_id=$3").bind(agent).bind(pending).bind(journal_id).fetch_optional(&mut **tx).await?
|
|
} else {
|
|
None
|
|
};
|
|
let previous_decision = previous_anchor(last_applied, pending_request, committed_pending);
|
|
if pending_request.is_some() && previous_decision.is_none() {
|
|
reason = "SERVER_CHECKPOINT_MISSING".into();
|
|
chain_valid = false;
|
|
}
|
|
if let Some(old) = &previous {
|
|
if old.get::<Uuid, _>("journal_id") != journal_id
|
|
|| checkpoint < old.get::<i64, _>("checkpoint")
|
|
{
|
|
reason = "JOURNAL_ROLLBACK_OR_REPLACEMENT".into();
|
|
chain_valid = false;
|
|
}
|
|
if previous_decision != Some(old.get::<Uuid, _>("decision_id"))
|
|
&& reason != "SERVER_CHECKPOINT_MISSING"
|
|
{
|
|
reason = "CHECKPOINT_CHAIN_MISMATCH".into();
|
|
chain_valid = false;
|
|
}
|
|
if let Some(helper) = helper_journal
|
|
&& (old
|
|
.get::<Option<Uuid>, _>("helper_journal_id")
|
|
.is_some_and(|id| id != helper)
|
|
|| helper_checkpoint < old.get::<i64, _>("helper_checkpoint"))
|
|
{
|
|
reason = "HELPER_JOURNAL_ROLLBACK".into();
|
|
chain_valid = false;
|
|
}
|
|
} else if previous_decision.is_some() || !r.helper_previous_decision_id.is_empty() {
|
|
reason = "SERVER_CHECKPOINT_MISSING".into();
|
|
chain_valid = false;
|
|
}
|
|
let mut helper_ready = helper_journal.is_some();
|
|
if helper_ready && !r.helper_previous_decision_id.is_empty() {
|
|
let previous_helper = sqlx::query_scalar::<_, Vec<u8>>(
|
|
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
|
|
)
|
|
.bind(agent)
|
|
.bind(Uuid::parse_str(&r.helper_previous_decision_id)?)
|
|
.fetch_optional(&mut **tx)
|
|
.await?;
|
|
if let Some(bytes) = previous_helper {
|
|
let old: ReconcileDecisionPayload = serde_json::from_slice(&bytes)?;
|
|
if old.helper_journal_id != helper_journal
|
|
|| old.helper_checkpoint > r.helper_checkpoint
|
|
{
|
|
helper_ready = false;
|
|
reason = "HELPER_CHECKPOINT_MISMATCH".into();
|
|
chain_valid = false;
|
|
}
|
|
} else {
|
|
helper_ready = false;
|
|
reason = "HELPER_CHECKPOINT_MISSING".into();
|
|
chain_valid = false;
|
|
}
|
|
} else if helper_ready
|
|
&& previous
|
|
.as_ref()
|
|
.is_some_and(|old| old.get::<Option<Uuid>, _>("helper_journal_id").is_some())
|
|
{
|
|
helper_ready = false;
|
|
reason = "HELPER_CHECKPOINT_REQUIRED".into();
|
|
chain_valid = false;
|
|
}
|
|
let mut locals = BTreeMap::new();
|
|
for local in &r.attempts {
|
|
let id = Uuid::parse_str(&local.job_id)?;
|
|
ensure!(
|
|
!id.is_nil() && locals.insert(id, local).is_none(),
|
|
"duplicate local attempt"
|
|
);
|
|
Uuid::parse_str(&local.attempt_id)?;
|
|
ensure!(
|
|
digest(&local.job_payload_hash)
|
|
&& matches!(
|
|
local.consumption.as_str(),
|
|
"unconsumed" | "consumed" | "uncertain"
|
|
),
|
|
"invalid attempt summary"
|
|
);
|
|
i64::try_from(local.revision)?;
|
|
ensure!(
|
|
matches!(
|
|
local.state.as_str(),
|
|
"queued"
|
|
| "accepted"
|
|
| "requesting"
|
|
| "running"
|
|
| "verifying"
|
|
| "awaiting_reboot"
|
|
| "unknown"
|
|
| "succeeded"
|
|
| "failed"
|
|
| "cancelled"
|
|
| "expired"
|
|
),
|
|
"invalid local state"
|
|
);
|
|
ensure!(
|
|
local.outcome_bytes.len() <= 512 * 1024,
|
|
"local outcome bound"
|
|
);
|
|
if !local.outcome_bytes.is_empty() {
|
|
ensure!(
|
|
digest(&local.outcome_sha256) && hash(&local.outcome_bytes) == local.outcome_sha256,
|
|
"local outcome hash"
|
|
);
|
|
let parsed: ReconciledOutcome = serde_json::from_slice(&local.outcome_bytes)?;
|
|
ensure!(
|
|
canonical_outcome_bytes(&parsed)? == local.outcome_bytes,
|
|
"noncanonical local outcome bytes"
|
|
);
|
|
ensure!(parsed.state == local.state, "local outcome state mismatch");
|
|
}
|
|
}
|
|
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
|
|
if rows.len() > 256 {
|
|
reason = "SYNC_SERVER_JOB_LIMIT".into();
|
|
chain_valid = false;
|
|
}
|
|
let mut server = BTreeMap::new();
|
|
for row in rows {
|
|
server.insert(row.get::<Uuid, _>("job_id"), row);
|
|
}
|
|
if previous.is_none() {
|
|
let missing:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND NOT(job_id=ANY($2)))").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_one(&mut **tx).await?;
|
|
if missing {
|
|
reason = "JOURNAL_HISTORY_MISSING".into();
|
|
chain_valid = false;
|
|
}
|
|
}
|
|
let mut ready = chain_valid && r.journal_complete;
|
|
let mut decisions = Vec::new();
|
|
let mut seen = BTreeSet::new();
|
|
for (job_id, local) in locals {
|
|
let attempt = Uuid::parse_str(&local.attempt_id)?;
|
|
let prior_hash = if local.outcome_sha256.is_empty() {
|
|
None
|
|
} else {
|
|
ensure!(digest(&local.outcome_sha256), "outcome digest");
|
|
Some(local.outcome_sha256.clone())
|
|
};
|
|
let mut decision = AttemptDecision {
|
|
job_id,
|
|
attempt_id: attempt,
|
|
job_payload_hash: local.job_payload_hash.clone(),
|
|
grant_id: optional_id(&local.grant_id)?,
|
|
authorization_id: None,
|
|
plan_hash: None,
|
|
action: String::new(),
|
|
execution_identity: String::new(),
|
|
revision: local.revision,
|
|
outcome_state: "unknown".into(),
|
|
previous_outcome_sha256: prior_hash.clone(),
|
|
outcome_sha256: None,
|
|
outcome_bytes: vec![],
|
|
original_boot_id: optional_id(&local.original_boot_id)?,
|
|
unlock: false,
|
|
reason: "MISSING_SERVER_JOB".into(),
|
|
};
|
|
let Some(row) = server.get(&job_id) else {
|
|
sqlx::query(
|
|
"UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1",
|
|
)
|
|
.bind(agent)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
ready = false;
|
|
reason = "MISSING_SERVER_JOB".into();
|
|
decisions.push(decision);
|
|
continue;
|
|
};
|
|
seen.insert(job_id);
|
|
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
|
decision.action = payload.action.name().into();
|
|
decision.execution_identity = payload.execution_identity.clone();
|
|
if row.get::<Uuid, _>("attempt_id") != attempt
|
|
|| row.get::<String, _>("payload_hash") != local.job_payload_hash
|
|
|| hash(&row.get::<Vec<u8>, _>("payload")) != local.job_payload_hash
|
|
|| payload.tenant_id != tenant
|
|
|| payload.agent_id != agent
|
|
|| payload.asset_id != device.get::<Uuid, _>("asset_id")
|
|
|| payload.enrollment_generation != device.get::<i64, _>("enrollment_generation") as u64
|
|
|| payload.assignment_version != device.get::<i64, _>("assignment_version") as u64
|
|
{
|
|
sqlx::query(
|
|
"UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1",
|
|
)
|
|
.bind(agent)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
ready = false;
|
|
decision.reason = "ATTEMPT_BINDING_MISMATCH".into();
|
|
decisions.push(decision);
|
|
continue;
|
|
}
|
|
let missing_helper =
|
|
crate::root_policy::needs_helper(&payload.execution_identity) && !helper_ready;
|
|
if missing_helper {
|
|
ready = false;
|
|
}
|
|
let grant = sqlx::query(
|
|
"SELECT grant_id,payload,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
|
)
|
|
.bind(job_id)
|
|
.bind(attempt)
|
|
.fetch_optional(&mut **tx)
|
|
.await?;
|
|
if let Some(grant) = &grant {
|
|
let p: ExecutionPermitPayload =
|
|
serde_json::from_slice(&grant.get::<Vec<u8>, _>("payload"))?;
|
|
decision.authorization_id = Some(p.authorization_id);
|
|
decision.plan_hash = Some(p.plan_hash);
|
|
}
|
|
let reserved = row.get::<bool, _>("start_reserved");
|
|
let server_state: String = row.get("state");
|
|
if clean_unstarted(local, reserved)
|
|
&& matches!(server_state.as_str(), "queued" | "accepted")
|
|
{
|
|
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=COALESCE(journal_proof,$2),revision=GREATEST(revision,$3) WHERE job_id=$1").bind(job_id).bind(format!("sync:{journal_id}:{}",r.checkpoint)).bind(i64::try_from(local.revision)?).execute(&mut **tx).await?;
|
|
sqlx::query(
|
|
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
|
|
)
|
|
.bind(job_id)
|
|
.execute(&mut **tx)
|
|
.await?;
|
|
decision.outcome_state = "accepted".into();
|
|
decision.reason = "ACCEPTED_NOT_STARTED".into();
|
|
decision.revision = (row.get::<i64, _>("revision") as u64).max(local.revision);
|
|
decisions.push(decision);
|
|
continue;
|
|
}
|
|
let grant_matches = grant
|
|
.as_ref()
|
|
.is_some_and(|g| Some(g.get::<Uuid, _>("grant_id")) == decision.grant_id);
|
|
let unsafe_claim =
|
|
local.state == "requesting" || (!grant_matches && local.consumption != "unconsumed");
|
|
if unsafe_claim {
|
|
if unsettled(&server_state) {
|
|
mark_unknown(tx, job_id, "SYNC_GRANT_OR_EFFECT_UNRESOLVED").await?;
|
|
}
|
|
ready = false;
|
|
decision.reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
|
|
decisions.push(decision);
|
|
continue;
|
|
}
|
|
// A free script with no bound postcondition never becomes successful by
|
|
// merely asserting a new state after unknown effect.
|
|
let manual = server_state == "unknown"
|
|
&& matches!(
|
|
payload.action,
|
|
Action::RunScript {
|
|
expected_stdout: None,
|
|
..
|
|
}
|
|
);
|
|
let embedded =
|
|
local
|
|
.result_envelope
|
|
.as_ref()
|
|
.and_then(|envelope| match &envelope.payload {
|
|
Some(Payload::JobResult(result)) => Some(result),
|
|
_ => None,
|
|
});
|
|
if let Some(result) = local.result.as_ref().or(embedded) {
|
|
ensure!(
|
|
result.job_id == local.job_id
|
|
&& result.attempt_id == local.attempt_id
|
|
&& result.revision == local.revision
|
|
&& result.state == local.state
|
|
&& result.grant_id == local.grant_id,
|
|
"sync result binding"
|
|
);
|
|
if !manual {
|
|
let evidence = local
|
|
.result_envelope
|
|
.as_ref()
|
|
.context("original durable result envelope required")?;
|
|
ensure!(
|
|
valid_envelope(evidence, &agent.to_string(), evidence.connection_generation)
|
|
&& evidence.message_id == local.result_message_id
|
|
&& embedded == Some(result),
|
|
"original result envelope binding"
|
|
);
|
|
let observed = chrono::DateTime::from_timestamp_millis(evidence.observed_at)
|
|
.context("result timestamp")?;
|
|
ensure!(
|
|
observed <= Utc::now() + Duration::seconds(45),
|
|
"future result timestamp"
|
|
);
|
|
crate::job_gateway::event(tx, tenant, agent, evidence).await?;
|
|
}
|
|
}
|
|
let mut final_row = sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1")
|
|
.bind(job_id)
|
|
.fetch_one(&mut **tx)
|
|
.await?;
|
|
// A reboot's postcondition is independently tied to persisted current-
|
|
// boot inventory and the original boot in the durable reservation audit.
|
|
let mut reboot_reconciled = false;
|
|
let mut reboot_witness = None;
|
|
if matches!(payload.action, Action::Reboot { .. })
|
|
&& grant_matches
|
|
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
|
|
{
|
|
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1").bind(tenant).bind(job_id.to_string()).bind(&local.grant_id).fetch_optional(&mut **tx).await?;
|
|
let reserved_at = grant
|
|
.as_ref()
|
|
.map(|g| g.get::<chrono::DateTime<Utc>, _>("created_at"));
|
|
let deadline = reboot_deadline(&payload.action, reserved_at);
|
|
let inventory = if let (Some(reserved), Some(until)) = (reserved_at, deadline) {
|
|
let existing_receipt = if final_row.get::<String, _>("state") == "succeeded" {
|
|
final_row
|
|
.get::<Option<Value>, _>("result")
|
|
.and_then(|value| {
|
|
value["evidence"]["inventory_message_id"]
|
|
.as_str()
|
|
.and_then(|v| Uuid::parse_str(v).ok())
|
|
})
|
|
} else {
|
|
None
|
|
};
|
|
reboot_inventory(
|
|
tx,
|
|
tenant,
|
|
agent,
|
|
boot,
|
|
&RebootWindow {
|
|
reserved_at: reserved,
|
|
deadline: until,
|
|
now: Utc::now(),
|
|
require_fresh: existing_receipt.is_none(),
|
|
},
|
|
existing_receipt,
|
|
)
|
|
.await?
|
|
} else {
|
|
None
|
|
};
|
|
if let (Some(original), Some((inventory, received_at)), Some(reserved), Some(until)) =
|
|
(original, inventory, reserved_at, deadline)
|
|
&& original != e.boot_id
|
|
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
|
|
{
|
|
reboot_witness = Some((original, inventory));
|
|
let verification =
|
|
reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?;
|
|
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
|
|
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()};
|
|
let mut evidence = e.clone();
|
|
evidence.message_id = Uuid::new_v4().to_string();
|
|
evidence.payload = Some(Payload::JobResult(result));
|
|
if unsettled(&final_row.get::<String, _>("state")) {
|
|
crate::job_gateway::event(tx, tenant, agent, &evidence).await?;
|
|
reboot_reconciled = true;
|
|
final_row =
|
|
sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1")
|
|
.bind(job_id)
|
|
.fetch_one(&mut **tx)
|
|
.await?;
|
|
}
|
|
}
|
|
}
|
|
let terminal: String = final_row.get("state");
|
|
if matches!(payload.action, Action::Reboot { .. })
|
|
&& grant_matches
|
|
&& terminal == "succeeded"
|
|
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
|
|
&& let Some(value) = final_row.get::<Option<Value>, _>("result")
|
|
&& let Some((original, inventory)) = reboot_witness
|
|
&& Uuid::parse_str(&original).ok() == decision.original_boot_id
|
|
&& decision.original_boot_id.is_some()
|
|
&& value["evidence"]["postconditions_passed"] == json!(true)
|
|
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
|
|
&& value["evidence"]["new_boot_id"]
|
|
.as_str()
|
|
.is_some_and(|id| id == e.boot_id)
|
|
&& value["evidence"]["inventory_message_id"] == json!(inventory)
|
|
{
|
|
reboot_reconciled = true;
|
|
}
|
|
decision.revision = final_row.get::<i64, _>("revision") as u64;
|
|
decision.outcome_state = terminal.clone();
|
|
if !unsettled(&terminal) && !manual {
|
|
if let Some(value) = final_row.get::<Option<Value>, _>("result") {
|
|
let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?;
|
|
let hash = hash(&bytes);
|
|
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled {
|
|
decision.outcome_sha256 = Some(hash);
|
|
decision.unlock = local.consumption == "consumed" && grant_matches;
|
|
if reboot_reconciled {
|
|
decision.outcome_bytes = bytes;
|
|
}
|
|
decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into();
|
|
} else {
|
|
ready = false;
|
|
decision.reason = "LOCAL_OUTCOME_MISMATCH".into();
|
|
}
|
|
} else if local.consumption == "unconsumed" && !reserved {
|
|
decision.reason = "DEFINITIVE_NONSTART".into();
|
|
} else {
|
|
ready = false;
|
|
decision.reason = "DEFINITIVE_OUTCOME_MISSING".into();
|
|
}
|
|
} else {
|
|
if unsettled(&terminal) {
|
|
mark_unknown(tx, job_id, "SYNC_EFFECT_UNRESOLVED").await?;
|
|
}
|
|
ready = false;
|
|
decision.outcome_state = "unknown".into();
|
|
decision.reason = if manual {
|
|
"MANUAL_RECONCILIATION_REQUIRED"
|
|
} else {
|
|
"EFFECT_UNRESOLVED"
|
|
}
|
|
.into();
|
|
}
|
|
if missing_helper {
|
|
decision.unlock = false;
|
|
decision.reason = "HELPER_JOURNAL_REQUIRED".into();
|
|
}
|
|
decisions.push(decision);
|
|
}
|
|
for (id, row) in server {
|
|
if !seen.contains(&id)
|
|
&& (row.get::<bool, _>("start_reserved")
|
|
|| row.get::<String, _>("state") != "queued"
|
|
|| row
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("acknowledged_at")
|
|
.is_some())
|
|
{
|
|
if unsettled(&row.get::<String, _>("state")) {
|
|
mark_unknown(tx, id, "SYNC_LOCAL_ATTEMPT_MISSING").await?;
|
|
}
|
|
ready = false;
|
|
reason = "LOCAL_ATTEMPT_MISSING".into();
|
|
}
|
|
}
|
|
if !ready && reason == "CONSISTENT" {
|
|
reason = "ATTEMPT_RECONCILIATION_REQUIRED".into();
|
|
}
|
|
helper_ready &= ready;
|
|
if !chain_valid {
|
|
for decision in &mut decisions {
|
|
decision.unlock = false;
|
|
}
|
|
}
|
|
let mut receipts = Vec::new();
|
|
for raw in &r.durable_message_ids {
|
|
let id = Uuid::parse_str(raw)?;
|
|
let persisted:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM job_receipts WHERE agent_id=$1 AND message_id=$2 UNION ALL SELECT 1 FROM observations WHERE agent_id=$1 AND message_id=$2)").bind(agent).bind(id).fetch_one(&mut **tx).await?;
|
|
if persisted {
|
|
receipts.push(id)
|
|
}
|
|
}
|
|
receipts.sort();
|
|
receipts.dedup();
|
|
let now = Utc::now();
|
|
let decision = ReconcileDecisionPayload {
|
|
domain: "rmm.reconciliation-decision".into(),
|
|
schema_version: 1,
|
|
decision_id: Uuid::new_v4(),
|
|
request_id,
|
|
request_hash: request_hash.clone(),
|
|
tenant_id: tenant,
|
|
asset_id: device.get("asset_id"),
|
|
agent_id: agent,
|
|
enrollment_generation: device.get::<i64, _>("enrollment_generation") as u64,
|
|
assignment_version: device.get::<i64, _>("assignment_version") as u64,
|
|
connection_generation: e.connection_generation,
|
|
boot_id: boot,
|
|
journal_id,
|
|
checkpoint: r.checkpoint,
|
|
helper_journal_id: helper_journal,
|
|
helper_checkpoint: r.helper_checkpoint,
|
|
issued_at: now,
|
|
valid_until: now + Duration::seconds(60),
|
|
ready,
|
|
helper_ready,
|
|
reason,
|
|
attempts: decisions,
|
|
acknowledged_message_ids: receipts,
|
|
key_id: state.job_signer.key_id.clone(),
|
|
};
|
|
let bytes = encode(&decision)?;
|
|
ensure!(bytes.len() <= 512 * 1024, "sync decision bound");
|
|
let signature = state.job_signer.sign(&bytes)?;
|
|
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)").bind(tenant).bind(agent).bind(request_id).bind(decision.decision_id).bind(request_hash).bind(journal_id).bind(checkpoint).bind(i64::try_from(e.connection_generation)?).bind(&bytes).bind(&signature).execute(&mut **tx).await?;
|
|
if chain_valid {
|
|
sqlx::query("INSERT INTO agent_sync_state(tenant_id,agent_id,journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint) VALUES($1,$2,$3,$4,$5,$6,$7) ON CONFLICT(tenant_id,agent_id) DO UPDATE SET checkpoint=GREATEST(agent_sync_state.checkpoint,EXCLUDED.checkpoint),decision_id=EXCLUDED.decision_id,helper_journal_id=COALESCE(agent_sync_state.helper_journal_id,EXCLUDED.helper_journal_id),helper_checkpoint=GREATEST(agent_sync_state.helper_checkpoint,EXCLUDED.helper_checkpoint)").bind(tenant).bind(agent).bind(journal_id).bind(checkpoint).bind(decision.decision_id).bind(helper_journal).bind(helper_checkpoint).execute(&mut **tx).await?;
|
|
}
|
|
sqlx::query("UPDATE agent_installations SET sync_ready=$2,helper_sync_ready=$3,restore_blocked=NOT $2 WHERE agent_id=$1").bind(agent).bind(ready).bind(helper_ready).execute(&mut **tx).await?;
|
|
Ok(SyncResponse {
|
|
decision_bytes: bytes,
|
|
signature,
|
|
})
|
|
}
|
|
async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> {
|
|
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?;
|
|
Ok(())
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
#[test]
|
|
fn reboot_deadline_uses_only_the_fixed_plan_and_trusted_reservation() {
|
|
let reserved = Utc::now();
|
|
let action = Action::Reboot {
|
|
reason: "Explicit final Cloud VM acceptance".into(),
|
|
reconnect_timeout_seconds: 180,
|
|
dependency_group: "single-test-vm".into(),
|
|
};
|
|
assert_eq!(
|
|
reboot_deadline(&action, Some(reserved)),
|
|
Some(reserved + Duration::seconds(180))
|
|
);
|
|
assert_eq!(reboot_deadline(&action, None), None);
|
|
assert_eq!(
|
|
reboot_deadline(
|
|
&Action::Reboot {
|
|
reason: "test".into(),
|
|
reconnect_timeout_seconds: 0,
|
|
dependency_group: "single-test-vm".into()
|
|
},
|
|
Some(reserved)
|
|
),
|
|
None
|
|
);
|
|
assert_eq!(
|
|
reboot_deadline(&action, Some(chrono::DateTime::<Utc>::MAX_UTC)),
|
|
None
|
|
);
|
|
}
|
|
#[tokio::test]
|
|
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
|
|
async fn reboot_inventory_requires_trusted_timely_same_identity_and_boot() {
|
|
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
|
|
.await
|
|
.unwrap();
|
|
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
|
|
.await
|
|
.unwrap();
|
|
rmm_persistence::verify_application_role(&app)
|
|
.await
|
|
.unwrap();
|
|
let tenant = Uuid::new_v4();
|
|
let asset = Uuid::new_v4();
|
|
let agent = Uuid::new_v4();
|
|
let boot = Uuid::new_v4();
|
|
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot deadline test')")
|
|
.bind(tenant)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO assets VALUES($1,$2)")
|
|
.bind(tenant)
|
|
.bind(asset)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)")
|
|
.bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
|
|
let now = Utc::now();
|
|
let reserved = now - Duration::seconds(200);
|
|
let deadline = reserved + Duration::seconds(180);
|
|
let window = RebootWindow {
|
|
reserved_at: reserved,
|
|
deadline,
|
|
now,
|
|
require_fresh: true,
|
|
};
|
|
let timely = Uuid::new_v4();
|
|
let late = Uuid::new_v4();
|
|
for (sequence, message, received) in [
|
|
(1, timely, deadline),
|
|
(2, late, deadline + Duration::microseconds(1)),
|
|
(3, Uuid::new_v4(), reserved),
|
|
] {
|
|
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
|
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&admin).await.unwrap();
|
|
}
|
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
|
assert_eq!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &window, None)
|
|
.await
|
|
.unwrap()
|
|
.map(|v| v.0),
|
|
Some(timely)
|
|
);
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &window, Some(late))
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, Uuid::new_v4(), boot, &window, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert!(
|
|
reboot_inventory(&mut tx, Uuid::new_v4(), agent, boot, &window, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, agent, Uuid::new_v4(), &window, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let later = RebootWindow {
|
|
now: now + Duration::minutes(6),
|
|
..window
|
|
};
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
// A terminal result reuses only its exact original timely receipt;
|
|
// later metadata cannot replace or extend that immutable proof.
|
|
let historical = RebootWindow {
|
|
require_fresh: false,
|
|
..later
|
|
};
|
|
assert_eq!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &historical, Some(timely))
|
|
.await
|
|
.unwrap()
|
|
.map(|v| v.0),
|
|
Some(timely)
|
|
);
|
|
let fresh = Uuid::new_v4();
|
|
let future = Uuid::new_v4();
|
|
for (sequence, message, received) in [
|
|
(4, fresh, later.now - Duration::seconds(1)),
|
|
(5, future, later.now + Duration::days(1)),
|
|
] {
|
|
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
|
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap();
|
|
}
|
|
// Delayed first verification keeps the original cutoff-bound ID and
|
|
// uses a DIFFERENT fresh same-boot receipt; late-only never suffices.
|
|
assert_eq!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
|
|
.await
|
|
.unwrap()
|
|
.map(|v| v.0),
|
|
Some(timely)
|
|
);
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert_eq!(
|
|
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None)
|
|
.await
|
|
.unwrap()
|
|
.map(|v| v.0),
|
|
Some(fresh)
|
|
);
|
|
assert!(
|
|
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future))
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert!(
|
|
reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert!(
|
|
reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert!(
|
|
reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let before_return = RebootWindow {
|
|
now: deadline - Duration::microseconds(1),
|
|
..window
|
|
};
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &before_return, None)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
tx.rollback().await.unwrap();
|
|
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs WHERE tenant_id=$1)+(SELECT count(*) FROM outbox WHERE tenant_id=$1)+(SELECT count(*) FROM execution_permits WHERE tenant_id=$1)").bind(tenant).fetch_one(&admin).await.unwrap();
|
|
assert_eq!(effects, 0);
|
|
}
|
|
#[tokio::test]
|
|
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
|
|
async fn reboot_success_ingress_rejects_forged_late_proof_and_preserves_exact_replay() {
|
|
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
|
|
.await
|
|
.unwrap();
|
|
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
|
|
.await
|
|
.unwrap();
|
|
rmm_persistence::verify_application_role(&app)
|
|
.await
|
|
.unwrap();
|
|
let tenant = Uuid::new_v4();
|
|
let asset = Uuid::new_v4();
|
|
let agent = Uuid::new_v4();
|
|
let boot = Uuid::new_v4();
|
|
let old = Uuid::new_v4();
|
|
let job = Uuid::new_v4();
|
|
let attempt = Uuid::new_v4();
|
|
let operation = Uuid::new_v4();
|
|
let plan = Uuid::new_v4();
|
|
let authorization = Uuid::new_v4();
|
|
let grant = Uuid::new_v4();
|
|
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
|
|
let reserved = now - Duration::seconds(600);
|
|
let deadline = reserved + Duration::seconds(180);
|
|
let payload = JobPayload {
|
|
schema_version: 1,
|
|
tenant_id: tenant,
|
|
asset_id: asset,
|
|
agent_id: agent,
|
|
enrollment_generation: 1,
|
|
assignment_version: 1,
|
|
job_id: job,
|
|
attempt_id: attempt,
|
|
action: Action::Reboot {
|
|
reason: "Isolated postcondition test; no actual reboot".into(),
|
|
reconnect_timeout_seconds: 180,
|
|
dependency_group: "single-test-vm".into(),
|
|
},
|
|
execution_identity: "root".into(),
|
|
not_before: reserved,
|
|
start_before: now + Duration::minutes(5),
|
|
max_runtime_seconds: 240,
|
|
max_output_bytes: 65536,
|
|
};
|
|
let bytes = serde_json::to_vec(&payload).unwrap();
|
|
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot ingress test')")
|
|
.bind(tenant)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO assets VALUES($1,$2)")
|
|
.bind(tenant)
|
|
.bind(asset)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
|
|
.bind(tenant)
|
|
.bind(operation)
|
|
.bind(authorization)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)")
|
|
.bind(tenant)
|
|
.bind(grant)
|
|
.bind(job)
|
|
.bind(attempt)
|
|
.bind(b"{}".as_slice())
|
|
.bind(vec![0u8; 64])
|
|
.bind(reserved)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"INSERT INTO audit_events VALUES($1,$2,'isolated-test','start_reserved',$3,$4)",
|
|
)
|
|
.bind(tenant)
|
|
.bind(Uuid::new_v4())
|
|
.bind(json!({"job_id":job,"grant_id":grant,"boot_id":old}))
|
|
.bind(reserved)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
let timely = Uuid::new_v4();
|
|
let late = Uuid::new_v4();
|
|
let fresh = Uuid::new_v4();
|
|
let wrong_boot = Uuid::new_v4();
|
|
let future = Uuid::new_v4();
|
|
for (sequence, message, received) in [
|
|
(1, timely, deadline),
|
|
(2, late, deadline + Duration::microseconds(1)),
|
|
] {
|
|
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
|
}
|
|
for (sequence, message, received, observed_boot) in [
|
|
(3, fresh, now - Duration::seconds(1), boot),
|
|
(4, wrong_boot, now - Duration::seconds(1), old),
|
|
(5, future, now + Duration::days(1), boot),
|
|
] {
|
|
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
|
.bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
|
}
|
|
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
|
|
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline,
|
|
"verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)});
|
|
let mut result = JobResult {
|
|
job_id: job.to_string(),
|
|
attempt_id: attempt.to_string(),
|
|
grant_id: grant.to_string(),
|
|
revision: 1,
|
|
state: "succeeded".into(),
|
|
evidence_json: evidence.to_string(),
|
|
..Default::default()
|
|
};
|
|
let mut envelope = Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: agent.to_string(),
|
|
connection_generation: 1,
|
|
boot_id: boot.to_string(),
|
|
payload: Some(Payload::JobResult(result.clone())),
|
|
..Default::default()
|
|
};
|
|
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
|
assert!(
|
|
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
|
.await
|
|
.is_err()
|
|
);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
|
|
.bind(job)
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
"unknown"
|
|
);
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
0
|
|
);
|
|
let mut good = evidence.clone();
|
|
good["inventory_message_id"] = json!(timely);
|
|
for (id, time) in [
|
|
(Some(wrong_boot), Some(now - Duration::seconds(1))),
|
|
(Some(future), Some(now + Duration::days(1))),
|
|
(None, None),
|
|
(Some(fresh), Some(now)),
|
|
] {
|
|
let mut bad = good.clone();
|
|
bad["verification_inventory_message_id"] = json!(id);
|
|
bad["verification_inventory_received_at"] = json!(time);
|
|
result.evidence_json = bad.to_string();
|
|
envelope.payload = Some(Payload::JobResult(result.clone()));
|
|
assert!(
|
|
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
envelope.connection_generation = 2;
|
|
result.evidence_json = good.to_string();
|
|
envelope.payload = Some(Payload::JobResult(result.clone()));
|
|
assert!(
|
|
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
|
.await
|
|
.is_err()
|
|
);
|
|
envelope.connection_generation = 1;
|
|
let mut false_clock = good.clone();
|
|
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
|
|
result.evidence_json = false_clock.to_string();
|
|
envelope.payload = Some(Payload::JobResult(result.clone()));
|
|
assert!(
|
|
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
|
.await
|
|
.is_err()
|
|
);
|
|
result.evidence_json = good.to_string();
|
|
envelope.payload = Some(Payload::JobResult(result.clone()));
|
|
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
|
|
.bind(job)
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
"succeeded"
|
|
);
|
|
// The original timely receipt survives later inventories and wall time.
|
|
let later = RebootWindow {
|
|
reserved_at: reserved,
|
|
deadline,
|
|
now: now + Duration::minutes(10),
|
|
require_fresh: false,
|
|
};
|
|
assert_eq!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(timely))
|
|
.await
|
|
.unwrap()
|
|
.map(|v| v.0),
|
|
Some(timely)
|
|
);
|
|
assert!(
|
|
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
// Exact durable-message replay ACKs the existing receipt, without
|
|
// accepting a new result or extending its original evidence window.
|
|
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
|
|
.fetch_one(&mut *tx)
|
|
.await
|
|
.unwrap(),
|
|
1
|
|
);
|
|
let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'")
|
|
.bind(tenant).fetch_one(&mut *tx).await.unwrap();
|
|
assert_eq!(audit["inventory_message_id"], json!(timely));
|
|
assert_eq!(audit["inventory_received_at"], json!(deadline));
|
|
assert_eq!(audit["verification_inventory_message_id"], json!(fresh));
|
|
assert_eq!(
|
|
audit["verification_inventory_received_at"],
|
|
json!(now - Duration::seconds(1))
|
|
);
|
|
assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1);
|
|
tx.rollback().await.unwrap();
|
|
}
|
|
#[test]
|
|
fn uncertain_request_resolves_only_to_a_known_checkpoint_anchor() {
|
|
let applied = Uuid::new_v4();
|
|
let pending = Uuid::new_v4();
|
|
let committed = Uuid::new_v4();
|
|
// Transport cut before server commit: retain the applied anchor.
|
|
assert_eq!(
|
|
previous_anchor(Some(applied), Some(pending), None),
|
|
Some(applied)
|
|
);
|
|
// Server committed but its reply was lost: use the newer checkpoint,
|
|
// even while the agent also retains its older applied witness.
|
|
assert_eq!(
|
|
previous_anchor(Some(applied), Some(pending), Some(committed)),
|
|
Some(committed)
|
|
);
|
|
// An absent pending request cannot invent an anchor for a legacy
|
|
// request that discarded its last applied decision.
|
|
assert_eq!(previous_anchor(None, Some(pending), None), None);
|
|
assert_eq!(
|
|
previous_anchor(None, Some(pending), Some(committed)),
|
|
Some(committed)
|
|
);
|
|
assert_eq!(previous_anchor(Some(applied), None, None), Some(applied));
|
|
}
|
|
#[test]
|
|
fn accepted_is_not_a_new_attempt_or_grant() {
|
|
let mut local = SyncAttempt {
|
|
state: "accepted".into(),
|
|
consumption: "unconsumed".into(),
|
|
..Default::default()
|
|
};
|
|
assert!(clean_unstarted(&local, false));
|
|
assert!(!clean_unstarted(&local, true));
|
|
local.state = "requesting".into();
|
|
assert!(!clean_unstarted(&local, false));
|
|
local.state = "running".into();
|
|
assert!(!clean_unstarted(&local, false));
|
|
}
|
|
}
|