feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,598 @@
|
||||
//! Authoritative reconciliation only: this module never inserts jobs, attempts,
|
||||
//! outbox entries or execution permits. A signed decision is not start authority.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{Duration, Utc};
|
||||
use prost::Message;
|
||||
use rmm_domain::{jobs::*, reconciliation::*};
|
||||
use rmm_protocol::*;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::{Postgres, Row, Transaction};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use uuid::Uuid;
|
||||
fn optional_id(raw: &str) -> Result<Option<Uuid>> {
|
||||
if raw.is_empty() {
|
||||
Ok(None)
|
||||
} else {
|
||||
let id = Uuid::parse_str(raw)?;
|
||||
ensure!(!id.is_nil(), "nil sync identity");
|
||||
Ok(Some(id))
|
||||
}
|
||||
}
|
||||
fn digest(raw: &str) -> bool {
|
||||
raw.len() == 64
|
||||
&& raw
|
||||
.bytes()
|
||||
.all(|c| c.is_ascii_digit() || b"abcdef".contains(&c))
|
||||
}
|
||||
fn outcome(state: &str, value: &Value) -> Result<ReconciledOutcome> {
|
||||
Ok(ReconciledOutcome {
|
||||
state: state.into(),
|
||||
stdout: serde_json::from_value(value["stdout"].clone())?,
|
||||
stderr: serde_json::from_value(value["stderr"].clone())?,
|
||||
exit_code: serde_json::from_value(value["exit_code"].clone())?,
|
||||
output_truncated: value["output_truncated"]
|
||||
.as_bool()
|
||||
.context("output bound evidence missing")?,
|
||||
reboot_required: value["reboot_required"]
|
||||
.as_bool()
|
||||
.context("reboot evidence missing")?,
|
||||
evidence: value["evidence"].clone(),
|
||||
error_code: value["error_code"]
|
||||
.as_str()
|
||||
.context("error evidence missing")?
|
||||
.into(),
|
||||
})
|
||||
}
|
||||
fn unsettled(state: &str) -> bool {
|
||||
!matches!(state, "succeeded" | "failed" | "cancelled" | "expired")
|
||||
}
|
||||
fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool {
|
||||
!reserved
|
||||
&& local.consumption == "unconsumed"
|
||||
&& local.grant_id.is_empty()
|
||||
&& matches!(local.state.as_str(), "queued" | "accepted")
|
||||
}
|
||||
pub async fn reconcile(
|
||||
state: &crate::State,
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
e: &Envelope,
|
||||
r: &SyncRequest,
|
||||
) -> Result<SyncResponse> {
|
||||
ensure!(
|
||||
r.attempts.len() <= 256 && r.durable_message_ids.len() <= 256,
|
||||
"sync count bound"
|
||||
);
|
||||
let request_id = optional_id(&r.request_id)?.context("sync request id required")?;
|
||||
let journal_id = optional_id(&r.journal_id)?.context("journal identity required")?;
|
||||
let helper_journal = optional_id(&r.helper_journal_id)?;
|
||||
let boot = Uuid::parse_str(&e.boot_id)?;
|
||||
let checkpoint = i64::try_from(r.checkpoint)?;
|
||||
let helper_checkpoint = i64::try_from(r.helper_checkpoint)?;
|
||||
let request_hash = hash(&r.encode_to_vec());
|
||||
if let Some(cached)=sqlx::query("SELECT request_hash,connection_generation,payload,signature FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2").bind(agent).bind(request_id).fetch_optional(&mut **tx).await?{
|
||||
ensure!(cached.get::<String,_>("request_hash")==request_hash,"sync request id conflict");
|
||||
ensure!(cached.get::<i64,_>("connection_generation")==i64::try_from(e.connection_generation)?,"old sync request: recover via previous_request_id in a fresh request");
|
||||
return Ok(SyncResponse{decision_bytes:cached.get("payload"),signature:cached.get("signature")});
|
||||
}
|
||||
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,sequence_regressed,sync_recovery_required,revoked_at FROM agent_installations WHERE agent_id=$1 FOR UPDATE").bind(agent).fetch_one(&mut **tx).await?;
|
||||
ensure!(
|
||||
device
|
||||
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
|
||||
.is_none()
|
||||
&& device.get::<i64, _>("connection_generation")
|
||||
== i64::try_from(e.connection_generation)?
|
||||
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(boot),
|
||||
"sync channel fenced"
|
||||
);
|
||||
let previous=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE agent_id=$1").bind(agent).fetch_optional(&mut **tx).await?;
|
||||
let mut reason = if r.journal_complete {
|
||||
"CONSISTENT"
|
||||
} else {
|
||||
"JOURNAL_INCOMPLETE"
|
||||
}
|
||||
.to_string();
|
||||
let mut chain_valid = true;
|
||||
if device.get::<bool, _>("sync_recovery_required") {
|
||||
reason = "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
if previous.is_none() {
|
||||
let history: bool =
|
||||
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE agent_id=$1)")
|
||||
.bind(agent)
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
if history {
|
||||
reason = "SERVER_STATE_ROLLBACK".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
}
|
||||
if device.get::<i64, _>("assignment_version") != i64::try_from(r.assignment_version)? {
|
||||
reason = "ASSIGNMENT_MISMATCH".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
if device.get::<bool, _>("sequence_regressed") {
|
||||
reason = "JOURNAL_SEQUENCE_REGRESSION".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
let mut previous_decision = optional_id(&r.previous_decision_id)?;
|
||||
if previous_decision.is_none() && !r.previous_request_id.is_empty() {
|
||||
previous_decision=sqlx::query_scalar("SELECT decision_id FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2 AND journal_id=$3").bind(agent).bind(Uuid::parse_str(&r.previous_request_id)?).bind(journal_id).fetch_optional(&mut **tx).await?;
|
||||
if previous_decision.is_none() {
|
||||
reason = "SERVER_CHECKPOINT_MISSING".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
}
|
||||
if let Some(old) = &previous {
|
||||
if old.get::<Uuid, _>("journal_id") != journal_id
|
||||
|| checkpoint < old.get::<i64, _>("checkpoint")
|
||||
{
|
||||
reason = "JOURNAL_ROLLBACK_OR_REPLACEMENT".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
if previous_decision != Some(old.get::<Uuid, _>("decision_id")) {
|
||||
reason = "CHECKPOINT_CHAIN_MISMATCH".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
if let Some(helper) = helper_journal
|
||||
&& (old
|
||||
.get::<Option<Uuid>, _>("helper_journal_id")
|
||||
.is_some_and(|id| id != helper)
|
||||
|| helper_checkpoint < old.get::<i64, _>("helper_checkpoint"))
|
||||
{
|
||||
reason = "HELPER_JOURNAL_ROLLBACK".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
} else if previous_decision.is_some() || !r.helper_previous_decision_id.is_empty() {
|
||||
reason = "SERVER_CHECKPOINT_MISSING".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
let mut helper_ready = helper_journal.is_some();
|
||||
if helper_ready && !r.helper_previous_decision_id.is_empty() {
|
||||
let previous_helper = sqlx::query_scalar::<_, Vec<u8>>(
|
||||
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
|
||||
)
|
||||
.bind(agent)
|
||||
.bind(Uuid::parse_str(&r.helper_previous_decision_id)?)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?;
|
||||
if let Some(bytes) = previous_helper {
|
||||
let old: ReconcileDecisionPayload = serde_json::from_slice(&bytes)?;
|
||||
if old.helper_journal_id != helper_journal
|
||||
|| old.helper_checkpoint > r.helper_checkpoint
|
||||
{
|
||||
helper_ready = false;
|
||||
reason = "HELPER_CHECKPOINT_MISMATCH".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
} else {
|
||||
helper_ready = false;
|
||||
reason = "HELPER_CHECKPOINT_MISSING".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
} else if helper_ready
|
||||
&& previous
|
||||
.as_ref()
|
||||
.is_some_and(|old| old.get::<Option<Uuid>, _>("helper_journal_id").is_some())
|
||||
{
|
||||
helper_ready = false;
|
||||
reason = "HELPER_CHECKPOINT_REQUIRED".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
let mut locals = BTreeMap::new();
|
||||
for local in &r.attempts {
|
||||
let id = Uuid::parse_str(&local.job_id)?;
|
||||
ensure!(
|
||||
!id.is_nil() && locals.insert(id, local).is_none(),
|
||||
"duplicate local attempt"
|
||||
);
|
||||
Uuid::parse_str(&local.attempt_id)?;
|
||||
ensure!(
|
||||
digest(&local.job_payload_hash)
|
||||
&& matches!(
|
||||
local.consumption.as_str(),
|
||||
"unconsumed" | "consumed" | "uncertain"
|
||||
),
|
||||
"invalid attempt summary"
|
||||
);
|
||||
i64::try_from(local.revision)?;
|
||||
ensure!(
|
||||
matches!(
|
||||
local.state.as_str(),
|
||||
"queued"
|
||||
| "accepted"
|
||||
| "requesting"
|
||||
| "running"
|
||||
| "verifying"
|
||||
| "awaiting_reboot"
|
||||
| "unknown"
|
||||
| "succeeded"
|
||||
| "failed"
|
||||
| "cancelled"
|
||||
| "expired"
|
||||
),
|
||||
"invalid local state"
|
||||
);
|
||||
ensure!(
|
||||
local.outcome_bytes.len() <= 512 * 1024,
|
||||
"local outcome bound"
|
||||
);
|
||||
if !local.outcome_bytes.is_empty() {
|
||||
ensure!(
|
||||
digest(&local.outcome_sha256) && hash(&local.outcome_bytes) == local.outcome_sha256,
|
||||
"local outcome hash"
|
||||
);
|
||||
let parsed: ReconciledOutcome = serde_json::from_slice(&local.outcome_bytes)?;
|
||||
ensure!(
|
||||
canonical_outcome_bytes(&parsed)? == local.outcome_bytes,
|
||||
"noncanonical local outcome bytes"
|
||||
);
|
||||
ensure!(parsed.state == local.state, "local outcome state mismatch");
|
||||
}
|
||||
}
|
||||
let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_all(&mut **tx).await?;
|
||||
if rows.len() > 256 {
|
||||
reason = "SYNC_SERVER_JOB_LIMIT".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
let mut server = BTreeMap::new();
|
||||
for row in rows {
|
||||
server.insert(row.get::<Uuid, _>("job_id"), row);
|
||||
}
|
||||
if previous.is_none() {
|
||||
let missing:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND NOT(job_id=ANY($2)))").bind(agent).bind(locals.keys().copied().collect::<Vec<_>>()).fetch_one(&mut **tx).await?;
|
||||
if missing {
|
||||
reason = "JOURNAL_HISTORY_MISSING".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
}
|
||||
let mut ready = chain_valid && r.journal_complete;
|
||||
let mut decisions = Vec::new();
|
||||
let mut seen = BTreeSet::new();
|
||||
for (job_id, local) in locals {
|
||||
let attempt = Uuid::parse_str(&local.attempt_id)?;
|
||||
let prior_hash = if local.outcome_sha256.is_empty() {
|
||||
None
|
||||
} else {
|
||||
ensure!(digest(&local.outcome_sha256), "outcome digest");
|
||||
Some(local.outcome_sha256.clone())
|
||||
};
|
||||
let mut decision = AttemptDecision {
|
||||
job_id,
|
||||
attempt_id: attempt,
|
||||
job_payload_hash: local.job_payload_hash.clone(),
|
||||
grant_id: optional_id(&local.grant_id)?,
|
||||
authorization_id: None,
|
||||
plan_hash: None,
|
||||
action: String::new(),
|
||||
execution_identity: String::new(),
|
||||
revision: local.revision,
|
||||
outcome_state: "unknown".into(),
|
||||
previous_outcome_sha256: prior_hash.clone(),
|
||||
outcome_sha256: None,
|
||||
outcome_bytes: vec![],
|
||||
original_boot_id: optional_id(&local.original_boot_id)?,
|
||||
unlock: false,
|
||||
reason: "MISSING_SERVER_JOB".into(),
|
||||
};
|
||||
let Some(row) = server.get(&job_id) else {
|
||||
sqlx::query(
|
||||
"UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1",
|
||||
)
|
||||
.bind(agent)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
ready = false;
|
||||
reason = "MISSING_SERVER_JOB".into();
|
||||
decisions.push(decision);
|
||||
continue;
|
||||
};
|
||||
seen.insert(job_id);
|
||||
let payload: JobPayload = serde_json::from_slice(&row.get::<Vec<u8>, _>("payload"))?;
|
||||
decision.action = payload.action.name().into();
|
||||
decision.execution_identity = payload.execution_identity.clone();
|
||||
if row.get::<Uuid, _>("attempt_id") != attempt
|
||||
|| row.get::<String, _>("payload_hash") != local.job_payload_hash
|
||||
|| hash(&row.get::<Vec<u8>, _>("payload")) != local.job_payload_hash
|
||||
|| payload.tenant_id != tenant
|
||||
|| payload.agent_id != agent
|
||||
|| payload.asset_id != device.get::<Uuid, _>("asset_id")
|
||||
|| payload.enrollment_generation != device.get::<i64, _>("enrollment_generation") as u64
|
||||
|| payload.assignment_version != device.get::<i64, _>("assignment_version") as u64
|
||||
{
|
||||
sqlx::query(
|
||||
"UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1",
|
||||
)
|
||||
.bind(agent)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
ready = false;
|
||||
decision.reason = "ATTEMPT_BINDING_MISMATCH".into();
|
||||
decisions.push(decision);
|
||||
continue;
|
||||
}
|
||||
let grant = sqlx::query(
|
||||
"SELECT grant_id,payload FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
||||
)
|
||||
.bind(job_id)
|
||||
.bind(attempt)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?;
|
||||
if let Some(grant) = &grant {
|
||||
let p: ExecutionPermitPayload =
|
||||
serde_json::from_slice(&grant.get::<Vec<u8>, _>("payload"))?;
|
||||
decision.authorization_id = Some(p.authorization_id);
|
||||
decision.plan_hash = Some(p.plan_hash);
|
||||
}
|
||||
let reserved = row.get::<bool, _>("start_reserved");
|
||||
let server_state: String = row.get("state");
|
||||
if clean_unstarted(local, reserved)
|
||||
&& matches!(server_state.as_str(), "queued" | "accepted")
|
||||
{
|
||||
sqlx::query("UPDATE jobs SET state='accepted',journal_proof=COALESCE(journal_proof,$2),revision=GREATEST(revision,$3) WHERE job_id=$1").bind(job_id).bind(format!("sync:{journal_id}:{}",r.checkpoint)).bind(i64::try_from(local.revision)?).execute(&mut **tx).await?;
|
||||
sqlx::query(
|
||||
"UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1",
|
||||
)
|
||||
.bind(job_id)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
decision.outcome_state = "accepted".into();
|
||||
decision.reason = "ACCEPTED_NOT_STARTED".into();
|
||||
decision.revision = (row.get::<i64, _>("revision") as u64).max(local.revision);
|
||||
decisions.push(decision);
|
||||
continue;
|
||||
}
|
||||
let grant_matches = grant
|
||||
.as_ref()
|
||||
.is_some_and(|g| Some(g.get::<Uuid, _>("grant_id")) == decision.grant_id);
|
||||
let unsafe_claim =
|
||||
local.state == "requesting" || (!grant_matches && local.consumption != "unconsumed");
|
||||
if unsafe_claim {
|
||||
if unsettled(&server_state) {
|
||||
mark_unknown(tx, job_id, "SYNC_GRANT_OR_EFFECT_UNRESOLVED").await?;
|
||||
}
|
||||
ready = false;
|
||||
decision.reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
|
||||
decisions.push(decision);
|
||||
continue;
|
||||
}
|
||||
// A free script with no bound postcondition never becomes successful by
|
||||
// merely asserting a new state after unknown effect.
|
||||
let manual = server_state == "unknown"
|
||||
&& matches!(
|
||||
payload.action,
|
||||
Action::RunScript {
|
||||
expected_stdout: None,
|
||||
..
|
||||
}
|
||||
);
|
||||
let embedded =
|
||||
local
|
||||
.result_envelope
|
||||
.as_ref()
|
||||
.and_then(|envelope| match &envelope.payload {
|
||||
Some(Payload::JobResult(result)) => Some(result),
|
||||
_ => None,
|
||||
});
|
||||
if let Some(result) = local.result.as_ref().or(embedded) {
|
||||
ensure!(
|
||||
result.job_id == local.job_id
|
||||
&& result.attempt_id == local.attempt_id
|
||||
&& result.revision == local.revision
|
||||
&& result.state == local.state
|
||||
&& result.grant_id == local.grant_id,
|
||||
"sync result binding"
|
||||
);
|
||||
if !manual {
|
||||
let evidence = local
|
||||
.result_envelope
|
||||
.as_ref()
|
||||
.context("original durable result envelope required")?;
|
||||
ensure!(
|
||||
valid_envelope(evidence, &agent.to_string(), evidence.connection_generation)
|
||||
&& evidence.message_id == local.result_message_id
|
||||
&& embedded == Some(result),
|
||||
"original result envelope binding"
|
||||
);
|
||||
let observed = chrono::DateTime::from_timestamp_millis(evidence.observed_at)
|
||||
.context("result timestamp")?;
|
||||
ensure!(
|
||||
observed <= Utc::now() + Duration::seconds(45),
|
||||
"future result timestamp"
|
||||
);
|
||||
crate::job_gateway::event(tx, tenant, agent, evidence).await?;
|
||||
}
|
||||
}
|
||||
let mut final_row = sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1")
|
||||
.bind(job_id)
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
// A reboot's postcondition is independently tied to persisted current-
|
||||
// boot inventory and the original boot in the durable reservation audit.
|
||||
let mut reboot_reconciled = false;
|
||||
if matches!(payload.action, Action::Reboot { .. })
|
||||
&& grant_matches
|
||||
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
|
||||
{
|
||||
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(job_id.to_string()).fetch_optional(&mut **tx).await?;
|
||||
let inventory:Option<Uuid>=sqlx::query_scalar("SELECT message_id FROM observations WHERE agent_id=$1 AND boot_id=$2 AND received_at>now()-interval '5 minutes' AND received_at>(SELECT created_at FROM execution_permits WHERE job_id=$3) ORDER BY received_at DESC LIMIT 1").bind(agent).bind(boot).bind(job_id).fetch_optional(&mut **tx).await?;
|
||||
if let (Some(original), Some(inventory)) = (original, inventory)
|
||||
&& original != e.boot_id
|
||||
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
|
||||
{
|
||||
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true}).to_string(),error_code:String::new()};
|
||||
let mut evidence = e.clone();
|
||||
evidence.message_id = Uuid::new_v4().to_string();
|
||||
evidence.payload = Some(Payload::JobResult(result));
|
||||
if unsettled(&final_row.get::<String, _>("state")) {
|
||||
crate::job_gateway::event(tx, tenant, agent, &evidence).await?;
|
||||
reboot_reconciled = true;
|
||||
final_row =
|
||||
sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1")
|
||||
.bind(job_id)
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
let terminal: String = final_row.get("state");
|
||||
if matches!(payload.action, Action::Reboot { .. })
|
||||
&& grant_matches
|
||||
&& terminal == "succeeded"
|
||||
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
|
||||
&& let Some(value) = final_row.get::<Option<Value>, _>("result")
|
||||
{
|
||||
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(job_id.to_string()).fetch_optional(&mut **tx).await?;
|
||||
if original.as_ref().and_then(|v| Uuid::parse_str(v).ok()) == decision.original_boot_id
|
||||
&& decision.original_boot_id.is_some()
|
||||
&& value["evidence"]["postconditions_passed"] == json!(true)
|
||||
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
|
||||
&& value["evidence"]["new_boot_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| Some(id) != original.as_deref())
|
||||
{
|
||||
reboot_reconciled = true;
|
||||
}
|
||||
}
|
||||
decision.revision = final_row.get::<i64, _>("revision") as u64;
|
||||
decision.outcome_state = terminal.clone();
|
||||
if !unsettled(&terminal) && !manual {
|
||||
if let Some(value) = final_row.get::<Option<Value>, _>("result") {
|
||||
let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?;
|
||||
let hash = hash(&bytes);
|
||||
if prior_hash.as_deref() == Some(&hash) || reboot_reconciled {
|
||||
decision.outcome_sha256 = Some(hash);
|
||||
decision.unlock = local.consumption == "consumed" && grant_matches;
|
||||
if reboot_reconciled {
|
||||
decision.outcome_bytes = bytes;
|
||||
}
|
||||
decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into();
|
||||
} else {
|
||||
ready = false;
|
||||
decision.reason = "LOCAL_OUTCOME_MISMATCH".into();
|
||||
}
|
||||
} else if local.consumption == "unconsumed" && !reserved {
|
||||
decision.reason = "DEFINITIVE_NONSTART".into();
|
||||
} else {
|
||||
ready = false;
|
||||
decision.reason = "DEFINITIVE_OUTCOME_MISSING".into();
|
||||
}
|
||||
} else {
|
||||
if unsettled(&terminal) {
|
||||
mark_unknown(tx, job_id, "SYNC_EFFECT_UNRESOLVED").await?;
|
||||
}
|
||||
ready = false;
|
||||
decision.outcome_state = "unknown".into();
|
||||
decision.reason = if manual {
|
||||
"MANUAL_RECONCILIATION_REQUIRED"
|
||||
} else {
|
||||
"EFFECT_UNRESOLVED"
|
||||
}
|
||||
.into();
|
||||
}
|
||||
if payload.execution_identity == "root" && !helper_ready {
|
||||
ready = false;
|
||||
decision.unlock = false;
|
||||
decision.reason = "HELPER_JOURNAL_REQUIRED".into();
|
||||
}
|
||||
decisions.push(decision);
|
||||
}
|
||||
for (id, row) in server {
|
||||
if !seen.contains(&id)
|
||||
&& (row.get::<bool, _>("start_reserved")
|
||||
|| row.get::<String, _>("state") != "queued"
|
||||
|| row
|
||||
.get::<Option<chrono::DateTime<Utc>>, _>("acknowledged_at")
|
||||
.is_some())
|
||||
{
|
||||
if unsettled(&row.get::<String, _>("state")) {
|
||||
mark_unknown(tx, id, "SYNC_LOCAL_ATTEMPT_MISSING").await?;
|
||||
}
|
||||
ready = false;
|
||||
reason = "LOCAL_ATTEMPT_MISSING".into();
|
||||
}
|
||||
}
|
||||
if !ready && reason == "CONSISTENT" {
|
||||
reason = "ATTEMPT_RECONCILIATION_REQUIRED".into();
|
||||
}
|
||||
helper_ready &= ready;
|
||||
if !chain_valid {
|
||||
for decision in &mut decisions {
|
||||
decision.unlock = false;
|
||||
}
|
||||
}
|
||||
let mut receipts = Vec::new();
|
||||
for raw in &r.durable_message_ids {
|
||||
let id = Uuid::parse_str(raw)?;
|
||||
let persisted:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM job_receipts WHERE agent_id=$1 AND message_id=$2 UNION ALL SELECT 1 FROM observations WHERE agent_id=$1 AND message_id=$2)").bind(agent).bind(id).fetch_one(&mut **tx).await?;
|
||||
if persisted {
|
||||
receipts.push(id)
|
||||
}
|
||||
}
|
||||
receipts.sort();
|
||||
receipts.dedup();
|
||||
let now = Utc::now();
|
||||
let decision = ReconcileDecisionPayload {
|
||||
domain: "rmm.reconciliation-decision".into(),
|
||||
schema_version: 1,
|
||||
decision_id: Uuid::new_v4(),
|
||||
request_id,
|
||||
request_hash: request_hash.clone(),
|
||||
tenant_id: tenant,
|
||||
asset_id: device.get("asset_id"),
|
||||
agent_id: agent,
|
||||
enrollment_generation: device.get::<i64, _>("enrollment_generation") as u64,
|
||||
assignment_version: device.get::<i64, _>("assignment_version") as u64,
|
||||
connection_generation: e.connection_generation,
|
||||
boot_id: boot,
|
||||
journal_id,
|
||||
checkpoint: r.checkpoint,
|
||||
helper_journal_id: helper_journal,
|
||||
helper_checkpoint: r.helper_checkpoint,
|
||||
issued_at: now,
|
||||
valid_until: now + Duration::seconds(60),
|
||||
ready,
|
||||
helper_ready,
|
||||
reason,
|
||||
attempts: decisions,
|
||||
acknowledged_message_ids: receipts,
|
||||
key_id: state.job_signer.key_id.clone(),
|
||||
};
|
||||
let bytes = encode(&decision)?;
|
||||
ensure!(bytes.len() <= 512 * 1024, "sync decision bound");
|
||||
let signature = state.job_signer.sign(&bytes)?;
|
||||
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)").bind(tenant).bind(agent).bind(request_id).bind(decision.decision_id).bind(request_hash).bind(journal_id).bind(checkpoint).bind(i64::try_from(e.connection_generation)?).bind(&bytes).bind(&signature).execute(&mut **tx).await?;
|
||||
if chain_valid {
|
||||
sqlx::query("INSERT INTO agent_sync_state(tenant_id,agent_id,journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint) VALUES($1,$2,$3,$4,$5,$6,$7) ON CONFLICT(tenant_id,agent_id) DO UPDATE SET checkpoint=GREATEST(agent_sync_state.checkpoint,EXCLUDED.checkpoint),decision_id=EXCLUDED.decision_id,helper_journal_id=COALESCE(agent_sync_state.helper_journal_id,EXCLUDED.helper_journal_id),helper_checkpoint=GREATEST(agent_sync_state.helper_checkpoint,EXCLUDED.helper_checkpoint)").bind(tenant).bind(agent).bind(journal_id).bind(checkpoint).bind(decision.decision_id).bind(helper_journal).bind(helper_checkpoint).execute(&mut **tx).await?;
|
||||
}
|
||||
sqlx::query("UPDATE agent_installations SET sync_ready=$2,helper_sync_ready=$3,restore_blocked=NOT $2 WHERE agent_id=$1").bind(agent).bind(ready).bind(helper_ready).execute(&mut **tx).await?;
|
||||
Ok(SyncResponse {
|
||||
decision_bytes: bytes,
|
||||
signature,
|
||||
})
|
||||
}
|
||||
async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> {
|
||||
sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn accepted_is_not_a_new_attempt_or_grant() {
|
||||
let mut local = SyncAttempt {
|
||||
state: "accepted".into(),
|
||||
consumption: "unconsumed".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(clean_unstarted(&local, false));
|
||||
assert!(!clean_unstarted(&local, true));
|
||||
local.state = "requesting".into();
|
||||
assert!(!clean_unstarted(&local, false));
|
||||
local.state = "running".into();
|
||||
assert!(!clean_unstarted(&local, false));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user