//! Authoritative reconciliation only: this module never inserts jobs, attempts, //! outbox entries or execution permits. A signed decision is not start authority. use anyhow::{Context, Result, ensure}; use chrono::{Duration, Utc}; use prost::Message; use rmm_domain::{jobs::*, reconciliation::*}; use rmm_protocol::*; use serde_json::{Value, json}; use sqlx::{Postgres, Row, Transaction}; use std::collections::{BTreeMap, BTreeSet}; use uuid::Uuid; fn optional_id(raw: &str) -> Result> { if raw.is_empty() { Ok(None) } else { let id = Uuid::parse_str(raw)?; ensure!(!id.is_nil(), "nil sync identity"); Ok(Some(id)) } } fn digest(raw: &str) -> bool { raw.len() == 64 && raw .bytes() .all(|c| c.is_ascii_digit() || b"abcdef".contains(&c)) } fn outcome(state: &str, value: &Value) -> Result { Ok(ReconciledOutcome { state: state.into(), stdout: serde_json::from_value(value["stdout"].clone())?, stderr: serde_json::from_value(value["stderr"].clone())?, exit_code: serde_json::from_value(value["exit_code"].clone())?, output_truncated: value["output_truncated"] .as_bool() .context("output bound evidence missing")?, reboot_required: value["reboot_required"] .as_bool() .context("reboot evidence missing")?, evidence: value["evidence"].clone(), error_code: value["error_code"] .as_str() .context("error evidence missing")? .into(), }) } fn unsettled(state: &str) -> bool { !matches!(state, "succeeded" | "failed" | "cancelled" | "expired") } fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool { !reserved && local.consumption == "unconsumed" && local.grant_id.is_empty() && matches!(local.state.as_str(), "queued" | "accepted") } /// A missing reply and a request that never committed are different cases. /// A committed pending request is the newer anchor. Otherwise only the last /// applied decision can anchor the chain; the caller still compares it with /// the authoritative head and performs all journal/generation checks. fn previous_anchor( last_applied: Option, pending_request: Option, committed_pending: Option, ) -> Option { if pending_request.is_some() { committed_pending.or(last_applied) } else { last_applied } } /// The server's durable grant reservation precedes the helper's saved start. /// Using that trusted timestamp is conservative: no device clock or late /// reconnect may extend the fixed, approved reconnect window. fn reboot_deadline( action: &Action, reserved_at: Option>, ) -> Option> { let Action::Reboot { reconnect_timeout_seconds, .. } = action else { return None; }; if !action.validate() { return None; } reserved_at?.checked_add_signed(Duration::seconds(i64::from(*reconnect_timeout_seconds))) } struct RebootWindow { reserved_at: chrono::DateTime, deadline: chrono::DateTime, now: chrono::DateTime, require_fresh: bool, } async fn reboot_fresh_inventory( tx: &mut Transaction<'_, Postgres>, tenant: Uuid, agent: Uuid, boot: Uuid, now: chrono::DateTime, receipt: Option, ) -> Result)>> { Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1") .bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt) .fetch_optional(&mut **tx).await?) } async fn reboot_inventory( tx: &mut Transaction<'_, Postgres>, tenant: Uuid, agent: Uuid, boot: Uuid, window: &RebootWindow, existing_receipt: Option, ) -> Result)>> { // Already-persisted definitive success keeps its original receipt. It need // not be younger than five minutes again, but its original server receipt // time must still prove the approved reconnect window and current boot. let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1") .bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt) .fetch_optional(&mut **tx).await?; // Verification may be delayed by local bootstrap readiness. Never move // the original reconnect cutoff: retain the actual timely receipt and // independently demand fresh evidence of the SAME current boot. if timely.is_some() && window.require_fresh && reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None) .await? .is_none() { return Ok(None); } Ok(timely) } /// First ingress of success needs the same independent, timely receipt as an /// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing. pub(crate) async fn verify_reboot_success( tx: &mut Transaction<'_, Postgres>, tenant: Uuid, agent: Uuid, envelope: &Envelope, job: &JobPayload, result: &JobResult, evidence: &Value, ) -> Result<()> { let boot = Uuid::parse_str(&envelope.boot_id)?; let grant = Uuid::parse_str(&result.grant_id)?; let now = Utc::now(); ensure!( job.tenant_id == tenant && job.agent_id == agent && job.job_id.to_string() == result.job_id && job.attempt_id.to_string() == result.attempt_id && evidence["postconditions_passed"] == true && evidence["reconciled_after_reboot"] == true && evidence["new_boot_id"] == envelope.boot_id, "REBOOT_RESULT_BINDING" ); let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)") .bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?; ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED"); let reserved:chrono::DateTime=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4") .bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?; let deadline = reboot_deadline(&job.action, Some(reserved)).context("REBOOT_FIXED_DEADLINE_REQUIRED")?; let original:Option=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1") .bind(tenant).bind(job.job_id.to_string()).bind(grant.to_string()).fetch_optional(&mut **tx).await?; ensure!( original .as_deref() .and_then(|v| Uuid::parse_str(v).ok()) .is_some_and(|old| !old.is_nil() && old != boot), "REBOOT_ORIGINAL_BOOT_REQUIRED" ); let receipt = evidence["inventory_message_id"] .as_str() .and_then(|v| Uuid::parse_str(v).ok()) .filter(|id| !id.is_nil()) .context("REBOOT_PERSISTED_INVENTORY_REQUIRED")?; let (_, received) = reboot_inventory( tx, tenant, agent, boot, &RebootWindow { reserved_at: reserved, deadline, now, require_fresh: true, }, Some(receipt), ) .await? .context("REBOOT_TIMELY_INVENTORY_REQUIRED")?; let verification_id = evidence .get("verification_inventory_message_id") .and_then(Value::as_str) .map(Uuid::parse_str) .transpose()?; let verification_time = evidence.get("verification_inventory_received_at"); let verification = if verification_id.is_some() || verification_time.is_some() || received <= now - Duration::minutes(5) { let id = verification_id .filter(|id| !id.is_nil()) .context("REBOOT_FRESH_VERIFICATION_REQUIRED")?; let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id)) .await? .context("REBOOT_FRESH_VERIFICATION_REQUIRED")?; ensure!( serde_json::from_value::>( verification_time.cloned().unwrap_or(Value::Null) ) .ok() == Some(time), "REBOOT_VERIFICATION_TIME_BINDING" ); (id, time) } else { (receipt, received) }; for (key, actual) in [ ("reboot_reserved_at", reserved), ("reconnect_deadline", deadline), ("inventory_received_at", received), ] { ensure!( serde_json::from_value::>(evidence[key].clone()).ok() == Some(actual), "REBOOT_SERVER_TIME_BINDING" ); } sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())") .bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot, "connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline, "inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1})) .execute(&mut **tx).await?; Ok(()) } pub async fn reconcile( state: &crate::State, tx: &mut Transaction<'_, Postgres>, tenant: Uuid, agent: Uuid, e: &Envelope, r: &SyncRequest, ) -> Result { ensure!( r.attempts.len() <= 256 && r.durable_message_ids.len() <= 256, "sync count bound" ); let request_id = optional_id(&r.request_id)?.context("sync request id required")?; let journal_id = optional_id(&r.journal_id)?.context("journal identity required")?; let helper_journal = optional_id(&r.helper_journal_id)?; let boot = Uuid::parse_str(&e.boot_id)?; let checkpoint = i64::try_from(r.checkpoint)?; let helper_checkpoint = i64::try_from(r.helper_checkpoint)?; let request_hash = hash(&r.encode_to_vec()); if let Some(cached)=sqlx::query("SELECT request_hash,connection_generation,payload,signature FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2").bind(agent).bind(request_id).fetch_optional(&mut **tx).await?{ ensure!(cached.get::("request_hash")==request_hash,"sync request id conflict"); ensure!(cached.get::("connection_generation")==i64::try_from(e.connection_generation)?,"old sync request: recover via previous_request_id in a fresh request"); return Ok(SyncResponse{decision_bytes:cached.get("payload"),signature:cached.get("signature")}); } let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,sequence_regressed,sync_recovery_required,revoked_at FROM agent_installations WHERE agent_id=$1 FOR UPDATE").bind(agent).fetch_one(&mut **tx).await?; ensure!( device .get::>, _>("revoked_at") .is_none() && device.get::("connection_generation") == i64::try_from(e.connection_generation)? && device.get::, _>("connection_boot_id") == Some(boot), "sync channel fenced" ); let previous=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE agent_id=$1").bind(agent).fetch_optional(&mut **tx).await?; let mut reason = if r.journal_complete { "CONSISTENT" } else { "JOURNAL_INCOMPLETE" } .to_string(); let mut chain_valid = true; if device.get::("sync_recovery_required") { reason = "MANUAL_SERVER_LEDGER_RECOVERY_REQUIRED".into(); chain_valid = false; } if previous.is_none() { let history: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE agent_id=$1)") .bind(agent) .fetch_one(&mut **tx) .await?; if history { reason = "SERVER_STATE_ROLLBACK".into(); chain_valid = false; } } if device.get::("assignment_version") != i64::try_from(r.assignment_version)? { reason = "ASSIGNMENT_MISMATCH".into(); chain_valid = false; } if device.get::("sequence_regressed") { reason = "JOURNAL_SEQUENCE_REGRESSION".into(); chain_valid = false; } let last_applied = optional_id(&r.previous_decision_id)?; let pending_request = optional_id(&r.previous_request_id)?; let committed_pending = if let Some(pending) = pending_request { sqlx::query_scalar("SELECT decision_id FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2 AND journal_id=$3").bind(agent).bind(pending).bind(journal_id).fetch_optional(&mut **tx).await? } else { None }; let previous_decision = previous_anchor(last_applied, pending_request, committed_pending); if pending_request.is_some() && previous_decision.is_none() { reason = "SERVER_CHECKPOINT_MISSING".into(); chain_valid = false; } if let Some(old) = &previous { if old.get::("journal_id") != journal_id || checkpoint < old.get::("checkpoint") { reason = "JOURNAL_ROLLBACK_OR_REPLACEMENT".into(); chain_valid = false; } if previous_decision != Some(old.get::("decision_id")) && reason != "SERVER_CHECKPOINT_MISSING" { reason = "CHECKPOINT_CHAIN_MISMATCH".into(); chain_valid = false; } if let Some(helper) = helper_journal && (old .get::, _>("helper_journal_id") .is_some_and(|id| id != helper) || helper_checkpoint < old.get::("helper_checkpoint")) { reason = "HELPER_JOURNAL_ROLLBACK".into(); chain_valid = false; } } else if previous_decision.is_some() || !r.helper_previous_decision_id.is_empty() { reason = "SERVER_CHECKPOINT_MISSING".into(); chain_valid = false; } let mut helper_ready = helper_journal.is_some(); if helper_ready && !r.helper_previous_decision_id.is_empty() { let previous_helper = sqlx::query_scalar::<_, Vec>( "SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2", ) .bind(agent) .bind(Uuid::parse_str(&r.helper_previous_decision_id)?) .fetch_optional(&mut **tx) .await?; if let Some(bytes) = previous_helper { let old: ReconcileDecisionPayload = serde_json::from_slice(&bytes)?; if old.helper_journal_id != helper_journal || old.helper_checkpoint > r.helper_checkpoint { helper_ready = false; reason = "HELPER_CHECKPOINT_MISMATCH".into(); chain_valid = false; } } else { helper_ready = false; reason = "HELPER_CHECKPOINT_MISSING".into(); chain_valid = false; } } else if helper_ready && previous .as_ref() .is_some_and(|old| old.get::, _>("helper_journal_id").is_some()) { helper_ready = false; reason = "HELPER_CHECKPOINT_REQUIRED".into(); chain_valid = false; } let mut locals = BTreeMap::new(); for local in &r.attempts { let id = Uuid::parse_str(&local.job_id)?; ensure!( !id.is_nil() && locals.insert(id, local).is_none(), "duplicate local attempt" ); Uuid::parse_str(&local.attempt_id)?; ensure!( digest(&local.job_payload_hash) && matches!( local.consumption.as_str(), "unconsumed" | "consumed" | "uncertain" ), "invalid attempt summary" ); i64::try_from(local.revision)?; ensure!( matches!( local.state.as_str(), "queued" | "accepted" | "requesting" | "running" | "verifying" | "awaiting_reboot" | "unknown" | "succeeded" | "failed" | "cancelled" | "expired" ), "invalid local state" ); ensure!( local.outcome_bytes.len() <= 512 * 1024, "local outcome bound" ); if !local.outcome_bytes.is_empty() { ensure!( digest(&local.outcome_sha256) && hash(&local.outcome_bytes) == local.outcome_sha256, "local outcome hash" ); let parsed: ReconciledOutcome = serde_json::from_slice(&local.outcome_bytes)?; ensure!( canonical_outcome_bytes(&parsed)? == local.outcome_bytes, "noncanonical local outcome bytes" ); ensure!(parsed.state == local.state, "local outcome state mismatch"); } } let rows=sqlx::query("SELECT j.job_id,j.attempt_id,j.payload_hash,j.payload,j.state,j.revision,j.result,j.start_reserved,b.acknowledged_at FROM jobs j LEFT JOIN outbox b ON b.tenant_id=j.tenant_id AND b.job_id=j.job_id WHERE j.agent_id=$1 AND (j.state NOT IN ('succeeded','failed','cancelled','expired') OR j.job_id=ANY($2)) ORDER BY j.job_id LIMIT 257 FOR UPDATE OF j").bind(agent).bind(locals.keys().copied().collect::>()).fetch_all(&mut **tx).await?; if rows.len() > 256 { reason = "SYNC_SERVER_JOB_LIMIT".into(); chain_valid = false; } let mut server = BTreeMap::new(); for row in rows { server.insert(row.get::("job_id"), row); } if previous.is_none() { let missing:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM jobs WHERE agent_id=$1 AND start_reserved AND NOT(job_id=ANY($2)))").bind(agent).bind(locals.keys().copied().collect::>()).fetch_one(&mut **tx).await?; if missing { reason = "JOURNAL_HISTORY_MISSING".into(); chain_valid = false; } } let mut ready = chain_valid && r.journal_complete; let mut decisions = Vec::new(); let mut seen = BTreeSet::new(); for (job_id, local) in locals { let attempt = Uuid::parse_str(&local.attempt_id)?; let prior_hash = if local.outcome_sha256.is_empty() { None } else { ensure!(digest(&local.outcome_sha256), "outcome digest"); Some(local.outcome_sha256.clone()) }; let mut decision = AttemptDecision { job_id, attempt_id: attempt, job_payload_hash: local.job_payload_hash.clone(), grant_id: optional_id(&local.grant_id)?, authorization_id: None, plan_hash: None, action: String::new(), execution_identity: String::new(), revision: local.revision, outcome_state: "unknown".into(), previous_outcome_sha256: prior_hash.clone(), outcome_sha256: None, outcome_bytes: vec![], original_boot_id: optional_id(&local.original_boot_id)?, unlock: false, reason: "MISSING_SERVER_JOB".into(), }; let Some(row) = server.get(&job_id) else { sqlx::query( "UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1", ) .bind(agent) .execute(&mut **tx) .await?; ready = false; reason = "MISSING_SERVER_JOB".into(); decisions.push(decision); continue; }; seen.insert(job_id); let payload: JobPayload = serde_json::from_slice(&row.get::, _>("payload"))?; decision.action = payload.action.name().into(); decision.execution_identity = payload.execution_identity.clone(); if row.get::("attempt_id") != attempt || row.get::("payload_hash") != local.job_payload_hash || hash(&row.get::, _>("payload")) != local.job_payload_hash || payload.tenant_id != tenant || payload.agent_id != agent || payload.asset_id != device.get::("asset_id") || payload.enrollment_generation != device.get::("enrollment_generation") as u64 || payload.assignment_version != device.get::("assignment_version") as u64 { sqlx::query( "UPDATE agent_installations SET sync_recovery_required=true WHERE agent_id=$1", ) .bind(agent) .execute(&mut **tx) .await?; ready = false; decision.reason = "ATTEMPT_BINDING_MISMATCH".into(); decisions.push(decision); continue; } let missing_helper = crate::root_policy::needs_helper(&payload.execution_identity) && !helper_ready; if missing_helper { ready = false; } let grant = sqlx::query( "SELECT grant_id,payload,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2", ) .bind(job_id) .bind(attempt) .fetch_optional(&mut **tx) .await?; if let Some(grant) = &grant { let p: ExecutionPermitPayload = serde_json::from_slice(&grant.get::, _>("payload"))?; decision.authorization_id = Some(p.authorization_id); decision.plan_hash = Some(p.plan_hash); } let reserved = row.get::("start_reserved"); let server_state: String = row.get("state"); if clean_unstarted(local, reserved) && matches!(server_state.as_str(), "queued" | "accepted") { sqlx::query("UPDATE jobs SET state='accepted',journal_proof=COALESCE(journal_proof,$2),revision=GREATEST(revision,$3) WHERE job_id=$1").bind(job_id).bind(format!("sync:{journal_id}:{}",r.checkpoint)).bind(i64::try_from(local.revision)?).execute(&mut **tx).await?; sqlx::query( "UPDATE outbox SET acknowledged_at=COALESCE(acknowledged_at,now()) WHERE job_id=$1", ) .bind(job_id) .execute(&mut **tx) .await?; decision.outcome_state = "accepted".into(); decision.reason = "ACCEPTED_NOT_STARTED".into(); decision.revision = (row.get::("revision") as u64).max(local.revision); decisions.push(decision); continue; } let grant_matches = grant .as_ref() .is_some_and(|g| Some(g.get::("grant_id")) == decision.grant_id); let unsafe_claim = local.state == "requesting" || (!grant_matches && local.consumption != "unconsumed"); if unsafe_claim { if unsettled(&server_state) { mark_unknown(tx, job_id, "SYNC_GRANT_OR_EFFECT_UNRESOLVED").await?; } ready = false; decision.reason = "GRANT_OR_EFFECT_UNRESOLVED".into(); decisions.push(decision); continue; } // A free script with no bound postcondition never becomes successful by // merely asserting a new state after unknown effect. let manual = server_state == "unknown" && matches!( payload.action, Action::RunScript { expected_stdout: None, .. } ); let embedded = local .result_envelope .as_ref() .and_then(|envelope| match &envelope.payload { Some(Payload::JobResult(result)) => Some(result), _ => None, }); if let Some(result) = local.result.as_ref().or(embedded) { ensure!( result.job_id == local.job_id && result.attempt_id == local.attempt_id && result.revision == local.revision && result.state == local.state && result.grant_id == local.grant_id, "sync result binding" ); if !manual { let evidence = local .result_envelope .as_ref() .context("original durable result envelope required")?; ensure!( valid_envelope(evidence, &agent.to_string(), evidence.connection_generation) && evidence.message_id == local.result_message_id && embedded == Some(result), "original result envelope binding" ); let observed = chrono::DateTime::from_timestamp_millis(evidence.observed_at) .context("result timestamp")?; ensure!( observed <= Utc::now() + Duration::seconds(45), "future result timestamp" ); crate::job_gateway::event(tx, tenant, agent, evidence).await?; } } let mut final_row = sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1") .bind(job_id) .fetch_one(&mut **tx) .await?; // A reboot's postcondition is independently tied to persisted current- // boot inventory and the original boot in the durable reservation audit. let mut reboot_reconciled = false; let mut reboot_witness = None; if matches!(payload.action, Action::Reboot { .. }) && grant_matches && matches!(local.state.as_str(), "awaiting_reboot" | "unknown") { let original:Option=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1").bind(tenant).bind(job_id.to_string()).bind(&local.grant_id).fetch_optional(&mut **tx).await?; let reserved_at = grant .as_ref() .map(|g| g.get::, _>("created_at")); let deadline = reboot_deadline(&payload.action, reserved_at); let inventory = if let (Some(reserved), Some(until)) = (reserved_at, deadline) { let existing_receipt = if final_row.get::("state") == "succeeded" { final_row .get::, _>("result") .and_then(|value| { value["evidence"]["inventory_message_id"] .as_str() .and_then(|v| Uuid::parse_str(v).ok()) }) } else { None }; reboot_inventory( tx, tenant, agent, boot, &RebootWindow { reserved_at: reserved, deadline: until, now: Utc::now(), require_fresh: existing_receipt.is_none(), }, existing_receipt, ) .await? } else { None }; if let (Some(original), Some((inventory, received_at)), Some(reserved), Some(until)) = (original, inventory, reserved_at, deadline) && original != e.boot_id && decision.original_boot_id == Some(Uuid::parse_str(&original)?) { reboot_witness = Some((original, inventory)); let verification = reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?; let revision = (final_row.get::("revision") as u64).max(local.revision) + 1; let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()}; let mut evidence = e.clone(); evidence.message_id = Uuid::new_v4().to_string(); evidence.payload = Some(Payload::JobResult(result)); if unsettled(&final_row.get::("state")) { crate::job_gateway::event(tx, tenant, agent, &evidence).await?; reboot_reconciled = true; final_row = sqlx::query("SELECT state,revision,result FROM jobs WHERE job_id=$1") .bind(job_id) .fetch_one(&mut **tx) .await?; } } } let terminal: String = final_row.get("state"); if matches!(payload.action, Action::Reboot { .. }) && grant_matches && terminal == "succeeded" && matches!(local.state.as_str(), "awaiting_reboot" | "unknown") && let Some(value) = final_row.get::, _>("result") && let Some((original, inventory)) = reboot_witness && Uuid::parse_str(&original).ok() == decision.original_boot_id && decision.original_boot_id.is_some() && value["evidence"]["postconditions_passed"] == json!(true) && value["evidence"]["reconciled_after_reboot"] == json!(true) && value["evidence"]["new_boot_id"] .as_str() .is_some_and(|id| id == e.boot_id) && value["evidence"]["inventory_message_id"] == json!(inventory) { reboot_reconciled = true; } decision.revision = final_row.get::("revision") as u64; decision.outcome_state = terminal.clone(); if !unsettled(&terminal) && !manual { if let Some(value) = final_row.get::, _>("result") { let bytes = canonical_outcome_bytes(&outcome(&terminal, &value)?)?; let hash = hash(&bytes); if prior_hash.as_deref() == Some(&hash) || reboot_reconciled { decision.outcome_sha256 = Some(hash); decision.unlock = local.consumption == "consumed" && grant_matches; if reboot_reconciled { decision.outcome_bytes = bytes; } decision.reason = "DEFINITIVE_OUTCOME_RECONCILED".into(); } else { ready = false; decision.reason = "LOCAL_OUTCOME_MISMATCH".into(); } } else if local.consumption == "unconsumed" && !reserved { decision.reason = "DEFINITIVE_NONSTART".into(); } else { ready = false; decision.reason = "DEFINITIVE_OUTCOME_MISSING".into(); } } else { if unsettled(&terminal) { mark_unknown(tx, job_id, "SYNC_EFFECT_UNRESOLVED").await?; } ready = false; decision.outcome_state = "unknown".into(); decision.reason = if manual { "MANUAL_RECONCILIATION_REQUIRED" } else { "EFFECT_UNRESOLVED" } .into(); } if missing_helper { decision.unlock = false; decision.reason = "HELPER_JOURNAL_REQUIRED".into(); } decisions.push(decision); } for (id, row) in server { if !seen.contains(&id) && (row.get::("start_reserved") || row.get::("state") != "queued" || row .get::>, _>("acknowledged_at") .is_some()) { if unsettled(&row.get::("state")) { mark_unknown(tx, id, "SYNC_LOCAL_ATTEMPT_MISSING").await?; } ready = false; reason = "LOCAL_ATTEMPT_MISSING".into(); } } if !ready && reason == "CONSISTENT" { reason = "ATTEMPT_RECONCILIATION_REQUIRED".into(); } helper_ready &= ready; if !chain_valid { for decision in &mut decisions { decision.unlock = false; } } let mut receipts = Vec::new(); for raw in &r.durable_message_ids { let id = Uuid::parse_str(raw)?; let persisted:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM job_receipts WHERE agent_id=$1 AND message_id=$2 UNION ALL SELECT 1 FROM observations WHERE agent_id=$1 AND message_id=$2)").bind(agent).bind(id).fetch_one(&mut **tx).await?; if persisted { receipts.push(id) } } receipts.sort(); receipts.dedup(); let now = Utc::now(); let decision = ReconcileDecisionPayload { domain: "rmm.reconciliation-decision".into(), schema_version: 1, decision_id: Uuid::new_v4(), request_id, request_hash: request_hash.clone(), tenant_id: tenant, asset_id: device.get("asset_id"), agent_id: agent, enrollment_generation: device.get::("enrollment_generation") as u64, assignment_version: device.get::("assignment_version") as u64, connection_generation: e.connection_generation, boot_id: boot, journal_id, checkpoint: r.checkpoint, helper_journal_id: helper_journal, helper_checkpoint: r.helper_checkpoint, issued_at: now, valid_until: now + Duration::seconds(60), ready, helper_ready, reason, attempts: decisions, acknowledged_message_ids: receipts, key_id: state.job_signer.key_id.clone(), }; let bytes = encode(&decision)?; ensure!(bytes.len() <= 512 * 1024, "sync decision bound"); let signature = state.job_signer.sign(&bytes)?; sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)").bind(tenant).bind(agent).bind(request_id).bind(decision.decision_id).bind(request_hash).bind(journal_id).bind(checkpoint).bind(i64::try_from(e.connection_generation)?).bind(&bytes).bind(&signature).execute(&mut **tx).await?; if chain_valid { sqlx::query("INSERT INTO agent_sync_state(tenant_id,agent_id,journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint) VALUES($1,$2,$3,$4,$5,$6,$7) ON CONFLICT(tenant_id,agent_id) DO UPDATE SET checkpoint=GREATEST(agent_sync_state.checkpoint,EXCLUDED.checkpoint),decision_id=EXCLUDED.decision_id,helper_journal_id=COALESCE(agent_sync_state.helper_journal_id,EXCLUDED.helper_journal_id),helper_checkpoint=GREATEST(agent_sync_state.helper_checkpoint,EXCLUDED.helper_checkpoint)").bind(tenant).bind(agent).bind(journal_id).bind(checkpoint).bind(decision.decision_id).bind(helper_journal).bind(helper_checkpoint).execute(&mut **tx).await?; } sqlx::query("UPDATE agent_installations SET sync_ready=$2,helper_sync_ready=$3,restore_blocked=NOT $2 WHERE agent_id=$1").bind(agent).bind(ready).bind(helper_ready).execute(&mut **tx).await?; Ok(SyncResponse { decision_bytes: bytes, signature, }) } async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &str) -> Result<()> { sqlx::query("UPDATE jobs SET state='unknown',start_reserved=true,pending_reason='reconciliation_required',result=COALESCE(result,'{}'::jsonb)||jsonb_build_object('error_code',$2::text,'source_id','authoritative_sync') WHERE job_id=$1 AND state NOT IN ('succeeded','failed','cancelled','expired')").bind(job).bind(reason).execute(&mut **tx).await?; Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn reboot_deadline_uses_only_the_fixed_plan_and_trusted_reservation() { let reserved = Utc::now(); let action = Action::Reboot { reason: "Explicit final Cloud VM acceptance".into(), reconnect_timeout_seconds: 180, dependency_group: "single-test-vm".into(), }; assert_eq!( reboot_deadline(&action, Some(reserved)), Some(reserved + Duration::seconds(180)) ); assert_eq!(reboot_deadline(&action, None), None); assert_eq!( reboot_deadline( &Action::Reboot { reason: "test".into(), reconnect_timeout_seconds: 0, dependency_group: "single-test-vm".into() }, Some(reserved) ), None ); assert_eq!( reboot_deadline(&action, Some(chrono::DateTime::::MAX_UTC)), None ); } #[tokio::test] #[ignore = "requires the existing dedicated isolated PostgreSQL harness"] async fn reboot_inventory_requires_trusted_timely_same_identity_and_boot() { let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap()) .await .unwrap(); let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap()) .await .unwrap(); rmm_persistence::verify_application_role(&app) .await .unwrap(); let tenant = Uuid::new_v4(); let asset = Uuid::new_v4(); let agent = Uuid::new_v4(); let boot = Uuid::new_v4(); sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot deadline test')") .bind(tenant) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO assets VALUES($1,$2)") .bind(tenant) .bind(asset) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)") .bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap(); let now = Utc::now(); let reserved = now - Duration::seconds(200); let deadline = reserved + Duration::seconds(180); let window = RebootWindow { reserved_at: reserved, deadline, now, require_fresh: true, }; let timely = Uuid::new_v4(); let late = Uuid::new_v4(); for (sequence, message, received) in [ (1, timely, deadline), (2, late, deadline + Duration::microseconds(1)), (3, Uuid::new_v4(), reserved), ] { sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)") .bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&admin).await.unwrap(); } let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); assert_eq!( reboot_inventory(&mut tx, tenant, agent, boot, &window, None) .await .unwrap() .map(|v| v.0), Some(timely) ); assert!( reboot_inventory(&mut tx, tenant, agent, boot, &window, Some(late)) .await .unwrap() .is_none() ); assert!( reboot_inventory(&mut tx, tenant, Uuid::new_v4(), boot, &window, None) .await .unwrap() .is_none() ); assert!( reboot_inventory(&mut tx, Uuid::new_v4(), agent, boot, &window, None) .await .unwrap() .is_none() ); assert!( reboot_inventory(&mut tx, tenant, agent, Uuid::new_v4(), &window, None) .await .unwrap() .is_none() ); let later = RebootWindow { now: now + Duration::minutes(6), ..window }; assert!( reboot_inventory(&mut tx, tenant, agent, boot, &later, None) .await .unwrap() .is_none() ); // A terminal result reuses only its exact original timely receipt; // later metadata cannot replace or extend that immutable proof. let historical = RebootWindow { require_fresh: false, ..later }; assert_eq!( reboot_inventory(&mut tx, tenant, agent, boot, &historical, Some(timely)) .await .unwrap() .map(|v| v.0), Some(timely) ); let fresh = Uuid::new_v4(); let future = Uuid::new_v4(); for (sequence, message, received) in [ (4, fresh, later.now - Duration::seconds(1)), (5, future, later.now + Duration::days(1)), ] { sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)") .bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap(); } // Delayed first verification keeps the original cutoff-bound ID and // uses a DIFFERENT fresh same-boot receipt; late-only never suffices. assert_eq!( reboot_inventory(&mut tx, tenant, agent, boot, &later, None) .await .unwrap() .map(|v| v.0), Some(timely) ); assert!( reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late)) .await .unwrap() .is_none() ); assert_eq!( reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None) .await .unwrap() .map(|v| v.0), Some(fresh) ); assert!( reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future)) .await .unwrap() .is_none() ); assert!( reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None) .await .unwrap() .is_none() ); assert!( reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None) .await .unwrap() .is_none() ); assert!( reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None) .await .unwrap() .is_none() ); let before_return = RebootWindow { now: deadline - Duration::microseconds(1), ..window }; assert!( reboot_inventory(&mut tx, tenant, agent, boot, &before_return, None) .await .unwrap() .is_none() ); tx.rollback().await.unwrap(); let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs WHERE tenant_id=$1)+(SELECT count(*) FROM outbox WHERE tenant_id=$1)+(SELECT count(*) FROM execution_permits WHERE tenant_id=$1)").bind(tenant).fetch_one(&admin).await.unwrap(); assert_eq!(effects, 0); } #[tokio::test] #[ignore = "requires the existing dedicated isolated PostgreSQL harness"] async fn reboot_success_ingress_rejects_forged_late_proof_and_preserves_exact_replay() { let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap()) .await .unwrap(); let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap()) .await .unwrap(); rmm_persistence::verify_application_role(&app) .await .unwrap(); let tenant = Uuid::new_v4(); let asset = Uuid::new_v4(); let agent = Uuid::new_v4(); let boot = Uuid::new_v4(); let old = Uuid::new_v4(); let job = Uuid::new_v4(); let attempt = Uuid::new_v4(); let operation = Uuid::new_v4(); let plan = Uuid::new_v4(); let authorization = Uuid::new_v4(); let grant = Uuid::new_v4(); let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap(); let reserved = now - Duration::seconds(600); let deadline = reserved + Duration::seconds(180); let payload = JobPayload { schema_version: 1, tenant_id: tenant, asset_id: asset, agent_id: agent, enrollment_generation: 1, assignment_version: 1, job_id: job, attempt_id: attempt, action: Action::Reboot { reason: "Isolated postcondition test; no actual reboot".into(), reconnect_timeout_seconds: 180, dependency_group: "single-test-vm".into(), }, execution_identity: "root".into(), not_before: reserved, start_before: now + Duration::minutes(5), max_runtime_seconds: 240, max_output_bytes: 65536, }; let bytes = serde_json::to_vec(&payload).unwrap(); sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot ingress test')") .bind(tenant) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO assets VALUES($1,$2)") .bind(tenant) .bind(asset) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO operations VALUES($1,$2,$3)") .bind(tenant) .bind(operation) .bind(authorization) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)") .bind(tenant) .bind(grant) .bind(job) .bind(attempt) .bind(b"{}".as_slice()) .bind(vec![0u8; 64]) .bind(reserved) .execute(&admin) .await .unwrap(); sqlx::query( "INSERT INTO audit_events VALUES($1,$2,'isolated-test','start_reserved',$3,$4)", ) .bind(tenant) .bind(Uuid::new_v4()) .bind(json!({"job_id":job,"grant_id":grant,"boot_id":old})) .bind(reserved) .execute(&admin) .await .unwrap(); let timely = Uuid::new_v4(); let late = Uuid::new_v4(); let fresh = Uuid::new_v4(); let wrong_boot = Uuid::new_v4(); let future = Uuid::new_v4(); for (sequence, message, received) in [ (1, timely, deadline), (2, late, deadline + Duration::microseconds(1)), ] { sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap(); } for (sequence, message, received, observed_boot) in [ (3, fresh, now - Duration::seconds(1), boot), (4, wrong_boot, now - Duration::seconds(1), old), (5, future, now + Duration::days(1), boot), ] { sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)") .bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap(); } let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late, "reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline, "verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)}); let mut result = JobResult { job_id: job.to_string(), attempt_id: attempt.to_string(), grant_id: grant.to_string(), revision: 1, state: "succeeded".into(), evidence_json: evidence.to_string(), ..Default::default() }; let mut envelope = Envelope { message_id: Uuid::new_v4().to_string(), agent_id: agent.to_string(), connection_generation: 1, boot_id: boot.to_string(), payload: Some(Payload::JobResult(result.clone())), ..Default::default() }; let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap(); assert!( crate::job_gateway::event(&mut tx, tenant, agent, &envelope) .await .is_err() ); assert_eq!( sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1") .bind(job) .fetch_one(&mut *tx) .await .unwrap(), "unknown" ); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts") .fetch_one(&mut *tx) .await .unwrap(), 0 ); let mut good = evidence.clone(); good["inventory_message_id"] = json!(timely); for (id, time) in [ (Some(wrong_boot), Some(now - Duration::seconds(1))), (Some(future), Some(now + Duration::days(1))), (None, None), (Some(fresh), Some(now)), ] { let mut bad = good.clone(); bad["verification_inventory_message_id"] = json!(id); bad["verification_inventory_received_at"] = json!(time); result.evidence_json = bad.to_string(); envelope.payload = Some(Payload::JobResult(result.clone())); assert!( crate::job_gateway::event(&mut tx, tenant, agent, &envelope) .await .is_err() ); } envelope.connection_generation = 2; result.evidence_json = good.to_string(); envelope.payload = Some(Payload::JobResult(result.clone())); assert!( crate::job_gateway::event(&mut tx, tenant, agent, &envelope) .await .is_err() ); envelope.connection_generation = 1; let mut false_clock = good.clone(); false_clock["reconnect_deadline"] = json!(now + Duration::days(1)); result.evidence_json = false_clock.to_string(); envelope.payload = Some(Payload::JobResult(result.clone())); assert!( crate::job_gateway::event(&mut tx, tenant, agent, &envelope) .await .is_err() ); result.evidence_json = good.to_string(); envelope.payload = Some(Payload::JobResult(result.clone())); crate::job_gateway::event(&mut tx, tenant, agent, &envelope) .await .unwrap(); assert_eq!( sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1") .bind(job) .fetch_one(&mut *tx) .await .unwrap(), "succeeded" ); // The original timely receipt survives later inventories and wall time. let later = RebootWindow { reserved_at: reserved, deadline, now: now + Duration::minutes(10), require_fresh: false, }; assert_eq!( reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(timely)) .await .unwrap() .map(|v| v.0), Some(timely) ); assert!( reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late)) .await .unwrap() .is_none() ); // Exact durable-message replay ACKs the existing receipt, without // accepting a new result or extending its original evidence window. crate::job_gateway::event(&mut tx, tenant, agent, &envelope) .await .unwrap(); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts") .fetch_one(&mut *tx) .await .unwrap(), 1 ); let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'") .bind(tenant).fetch_one(&mut *tx).await.unwrap(); assert_eq!(audit["inventory_message_id"], json!(timely)); assert_eq!(audit["inventory_received_at"], json!(deadline)); assert_eq!(audit["verification_inventory_message_id"], json!(fresh)); assert_eq!( audit["verification_inventory_received_at"], json!(now - Duration::seconds(1)) ); assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1); tx.rollback().await.unwrap(); } #[test] fn uncertain_request_resolves_only_to_a_known_checkpoint_anchor() { let applied = Uuid::new_v4(); let pending = Uuid::new_v4(); let committed = Uuid::new_v4(); // Transport cut before server commit: retain the applied anchor. assert_eq!( previous_anchor(Some(applied), Some(pending), None), Some(applied) ); // Server committed but its reply was lost: use the newer checkpoint, // even while the agent also retains its older applied witness. assert_eq!( previous_anchor(Some(applied), Some(pending), Some(committed)), Some(committed) ); // An absent pending request cannot invent an anchor for a legacy // request that discarded its last applied decision. assert_eq!(previous_anchor(None, Some(pending), None), None); assert_eq!( previous_anchor(None, Some(pending), Some(committed)), Some(committed) ); assert_eq!(previous_anchor(Some(applied), None, None), Some(applied)); } #[test] fn accepted_is_not_a_new_attempt_or_grant() { let mut local = SyncAttempt { state: "accepted".into(), consumption: "unconsumed".into(), ..Default::default() }; assert!(clean_unstarted(&local, false)); assert!(!clean_unstarted(&local, true)); local.state = "requesting".into(); assert!(!clean_unstarted(&local, false)); local.state = "running".into(); assert!(!clean_unstarted(&local, false)); } }