fix: bind reboot success to independently received inventory within reserved deadline
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:49:12 +01:00
parent d8bf8e238b
commit b3a22d464b
9 changed files with 1048 additions and 27 deletions
+4 -7
View File
@@ -189,13 +189,10 @@ pub async fn event(
}
}
if matches!(payload.action, Action::Reboot { .. }) {
ensure!(
evidence.get("new_boot_id").and_then(Value::as_str)
== Some(e.boot_id.as_str()),
"new boot evidence required"
);
let old_boot=sqlx::query_scalar::<_,String>("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(&r.job_id).fetch_one(&mut **tx).await?;
ensure!(old_boot != e.boot_id, "reboot needs new boot");
crate::synchronization::verify_reboot_success(
tx, tenant, agent, e, &payload, r, &evidence,
)
.await?;
}
}
sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1")
+501 -16
View File
@@ -67,6 +67,115 @@ fn previous_anchor(
last_applied
}
}
/// The server's durable grant reservation precedes the helper's saved start.
/// Using that trusted timestamp is conservative: no device clock or late
/// reconnect may extend the fixed, approved reconnect window.
fn reboot_deadline(
action: &Action,
reserved_at: Option<chrono::DateTime<Utc>>,
) -> Option<chrono::DateTime<Utc>> {
let Action::Reboot {
reconnect_timeout_seconds,
..
} = action
else {
return None;
};
if !action.validate() {
return None;
}
reserved_at?.checked_add_signed(Duration::seconds(i64::from(*reconnect_timeout_seconds)))
}
struct RebootWindow {
reserved_at: chrono::DateTime<Utc>,
deadline: chrono::DateTime<Utc>,
now: chrono::DateTime<Utc>,
require_fresh: bool,
}
async fn reboot_inventory(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
boot: Uuid,
window: &RebootWindow,
existing_receipt: Option<Uuid>,
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
// Already-persisted definitive success keeps its original receipt. It need
// not be younger than five minutes again, but its original server receipt
// time must still prove the approved reconnect window and current boot.
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND (NOT $9::boolean OR received_at>$6) AND received_at<=$7 AND ($8::uuid IS NULL OR message_id=$8) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now-Duration::minutes(5)).bind(window.now).bind(existing_receipt).bind(window.require_fresh)
.fetch_optional(&mut **tx).await?)
}
/// First ingress of success needs the same independent, timely receipt as an
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
pub(crate) async fn verify_reboot_success(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
envelope: &Envelope,
job: &JobPayload,
result: &JobResult,
evidence: &Value,
) -> Result<()> {
let boot = Uuid::parse_str(&envelope.boot_id)?;
let grant = Uuid::parse_str(&result.grant_id)?;
ensure!(
job.tenant_id == tenant
&& job.agent_id == agent
&& job.job_id.to_string() == result.job_id
&& job.attempt_id.to_string() == result.attempt_id
&& evidence["postconditions_passed"] == true
&& evidence["reconciled_after_reboot"] == true
&& evidence["new_boot_id"] == envelope.boot_id,
"REBOOT_RESULT_BINDING"
);
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
let deadline =
reboot_deadline(&job.action, Some(reserved)).context("REBOOT_FIXED_DEADLINE_REQUIRED")?;
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1")
.bind(tenant).bind(job.job_id.to_string()).bind(grant.to_string()).fetch_optional(&mut **tx).await?;
ensure!(
original
.as_deref()
.and_then(|v| Uuid::parse_str(v).ok())
.is_some_and(|old| !old.is_nil() && old != boot),
"REBOOT_ORIGINAL_BOOT_REQUIRED"
);
let receipt = evidence["inventory_message_id"]
.as_str()
.and_then(|v| Uuid::parse_str(v).ok())
.filter(|id| !id.is_nil())
.context("REBOOT_PERSISTED_INVENTORY_REQUIRED")?;
let (_, received) = reboot_inventory(
tx,
tenant,
agent,
boot,
&RebootWindow {
reserved_at: reserved,
deadline,
now: Utc::now(),
require_fresh: true,
},
Some(receipt),
)
.await?
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
for (key, actual) in [
("reboot_reserved_at", reserved),
("reconnect_deadline", deadline),
("inventory_received_at", received),
] {
ensure!(
serde_json::from_value::<chrono::DateTime<Utc>>(evidence[key].clone()).ok()
== Some(actual),
"REBOOT_SERVER_TIME_BINDING"
);
}
Ok(())
}
pub async fn reconcile(
state: &crate::State,
tx: &mut Transaction<'_, Postgres>,
@@ -335,7 +444,7 @@ pub async fn reconcile(
continue;
}
let grant = sqlx::query(
"SELECT grant_id,payload FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
"SELECT grant_id,payload,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
)
.bind(job_id)
.bind(attempt)
@@ -433,18 +542,53 @@ pub async fn reconcile(
// A reboot's postcondition is independently tied to persisted current-
// boot inventory and the original boot in the durable reservation audit.
let mut reboot_reconciled = false;
let mut reboot_witness = None;
if matches!(payload.action, Action::Reboot { .. })
&& grant_matches
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
{
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(job_id.to_string()).fetch_optional(&mut **tx).await?;
let inventory:Option<Uuid>=sqlx::query_scalar("SELECT message_id FROM observations WHERE agent_id=$1 AND boot_id=$2 AND received_at>now()-interval '5 minutes' AND received_at>(SELECT created_at FROM execution_permits WHERE job_id=$3) ORDER BY received_at DESC LIMIT 1").bind(agent).bind(boot).bind(job_id).fetch_optional(&mut **tx).await?;
if let (Some(original), Some(inventory)) = (original, inventory)
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1").bind(tenant).bind(job_id.to_string()).bind(&local.grant_id).fetch_optional(&mut **tx).await?;
let reserved_at = grant
.as_ref()
.map(|g| g.get::<chrono::DateTime<Utc>, _>("created_at"));
let deadline = reboot_deadline(&payload.action, reserved_at);
let inventory = if let (Some(reserved), Some(until)) = (reserved_at, deadline) {
let existing_receipt = if final_row.get::<String, _>("state") == "succeeded" {
final_row
.get::<Option<Value>, _>("result")
.and_then(|value| {
value["evidence"]["inventory_message_id"]
.as_str()
.and_then(|v| Uuid::parse_str(v).ok())
})
} else {
None
};
reboot_inventory(
tx,
tenant,
agent,
boot,
&RebootWindow {
reserved_at: reserved,
deadline: until,
now: Utc::now(),
require_fresh: existing_receipt.is_none(),
},
existing_receipt,
)
.await?
} else {
None
};
if let (Some(original), Some((inventory, received_at)), Some(reserved), Some(until)) =
(original, inventory, reserved_at, deadline)
&& original != e.boot_id
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
{
reboot_witness = Some((original, inventory));
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true}).to_string(),error_code:String::new()};
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at}).to_string(),error_code:String::new()};
let mut evidence = e.clone();
evidence.message_id = Uuid::new_v4().to_string();
evidence.payload = Some(Payload::JobResult(result));
@@ -465,18 +609,17 @@ pub async fn reconcile(
&& terminal == "succeeded"
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
&& let Some(value) = final_row.get::<Option<Value>, _>("result")
&& let Some((original, inventory)) = reboot_witness
&& Uuid::parse_str(&original).ok() == decision.original_boot_id
&& decision.original_boot_id.is_some()
&& value["evidence"]["postconditions_passed"] == json!(true)
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
&& value["evidence"]["new_boot_id"]
.as_str()
.is_some_and(|id| id == e.boot_id)
&& value["evidence"]["inventory_message_id"] == json!(inventory)
{
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(job_id.to_string()).fetch_optional(&mut **tx).await?;
if original.as_ref().and_then(|v| Uuid::parse_str(v).ok()) == decision.original_boot_id
&& decision.original_boot_id.is_some()
&& value["evidence"]["postconditions_passed"] == json!(true)
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
&& value["evidence"]["new_boot_id"]
.as_str()
.is_some_and(|id| Some(id) != original.as_deref())
{
reboot_reconciled = true;
}
reboot_reconciled = true;
}
decision.revision = final_row.get::<i64, _>("revision") as u64;
decision.outcome_state = terminal.clone();
@@ -603,6 +746,348 @@ async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &st
mod tests {
use super::*;
#[test]
fn reboot_deadline_uses_only_the_fixed_plan_and_trusted_reservation() {
let reserved = Utc::now();
let action = Action::Reboot {
reason: "Explicit final Cloud VM acceptance".into(),
reconnect_timeout_seconds: 180,
dependency_group: "single-test-vm".into(),
};
assert_eq!(
reboot_deadline(&action, Some(reserved)),
Some(reserved + Duration::seconds(180))
);
assert_eq!(reboot_deadline(&action, None), None);
assert_eq!(
reboot_deadline(
&Action::Reboot {
reason: "test".into(),
reconnect_timeout_seconds: 0,
dependency_group: "single-test-vm".into()
},
Some(reserved)
),
None
);
assert_eq!(
reboot_deadline(&action, Some(chrono::DateTime::<Utc>::MAX_UTC)),
None
);
}
#[tokio::test]
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
async fn reboot_inventory_requires_trusted_timely_same_identity_and_boot() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await
.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await
.unwrap();
rmm_persistence::verify_application_role(&app)
.await
.unwrap();
let tenant = Uuid::new_v4();
let asset = Uuid::new_v4();
let agent = Uuid::new_v4();
let boot = Uuid::new_v4();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot deadline test')")
.bind(tenant)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets VALUES($1,$2)")
.bind(tenant)
.bind(asset)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)")
.bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
let now = Utc::now();
let reserved = now - Duration::seconds(200);
let deadline = reserved + Duration::seconds(180);
let window = RebootWindow {
reserved_at: reserved,
deadline,
now,
require_fresh: true,
};
let timely = Uuid::new_v4();
let late = Uuid::new_v4();
for (sequence, message, received) in [
(1, timely, deadline),
(2, late, deadline + Duration::microseconds(1)),
(3, Uuid::new_v4(), reserved),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&admin).await.unwrap();
}
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &window, None)
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &window, Some(late))
.await
.unwrap()
.is_none()
);
assert!(
reboot_inventory(&mut tx, tenant, Uuid::new_v4(), boot, &window, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_inventory(&mut tx, Uuid::new_v4(), agent, boot, &window, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_inventory(&mut tx, tenant, agent, Uuid::new_v4(), &window, None)
.await
.unwrap()
.is_none()
);
let later = RebootWindow {
now: now + Duration::minutes(6),
..window
};
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
.await
.unwrap()
.is_none()
);
// A terminal result reuses only its exact original timely receipt;
// later metadata cannot replace or extend that immutable proof.
let historical = RebootWindow {
require_fresh: false,
..later
};
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &historical, Some(timely))
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
let before_return = RebootWindow {
now: deadline - Duration::microseconds(1),
..window
};
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &before_return, None)
.await
.unwrap()
.is_none()
);
tx.rollback().await.unwrap();
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs WHERE tenant_id=$1)+(SELECT count(*) FROM outbox WHERE tenant_id=$1)+(SELECT count(*) FROM execution_permits WHERE tenant_id=$1)").bind(tenant).fetch_one(&admin).await.unwrap();
assert_eq!(effects, 0);
}
#[tokio::test]
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
async fn reboot_success_ingress_rejects_forged_late_proof_and_preserves_exact_replay() {
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
.await
.unwrap();
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
.await
.unwrap();
rmm_persistence::verify_application_role(&app)
.await
.unwrap();
let tenant = Uuid::new_v4();
let asset = Uuid::new_v4();
let agent = Uuid::new_v4();
let boot = Uuid::new_v4();
let old = Uuid::new_v4();
let job = Uuid::new_v4();
let attempt = Uuid::new_v4();
let operation = Uuid::new_v4();
let plan = Uuid::new_v4();
let authorization = Uuid::new_v4();
let grant = Uuid::new_v4();
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
let reserved = now - Duration::seconds(200);
let deadline = reserved + Duration::seconds(180);
let payload = JobPayload {
schema_version: 1,
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
job_id: job,
attempt_id: attempt,
action: Action::Reboot {
reason: "Isolated postcondition test; no actual reboot".into(),
reconnect_timeout_seconds: 180,
dependency_group: "single-test-vm".into(),
},
execution_identity: "root".into(),
not_before: reserved,
start_before: now + Duration::minutes(5),
max_runtime_seconds: 240,
max_output_bytes: 65536,
};
let bytes = serde_json::to_vec(&payload).unwrap();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot ingress test')")
.bind(tenant)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets VALUES($1,$2)")
.bind(tenant)
.bind(asset)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
.bind(tenant)
.bind(operation)
.bind(authorization)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)")
.bind(tenant)
.bind(grant)
.bind(job)
.bind(attempt)
.bind(b"{}".as_slice())
.bind(vec![0u8; 64])
.bind(reserved)
.execute(&admin)
.await
.unwrap();
sqlx::query(
"INSERT INTO audit_events VALUES($1,$2,'isolated-test','start_reserved',$3,$4)",
)
.bind(tenant)
.bind(Uuid::new_v4())
.bind(json!({"job_id":job,"grant_id":grant,"boot_id":old}))
.bind(reserved)
.execute(&admin)
.await
.unwrap();
let timely = Uuid::new_v4();
let late = Uuid::new_v4();
for (sequence, message, received) in [
(1, timely, deadline),
(2, late, deadline + Duration::microseconds(1)),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
}
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline});
let mut result = JobResult {
job_id: job.to_string(),
attempt_id: attempt.to_string(),
grant_id: grant.to_string(),
revision: 1,
state: "succeeded".into(),
evidence_json: evidence.to_string(),
..Default::default()
};
let mut envelope = Envelope {
message_id: Uuid::new_v4().to_string(),
agent_id: agent.to_string(),
boot_id: boot.to_string(),
payload: Some(Payload::JobResult(result.clone())),
..Default::default()
};
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
assert_eq!(
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
.bind(job)
.fetch_one(&mut *tx)
.await
.unwrap(),
"unknown"
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
.fetch_one(&mut *tx)
.await
.unwrap(),
0
);
let mut good = evidence.clone();
good["inventory_message_id"] = json!(timely);
let mut false_clock = good.clone();
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
result.evidence_json = false_clock.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
result.evidence_json = good.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
.bind(job)
.fetch_one(&mut *tx)
.await
.unwrap(),
"succeeded"
);
// The original timely receipt survives later inventories and wall time.
let later = RebootWindow {
reserved_at: reserved,
deadline,
now: now + Duration::minutes(10),
require_fresh: false,
};
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(timely))
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
.await
.unwrap()
.is_none()
);
// Exact durable-message replay ACKs the existing receipt, without
// accepting a new result or extending its original evidence window.
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.unwrap();
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
.fetch_one(&mut *tx)
.await
.unwrap(),
1
);
tx.rollback().await.unwrap();
}
#[test]
fn uncertain_request_resolves_only_to_a_known_checkpoint_anchor() {
let applied = Uuid::new_v4();
let pending = Uuid::new_v4();