Preserve timely reboot evidence and document blocked original reconciliation
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 22:03:24 +01:00
parent 80906a3eb6
commit 4102b97fb9
9 changed files with 387 additions and 13 deletions
+175 -8
View File
@@ -92,6 +92,18 @@ struct RebootWindow {
now: chrono::DateTime<Utc>,
require_fresh: bool,
}
async fn reboot_fresh_inventory(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
boot: Uuid,
now: chrono::DateTime<Utc>,
receipt: Option<Uuid>,
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt)
.fetch_optional(&mut **tx).await?)
}
async fn reboot_inventory(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
@@ -103,9 +115,21 @@ async fn reboot_inventory(
// Already-persisted definitive success keeps its original receipt. It need
// not be younger than five minutes again, but its original server receipt
// time must still prove the approved reconnect window and current boot.
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND (NOT $9::boolean OR received_at>$6) AND received_at<=$7 AND ($8::uuid IS NULL OR message_id=$8) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now-Duration::minutes(5)).bind(window.now).bind(existing_receipt).bind(window.require_fresh)
.fetch_optional(&mut **tx).await?)
let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt)
.fetch_optional(&mut **tx).await?;
// Verification may be delayed by local bootstrap readiness. Never move
// the original reconnect cutoff: retain the actual timely receipt and
// independently demand fresh evidence of the SAME current boot.
if timely.is_some()
&& window.require_fresh
&& reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None)
.await?
.is_none()
{
return Ok(None);
}
Ok(timely)
}
/// First ingress of success needs the same independent, timely receipt as an
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
@@ -120,6 +144,7 @@ pub(crate) async fn verify_reboot_success(
) -> Result<()> {
let boot = Uuid::parse_str(&envelope.boot_id)?;
let grant = Uuid::parse_str(&result.grant_id)?;
let now = Utc::now();
ensure!(
job.tenant_id == tenant
&& job.agent_id == agent
@@ -130,6 +155,9 @@ pub(crate) async fn verify_reboot_success(
&& evidence["new_boot_id"] == envelope.boot_id,
"REBOOT_RESULT_BINDING"
);
let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)")
.bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?;
ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED");
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
let deadline =
@@ -156,13 +184,41 @@ pub(crate) async fn verify_reboot_success(
&RebootWindow {
reserved_at: reserved,
deadline,
now: Utc::now(),
now,
require_fresh: true,
},
Some(receipt),
)
.await?
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
let verification_id = evidence
.get("verification_inventory_message_id")
.and_then(Value::as_str)
.map(Uuid::parse_str)
.transpose()?;
let verification_time = evidence.get("verification_inventory_received_at");
let verification = if verification_id.is_some()
|| verification_time.is_some()
|| received <= now - Duration::minutes(5)
{
let id = verification_id
.filter(|id| !id.is_nil())
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id))
.await?
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
ensure!(
serde_json::from_value::<chrono::DateTime<Utc>>(
verification_time.cloned().unwrap_or(Value::Null)
)
.ok()
== Some(time),
"REBOOT_VERIFICATION_TIME_BINDING"
);
(id, time)
} else {
(receipt, received)
};
for (key, actual) in [
("reboot_reserved_at", reserved),
("reconnect_deadline", deadline),
@@ -174,6 +230,11 @@ pub(crate) async fn verify_reboot_success(
"REBOOT_SERVER_TIME_BINDING"
);
}
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())")
.bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot,
"connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline,
"inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1}))
.execute(&mut **tx).await?;
Ok(())
}
pub async fn reconcile(
@@ -587,8 +648,10 @@ pub async fn reconcile(
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
{
reboot_witness = Some((original, inventory));
let verification =
reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?;
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at}).to_string(),error_code:String::new()};
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()};
let mut evidence = e.clone();
evidence.message_id = Uuid::new_v4().to_string();
evidence.payload = Some(Payload::JobResult(result));
@@ -877,6 +940,61 @@ mod tests {
.map(|v| v.0),
Some(timely)
);
let fresh = Uuid::new_v4();
let future = Uuid::new_v4();
for (sequence, message, received) in [
(4, fresh, later.now - Duration::seconds(1)),
(5, future, later.now + Duration::days(1)),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap();
}
// Delayed first verification keeps the original cutoff-bound ID and
// uses a DIFFERENT fresh same-boot receipt; late-only never suffices.
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
.await
.unwrap()
.is_none()
);
assert_eq!(
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None)
.await
.unwrap()
.map(|v| v.0),
Some(fresh)
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future))
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None)
.await
.unwrap()
.is_none()
);
let before_return = RebootWindow {
now: deadline - Duration::microseconds(1),
..window
@@ -915,7 +1033,7 @@ mod tests {
let authorization = Uuid::new_v4();
let grant = Uuid::new_v4();
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
let reserved = now - Duration::seconds(200);
let reserved = now - Duration::seconds(600);
let deadline = reserved + Duration::seconds(180);
let payload = JobPayload {
schema_version: 1,
@@ -949,7 +1067,7 @@ mod tests {
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
@@ -984,14 +1102,26 @@ mod tests {
.unwrap();
let timely = Uuid::new_v4();
let late = Uuid::new_v4();
let fresh = Uuid::new_v4();
let wrong_boot = Uuid::new_v4();
let future = Uuid::new_v4();
for (sequence, message, received) in [
(1, timely, deadline),
(2, late, deadline + Duration::microseconds(1)),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
}
for (sequence, message, received, observed_boot) in [
(3, fresh, now - Duration::seconds(1), boot),
(4, wrong_boot, now - Duration::seconds(1), old),
(5, future, now + Duration::days(1), boot),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
}
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline});
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline,
"verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)});
let mut result = JobResult {
job_id: job.to_string(),
attempt_id: attempt.to_string(),
@@ -1004,6 +1134,7 @@ mod tests {
let mut envelope = Envelope {
message_id: Uuid::new_v4().to_string(),
agent_id: agent.to_string(),
connection_generation: 1,
boot_id: boot.to_string(),
payload: Some(Payload::JobResult(result.clone())),
..Default::default()
@@ -1031,6 +1162,32 @@ mod tests {
);
let mut good = evidence.clone();
good["inventory_message_id"] = json!(timely);
for (id, time) in [
(Some(wrong_boot), Some(now - Duration::seconds(1))),
(Some(future), Some(now + Duration::days(1))),
(None, None),
(Some(fresh), Some(now)),
] {
let mut bad = good.clone();
bad["verification_inventory_message_id"] = json!(id);
bad["verification_inventory_received_at"] = json!(time);
result.evidence_json = bad.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
}
envelope.connection_generation = 2;
result.evidence_json = good.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
envelope.connection_generation = 1;
let mut false_clock = good.clone();
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
result.evidence_json = false_clock.to_string();
@@ -1085,6 +1242,16 @@ mod tests {
.unwrap(),
1
);
let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'")
.bind(tenant).fetch_one(&mut *tx).await.unwrap();
assert_eq!(audit["inventory_message_id"], json!(timely));
assert_eq!(audit["inventory_received_at"], json!(deadline));
assert_eq!(audit["verification_inventory_message_id"], json!(fresh));
assert_eq!(
audit["verification_inventory_received_at"],
json!(now - Duration::seconds(1))
);
assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1);
tx.rollback().await.unwrap();
}
#[test]