Preserve timely reboot evidence and document blocked original reconciliation
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -92,6 +92,18 @@ struct RebootWindow {
|
||||
now: chrono::DateTime<Utc>,
|
||||
require_fresh: bool,
|
||||
}
|
||||
async fn reboot_fresh_inventory(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
boot: Uuid,
|
||||
now: chrono::DateTime<Utc>,
|
||||
receipt: Option<Uuid>,
|
||||
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
|
||||
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt)
|
||||
.fetch_optional(&mut **tx).await?)
|
||||
}
|
||||
async fn reboot_inventory(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
@@ -103,9 +115,21 @@ async fn reboot_inventory(
|
||||
// Already-persisted definitive success keeps its original receipt. It need
|
||||
// not be younger than five minutes again, but its original server receipt
|
||||
// time must still prove the approved reconnect window and current boot.
|
||||
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND (NOT $9::boolean OR received_at>$6) AND received_at<=$7 AND ($8::uuid IS NULL OR message_id=$8) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now-Duration::minutes(5)).bind(window.now).bind(existing_receipt).bind(window.require_fresh)
|
||||
.fetch_optional(&mut **tx).await?)
|
||||
let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt)
|
||||
.fetch_optional(&mut **tx).await?;
|
||||
// Verification may be delayed by local bootstrap readiness. Never move
|
||||
// the original reconnect cutoff: retain the actual timely receipt and
|
||||
// independently demand fresh evidence of the SAME current boot.
|
||||
if timely.is_some()
|
||||
&& window.require_fresh
|
||||
&& reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None)
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(timely)
|
||||
}
|
||||
/// First ingress of success needs the same independent, timely receipt as an
|
||||
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
|
||||
@@ -120,6 +144,7 @@ pub(crate) async fn verify_reboot_success(
|
||||
) -> Result<()> {
|
||||
let boot = Uuid::parse_str(&envelope.boot_id)?;
|
||||
let grant = Uuid::parse_str(&result.grant_id)?;
|
||||
let now = Utc::now();
|
||||
ensure!(
|
||||
job.tenant_id == tenant
|
||||
&& job.agent_id == agent
|
||||
@@ -130,6 +155,9 @@ pub(crate) async fn verify_reboot_success(
|
||||
&& evidence["new_boot_id"] == envelope.boot_id,
|
||||
"REBOOT_RESULT_BINDING"
|
||||
);
|
||||
let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)")
|
||||
.bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?;
|
||||
ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED");
|
||||
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
|
||||
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
|
||||
let deadline =
|
||||
@@ -156,13 +184,41 @@ pub(crate) async fn verify_reboot_success(
|
||||
&RebootWindow {
|
||||
reserved_at: reserved,
|
||||
deadline,
|
||||
now: Utc::now(),
|
||||
now,
|
||||
require_fresh: true,
|
||||
},
|
||||
Some(receipt),
|
||||
)
|
||||
.await?
|
||||
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
|
||||
let verification_id = evidence
|
||||
.get("verification_inventory_message_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(Uuid::parse_str)
|
||||
.transpose()?;
|
||||
let verification_time = evidence.get("verification_inventory_received_at");
|
||||
let verification = if verification_id.is_some()
|
||||
|| verification_time.is_some()
|
||||
|| received <= now - Duration::minutes(5)
|
||||
{
|
||||
let id = verification_id
|
||||
.filter(|id| !id.is_nil())
|
||||
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
|
||||
let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id))
|
||||
.await?
|
||||
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
|
||||
ensure!(
|
||||
serde_json::from_value::<chrono::DateTime<Utc>>(
|
||||
verification_time.cloned().unwrap_or(Value::Null)
|
||||
)
|
||||
.ok()
|
||||
== Some(time),
|
||||
"REBOOT_VERIFICATION_TIME_BINDING"
|
||||
);
|
||||
(id, time)
|
||||
} else {
|
||||
(receipt, received)
|
||||
};
|
||||
for (key, actual) in [
|
||||
("reboot_reserved_at", reserved),
|
||||
("reconnect_deadline", deadline),
|
||||
@@ -174,6 +230,11 @@ pub(crate) async fn verify_reboot_success(
|
||||
"REBOOT_SERVER_TIME_BINDING"
|
||||
);
|
||||
}
|
||||
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())")
|
||||
.bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot,
|
||||
"connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline,
|
||||
"inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1}))
|
||||
.execute(&mut **tx).await?;
|
||||
Ok(())
|
||||
}
|
||||
pub async fn reconcile(
|
||||
@@ -587,8 +648,10 @@ pub async fn reconcile(
|
||||
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
|
||||
{
|
||||
reboot_witness = Some((original, inventory));
|
||||
let verification =
|
||||
reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?;
|
||||
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at}).to_string(),error_code:String::new()};
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()};
|
||||
let mut evidence = e.clone();
|
||||
evidence.message_id = Uuid::new_v4().to_string();
|
||||
evidence.payload = Some(Payload::JobResult(result));
|
||||
@@ -877,6 +940,61 @@ mod tests {
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
let fresh = Uuid::new_v4();
|
||||
let future = Uuid::new_v4();
|
||||
for (sequence, message, received) in [
|
||||
(4, fresh, later.now - Duration::seconds(1)),
|
||||
(5, future, later.now + Duration::days(1)),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
||||
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap();
|
||||
}
|
||||
// Delayed first verification keeps the original cutoff-bound ID and
|
||||
// uses a DIFFERENT fresh same-boot receipt; late-only never suffices.
|
||||
assert_eq!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(fresh)
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let before_return = RebootWindow {
|
||||
now: deadline - Duration::microseconds(1),
|
||||
..window
|
||||
@@ -915,7 +1033,7 @@ mod tests {
|
||||
let authorization = Uuid::new_v4();
|
||||
let grant = Uuid::new_v4();
|
||||
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
|
||||
let reserved = now - Duration::seconds(200);
|
||||
let reserved = now - Duration::seconds(600);
|
||||
let deadline = reserved + Duration::seconds(180);
|
||||
let payload = JobPayload {
|
||||
schema_version: 1,
|
||||
@@ -949,7 +1067,7 @@ mod tests {
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
|
||||
@@ -984,14 +1102,26 @@ mod tests {
|
||||
.unwrap();
|
||||
let timely = Uuid::new_v4();
|
||||
let late = Uuid::new_v4();
|
||||
let fresh = Uuid::new_v4();
|
||||
let wrong_boot = Uuid::new_v4();
|
||||
let future = Uuid::new_v4();
|
||||
for (sequence, message, received) in [
|
||||
(1, timely, deadline),
|
||||
(2, late, deadline + Duration::microseconds(1)),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
||||
}
|
||||
for (sequence, message, received, observed_boot) in [
|
||||
(3, fresh, now - Duration::seconds(1), boot),
|
||||
(4, wrong_boot, now - Duration::seconds(1), old),
|
||||
(5, future, now + Duration::days(1), boot),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
||||
.bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
||||
}
|
||||
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
|
||||
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline});
|
||||
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline,
|
||||
"verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)});
|
||||
let mut result = JobResult {
|
||||
job_id: job.to_string(),
|
||||
attempt_id: attempt.to_string(),
|
||||
@@ -1004,6 +1134,7 @@ mod tests {
|
||||
let mut envelope = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: agent.to_string(),
|
||||
connection_generation: 1,
|
||||
boot_id: boot.to_string(),
|
||||
payload: Some(Payload::JobResult(result.clone())),
|
||||
..Default::default()
|
||||
@@ -1031,6 +1162,32 @@ mod tests {
|
||||
);
|
||||
let mut good = evidence.clone();
|
||||
good["inventory_message_id"] = json!(timely);
|
||||
for (id, time) in [
|
||||
(Some(wrong_boot), Some(now - Duration::seconds(1))),
|
||||
(Some(future), Some(now + Duration::days(1))),
|
||||
(None, None),
|
||||
(Some(fresh), Some(now)),
|
||||
] {
|
||||
let mut bad = good.clone();
|
||||
bad["verification_inventory_message_id"] = json!(id);
|
||||
bad["verification_inventory_received_at"] = json!(time);
|
||||
result.evidence_json = bad.to_string();
|
||||
envelope.payload = Some(Payload::JobResult(result.clone()));
|
||||
assert!(
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
envelope.connection_generation = 2;
|
||||
result.evidence_json = good.to_string();
|
||||
envelope.payload = Some(Payload::JobResult(result.clone()));
|
||||
assert!(
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
envelope.connection_generation = 1;
|
||||
let mut false_clock = good.clone();
|
||||
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
|
||||
result.evidence_json = false_clock.to_string();
|
||||
@@ -1085,6 +1242,16 @@ mod tests {
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'")
|
||||
.bind(tenant).fetch_one(&mut *tx).await.unwrap();
|
||||
assert_eq!(audit["inventory_message_id"], json!(timely));
|
||||
assert_eq!(audit["inventory_received_at"], json!(deadline));
|
||||
assert_eq!(audit["verification_inventory_message_id"], json!(fresh));
|
||||
assert_eq!(
|
||||
audit["verification_inventory_received_at"],
|
||||
json!(now - Duration::seconds(1))
|
||||
);
|
||||
assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1);
|
||||
tx.rollback().await.unwrap();
|
||||
}
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user