fix: bind reboot success to independently received inventory within reserved deadline
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -189,13 +189,10 @@ pub async fn event(
|
||||
}
|
||||
}
|
||||
if matches!(payload.action, Action::Reboot { .. }) {
|
||||
ensure!(
|
||||
evidence.get("new_boot_id").and_then(Value::as_str)
|
||||
== Some(e.boot_id.as_str()),
|
||||
"new boot evidence required"
|
||||
);
|
||||
let old_boot=sqlx::query_scalar::<_,String>("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(&r.job_id).fetch_one(&mut **tx).await?;
|
||||
ensure!(old_boot != e.boot_id, "reboot needs new boot");
|
||||
crate::synchronization::verify_reboot_success(
|
||||
tx, tenant, agent, e, &payload, r, &evidence,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
sqlx::query("UPDATE jobs SET state=$2,revision=$3,result=$4 WHERE job_id=$1")
|
||||
|
||||
@@ -67,6 +67,115 @@ fn previous_anchor(
|
||||
last_applied
|
||||
}
|
||||
}
|
||||
/// The server's durable grant reservation precedes the helper's saved start.
|
||||
/// Using that trusted timestamp is conservative: no device clock or late
|
||||
/// reconnect may extend the fixed, approved reconnect window.
|
||||
fn reboot_deadline(
|
||||
action: &Action,
|
||||
reserved_at: Option<chrono::DateTime<Utc>>,
|
||||
) -> Option<chrono::DateTime<Utc>> {
|
||||
let Action::Reboot {
|
||||
reconnect_timeout_seconds,
|
||||
..
|
||||
} = action
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
if !action.validate() {
|
||||
return None;
|
||||
}
|
||||
reserved_at?.checked_add_signed(Duration::seconds(i64::from(*reconnect_timeout_seconds)))
|
||||
}
|
||||
struct RebootWindow {
|
||||
reserved_at: chrono::DateTime<Utc>,
|
||||
deadline: chrono::DateTime<Utc>,
|
||||
now: chrono::DateTime<Utc>,
|
||||
require_fresh: bool,
|
||||
}
|
||||
async fn reboot_inventory(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
boot: Uuid,
|
||||
window: &RebootWindow,
|
||||
existing_receipt: Option<Uuid>,
|
||||
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
|
||||
// Already-persisted definitive success keeps its original receipt. It need
|
||||
// not be younger than five minutes again, but its original server receipt
|
||||
// time must still prove the approved reconnect window and current boot.
|
||||
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND (NOT $9::boolean OR received_at>$6) AND received_at<=$7 AND ($8::uuid IS NULL OR message_id=$8) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now-Duration::minutes(5)).bind(window.now).bind(existing_receipt).bind(window.require_fresh)
|
||||
.fetch_optional(&mut **tx).await?)
|
||||
}
|
||||
/// First ingress of success needs the same independent, timely receipt as an
|
||||
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
|
||||
pub(crate) async fn verify_reboot_success(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
envelope: &Envelope,
|
||||
job: &JobPayload,
|
||||
result: &JobResult,
|
||||
evidence: &Value,
|
||||
) -> Result<()> {
|
||||
let boot = Uuid::parse_str(&envelope.boot_id)?;
|
||||
let grant = Uuid::parse_str(&result.grant_id)?;
|
||||
ensure!(
|
||||
job.tenant_id == tenant
|
||||
&& job.agent_id == agent
|
||||
&& job.job_id.to_string() == result.job_id
|
||||
&& job.attempt_id.to_string() == result.attempt_id
|
||||
&& evidence["postconditions_passed"] == true
|
||||
&& evidence["reconciled_after_reboot"] == true
|
||||
&& evidence["new_boot_id"] == envelope.boot_id,
|
||||
"REBOOT_RESULT_BINDING"
|
||||
);
|
||||
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
|
||||
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
|
||||
let deadline =
|
||||
reboot_deadline(&job.action, Some(reserved)).context("REBOOT_FIXED_DEADLINE_REQUIRED")?;
|
||||
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(job.job_id.to_string()).bind(grant.to_string()).fetch_optional(&mut **tx).await?;
|
||||
ensure!(
|
||||
original
|
||||
.as_deref()
|
||||
.and_then(|v| Uuid::parse_str(v).ok())
|
||||
.is_some_and(|old| !old.is_nil() && old != boot),
|
||||
"REBOOT_ORIGINAL_BOOT_REQUIRED"
|
||||
);
|
||||
let receipt = evidence["inventory_message_id"]
|
||||
.as_str()
|
||||
.and_then(|v| Uuid::parse_str(v).ok())
|
||||
.filter(|id| !id.is_nil())
|
||||
.context("REBOOT_PERSISTED_INVENTORY_REQUIRED")?;
|
||||
let (_, received) = reboot_inventory(
|
||||
tx,
|
||||
tenant,
|
||||
agent,
|
||||
boot,
|
||||
&RebootWindow {
|
||||
reserved_at: reserved,
|
||||
deadline,
|
||||
now: Utc::now(),
|
||||
require_fresh: true,
|
||||
},
|
||||
Some(receipt),
|
||||
)
|
||||
.await?
|
||||
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
|
||||
for (key, actual) in [
|
||||
("reboot_reserved_at", reserved),
|
||||
("reconnect_deadline", deadline),
|
||||
("inventory_received_at", received),
|
||||
] {
|
||||
ensure!(
|
||||
serde_json::from_value::<chrono::DateTime<Utc>>(evidence[key].clone()).ok()
|
||||
== Some(actual),
|
||||
"REBOOT_SERVER_TIME_BINDING"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub async fn reconcile(
|
||||
state: &crate::State,
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
@@ -335,7 +444,7 @@ pub async fn reconcile(
|
||||
continue;
|
||||
}
|
||||
let grant = sqlx::query(
|
||||
"SELECT grant_id,payload FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
||||
"SELECT grant_id,payload,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
||||
)
|
||||
.bind(job_id)
|
||||
.bind(attempt)
|
||||
@@ -433,18 +542,53 @@ pub async fn reconcile(
|
||||
// A reboot's postcondition is independently tied to persisted current-
|
||||
// boot inventory and the original boot in the durable reservation audit.
|
||||
let mut reboot_reconciled = false;
|
||||
let mut reboot_witness = None;
|
||||
if matches!(payload.action, Action::Reboot { .. })
|
||||
&& grant_matches
|
||||
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
|
||||
{
|
||||
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(job_id.to_string()).fetch_optional(&mut **tx).await?;
|
||||
let inventory:Option<Uuid>=sqlx::query_scalar("SELECT message_id FROM observations WHERE agent_id=$1 AND boot_id=$2 AND received_at>now()-interval '5 minutes' AND received_at>(SELECT created_at FROM execution_permits WHERE job_id=$3) ORDER BY received_at DESC LIMIT 1").bind(agent).bind(boot).bind(job_id).fetch_optional(&mut **tx).await?;
|
||||
if let (Some(original), Some(inventory)) = (original, inventory)
|
||||
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE tenant_id=$1 AND event_type='start_reserved' AND data->>'job_id'=$2 AND data->>'grant_id'=$3 ORDER BY created_at DESC LIMIT 1").bind(tenant).bind(job_id.to_string()).bind(&local.grant_id).fetch_optional(&mut **tx).await?;
|
||||
let reserved_at = grant
|
||||
.as_ref()
|
||||
.map(|g| g.get::<chrono::DateTime<Utc>, _>("created_at"));
|
||||
let deadline = reboot_deadline(&payload.action, reserved_at);
|
||||
let inventory = if let (Some(reserved), Some(until)) = (reserved_at, deadline) {
|
||||
let existing_receipt = if final_row.get::<String, _>("state") == "succeeded" {
|
||||
final_row
|
||||
.get::<Option<Value>, _>("result")
|
||||
.and_then(|value| {
|
||||
value["evidence"]["inventory_message_id"]
|
||||
.as_str()
|
||||
.and_then(|v| Uuid::parse_str(v).ok())
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
reboot_inventory(
|
||||
tx,
|
||||
tenant,
|
||||
agent,
|
||||
boot,
|
||||
&RebootWindow {
|
||||
reserved_at: reserved,
|
||||
deadline: until,
|
||||
now: Utc::now(),
|
||||
require_fresh: existing_receipt.is_none(),
|
||||
},
|
||||
existing_receipt,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let (Some(original), Some((inventory, received_at)), Some(reserved), Some(until)) =
|
||||
(original, inventory, reserved_at, deadline)
|
||||
&& original != e.boot_id
|
||||
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
|
||||
{
|
||||
reboot_witness = Some((original, inventory));
|
||||
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true}).to_string(),error_code:String::new()};
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at}).to_string(),error_code:String::new()};
|
||||
let mut evidence = e.clone();
|
||||
evidence.message_id = Uuid::new_v4().to_string();
|
||||
evidence.payload = Some(Payload::JobResult(result));
|
||||
@@ -465,18 +609,17 @@ pub async fn reconcile(
|
||||
&& terminal == "succeeded"
|
||||
&& matches!(local.state.as_str(), "awaiting_reboot" | "unknown")
|
||||
&& let Some(value) = final_row.get::<Option<Value>, _>("result")
|
||||
&& let Some((original, inventory)) = reboot_witness
|
||||
&& Uuid::parse_str(&original).ok() == decision.original_boot_id
|
||||
&& decision.original_boot_id.is_some()
|
||||
&& value["evidence"]["postconditions_passed"] == json!(true)
|
||||
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
|
||||
&& value["evidence"]["new_boot_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| id == e.boot_id)
|
||||
&& value["evidence"]["inventory_message_id"] == json!(inventory)
|
||||
{
|
||||
let original:Option<String>=sqlx::query_scalar("SELECT data->>'boot_id' FROM audit_events WHERE event_type='start_reserved' AND data->>'job_id'=$1 ORDER BY created_at DESC LIMIT 1").bind(job_id.to_string()).fetch_optional(&mut **tx).await?;
|
||||
if original.as_ref().and_then(|v| Uuid::parse_str(v).ok()) == decision.original_boot_id
|
||||
&& decision.original_boot_id.is_some()
|
||||
&& value["evidence"]["postconditions_passed"] == json!(true)
|
||||
&& value["evidence"]["reconciled_after_reboot"] == json!(true)
|
||||
&& value["evidence"]["new_boot_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| Some(id) != original.as_deref())
|
||||
{
|
||||
reboot_reconciled = true;
|
||||
}
|
||||
reboot_reconciled = true;
|
||||
}
|
||||
decision.revision = final_row.get::<i64, _>("revision") as u64;
|
||||
decision.outcome_state = terminal.clone();
|
||||
@@ -603,6 +746,348 @@ async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &st
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn reboot_deadline_uses_only_the_fixed_plan_and_trusted_reservation() {
|
||||
let reserved = Utc::now();
|
||||
let action = Action::Reboot {
|
||||
reason: "Explicit final Cloud VM acceptance".into(),
|
||||
reconnect_timeout_seconds: 180,
|
||||
dependency_group: "single-test-vm".into(),
|
||||
};
|
||||
assert_eq!(
|
||||
reboot_deadline(&action, Some(reserved)),
|
||||
Some(reserved + Duration::seconds(180))
|
||||
);
|
||||
assert_eq!(reboot_deadline(&action, None), None);
|
||||
assert_eq!(
|
||||
reboot_deadline(
|
||||
&Action::Reboot {
|
||||
reason: "test".into(),
|
||||
reconnect_timeout_seconds: 0,
|
||||
dependency_group: "single-test-vm".into()
|
||||
},
|
||||
Some(reserved)
|
||||
),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
reboot_deadline(&action, Some(chrono::DateTime::<Utc>::MAX_UTC)),
|
||||
None
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
|
||||
async fn reboot_inventory_requires_trusted_timely_same_identity_and_boot() {
|
||||
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
rmm_persistence::verify_application_role(&app)
|
||||
.await
|
||||
.unwrap();
|
||||
let tenant = Uuid::new_v4();
|
||||
let asset = Uuid::new_v4();
|
||||
let agent = Uuid::new_v4();
|
||||
let boot = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot deadline test')")
|
||||
.bind(tenant)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO assets VALUES($1,$2)")
|
||||
.bind(tenant)
|
||||
.bind(asset)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)")
|
||||
.bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
|
||||
let now = Utc::now();
|
||||
let reserved = now - Duration::seconds(200);
|
||||
let deadline = reserved + Duration::seconds(180);
|
||||
let window = RebootWindow {
|
||||
reserved_at: reserved,
|
||||
deadline,
|
||||
now,
|
||||
require_fresh: true,
|
||||
};
|
||||
let timely = Uuid::new_v4();
|
||||
let late = Uuid::new_v4();
|
||||
for (sequence, message, received) in [
|
||||
(1, timely, deadline),
|
||||
(2, late, deadline + Duration::microseconds(1)),
|
||||
(3, Uuid::new_v4(), reserved),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
||||
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&admin).await.unwrap();
|
||||
}
|
||||
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
||||
assert_eq!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &window, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &window, Some(late))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, Uuid::new_v4(), boot, &window, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, Uuid::new_v4(), agent, boot, &window, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, Uuid::new_v4(), &window, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let later = RebootWindow {
|
||||
now: now + Duration::minutes(6),
|
||||
..window
|
||||
};
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
// A terminal result reuses only its exact original timely receipt;
|
||||
// later metadata cannot replace or extend that immutable proof.
|
||||
let historical = RebootWindow {
|
||||
require_fresh: false,
|
||||
..later
|
||||
};
|
||||
assert_eq!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &historical, Some(timely))
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
let before_return = RebootWindow {
|
||||
now: deadline - Duration::microseconds(1),
|
||||
..window
|
||||
};
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &before_return, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
tx.rollback().await.unwrap();
|
||||
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs WHERE tenant_id=$1)+(SELECT count(*) FROM outbox WHERE tenant_id=$1)+(SELECT count(*) FROM execution_permits WHERE tenant_id=$1)").bind(tenant).fetch_one(&admin).await.unwrap();
|
||||
assert_eq!(effects, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the existing dedicated isolated PostgreSQL harness"]
|
||||
async fn reboot_success_ingress_rejects_forged_late_proof_and_preserves_exact_replay() {
|
||||
let admin = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let app = sqlx::PgPool::connect(&std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
rmm_persistence::verify_application_role(&app)
|
||||
.await
|
||||
.unwrap();
|
||||
let tenant = Uuid::new_v4();
|
||||
let asset = Uuid::new_v4();
|
||||
let agent = Uuid::new_v4();
|
||||
let boot = Uuid::new_v4();
|
||||
let old = Uuid::new_v4();
|
||||
let job = Uuid::new_v4();
|
||||
let attempt = Uuid::new_v4();
|
||||
let operation = Uuid::new_v4();
|
||||
let plan = Uuid::new_v4();
|
||||
let authorization = Uuid::new_v4();
|
||||
let grant = Uuid::new_v4();
|
||||
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
|
||||
let reserved = now - Duration::seconds(200);
|
||||
let deadline = reserved + Duration::seconds(180);
|
||||
let payload = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: tenant,
|
||||
asset_id: asset,
|
||||
agent_id: agent,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: job,
|
||||
attempt_id: attempt,
|
||||
action: Action::Reboot {
|
||||
reason: "Isolated postcondition test; no actual reboot".into(),
|
||||
reconnect_timeout_seconds: 180,
|
||||
dependency_group: "single-test-vm".into(),
|
||||
},
|
||||
execution_identity: "root".into(),
|
||||
not_before: reserved,
|
||||
start_before: now + Duration::minutes(5),
|
||||
max_runtime_seconds: 240,
|
||||
max_output_bytes: 65536,
|
||||
};
|
||||
let bytes = serde_json::to_vec(&payload).unwrap();
|
||||
sqlx::query("INSERT INTO tenants VALUES($1,'isolated reboot ingress test')")
|
||||
.bind(tenant)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO assets VALUES($1,$2)")
|
||||
.bind(tenant)
|
||||
.bind(asset)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
|
||||
.bind(tenant)
|
||||
.bind(operation)
|
||||
.bind(authorization)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,state,start_reserved,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'unknown',true,'device_execution')").bind(tenant).bind(job).bind(attempt).bind(operation).bind(agent).bind(asset).bind(&bytes).bind(hash(&bytes)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO execution_permits VALUES($1,$2,$3,$4,$5,$6,$7)")
|
||||
.bind(tenant)
|
||||
.bind(grant)
|
||||
.bind(job)
|
||||
.bind(attempt)
|
||||
.bind(b"{}".as_slice())
|
||||
.bind(vec![0u8; 64])
|
||||
.bind(reserved)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO audit_events VALUES($1,$2,'isolated-test','start_reserved',$3,$4)",
|
||||
)
|
||||
.bind(tenant)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({"job_id":job,"grant_id":grant,"boot_id":old}))
|
||||
.bind(reserved)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let timely = Uuid::new_v4();
|
||||
let late = Uuid::new_v4();
|
||||
for (sequence, message, received) in [
|
||||
(1, timely, deadline),
|
||||
(2, late, deadline + Duration::microseconds(1)),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
||||
}
|
||||
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
|
||||
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline});
|
||||
let mut result = JobResult {
|
||||
job_id: job.to_string(),
|
||||
attempt_id: attempt.to_string(),
|
||||
grant_id: grant.to_string(),
|
||||
revision: 1,
|
||||
state: "succeeded".into(),
|
||||
evidence_json: evidence.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut envelope = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: agent.to_string(),
|
||||
boot_id: boot.to_string(),
|
||||
payload: Some(Payload::JobResult(result.clone())),
|
||||
..Default::default()
|
||||
};
|
||||
let mut tx = rmm_persistence::tenant_tx(&app, tenant).await.unwrap();
|
||||
assert!(
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
|
||||
.bind(job)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.unwrap(),
|
||||
"unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
let mut good = evidence.clone();
|
||||
good["inventory_message_id"] = json!(timely);
|
||||
let mut false_clock = good.clone();
|
||||
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
|
||||
result.evidence_json = false_clock.to_string();
|
||||
envelope.payload = Some(Payload::JobResult(result.clone()));
|
||||
assert!(
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
result.evidence_json = good.to_string();
|
||||
envelope.payload = Some(Payload::JobResult(result.clone()));
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, String>("SELECT state FROM jobs WHERE job_id=$1")
|
||||
.bind(job)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.unwrap(),
|
||||
"succeeded"
|
||||
);
|
||||
// The original timely receipt survives later inventories and wall time.
|
||||
let later = RebootWindow {
|
||||
reserved_at: reserved,
|
||||
deadline,
|
||||
now: now + Duration::minutes(10),
|
||||
require_fresh: false,
|
||||
};
|
||||
assert_eq!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(timely))
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
// Exact durable-message replay ACKs the existing receipt, without
|
||||
// accepting a new result or extending its original evidence window.
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM job_receipts")
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
tx.rollback().await.unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn uncertain_request_resolves_only_to_a_known_checkpoint_anchor() {
|
||||
let applied = Uuid::new_v4();
|
||||
let pending = Uuid::new_v4();
|
||||
|
||||
Reference in New Issue
Block a user