Add gated agent update planning, preserved handoff and Linux worker ports
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-14 21:48:21 +01:00
parent dc9cb1b096
commit 1e89c4577f
82 changed files with 25052 additions and 69 deletions
+156 -1
View File
@@ -372,6 +372,23 @@ pub async fn reconcile(
reason = "HELPER_CHECKPOINT_REQUIRED".into();
chain_valid = false;
}
let previous_payload: Option<Vec<u8>> = if let Some(old) = &previous {
sqlx::query_scalar(
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
)
.bind(agent)
.bind(old.get::<Uuid, _>("decision_id"))
.fetch_optional(&mut **tx)
.await?
} else {
None
};
let previous_update_decisions: Vec<AttemptDecision> = previous_payload
.as_deref()
.map(serde_json::from_slice::<ReconcileDecisionPayload>)
.transpose()?
.map(|p| p.attempts)
.unwrap_or_default();
let mut locals = BTreeMap::new();
for local in &r.attempts {
let id = Uuid::parse_str(&local.job_id)?;
@@ -389,6 +406,10 @@ pub async fn reconcile(
"invalid attempt summary"
);
i64::try_from(local.revision)?;
if !local.update_handoff_bytes.is_empty() {
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
}
ensure!(
matches!(
local.state.as_str(),
@@ -451,6 +472,7 @@ pub async fn reconcile(
Some(local.outcome_sha256.clone())
};
let mut decision = AttemptDecision {
update_inflight: None,
job_id,
attempt_id: attempt,
job_payload_hash: local.job_payload_hash.clone(),
@@ -510,7 +532,7 @@ pub async fn reconcile(
ready = false;
}
let grant = sqlx::query(
"SELECT grant_id,payload,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
"SELECT grant_id,payload,signature,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
)
.bind(job_id)
.bind(attempt)
@@ -554,6 +576,138 @@ pub async fn reconcile(
decisions.push(decision);
continue;
}
if !local.update_handoff_bytes.is_empty() {
use rmm_domain::agent_update as update;
ensure!(
chain_valid
&& r.journal_complete
&& reserved
&& grant_matches
&& helper_journal.is_some()
&& matches!(
server_state.as_str(),
"queued" | "accepted" | "running" | "verifying"
)
&& row.get::<Option<Value>, _>("result").is_none()
&& local.state == "running"
&& local.consumption == "consumed"
&& local.outcome_bytes.is_empty()
&& local.outcome_sha256.is_empty()
&& local.result.is_none()
&& local.result_envelope.is_none(),
"UPDATE_INFLIGHT_NOT_ORIGINAL_ACTIVE_ATTEMPT"
);
let proof = update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
.map_err(anyhow::Error::msg)?;
let key = update::job_health_public_key(&payload).map_err(anyhow::Error::msg)?;
let (header, _) =
update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
let grant = grant.as_ref().context("UPDATE_ORIGINAL_GRANT_REQUIRED")?;
let permit_bytes: Vec<u8> = grant.get("payload");
let signature: Vec<u8> = grant.get("signature");
let permit: ExecutionPermitPayload = serde_json::from_slice(&permit_bytes)?;
let job_bytes: Vec<u8> = row.get("payload");
let original = update::OriginalHelperRequest {
job_bytes: &job_bytes,
permit_bytes: &permit_bytes,
signature: &signature,
plan_hash: &permit.plan_hash,
authorization_id: permit.authorization_id,
};
let authorized_plan:Option<Uuid>=sqlx::query_scalar("SELECT plan_id FROM authorizations WHERE tenant_id=$1 AND authorization_id=$2 AND plan_hash=$3")
.bind(tenant).bind(permit.authorization_id).bind(&permit.plan_hash).fetch_optional(&mut **tx).await?;
let original_sync: Vec<u8> = sqlx::query_scalar(
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
)
.bind(agent)
.bind(header.previous_decision_id)
.fetch_optional(&mut **tx)
.await?
.context("UPDATE_ORIGINAL_READY_SYNC_REQUIRED")?;
let original_sync: ReconcileDecisionPayload = serde_json::from_slice(&original_sync)?;
ensure!(
original_sync.ready
&& original_sync.helper_ready
&& original_sync.agent_id == agent
&& original_sync.tenant_id == tenant
&& original_sync.journal_id == header.agent_journal_id
&& original_sync.helper_journal_id == Some(header.helper_journal_id)
&& original_sync.helper_checkpoint == header.helper_checkpoint
&& original_sync.boot_id == header.boot_id
&& original_sync.connection_generation == header.connection_generation
&& original_sync.issued_at <= proof.delegation.prepared_at
&& proof.delegation.prepared_at < original_sync.valid_until,
"UPDATE_ORIGINAL_READY_SYNC_BINDING"
);
let seconds = u64::try_from(Utc::now().timestamp())?;
ensure!(
update::handoff_matches_job(&header, &payload)
&& header.boot_id == boot
&& header.attempt.plan_id
== authorized_plan.context("UPDATE_ORIGINAL_PLAN_REQUIRED")?
&& permit.agent_update_plan_id == Some(header.attempt.plan_id)
&& header.agent_journal_id == journal_id
&& Some(header.helper_journal_id) == helper_journal
&& header.helper_checkpoint <= r.helper_checkpoint
&& header.connection_generation <= e.connection_generation
&& header.attempt.grant_id == permit.grant_id
&& header.attempt.authorization_id == permit.authorization_id
&& header.attempt.plan_sha256 == permit.plan_hash
&& header.attempt.job_payload_sha256 == permit.job_payload_hash
&& header.attempt.original_request_sha256
== hash(&serde_json::to_vec(&original)?)
&& permit.action == "update_agent"
&& permit.max_runtime_seconds == 600
&& permit.key_id == state.job_signer.key_id
&& header.finish_before
== header
.consumed_at
.checked_add(600)
.context("UPDATE_DEADLINE_OVERFLOW")?
&& header.consumed_at <= seconds
&& seconds < header.finish_before
&& proof.delegation.prepared_at >= permit.not_before
&& proof.delegation.prepared_at < permit.start_before
&& local.original_boot_id == boot.to_string(),
"UPDATE_INFLIGHT_ORIGINAL_BINDING"
);
let ack = update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
if let Some(old) = previous_update_decisions
.iter()
.find(|d| d.job_id == job_id && d.attempt_id == attempt)
.and_then(|d| d.update_inflight.as_ref())
{
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
ensure!(
old.handoff_sha256 == ack.handoff_sha256
&& old.delegation_sha256 == ack.delegation_sha256
&& old.owner_nonce == ack.owner_nonce
&& old.finish_before == ack.finish_before
&& (hash(&decoded.handoff_record) == old.head_sha256
|| decoded
.events
.iter()
.any(|bytes| hash(bytes) == old.head_sha256)),
"UPDATE_INFLIGHT_PREFIX_REGRESSION"
);
}
let revision = local.revision.max(row.get::<i64, _>("revision") as u64);
sqlx::query(
"UPDATE jobs SET state='running',revision=GREATEST(revision,$2) WHERE job_id=$1",
)
.bind(job_id)
.bind(i64::try_from(revision)?)
.execute(&mut **tx)
.await?;
decision.revision = revision;
decision.outcome_state = "running".into();
decision.update_inflight = Some(ack);
decision.reason = "AGENT_UPDATE_INFLIGHT".into();
decision.unlock = false;
ready = false;
decisions.push(decision);
continue;
}
// A free script with no bound postcondition never becomes successful by
// merely asserting a new state after unknown effect.
let manual = server_state == "unknown"
@@ -1286,6 +1440,7 @@ mod tests {
#[test]
fn accepted_is_not_a_new_attempt_or_grant() {
let mut local = SyncAttempt {
update_handoff_bytes: Vec::new(),
state: "accepted".into(),
consumption: "unconsumed".into(),
..Default::default()