Add gated agent update planning, preserved handoff and Linux worker ports
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -372,6 +372,23 @@ pub async fn reconcile(
|
||||
reason = "HELPER_CHECKPOINT_REQUIRED".into();
|
||||
chain_valid = false;
|
||||
}
|
||||
let previous_payload: Option<Vec<u8>> = if let Some(old) = &previous {
|
||||
sqlx::query_scalar(
|
||||
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
|
||||
)
|
||||
.bind(agent)
|
||||
.bind(old.get::<Uuid, _>("decision_id"))
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let previous_update_decisions: Vec<AttemptDecision> = previous_payload
|
||||
.as_deref()
|
||||
.map(serde_json::from_slice::<ReconcileDecisionPayload>)
|
||||
.transpose()?
|
||||
.map(|p| p.attempts)
|
||||
.unwrap_or_default();
|
||||
let mut locals = BTreeMap::new();
|
||||
for local in &r.attempts {
|
||||
let id = Uuid::parse_str(&local.job_id)?;
|
||||
@@ -389,6 +406,10 @@ pub async fn reconcile(
|
||||
"invalid attempt summary"
|
||||
);
|
||||
i64::try_from(local.revision)?;
|
||||
if !local.update_handoff_bytes.is_empty() {
|
||||
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
}
|
||||
ensure!(
|
||||
matches!(
|
||||
local.state.as_str(),
|
||||
@@ -451,6 +472,7 @@ pub async fn reconcile(
|
||||
Some(local.outcome_sha256.clone())
|
||||
};
|
||||
let mut decision = AttemptDecision {
|
||||
update_inflight: None,
|
||||
job_id,
|
||||
attempt_id: attempt,
|
||||
job_payload_hash: local.job_payload_hash.clone(),
|
||||
@@ -510,7 +532,7 @@ pub async fn reconcile(
|
||||
ready = false;
|
||||
}
|
||||
let grant = sqlx::query(
|
||||
"SELECT grant_id,payload,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
||||
"SELECT grant_id,payload,signature,created_at FROM execution_permits WHERE job_id=$1 AND attempt_id=$2",
|
||||
)
|
||||
.bind(job_id)
|
||||
.bind(attempt)
|
||||
@@ -554,6 +576,138 @@ pub async fn reconcile(
|
||||
decisions.push(decision);
|
||||
continue;
|
||||
}
|
||||
if !local.update_handoff_bytes.is_empty() {
|
||||
use rmm_domain::agent_update as update;
|
||||
ensure!(
|
||||
chain_valid
|
||||
&& r.journal_complete
|
||||
&& reserved
|
||||
&& grant_matches
|
||||
&& helper_journal.is_some()
|
||||
&& matches!(
|
||||
server_state.as_str(),
|
||||
"queued" | "accepted" | "running" | "verifying"
|
||||
)
|
||||
&& row.get::<Option<Value>, _>("result").is_none()
|
||||
&& local.state == "running"
|
||||
&& local.consumption == "consumed"
|
||||
&& local.outcome_bytes.is_empty()
|
||||
&& local.outcome_sha256.is_empty()
|
||||
&& local.result.is_none()
|
||||
&& local.result_envelope.is_none(),
|
||||
"UPDATE_INFLIGHT_NOT_ORIGINAL_ACTIVE_ATTEMPT"
|
||||
);
|
||||
let proof = update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let key = update::job_health_public_key(&payload).map_err(anyhow::Error::msg)?;
|
||||
let (header, _) =
|
||||
update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
|
||||
let grant = grant.as_ref().context("UPDATE_ORIGINAL_GRANT_REQUIRED")?;
|
||||
let permit_bytes: Vec<u8> = grant.get("payload");
|
||||
let signature: Vec<u8> = grant.get("signature");
|
||||
let permit: ExecutionPermitPayload = serde_json::from_slice(&permit_bytes)?;
|
||||
let job_bytes: Vec<u8> = row.get("payload");
|
||||
let original = update::OriginalHelperRequest {
|
||||
job_bytes: &job_bytes,
|
||||
permit_bytes: &permit_bytes,
|
||||
signature: &signature,
|
||||
plan_hash: &permit.plan_hash,
|
||||
authorization_id: permit.authorization_id,
|
||||
};
|
||||
let authorized_plan:Option<Uuid>=sqlx::query_scalar("SELECT plan_id FROM authorizations WHERE tenant_id=$1 AND authorization_id=$2 AND plan_hash=$3")
|
||||
.bind(tenant).bind(permit.authorization_id).bind(&permit.plan_hash).fetch_optional(&mut **tx).await?;
|
||||
let original_sync: Vec<u8> = sqlx::query_scalar(
|
||||
"SELECT payload FROM sync_checkpoints WHERE agent_id=$1 AND decision_id=$2",
|
||||
)
|
||||
.bind(agent)
|
||||
.bind(header.previous_decision_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?
|
||||
.context("UPDATE_ORIGINAL_READY_SYNC_REQUIRED")?;
|
||||
let original_sync: ReconcileDecisionPayload = serde_json::from_slice(&original_sync)?;
|
||||
ensure!(
|
||||
original_sync.ready
|
||||
&& original_sync.helper_ready
|
||||
&& original_sync.agent_id == agent
|
||||
&& original_sync.tenant_id == tenant
|
||||
&& original_sync.journal_id == header.agent_journal_id
|
||||
&& original_sync.helper_journal_id == Some(header.helper_journal_id)
|
||||
&& original_sync.helper_checkpoint == header.helper_checkpoint
|
||||
&& original_sync.boot_id == header.boot_id
|
||||
&& original_sync.connection_generation == header.connection_generation
|
||||
&& original_sync.issued_at <= proof.delegation.prepared_at
|
||||
&& proof.delegation.prepared_at < original_sync.valid_until,
|
||||
"UPDATE_ORIGINAL_READY_SYNC_BINDING"
|
||||
);
|
||||
let seconds = u64::try_from(Utc::now().timestamp())?;
|
||||
ensure!(
|
||||
update::handoff_matches_job(&header, &payload)
|
||||
&& header.boot_id == boot
|
||||
&& header.attempt.plan_id
|
||||
== authorized_plan.context("UPDATE_ORIGINAL_PLAN_REQUIRED")?
|
||||
&& permit.agent_update_plan_id == Some(header.attempt.plan_id)
|
||||
&& header.agent_journal_id == journal_id
|
||||
&& Some(header.helper_journal_id) == helper_journal
|
||||
&& header.helper_checkpoint <= r.helper_checkpoint
|
||||
&& header.connection_generation <= e.connection_generation
|
||||
&& header.attempt.grant_id == permit.grant_id
|
||||
&& header.attempt.authorization_id == permit.authorization_id
|
||||
&& header.attempt.plan_sha256 == permit.plan_hash
|
||||
&& header.attempt.job_payload_sha256 == permit.job_payload_hash
|
||||
&& header.attempt.original_request_sha256
|
||||
== hash(&serde_json::to_vec(&original)?)
|
||||
&& permit.action == "update_agent"
|
||||
&& permit.max_runtime_seconds == 600
|
||||
&& permit.key_id == state.job_signer.key_id
|
||||
&& header.finish_before
|
||||
== header
|
||||
.consumed_at
|
||||
.checked_add(600)
|
||||
.context("UPDATE_DEADLINE_OVERFLOW")?
|
||||
&& header.consumed_at <= seconds
|
||||
&& seconds < header.finish_before
|
||||
&& proof.delegation.prepared_at >= permit.not_before
|
||||
&& proof.delegation.prepared_at < permit.start_before
|
||||
&& local.original_boot_id == boot.to_string(),
|
||||
"UPDATE_INFLIGHT_ORIGINAL_BINDING"
|
||||
);
|
||||
let ack = update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
|
||||
if let Some(old) = previous_update_decisions
|
||||
.iter()
|
||||
.find(|d| d.job_id == job_id && d.attempt_id == attempt)
|
||||
.and_then(|d| d.update_inflight.as_ref())
|
||||
{
|
||||
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
old.handoff_sha256 == ack.handoff_sha256
|
||||
&& old.delegation_sha256 == ack.delegation_sha256
|
||||
&& old.owner_nonce == ack.owner_nonce
|
||||
&& old.finish_before == ack.finish_before
|
||||
&& (hash(&decoded.handoff_record) == old.head_sha256
|
||||
|| decoded
|
||||
.events
|
||||
.iter()
|
||||
.any(|bytes| hash(bytes) == old.head_sha256)),
|
||||
"UPDATE_INFLIGHT_PREFIX_REGRESSION"
|
||||
);
|
||||
}
|
||||
let revision = local.revision.max(row.get::<i64, _>("revision") as u64);
|
||||
sqlx::query(
|
||||
"UPDATE jobs SET state='running',revision=GREATEST(revision,$2) WHERE job_id=$1",
|
||||
)
|
||||
.bind(job_id)
|
||||
.bind(i64::try_from(revision)?)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
decision.revision = revision;
|
||||
decision.outcome_state = "running".into();
|
||||
decision.update_inflight = Some(ack);
|
||||
decision.reason = "AGENT_UPDATE_INFLIGHT".into();
|
||||
decision.unlock = false;
|
||||
ready = false;
|
||||
decisions.push(decision);
|
||||
continue;
|
||||
}
|
||||
// A free script with no bound postcondition never becomes successful by
|
||||
// merely asserting a new state after unknown effect.
|
||||
let manual = server_state == "unknown"
|
||||
@@ -1286,6 +1440,7 @@ mod tests {
|
||||
#[test]
|
||||
fn accepted_is_not_a_new_attempt_or_grant() {
|
||||
let mut local = SyncAttempt {
|
||||
update_handoff_bytes: Vec::new(),
|
||||
state: "accepted".into(),
|
||||
consumption: "unconsumed".into(),
|
||||
..Default::default()
|
||||
|
||||
Reference in New Issue
Block a user