Add gated agent update planning, preserved handoff and Linux worker ports
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -142,6 +142,66 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
gate.helper = None;
|
||||
Ok(())
|
||||
}
|
||||
/// Typed read-only ACK for exactly the own original update handoff. It
|
||||
/// clears only the control delivery acknowledgement; both start flags stay
|
||||
/// false and can_start remains false for ALL jobs, including this attempt.
|
||||
pub fn confirm_update_helper(&mut self, snapshot: &HelperSnapshot) -> Result<()> {
|
||||
let gate = self.gate.as_mut().context("NO_SIGNED_DECISION")?;
|
||||
let expected = gate.helper.context("NO_HELPER_APPLY_PENDING")?;
|
||||
ensure!(
|
||||
!gate.ready
|
||||
&& !gate.helper_ready
|
||||
&& snapshot.journal_id == expected.0
|
||||
&& snapshot.checkpoint == expected.1
|
||||
&& snapshot.previous_decision_id == Some(expected.2),
|
||||
"UPDATE_CONTROL_NOT_APPLIED"
|
||||
);
|
||||
let updates: Vec<_> = snapshot
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| !a.update_handoff_bytes.is_empty())
|
||||
.collect();
|
||||
ensure!(updates.len() == 1, "UPDATE_CONTROL_EXACTLY_ONE_REQUIRED");
|
||||
let local = updates[0];
|
||||
let requested = gate
|
||||
.request
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|a| a.job_id == local.job_id && a.attempt_id == local.attempt_id)
|
||||
.context("UPDATE_CONTROL_NOT_REQUESTED")?;
|
||||
ensure!(
|
||||
local.state == "running"
|
||||
&& local.consumption == "consumed"
|
||||
&& local.outcome_bytes.is_empty()
|
||||
&& local.update_handoff_bytes == requested.update_handoff_bytes
|
||||
&& local.grant_id == requested.grant_id
|
||||
&& local.job_payload_hash == requested.job_payload_hash,
|
||||
"UPDATE_CONTROL_CHANGED"
|
||||
);
|
||||
let decision: ReconcileDecisionPayload =
|
||||
serde_json::from_slice(&gate.response.decision_bytes)?;
|
||||
ensure!(
|
||||
decision
|
||||
.attempts
|
||||
.iter()
|
||||
.any(|d| d.job_id.to_string() == local.job_id
|
||||
&& d.attempt_id.to_string() == local.attempt_id
|
||||
&& d.update_inflight.is_some()
|
||||
&& !d.unlock),
|
||||
"UPDATE_CONTROL_NOT_SIGNED"
|
||||
);
|
||||
ensure!(
|
||||
snapshot
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| a.job_id != local.job_id || a.attempt_id != local.attempt_id)
|
||||
.all(|a| terminal(&a.state)
|
||||
|| (a.state == "accepted" && a.consumption == "unconsumed")),
|
||||
"UPDATE_OTHER_EFFECT_UNRESOLVED"
|
||||
);
|
||||
gate.helper = None;
|
||||
Ok(())
|
||||
}
|
||||
pub fn prepare(
|
||||
&mut self,
|
||||
db: &mut Connection,
|
||||
@@ -216,7 +276,7 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
ingest_helper(
|
||||
&tx,
|
||||
ledger,
|
||||
&self.binding,
|
||||
(&self.binding, &self.verifier),
|
||||
&channel,
|
||||
h,
|
||||
now,
|
||||
@@ -230,7 +290,17 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
)?;
|
||||
// A possibly started attempt is NEVER returned to accepted. Preserve
|
||||
// revisions and results; only ambiguous no-result states become unknown.
|
||||
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?;
|
||||
let inflight: Vec<_> = helper
|
||||
.into_iter()
|
||||
.flat_map(|h| h.attempts.iter())
|
||||
.filter(|a| !a.update_handoff_bytes.is_empty())
|
||||
.collect();
|
||||
ensure!(inflight.len() <= 1, "UPDATE_MULTIPLE_INFLIGHT");
|
||||
if let Some(update) = inflight.first() {
|
||||
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT (job=?1 AND attempt=?2) AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",params![update.job_id,update.attempt_id])?;
|
||||
} else {
|
||||
tx.execute("UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE state IN ('requesting','running','verifying') AND NOT EXISTS(SELECT 1 FROM reconcile_results r WHERE r.job=job_attempts.job AND r.attempt=job_attempts.attempt AND r.revision>=job_attempts.revision)",[])?;
|
||||
}
|
||||
let mut attempts = attempts(&tx, ledger, &self.binding)?;
|
||||
if let Some(h) = helper {
|
||||
merge_helper(&mut attempts, h)?;
|
||||
@@ -436,7 +506,8 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
safe = false;
|
||||
continue;
|
||||
}
|
||||
let outcome = validate_attempt(&job, &offer, local, d, channel.boot_id)?;
|
||||
let outcome =
|
||||
validate_attempt(&job, &offer, local, d, channel.boot_id, &self.binding, now)?;
|
||||
safe &= (d.outcome_state == "accepted"
|
||||
&& local.consumption == "unconsumed"
|
||||
&& local.grant_id.is_empty())
|
||||
@@ -866,6 +937,7 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|
||||
}
|
||||
}
|
||||
summaries.push(SyncAttempt {
|
||||
update_handoff_bytes: Vec::new(),
|
||||
job_id: offer.job_id,
|
||||
attempt_id: offer.attempt_id,
|
||||
job_payload_hash: offer.job_payload_hash,
|
||||
@@ -897,12 +969,13 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|
||||
fn ingest_helper(
|
||||
db: &Connection,
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
binding_and_verifier: (&Binding, &impl Ed25519Verifier),
|
||||
channel: &Channel,
|
||||
helper: &HelperSnapshot,
|
||||
now: DateTime<Utc>,
|
||||
identity: HelperIdentity,
|
||||
) -> Result<()> {
|
||||
let (binding, verifier) = binding_and_verifier;
|
||||
let mut statement = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
|
||||
for row in statement.query_map([], |r| {
|
||||
Ok((
|
||||
@@ -952,6 +1025,65 @@ fn ingest_helper(
|
||||
"HELPER_ATTEMPT_BINDING"
|
||||
);
|
||||
if h.outcome_bytes.is_empty() {
|
||||
if !h.update_handoff_bytes.is_empty() {
|
||||
let header =
|
||||
validate_update_summary(binding, &job, &offer, h, channel.boot_id, now)?;
|
||||
ensure!(
|
||||
header.helper_journal_id == helper.journal_id
|
||||
&& header.agent_journal_id == metadata(db)?.journal_id,
|
||||
"UPDATE_HELPER_JOURNAL_BINDING"
|
||||
);
|
||||
let state: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![h.job_id, h.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
matches!(
|
||||
state.as_str(),
|
||||
"accepted" | "requesting" | "running" | "verifying"
|
||||
),
|
||||
"UPDATE_CANNOT_HEAL_UNKNOWN"
|
||||
);
|
||||
type OriginalEvidence = (
|
||||
String,
|
||||
u64,
|
||||
String,
|
||||
String,
|
||||
Vec<u8>,
|
||||
Option<Vec<u8>>,
|
||||
Option<Vec<u8>>,
|
||||
);
|
||||
let original:Option<OriginalEvidence>=ledger.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start,e.outcome,c.result FROM execution_evidence e JOIN consumed c USING(grant_id) WHERE grant_id=?1",[&h.grant_id],
|
||||
|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?;
|
||||
let (agent, generation, job_id, attempt_id, start, outcome, result) =
|
||||
original.context("UPDATE_LOCAL_ORIGINAL_MISSING")?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == h.job_id
|
||||
&& attempt_id == h.attempt_id,
|
||||
"UPDATE_LOCAL_CONSUMPTION_BINDING"
|
||||
);
|
||||
ensure!(
|
||||
outcome.is_none() && result.is_none(),
|
||||
"UPDATE_CANNOT_REPLACE_OUTCOME"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: crate::auth::Request =
|
||||
serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
binding.verify_with(&request, permit.not_before, verifier)?;
|
||||
ensure!(
|
||||
hash(&serde_json::to_vec(&request)?) == header.attempt.original_request_sha256
|
||||
&& permit.grant_id == header.attempt.grant_id
|
||||
&& permit.agent_update_plan_id == Some(header.attempt.plan_id),
|
||||
"UPDATE_LOCAL_ORIGINAL_CHANGED"
|
||||
);
|
||||
db.execute("UPDATE job_attempts SET state='running',revision=MAX(revision,?3) WHERE job=?1 AND attempt=?2",params![h.job_id,h.attempt_id,h.revision])?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let o: ReconciledOutcome = serde_json::from_slice(&h.outcome_bytes)?;
|
||||
@@ -1084,6 +1216,13 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
|
||||
"HELPER_ATTEMPT_BINDING"
|
||||
);
|
||||
if h.consumption != "unconsumed" {
|
||||
if !h.update_handoff_bytes.is_empty() {
|
||||
ensure!(
|
||||
h.state == "running" && a.state == "running" && a.outcome_bytes.is_empty(),
|
||||
"UPDATE_CANNOT_HEAL_UNKNOWN"
|
||||
);
|
||||
a.update_handoff_bytes.clone_from(&h.update_handoff_bytes);
|
||||
}
|
||||
a.grant_id = h.grant_id.clone();
|
||||
a.consumption = h.consumption.clone();
|
||||
ensure!(
|
||||
@@ -1101,6 +1240,42 @@ fn merge_helper(local: &mut [SyncAttempt], helper: &HelperSnapshot) -> Result<()
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn validate_update_summary(
|
||||
_binding: &Binding,
|
||||
job: &JobPayload,
|
||||
offer: &JobOffer,
|
||||
local: &SyncAttempt,
|
||||
boot: Uuid,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<rmm_updater_handoff::Handoff> {
|
||||
let proof =
|
||||
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let key = rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
|
||||
let (h, _) =
|
||||
rmm_domain::agent_update::verify_inflight_proof(&proof, key).map_err(anyhow::Error::msg)?;
|
||||
let now = u64::try_from(now.timestamp())?;
|
||||
ensure!(
|
||||
rmm_domain::agent_update::handoff_matches_job(&h, job)
|
||||
&& h.boot_id == boot
|
||||
&& h.attempt.grant_id.to_string() == local.grant_id
|
||||
&& h.attempt.authorization_id.to_string() == offer.authorization_id
|
||||
&& h.attempt.plan_sha256 == offer.plan_hash
|
||||
&& h.attempt.job_payload_sha256 == offer.job_payload_hash
|
||||
&& local.original_boot_id == boot.to_string()
|
||||
&& local.consumption == "consumed"
|
||||
&& local.state == "running"
|
||||
&& local.outcome_bytes.is_empty()
|
||||
&& local.outcome_sha256.is_empty()
|
||||
&& local.result.is_none()
|
||||
&& local.result_envelope.is_none()
|
||||
&& h.consumed_at <= now
|
||||
&& now < h.finish_before,
|
||||
"UPDATE_INFLIGHT_SUMMARY_BINDING"
|
||||
);
|
||||
Ok(h)
|
||||
}
|
||||
|
||||
pub fn verify_response(
|
||||
binding: &Binding,
|
||||
request: &SyncRequest,
|
||||
@@ -1176,6 +1351,8 @@ fn validate_attempt(
|
||||
local: &SyncAttempt,
|
||||
d: &AttemptDecision,
|
||||
boot: Uuid,
|
||||
binding: &Binding,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
ensure!(
|
||||
d.job_payload_hash == offer.job_payload_hash
|
||||
@@ -1199,6 +1376,25 @@ fn validate_attempt(
|
||||
!terminal(&local.state) || d.outcome_state == local.state,
|
||||
"TERMINAL_STATE_REGRESSION"
|
||||
);
|
||||
if d.update_inflight.is_some() || !local.update_handoff_bytes.is_empty() {
|
||||
validate_update_summary(binding, job, offer, local, boot, now)?;
|
||||
let proof =
|
||||
rmm_domain::agent_update::UpdateInflightProof::from_bytes(&local.update_handoff_bytes)
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let key =
|
||||
rmm_domain::agent_update::job_health_public_key(job).map_err(anyhow::Error::msg)?;
|
||||
let expected =
|
||||
rmm_domain::agent_update::inflight_decision(&proof, key).map_err(anyhow::Error::msg)?;
|
||||
ensure!(
|
||||
d.update_inflight.as_ref() == Some(&expected)
|
||||
&& d.outcome_state == "running"
|
||||
&& !d.unlock
|
||||
&& d.outcome_bytes.is_empty()
|
||||
&& d.outcome_sha256.is_none(),
|
||||
"UPDATE_INFLIGHT_DECISION_CHANGED"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
if d.outcome_state == "accepted" {
|
||||
ensure!(
|
||||
local.state == "accepted"
|
||||
|
||||
Reference in New Issue
Block a user