feat: qualify bound configure-only recovery runtime on cloud VM
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 18:07:46 +01:00
parent b24314da9e
commit 2230235dfb
21 changed files with 1749 additions and 80 deletions
@@ -132,6 +132,120 @@ struct LedgerRow {
start: Vec<u8>,
outcome: Vec<u8>,
}
pub struct ClosedPartial {
pub reference: rmm_domain::apt_transaction::AptNativeRecovery,
pub proof: Vec<u8>,
pub job_id: Uuid,
pub authorization_id: Uuid,
pub plan_sha256: String,
pub previous_start_before: DateTime<Utc>,
pub baseline_sha256: String,
pub status_fingerprint: String,
}
/// Only a previously classified partial whose exact outcome was closed by
/// signed RootSync can seed a new, separately approved configure-only job.
pub fn closed_partial(
db: &Connection,
binding: &Binding,
wanted: Option<&rmm_domain::apt_transaction::AptNativeRecovery>,
) -> Result<ClosedPartial> {
closed_partial_from(db, binding, wanted, read_evidence)
}
fn closed_partial_from(
db: &Connection,
binding: &Binding,
wanted: Option<&rmm_domain::apt_transaction::AptNativeRecovery>,
read: impl Fn() -> Result<BTreeMap<String, Vec<u8>>>,
) -> Result<ClosedPartial> {
let mut q=db.prepare("SELECT c.rowid,c.grant_id,c.agent,c.generation,c.job,c.attempt,e.start,c.result,e.outcome,p.original_request_sha256,p.previous_outcome_sha256,p.previous_outcome,p.proof_sha256,p.proof,p.classified_outcome,p.checked_at_ms FROM apt_partial_reconciliations p JOIN consumed c USING(grant_id) JOIN execution_evidence e USING(grant_id) ORDER BY c.rowid")?;
let rows = q
.query_map([], |r| {
Ok((
LedgerRow {
rowid: r.get(0)?,
grant: r.get(1)?,
agent: r.get(2)?,
generation: r.get(3)?,
job: r.get(4)?,
attempt: r.get(5)?,
start: r.get(6)?,
outcome: r.get(11)?,
},
r.get::<_, Option<Vec<u8>>>(7)?,
r.get::<_, Option<Vec<u8>>>(8)?,
r.get::<_, String>(9)?,
r.get::<_, String>(10)?,
r.get::<_, String>(12)?,
r.get::<_, Vec<u8>>(13)?,
r.get::<_, Vec<u8>>(14)?,
r.get::<_, i64>(15)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
ensure!(rows.len() <= 256, "APT_RECOVERY_PROOF_LIMIT");
let mut found = None;
for (row, result, outcome, request_sha, old_sha, proof_sha, proof, classified, checked) in rows
{
if wanted.is_some_and(|w| w.previous_grant_id.to_string() != row.grant) {
continue;
}
let Some(c) = eligible(binding, row)? else {
continue;
};
let reference = rmm_domain::apt_transaction::AptNativeRecovery {
schema_version: 1,
previous_grant_id: c.probe.grant_id,
previous_attempt_id: c.probe.attempt_id,
partial_proof_sha256: proof_sha.clone(),
};
ensure!(
wanted.is_none_or(|w| w == &reference)
&& reference.validate()
&& hash(&proof) == proof_sha
&& hash(&c.original_outcome) == old_sha
&& hash(&serde_json::to_vec(&c.request)?) == request_sha,
"APT_RECOVERY_IMMUTABLE_PROOF"
);
let result: Outcome =
serde_json::from_slice(&result.context("APT_RECOVERY_PRIOR_UNRESOLVED")?)?;
let classified_value: Value = serde_json::from_slice(&classified)?;
ensure!(
result.state == "failed"
&& result.error_code == CODE
&& !result.reboot_required
&& serde_json::to_value(&result)? == classified_value
&& serde_json::from_slice::<Value>(
&outcome.context("APT_RECOVERY_PRIOR_OUTCOME")?
)? == classified_value,
"APT_RECOVERY_PRIOR_NOT_CLOSED_PARTIAL"
);
let p: PartialProof = serde_json::from_slice(&proof)?;
ensure!(p.checked_at_ms == checked, "APT_RECOVERY_PROOF_TIME");
let files = read()?;
verify_proof(
&c,
&files,
&ProbeOutput {
effect_state: "known_failed".into(),
code: CODE.into(),
proof: p.clone(),
},
DateTime::from_timestamp_millis(checked).context("APT_RECOVERY_PROOF_TIME")?,
)?;
ensure!(found.is_none(), "APT_RECOVERY_AMBIGUOUS_PROOFS");
found = Some(ClosedPartial {
reference,
proof,
job_id: c.probe.job_id,
authorization_id: c.probe.authorization_id,
plan_sha256: c.probe.plan_sha256,
previous_start_before: c.permit.start_before,
baseline_sha256: p.observed_snapshot_sha256,
status_fingerprint: p.status_fingerprint,
});
}
found.context("APT_RECOVERY_CLOSED_PROOF_REQUIRED")
}
fn digest(s: &str) -> bool {
s.len() == 64
&& s.bytes()
@@ -176,6 +290,9 @@ fn eligible(binding: &Binding, row: LedgerRow) -> Result<Option<Candidate>> {
else {
return Ok(None);
};
if apt_manifest.schema_version != 3 || apt_manifest.native_retry.is_none() {
return Ok(None);
}
ensure!(
recipe_id == apt::INSTALL_RECIPE
&& recipe_version == "1"
@@ -847,4 +964,38 @@ mod tests {
classified
);
}
#[test]
fn new_recovery_requires_exact_closed_partial_signed_history_and_immutable_proof() {
let (binding, row, files, now) = fixture();
let c = eligible(&binding, row.clone()).unwrap().unwrap();
let proof = output(&c, &files, now).proof;
let mut db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,agent TEXT,generation INTEGER,job TEXT,attempt TEXT,result BLOB);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,start BLOB,outcome BLOB);").unwrap();
db.execute("INSERT INTO consumed(rowid,grant_id,agent,generation,job,attempt,result) VALUES(?1,?2,?3,?4,?5,?6,NULL)",params![row.rowid,row.grant,row.agent,row.generation,row.job,row.attempt]).unwrap();
db.execute(
"INSERT INTO execution_evidence VALUES(?1,?2,?3)",
params![row.grant, row.start, row.outcome],
)
.unwrap();
persist(&mut db, &c, &proof).unwrap();
assert!(closed_partial_from(&db, &binding, None, || Ok(files.clone())).is_err());
db.execute(
"UPDATE consumed SET result=(SELECT outcome FROM execution_evidence)",
[],
)
.unwrap();
let closed = closed_partial_from(&db, &binding, None, || Ok(files.clone())).unwrap();
assert_eq!(closed.reference.previous_grant_id, c.probe.grant_id);
assert_eq!(closed.reference.previous_attempt_id, c.probe.attempt_id);
assert_eq!(closed.baseline_sha256, proof.observed_snapshot_sha256);
let mut changed = closed.reference.clone();
changed.previous_attempt_id = Uuid::new_v4();
assert!(closed_partial_from(&db, &binding, Some(&changed), || Ok(files.clone())).is_err());
let mut changed_files = files.clone();
changed_files.insert("signed-action.json".into(), b"{}".to_vec());
assert!(closed_partial_from(&db, &binding, None, || Ok(changed_files.clone())).is_err());
db.execute("UPDATE apt_partial_reconciliations SET proof=x'7b7d'", [])
.unwrap();
assert!(closed_partial_from(&db, &binding, None, || Ok(files.clone())).is_err());
}
}