feat: qualify bound configure-only recovery runtime on cloud VM
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -132,6 +132,120 @@ struct LedgerRow {
|
||||
start: Vec<u8>,
|
||||
outcome: Vec<u8>,
|
||||
}
|
||||
pub struct ClosedPartial {
|
||||
pub reference: rmm_domain::apt_transaction::AptNativeRecovery,
|
||||
pub proof: Vec<u8>,
|
||||
pub job_id: Uuid,
|
||||
pub authorization_id: Uuid,
|
||||
pub plan_sha256: String,
|
||||
pub previous_start_before: DateTime<Utc>,
|
||||
pub baseline_sha256: String,
|
||||
pub status_fingerprint: String,
|
||||
}
|
||||
/// Only a previously classified partial whose exact outcome was closed by
|
||||
/// signed RootSync can seed a new, separately approved configure-only job.
|
||||
pub fn closed_partial(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
wanted: Option<&rmm_domain::apt_transaction::AptNativeRecovery>,
|
||||
) -> Result<ClosedPartial> {
|
||||
closed_partial_from(db, binding, wanted, read_evidence)
|
||||
}
|
||||
fn closed_partial_from(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
wanted: Option<&rmm_domain::apt_transaction::AptNativeRecovery>,
|
||||
read: impl Fn() -> Result<BTreeMap<String, Vec<u8>>>,
|
||||
) -> Result<ClosedPartial> {
|
||||
let mut q=db.prepare("SELECT c.rowid,c.grant_id,c.agent,c.generation,c.job,c.attempt,e.start,c.result,e.outcome,p.original_request_sha256,p.previous_outcome_sha256,p.previous_outcome,p.proof_sha256,p.proof,p.classified_outcome,p.checked_at_ms FROM apt_partial_reconciliations p JOIN consumed c USING(grant_id) JOIN execution_evidence e USING(grant_id) ORDER BY c.rowid")?;
|
||||
let rows = q
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
LedgerRow {
|
||||
rowid: r.get(0)?,
|
||||
grant: r.get(1)?,
|
||||
agent: r.get(2)?,
|
||||
generation: r.get(3)?,
|
||||
job: r.get(4)?,
|
||||
attempt: r.get(5)?,
|
||||
start: r.get(6)?,
|
||||
outcome: r.get(11)?,
|
||||
},
|
||||
r.get::<_, Option<Vec<u8>>>(7)?,
|
||||
r.get::<_, Option<Vec<u8>>>(8)?,
|
||||
r.get::<_, String>(9)?,
|
||||
r.get::<_, String>(10)?,
|
||||
r.get::<_, String>(12)?,
|
||||
r.get::<_, Vec<u8>>(13)?,
|
||||
r.get::<_, Vec<u8>>(14)?,
|
||||
r.get::<_, i64>(15)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ensure!(rows.len() <= 256, "APT_RECOVERY_PROOF_LIMIT");
|
||||
let mut found = None;
|
||||
for (row, result, outcome, request_sha, old_sha, proof_sha, proof, classified, checked) in rows
|
||||
{
|
||||
if wanted.is_some_and(|w| w.previous_grant_id.to_string() != row.grant) {
|
||||
continue;
|
||||
}
|
||||
let Some(c) = eligible(binding, row)? else {
|
||||
continue;
|
||||
};
|
||||
let reference = rmm_domain::apt_transaction::AptNativeRecovery {
|
||||
schema_version: 1,
|
||||
previous_grant_id: c.probe.grant_id,
|
||||
previous_attempt_id: c.probe.attempt_id,
|
||||
partial_proof_sha256: proof_sha.clone(),
|
||||
};
|
||||
ensure!(
|
||||
wanted.is_none_or(|w| w == &reference)
|
||||
&& reference.validate()
|
||||
&& hash(&proof) == proof_sha
|
||||
&& hash(&c.original_outcome) == old_sha
|
||||
&& hash(&serde_json::to_vec(&c.request)?) == request_sha,
|
||||
"APT_RECOVERY_IMMUTABLE_PROOF"
|
||||
);
|
||||
let result: Outcome =
|
||||
serde_json::from_slice(&result.context("APT_RECOVERY_PRIOR_UNRESOLVED")?)?;
|
||||
let classified_value: Value = serde_json::from_slice(&classified)?;
|
||||
ensure!(
|
||||
result.state == "failed"
|
||||
&& result.error_code == CODE
|
||||
&& !result.reboot_required
|
||||
&& serde_json::to_value(&result)? == classified_value
|
||||
&& serde_json::from_slice::<Value>(
|
||||
&outcome.context("APT_RECOVERY_PRIOR_OUTCOME")?
|
||||
)? == classified_value,
|
||||
"APT_RECOVERY_PRIOR_NOT_CLOSED_PARTIAL"
|
||||
);
|
||||
let p: PartialProof = serde_json::from_slice(&proof)?;
|
||||
ensure!(p.checked_at_ms == checked, "APT_RECOVERY_PROOF_TIME");
|
||||
let files = read()?;
|
||||
verify_proof(
|
||||
&c,
|
||||
&files,
|
||||
&ProbeOutput {
|
||||
effect_state: "known_failed".into(),
|
||||
code: CODE.into(),
|
||||
proof: p.clone(),
|
||||
},
|
||||
DateTime::from_timestamp_millis(checked).context("APT_RECOVERY_PROOF_TIME")?,
|
||||
)?;
|
||||
ensure!(found.is_none(), "APT_RECOVERY_AMBIGUOUS_PROOFS");
|
||||
found = Some(ClosedPartial {
|
||||
reference,
|
||||
proof,
|
||||
job_id: c.probe.job_id,
|
||||
authorization_id: c.probe.authorization_id,
|
||||
plan_sha256: c.probe.plan_sha256,
|
||||
previous_start_before: c.permit.start_before,
|
||||
baseline_sha256: p.observed_snapshot_sha256,
|
||||
status_fingerprint: p.status_fingerprint,
|
||||
});
|
||||
}
|
||||
found.context("APT_RECOVERY_CLOSED_PROOF_REQUIRED")
|
||||
}
|
||||
fn digest(s: &str) -> bool {
|
||||
s.len() == 64
|
||||
&& s.bytes()
|
||||
@@ -176,6 +290,9 @@ fn eligible(binding: &Binding, row: LedgerRow) -> Result<Option<Candidate>> {
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if apt_manifest.schema_version != 3 || apt_manifest.native_retry.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
ensure!(
|
||||
recipe_id == apt::INSTALL_RECIPE
|
||||
&& recipe_version == "1"
|
||||
@@ -847,4 +964,38 @@ mod tests {
|
||||
classified
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn new_recovery_requires_exact_closed_partial_signed_history_and_immutable_proof() {
|
||||
let (binding, row, files, now) = fixture();
|
||||
let c = eligible(&binding, row.clone()).unwrap().unwrap();
|
||||
let proof = output(&c, &files, now).proof;
|
||||
let mut db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,agent TEXT,generation INTEGER,job TEXT,attempt TEXT,result BLOB);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,start BLOB,outcome BLOB);").unwrap();
|
||||
db.execute("INSERT INTO consumed(rowid,grant_id,agent,generation,job,attempt,result) VALUES(?1,?2,?3,?4,?5,?6,NULL)",params![row.rowid,row.grant,row.agent,row.generation,row.job,row.attempt]).unwrap();
|
||||
db.execute(
|
||||
"INSERT INTO execution_evidence VALUES(?1,?2,?3)",
|
||||
params![row.grant, row.start, row.outcome],
|
||||
)
|
||||
.unwrap();
|
||||
persist(&mut db, &c, &proof).unwrap();
|
||||
assert!(closed_partial_from(&db, &binding, None, || Ok(files.clone())).is_err());
|
||||
db.execute(
|
||||
"UPDATE consumed SET result=(SELECT outcome FROM execution_evidence)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let closed = closed_partial_from(&db, &binding, None, || Ok(files.clone())).unwrap();
|
||||
assert_eq!(closed.reference.previous_grant_id, c.probe.grant_id);
|
||||
assert_eq!(closed.reference.previous_attempt_id, c.probe.attempt_id);
|
||||
assert_eq!(closed.baseline_sha256, proof.observed_snapshot_sha256);
|
||||
let mut changed = closed.reference.clone();
|
||||
changed.previous_attempt_id = Uuid::new_v4();
|
||||
assert!(closed_partial_from(&db, &binding, Some(&changed), || Ok(files.clone())).is_err());
|
||||
let mut changed_files = files.clone();
|
||||
changed_files.insert("signed-action.json".into(), b"{}".to_vec());
|
||||
assert!(closed_partial_from(&db, &binding, None, || Ok(changed_files.clone())).is_err());
|
||||
db.execute("UPDATE apt_partial_reconciliations SET proof=x'7b7d'", [])
|
||||
.unwrap();
|
||||
assert!(closed_partial_from(&db, &binding, None, || Ok(files.clone())).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user