feat: preserve failed APT history for one signed UI-approved recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 15:50:34 +01:00
parent a5c2dce4fb
commit fd7225600b
18 changed files with 1345 additions and 34 deletions
+248 -1
View File
@@ -131,6 +131,8 @@ pub struct AptExecutionManifest {
pub apt_config_sha256: String,
pub runtime_sha256: String,
pub test_result_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub native_retry: Option<rmm_domain::apt_transaction::AptNativeRetry>,
}
#[derive(Deserialize)]
@@ -180,6 +182,7 @@ pub fn policy_header(job: &JobPayload, config: &ProviderConfig) -> Result<()> {
/// Opaque verified capability: callers cannot manufacture one from a manifest.
pub struct PreparedApt {
binding: Binding,
job: JobPayload,
permit: ExecutionPermitPayload,
request: Request,
@@ -197,6 +200,7 @@ pub fn prepare(
verify_installation()?;
verify_prefetched(&manifest, &permit)?;
Ok(PreparedApt {
binding: binding.clone(),
job,
permit,
request: request.clone(),
@@ -295,7 +299,13 @@ fn validate_manifest(
config.validate()?;
ensure!(
m.kind == "rmm.apt.helper-action"
&& m.schema_version == 2
&& match (&m.native_retry, m.schema_version) {
(None, 2) => true,
(Some(retry), 3) => {
action == "apt_native_acceptance" && retry.validate()
}
_ => false,
}
&& m.execution_profile == config.execution_profile
&& !m.test_result_id.is_empty()
&& m.test_result_id.len() <= 128
@@ -549,7 +559,243 @@ fn verify_prefetched(m: &AptExecutionManifest, permit: &ExecutionPermitPayload)
Ok(())
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct BootstrapAttempt {
transaction_sha256: String,
job_payload_sha256: String,
grant_id: uuid::Uuid,
attempt_id: uuid::Uuid,
}
struct RetryEvidence {
marker: Vec<u8>,
context: Vec<u8>,
signed_action: Vec<u8>,
expectation: Vec<u8>,
baseline: Vec<u8>,
launch: Vec<u8>,
sandbox_result: Vec<u8>,
}
struct RetryLedgerRecord {
agent: String,
generation: u64,
job: String,
attempt: String,
result: Option<Vec<u8>>,
start: Vec<u8>,
outcome: Option<Vec<u8>>,
}
fn absent_nofollow(path: &Path) -> Result<bool> {
match fs::symlink_metadata(path) {
Ok(_) => Ok(false),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true),
Err(error) => Err(error.into()),
}
}
impl PreparedApt {
/// Read-only prerequisite under the caller's device ExecutorGuard. A new
/// signed retry does not clear an unknown effect or change an old record.
pub fn verify_retry_history(&self, db: &rusqlite::Connection) -> Result<()> {
let Some(retry) = &self.manifest.native_retry else {
return Ok(());
};
let recipe = &self.manifest.resolved_apt_transaction.recipe_id;
self.manifest.resolved_apt_transaction.validate_fixture()?;
let base = Path::new(BASE);
for name in [
format!("bootstrap-retry-{recipe}.json"),
format!("acceptance-{recipe}.json"),
format!("pending-acceptance-{recipe}.json"),
] {
ensure!(absent_nofollow(&base.join(name))?, "APT_RETRY_ALREADY_USED");
}
let active = base.join("active");
let history = base
.join("history")
.join(retry.previous_grant_id.to_string());
let active_is_prior = if absent_nofollow(&active)? {
false
} else {
secure(&active, 0, true, true)?;
let context: serde_json::Value =
serde_json::from_slice(&read_sealed(&active.join("context.json"), 8192)?)?;
context.get("grant_id") == Some(&serde_json::json!(retry.previous_grant_id))
};
ensure!(
!(active_is_prior && !absent_nofollow(&history)?),
"APT_RETRY_AMBIGUOUS_HISTORY"
);
let prior = if active_is_prior { active } else { history };
secure(&prior, 0, true, true)?;
ensure!(
absent_nofollow(&prior.join("hook-armed.json"))?,
"APT_RETRY_PRIOR_HOOK_ARMED"
);
let evidence = RetryEvidence {
marker: read_sealed(&base.join(format!("bootstrap-attempt-{recipe}.json")), 8192)?,
context: read_sealed(&prior.join("context.json"), 8192)?,
signed_action: read_sealed(&prior.join("signed-action.json"), 65536)?,
expectation: read_sealed(&prior.join("expectation.json"), 65536)?,
baseline: read_sealed(&prior.join("baseline.json"), 1024 * 1024)?,
launch: read_sealed(&prior.join("launch.json"), 8192)?,
sandbox_result: read_sealed(&prior.join("sandbox-result.json"), 8192)?,
};
self.verify_retry_evidence(db, &evidence)
}
fn verify_retry_evidence(
&self,
db: &rusqlite::Connection,
evidence: &RetryEvidence,
) -> Result<()> {
let retry = self
.manifest
.native_retry
.as_ref()
.context("APT_RETRY_REQUIRED")?;
ensure!(
self.manifest.schema_version == 3
&& self.manifest.action == "apt-native-acceptance"
&& self.manifest.execution_profile == "dev-test"
&& retry.validate()
&& retry.previous_grant_id != self.permit.grant_id
&& retry.previous_attempt_id != self.job.attempt_id
&& hash(&evidence.marker) == retry.bootstrap_marker_sha256,
"APT_RETRY_BINDING"
);
let marker: BootstrapAttempt = serde_json::from_slice(&evidence.marker)?;
ensure!(
marker.grant_id == retry.previous_grant_id
&& marker.attempt_id == retry.previous_attempt_id
&& marker.job_payload_sha256 == hash(&evidence.signed_action)
&& marker.transaction_sha256 == hash(&evidence.expectation)
&& marker.transaction_sha256 == self.manifest.transaction_sha256
&& evidence.expectation == self.manifest.transaction_bytes,
"APT_RETRY_ORIGINAL_MARKER"
);
let prior: RetryLedgerRecord = db.query_row(
"SELECT c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e ON e.grant_id=c.grant_id WHERE c.grant_id=?1",
[retry.previous_grant_id.to_string()],
|row| Ok(RetryLedgerRecord {agent: row.get(0)?, generation: row.get(1)?,
job: row.get(2)?, attempt: row.get(3)?, result: row.get(4)?,
start: row.get(5)?, outcome: row.get(6)?}),
)?;
let result_bytes = prior.result.context("APT_RETRY_PREVIOUS_UNRESOLVED")?;
let result: Outcome = serde_json::from_slice(&result_bytes)?;
let recorded_outcome: serde_json::Value =
serde_json::from_slice(&prior.outcome.context("APT_RETRY_OUTCOME_REQUIRED")?)?;
ensure!(
prior.agent == self.job.agent_id.to_string()
&& prior.generation == self.job.enrollment_generation
&& prior.attempt == retry.previous_attempt_id.to_string()
&& prior.job != self.job.job_id.to_string()
&& result.state == "failed"
&& result.error_code == "HOOK_NOT_HANDED_OFF"
&& !result.reboot_required
&& result.evidence.get("test_acceptance") == Some(&serde_json::json!(true))
&& result.evidence.get("noProductionEvidence") == Some(&serde_json::json!(true))
&& recorded_outcome == serde_json::from_slice::<serde_json::Value>(&result_bytes)?,
"APT_RETRY_PREVIOUS_NOT_KNOWN_FAILED"
);
let start: serde_json::Value = serde_json::from_slice(&prior.start)?;
let request: Request = serde_json::from_value(
start
.get("request")
.context("APT_RETRY_START_REQUEST")?
.clone(),
)?;
let prior_permit: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?;
// Historical verification validates the original signature/bindings at
// its original not-before time; it grants no present execution right.
let (prior_job, prior_permit) = self.binding.verify(&request, prior_permit.not_before)?;
ensure!(
prior_permit.grant_id == retry.previous_grant_id
&& prior_permit.start_before <= self.permit.not_before
&& prior_permit.authorization_id != self.permit.authorization_id
&& prior_permit.plan_hash != self.permit.plan_hash
&& prior_job.job_id.to_string() == prior.job
&& prior_job.attempt_id == retry.previous_attempt_id
&& prior_job.execution_identity == "root"
&& request.job_bytes == evidence.signed_action,
"APT_RETRY_PRIOR_SIGNED_REQUEST"
);
let rmm_domain::jobs::Action::AptNativeAcceptance {
recipe_id,
recipe_version,
apt_manifest,
} = &prior_job.action
else {
bail!("APT_RETRY_PRIOR_NATIVE_ACTION_REQUIRED");
};
ensure!(
apt_manifest.schema_version == 2
&& apt_manifest.native_retry.is_none()
&& recipe_id == &self.manifest.resolved_apt_transaction.recipe_id
&& recipe_version == &self.manifest.resolved_apt_transaction.recipe_version
&& apt_manifest.transaction_bytes == evidence.expectation
&& apt_manifest.transaction_sha256 == marker.transaction_sha256
&& hash(&evidence.baseline)
== self.manifest.resolved_apt_transaction.baseline_sha256,
"APT_RETRY_PRIOR_RECIPE_OR_BASELINE"
);
let context: serde_json::Value = serde_json::from_slice(&evidence.context)?;
for (name, expected) in [
("schema_version", serde_json::json!(2)),
(
"transaction_sha256",
serde_json::json!(marker.transaction_sha256),
),
(
"job_payload_sha256",
serde_json::json!(marker.job_payload_sha256),
),
("grant_id", serde_json::json!(prior_permit.grant_id)),
("job_id", serde_json::json!(prior_job.job_id)),
("attempt_id", serde_json::json!(prior_job.attempt_id)),
(
"authorization_id",
serde_json::json!(prior_permit.authorization_id),
),
("plan_sha256", serde_json::json!(prior_permit.plan_hash)),
("action", serde_json::json!("apt-native-acceptance")),
("execution_profile", serde_json::json!("dev-test")),
(
"not_before_ms",
serde_json::json!(prior_permit.not_before.timestamp_millis()),
),
(
"start_before_ms",
serde_json::json!(prior_permit.start_before.timestamp_millis()),
),
] {
ensure!(
context.get(name) == Some(&expected),
"APT_RETRY_PRIOR_CONTEXT"
);
}
let ledger_id = context
.get("effect_armed_ledger_id")
.context("APT_RETRY_PRIOR_LEDGER")?;
ensure!(
result.evidence.get("effect_armed_ledger_id") == Some(ledger_id),
"APT_RETRY_PRIOR_LEDGER"
);
ensure!(
serde_json::from_slice::<serde_json::Value>(&evidence.launch)?
== serde_json::json!({"transaction_sha256":marker.transaction_sha256,
"effect_armed_ledger_id":ledger_id})
&& serde_json::from_slice::<serde_json::Value>(&evidence.sandbox_result)?
== serde_json::json!({"transaction_sha256":marker.transaction_sha256,
"apt_exit_code":100,"reboot_required":false}),
"APT_RETRY_PRIOR_NO_EFFECT_PROOF"
);
Ok(())
}
fn verify_parent_consumption(&self, db: &rusqlite::Connection) -> Result<String> {
let row: (i64, String, u64, String, String, Option<Vec<u8>>) = db.query_row(
"SELECT rowid,agent,generation,job,attempt,result FROM consumed WHERE grant_id=?1",
@@ -596,6 +842,7 @@ impl PreparedApt {
pub fn execute(self, db: &rusqlite::Connection) -> Result<Outcome> {
ensure_root()?;
let ledger_id = self.verify_parent_consumption(db)?;
self.verify_retry_history(db)?;
let active = self.stage(db, &ledger_id)?;
// No timeout killing the native package manager. Client timeout means
// unknown; this helper continues and journals the eventual real outcome.