feat: preserve failed APT history for one signed UI-approved recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -131,6 +131,8 @@ pub struct AptExecutionManifest {
|
||||
pub apt_config_sha256: String,
|
||||
pub runtime_sha256: String,
|
||||
pub test_result_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub native_retry: Option<rmm_domain::apt_transaction::AptNativeRetry>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -180,6 +182,7 @@ pub fn policy_header(job: &JobPayload, config: &ProviderConfig) -> Result<()> {
|
||||
|
||||
/// Opaque verified capability: callers cannot manufacture one from a manifest.
|
||||
pub struct PreparedApt {
|
||||
binding: Binding,
|
||||
job: JobPayload,
|
||||
permit: ExecutionPermitPayload,
|
||||
request: Request,
|
||||
@@ -197,6 +200,7 @@ pub fn prepare(
|
||||
verify_installation()?;
|
||||
verify_prefetched(&manifest, &permit)?;
|
||||
Ok(PreparedApt {
|
||||
binding: binding.clone(),
|
||||
job,
|
||||
permit,
|
||||
request: request.clone(),
|
||||
@@ -295,7 +299,13 @@ fn validate_manifest(
|
||||
config.validate()?;
|
||||
ensure!(
|
||||
m.kind == "rmm.apt.helper-action"
|
||||
&& m.schema_version == 2
|
||||
&& match (&m.native_retry, m.schema_version) {
|
||||
(None, 2) => true,
|
||||
(Some(retry), 3) => {
|
||||
action == "apt_native_acceptance" && retry.validate()
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
&& m.execution_profile == config.execution_profile
|
||||
&& !m.test_result_id.is_empty()
|
||||
&& m.test_result_id.len() <= 128
|
||||
@@ -549,7 +559,243 @@ fn verify_prefetched(m: &AptExecutionManifest, permit: &ExecutionPermitPayload)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct BootstrapAttempt {
|
||||
transaction_sha256: String,
|
||||
job_payload_sha256: String,
|
||||
grant_id: uuid::Uuid,
|
||||
attempt_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
struct RetryEvidence {
|
||||
marker: Vec<u8>,
|
||||
context: Vec<u8>,
|
||||
signed_action: Vec<u8>,
|
||||
expectation: Vec<u8>,
|
||||
baseline: Vec<u8>,
|
||||
launch: Vec<u8>,
|
||||
sandbox_result: Vec<u8>,
|
||||
}
|
||||
|
||||
struct RetryLedgerRecord {
|
||||
agent: String,
|
||||
generation: u64,
|
||||
job: String,
|
||||
attempt: String,
|
||||
result: Option<Vec<u8>>,
|
||||
start: Vec<u8>,
|
||||
outcome: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
fn absent_nofollow(path: &Path) -> Result<bool> {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(_) => Ok(false),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
|
||||
impl PreparedApt {
|
||||
/// Read-only prerequisite under the caller's device ExecutorGuard. A new
|
||||
/// signed retry does not clear an unknown effect or change an old record.
|
||||
pub fn verify_retry_history(&self, db: &rusqlite::Connection) -> Result<()> {
|
||||
let Some(retry) = &self.manifest.native_retry else {
|
||||
return Ok(());
|
||||
};
|
||||
let recipe = &self.manifest.resolved_apt_transaction.recipe_id;
|
||||
self.manifest.resolved_apt_transaction.validate_fixture()?;
|
||||
let base = Path::new(BASE);
|
||||
for name in [
|
||||
format!("bootstrap-retry-{recipe}.json"),
|
||||
format!("acceptance-{recipe}.json"),
|
||||
format!("pending-acceptance-{recipe}.json"),
|
||||
] {
|
||||
ensure!(absent_nofollow(&base.join(name))?, "APT_RETRY_ALREADY_USED");
|
||||
}
|
||||
let active = base.join("active");
|
||||
let history = base
|
||||
.join("history")
|
||||
.join(retry.previous_grant_id.to_string());
|
||||
let active_is_prior = if absent_nofollow(&active)? {
|
||||
false
|
||||
} else {
|
||||
secure(&active, 0, true, true)?;
|
||||
let context: serde_json::Value =
|
||||
serde_json::from_slice(&read_sealed(&active.join("context.json"), 8192)?)?;
|
||||
context.get("grant_id") == Some(&serde_json::json!(retry.previous_grant_id))
|
||||
};
|
||||
ensure!(
|
||||
!(active_is_prior && !absent_nofollow(&history)?),
|
||||
"APT_RETRY_AMBIGUOUS_HISTORY"
|
||||
);
|
||||
let prior = if active_is_prior { active } else { history };
|
||||
secure(&prior, 0, true, true)?;
|
||||
ensure!(
|
||||
absent_nofollow(&prior.join("hook-armed.json"))?,
|
||||
"APT_RETRY_PRIOR_HOOK_ARMED"
|
||||
);
|
||||
let evidence = RetryEvidence {
|
||||
marker: read_sealed(&base.join(format!("bootstrap-attempt-{recipe}.json")), 8192)?,
|
||||
context: read_sealed(&prior.join("context.json"), 8192)?,
|
||||
signed_action: read_sealed(&prior.join("signed-action.json"), 65536)?,
|
||||
expectation: read_sealed(&prior.join("expectation.json"), 65536)?,
|
||||
baseline: read_sealed(&prior.join("baseline.json"), 1024 * 1024)?,
|
||||
launch: read_sealed(&prior.join("launch.json"), 8192)?,
|
||||
sandbox_result: read_sealed(&prior.join("sandbox-result.json"), 8192)?,
|
||||
};
|
||||
self.verify_retry_evidence(db, &evidence)
|
||||
}
|
||||
|
||||
fn verify_retry_evidence(
|
||||
&self,
|
||||
db: &rusqlite::Connection,
|
||||
evidence: &RetryEvidence,
|
||||
) -> Result<()> {
|
||||
let retry = self
|
||||
.manifest
|
||||
.native_retry
|
||||
.as_ref()
|
||||
.context("APT_RETRY_REQUIRED")?;
|
||||
ensure!(
|
||||
self.manifest.schema_version == 3
|
||||
&& self.manifest.action == "apt-native-acceptance"
|
||||
&& self.manifest.execution_profile == "dev-test"
|
||||
&& retry.validate()
|
||||
&& retry.previous_grant_id != self.permit.grant_id
|
||||
&& retry.previous_attempt_id != self.job.attempt_id
|
||||
&& hash(&evidence.marker) == retry.bootstrap_marker_sha256,
|
||||
"APT_RETRY_BINDING"
|
||||
);
|
||||
let marker: BootstrapAttempt = serde_json::from_slice(&evidence.marker)?;
|
||||
ensure!(
|
||||
marker.grant_id == retry.previous_grant_id
|
||||
&& marker.attempt_id == retry.previous_attempt_id
|
||||
&& marker.job_payload_sha256 == hash(&evidence.signed_action)
|
||||
&& marker.transaction_sha256 == hash(&evidence.expectation)
|
||||
&& marker.transaction_sha256 == self.manifest.transaction_sha256
|
||||
&& evidence.expectation == self.manifest.transaction_bytes,
|
||||
"APT_RETRY_ORIGINAL_MARKER"
|
||||
);
|
||||
let prior: RetryLedgerRecord = db.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e ON e.grant_id=c.grant_id WHERE c.grant_id=?1",
|
||||
[retry.previous_grant_id.to_string()],
|
||||
|row| Ok(RetryLedgerRecord {agent: row.get(0)?, generation: row.get(1)?,
|
||||
job: row.get(2)?, attempt: row.get(3)?, result: row.get(4)?,
|
||||
start: row.get(5)?, outcome: row.get(6)?}),
|
||||
)?;
|
||||
let result_bytes = prior.result.context("APT_RETRY_PREVIOUS_UNRESOLVED")?;
|
||||
let result: Outcome = serde_json::from_slice(&result_bytes)?;
|
||||
let recorded_outcome: serde_json::Value =
|
||||
serde_json::from_slice(&prior.outcome.context("APT_RETRY_OUTCOME_REQUIRED")?)?;
|
||||
ensure!(
|
||||
prior.agent == self.job.agent_id.to_string()
|
||||
&& prior.generation == self.job.enrollment_generation
|
||||
&& prior.attempt == retry.previous_attempt_id.to_string()
|
||||
&& prior.job != self.job.job_id.to_string()
|
||||
&& result.state == "failed"
|
||||
&& result.error_code == "HOOK_NOT_HANDED_OFF"
|
||||
&& !result.reboot_required
|
||||
&& result.evidence.get("test_acceptance") == Some(&serde_json::json!(true))
|
||||
&& result.evidence.get("noProductionEvidence") == Some(&serde_json::json!(true))
|
||||
&& recorded_outcome == serde_json::from_slice::<serde_json::Value>(&result_bytes)?,
|
||||
"APT_RETRY_PREVIOUS_NOT_KNOWN_FAILED"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&prior.start)?;
|
||||
let request: Request = serde_json::from_value(
|
||||
start
|
||||
.get("request")
|
||||
.context("APT_RETRY_START_REQUEST")?
|
||||
.clone(),
|
||||
)?;
|
||||
let prior_permit: ExecutionPermitPayload = serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Historical verification validates the original signature/bindings at
|
||||
// its original not-before time; it grants no present execution right.
|
||||
let (prior_job, prior_permit) = self.binding.verify(&request, prior_permit.not_before)?;
|
||||
ensure!(
|
||||
prior_permit.grant_id == retry.previous_grant_id
|
||||
&& prior_permit.start_before <= self.permit.not_before
|
||||
&& prior_permit.authorization_id != self.permit.authorization_id
|
||||
&& prior_permit.plan_hash != self.permit.plan_hash
|
||||
&& prior_job.job_id.to_string() == prior.job
|
||||
&& prior_job.attempt_id == retry.previous_attempt_id
|
||||
&& prior_job.execution_identity == "root"
|
||||
&& request.job_bytes == evidence.signed_action,
|
||||
"APT_RETRY_PRIOR_SIGNED_REQUEST"
|
||||
);
|
||||
let rmm_domain::jobs::Action::AptNativeAcceptance {
|
||||
recipe_id,
|
||||
recipe_version,
|
||||
apt_manifest,
|
||||
} = &prior_job.action
|
||||
else {
|
||||
bail!("APT_RETRY_PRIOR_NATIVE_ACTION_REQUIRED");
|
||||
};
|
||||
ensure!(
|
||||
apt_manifest.schema_version == 2
|
||||
&& apt_manifest.native_retry.is_none()
|
||||
&& recipe_id == &self.manifest.resolved_apt_transaction.recipe_id
|
||||
&& recipe_version == &self.manifest.resolved_apt_transaction.recipe_version
|
||||
&& apt_manifest.transaction_bytes == evidence.expectation
|
||||
&& apt_manifest.transaction_sha256 == marker.transaction_sha256
|
||||
&& hash(&evidence.baseline)
|
||||
== self.manifest.resolved_apt_transaction.baseline_sha256,
|
||||
"APT_RETRY_PRIOR_RECIPE_OR_BASELINE"
|
||||
);
|
||||
let context: serde_json::Value = serde_json::from_slice(&evidence.context)?;
|
||||
for (name, expected) in [
|
||||
("schema_version", serde_json::json!(2)),
|
||||
(
|
||||
"transaction_sha256",
|
||||
serde_json::json!(marker.transaction_sha256),
|
||||
),
|
||||
(
|
||||
"job_payload_sha256",
|
||||
serde_json::json!(marker.job_payload_sha256),
|
||||
),
|
||||
("grant_id", serde_json::json!(prior_permit.grant_id)),
|
||||
("job_id", serde_json::json!(prior_job.job_id)),
|
||||
("attempt_id", serde_json::json!(prior_job.attempt_id)),
|
||||
(
|
||||
"authorization_id",
|
||||
serde_json::json!(prior_permit.authorization_id),
|
||||
),
|
||||
("plan_sha256", serde_json::json!(prior_permit.plan_hash)),
|
||||
("action", serde_json::json!("apt-native-acceptance")),
|
||||
("execution_profile", serde_json::json!("dev-test")),
|
||||
(
|
||||
"not_before_ms",
|
||||
serde_json::json!(prior_permit.not_before.timestamp_millis()),
|
||||
),
|
||||
(
|
||||
"start_before_ms",
|
||||
serde_json::json!(prior_permit.start_before.timestamp_millis()),
|
||||
),
|
||||
] {
|
||||
ensure!(
|
||||
context.get(name) == Some(&expected),
|
||||
"APT_RETRY_PRIOR_CONTEXT"
|
||||
);
|
||||
}
|
||||
let ledger_id = context
|
||||
.get("effect_armed_ledger_id")
|
||||
.context("APT_RETRY_PRIOR_LEDGER")?;
|
||||
ensure!(
|
||||
result.evidence.get("effect_armed_ledger_id") == Some(ledger_id),
|
||||
"APT_RETRY_PRIOR_LEDGER"
|
||||
);
|
||||
ensure!(
|
||||
serde_json::from_slice::<serde_json::Value>(&evidence.launch)?
|
||||
== serde_json::json!({"transaction_sha256":marker.transaction_sha256,
|
||||
"effect_armed_ledger_id":ledger_id})
|
||||
&& serde_json::from_slice::<serde_json::Value>(&evidence.sandbox_result)?
|
||||
== serde_json::json!({"transaction_sha256":marker.transaction_sha256,
|
||||
"apt_exit_code":100,"reboot_required":false}),
|
||||
"APT_RETRY_PRIOR_NO_EFFECT_PROOF"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn verify_parent_consumption(&self, db: &rusqlite::Connection) -> Result<String> {
|
||||
let row: (i64, String, u64, String, String, Option<Vec<u8>>) = db.query_row(
|
||||
"SELECT rowid,agent,generation,job,attempt,result FROM consumed WHERE grant_id=?1",
|
||||
@@ -596,6 +842,7 @@ impl PreparedApt {
|
||||
pub fn execute(self, db: &rusqlite::Connection) -> Result<Outcome> {
|
||||
ensure_root()?;
|
||||
let ledger_id = self.verify_parent_consumption(db)?;
|
||||
self.verify_retry_history(db)?;
|
||||
let active = self.stage(db, &ledger_id)?;
|
||||
// No timeout killing the native package manager. Client timeout means
|
||||
// unknown; this helper continues and journals the eventual real outcome.
|
||||
|
||||
Reference in New Issue
Block a user