fix: retain sync anchors and verify bound APT partial postconditions
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -0,0 +1,850 @@
|
||||
//! Recheck only the original fixture job's bound postcondition. No executor,
|
||||
//! approval, new grant, package command or consumption unlock exists here.
|
||||
use crate::{Binding, Outcome, Request, apt, reconcile::ExecutorGuard};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rmm_domain::{apt_transaction::AptExecutionManifest, jobs::*};
|
||||
use rusqlite::{Connection, TransactionBehavior, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs::{self, OpenOptions},
|
||||
io::{Read, Write},
|
||||
os::unix::fs::{MetadataExt, OpenOptionsExt},
|
||||
path::Path,
|
||||
process::{Command, Stdio},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
const BASE: &str = "/var/lib/rmm-apt-transaction";
|
||||
const CODE: &str = "APT_PARTIAL_UNCONFIGURED_VERIFIED";
|
||||
const EVIDENCE_NAMES: [&str; 11] = [
|
||||
"context.json",
|
||||
"signed-action.json",
|
||||
"expectation.json",
|
||||
"baseline.json",
|
||||
"launch.json",
|
||||
"native-prechecks.json",
|
||||
"sandbox-ready.json",
|
||||
"sandbox-result.json",
|
||||
"hook-armed.json",
|
||||
"bootstrap-attempt-rmm-apt-fixture-install-1.0.json",
|
||||
"bootstrap-retry-rmm-apt-fixture-install-1.0.json",
|
||||
];
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SavedStart {
|
||||
previous_boot_id: Uuid,
|
||||
reserved_at: DateTime<Utc>,
|
||||
reconnect_deadline: Option<DateTime<Utc>>,
|
||||
request: Request,
|
||||
reauthentication_required: bool,
|
||||
postconditions_passed: bool,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PackageState {
|
||||
version: String,
|
||||
status: String,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ProbeRequest {
|
||||
transaction_sha256: String,
|
||||
job_payload_sha256: String,
|
||||
effect_armed_ledger_id: String,
|
||||
grant_id: Uuid,
|
||||
job_id: Uuid,
|
||||
attempt_id: Uuid,
|
||||
authorization_id: Uuid,
|
||||
plan_sha256: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ContextRecord {
|
||||
transaction_sha256: String,
|
||||
job_payload_sha256: String,
|
||||
effect_armed_ledger_id: String,
|
||||
grant_id: Uuid,
|
||||
job_id: Uuid,
|
||||
attempt_id: Uuid,
|
||||
authorization_id: Uuid,
|
||||
plan_sha256: String,
|
||||
schema_version: u32,
|
||||
action: String,
|
||||
execution_profile: String,
|
||||
not_before_ms: i64,
|
||||
start_before_ms: i64,
|
||||
boot_id: Uuid,
|
||||
monotonic_start_before_ns: u64,
|
||||
}
|
||||
#[derive(Clone, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PartialProof {
|
||||
schema_version: u32,
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
transaction_sha256: String,
|
||||
job_payload_sha256: String,
|
||||
effect_armed_ledger_id: String,
|
||||
grant_id: Uuid,
|
||||
job_id: Uuid,
|
||||
attempt_id: Uuid,
|
||||
authorization_id: Uuid,
|
||||
plan_sha256: String,
|
||||
original_evidence_sha256: String,
|
||||
original_baseline_sha256: String,
|
||||
observed_snapshot_sha256: String,
|
||||
status_fingerprint: String,
|
||||
fixture_payload_sha256: String,
|
||||
snapshot: BTreeMap<String, PackageState>,
|
||||
checked_at_ms: i64,
|
||||
partial_effect: bool,
|
||||
postconditions_passed: bool,
|
||||
reboot_required: bool,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ProbeOutput {
|
||||
effect_state: String,
|
||||
code: String,
|
||||
proof: PartialProof,
|
||||
}
|
||||
|
||||
struct Candidate {
|
||||
request: Request,
|
||||
permit: ExecutionPermitPayload,
|
||||
manifest: AptExecutionManifest,
|
||||
probe: ProbeRequest,
|
||||
original_boot_id: Uuid,
|
||||
original_outcome: Vec<u8>,
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct LedgerRow {
|
||||
rowid: i64,
|
||||
grant: String,
|
||||
agent: String,
|
||||
generation: u64,
|
||||
job: String,
|
||||
attempt: String,
|
||||
start: Vec<u8>,
|
||||
outcome: Vec<u8>,
|
||||
}
|
||||
fn digest(s: &str) -> bool {
|
||||
s.len() == 64
|
||||
&& s.bytes()
|
||||
.all(|b| b.is_ascii_digit() || b"abcdef".contains(&b))
|
||||
}
|
||||
fn eligible(binding: &Binding, row: LedgerRow) -> Result<Option<Candidate>> {
|
||||
ensure!(
|
||||
row.start.len() <= 512 * 1024 && row.outcome.len() <= 512 * 1024,
|
||||
"APT_RECONCILE_RECORD_LIMIT"
|
||||
);
|
||||
let old: Outcome = serde_json::from_slice(&row.outcome)?;
|
||||
if old.state != "unknown" || old.error_code != "DPKG_INCONSISTENT" {
|
||||
return Ok(None);
|
||||
}
|
||||
let start: SavedStart = serde_json::from_slice(&row.start)?;
|
||||
ensure!(
|
||||
!start.previous_boot_id.is_nil()
|
||||
&& start.reauthentication_required
|
||||
&& !start.postconditions_passed
|
||||
&& start.reconnect_deadline.is_none(),
|
||||
"APT_RECONCILE_ORIGINAL_START"
|
||||
);
|
||||
let permit: ExecutionPermitPayload = serde_json::from_slice(&start.request.permit_bytes)?;
|
||||
// The old start window is verified historically. It never authorizes a
|
||||
// present process start, and the consumed tombstone is never replaced.
|
||||
let (job, permit) = binding.verify(&start.request, permit.not_before)?;
|
||||
ensure!(
|
||||
row.agent == binding.agent_id.to_string()
|
||||
&& row.generation == binding.enrollment_generation
|
||||
&& row.grant == permit.grant_id.to_string()
|
||||
&& row.job == job.job_id.to_string()
|
||||
&& row.attempt == job.attempt_id.to_string()
|
||||
&& job.execution_identity == "root"
|
||||
&& start.reserved_at <= permit.start_before,
|
||||
"APT_RECONCILE_ORIGINAL_BINDING"
|
||||
);
|
||||
let Action::AptNativeAcceptance {
|
||||
recipe_id,
|
||||
recipe_version,
|
||||
apt_manifest,
|
||||
} = &job.action
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
ensure!(
|
||||
recipe_id == apt::INSTALL_RECIPE
|
||||
&& recipe_version == "1"
|
||||
&& apt_manifest.validate()
|
||||
&& apt_manifest.schema_version == 3
|
||||
&& apt_manifest.native_retry.is_some()
|
||||
&& apt_manifest.action == "apt-native-acceptance"
|
||||
&& apt_manifest.execution_profile == "dev-test",
|
||||
"APT_RECONCILE_FIXED_NATIVE_FIXTURE_ONLY"
|
||||
);
|
||||
let ledger = format!("apt-{}-{}", permit.grant_id, row.rowid);
|
||||
ensure!(
|
||||
old.exit_code == Some(0)
|
||||
&& old.stdout.is_empty()
|
||||
&& old.stderr.is_empty()
|
||||
&& !old.output_truncated
|
||||
&& !old.reboot_required
|
||||
&& old.evidence["postconditions_passed"] == false
|
||||
&& old.evidence["test_acceptance"] == true
|
||||
&& old.evidence["noProductionEvidence"] == true
|
||||
&& old.evidence["effect_armed_ledger_id"] == ledger,
|
||||
"APT_RECONCILE_EXPECTED_UNKNOWN_ONLY"
|
||||
);
|
||||
let probe = ProbeRequest {
|
||||
transaction_sha256: apt_manifest.transaction_sha256.clone(),
|
||||
job_payload_sha256: hash(&start.request.job_bytes),
|
||||
effect_armed_ledger_id: ledger,
|
||||
grant_id: permit.grant_id,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
authorization_id: permit.authorization_id,
|
||||
plan_sha256: permit.plan_hash.clone(),
|
||||
};
|
||||
Ok(Some(Candidate {
|
||||
request: start.request,
|
||||
permit,
|
||||
manifest: apt_manifest.clone(),
|
||||
probe,
|
||||
original_boot_id: start.previous_boot_id,
|
||||
original_outcome: row.outcome,
|
||||
}))
|
||||
}
|
||||
fn sealed(path: &Path, limit: u64) -> Result<Vec<u8>> {
|
||||
for component in path.ancestors() {
|
||||
let meta = fs::symlink_metadata(component)?;
|
||||
ensure!(
|
||||
!meta.file_type().is_symlink() && meta.uid() == 0 && meta.mode() & 0o022 == 0,
|
||||
"APT_RECONCILE_UNTRUSTED_PATH"
|
||||
);
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(path)?;
|
||||
ensure!(
|
||||
file.metadata()?.is_file() && file.metadata()?.len() <= limit,
|
||||
"APT_RECONCILE_FILE_LIMIT"
|
||||
);
|
||||
let mut bytes = vec![];
|
||||
(&mut file).take(limit + 1).read_to_end(&mut bytes)?;
|
||||
ensure!(bytes.len() as u64 <= limit, "APT_RECONCILE_FILE_LIMIT");
|
||||
Ok(bytes)
|
||||
}
|
||||
fn read_evidence() -> Result<BTreeMap<String, Vec<u8>>> {
|
||||
let base = Path::new(BASE);
|
||||
crate::private_path(base, 0, true)?;
|
||||
crate::private_path(&base.join("active"), 0, true)?;
|
||||
for name in [
|
||||
"acceptance-rmm-apt-fixture-install-1.0.json",
|
||||
"pending-acceptance-rmm-apt-fixture-install-1.0.json",
|
||||
] {
|
||||
ensure!(
|
||||
matches!(fs::symlink_metadata(base.join(name)),Err(e) if e.kind()==std::io::ErrorKind::NotFound),
|
||||
"APT_RECONCILE_ACCEPTANCE_EXISTS"
|
||||
);
|
||||
}
|
||||
let mut result = BTreeMap::new();
|
||||
for name in EVIDENCE_NAMES.into_iter().chain(["apt.conf"]) {
|
||||
let path = if name.starts_with("bootstrap-") {
|
||||
base.join(name)
|
||||
} else {
|
||||
base.join("active").join(name)
|
||||
};
|
||||
result.insert(name.to_owned(), sealed(&path, 1024 * 1024)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
fn verify_evidence(
|
||||
c: &Candidate,
|
||||
files: &BTreeMap<String, Vec<u8>>,
|
||||
) -> Result<BTreeMap<String, PackageState>> {
|
||||
let read = |name: &str| files.get(name).context("APT_RECONCILE_EVIDENCE_MISSING");
|
||||
ensure!(
|
||||
read("signed-action.json")? == &c.request.job_bytes
|
||||
&& read("expectation.json")? == &c.manifest.transaction_bytes
|
||||
&& hash(read("baseline.json")?) == c.manifest.resolved_apt_transaction.baseline_sha256
|
||||
&& hash(read("apt.conf")?) == c.manifest.apt_config_sha256,
|
||||
"APT_RECONCILE_ORIGINAL_FILE_HASH"
|
||||
);
|
||||
let context: ContextRecord = serde_json::from_slice(read("context.json")?)?;
|
||||
ensure!(
|
||||
context.schema_version == 2
|
||||
&& context.action == "apt-native-acceptance"
|
||||
&& context.execution_profile == "dev-test"
|
||||
&& context.transaction_sha256 == c.probe.transaction_sha256
|
||||
&& context.job_payload_sha256 == c.probe.job_payload_sha256
|
||||
&& context.effect_armed_ledger_id == c.probe.effect_armed_ledger_id
|
||||
&& context.grant_id == c.probe.grant_id
|
||||
&& context.job_id == c.probe.job_id
|
||||
&& context.attempt_id == c.probe.attempt_id
|
||||
&& context.authorization_id == c.probe.authorization_id
|
||||
&& context.plan_sha256 == c.probe.plan_sha256
|
||||
&& context.boot_id == c.original_boot_id
|
||||
&& context.not_before_ms == c.permit.not_before.timestamp_millis()
|
||||
&& context.start_before_ms == c.permit.start_before.timestamp_millis()
|
||||
&& context.monotonic_start_before_ns > 0,
|
||||
"APT_RECONCILE_ORIGINAL_CONTEXT"
|
||||
);
|
||||
let value = |name: &str| -> Result<Value> { Ok(serde_json::from_slice(read(name)?)?) };
|
||||
ensure!(
|
||||
value("launch.json")?
|
||||
== json!({"transaction_sha256":c.probe.transaction_sha256,"effect_armed_ledger_id":c.probe.effect_armed_ledger_id})
|
||||
&& value("sandbox-result.json")?
|
||||
== json!({"transaction_sha256":c.probe.transaction_sha256,"apt_exit_code":0,"reboot_required":false})
|
||||
&& value("sandbox-ready.json")?
|
||||
== json!({"transaction_sha256":c.probe.transaction_sha256,"sandbox_version":1})
|
||||
&& value("native-prechecks.json")? == native_prechecks(c),
|
||||
"APT_RECONCILE_NATIVE_RESULT"
|
||||
);
|
||||
let hook = value("hook-armed.json")?;
|
||||
ensure!(hook["transaction_sha256"]==c.probe.transaction_sha256 && hook["grant_id"]==c.probe.grant_id.to_string() &&
|
||||
hook["effect_armed_ledger_id"]==c.probe.effect_armed_ledger_id &&
|
||||
hook["protocol_sha256"].as_str().is_some_and(digest) && hook["armed_at_ms"].as_i64().is_some_and(|time|
|
||||
time>=context.not_before_ms && time<context.start_before_ms), "APT_RECONCILE_ARMED_HOOK");
|
||||
let retry = c
|
||||
.manifest
|
||||
.native_retry
|
||||
.as_ref()
|
||||
.context("APT_RECONCILE_RETRY_REQUIRED")?;
|
||||
ensure!(
|
||||
hash(read("bootstrap-attempt-rmm-apt-fixture-install-1.0.json")?)
|
||||
== retry.bootstrap_marker_sha256,
|
||||
"APT_RECONCILE_ORIGINAL_MARKER"
|
||||
);
|
||||
let retry_marker = value("bootstrap-retry-rmm-apt-fixture-install-1.0.json")?;
|
||||
ensure!(
|
||||
retry_marker["transaction_sha256"] == c.probe.transaction_sha256
|
||||
&& retry_marker["job_payload_sha256"] == c.probe.job_payload_sha256
|
||||
&& retry_marker["grant_id"] == c.probe.grant_id.to_string()
|
||||
&& retry_marker["attempt_id"] == c.probe.attempt_id.to_string()
|
||||
&& retry_marker["native_retry"] == serde_json::to_value(retry)?
|
||||
&& retry_marker["previous_evidence_sha256"]
|
||||
.as_str()
|
||||
.is_some_and(digest),
|
||||
"APT_RECONCILE_RETRY_MARKER"
|
||||
);
|
||||
let baseline: BTreeMap<String, PackageState> = serde_json::from_slice(read("baseline.json")?)?;
|
||||
ensure!(
|
||||
!baseline.contains_key("rmm-mvp-test:all"),
|
||||
"APT_RECONCILE_ORIGINAL_ABSENCE"
|
||||
);
|
||||
Ok(baseline)
|
||||
}
|
||||
fn native_prechecks(c: &Candidate) -> Value {
|
||||
json!({"schema_version":1,"transaction_sha256":c.probe.transaction_sha256,
|
||||
"job_payload_sha256":c.probe.job_payload_sha256,"boot_id":c.original_boot_id,
|
||||
"checks":{"namespaces_caps_nnp_seccomp":true,"readonly_controlpaths":true,
|
||||
"private_tmpfs_sockets":true,"control_syscalls":{"invalid_reboot_magic":"EPERM",
|
||||
"invalid_kexec_load_flags":"EPERM","invalid_kexec_file_fds_flags":"EPERM"},
|
||||
"frontend_lock":"native-apt-check-conflict"}})
|
||||
}
|
||||
fn evidence_hash(files: &BTreeMap<String, Vec<u8>>) -> Result<String> {
|
||||
let mut hashes = BTreeMap::new();
|
||||
for name in EVIDENCE_NAMES {
|
||||
hashes.insert(
|
||||
name,
|
||||
hash(files.get(name).context("APT_RECONCILE_EVIDENCE_MISSING")?),
|
||||
);
|
||||
}
|
||||
Ok(hash(&serde_json::to_vec(&hashes)?))
|
||||
}
|
||||
fn verify_proof(
|
||||
c: &Candidate,
|
||||
files: &BTreeMap<String, Vec<u8>>,
|
||||
output: &ProbeOutput,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<()> {
|
||||
let baseline = verify_evidence(c, files)?;
|
||||
let p = &output.proof;
|
||||
ensure!(
|
||||
output.effect_state == "known_failed"
|
||||
&& output.code == CODE
|
||||
&& p.schema_version == 1
|
||||
&& p.kind == "rmm.apt.partial-reconciliation"
|
||||
&& p.partial_effect
|
||||
&& !p.postconditions_passed
|
||||
&& !p.reboot_required,
|
||||
"APT_RECONCILE_PROOF_TYPE"
|
||||
);
|
||||
let actual = ProbeRequest {
|
||||
transaction_sha256: p.transaction_sha256.clone(),
|
||||
job_payload_sha256: p.job_payload_sha256.clone(),
|
||||
effect_armed_ledger_id: p.effect_armed_ledger_id.clone(),
|
||||
grant_id: p.grant_id,
|
||||
job_id: p.job_id,
|
||||
attempt_id: p.attempt_id,
|
||||
authorization_id: p.authorization_id,
|
||||
plan_sha256: p.plan_sha256.clone(),
|
||||
};
|
||||
let mut expected = baseline;
|
||||
expected.insert(
|
||||
"rmm-mvp-test:all".into(),
|
||||
PackageState {
|
||||
version: "1.0".into(),
|
||||
status: "install ok unpacked".into(),
|
||||
},
|
||||
);
|
||||
ensure!(
|
||||
actual == c.probe
|
||||
&& p.original_evidence_sha256 == evidence_hash(files)?
|
||||
&& p.original_baseline_sha256 == c.manifest.resolved_apt_transaction.baseline_sha256
|
||||
&& p.snapshot == expected
|
||||
&& p.observed_snapshot_sha256
|
||||
== hash(&serde_json::to_vec(&serde_json::to_value(&p.snapshot)?)?)
|
||||
&& p.fixture_payload_sha256 == hash(b"1.0\n")
|
||||
&& digest(&p.status_fingerprint)
|
||||
&& p.checked_at_ms >= now.timestamp_millis() - 1000
|
||||
&& p.checked_at_ms <= Utc::now().timestamp_millis() + 1000,
|
||||
"APT_RECONCILE_PROOF_BINDING"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn probe(c: &Candidate) -> Result<ProbeOutput> {
|
||||
apt::verify_installation()?;
|
||||
let mut child = Command::new("/usr/bin/python3")
|
||||
.args(["-I", apt::CODE, "verify-partial"])
|
||||
.env_clear()
|
||||
.env("PATH", "/usr/bin:/bin")
|
||||
.env("LC_ALL", "C")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()?;
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.context("APT_RECONCILE_STDIN")?
|
||||
.write_all(&serde_json::to_vec(&c.probe)?)?;
|
||||
let mut stdout = child.stdout.take().context("APT_RECONCILE_STDOUT")?;
|
||||
let mut bytes = vec![];
|
||||
let mut chunk = [0; 8192];
|
||||
let mut overflow = false;
|
||||
loop {
|
||||
let count = stdout.read(&mut chunk)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
if bytes.len() + count <= 1024 * 1024 {
|
||||
bytes.extend_from_slice(&chunk[..count]);
|
||||
} else {
|
||||
overflow = true;
|
||||
}
|
||||
}
|
||||
let status = child.wait()?;
|
||||
ensure!(status.success() && !overflow, "APT_RECONCILE_NOT_PROVEN");
|
||||
Ok(serde_json::from_slice(&bytes)?)
|
||||
}
|
||||
fn persist(db: &mut Connection, c: &Candidate, proof: &PartialProof) -> Result<()> {
|
||||
let proof_bytes = serde_json::to_vec(&serde_json::to_value(proof)?)?;
|
||||
let proof_sha = hash(&proof_bytes);
|
||||
let outcome = Outcome {
|
||||
state: "failed".into(),
|
||||
stdout: vec![],
|
||||
stderr: vec![],
|
||||
exit_code: Some(0),
|
||||
output_truncated: false,
|
||||
reboot_required: false,
|
||||
evidence: json!({"postconditions_passed":false,"partial_effect":true,"reconciled_postconditions":true,
|
||||
"effect_armed_ledger_id":c.probe.effect_armed_ledger_id,"test_acceptance":true,"noProductionEvidence":true,
|
||||
"apt":{"test_acceptance":true,"noProductionEvidence":true,"partial_effect":true,
|
||||
"fixture_status":"install ok unpacked","fixture_version":"1.0","fixture_payload_sha256":proof.fixture_payload_sha256,
|
||||
"snapshot_sha256":proof.observed_snapshot_sha256,"status_fingerprint":proof.status_fingerprint,
|
||||
"reconciliation_proof_sha256":proof_sha}}),
|
||||
error_code: CODE.into(),
|
||||
};
|
||||
let classified = serde_json::to_vec(&outcome)?;
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute_batch("CREATE TABLE IF NOT EXISTS apt_partial_reconciliations(grant_id TEXT PRIMARY KEY,
|
||||
original_request_sha256 TEXT NOT NULL,previous_outcome_sha256 TEXT NOT NULL,previous_outcome BLOB NOT NULL,
|
||||
proof_sha256 TEXT NOT NULL,proof BLOB NOT NULL,classified_outcome BLOB NOT NULL,checked_at_ms INTEGER NOT NULL);")?;
|
||||
let current:(Option<Vec<u8>>,Option<Vec<u8>>)=tx.query_row("SELECT c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE grant_id=?1",
|
||||
[c.probe.grant_id.to_string()],|r|Ok((r.get(0)?,r.get(1)?)))?;
|
||||
ensure!(
|
||||
current.0.is_none() && current.1.as_deref() == Some(c.original_outcome.as_slice()),
|
||||
"APT_RECONCILE_OUTCOME_CHANGED"
|
||||
);
|
||||
tx.execute("INSERT INTO apt_partial_reconciliations(grant_id,original_request_sha256,previous_outcome_sha256,previous_outcome,proof_sha256,proof,classified_outcome,checked_at_ms) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",
|
||||
params![c.probe.grant_id.to_string(),hash(&serde_json::to_vec(&c.request)?),hash(&c.original_outcome),c.original_outcome,proof_sha,proof_bytes,classified,proof.checked_at_ms])?;
|
||||
ensure!(
|
||||
tx.execute(
|
||||
"UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1 AND outcome=?3",
|
||||
params![c.probe.grant_id.to_string(), classified, c.original_outcome]
|
||||
)? == 1,
|
||||
"APT_RECONCILE_OUTCOME_CHANGED"
|
||||
);
|
||||
// consumed.result stays NULL. Only the normal independently checked signed
|
||||
// RootSync decision may close this particular consumed resource afterward.
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Invoke under the same executor lock as RootSync capture. Failure to prove a
|
||||
/// postcondition leaves the existing unknown outcome and resource fence intact.
|
||||
pub fn refresh(
|
||||
db: &mut Connection,
|
||||
binding: &Binding,
|
||||
guard: &ExecutorGuard,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<()> {
|
||||
guard.check(db)?;
|
||||
// SAFETY: geteuid has no pointers or preconditions.
|
||||
ensure!(
|
||||
unsafe { libc::geteuid() } == 0,
|
||||
"APT_RECONCILE_ROOT_REQUIRED"
|
||||
);
|
||||
let mut statement=db.prepare("SELECT c.rowid,c.grant_id,c.agent,c.generation,c.job,c.attempt,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL AND e.outcome IS NOT NULL")?;
|
||||
let rows = statement
|
||||
.query_map([], |r| {
|
||||
Ok(LedgerRow {
|
||||
rowid: r.get(0)?,
|
||||
grant: r.get(1)?,
|
||||
agent: r.get(2)?,
|
||||
generation: r.get(3)?,
|
||||
job: r.get(4)?,
|
||||
attempt: r.get(5)?,
|
||||
start: r.get(6)?,
|
||||
outcome: r.get(7)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
drop(statement);
|
||||
ensure!(rows.len() <= 256, "APT_RECONCILE_CAPACITY");
|
||||
for row in rows {
|
||||
let Some(candidate) = eligible(binding, row)? else {
|
||||
continue;
|
||||
};
|
||||
// These checks are read-only. Any missing/ambiguous evidence is not a
|
||||
// definitive failure and never releases the unknown attempt.
|
||||
let Ok(files) = read_evidence() else {
|
||||
continue;
|
||||
};
|
||||
if verify_evidence(&candidate, &files).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(output) = probe(&candidate) else {
|
||||
continue;
|
||||
};
|
||||
if verify_proof(&candidate, &files, &output, now).is_err() {
|
||||
continue;
|
||||
}
|
||||
persist(db, &candidate, &output.proof)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use openssl::{pkey::PKey, sign::Signer};
|
||||
|
||||
fn fixture() -> (Binding, LedgerRow, BTreeMap<String, Vec<u8>>, DateTime<Utc>) {
|
||||
let now = Utc::now();
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let binding = Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
|
||||
};
|
||||
let baseline = json!({"base-files:amd64":{"version":"1","status":"install ok installed"}});
|
||||
let tx = json!({"schema_version":1,"recipe_id":apt::INSTALL_RECIPE,"recipe_version":"1","platform":"ubuntu-24.04-amd64",
|
||||
"resolver_config_version":"rmm-apt-local-v1","no_reboot":true,"status_fingerprint":"a".repeat(64),
|
||||
"baseline_sha256":hash(&serde_json::to_vec(&baseline).unwrap()),"packages":[{"name":"rmm-mvp-test","architecture":"all",
|
||||
"old_version":null,"new_version":"1.0","origin":"rmm-test",
|
||||
"sha256":"74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867","size":20672,
|
||||
"archive_path":"/var/lib/rmm-apt-transaction/archives/74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867.deb"}]});
|
||||
let txbytes = serde_json::to_vec(&tx).unwrap();
|
||||
let original_grant = Uuid::new_v4();
|
||||
let original_attempt = Uuid::new_v4();
|
||||
let marker = serde_json::to_vec(
|
||||
&json!({"transaction_sha256":hash(&txbytes),"job_payload_sha256":"b".repeat(64),
|
||||
"grant_id":original_grant,"attempt_id":original_attempt}),
|
||||
)
|
||||
.unwrap();
|
||||
let native_retry = json!({"schema_version":1,"previous_grant_id":original_grant,"previous_attempt_id":original_attempt,
|
||||
"bootstrap_marker_sha256":hash(&marker)});
|
||||
let manifest = json!({"type":"rmm.apt.helper-action","schema_version":3,"action":"apt-native-acceptance",
|
||||
"execution_profile":"dev-test","transaction_sha256":hash(&txbytes),"resolved_apt_transaction":tx,
|
||||
"transaction_bytes":txbytes,"provider_sha256":"c".repeat(64),"hook_sha256":"d".repeat(64),
|
||||
"apt_config_sha256":hash(b"historical config"),"runtime_sha256":"e".repeat(64),"test_result_id":"partial-test",
|
||||
"native_retry":native_retry});
|
||||
let job:JobPayload=serde_json::from_value(json!({"schema_version":1,"tenant_id":binding.tenant_id,
|
||||
"asset_id":binding.asset_id,"agent_id":binding.agent_id,"enrollment_generation":1,"assignment_version":1,
|
||||
"job_id":Uuid::new_v4(),"attempt_id":Uuid::new_v4(),"action":{"action":"apt_native_acceptance",
|
||||
"recipe_id":apt::INSTALL_RECIPE,"recipe_version":"1","apt_manifest":manifest},"execution_identity":"root",
|
||||
"not_before":now,"start_before":now+chrono::Duration::minutes(15),"max_runtime_seconds":300,"max_output_bytes":65536})).unwrap();
|
||||
let job_bytes = encode(&job).unwrap();
|
||||
let permit = ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: Uuid::new_v4(),
|
||||
authorization_id: Uuid::new_v4(),
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
plan_hash: "f".repeat(64),
|
||||
job_payload_hash: hash(&job_bytes),
|
||||
action: "apt_native_acceptance".into(),
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(60),
|
||||
max_runtime_seconds: 300,
|
||||
key_id: hash(&key.raw_public_key().unwrap()),
|
||||
};
|
||||
let permit_bytes = encode(&permit).unwrap();
|
||||
let request = Request {
|
||||
job_bytes,
|
||||
signature: Signer::new_without_digest(&key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&permit_bytes)
|
||||
.unwrap(),
|
||||
permit_bytes,
|
||||
authorization_id: permit.authorization_id,
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
};
|
||||
let boot = Uuid::new_v4();
|
||||
let ledger = format!("apt-{}-1", permit.grant_id);
|
||||
let old = Outcome {
|
||||
exit_code: Some(0),
|
||||
evidence: json!({"postconditions_passed":false,"test_acceptance":true,"noProductionEvidence":true,
|
||||
"effect_armed_ledger_id":ledger,"apt":{"test_acceptance":true,"noProductionEvidence":true}}),
|
||||
..Outcome::unknown("DPKG_INCONSISTENT")
|
||||
};
|
||||
let row = LedgerRow {
|
||||
rowid: 1,
|
||||
grant: permit.grant_id.to_string(),
|
||||
agent: binding.agent_id.to_string(),
|
||||
generation: 1,
|
||||
job: job.job_id.to_string(),
|
||||
attempt: job.attempt_id.to_string(),
|
||||
outcome: serde_json::to_vec(&old).unwrap(),
|
||||
start: serde_json::to_vec(
|
||||
&json!({"previous_boot_id":boot,"reserved_at":now,"reconnect_deadline":null,
|
||||
"request":request,"reauthentication_required":true,"postconditions_passed":false}),
|
||||
)
|
||||
.unwrap(),
|
||||
};
|
||||
let c = eligible(&binding, row.clone()).unwrap().unwrap();
|
||||
let mut files = BTreeMap::new();
|
||||
files.insert("signed-action.json".into(), c.request.job_bytes.clone());
|
||||
files.insert(
|
||||
"expectation.json".into(),
|
||||
c.manifest.transaction_bytes.clone(),
|
||||
);
|
||||
files.insert(
|
||||
"baseline.json".into(),
|
||||
serde_json::to_vec(&baseline).unwrap(),
|
||||
);
|
||||
files.insert("apt.conf".into(), b"historical config".to_vec());
|
||||
let context = json!({"schema_version":2,"transaction_sha256":c.probe.transaction_sha256,"job_payload_sha256":c.probe.job_payload_sha256,
|
||||
"effect_armed_ledger_id":ledger,"grant_id":permit.grant_id,"job_id":job.job_id,"attempt_id":job.attempt_id,
|
||||
"authorization_id":permit.authorization_id,"plan_sha256":permit.plan_hash,"action":"apt-native-acceptance",
|
||||
"execution_profile":"dev-test","not_before_ms":permit.not_before.timestamp_millis(),"start_before_ms":permit.start_before.timestamp_millis(),
|
||||
"boot_id":boot,"monotonic_start_before_ns":1});
|
||||
let objects = [
|
||||
("context.json", context),
|
||||
(
|
||||
"launch.json",
|
||||
json!({"transaction_sha256":c.probe.transaction_sha256,"effect_armed_ledger_id":ledger}),
|
||||
),
|
||||
(
|
||||
"sandbox-result.json",
|
||||
json!({"transaction_sha256":c.probe.transaction_sha256,"apt_exit_code":0,"reboot_required":false}),
|
||||
),
|
||||
(
|
||||
"sandbox-ready.json",
|
||||
json!({"transaction_sha256":c.probe.transaction_sha256,"sandbox_version":1}),
|
||||
),
|
||||
("native-prechecks.json", native_prechecks(&c)),
|
||||
(
|
||||
"hook-armed.json",
|
||||
json!({"transaction_sha256":c.probe.transaction_sha256,"effect_armed_ledger_id":ledger,"grant_id":permit.grant_id,
|
||||
"protocol_sha256":"a".repeat(64),"armed_at_ms":now.timestamp_millis()}),
|
||||
),
|
||||
(
|
||||
"bootstrap-retry-rmm-apt-fixture-install-1.0.json",
|
||||
json!({"transaction_sha256":c.probe.transaction_sha256,
|
||||
"job_payload_sha256":c.probe.job_payload_sha256,"effect_armed_ledger_id":ledger,"grant_id":permit.grant_id,"attempt_id":job.attempt_id,
|
||||
"native_retry":native_retry,"previous_evidence_sha256":"b".repeat(64)}),
|
||||
),
|
||||
];
|
||||
for (name, value) in objects {
|
||||
files.insert(name.into(), serde_json::to_vec(&value).unwrap());
|
||||
}
|
||||
files.insert(
|
||||
"bootstrap-attempt-rmm-apt-fixture-install-1.0.json".into(),
|
||||
marker,
|
||||
);
|
||||
(binding, row, files, now)
|
||||
}
|
||||
fn output(c: &Candidate, files: &BTreeMap<String, Vec<u8>>, now: DateTime<Utc>) -> ProbeOutput {
|
||||
let mut snapshot = verify_evidence(c, files).unwrap();
|
||||
snapshot.insert(
|
||||
"rmm-mvp-test:all".into(),
|
||||
PackageState {
|
||||
version: "1.0".into(),
|
||||
status: "install ok unpacked".into(),
|
||||
},
|
||||
);
|
||||
ProbeOutput {
|
||||
effect_state: "known_failed".into(),
|
||||
code: CODE.into(),
|
||||
proof: PartialProof {
|
||||
schema_version: 1,
|
||||
kind: "rmm.apt.partial-reconciliation".into(),
|
||||
transaction_sha256: c.probe.transaction_sha256.clone(),
|
||||
job_payload_sha256: c.probe.job_payload_sha256.clone(),
|
||||
effect_armed_ledger_id: c.probe.effect_armed_ledger_id.clone(),
|
||||
grant_id: c.probe.grant_id,
|
||||
job_id: c.probe.job_id,
|
||||
attempt_id: c.probe.attempt_id,
|
||||
authorization_id: c.probe.authorization_id,
|
||||
plan_sha256: c.probe.plan_sha256.clone(),
|
||||
original_evidence_sha256: evidence_hash(files).unwrap(),
|
||||
original_baseline_sha256: c
|
||||
.manifest
|
||||
.resolved_apt_transaction
|
||||
.baseline_sha256
|
||||
.clone(),
|
||||
observed_snapshot_sha256: hash(
|
||||
&serde_json::to_vec(&serde_json::to_value(&snapshot).unwrap()).unwrap(),
|
||||
),
|
||||
snapshot,
|
||||
status_fingerprint: "a".repeat(64),
|
||||
fixture_payload_sha256: hash(b"1.0\n"),
|
||||
checked_at_ms: now.timestamp_millis(),
|
||||
partial_effect: true,
|
||||
postconditions_passed: false,
|
||||
reboot_required: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn original_signature_identity_marker_and_complete_evidence_are_required() {
|
||||
let (binding, row, files, now) = fixture();
|
||||
let c = eligible(&binding, row.clone()).unwrap().unwrap();
|
||||
verify_proof(&c, &files, &output(&c, &files, now), now).unwrap();
|
||||
let mut wrong = row.clone();
|
||||
wrong.agent = Uuid::new_v4().to_string();
|
||||
assert!(eligible(&binding, wrong).is_err());
|
||||
let mut wrong = row;
|
||||
let mut start: Value = serde_json::from_slice(&wrong.start).unwrap();
|
||||
start["request"]["signature"][0] =
|
||||
json!((start["request"]["signature"][0].as_u64().unwrap() + 1) % 256);
|
||||
wrong.start = serde_json::to_vec(&start).unwrap();
|
||||
assert!(eligible(&binding, wrong).is_err());
|
||||
for name in [
|
||||
"context.json",
|
||||
"signed-action.json",
|
||||
"expectation.json",
|
||||
"baseline.json",
|
||||
"launch.json",
|
||||
"sandbox-result.json",
|
||||
"native-prechecks.json",
|
||||
"hook-armed.json",
|
||||
"bootstrap-attempt-rmm-apt-fixture-install-1.0.json",
|
||||
] {
|
||||
let mut altered = files.clone();
|
||||
altered.insert(name.into(), b"{}".to_vec());
|
||||
assert!(verify_evidence(&c, &altered).is_err(), "{name}");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn partial_snapshot_cannot_hide_other_changes_or_turn_into_success() {
|
||||
let (binding, row, files, now) = fixture();
|
||||
let c = eligible(&binding, row).unwrap().unwrap();
|
||||
for variation in 0..6 {
|
||||
let mut o = output(&c, &files, now);
|
||||
match variation {
|
||||
0 => {
|
||||
o.proof
|
||||
.snapshot
|
||||
.get_mut("base-files:amd64")
|
||||
.unwrap()
|
||||
.version = "changed".into()
|
||||
}
|
||||
1 => {
|
||||
o.proof.snapshot.get_mut("rmm-mvp-test:all").unwrap().status =
|
||||
"install ok installed".into()
|
||||
}
|
||||
2 => o.proof.job_id = Uuid::new_v4(),
|
||||
3 => o.proof.partial_effect = false,
|
||||
4 => o.proof.checked_at_ms -= 60000,
|
||||
_ => o.proof.postconditions_passed = true,
|
||||
}
|
||||
assert!(verify_proof(&c, &files, &o, now).is_err());
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn append_only_proof_preserves_unknown_and_consumption_until_signed_sync() {
|
||||
let (binding, row, files, now) = fixture();
|
||||
let c = eligible(&binding, row.clone()).unwrap().unwrap();
|
||||
let output = output(&c, &files, now);
|
||||
verify_proof(&c, &files, &output, now).unwrap();
|
||||
let mut db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB); CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);").unwrap();
|
||||
db.execute(
|
||||
"INSERT INTO consumed VALUES(?1,NULL)",
|
||||
[c.probe.grant_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
db.execute(
|
||||
"INSERT INTO execution_evidence VALUES(?1,?2)",
|
||||
params![c.probe.grant_id.to_string(), row.outcome],
|
||||
)
|
||||
.unwrap();
|
||||
persist(&mut db, &c, &output.proof).unwrap();
|
||||
assert!(
|
||||
db.query_row("SELECT result IS NULL FROM consumed", [], |r| r
|
||||
.get::<_, bool>(0))
|
||||
.unwrap()
|
||||
);
|
||||
let saved: Vec<u8> = db
|
||||
.query_row(
|
||||
"SELECT previous_outcome FROM apt_partial_reconciliations",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(saved, c.original_outcome);
|
||||
let classified: Vec<u8> = db
|
||||
.query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
let outcome: Outcome = serde_json::from_slice(&classified).unwrap();
|
||||
assert_eq!(outcome.state, "failed");
|
||||
assert_eq!(outcome.error_code, CODE);
|
||||
assert_eq!(outcome.evidence["postconditions_passed"], false);
|
||||
assert_eq!(outcome.evidence["partial_effect"], true);
|
||||
assert!(persist(&mut db, &c, &output.proof).is_err());
|
||||
assert_eq!(
|
||||
db.query_row(
|
||||
"SELECT count(*) FROM apt_partial_reconciliations",
|
||||
[],
|
||||
|r| r.get::<_, i64>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT outcome FROM execution_evidence", [], |r| r
|
||||
.get::<_, Vec<u8>>(0))
|
||||
.unwrap(),
|
||||
classified
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
//! received with a job are trusted. Both processes keep their own consumption DB.
|
||||
extern crate self as rmm_privileged_helper;
|
||||
pub mod apt;
|
||||
pub mod apt_reconcile;
|
||||
pub mod control;
|
||||
pub mod readiness;
|
||||
pub mod reconcile;
|
||||
|
||||
@@ -298,6 +298,12 @@ fn handle(
|
||||
let (mut snapshot, mut unlocked) = (None, vec![]);
|
||||
match &request {
|
||||
control::Request::Snapshot { generation, .. } => {
|
||||
rmm_privileged_helper::apt_reconcile::refresh(
|
||||
db,
|
||||
&config.binding,
|
||||
&guard,
|
||||
reconcile::Clock::now(sync_clock)?,
|
||||
)?;
|
||||
snapshot = Some(sync.snapshot(
|
||||
db,
|
||||
&guard,
|
||||
|
||||
@@ -62,7 +62,7 @@ impl ExecutorGuard {
|
||||
_not_sync: std::marker::PhantomData,
|
||||
})
|
||||
}
|
||||
fn check(&self, db: &Connection) -> Result<()> {
|
||||
pub(crate) fn check(&self, db: &Connection) -> Result<()> {
|
||||
let path: String = db.query_row(
|
||||
"SELECT file FROM pragma_database_list WHERE name='main'",
|
||||
[],
|
||||
|
||||
Reference in New Issue
Block a user