From 998f16fde4771b8b6ba5e65512146c1816d99ed4 Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Fri, 13 Nov 2026 17:20:50 +0100 Subject: [PATCH] fix: retain sync anchors and verify bound APT partial postconditions --- crates/agent/src/reconcile.rs | 10 +- crates/agent/tests/reconcile.rs | 7 +- crates/privileged-helper/src/apt_reconcile.rs | 850 ++++++++++++++++++ crates/privileged-helper/src/lib.rs | 1 + crates/privileged-helper/src/main.rs | 6 + crates/privileged-helper/src/reconcile.rs | 2 +- crates/server/src/synchronization.rs | 62 +- deploy/cloud_apt_reconcile_upgrade.py | 154 ++++ deploy/cloud_sync_evidence.py | 67 ++ docs/implementation-status.md | 44 + docs/mvp-gates.md | 7 + .../test_partial_reconciliation.py | 118 +++ tools/apt-transaction/apt_transaction.py | 157 ++++ 13 files changed, 1469 insertions(+), 16 deletions(-) create mode 100644 crates/privileged-helper/src/apt_reconcile.rs create mode 100644 deploy/cloud_apt_reconcile_upgrade.py create mode 100644 deploy/cloud_sync_evidence.py create mode 100644 tests/apt_transaction/test_partial_reconciliation.py diff --git a/crates/agent/src/reconcile.rs b/crates/agent/src/reconcile.rs index 77450be..01bd225 100644 --- a/crates/agent/src/reconcile.rs +++ b/crates/agent/src/reconcile.rs @@ -238,11 +238,11 @@ impl ClientSync { request_id: Uuid::new_v4().to_string(), journal_id: meta.journal_id.to_string(), checkpoint: u64::try_from(next)?, - previous_decision_id: if meta.pending_request.is_some() { - String::new() - } else { - meta.decision.map(|id| id.to_string()).unwrap_or_default() - }, + // A pending request may have committed with a lost reply, or may + // never have reached the server. Preserve both witnesses: the + // server resolves the pending request first and otherwise checks + // this last applied decision against its authoritative head. + previous_decision_id: meta.decision.map(|id| id.to_string()).unwrap_or_default(), assignment_version: self.binding.assignment_version, attempts, durable_message_ids: ids, diff --git a/crates/agent/tests/reconcile.rs b/crates/agent/tests/reconcile.rs index 614e64a..54b9788 100644 --- a/crates/agent/tests/reconcile.rs +++ b/crates/agent/tests/reconcile.rs @@ -788,7 +788,7 @@ fn rollback_and_false_clock_close_mutations_but_do_not_destroy_journal() { let _native = NativeClock::default(); } #[test] -fn lost_response_uses_fresh_request_and_previous_request_not_old_decision() { +fn uncertain_request_preserves_pending_request_and_last_applied_anchor() { let mut f = Fixture::new(); let mut c = f.client(); let first = f.prepare(&mut c); @@ -804,7 +804,10 @@ fn lost_response_uses_fresh_request_and_previous_request_not_old_decision() { recovered.request.previous_request_id, lost.request.request_id ); - assert!(recovered.request.previous_decision_id.is_empty()); + assert_eq!( + recovered.request.previous_decision_id, + d.decision_id.to_string() + ); } #[test] diff --git a/crates/privileged-helper/src/apt_reconcile.rs b/crates/privileged-helper/src/apt_reconcile.rs new file mode 100644 index 0000000..e994a13 --- /dev/null +++ b/crates/privileged-helper/src/apt_reconcile.rs @@ -0,0 +1,850 @@ +//! Recheck only the original fixture job's bound postcondition. No executor, +//! approval, new grant, package command or consumption unlock exists here. +use crate::{Binding, Outcome, Request, apt, reconcile::ExecutorGuard}; +use anyhow::{Context, Result, ensure}; +use chrono::{DateTime, Utc}; +use rmm_domain::{apt_transaction::AptExecutionManifest, jobs::*}; +use rusqlite::{Connection, TransactionBehavior, params}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + collections::BTreeMap, + fs::{self, OpenOptions}, + io::{Read, Write}, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::Path, + process::{Command, Stdio}, +}; +use uuid::Uuid; + +const BASE: &str = "/var/lib/rmm-apt-transaction"; +const CODE: &str = "APT_PARTIAL_UNCONFIGURED_VERIFIED"; +const EVIDENCE_NAMES: [&str; 11] = [ + "context.json", + "signed-action.json", + "expectation.json", + "baseline.json", + "launch.json", + "native-prechecks.json", + "sandbox-ready.json", + "sandbox-result.json", + "hook-armed.json", + "bootstrap-attempt-rmm-apt-fixture-install-1.0.json", + "bootstrap-retry-rmm-apt-fixture-install-1.0.json", +]; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SavedStart { + previous_boot_id: Uuid, + reserved_at: DateTime, + reconnect_deadline: Option>, + request: Request, + reauthentication_required: bool, + postconditions_passed: bool, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct PackageState { + version: String, + status: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct ProbeRequest { + transaction_sha256: String, + job_payload_sha256: String, + effect_armed_ledger_id: String, + grant_id: Uuid, + job_id: Uuid, + attempt_id: Uuid, + authorization_id: Uuid, + plan_sha256: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ContextRecord { + transaction_sha256: String, + job_payload_sha256: String, + effect_armed_ledger_id: String, + grant_id: Uuid, + job_id: Uuid, + attempt_id: Uuid, + authorization_id: Uuid, + plan_sha256: String, + schema_version: u32, + action: String, + execution_profile: String, + not_before_ms: i64, + start_before_ms: i64, + boot_id: Uuid, + monotonic_start_before_ns: u64, +} +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct PartialProof { + schema_version: u32, + #[serde(rename = "type")] + kind: String, + transaction_sha256: String, + job_payload_sha256: String, + effect_armed_ledger_id: String, + grant_id: Uuid, + job_id: Uuid, + attempt_id: Uuid, + authorization_id: Uuid, + plan_sha256: String, + original_evidence_sha256: String, + original_baseline_sha256: String, + observed_snapshot_sha256: String, + status_fingerprint: String, + fixture_payload_sha256: String, + snapshot: BTreeMap, + checked_at_ms: i64, + partial_effect: bool, + postconditions_passed: bool, + reboot_required: bool, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ProbeOutput { + effect_state: String, + code: String, + proof: PartialProof, +} + +struct Candidate { + request: Request, + permit: ExecutionPermitPayload, + manifest: AptExecutionManifest, + probe: ProbeRequest, + original_boot_id: Uuid, + original_outcome: Vec, +} +#[derive(Clone)] +struct LedgerRow { + rowid: i64, + grant: String, + agent: String, + generation: u64, + job: String, + attempt: String, + start: Vec, + outcome: Vec, +} +fn digest(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || b"abcdef".contains(&b)) +} +fn eligible(binding: &Binding, row: LedgerRow) -> Result> { + ensure!( + row.start.len() <= 512 * 1024 && row.outcome.len() <= 512 * 1024, + "APT_RECONCILE_RECORD_LIMIT" + ); + let old: Outcome = serde_json::from_slice(&row.outcome)?; + if old.state != "unknown" || old.error_code != "DPKG_INCONSISTENT" { + return Ok(None); + } + let start: SavedStart = serde_json::from_slice(&row.start)?; + ensure!( + !start.previous_boot_id.is_nil() + && start.reauthentication_required + && !start.postconditions_passed + && start.reconnect_deadline.is_none(), + "APT_RECONCILE_ORIGINAL_START" + ); + let permit: ExecutionPermitPayload = serde_json::from_slice(&start.request.permit_bytes)?; + // The old start window is verified historically. It never authorizes a + // present process start, and the consumed tombstone is never replaced. + let (job, permit) = binding.verify(&start.request, permit.not_before)?; + ensure!( + row.agent == binding.agent_id.to_string() + && row.generation == binding.enrollment_generation + && row.grant == permit.grant_id.to_string() + && row.job == job.job_id.to_string() + && row.attempt == job.attempt_id.to_string() + && job.execution_identity == "root" + && start.reserved_at <= permit.start_before, + "APT_RECONCILE_ORIGINAL_BINDING" + ); + let Action::AptNativeAcceptance { + recipe_id, + recipe_version, + apt_manifest, + } = &job.action + else { + return Ok(None); + }; + ensure!( + recipe_id == apt::INSTALL_RECIPE + && recipe_version == "1" + && apt_manifest.validate() + && apt_manifest.schema_version == 3 + && apt_manifest.native_retry.is_some() + && apt_manifest.action == "apt-native-acceptance" + && apt_manifest.execution_profile == "dev-test", + "APT_RECONCILE_FIXED_NATIVE_FIXTURE_ONLY" + ); + let ledger = format!("apt-{}-{}", permit.grant_id, row.rowid); + ensure!( + old.exit_code == Some(0) + && old.stdout.is_empty() + && old.stderr.is_empty() + && !old.output_truncated + && !old.reboot_required + && old.evidence["postconditions_passed"] == false + && old.evidence["test_acceptance"] == true + && old.evidence["noProductionEvidence"] == true + && old.evidence["effect_armed_ledger_id"] == ledger, + "APT_RECONCILE_EXPECTED_UNKNOWN_ONLY" + ); + let probe = ProbeRequest { + transaction_sha256: apt_manifest.transaction_sha256.clone(), + job_payload_sha256: hash(&start.request.job_bytes), + effect_armed_ledger_id: ledger, + grant_id: permit.grant_id, + job_id: job.job_id, + attempt_id: job.attempt_id, + authorization_id: permit.authorization_id, + plan_sha256: permit.plan_hash.clone(), + }; + Ok(Some(Candidate { + request: start.request, + permit, + manifest: apt_manifest.clone(), + probe, + original_boot_id: start.previous_boot_id, + original_outcome: row.outcome, + })) +} +fn sealed(path: &Path, limit: u64) -> Result> { + for component in path.ancestors() { + let meta = fs::symlink_metadata(component)?; + ensure!( + !meta.file_type().is_symlink() && meta.uid() == 0 && meta.mode() & 0o022 == 0, + "APT_RECONCILE_UNTRUSTED_PATH" + ); + } + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?; + ensure!( + file.metadata()?.is_file() && file.metadata()?.len() <= limit, + "APT_RECONCILE_FILE_LIMIT" + ); + let mut bytes = vec![]; + (&mut file).take(limit + 1).read_to_end(&mut bytes)?; + ensure!(bytes.len() as u64 <= limit, "APT_RECONCILE_FILE_LIMIT"); + Ok(bytes) +} +fn read_evidence() -> Result>> { + let base = Path::new(BASE); + crate::private_path(base, 0, true)?; + crate::private_path(&base.join("active"), 0, true)?; + for name in [ + "acceptance-rmm-apt-fixture-install-1.0.json", + "pending-acceptance-rmm-apt-fixture-install-1.0.json", + ] { + ensure!( + matches!(fs::symlink_metadata(base.join(name)),Err(e) if e.kind()==std::io::ErrorKind::NotFound), + "APT_RECONCILE_ACCEPTANCE_EXISTS" + ); + } + let mut result = BTreeMap::new(); + for name in EVIDENCE_NAMES.into_iter().chain(["apt.conf"]) { + let path = if name.starts_with("bootstrap-") { + base.join(name) + } else { + base.join("active").join(name) + }; + result.insert(name.to_owned(), sealed(&path, 1024 * 1024)?); + } + Ok(result) +} +fn verify_evidence( + c: &Candidate, + files: &BTreeMap>, +) -> Result> { + let read = |name: &str| files.get(name).context("APT_RECONCILE_EVIDENCE_MISSING"); + ensure!( + read("signed-action.json")? == &c.request.job_bytes + && read("expectation.json")? == &c.manifest.transaction_bytes + && hash(read("baseline.json")?) == c.manifest.resolved_apt_transaction.baseline_sha256 + && hash(read("apt.conf")?) == c.manifest.apt_config_sha256, + "APT_RECONCILE_ORIGINAL_FILE_HASH" + ); + let context: ContextRecord = serde_json::from_slice(read("context.json")?)?; + ensure!( + context.schema_version == 2 + && context.action == "apt-native-acceptance" + && context.execution_profile == "dev-test" + && context.transaction_sha256 == c.probe.transaction_sha256 + && context.job_payload_sha256 == c.probe.job_payload_sha256 + && context.effect_armed_ledger_id == c.probe.effect_armed_ledger_id + && context.grant_id == c.probe.grant_id + && context.job_id == c.probe.job_id + && context.attempt_id == c.probe.attempt_id + && context.authorization_id == c.probe.authorization_id + && context.plan_sha256 == c.probe.plan_sha256 + && context.boot_id == c.original_boot_id + && context.not_before_ms == c.permit.not_before.timestamp_millis() + && context.start_before_ms == c.permit.start_before.timestamp_millis() + && context.monotonic_start_before_ns > 0, + "APT_RECONCILE_ORIGINAL_CONTEXT" + ); + let value = |name: &str| -> Result { Ok(serde_json::from_slice(read(name)?)?) }; + ensure!( + value("launch.json")? + == json!({"transaction_sha256":c.probe.transaction_sha256,"effect_armed_ledger_id":c.probe.effect_armed_ledger_id}) + && value("sandbox-result.json")? + == json!({"transaction_sha256":c.probe.transaction_sha256,"apt_exit_code":0,"reboot_required":false}) + && value("sandbox-ready.json")? + == json!({"transaction_sha256":c.probe.transaction_sha256,"sandbox_version":1}) + && value("native-prechecks.json")? == native_prechecks(c), + "APT_RECONCILE_NATIVE_RESULT" + ); + let hook = value("hook-armed.json")?; + ensure!(hook["transaction_sha256"]==c.probe.transaction_sha256 && hook["grant_id"]==c.probe.grant_id.to_string() && + hook["effect_armed_ledger_id"]==c.probe.effect_armed_ledger_id && + hook["protocol_sha256"].as_str().is_some_and(digest) && hook["armed_at_ms"].as_i64().is_some_and(|time| + time>=context.not_before_ms && time = serde_json::from_slice(read("baseline.json")?)?; + ensure!( + !baseline.contains_key("rmm-mvp-test:all"), + "APT_RECONCILE_ORIGINAL_ABSENCE" + ); + Ok(baseline) +} +fn native_prechecks(c: &Candidate) -> Value { + json!({"schema_version":1,"transaction_sha256":c.probe.transaction_sha256, + "job_payload_sha256":c.probe.job_payload_sha256,"boot_id":c.original_boot_id, + "checks":{"namespaces_caps_nnp_seccomp":true,"readonly_controlpaths":true, + "private_tmpfs_sockets":true,"control_syscalls":{"invalid_reboot_magic":"EPERM", + "invalid_kexec_load_flags":"EPERM","invalid_kexec_file_fds_flags":"EPERM"}, + "frontend_lock":"native-apt-check-conflict"}}) +} +fn evidence_hash(files: &BTreeMap>) -> Result { + let mut hashes = BTreeMap::new(); + for name in EVIDENCE_NAMES { + hashes.insert( + name, + hash(files.get(name).context("APT_RECONCILE_EVIDENCE_MISSING")?), + ); + } + Ok(hash(&serde_json::to_vec(&hashes)?)) +} +fn verify_proof( + c: &Candidate, + files: &BTreeMap>, + output: &ProbeOutput, + now: DateTime, +) -> Result<()> { + let baseline = verify_evidence(c, files)?; + let p = &output.proof; + ensure!( + output.effect_state == "known_failed" + && output.code == CODE + && p.schema_version == 1 + && p.kind == "rmm.apt.partial-reconciliation" + && p.partial_effect + && !p.postconditions_passed + && !p.reboot_required, + "APT_RECONCILE_PROOF_TYPE" + ); + let actual = ProbeRequest { + transaction_sha256: p.transaction_sha256.clone(), + job_payload_sha256: p.job_payload_sha256.clone(), + effect_armed_ledger_id: p.effect_armed_ledger_id.clone(), + grant_id: p.grant_id, + job_id: p.job_id, + attempt_id: p.attempt_id, + authorization_id: p.authorization_id, + plan_sha256: p.plan_sha256.clone(), + }; + let mut expected = baseline; + expected.insert( + "rmm-mvp-test:all".into(), + PackageState { + version: "1.0".into(), + status: "install ok unpacked".into(), + }, + ); + ensure!( + actual == c.probe + && p.original_evidence_sha256 == evidence_hash(files)? + && p.original_baseline_sha256 == c.manifest.resolved_apt_transaction.baseline_sha256 + && p.snapshot == expected + && p.observed_snapshot_sha256 + == hash(&serde_json::to_vec(&serde_json::to_value(&p.snapshot)?)?) + && p.fixture_payload_sha256 == hash(b"1.0\n") + && digest(&p.status_fingerprint) + && p.checked_at_ms >= now.timestamp_millis() - 1000 + && p.checked_at_ms <= Utc::now().timestamp_millis() + 1000, + "APT_RECONCILE_PROOF_BINDING" + ); + Ok(()) +} +fn probe(c: &Candidate) -> Result { + apt::verify_installation()?; + let mut child = Command::new("/usr/bin/python3") + .args(["-I", apt::CODE, "verify-partial"]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("LC_ALL", "C") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + child + .stdin + .take() + .context("APT_RECONCILE_STDIN")? + .write_all(&serde_json::to_vec(&c.probe)?)?; + let mut stdout = child.stdout.take().context("APT_RECONCILE_STDOUT")?; + let mut bytes = vec![]; + let mut chunk = [0; 8192]; + let mut overflow = false; + loop { + let count = stdout.read(&mut chunk)?; + if count == 0 { + break; + } + if bytes.len() + count <= 1024 * 1024 { + bytes.extend_from_slice(&chunk[..count]); + } else { + overflow = true; + } + } + let status = child.wait()?; + ensure!(status.success() && !overflow, "APT_RECONCILE_NOT_PROVEN"); + Ok(serde_json::from_slice(&bytes)?) +} +fn persist(db: &mut Connection, c: &Candidate, proof: &PartialProof) -> Result<()> { + let proof_bytes = serde_json::to_vec(&serde_json::to_value(proof)?)?; + let proof_sha = hash(&proof_bytes); + let outcome = Outcome { + state: "failed".into(), + stdout: vec![], + stderr: vec![], + exit_code: Some(0), + output_truncated: false, + reboot_required: false, + evidence: json!({"postconditions_passed":false,"partial_effect":true,"reconciled_postconditions":true, + "effect_armed_ledger_id":c.probe.effect_armed_ledger_id,"test_acceptance":true,"noProductionEvidence":true, + "apt":{"test_acceptance":true,"noProductionEvidence":true,"partial_effect":true, + "fixture_status":"install ok unpacked","fixture_version":"1.0","fixture_payload_sha256":proof.fixture_payload_sha256, + "snapshot_sha256":proof.observed_snapshot_sha256,"status_fingerprint":proof.status_fingerprint, + "reconciliation_proof_sha256":proof_sha}}), + error_code: CODE.into(), + }; + let classified = serde_json::to_vec(&outcome)?; + let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute_batch("CREATE TABLE IF NOT EXISTS apt_partial_reconciliations(grant_id TEXT PRIMARY KEY, + original_request_sha256 TEXT NOT NULL,previous_outcome_sha256 TEXT NOT NULL,previous_outcome BLOB NOT NULL, + proof_sha256 TEXT NOT NULL,proof BLOB NOT NULL,classified_outcome BLOB NOT NULL,checked_at_ms INTEGER NOT NULL);")?; + let current:(Option>,Option>)=tx.query_row("SELECT c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE grant_id=?1", + [c.probe.grant_id.to_string()],|r|Ok((r.get(0)?,r.get(1)?)))?; + ensure!( + current.0.is_none() && current.1.as_deref() == Some(c.original_outcome.as_slice()), + "APT_RECONCILE_OUTCOME_CHANGED" + ); + tx.execute("INSERT INTO apt_partial_reconciliations(grant_id,original_request_sha256,previous_outcome_sha256,previous_outcome,proof_sha256,proof,classified_outcome,checked_at_ms) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)", + params![c.probe.grant_id.to_string(),hash(&serde_json::to_vec(&c.request)?),hash(&c.original_outcome),c.original_outcome,proof_sha,proof_bytes,classified,proof.checked_at_ms])?; + ensure!( + tx.execute( + "UPDATE execution_evidence SET outcome=?2 WHERE grant_id=?1 AND outcome=?3", + params![c.probe.grant_id.to_string(), classified, c.original_outcome] + )? == 1, + "APT_RECONCILE_OUTCOME_CHANGED" + ); + // consumed.result stays NULL. Only the normal independently checked signed + // RootSync decision may close this particular consumed resource afterward. + tx.commit()?; + Ok(()) +} + +/// Invoke under the same executor lock as RootSync capture. Failure to prove a +/// postcondition leaves the existing unknown outcome and resource fence intact. +pub fn refresh( + db: &mut Connection, + binding: &Binding, + guard: &ExecutorGuard, + now: DateTime, +) -> Result<()> { + guard.check(db)?; + // SAFETY: geteuid has no pointers or preconditions. + ensure!( + unsafe { libc::geteuid() } == 0, + "APT_RECONCILE_ROOT_REQUIRED" + ); + let mut statement=db.prepare("SELECT c.rowid,c.grant_id,c.agent,c.generation,c.job,c.attempt,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL AND e.outcome IS NOT NULL")?; + let rows = statement + .query_map([], |r| { + Ok(LedgerRow { + rowid: r.get(0)?, + grant: r.get(1)?, + agent: r.get(2)?, + generation: r.get(3)?, + job: r.get(4)?, + attempt: r.get(5)?, + start: r.get(6)?, + outcome: r.get(7)?, + }) + })? + .collect::>>()?; + drop(statement); + ensure!(rows.len() <= 256, "APT_RECONCILE_CAPACITY"); + for row in rows { + let Some(candidate) = eligible(binding, row)? else { + continue; + }; + // These checks are read-only. Any missing/ambiguous evidence is not a + // definitive failure and never releases the unknown attempt. + let Ok(files) = read_evidence() else { + continue; + }; + if verify_evidence(&candidate, &files).is_err() { + continue; + } + let Ok(output) = probe(&candidate) else { + continue; + }; + if verify_proof(&candidate, &files, &output, now).is_err() { + continue; + } + persist(db, &candidate, &output.proof)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use openssl::{pkey::PKey, sign::Signer}; + + fn fixture() -> (Binding, LedgerRow, BTreeMap>, DateTime) { + let now = Utc::now(); + let key = PKey::generate_ed25519().unwrap(); + let binding = Binding { + tenant_id: Uuid::new_v4(), + asset_id: Uuid::new_v4(), + agent_id: Uuid::new_v4(), + enrollment_generation: 1, + assignment_version: 1, + public_key_hex: hex::encode(key.raw_public_key().unwrap()), + }; + let baseline = json!({"base-files:amd64":{"version":"1","status":"install ok installed"}}); + let tx = json!({"schema_version":1,"recipe_id":apt::INSTALL_RECIPE,"recipe_version":"1","platform":"ubuntu-24.04-amd64", + "resolver_config_version":"rmm-apt-local-v1","no_reboot":true,"status_fingerprint":"a".repeat(64), + "baseline_sha256":hash(&serde_json::to_vec(&baseline).unwrap()),"packages":[{"name":"rmm-mvp-test","architecture":"all", + "old_version":null,"new_version":"1.0","origin":"rmm-test", + "sha256":"74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867","size":20672, + "archive_path":"/var/lib/rmm-apt-transaction/archives/74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867.deb"}]}); + let txbytes = serde_json::to_vec(&tx).unwrap(); + let original_grant = Uuid::new_v4(); + let original_attempt = Uuid::new_v4(); + let marker = serde_json::to_vec( + &json!({"transaction_sha256":hash(&txbytes),"job_payload_sha256":"b".repeat(64), + "grant_id":original_grant,"attempt_id":original_attempt}), + ) + .unwrap(); + let native_retry = json!({"schema_version":1,"previous_grant_id":original_grant,"previous_attempt_id":original_attempt, + "bootstrap_marker_sha256":hash(&marker)}); + let manifest = json!({"type":"rmm.apt.helper-action","schema_version":3,"action":"apt-native-acceptance", + "execution_profile":"dev-test","transaction_sha256":hash(&txbytes),"resolved_apt_transaction":tx, + "transaction_bytes":txbytes,"provider_sha256":"c".repeat(64),"hook_sha256":"d".repeat(64), + "apt_config_sha256":hash(b"historical config"),"runtime_sha256":"e".repeat(64),"test_result_id":"partial-test", + "native_retry":native_retry}); + let job:JobPayload=serde_json::from_value(json!({"schema_version":1,"tenant_id":binding.tenant_id, + "asset_id":binding.asset_id,"agent_id":binding.agent_id,"enrollment_generation":1,"assignment_version":1, + "job_id":Uuid::new_v4(),"attempt_id":Uuid::new_v4(),"action":{"action":"apt_native_acceptance", + "recipe_id":apt::INSTALL_RECIPE,"recipe_version":"1","apt_manifest":manifest},"execution_identity":"root", + "not_before":now,"start_before":now+chrono::Duration::minutes(15),"max_runtime_seconds":300,"max_output_bytes":65536})).unwrap(); + let job_bytes = encode(&job).unwrap(); + let permit = ExecutionPermitPayload { + domain: "rmm.execution-permit".into(), + schema_version: 1, + grant_id: Uuid::new_v4(), + authorization_id: Uuid::new_v4(), + tenant_id: binding.tenant_id, + asset_id: binding.asset_id, + agent_id: binding.agent_id, + enrollment_generation: 1, + assignment_version: 1, + job_id: job.job_id, + attempt_id: job.attempt_id, + plan_hash: "f".repeat(64), + job_payload_hash: hash(&job_bytes), + action: "apt_native_acceptance".into(), + execution_identity: "root".into(), + not_before: now, + start_before: now + chrono::Duration::seconds(60), + max_runtime_seconds: 300, + key_id: hash(&key.raw_public_key().unwrap()), + }; + let permit_bytes = encode(&permit).unwrap(); + let request = Request { + job_bytes, + signature: Signer::new_without_digest(&key) + .unwrap() + .sign_oneshot_to_vec(&permit_bytes) + .unwrap(), + permit_bytes, + authorization_id: permit.authorization_id, + plan_hash: permit.plan_hash.clone(), + }; + let boot = Uuid::new_v4(); + let ledger = format!("apt-{}-1", permit.grant_id); + let old = Outcome { + exit_code: Some(0), + evidence: json!({"postconditions_passed":false,"test_acceptance":true,"noProductionEvidence":true, + "effect_armed_ledger_id":ledger,"apt":{"test_acceptance":true,"noProductionEvidence":true}}), + ..Outcome::unknown("DPKG_INCONSISTENT") + }; + let row = LedgerRow { + rowid: 1, + grant: permit.grant_id.to_string(), + agent: binding.agent_id.to_string(), + generation: 1, + job: job.job_id.to_string(), + attempt: job.attempt_id.to_string(), + outcome: serde_json::to_vec(&old).unwrap(), + start: serde_json::to_vec( + &json!({"previous_boot_id":boot,"reserved_at":now,"reconnect_deadline":null, + "request":request,"reauthentication_required":true,"postconditions_passed":false}), + ) + .unwrap(), + }; + let c = eligible(&binding, row.clone()).unwrap().unwrap(); + let mut files = BTreeMap::new(); + files.insert("signed-action.json".into(), c.request.job_bytes.clone()); + files.insert( + "expectation.json".into(), + c.manifest.transaction_bytes.clone(), + ); + files.insert( + "baseline.json".into(), + serde_json::to_vec(&baseline).unwrap(), + ); + files.insert("apt.conf".into(), b"historical config".to_vec()); + let context = json!({"schema_version":2,"transaction_sha256":c.probe.transaction_sha256,"job_payload_sha256":c.probe.job_payload_sha256, + "effect_armed_ledger_id":ledger,"grant_id":permit.grant_id,"job_id":job.job_id,"attempt_id":job.attempt_id, + "authorization_id":permit.authorization_id,"plan_sha256":permit.plan_hash,"action":"apt-native-acceptance", + "execution_profile":"dev-test","not_before_ms":permit.not_before.timestamp_millis(),"start_before_ms":permit.start_before.timestamp_millis(), + "boot_id":boot,"monotonic_start_before_ns":1}); + let objects = [ + ("context.json", context), + ( + "launch.json", + json!({"transaction_sha256":c.probe.transaction_sha256,"effect_armed_ledger_id":ledger}), + ), + ( + "sandbox-result.json", + json!({"transaction_sha256":c.probe.transaction_sha256,"apt_exit_code":0,"reboot_required":false}), + ), + ( + "sandbox-ready.json", + json!({"transaction_sha256":c.probe.transaction_sha256,"sandbox_version":1}), + ), + ("native-prechecks.json", native_prechecks(&c)), + ( + "hook-armed.json", + json!({"transaction_sha256":c.probe.transaction_sha256,"effect_armed_ledger_id":ledger,"grant_id":permit.grant_id, + "protocol_sha256":"a".repeat(64),"armed_at_ms":now.timestamp_millis()}), + ), + ( + "bootstrap-retry-rmm-apt-fixture-install-1.0.json", + json!({"transaction_sha256":c.probe.transaction_sha256, + "job_payload_sha256":c.probe.job_payload_sha256,"effect_armed_ledger_id":ledger,"grant_id":permit.grant_id,"attempt_id":job.attempt_id, + "native_retry":native_retry,"previous_evidence_sha256":"b".repeat(64)}), + ), + ]; + for (name, value) in objects { + files.insert(name.into(), serde_json::to_vec(&value).unwrap()); + } + files.insert( + "bootstrap-attempt-rmm-apt-fixture-install-1.0.json".into(), + marker, + ); + (binding, row, files, now) + } + fn output(c: &Candidate, files: &BTreeMap>, now: DateTime) -> ProbeOutput { + let mut snapshot = verify_evidence(c, files).unwrap(); + snapshot.insert( + "rmm-mvp-test:all".into(), + PackageState { + version: "1.0".into(), + status: "install ok unpacked".into(), + }, + ); + ProbeOutput { + effect_state: "known_failed".into(), + code: CODE.into(), + proof: PartialProof { + schema_version: 1, + kind: "rmm.apt.partial-reconciliation".into(), + transaction_sha256: c.probe.transaction_sha256.clone(), + job_payload_sha256: c.probe.job_payload_sha256.clone(), + effect_armed_ledger_id: c.probe.effect_armed_ledger_id.clone(), + grant_id: c.probe.grant_id, + job_id: c.probe.job_id, + attempt_id: c.probe.attempt_id, + authorization_id: c.probe.authorization_id, + plan_sha256: c.probe.plan_sha256.clone(), + original_evidence_sha256: evidence_hash(files).unwrap(), + original_baseline_sha256: c + .manifest + .resolved_apt_transaction + .baseline_sha256 + .clone(), + observed_snapshot_sha256: hash( + &serde_json::to_vec(&serde_json::to_value(&snapshot).unwrap()).unwrap(), + ), + snapshot, + status_fingerprint: "a".repeat(64), + fixture_payload_sha256: hash(b"1.0\n"), + checked_at_ms: now.timestamp_millis(), + partial_effect: true, + postconditions_passed: false, + reboot_required: false, + }, + } + } + #[test] + fn original_signature_identity_marker_and_complete_evidence_are_required() { + let (binding, row, files, now) = fixture(); + let c = eligible(&binding, row.clone()).unwrap().unwrap(); + verify_proof(&c, &files, &output(&c, &files, now), now).unwrap(); + let mut wrong = row.clone(); + wrong.agent = Uuid::new_v4().to_string(); + assert!(eligible(&binding, wrong).is_err()); + let mut wrong = row; + let mut start: Value = serde_json::from_slice(&wrong.start).unwrap(); + start["request"]["signature"][0] = + json!((start["request"]["signature"][0].as_u64().unwrap() + 1) % 256); + wrong.start = serde_json::to_vec(&start).unwrap(); + assert!(eligible(&binding, wrong).is_err()); + for name in [ + "context.json", + "signed-action.json", + "expectation.json", + "baseline.json", + "launch.json", + "sandbox-result.json", + "native-prechecks.json", + "hook-armed.json", + "bootstrap-attempt-rmm-apt-fixture-install-1.0.json", + ] { + let mut altered = files.clone(); + altered.insert(name.into(), b"{}".to_vec()); + assert!(verify_evidence(&c, &altered).is_err(), "{name}"); + } + } + #[test] + fn partial_snapshot_cannot_hide_other_changes_or_turn_into_success() { + let (binding, row, files, now) = fixture(); + let c = eligible(&binding, row).unwrap().unwrap(); + for variation in 0..6 { + let mut o = output(&c, &files, now); + match variation { + 0 => { + o.proof + .snapshot + .get_mut("base-files:amd64") + .unwrap() + .version = "changed".into() + } + 1 => { + o.proof.snapshot.get_mut("rmm-mvp-test:all").unwrap().status = + "install ok installed".into() + } + 2 => o.proof.job_id = Uuid::new_v4(), + 3 => o.proof.partial_effect = false, + 4 => o.proof.checked_at_ms -= 60000, + _ => o.proof.postconditions_passed = true, + } + assert!(verify_proof(&c, &files, &o, now).is_err()); + } + } + #[test] + fn append_only_proof_preserves_unknown_and_consumption_until_signed_sync() { + let (binding, row, files, now) = fixture(); + let c = eligible(&binding, row.clone()).unwrap().unwrap(); + let output = output(&c, &files, now); + verify_proof(&c, &files, &output, now).unwrap(); + let mut db = Connection::open_in_memory().unwrap(); + db.execute_batch("CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB); CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);").unwrap(); + db.execute( + "INSERT INTO consumed VALUES(?1,NULL)", + [c.probe.grant_id.to_string()], + ) + .unwrap(); + db.execute( + "INSERT INTO execution_evidence VALUES(?1,?2)", + params![c.probe.grant_id.to_string(), row.outcome], + ) + .unwrap(); + persist(&mut db, &c, &output.proof).unwrap(); + assert!( + db.query_row("SELECT result IS NULL FROM consumed", [], |r| r + .get::<_, bool>(0)) + .unwrap() + ); + let saved: Vec = db + .query_row( + "SELECT previous_outcome FROM apt_partial_reconciliations", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(saved, c.original_outcome); + let classified: Vec = db + .query_row("SELECT outcome FROM execution_evidence", [], |r| r.get(0)) + .unwrap(); + let outcome: Outcome = serde_json::from_slice(&classified).unwrap(); + assert_eq!(outcome.state, "failed"); + assert_eq!(outcome.error_code, CODE); + assert_eq!(outcome.evidence["postconditions_passed"], false); + assert_eq!(outcome.evidence["partial_effect"], true); + assert!(persist(&mut db, &c, &output.proof).is_err()); + assert_eq!( + db.query_row( + "SELECT count(*) FROM apt_partial_reconciliations", + [], + |r| r.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); + assert_eq!( + db.query_row("SELECT outcome FROM execution_evidence", [], |r| r + .get::<_, Vec>(0)) + .unwrap(), + classified + ); + } +} diff --git a/crates/privileged-helper/src/lib.rs b/crates/privileged-helper/src/lib.rs index 2a024e8..0c06342 100644 --- a/crates/privileged-helper/src/lib.rs +++ b/crates/privileged-helper/src/lib.rs @@ -2,6 +2,7 @@ //! received with a job are trusted. Both processes keep their own consumption DB. extern crate self as rmm_privileged_helper; pub mod apt; +pub mod apt_reconcile; pub mod control; pub mod readiness; pub mod reconcile; diff --git a/crates/privileged-helper/src/main.rs b/crates/privileged-helper/src/main.rs index cbe8562..4211742 100644 --- a/crates/privileged-helper/src/main.rs +++ b/crates/privileged-helper/src/main.rs @@ -298,6 +298,12 @@ fn handle( let (mut snapshot, mut unlocked) = (None, vec![]); match &request { control::Request::Snapshot { generation, .. } => { + rmm_privileged_helper::apt_reconcile::refresh( + db, + &config.binding, + &guard, + reconcile::Clock::now(sync_clock)?, + )?; snapshot = Some(sync.snapshot( db, &guard, diff --git a/crates/privileged-helper/src/reconcile.rs b/crates/privileged-helper/src/reconcile.rs index 46855ac..07f30f9 100644 --- a/crates/privileged-helper/src/reconcile.rs +++ b/crates/privileged-helper/src/reconcile.rs @@ -62,7 +62,7 @@ impl ExecutorGuard { _not_sync: std::marker::PhantomData, }) } - fn check(&self, db: &Connection) -> Result<()> { + pub(crate) fn check(&self, db: &Connection) -> Result<()> { let path: String = db.query_row( "SELECT file FROM pragma_database_list WHERE name='main'", [], diff --git a/crates/server/src/synchronization.rs b/crates/server/src/synchronization.rs index d3e485c..3cecdb3 100644 --- a/crates/server/src/synchronization.rs +++ b/crates/server/src/synchronization.rs @@ -52,6 +52,21 @@ fn clean_unstarted(local: &SyncAttempt, reserved: bool) -> bool { && local.grant_id.is_empty() && matches!(local.state.as_str(), "queued" | "accepted") } +/// A missing reply and a request that never committed are different cases. +/// A committed pending request is the newer anchor. Otherwise only the last +/// applied decision can anchor the chain; the caller still compares it with +/// the authoritative head and performs all journal/generation checks. +fn previous_anchor( + last_applied: Option, + pending_request: Option, + committed_pending: Option, +) -> Option { + if pending_request.is_some() { + committed_pending.or(last_applied) + } else { + last_applied + } +} pub async fn reconcile( state: &crate::State, tx: &mut Transaction<'_, Postgres>, @@ -117,13 +132,17 @@ pub async fn reconcile( reason = "JOURNAL_SEQUENCE_REGRESSION".into(); chain_valid = false; } - let mut previous_decision = optional_id(&r.previous_decision_id)?; - if previous_decision.is_none() && !r.previous_request_id.is_empty() { - previous_decision=sqlx::query_scalar("SELECT decision_id FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2 AND journal_id=$3").bind(agent).bind(Uuid::parse_str(&r.previous_request_id)?).bind(journal_id).fetch_optional(&mut **tx).await?; - if previous_decision.is_none() { - reason = "SERVER_CHECKPOINT_MISSING".into(); - chain_valid = false; - } + let last_applied = optional_id(&r.previous_decision_id)?; + let pending_request = optional_id(&r.previous_request_id)?; + let committed_pending = if let Some(pending) = pending_request { + sqlx::query_scalar("SELECT decision_id FROM sync_checkpoints WHERE agent_id=$1 AND request_id=$2 AND journal_id=$3").bind(agent).bind(pending).bind(journal_id).fetch_optional(&mut **tx).await? + } else { + None + }; + let previous_decision = previous_anchor(last_applied, pending_request, committed_pending); + if pending_request.is_some() && previous_decision.is_none() { + reason = "SERVER_CHECKPOINT_MISSING".into(); + chain_valid = false; } if let Some(old) = &previous { if old.get::("journal_id") != journal_id @@ -132,7 +151,9 @@ pub async fn reconcile( reason = "JOURNAL_ROLLBACK_OR_REPLACEMENT".into(); chain_valid = false; } - if previous_decision != Some(old.get::("decision_id")) { + if previous_decision != Some(old.get::("decision_id")) + && reason != "SERVER_CHECKPOINT_MISSING" + { reason = "CHECKPOINT_CHAIN_MISMATCH".into(); chain_valid = false; } @@ -582,6 +603,31 @@ async fn mark_unknown(tx: &mut Transaction<'_, Postgres>, job: Uuid, reason: &st mod tests { use super::*; #[test] + fn uncertain_request_resolves_only_to_a_known_checkpoint_anchor() { + let applied = Uuid::new_v4(); + let pending = Uuid::new_v4(); + let committed = Uuid::new_v4(); + // Transport cut before server commit: retain the applied anchor. + assert_eq!( + previous_anchor(Some(applied), Some(pending), None), + Some(applied) + ); + // Server committed but its reply was lost: use the newer checkpoint, + // even while the agent also retains its older applied witness. + assert_eq!( + previous_anchor(Some(applied), Some(pending), Some(committed)), + Some(committed) + ); + // An absent pending request cannot invent an anchor for a legacy + // request that discarded its last applied decision. + assert_eq!(previous_anchor(None, Some(pending), None), None); + assert_eq!( + previous_anchor(None, Some(pending), Some(committed)), + Some(committed) + ); + assert_eq!(previous_anchor(Some(applied), None, None), Some(applied)); + } + #[test] fn accepted_is_not_a_new_attempt_or_grant() { let mut local = SyncAttempt { state: "accepted".into(), diff --git a/deploy/cloud_apt_reconcile_upgrade.py b/deploy/cloud_apt_reconcile_upgrade.py new file mode 100644 index 0000000..2aef12c --- /dev/null +++ b/deploy/cloud_apt_reconcile_upgrade.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Preserving helper upgrade for the original APT bound readonly recheck. + +APT execution endpoint stays stopped. General helper snapshots may prove the +existing failed partial postcondition; only existing signed Sync closes it. +No package command, new grant, approval or consumption reset exists here. +""" +import base64 +import hashlib +import json +import os +from pathlib import Path +import uuid + +import cloud_apt_retry as apt +cloud=apt.cloud +require=cloud.require +STATE=cloud.ROOT/'devstate/cloud-apt-recovery' + + +def upgrade(payload): + import fcntl + import sqlite3 + os.umask(0o077) + base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote') + ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper') + provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json' + dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf') + full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n' + 'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n' + 'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n' + 'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n' + 'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n') + reduced=''.join(line+'\n' for line in full.splitlines() if not line.startswith('Environment=RMM_APT_HELPER_')) + def run(args,**kw):return subprocess.run(args,capture_output=True,check=True,timeout=20,**kw).stdout + def sha(data):return hashlib.sha256(data).hexdigest() + def manifest(path): + result={} + for p in path.rglob('*'): + require(not p.is_symlink(),'preserved state symlink') + if p.is_file():result[str(p.relative_to(path))]=sha(p.read_bytes()) + return result + def guard(): + with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db: + require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity') + require({r[0] for r in db.execute('SELECT grant_id FROM consumed')}==set(payload['expected_grants']),'exact preserved grants') + rows=db.execute('SELECT c.grant_id,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall() + require(len(rows)==1 and rows[0][0]==payload['unknown_grant'],'one exact existing unknown') + outcome=json.loads(rows[0][1]);require(outcome['state']=='unknown' and outcome['error_code']=='DPKG_INCONSISTENT','exact known partial outcome') + require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'no sticky faults') + return sha(rows[0][1]) + guard() + require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved') + require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required') + require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required') + original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json' + retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json' + require(sha(original.read_bytes())=='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053' and + sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required') + require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected') + raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True) + require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts') + backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700) + def create(path,data,mode=0o600): + fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode) + with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode) + report=dict(stage='validated',completed=False,package_effects=False,backup=str(backup)) + def save(): + next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json') + save();staged=backup/'helper-next';create(staged,raw,0o755) + run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)]) + run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service']) + fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB) + try: + original_unknown=guard();inode=(ledger.stat().st_dev,ledger.stat().st_ino) + preserved={str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent,base/'active',base/'history',base/'archives',base/'baselines',base/'log',base/'state')} + witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))} + old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata') + general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() + for p in (binary,provider,metadata,dropin): + target=backup/str(p).lstrip('/').replace('/','__') + require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup') + os.rename(p,target) + os.rename(staged,binary);create(provider,code,0o755) + create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644) + inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json'))) + require(inspected==json.loads(general),'general policy unchanged') + require(all(manifest(Path(p))==values for p,values in preserved.items()) and + all(sha(Path(p).read_bytes())==value for p,value in witnesses.items()) and + (ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift') + report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown, + helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save() + finally: + fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd) + run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service']) + report.update(stage='started_bound_readonly_recheck',completed=True);save() + return report + + +def result_evidence(payload): + import sqlite3 + ledger=Path('/var/lib/rmm-helper/consumption.sqlite') + with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db: + row=db.execute('SELECT result FROM consumed WHERE grant_id=?',(payload['grant_id'],)).fetchone() + tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + proof=[] + if 'apt_partial_reconciliations' in tables: + columns=[r[1] for r in db.execute('PRAGMA table_info(apt_partial_reconciliations)')] + proof=[dict(zip(columns,r)) for r in db.execute('SELECT * FROM apt_partial_reconciliations')] + # Opaque witness bytes remain private, JSON-safe for protected file. + for item in proof: + for k,v in list(item.items()): + if isinstance(v,bytes):item[k]=dict(sha256=hashlib.sha256(v).hexdigest(),size=len(v)) + return dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0], + unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0], + faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0], + fault_codes=[r[0] for r in db.execute('SELECT fault FROM helper_sync_meta WHERE fault IS NOT NULL')], + original_result=json.loads(row[0]) if row and row[0] else None,proof=proof, + apt_active=subprocess.run(['systemctl','is-active','rmm-apt-helper.service'],capture_output=True).stdout.strip().decode()) + + +def installation_status(payload): + root=Path('/var/lib/rmm-bootstrap-history') + reports=[] + for p in sorted(root.glob('cloud-readonly-reconcile-*/report.json')): + reports.append(json.loads(p.read_text())) + services={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True).stdout.strip().decode() + for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')} + hashes={name:hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in + ('/usr/local/libexec/rmm-privileged-helper','/usr/lib/rmm-apt-transaction/apt_transaction.py')} + return dict(reports=reports,services=services,hashes=hashes) + + +def main(): + m=cloud.contract;os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True) + checkpoint=m.protected_json(apt.STATE/'apt_install_retry/checkpoint.json');grant=checkpoint['result']['result']['grant_id'] + require(apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact closed-policy unknown gate') + folders=[cloud.ROOT/'devstate/cloud-root/cases'/name for name in ('service_setup','service_start','service_stop','apt_native_install')] + folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')] + expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant] + ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes() + payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant, + old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c', + old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945', + binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(), + helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest()) + report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report) + print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')}) + + +if __name__=='__main__': + try:main() + except Exception as error: + print('CLOUD_APT_RECONCILE_UPGRADE_INCOMPLETE:'+type(error).__name__);raise SystemExit(2) from None diff --git a/deploy/cloud_sync_evidence.py b/deploy/cloud_sync_evidence.py new file mode 100644 index 0000000..2d2e1ad --- /dev/null +++ b/deploy/cloud_sync_evidence.py @@ -0,0 +1,67 @@ +"""Read-only stored checkpoint metadata; no effects or trust-state reset.""" + + +def vm_chain(payload): + import sqlite3 + def wire(raw): + result={};i=0 + def number(): + nonlocal i + value=0;shift=0 + while True: + require(i>3;kind=key&7 + if kind==0:value=number() + elif kind==2: + length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length + else:raise RuntimeError('stored protobuf wire type') + result.setdefault(field,[]).append(value) + return result + def request(raw): + fields=wire(raw);result={} + for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')): + result[k]=fields.get(n,[b''])[0].decode('ascii') + for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0] + return result + def decision(value): + names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation') + return {k:value.get(k) for k in names} + result={} + path=Path('/var/lib/rmm-agent-remote/journal.sqlite') + with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db: + require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity') + names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')] + result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone())) + requests=[] + for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'): + item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None) + if response:item['decision']=decision(json.loads(wire(response)[1][0])) + requests.append(item) + result['agent_requests']=requests + path=Path('/var/lib/rmm-helper/consumption.sqlite') + with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db: + require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity') + names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')] + result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone())) + decisions=[] + for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'): + signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes'])) + decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value))) + result['helper_decisions']=decisions + return result + + +def server_chain(payload): + root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id'])) + cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), + 'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'] + scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'" + query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+")," + "'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at," + "convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready," + "convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));") + raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip() + return json.loads(raw) diff --git a/docs/implementation-status.md b/docs/implementation-status.md index e1ef932..f5752df 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -1,5 +1,49 @@ # Tatsächlicher Implementierungsstand +## Nicht persistierter Sync-Request als MVP-Blocker eingegrenzt — 2026-11-13 + +Vorprüfung des geplanten Readonly-APT-Helperupgrades stoppte **vor jeder +Datei-/Dienständerung**: Agent und Sharedhelper tragen +`CHECKPOINT_CHAIN_MISMATCH`. Installierte Helper-/Providerhashes unverändert, +alle drei Dienste aktiv, keine Upgrade-Backups/Installation angelegt. Gespeicherte +Checkpointmetadaten ausschließlich gelesen, keine Fault-, Ledger- oder +Identitätsrücksetzung. Rootpolicy weiter geschlossen, APT weiterhin unknown. + +Gespeicherte Kette zeigt: Ein lokal dauerhaft vorbereiteter Request wurde +serverseitig nicht committed. Der folgende Request referenziert diesen +unsicheren Vorgänger, lässt aber den tatsächlich bekannten letzten signierten +Anker weg. Der Server findet den Vorgänger nicht, behandelt dies als Kettenfehler +und überschreibt zusätzlich den präziseren Grund mit Chain-Mismatch. Beide +lokalen Journale haben die signierte Ablehnung bereits gespeichert; der Server +hält noch den älteren gültigen Head. Dies ist ein Fehler im aktuellen +MVP-Wiederverbindungspfad, keine nachgelagerte Erweiterung. + +Prävention implementiert: Agent behält bestätigten Anker **und** unsicheren +Vorgänger; Server benutzt einen tatsächlich gespeicherten Vorgänger, sonst nur +den expliziten noch passenden bestätigten Anker. Exakter Vergleich mit dem +autoritativen Serverhead bleibt Pflicht. Fehlender/veralteter Anker wird +weiter verweigert; Fehlergrund bleibt präzise. **18 Agent-Reconcile- und +2 Server-Sync-Tests**, Format und Clippy mit `-D warnings` bestanden. +Diese Prävention ist noch nicht auf Cloud ausgeliefert und löscht keine Sperre. + +Zusätzlich gebundener Readonly-APT-Verifier implementiert: vollständige Baseline +plus ausschließlich Fixture1.0 `unpacked`, original signierter Auftrag, +Kontext/Hook/APT0 und beide unveränderten Marker. Immutable alte Unknownbytes +und neuer Prüfbeleg werden angehängt; nur bestehender signierter RootSync darf +danach den einzelnen Verbrauch schließen. **70 Python-Tests** sowie +**31 Helper-Library / 3 Binary / 20 RootSync-Tests**, Format/Clippy/Releasebuild +bestanden. **Noch keine native Verifierabnahme**, weil die Kettensperre vorher +korrekt blockierte. Kein Paket-/Configure-/Retryaufruf ausgeführt. + +Nächster zwingender Schritt ist eine getrennt signierte, exakt an diese +gespeicherten Zeugen gebundene Kontrollquittung: vorhandene wirkungslose +Ablehnung als gemeinsamen Head ausrichten, alte Sperre und alle Zeugen dauerhaft +archivieren, weder Outcome/Grant noch Bereit-Lease ändern. Danach gewöhnlicher +Sync und erst dann der gebundene APT-Abgleich. Keine manuelle SQL-Faultlöschung. +Private Metadaten: `devstate/cloud-apt-recovery/sync-chain-{vm,server}.json`. + +**MVP-Prüfung: weiterhin offen; Kettenreparatur hat Vorrang vor Paket-Recovery.** + ## Linux-TLS-Lebenszyklus nativ geprüft; MVP bleibt offen — 2026-11-13 Auf derselben erlaubten Cloud-Linux-VM eine **separate ausschließlich passive diff --git a/docs/mvp-gates.md b/docs/mvp-gates.md index 3eb3683..08234c7 100644 --- a/docs/mvp-gates.md +++ b/docs/mvp-gates.md @@ -39,6 +39,13 @@ bestanden. Dauerhafte APT-Unknown-Evidenz unverändert, keine Jobs, Testservice gestoppt. M1 braucht weiterhin tatsächlichen WSS-Ausfall/Fallback; Gate 0 bleibt die nächste blockierende Verwaltungsstrecke. MVP insgesamt weiterhin offen. +**Aktueller Gate-0-Blocker:** Upgradevorprüfung fand in Agent und Sharedhelper +eine gespeicherte `CHECKPOINT_CHAIN_MISMATCH`-Sperre. Nicht serverseitig +persistierter Vorgänger plus verworfener bestätigter Anker ist anhand der +gespeicherten Kette eingegrenzt; Prävention lokal geprüft, nicht deployt. +Vor APT-Abgleich ist eine exakt signierte Kontrollquittung mit erhaltener +Historie nötig. Keine direkte Faultlöschung oder pauschale Startfreigabe. + Alle folgenden Schritte sind **geplant, nicht durch dieses Dokument ausgeführt**. Jeder Ausführungsjob braucht weiterhin seinen eigenen festen WebUI-genehmigten Plan. Zulässige dev/test-Browserbedienung bleibt sichtbar simuliert und auditiert. diff --git a/tests/apt_transaction/test_partial_reconciliation.py b/tests/apt_transaction/test_partial_reconciliation.py new file mode 100644 index 0000000..23b52b0 --- /dev/null +++ b/tests/apt_transaction/test_partial_reconciliation.py @@ -0,0 +1,118 @@ +"""Original-bound readonly proof only; all native package queries mocked.""" +from dataclasses import replace +from pathlib import Path +import tempfile +import uuid +from unittest import TestCase +from unittest.mock import patch + +from test_guard import apt +from test_native_acceptance import signed_job, prechecks +import test_native_retry as retry_fixture + + +class PartialReconciliationTests(TestCase): + def scenario(self, variation=None): + with tempfile.TemporaryDirectory(dir='/tmp/opencode') as temp: + base = Path(temp) + with patch.object(apt, 'BASE', base), patch.object(apt, 'ACTIVE', base/'active'), \ + patch.object(apt, 'secure_path', side_effect=lambda path, **kwargs: path), \ + patch.object(apt, 'sealed_read', side_effect=lambda path, *args, **kwargs: + b'1.0\n' if path == Path('/usr/share/rmm-mvp-test/version') else path.read_bytes()): + tx, ctx, action, old, previous, original = retry_fixture.NativeRetryTests().setup_history(base) + ctx = replace(ctx, job_id=str(uuid.uuid4()), authorization_id=str(uuid.uuid4())) + job_bytes = apt.canonical(signed_job(ctx, action)) + ctx = replace(ctx, job_payload_sha256=apt.digest(job_bytes)) + active = base/'active' + (active/'context.json').write_bytes(apt.canonical(ctx.__dict__)) + (active/'signed-action.json').write_bytes(job_bytes) + (active/'apt.conf').write_bytes(apt.apt_config()) + # setup_history's manifest uses the current config only as a + # synthetic historical fixture; actual VM history may differ. + path, binding = apt.bootstrap_binding(tx, ctx, action) + apt.durable_once(path, binding) + files = { + 'launch.json': {'transaction_sha256': ctx.transaction_sha256, + 'effect_armed_ledger_id': ctx.effect_armed_ledger_id}, + 'native-prechecks.json': prechecks(ctx), + 'sandbox-ready.json': {'sandbox_version': 1, 'transaction_sha256': ctx.transaction_sha256}, + 'sandbox-result.json': {'transaction_sha256': ctx.transaction_sha256, + 'apt_exit_code': 0, 'reboot_required': False}, + 'hook-armed.json': {'armed_at_ms': 2000, 'effect_armed_ledger_id': ctx.effect_armed_ledger_id, + 'grant_id': ctx.grant_id, 'transaction_sha256': ctx.transaction_sha256, + 'protocol_sha256': 'e'*64}, + } + for name, value in files.items(): (active/name).write_bytes(apt.canonical(value)) + fields = ('transaction_sha256 job_payload_sha256 effect_armed_ledger_id grant_id ' + 'job_id attempt_id authorization_id plan_sha256') + request = {key: getattr(ctx, key) for key in fields.split()} + observed = {'rmm-mvp-test:all': {'version': '1.0', 'status': 'install ok unpacked'}} + expected_files = {str(p.relative_to(base)): p.read_bytes() for p in base.rglob('*') if p.is_file()} + if variation == 'foreign-lock': pass + elif variation == 'different-package': observed['other:amd64'] = {'version': '1', 'status': 'install ok installed'} + elif variation == 'installed': observed['rmm-mvp-test:all']['status'] = 'install ok installed' + elif variation == 'other-version': observed['rmm-mvp-test:all']['version'] = '1.1' + elif variation == 'missing-hook': (active/'hook-armed.json').unlink() + elif variation == 'false-exit': + files['sandbox-result.json']['apt_exit_code'] = False + (active/'sandbox-result.json').write_bytes(apt.canonical(files['sandbox-result.json'])) + elif variation == 'apt-failed': + files['sandbox-result.json']['apt_exit_code'] = 100 + (active/'sandbox-result.json').write_bytes(apt.canonical(files['sandbox-result.json'])) + elif variation == 'wrong-context': request['grant_id'] = str(uuid.uuid4()) + elif variation == 'config-tamper': (active/'apt.conf').write_bytes(b'changed') + elif variation == 'accepted': apt.record_path(tx).write_bytes(b'{}') + elif variation == 'precheck': + files['native-prechecks.json']['checks']['readonly_controlpaths'] = False + (active/'native-prechecks.json').write_bytes(apt.canonical(files['native-prechecks.json'])) + before = {str(p.relative_to(base)): p.read_bytes() for p in base.rglob('*') if p.is_file()} + with patch.object(apt.os, 'geteuid', return_value=0), \ + patch.object(Path, 'read_text', return_value=ctx.boot_id if variation != 'boot' else 'different-boot'), \ + patch.object(apt, 'observe_host_frontend_lock', return_value=variation == 'foreign-lock'), \ + patch.object(apt, 'status_fingerprint', side_effect=['a'*64, 'b'*64] if variation == 'changed' else None, + return_value='a'*64), \ + patch.object(apt, 'partial_installed_snapshot', return_value=observed), \ + patch.object(apt, 'verify_archive') as archive, \ + patch.object(apt.subprocess, 'Popen', side_effect=AssertionError('No effect process permitted')): + if variation is not None: + with self.assertRaises((apt.Veto, OSError, ValueError)): + apt.verify_partial(request) + else: + result = apt.verify_partial(request) + self.assertEqual(result['effect_state'], 'known_failed') + self.assertEqual(result['code'], 'APT_PARTIAL_UNCONFIGURED_VERIFIED') + proof = result['proof'] + self.assertEqual(proof['snapshot'], observed) + self.assertEqual(proof['observed_snapshot_sha256'], apt.digest(apt.canonical(observed))) + self.assertTrue(proof['partial_effect']) + self.assertFalse(proof['postconditions_passed']) + self.assertFalse(proof['reboot_required']) + self.assertEqual(proof['fixture_payload_sha256'], apt.digest(b'1.0\n')) + self.assertEqual({key: proof[key] for key in fields.split()}, request) + names = ('context.json', 'signed-action.json', 'expectation.json', 'baseline.json', + 'launch.json', 'native-prechecks.json', 'sandbox-ready.json', 'sandbox-result.json', 'hook-armed.json') + raw = {name: (active/name).read_bytes() for name in names} + raw[original.name] = original.read_bytes(); raw[path.name] = path.read_bytes() + self.assertEqual(proof['original_evidence_sha256'], apt.digest(apt.canonical({n: apt.digest(v) for n,v in raw.items()}))) + archive.assert_called_once_with(tx.packages[0]) + after = {str(p.relative_to(base)): p.read_bytes() for p in base.rglob('*') if p.is_file()} + self.assertEqual(before, after, 'verification must never create/change/delete evidence') + if variation is None: self.assertEqual(expected_files, after) + + def test_original_bound_partial_is_failed_not_success_and_has_no_writes(self): + self.scenario() + + def test_uncertain_or_mismatched_partial_state_never_closes_unknown(self): + for variation in ('foreign-lock','different-package','installed','other-version','missing-hook', + 'false-exit','apt-failed','wrong-context','config-tamper','accepted','precheck','boot','changed'): + with self.subTest(variation=variation): self.scenario(variation) + + def test_partial_snapshot_does_not_relax_other_packages_or_ordinary_snapshot(self): + query = b'rmm-mvp-test\tall\t1.0\tinstall ok unpacked\n' + with patch.object(apt, 'run_read', return_value=query): + self.assertEqual(apt.partial_installed_snapshot()['rmm-mvp-test:all']['status'], 'install ok unpacked') + with self.assertRaisesRegex(apt.Veto, 'DPKG_INCONSISTENT'): apt.installed_snapshot() + for data in (query+query, b'other\tamd64\t1\tinstall ok unpacked\n', + b'rmm-mvp-test\tall\t1.1\tinstall ok unpacked\n', b'malformed\n'): + with self.subTest(data=data), patch.object(apt, 'run_read', return_value=data), self.assertRaises(apt.Veto): + apt.partial_installed_snapshot() diff --git a/tools/apt-transaction/apt_transaction.py b/tools/apt-transaction/apt_transaction.py index 9a17452..54dd3c1 100644 --- a/tools/apt-transaction/apt_transaction.py +++ b/tools/apt-transaction/apt_transaction.py @@ -1232,6 +1232,161 @@ def verify_after(transaction, baseline): return evidence +def partial_installed_snapshot(): + """Readonly original-job verification, never a general planning baseline. + + Only this exact script/trigger/dependency-free fixture may be unpacked. + Normal inventory/planning/execution keep rejecting inconsistent dpkg state. + """ + data = run_read(['/usr/bin/dpkg-query', '--show', + '--showformat=${Package}\t${Architecture}\t${Version}\t${Status}\n'], + limit=32 * MAX_INPUT) + snapshot = {} + for line in data.decode('utf-8', 'strict').splitlines(): + fields = line.split('\t') + require(len(fields) == 4, 'DPKG_QUERY_FORMAT') + name, arch, version, status = fields + key = name + ':' + arch + require(key not in snapshot, 'DPKG_QUERY_DUPLICATE') + ordinary = status in ('install ok installed', 'deinstall ok config-files', + 'purge ok not-installed', 'unknown ok not-installed') + partial = key == 'rmm-mvp-test:all' and version == '1.0' and status == 'install ok unpacked' + require(ordinary or partial, 'PARTIAL_OTHER_INCONSISTENT_PACKAGE') + snapshot[key] = {'version': version, 'status': status} + return snapshot + + +def verify_partial(request): + """Recheck ONLY the fixed original job's postcondition, with no mutation. + + RootHelper independently verifies its historical signed Request and the + unresolved consumed row first. These caller bindings are not authority to + execute: this mode has no APT/dpkg effect call or writable metadata path. + Current code/config/runtime need not equal the historical execution runtime; + original signed bytes and original apt.conf remain immutable evidence. + """ + require(os.geteuid() == 0, 'ROOT_REQUIRED') + fields = ('transaction_sha256 job_payload_sha256 effect_armed_ledger_id grant_id ' + 'job_id attempt_id authorization_id plan_sha256') + exact(request, fields) + for key in ('transaction_sha256', 'job_payload_sha256', 'plan_sha256'): + token(request[key], HASH, 'PARTIAL_REQUEST_HASH') + token(request['effect_armed_ledger_id'], IDENT, 'PARTIAL_REQUEST_LEDGER') + for key in ('grant_id', 'job_id', 'attempt_id', 'authorization_id'): + value = request[key] + require(type(value) is str and str(uuid.UUID(value)) == value and + uuid.UUID(value).int != 0, 'PARTIAL_REQUEST_ID') + secure_path(BASE, directory=True, private=True) + secure_path(ACTIVE, directory=True, private=True) + names = ('context.json', 'signed-action.json', 'expectation.json', 'baseline.json', + 'launch.json', 'native-prechecks.json', 'sandbox-ready.json', + 'sandbox-result.json', 'hook-armed.json') + raw = {name: sealed_read(ACTIVE / name) for name in names} + ctx = VerifiedHelperContext.parse(json_bytes(raw['context.json'])) + require(ctx.action == 'apt-native-acceptance' and ctx.execution_profile == 'dev-test' and + all(getattr(ctx, key) == request[key] for key in fields.split()), 'PARTIAL_CONTEXT_BINDING') + require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == ctx.boot_id, + 'PARTIAL_BOOT_CHANGED') + require(digest(raw['signed-action.json']) == ctx.job_payload_sha256 and + digest(raw['expectation.json']) == ctx.transaction_sha256, 'PARTIAL_SIGNED_BYTES') + tx = ResolvedAptTransaction.parse(json_bytes(raw['expectation.json'])) + fixture_scope(tx) + require(tx.recipe_id == 'rmm-apt-fixture-install-1.0', 'PARTIAL_FIXED_INSTALL_ONLY') + baseline = json_bytes(raw['baseline.json']) + require(type(baseline) is dict and raw['baseline.json'] == canonical(baseline) and + digest(raw['baseline.json']) == tx.baseline_sha256 and + 'rmm-mvp-test:all' not in baseline, 'PARTIAL_ORIGINAL_BASELINE') + job = json_bytes(raw['signed-action.json']) + exact(job, 'schema_version tenant_id asset_id agent_id enrollment_generation assignment_version ' + 'job_id attempt_id action execution_identity not_before start_before max_runtime_seconds max_output_bytes') + require(type(job['schema_version']) is int and job['schema_version'] == 1 and + job['job_id'] == ctx.job_id and job['attempt_id'] == ctx.attempt_id and + job['execution_identity'] == 'root', 'PARTIAL_ORIGINAL_JOB') + outer = job['action'] + exact(outer, 'action recipe_id recipe_version apt_manifest') + require(outer['action'] == 'apt_native_acceptance' and outer['recipe_id'] == tx.recipe_id and + outer['recipe_version'] == tx.recipe_version, 'PARTIAL_ORIGINAL_ACTION') + action = outer['apt_manifest'] + exact(action, 'type schema_version action execution_profile transaction_sha256 ' + 'resolved_apt_transaction provider_sha256 hook_sha256 apt_config_sha256 ' + 'runtime_sha256 test_result_id transaction_bytes native_retry') + require(action['type'] == 'rmm.apt.helper-action' and type(action['schema_version']) is int and + action['schema_version'] == 3 and action['action'] == ctx.action and + action['execution_profile'] == ctx.execution_profile and + action['transaction_sha256'] == ctx.transaction_sha256 and + canonical(action['resolved_apt_transaction']) == canonical(json_bytes(raw['expectation.json'])) and + type(action['transaction_bytes']) is list and + all(type(n) is int and 0 <= n <= 255 for n in action['transaction_bytes']) and + bytes(action['transaction_bytes']) == raw['expectation.json'], 'PARTIAL_ORIGINAL_MANIFEST') + for key in ('provider_sha256', 'hook_sha256', 'apt_config_sha256', 'runtime_sha256'): + token(action[key], HASH, 'PARTIAL_ORIGINAL_RUNTIME_HASH') + token(action['test_result_id'], IDENT, 'PARTIAL_ORIGINAL_TEST_ID') + require(digest(sealed_read(ACTIVE / 'apt.conf')) == action['apt_config_sha256'], + 'PARTIAL_ORIGINAL_CONFIG') + # This reuses only immutable historical binding checks, never the current + # executable gate or an acceptance publication. It also verifies both once + # markers and the original failed no-handoff history referenced by retry. + check_bootstrap_attempt(tx, ctx, action) + for prefix in ('bootstrap-attempt-', 'bootstrap-retry-'): + name = prefix + tx.recipe_id + '.json' + raw[name] = sealed_read(BASE / name) + for prefix in ('acceptance-', 'pending-acceptance-'): + require(not os.path.lexists(BASE / (prefix + tx.recipe_id + '.json')), + 'PARTIAL_ACCEPTANCE_ALREADY_EXISTS') + require(canonical(json_bytes(raw['launch.json'])) == canonical({ + 'transaction_sha256': ctx.transaction_sha256, + 'effect_armed_ledger_id': ctx.effect_armed_ledger_id}), 'PARTIAL_LAUNCH_BINDING') + prechecks = {'schema_version': 1, 'transaction_sha256': ctx.transaction_sha256, + 'job_payload_sha256': ctx.job_payload_sha256, 'boot_id': ctx.boot_id, + 'checks': {'namespaces_caps_nnp_seccomp': True, 'readonly_controlpaths': True, + 'private_tmpfs_sockets': True, + 'control_syscalls': {'invalid_reboot_magic': 'EPERM', + 'invalid_kexec_load_flags': 'EPERM', + 'invalid_kexec_file_fds_flags': 'EPERM'}, + 'frontend_lock': 'native-apt-check-conflict'}} + require(canonical(json_bytes(raw['native-prechecks.json'])) == canonical(prechecks) and + canonical(json_bytes(raw['sandbox-ready.json'])) == canonical({ + 'sandbox_version': 1, 'transaction_sha256': ctx.transaction_sha256}) and + canonical(json_bytes(raw['sandbox-result.json'])) == canonical({ + 'transaction_sha256': ctx.transaction_sha256, 'apt_exit_code': 0, + 'reboot_required': False}), 'PARTIAL_NATIVE_COMPLETION_REQUIRED') + marker = json_bytes(raw['hook-armed.json']) + exact(marker, 'armed_at_ms effect_armed_ledger_id grant_id protocol_sha256 transaction_sha256') + require(marker['effect_armed_ledger_id'] == ctx.effect_armed_ledger_id and + marker['grant_id'] == ctx.grant_id and marker['transaction_sha256'] == ctx.transaction_sha256 and + type(marker['armed_at_ms']) is int and + ctx.not_before_ms <= marker['armed_at_ms'] < ctx.start_before_ms, 'PARTIAL_HOOK_BINDING') + token(marker['protocol_sha256'], HASH, 'PARTIAL_HOOK_PROTOCOL_HASH') + require(not observe_host_frontend_lock() and + not observe_host_frontend_lock(Path('/var/lib/dpkg/lock')), 'FOREIGN_PACKAGE_LOCK') + fingerprint = status_fingerprint() + observed = partial_installed_snapshot() + expected = dict(baseline) + expected['rmm-mvp-test:all'] = {'version': '1.0', 'status': 'install ok unpacked'} + require(observed == expected, 'PARTIAL_UNEXPECTED_PACKAGE_CHANGE') + verify_archive(tx.packages[0]) + payload = sealed_read(Path('/usr/share/rmm-mvp-test/version'), limit=32) + require(payload == b'1.0\n', 'PARTIAL_FIXTURE_PAYLOAD') + require(status_fingerprint() == fingerprint and partial_installed_snapshot() == observed and + not observe_host_frontend_lock() and + not observe_host_frontend_lock(Path('/var/lib/dpkg/lock')), 'PARTIAL_STATE_CHANGED') + # Re-read the historical witnesses too: a stable live package projection + # cannot compensate for changed/tampered original evidence. + for name, data in raw.items(): + path = ACTIVE / name if name in names else BASE / name + require(sealed_read(path) == data, 'PARTIAL_HISTORICAL_EVIDENCE_CHANGED') + proof = dict(schema_version=1, type='rmm.apt.partial-reconciliation', **request, + original_evidence_sha256=digest(canonical({name: digest(data) for name, data in raw.items()})), + original_baseline_sha256=tx.baseline_sha256, + observed_snapshot_sha256=digest(canonical(observed)), status_fingerprint=fingerprint, + fixture_payload_sha256=digest(payload), snapshot=observed, + checked_at_ms=time.time_ns() // 1000000, partial_effect=True, + postconditions_passed=False, reboot_required=False) + result = dict(effect_state='known_failed', code='APT_PARTIAL_UNCONFIGURED_VERIFIED', proof=proof) + require(len(canonical(result)) <= MAX_INPUT, 'PARTIAL_PROOF_LIMIT') + return result + + def verified_no_change(transaction, baseline, context): """Independently verify the full selected postcondition, never run APT.""" fixture_scope(transaction) # Cannot promote unknown catalog recipes. @@ -1690,6 +1845,8 @@ def main(): probe_worker() elif mode == 'readiness-status': sys.stdout.buffer.write(canonical(readiness_status()) + b'\n') + elif mode == 'verify-partial': + sys.stdout.buffer.write(canonical(verify_partial(json_bytes(sys.stdin.buffer.read(MAX_INPUT + 1)))) + b'\n') elif mode == 'verify-controller': verify_controller_profile() elif mode == 'inspect-sandbox':