From 359233c64d7ff5061811b3115a41721b946bbb02 Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Fri, 13 Nov 2026 17:43:38 +0100 Subject: [PATCH] fix: repair proven unsent sync chain with signed archived receipt --- Rust_RMM_Implementierungsspezifikation.md | 17 +- crates/agent/Cargo.toml | 1 + crates/agent/src/bin/rmm-checkpoint-repair.rs | 497 +++++++++++ crates/domain/src/checkpoint_repair.rs | 139 ++++ crates/domain/src/lib.rs | 1 + .../src/checkpoint_repair.rs | 458 +++++++++++ crates/privileged-helper/src/lib.rs | 1 + crates/privileged-helper/src/reconcile.rs | 143 ++++ crates/privileged-helper/tests/reconcile.rs | 26 + crates/server/src/agent_http.rs | 56 +- crates/server/src/checkpoint_repair.rs | 772 ++++++++++++++++++ crates/server/src/dev_delivery_faults.rs | 570 +++++++++++++ crates/server/src/gateway.rs | 34 +- crates/server/src/main.rs | 24 + deploy/cloud_apt_reconcile_upgrade.py | 15 +- deploy/cloud_checkpoint_repair.py | 158 ++++ deploy/cloud_server_upgrade.py | 6 +- docs/implementation-status.md | 45 + docs/mvp-gates.md | 14 +- docs/project-decisions.md | 14 +- tests/checkpoint_repair_database.py | 69 ++ 21 files changed, 3030 insertions(+), 30 deletions(-) create mode 100644 crates/agent/src/bin/rmm-checkpoint-repair.rs create mode 100644 crates/domain/src/checkpoint_repair.rs create mode 100644 crates/privileged-helper/src/checkpoint_repair.rs create mode 100644 crates/server/src/checkpoint_repair.rs create mode 100644 crates/server/src/dev_delivery_faults.rs create mode 100644 deploy/cloud_checkpoint_repair.py create mode 100644 tests/checkpoint_repair_database.py diff --git a/Rust_RMM_Implementierungsspezifikation.md b/Rust_RMM_Implementierungsspezifikation.md index cab7adf..5b3f70d 100644 --- a/Rust_RMM_Implementierungsspezifikation.md +++ b/Rust_RMM_Implementierungsspezifikation.md @@ -1,6 +1,6 @@ # Rust RMM: Architektur und Implementierungsspezifikation -Version 1.9 · 2026-11-13 · Sprache: Deutsch +Version 1.10 · 2026-11-13 · Sprache: Deutsch **Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam. @@ -66,6 +66,21 @@ Deploymentdaten, offene Konfiguration und reale Nachweise: ### 0.1 Auftrag an die implementierende KI +**Technische Präzisierung Version 1.10 — wirkungslose Kontrollkettenreparatur:** +Ein nachweislich nicht serverseitig persistierter Sync-Request darf im expliziten +Entwicklungsprofil über ein eng begrenztes Root-Bootstrapwerkzeug abgeglichen +werden. Server und beide lokalen Verifier prüfen die vollständigen erhaltenen +Journale, ursprünglichen Signaturen, exakten Geräte-/Boot-/Checkpointbindungen +und den unveränderten offenen Verbrauch. Nur die bereits gespeicherte signierte +Ablehnung wird zum monotonen gemeinsamen Kontrollhead; eine kurzlebige, +domänengetrennte Signatur bindet sämtliche Zeugen. Lokale alte Faults und +Quittung werden dauerhaft archiviert. Keine Bereitschaft, Genehmigung, Jobs, +Grants, Verbräuche oder Outcomes entstehen oder ändern sich dadurch. Kein +manuelles Löschen einer Sperre, kein Journal-/Identitätsreset. Erst der normale +signierte Sync und eine unabhängig gebundene Nachprüfung dürfen anschließend +ein unbekanntes Ausführungsergebnis klassifizieren. Dies konkretisiert den +bestehenden Abgleichvertrag und ist keine Ausnahme von der WebUI-Jobgenehmigung. + **Präzisierung Version 1.9, 2026-11-13 — expliziter neuer APT-Testversuch:** Ein bekannter Fehler vor der vertrauenswürdigen Hookübergabe darf nach belegtem unverändertem vollständigem Paketbestand durch genau einen neuen festen, diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index fed8949..284ad3e 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -3,6 +3,7 @@ name = "rmm-agent" version.workspace = true edition.workspace = true license.workspace = true +default-run = "rmm-agent" [dependencies] anyhow.workspace = true chrono.workspace = true diff --git a/crates/agent/src/bin/rmm-checkpoint-repair.rs b/crates/agent/src/bin/rmm-checkpoint-repair.rs new file mode 100644 index 0000000..c5194d6 --- /dev/null +++ b/crates/agent/src/bin/rmm-checkpoint-repair.rs @@ -0,0 +1,497 @@ +//! Narrow, root-only development bootstrap. No job execution or ready lease. +use anyhow::{Context, Result, ensure}; +use fs2::FileExt; +use prost::Message; +use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload}; +use rmm_privileged_helper::{Binding, checkpoint_repair as repair}; +use rmm_protocol::{Envelope, Payload, SyncRequest, SyncResponse}; +use rusqlite::{Connection, OpenFlags, OptionalExtension}; +use std::{ + fs::{self, OpenOptions}, + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::Path, +}; +const AGENT: &str = "/var/lib/rmm-agent-remote"; +const HELPER: &str = "/var/lib/rmm-helper"; +fn read_private(path: &Path, owner: u32, limit: usize) -> Result> { + rmm_privileged_helper::private_path(path, owner, false)?; + let m = fs::metadata(path)?; + ensure!(m.len() <= limit as u64, "REPAIR_FILE_SIZE"); + Ok(fs::read(path)?) +} +fn open_existing(path: &Path, owner: u32) -> Result { + rmm_privileged_helper::private_path(path, owner, false)?; + // Require the existing DELETE journal mode. This bootstrap never opens WAL + // state or creates privileged sidecars in the unprivileged agent directory. + for suffix in ["-wal", "-shm", "-journal"] { + ensure!( + !Path::new(&format!("{}{suffix}", path.display())).try_exists()?, + "REPAIR_DATABASE_SIDECAR_PRESENT" + ); + } + let db = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + let mode: String = db.query_row("PRAGMA journal_mode", [], |v| v.get(0))?; + ensure!(mode == "delete", "REPAIR_DELETE_JOURNAL_REQUIRED"); + db.execute_batch( + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=OFF; PRAGMA trusted_schema=OFF;", + )?; + Ok(db) +} +fn signed(raw: &[u8]) -> Result { + let r = SyncResponse::decode(raw)?; + ensure!(r.encode_to_vec() == raw, "REPAIR_NONCANONICAL_RESPONSE"); + Ok(SignedDecision { + decision_bytes: r.decision_bytes, + signature: r.signature, + }) +} +fn check_response( + binding: &Binding, + request: &SyncRequest, + envelope: &Envelope, + response: &SignedDecision, +) -> Result { + let d = repair::verify_historical_decision(binding, response)?; + ensure!( + d.request_id.to_string() == request.request_id + && d.request_hash == hash(&request.encode_to_vec()) + && d.journal_id.to_string() == request.journal_id + && d.checkpoint == request.checkpoint + && d.connection_generation == envelope.connection_generation + && d.boot_id.to_string() == envelope.boot_id + && d.helper_journal_id + .map(|id| id.to_string()) + .unwrap_or_default() + == request.helper_journal_id + && d.helper_checkpoint == request.helper_checkpoint, + "REPAIR_AGENT_SIGNED_REQUEST_BINDING" + ); + Ok(d) +} +fn agent_history(db: &Connection, binding: &Binding) -> Result> { + let mut s = db.prepare( + "SELECT request,envelope,generation,response FROM reconcile_requests ORDER BY rowid", + )?; + let rows = s + .query_map([], |v| { + Ok(( + v.get::<_, Vec>(0)?, + v.get::<_, Vec>(1)?, + v.get::<_, u64>(2)?, + v.get::<_, Option>>(3)?, + )) + })? + .collect::>>()?; + ensure!(rows.len() <= 100000, "REPAIR_AGENT_HISTORY_LIMIT"); + let mut result = vec![]; + for (raw, outer, generation, response) in rows { + ensure!( + raw.len() <= rmm_domain::MAX_MESSAGE && outer.len() <= rmm_domain::MAX_MESSAGE, + "REPAIR_REQUEST_SIZE" + ); + let request = SyncRequest::decode(raw.as_slice())?; + let envelope = Envelope::decode(outer.as_slice())?; + ensure!( + request.encode_to_vec() == raw + && envelope.encode_to_vec() == outer + && envelope.payload == Some(Payload::SyncRequest(request.clone())) + && envelope.agent_id == binding.agent_id.to_string() + && envelope.enrollment_generation == binding.enrollment_generation + && envelope.connection_generation == generation + && envelope.sequence == request.checkpoint + && request.assignment_version == binding.assignment_version, + "REPAIR_AGENT_DURABLE_REQUEST_BINDING" + ); + if let Some(response) = response { + let response = signed(&response)?; + check_response(binding, &request, &envelope, &response)?; + result.push((request, response)); + } + } + let mut offers = db.prepare("SELECT job,attempt,offer FROM job_attempts")?; + for row in offers.query_map([], |v| { + Ok(( + v.get::<_, String>(0)?, + v.get::<_, String>(1)?, + v.get::<_, Vec>(2)?, + )) + })? { + let (job, attempt, bytes) = row?; + let offer = rmm_protocol::JobOffer::decode(bytes.as_slice())?; + let payload = binding.job(&offer.payload_bytes)?; + ensure!( + offer.job_id == job + && offer.attempt_id == attempt + && payload.job_id.to_string() == job + && payload.attempt_id.to_string() == attempt + && offer.job_payload_hash == hash(&offer.payload_bytes), + "REPAIR_AGENT_OFFER_BINDING" + ); + } + Ok(result) +} +fn combined_inventory(agent: &Connection, consumption: &Connection) -> Result { + let mut value = repair::inventory(agent, "reconcile_meta")?; + value + .as_object_mut() + .context("REPAIR_AGENT_INVENTORY")? + .insert( + "agent_consumption".into(), + repair::inventory(consumption, "")?, + ); + Ok(value) +} +fn evidence( + agent: &Connection, + consumption: &Connection, + helper: &Connection, + binding: &Binding, +) -> Result { + rmm_privileged_helper::reconcile::verify_checkpoint_repair_history(helper, binding, None)?; + let history = agent_history(agent, binding)?; + let (head, fault, pending): (String, Option, Option) = agent.query_row( + "SELECT last_decision,fault,pending_request FROM reconcile_meta WHERE id=1", + [], + |v| Ok((v.get(0)?, v.get(1)?, v.get(2)?)), + )?; + ensure!( + fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH") && pending.is_none(), + "REPAIR_AGENT_EXACT_FAULT_REQUIRED" + ); + let (y, denial) = history + .iter() + .find(|(_, r)| { + serde_json::from_slice::(&r.decision_bytes) + .is_ok_and(|d| d.decision_id.to_string() == head) + }) + .context("REPAIR_REJECTION_MISSING")?; + let raw_x: Vec = agent.query_row( + "SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL", + [&y.previous_request_id], + |v| v.get(0), + )?; + let x = SyncRequest::decode(raw_x.as_slice())?; + let (_, a) = history + .iter() + .find(|(_, r)| { + serde_json::from_slice::(&r.decision_bytes) + .is_ok_and(|d| d.decision_id.to_string() == x.previous_decision_id) + }) + .context("REPAIR_PREVIOUS_VALID_MISSING")?; + Ok(Evidence { + schema_version: 1, + tenant_id: binding.tenant_id, + asset_id: binding.asset_id, + agent_id: binding.agent_id, + enrollment_generation: binding.enrollment_generation, + assignment_version: binding.assignment_version, + agent_inventory: combined_inventory(agent, consumption)?, + helper_inventory: repair::inventory(helper, "helper_sync_meta")?, + uncommitted_request: raw_x, + rejected_request: y.encode_to_vec(), + rejected_response: denial.clone(), + previous_valid_response: a.clone(), + }) +} +fn verify_agent( + agent: &Connection, + consumption: &Connection, + binding: &Binding, + signed: &SignedReceipt, + e: &Evidence, +) -> Result<()> { + let r = repair::verify_receipt(binding, signed)?; + let _ = agent_history(agent, binding)?; + ensure!( + e.schema_version == 1 + && e.tenant_id == binding.tenant_id + && e.asset_id == binding.asset_id + && e.agent_id == binding.agent_id + && e.enrollment_generation == binding.enrollment_generation + && e.assignment_version == binding.assignment_version, + "REPAIR_EVIDENCE_IDENTITY" + ); + let actual = combined_inventory(agent, consumption)?; + ensure!( + actual == e.agent_inventory && inventory_sha256(&actual)? == r.agent_inventory_sha256, + "REPAIR_AGENT_INVENTORY" + ); + let (journal,checkpoint,high,head,pending,helperid,helpercp,fault):(String,u64,u64,String,Option,String,u64,Option)=agent.query_row("SELECT journal_id,checkpoint,high_sequence,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|v|Ok((v.get(0)?,v.get(1)?,v.get(2)?,v.get(3)?,v.get(4)?,v.get(5)?,v.get(6)?,v.get(7)?)))?; + let counter: u64 = agent.query_row("SELECT seq FROM counter WHERE id=1", [], |v| v.get(0))?; + ensure!( + journal == r.agent_journal_id.to_string() + && checkpoint == r.agent_checkpoint + && counter >= high + && counter >= checkpoint + && head == r.rejected_decision_id.to_string() + && pending.is_none() + && helperid == r.helper_journal_id.to_string() + && helpercp == r.helper_checkpoint + && (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH") + || (fault.is_none() && repair::already_applied(agent, signed)?)), + "REPAIR_AGENT_HEAD" + ); + let y = SyncRequest::decode(e.rejected_request.as_slice())?; + let x = SyncRequest::decode(e.uncommitted_request.as_slice())?; + let stored_y: (Vec, Vec) = agent.query_row( + "SELECT request,response FROM reconcile_requests WHERE id=?1", + [&y.request_id], + |v| Ok((v.get(0)?, v.get(1)?)), + )?; + let stored_x: Option> = agent + .query_row( + "SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL", + [&x.request_id], + |v| v.get(0), + ) + .optional()?; + let d = repair::verify_historical_decision(binding, &e.rejected_response)?; + let a = repair::verify_historical_decision(binding, &e.previous_valid_response)?; + ensure!( + stored_y.0 == e.rejected_request + && signed_response_eq(&stored_y.1, &e.rejected_response)? + && stored_x.as_deref() == Some(e.uncommitted_request.as_slice()) + && d.decision_id == r.rejected_decision_id + && hash(&e.rejected_response.decision_bytes) == r.rejected_decision_sha256 + && d.reason == r.fault + && !d.ready + && !d.helper_ready + && !d.attempts.iter().any(|v| v.unlock) + && a.decision_id == r.previous_valid_decision_id + && hash(&e.previous_valid_response.decision_bytes) == r.previous_valid_decision_sha256 + && hash(&e.rejected_request) == r.rejected_request_sha256 + && y.request_id == r.rejected_request_id.to_string() + && hash(&e.uncommitted_request) == r.uncommitted_request_sha256 + && x.request_id == r.uncommitted_request_id.to_string() + && x.previous_decision_id == a.decision_id.to_string() + && x.helper_previous_decision_id == a.decision_id.to_string() + && y.previous_decision_id.is_empty() + && y.previous_request_id == x.request_id + && y.helper_previous_decision_id == a.decision_id.to_string(), + "REPAIR_AGENT_EXACT_UNSENT_CHAIN" + ); + Ok(()) +} +fn signed_response_eq(raw: &[u8], expected: &SignedDecision) -> Result { + Ok(signed(raw)? == *expected) +} +fn main() -> Result<()> { + ensure!( + unsafe { libc::geteuid() } == 0, + "REPAIR_ROOT_BOOTSTRAP_ONLY" + ); + ensure!( + std::env::var("RMM_CHECKPOINT_REPAIR_PROFILE").as_deref() == Ok("development"), + "REPAIR_EXPLICIT_DEVELOPMENT_PROFILE_REQUIRED" + ); + ensure!( + rmm_privileged_helper::clock_trusted(), + "REPAIR_CLOCK_UNTRUSTED" + ); + let args = std::env::args_os().skip(1).collect::>(); + ensure!( + matches!(args.len(), 3 | 4), + "usage: rmm-checkpoint-repair prepare BINDING_FILE EVIDENCE_FILE | apply BINDING_FILE EVIDENCE_FILE RECEIPT_FILE" + ); + let binding = rmm_privileged_helper::read_binding(Path::new(&args[1]), 0)?; + let owner = fs::metadata(AGENT)?.uid(); + ensure!(owner != 0, "REPAIR_UNPRIVILEGED_AGENT_REQUIRED"); + rmm_privileged_helper::private_path(Path::new(AGENT), owner, true)?; + rmm_privileged_helper::private_path(Path::new(HELPER), 0, true)?; + let lock_path = Path::new(AGENT).join("agent.lock"); + rmm_privileged_helper::private_path(&lock_path, owner, false)?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NOFOLLOW) + .open(lock_path)?; + lock.try_lock_exclusive()?; + let _guard = rmm_privileged_helper::reconcile::ExecutorGuard::acquire(Path::new(HELPER), 0)?; + let mut agent = open_existing(&Path::new(AGENT).join("journal.sqlite"), owner)?; + let consumption = open_existing(&Path::new(AGENT).join("consumption.sqlite"), owner)?; + let mut helper = open_existing(&Path::new(HELPER).join("consumption.sqlite"), 0)?; + let mode = args[0].to_str().context("REPAIR_MODE")?; + match (mode, args.len()) { + ("prepare", 3) => { + let e = evidence(&agent, &consumption, &helper, &binding)?; + let bytes = serde_json::to_vec(&e)?; + ensure!(bytes.len() <= 32 * 1024 * 1024, "REPAIR_EVIDENCE_SIZE"); + let mut f = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&args[2])?; + f.write_all(&bytes)?; + f.sync_all()?; + println!("checkpoint repair evidence prepared; no state changed"); + } + ("apply", 4) => { + let e: Evidence = + serde_json::from_slice(&read_private(Path::new(&args[2]), 0, 32 * 1024 * 1024)?)?; + let signed: SignedReceipt = + serde_json::from_slice(&read_private(Path::new(&args[3]), 0, 65536)?)?; + verify_agent(&agent, &consumption, &binding, &signed, &e)?; + repair::verify_helper(&helper, &binding, &signed, &e)?; + let r = repair::verify_receipt(&binding, &signed)?; + // Every witness is verified before either durable fault archive is written. + // A stopped bootstrap can replay the identical receipt after interruption. + repair::commit_fault_repair( + &mut helper, + "helper_sync_meta", + &signed, + &r.helper_inventory_sha256, + None, + )?; + repair::commit_fault_repair( + &mut agent, + "reconcile_meta", + &signed, + &r.agent_inventory_sha256, + Some(("agent_consumption", repair::inventory(&consumption, "")?)), + )?; + println!( + "exact checkpoint faults archived and repaired; no execution readiness granted" + ); + } + _ => anyhow::bail!("REPAIR_UNKNOWN_MODE"), + } + drop(helper); + drop(consumption); + drop(agent); + // DELETE journaling leaves no root-owned WAL/SHM or journal behind. + for name in ["journal.sqlite", "consumption.sqlite"] { + for suffix in ["-wal", "-shm", "-journal"] { + ensure!( + !Path::new(AGENT) + .join(format!("{name}{suffix}")) + .try_exists()?, + "REPAIR_UNEXPECTED_AGENT_SIDECAR" + ); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use openssl::{pkey::PKey, sign::Signer}; + use uuid::Uuid; + #[test] + fn durable_signed_request_history_rejects_mutation_before_any_repair() { + let key = PKey::generate_ed25519().unwrap(); + let id = Uuid::new_v4(); + let boot = Uuid::new_v4(); + let binding = Binding { + tenant_id: id, + asset_id: id, + agent_id: id, + enrollment_generation: 1, + assignment_version: 1, + public_key_hex: key + .raw_public_key() + .unwrap() + .iter() + .map(|b| format!("{b:02x}")) + .collect(), + }; + let q = SyncRequest { + request_id: Uuid::new_v4().to_string(), + journal_id: id.to_string(), + checkpoint: 10, + assignment_version: 1, + journal_complete: true, + helper_journal_id: id.to_string(), + helper_checkpoint: 20, + ..Default::default() + }; + let e = Envelope { + protocol_major: 1, + protocol_minor: 0, + message_id: Uuid::new_v4().to_string(), + agent_id: id.to_string(), + enrollment_generation: 1, + connection_generation: 2, + boot_id: boot.to_string(), + sequence: 10, + observed_at: Utc::now().timestamp_millis(), + payload: Some(Payload::SyncRequest(q.clone())), + }; + let now = Utc::now(); + let d = ReconcileDecisionPayload { + domain: "rmm.reconciliation-decision".into(), + schema_version: 1, + decision_id: Uuid::new_v4(), + request_id: Uuid::parse_str(&q.request_id).unwrap(), + request_hash: hash(&q.encode_to_vec()), + tenant_id: id, + asset_id: id, + agent_id: id, + enrollment_generation: 1, + assignment_version: 1, + connection_generation: 2, + boot_id: boot, + journal_id: id, + checkpoint: 10, + helper_journal_id: Some(id), + helper_checkpoint: 20, + issued_at: now, + valid_until: now + chrono::Duration::seconds(60), + ready: false, + helper_ready: false, + reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(), + attempts: vec![], + acknowledged_message_ids: vec![], + key_id: hash(&key.raw_public_key().unwrap()), + }; + let bytes = serde_json::to_vec(&d).unwrap(); + let signature = Signer::new_without_digest(&key) + .unwrap() + .sign_oneshot_to_vec(&bytes) + .unwrap(); + let response = SyncResponse { + decision_bytes: bytes, + signature, + }; + let db = Connection::open_in_memory().unwrap(); + db.execute_batch("CREATE TABLE reconcile_requests(id TEXT,request BLOB,envelope BLOB,generation INTEGER,response BLOB);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB);").unwrap(); + db.execute( + "INSERT INTO reconcile_requests VALUES(?1,?2,?3,2,?4)", + rusqlite::params![ + q.request_id, + q.encode_to_vec(), + e.encode_to_vec(), + response.encode_to_vec() + ], + ) + .unwrap(); + assert_eq!(agent_history(&db, &binding).unwrap().len(), 1); + let mut bad = response.clone(); + bad.signature[0] ^= 1; + db.execute( + "UPDATE reconcile_requests SET response=?1", + [bad.encode_to_vec()], + ) + .unwrap(); + assert!(agent_history(&db, &binding).is_err()); + db.execute( + "UPDATE reconcile_requests SET response=?1", + [response.encode_to_vec()], + ) + .unwrap(); + let mut changed = e; + changed.sequence = 11; + db.execute( + "UPDATE reconcile_requests SET envelope=?1", + [changed.encode_to_vec()], + ) + .unwrap(); + assert!(agent_history(&db, &binding).is_err()); + } +} diff --git a/crates/domain/src/checkpoint_repair.rs b/crates/domain/src/checkpoint_repair.rs new file mode 100644 index 0000000..ba9293d --- /dev/null +++ b/crates/domain/src/checkpoint_repair.rs @@ -0,0 +1,139 @@ +//! Development bootstrap receipt for one proven unsent reconciliation request. +//! This is neither a job approval, an execution permit nor a ready lease. +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SignedDecision { + pub decision_bytes: Vec, + pub signature: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SignedReceipt { + pub payload: Vec, + pub signature: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Evidence { + pub schema_version: u32, + pub tenant_id: Uuid, + pub asset_id: Uuid, + pub agent_id: Uuid, + pub enrollment_generation: u64, + pub assignment_version: u64, + pub agent_inventory: Value, + pub helper_inventory: Value, + pub uncommitted_request: Vec, + pub rejected_request: Vec, + pub rejected_response: SignedDecision, + pub previous_valid_response: SignedDecision, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Receipt { + pub domain: String, + pub schema_version: u32, + pub receipt_id: Uuid, + pub tenant_id: Uuid, + pub asset_id: Uuid, + pub agent_id: Uuid, + pub enrollment_generation: u64, + pub assignment_version: u64, + pub issued_at: DateTime, + pub valid_until: DateTime, + pub key_id: String, + pub fault: String, + pub agent_journal_id: Uuid, + pub agent_checkpoint: u64, + pub helper_journal_id: Uuid, + pub helper_checkpoint: u64, + pub rejected_decision_id: Uuid, + pub rejected_decision_sha256: String, + pub rejected_request_id: Uuid, + pub rejected_request_sha256: String, + pub uncommitted_request_id: Uuid, + pub uncommitted_request_sha256: String, + pub previous_valid_decision_id: Uuid, + pub previous_valid_decision_sha256: String, + pub agent_inventory_sha256: String, + pub helper_inventory_sha256: String, + pub helper_ledger_sha256: String, + pub ready: bool, + pub helper_ready: bool, + pub unlock: bool, +} +pub fn canonical_bytes(value: &Value) -> Result, serde_json::Error> { + fn ordered(v: Value) -> Value { + match v { + Value::Object(m) => { + let mut items = m.into_iter().collect::>(); + items.sort_by(|a, b| a.0.cmp(&b.0)); + Value::Object(items.into_iter().map(|(k, v)| (k, ordered(v))).collect()) + } + Value::Array(a) => Value::Array(a.into_iter().map(ordered).collect()), + other => other, + } + } + serde_json::to_vec(&ordered(value.clone())) +} +pub fn inventory_sha256(value: &Value) -> Result { + Ok(crate::jobs::hash(&canonical_bytes(value)?)) +} +pub fn helper_ledger_sha256(inventory: &Value) -> Result { + inventory_sha256( + &serde_json::json!({"consumed":inventory["consumed"],"execution_evidence":inventory["execution_evidence"]}), + ) +} +impl Receipt { + pub fn valid_profile(&self, now: DateTime) -> bool { + let digest = |s: &str| { + s.len() == 64 + && s.bytes() + .all(|c| c.is_ascii_digit() || b"abcdef".contains(&c)) + }; + self.domain == "rmm.sync.unsent-request-repair" + && self.schema_version == 1 + && !self.receipt_id.is_nil() + && self.fault == "CHECKPOINT_CHAIN_MISMATCH" + && !self.ready + && !self.helper_ready + && !self.unlock + && self.issued_at <= now + && now < self.valid_until + && self.valid_until > self.issued_at + && self.valid_until - self.issued_at <= chrono::Duration::minutes(15) + && [ + self.tenant_id, + self.asset_id, + self.agent_id, + self.agent_journal_id, + self.helper_journal_id, + self.rejected_decision_id, + self.rejected_request_id, + self.uncommitted_request_id, + self.previous_valid_decision_id, + ] + .iter() + .all(|id| !id.is_nil()) + && self.enrollment_generation > 0 + && self.assignment_version > 0 + && self.agent_checkpoint > 0 + && self.helper_checkpoint > 0 + && [ + &self.key_id, + &self.rejected_decision_sha256, + &self.rejected_request_sha256, + &self.uncommitted_request_sha256, + &self.previous_valid_decision_sha256, + &self.agent_inventory_sha256, + &self.helper_inventory_sha256, + &self.helper_ledger_sha256, + ] + .iter() + .all(|s| digest(s)) + } +} diff --git a/crates/domain/src/lib.rs b/crates/domain/src/lib.rs index 956d422..5f394f7 100644 --- a/crates/domain/src/lib.rs +++ b/crates/domain/src/lib.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; pub mod apt_transaction; +pub mod checkpoint_repair; pub mod inventory_document; pub mod jobs; pub mod reconciliation; diff --git a/crates/privileged-helper/src/checkpoint_repair.rs b/crates/privileged-helper/src/checkpoint_repair.rs new file mode 100644 index 0000000..a389483 --- /dev/null +++ b/crates/privileged-helper/src/checkpoint_repair.rs @@ -0,0 +1,458 @@ +//! Fixed bootstrap verification of one signed no-effect checkpoint repair. +use crate::Binding; +use anyhow::{Context, Result, ensure}; +use chrono::Utc; +use openssl::{ + pkey::{Id, PKey}, + sign::Verifier, +}; +use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload}; +use rusqlite::{Connection, OptionalExtension, params, types::ValueRef}; +use serde_json::{Value, json}; +pub const AUDIT: &str = "checkpoint_repair_receipts"; +pub fn verify_historical_decision( + binding: &Binding, + signed: &SignedDecision, +) -> Result { + ensure!( + signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64, + "REPAIR_DECISION_SIZE" + ); + let raw = hex::decode(&binding.public_key_hex)?; + let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?; + ensure!( + Verifier::new_without_digest(&key)? + .verify_oneshot(&signed.signature, &signed.decision_bytes)?, + "REPAIR_HISTORICAL_SIGNATURE" + ); + let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?; + ensure!( + d.valid_profile(d.issued_at) + && d.key_id == hash(&raw) + && d.tenant_id == binding.tenant_id + && d.asset_id == binding.asset_id + && d.agent_id == binding.agent_id + && d.enrollment_generation == binding.enrollment_generation + && d.assignment_version == binding.assignment_version, + "REPAIR_HISTORICAL_IDENTITY" + ); + Ok(d) +} +pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result { + ensure!( + signed.payload.len() <= 16384 && signed.signature.len() == 64, + "REPAIR_RECEIPT_SIZE" + ); + let raw = hex::decode(&binding.public_key_hex)?; + let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?; + ensure!( + Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?, + "REPAIR_RECEIPT_SIGNATURE" + ); + let r: Receipt = serde_json::from_slice(&signed.payload)?; + ensure!( + r.valid_profile(Utc::now()) + && r.key_id == hash(&raw) + && r.tenant_id == binding.tenant_id + && r.asset_id == binding.asset_id + && r.agent_id == binding.agent_id + && r.enrollment_generation == binding.enrollment_generation + && r.assignment_version == binding.assignment_version, + "REPAIR_RECEIPT_BINDING" + ); + Ok(r) +} +/// Full database inventory with typed values; byte fields are bounded hashes. +/// The append-only repair archive is excluded; only the repaired fault cell is +/// normalized for exact idempotent replay of the same signed receipt. +pub fn inventory(db: &Connection, meta: &str) -> Result { + let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; + ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY"); + let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?; + ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM"); + let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?; + let names = q + .query_map([AUDIT], |r| r.get::<_, String>(0))? + .collect::>>()?; + let mut tables = serde_json::Map::new(); + let mut count = 0; + for name in names { + ensure!( + name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'), + "REPAIR_TABLE_NAME" + ); + let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?; + let columns = s + .column_names() + .into_iter() + .map(str::to_owned) + .collect::>(); + let mut rows = vec![]; + let mut cursor = s.query([])?; + while let Some(row) = cursor.next()? { + count += 1; + ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT"); + let mut cells = vec![]; + for (i, col) in columns.iter().enumerate() { + let cell = if name == meta && col == "fault" { + let v = row.get_ref(i)?; + ensure!( + matches!(v, ValueRef::Null) + || matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")), + "REPAIR_DIFFERENT_FAULT" + ); + json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"}) + } else { + match row.get_ref(i)? { + ValueRef::Null => json!({"type":"null"}), + ValueRef::Integer(v) => json!({"type":"integer","value":v}), + ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"), + ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}), + ValueRef::Blob(v) => { + json!({"type":"blob","sha256":hash(v),"length":v.len()}) + } + } + }; + cells.push(cell); + } + rows.push(cells); + } + tables.insert(name, json!({"columns":columns,"rows":rows})); + } + Ok(Value::Object(tables)) +} +pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result { + let exists: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)", + [AUDIT], + |r| r.get(0), + )?; + if !exists { + return Ok(false); + } + let r: Receipt = serde_json::from_slice(&signed.payload)?; + let old: Option> = db + .query_row( + "SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1", + [r.receipt_id.to_string()], + |v| v.get(0), + ) + .optional()?; + if let Some(old) = old { + ensure!( + old == serde_json::to_vec(signed)?, + "REPAIR_RECEIPT_CONFLICT" + ); + Ok(true) + } else { + Ok(false) + } +} +pub fn commit_fault_repair( + db: &mut Connection, + meta: &str, + signed: &SignedReceipt, + expected_hash: &str, + additional: Option<(&str, Value)>, +) -> Result<()> { + ensure!( + matches!(meta, "reconcile_meta" | "helper_sync_meta"), + "REPAIR_META_TABLE" + ); + let r: Receipt = serde_json::from_slice(&signed.payload)?; + let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let mut actual = inventory(&tx, meta)?; + if let Some((key, value)) = additional { + actual + .as_object_mut() + .context("REPAIR_INVENTORY_OBJECT")? + .insert(key.into(), value); + } + ensure!( + inventory_sha256(&actual)? == expected_hash, + "REPAIR_WITNESS_CHANGED" + ); + let fault: Option = + tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| { + v.get(0) + })?; + if already_applied(&tx, signed)? { + ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED"); + tx.commit()?; + return Ok(()); + } + ensure!( + fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"), + "REPAIR_EXACT_FAULT_REQUIRED" + ); + tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?; + tx.execute( + "INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)", + params![ + r.receipt_id.to_string(), + serde_json::to_vec(signed)?, + fault, + expected_hash, + Utc::now().to_rfc3339() + ], + )?; + ensure!( + tx.execute( + &format!( + "UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'" + ), + [] + )? == 1, + "REPAIR_FAULT_CHANGED" + ); + let mut after = inventory(&tx, meta)?; + if let Some(value) = actual.get("agent_consumption") { + after + .as_object_mut() + .context("REPAIR_INVENTORY_OBJECT")? + .insert("agent_consumption".into(), value.clone()); + } + ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED"); + tx.commit()?; + Ok(()) +} +pub fn verify_helper( + db: &Connection, + binding: &Binding, + signed: &SignedReceipt, + e: &Evidence, +) -> Result<()> { + let r = verify_receipt(binding, signed)?; + ensure!( + e.schema_version == 1 + && e.tenant_id == binding.tenant_id + && e.asset_id == binding.asset_id + && e.agent_id == binding.agent_id + && e.enrollment_generation == binding.enrollment_generation + && e.assignment_version == binding.assignment_version, + "REPAIR_HELPER_EVIDENCE_IDENTITY" + ); + crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?; + let inv = inventory(db, "helper_sync_meta")?; + ensure!( + inventory_sha256(&inv)? == r.helper_inventory_sha256 + && inv == e.helper_inventory + && helper_ledger_sha256(&inv)? == r.helper_ledger_sha256, + "REPAIR_HELPER_INVENTORY" + ); + let (journal, checkpoint, decision, fault): (String, u64, String, Option) = db + .query_row( + "SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1", + [], + |v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)), + )?; + ensure!( + journal == r.helper_journal_id.to_string() + && checkpoint == r.helper_checkpoint + && decision == r.rejected_decision_id.to_string() + && (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH") + || (fault.is_none() && already_applied(db, signed)?)), + "REPAIR_HELPER_HEAD" + ); + let (response, request): (Vec, Vec) = db.query_row( + "SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1", + [decision], + |v| Ok((v.get(0)?, v.get(1)?)), + )?; + let stored: SignedDecision = serde_json::from_slice(&response)?; + ensure!( + stored == e.rejected_response + && request == e.rejected_request + && hash(&request) == r.rejected_request_sha256 + && hash(&stored.decision_bytes) == r.rejected_decision_sha256, + "REPAIR_HELPER_REJECTION" + ); + let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?; + ensure!( + rejection.reason == r.fault + && !rejection.ready + && !rejection.helper_ready + && !rejection.attempts.iter().any(|v| v.unlock), + "REPAIR_REJECTION_HAS_AUTHORITY" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use openssl::{pkey::Private, sign::Signer}; + use uuid::Uuid; + fn fixture() -> (Binding, PKey, Receipt) { + let key = PKey::generate_ed25519().unwrap(); + let id = Uuid::new_v4(); + let now = Utc::now(); + let binding = Binding { + tenant_id: id, + asset_id: id, + agent_id: id, + enrollment_generation: 1, + assignment_version: 1, + public_key_hex: hex::encode(key.raw_public_key().unwrap()), + }; + let r = Receipt { + domain: "rmm.sync.unsent-request-repair".into(), + schema_version: 1, + receipt_id: Uuid::new_v4(), + tenant_id: id, + asset_id: id, + agent_id: id, + enrollment_generation: 1, + assignment_version: 1, + issued_at: now, + valid_until: now + chrono::Duration::minutes(15), + key_id: hash(&key.raw_public_key().unwrap()), + fault: "CHECKPOINT_CHAIN_MISMATCH".into(), + agent_journal_id: id, + agent_checkpoint: 13, + helper_journal_id: id, + helper_checkpoint: 22, + rejected_decision_id: id, + rejected_decision_sha256: hash(b"denied"), + rejected_request_id: id, + rejected_request_sha256: hash(b"request"), + uncommitted_request_id: id, + uncommitted_request_sha256: hash(b"unsent"), + previous_valid_decision_id: id, + previous_valid_decision_sha256: hash(b"previous"), + agent_inventory_sha256: hash(b"agent"), + helper_inventory_sha256: hash(b"helper"), + helper_ledger_sha256: hash(b"ledger"), + ready: false, + helper_ready: false, + unlock: false, + }; + (binding, key, r) + } + fn sign(key: &PKey, r: &Receipt) -> SignedReceipt { + let payload = serde_json::to_vec(r).unwrap(); + let signature = Signer::new_without_digest(key) + .unwrap() + .sign_oneshot_to_vec(&payload) + .unwrap(); + SignedReceipt { payload, signature } + } + #[test] + fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() { + let (binding, key, r) = fixture(); + assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok()); + let mut bad = sign(&key, &r); + bad.payload[0] ^= 1; + assert!(verify_receipt(&binding, &bad).is_err()); + for changed in [0, 1, 2, 3, 4, 5] { + let mut bad = r.clone(); + match changed { + 0 => bad.ready = true, + 1 => bad.helper_ready = true, + 2 => bad.unlock = true, + 3 => bad.agent_id = Uuid::new_v4(), + 4 => bad.domain = "rmm.execution-permit".into(), + _ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16), + }; + assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err()); + } + let mut expired = r.clone(); + expired.issued_at = Utc::now() - chrono::Duration::minutes(2); + expired.valid_until = Utc::now() - chrono::Duration::minutes(1); + assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err()); + } + fn database() -> Connection { + let db = Connection::open_in_memory().unwrap(); + db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap(); + db + } + #[test] + fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() { + let (_, key, mut r) = fixture(); + let mut db = database(); + let before = inventory(&db, "helper_sync_meta").unwrap(); + r.helper_inventory_sha256 = inventory_sha256(&before).unwrap(); + r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap(); + let signed = sign(&key, &r); + commit_fault_repair( + &mut db, + "helper_sync_meta", + &signed, + &r.helper_inventory_sha256, + None, + ) + .unwrap(); + assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before); + assert!( + db.query_row("SELECT result IS NULL FROM consumed", [], |v| v + .get::<_, bool>(0)) + .unwrap() + ); + assert_eq!( + db.query_row("SELECT outcome FROM execution_evidence", [], |v| v + .get::<_, Vec>(0)) + .unwrap(), + b"unknown" + ); + assert_eq!( + db.query_row( + "SELECT old_fault FROM checkpoint_repair_receipts", + [], + |v| v.get::<_, String>(0) + ) + .unwrap(), + "CHECKPOINT_CHAIN_MISMATCH" + ); + commit_fault_repair( + &mut db, + "helper_sync_meta", + &signed, + &r.helper_inventory_sha256, + None, + ) + .unwrap(); + assert_eq!( + db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v + .get::<_, u64>(0)) + .unwrap(), + 1 + ); + db.execute("UPDATE consumed SET result=x'01'", []).unwrap(); + assert!( + commit_fault_repair( + &mut db, + "helper_sync_meta", + &signed, + &r.helper_inventory_sha256, + None + ) + .is_err() + ); + } + #[test] + fn changed_witness_or_other_fault_is_never_cleared() { + let (_, key, r) = fixture(); + let mut db = database(); + let signed = sign(&key, &r); + assert!( + commit_fault_repair( + &mut db, + "helper_sync_meta", + &signed, + &r.helper_inventory_sha256, + None + ) + .is_err() + ); + db.execute( + "UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'", + [], + ) + .unwrap(); + assert!(inventory(&db, "helper_sync_meta").is_err()); + assert!(!already_applied(&db, &signed).unwrap()); + let db = database(); + db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap(); + assert!(inventory(&db, "helper_sync_meta").is_err()); + } +} diff --git a/crates/privileged-helper/src/lib.rs b/crates/privileged-helper/src/lib.rs index 0c06342..8b60189 100644 --- a/crates/privileged-helper/src/lib.rs +++ b/crates/privileged-helper/src/lib.rs @@ -3,6 +3,7 @@ extern crate self as rmm_privileged_helper; pub mod apt; pub mod apt_reconcile; +pub mod checkpoint_repair; pub mod control; pub mod readiness; pub mod reconcile; diff --git a/crates/privileged-helper/src/reconcile.rs b/crates/privileged-helper/src/reconcile.rs index 07f30f9..3ce7e2d 100644 --- a/crates/privileged-helper/src/reconcile.rs +++ b/crates/privileged-helper/src/reconcile.rs @@ -681,6 +681,142 @@ fn validate_history( } Ok(()) } +/// Bootstrap-only historical verification; this never creates a lease. +pub fn verify_checkpoint_repair_history( + db: &Connection, + binding: &Binding, + links: Option<( + &rmm_domain::checkpoint_repair::Receipt, + &rmm_domain::checkpoint_repair::Evidence, + )>, +) -> Result<()> { + let meta = metadata(db)?; + validate_history(db, binding, &meta, meta.generation)?; + let mut statement = db.prepare( + "SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint", + )?; + let entries = statement + .query_map([], |r| { + Ok(( + r.get::<_, Vec>(0)?, + r.get::<_, Vec>(1)?, + r.get::<_, u64>(2)?, + )) + })? + .collect::>>()?; + ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT"); + let mut previous = None; + let mut previous_checkpoint = 0; + let mut previous_agent_checkpoint = 0; + for (signed, raw, checkpoint) in entries { + let response: SignedResponse = serde_json::from_slice(&signed)?; + let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?; + let decision = verify(binding, &response, parsed.issued_at)?; + let request = RequestView::decode(&raw)?; + let captured: Vec = db.query_row( + "SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1", + [checkpoint], + |r| r.get(0), + )?; + let capture: Snapshot = serde_json::from_slice(&captured)?; + ensure!( + decision.helper_journal_id == Some(meta.journal_id) + && decision.helper_checkpoint == checkpoint + && request.helper_journal_id == meta.journal_id.to_string() + && request.helper_checkpoint == checkpoint + && request.assignment_version == binding.assignment_version + && request.journal_complete + && decision.request_hash == hash(&raw) + && decision.request_id.to_string() == request.request_id + && decision.journal_id.to_string() == request.journal_id + && decision.checkpoint == request.checkpoint + && capture.journal_id == meta.journal_id + && capture.checkpoint == checkpoint + && capture.boot_id == decision.boot_id + && capture.captured_at <= decision.issued_at + && capture.previous_decision_id == previous + && checkpoint > previous_checkpoint + && decision.checkpoint >= previous_agent_checkpoint + && decision.connection_generation <= meta.generation + && request.helper_previous_decision_id + == capture + .previous_decision_id + .map(|v| v.to_string()) + .unwrap_or_default(), + "REPAIR_HELPER_HISTORY_BINDING" + ); + previous = Some(decision.decision_id); + previous_checkpoint = checkpoint; + previous_agent_checkpoint = decision.checkpoint; + } + let _ = load_rows(db, binding, false)?; + if let Some((r, e)) = links { + verify_checkpoint_repair_links(db, binding, r, e)?; + } + Ok(()) +} +fn verify_checkpoint_repair_links( + db: &Connection, + binding: &Binding, + r: &rmm_domain::checkpoint_repair::Receipt, + e: &rmm_domain::checkpoint_repair::Evidence, +) -> Result<()> { + let x = RequestView::decode(&e.uncommitted_request)?; + let y = RequestView::decode(&e.rejected_request)?; + let a: ReconcileDecisionPayload = + serde_json::from_slice(&e.previous_valid_response.decision_bytes)?; + let a = verify( + binding, + &SignedResponse { + decision_bytes: e.previous_valid_response.decision_bytes.clone(), + signature: e.previous_valid_response.signature.clone(), + }, + a.issued_at, + )?; + let old: Vec = db.query_row( + "SELECT response FROM helper_sync_decisions WHERE decision_id=?1", + [a.decision_id.to_string()], + |v| v.get(0), + )?; + let old: SignedResponse = serde_json::from_slice(&old)?; + ensure!( + old.decision_bytes == e.previous_valid_response.decision_bytes + && old.signature == e.previous_valid_response.signature + && a.decision_id == r.previous_valid_decision_id + && hash(&old.decision_bytes) == r.previous_valid_decision_sha256 + && a.reason == "ATTEMPT_RECONCILIATION_REQUIRED" + && !a.ready + && !a.helper_ready + && x.request_id == r.uncommitted_request_id.to_string() + && hash(&e.uncommitted_request) == r.uncommitted_request_sha256 + && y.request_id == r.rejected_request_id.to_string() + && hash(&e.rejected_request) == r.rejected_request_sha256 + && x.previous_decision_id == a.decision_id.to_string() + && x.previous_request_id.is_empty() + && x.helper_previous_decision_id == a.decision_id.to_string() + && y.helper_previous_decision_id == a.decision_id.to_string() + && y.previous_decision_id.is_empty() + && y.previous_request_id == x.request_id + && x.journal_id == r.agent_journal_id.to_string() + && y.journal_id == x.journal_id + && a.journal_id == r.agent_journal_id + && x.helper_journal_id == r.helper_journal_id.to_string() + && y.helper_journal_id == x.helper_journal_id + && a.helper_journal_id == Some(r.helper_journal_id) + && a.checkpoint < x.checkpoint + && x.checkpoint < y.checkpoint + && y.checkpoint == r.agent_checkpoint + && a.helper_checkpoint < x.helper_checkpoint + && x.helper_checkpoint < y.helper_checkpoint + && y.helper_checkpoint == r.helper_checkpoint + && x.assignment_version == binding.assignment_version + && y.assignment_version == binding.assignment_version + && x.journal_complete + && y.journal_complete, + "REPAIR_EXACT_UNSENT_CHAIN_REQUIRED" + ); + Ok(()) +} fn persist_fault(db: &Connection, error: &anyhow::Error) { let text = error.to_string(); if [ @@ -991,6 +1127,8 @@ struct RequestView { helper_journal_id: String, helper_checkpoint: u64, helper_previous_decision_id: String, + previous_decision_id: String, + previous_request_id: String, } #[derive(Default)] struct AttemptView { @@ -1089,6 +1227,11 @@ impl RequestView { if !s.is_empty() { Uuid::parse_str(&s)?; } + if tag == 4 { + r.previous_decision_id = s; + } else { + r.previous_request_id = s; + } } 5 => r.assignment_version = number(value)?, 6 => { diff --git a/crates/privileged-helper/tests/reconcile.rs b/crates/privileged-helper/tests/reconcile.rs index 26c43bc..360e947 100644 --- a/crates/privileged-helper/tests/reconcile.rs +++ b/crates/privileged-helper/tests/reconcile.rs @@ -39,6 +39,32 @@ struct Fixture { boot: TestBoot, agent_journal: Uuid, } + +#[test] +fn checkpoint_repair_verifies_complete_signed_history_and_detects_removed_predecessor() { + let mut f = Fixture::new(); + let mut sync = f.sync(); + let first = f.snapshot(&mut sync, 7); + let request = f.request(&first); + let d = f.decision(&first, &request, 7, false); + let signed = f.sign(&d); + sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot) + .unwrap(); + f.clock.now += Duration::seconds(1); + let second = f.snapshot(&mut sync, 7); + let request = f.request(&second); + let d = f.decision(&second, &request, 7, false); + let signed = f.sign(&d); + sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot) + .unwrap(); + verify_checkpoint_repair_history(&f.db, &f.binding, None).unwrap(); + f.db.execute( + "DELETE FROM helper_sync_decisions WHERE checkpoint=?1", + [first.checkpoint], + ) + .unwrap(); + assert!(verify_checkpoint_repair_history(&f.db, &f.binding, None).is_err()); +} impl Drop for Fixture { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.dir); diff --git a/crates/server/src/agent_http.rs b/crates/server/src/agent_http.rs index 6afcdbf..745aea5 100644 --- a/crates/server/src/agent_http.rs +++ b/crates/server/src/agent_http.rs @@ -38,6 +38,35 @@ fn response(messages: Vec) -> Response { ) .into_response() } +async fn append_work( + s: &crate::State, + tenant: Uuid, + agent: Uuid, + source: &Envelope, + payload: Payload, + generation: u64, + replies: &mut Vec, +) -> anyhow::Result<()> { + let reply = crate::agent_service::reply(source, payload, generation); + // Preserve the protocol's 16-envelope response limit. An offer that cannot + // fit twice is delivered normally without consuming the optional fault. + let claim = if replies.len() <= 14 { + crate::gateway::dev_delivery_faults::duplicate_offer( + &s.pool, tenant, agent, &reply, "https", + ) + .await? + } else { + None + }; + if claim.is_some() { + replies.push(reply.clone()); + } + replies.push(reply); + if let Some(claim) = claim { + crate::gateway::dev_delivery_faults::duplicate_emitted(&s.pool, claim).await?; + } + Ok(()) +} pub async fn messages( State(s): State, Extension(peer): Extension, @@ -83,13 +112,21 @@ pub async fn messages( } .map_err(|_| StatusCode::CONFLICT)?; generation = reply.connection_generation; - replies.push(reply); + if !crate::gateway::dev_delivery_faults::after_ingest(&s.pool, t, a, &e, &reply, "https") + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)? + { + replies.push(reply); + } } - if let Some(work) = crate::job_gateway::next(&s, t, a, generation) - .await - .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)? + if replies.len() < 16 + && let Some(work) = crate::job_gateway::next(&s, t, a, generation) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)? { - replies.push(crate::agent_service::reply(&last, work, generation)); + append_work(&s, t, a, &last, work, generation, &mut replies) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; } Ok(response(replies)) } @@ -119,10 +156,11 @@ pub async fn poll( .await .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)? { - return Ok(response(vec![ - ack, - crate::agent_service::reply(&e, work, e.connection_generation), - ])); + let mut replies = vec![ack]; + append_work(&s, t, a, &e, work, e.connection_generation, &mut replies) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + return Ok(response(replies)); } if tokio::time::Instant::now() >= deadline { return Ok(response(vec![ack])); diff --git a/crates/server/src/checkpoint_repair.rs b/crates/server/src/checkpoint_repair.rs new file mode 100644 index 0000000..c80a445 --- /dev/null +++ b/crates/server/src/checkpoint_repair.rs @@ -0,0 +1,772 @@ +//! Fixed development bootstrap for a proven unsent sync request. This promotes +//! an existing signed denial only; it grants no readiness or execution rights. +use anyhow::{Context, Result, ensure}; +use chrono::{Duration, Utc}; +use openssl::{ + pkey::{Id, PKey}, + sign::Verifier, +}; +use prost::Message; +use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload}; +use rmm_protocol::SyncRequest; +use serde_json::{Value, json}; +use sqlx::Row; +use std::{ + fs, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::Path, +}; +use uuid::Uuid; + +const EVENT: &str = "unsent_request_repair_issued"; +const FAULT: &str = "CHECKPOINT_CHAIN_MISMATCH"; + +pub fn closed_policy() -> Result<()> { + ensure!( + std::env::var("RMM_PROFILE").as_deref() == Ok("development") + && std::env::var("RMM_DEV_AUTH").as_deref() == Ok("true") + && std::env::var("RMM_SERVER_ROLE").as_deref() == Ok("api"), + "REPAIR_DEVELOPMENT_API_ONLY" + ); + let path = std::env::var("RMM_DEV_ROOT_POLICY_FILE").context("REPAIR_POLICY_REQUIRED")?; + let m = fs::symlink_metadata(&path)?; + let parent = fs::symlink_metadata(Path::new(&path).parent().context("REPAIR_POLICY_PARENT")?)?; + let key = std::env::var("RMM_JOB_KEY_FILE").context("REPAIR_EXISTING_KEY_REQUIRED")?; + ensure!( + m.is_file() + && m.len() <= 65536 + && m.permissions().mode() & 0o077 == 0 + && m.uid() == fs::metadata(key)?.uid() + && parent.is_dir() + && parent.permissions().mode() & 0o022 == 0, + "REPAIR_POLICY_PERMISSIONS" + ); + let p: Value = serde_json::from_slice(&fs::read(path)?)?; + ensure!( + p["schema_version"] == 1 + && p["profile"] == "development" + && p["expires_at"] + .as_str() + .and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok()) + .is_some_and(|until| until <= Utc::now()), + "REPAIR_POLICY_MUST_BE_CLOSED" + ); + Ok(()) +} + +fn cell<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a Value> { + let columns = table["columns"] + .as_array() + .context("REPAIR_INVENTORY_COLUMNS")?; + let i = columns + .iter() + .position(|v| v.as_str() == Some(column)) + .context("REPAIR_INVENTORY_COLUMN")?; + row.as_array() + .and_then(|r| r.get(i)) + .context("REPAIR_INVENTORY_CELL") +} +fn rows(table: &Value) -> Result<&Vec> { + table["rows"].as_array().context("REPAIR_INVENTORY_ROWS") +} +fn text<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a str> { + let v = cell(table, row, column)?; + ensure!(v["type"] == "text", "REPAIR_CELL_TYPE"); + v["value"].as_str().context("REPAIR_CELL_TEXT") +} +fn integer(table: &Value, row: &Value, column: &str) -> Result { + let v = cell(table, row, column)?; + ensure!(v["type"] == "integer", "REPAIR_CELL_TYPE"); + v["value"].as_u64().context("REPAIR_CELL_INTEGER") +} +fn blob(table: &Value, row: &Value, column: &str, bytes: &[u8]) -> Result<()> { + let v = cell(table, row, column)?; + ensure!( + v == &json!({"type":"blob","sha256":hash(bytes),"length":bytes.len()}), + "REPAIR_WITNESS_BLOB" + ); + Ok(()) +} +fn row_by<'a>(table: &'a Value, column: &str, id: &str) -> Result<&'a Value> { + let found = rows(table)? + .iter() + .filter(|row| text(table, row, column).ok() == Some(id)) + .collect::>(); + ensure!(found.len() == 1, "REPAIR_WITNESS_ROW"); + Ok(found[0]) +} +fn decision( + signer: &crate::job_signer::JobSigner, + s: &SignedDecision, +) -> Result { + ensure!( + s.decision_bytes.len() <= 1024 * 1024 && s.signature.len() == 64, + "REPAIR_DECISION_SIZE" + ); + let key = PKey::public_key_from_raw_bytes(&signer.public_key()?, Id::ED25519)?; + ensure!( + Verifier::new_without_digest(&key)?.verify_oneshot(&s.signature, &s.decision_bytes)?, + "REPAIR_DECISION_SIGNATURE" + ); + let d: ReconcileDecisionPayload = serde_json::from_slice(&s.decision_bytes)?; + ensure!( + d.valid_profile(d.issued_at) && d.key_id == signer.key_id, + "REPAIR_DECISION_PROFILE" + ); + Ok(d) +} + +struct Case { + old: ReconcileDecisionPayload, + rejected: ReconcileDecisionPayload, + unsent: SyncRequest, + request: SyncRequest, +} +fn validate(signer: &crate::job_signer::JobSigner, e: &Evidence) -> Result { + ensure!( + e.schema_version == 1 + && e.uncommitted_request.len() <= 1024 * 1024 + && e.rejected_request.len() <= 1024 * 1024, + "REPAIR_EVIDENCE_PROFILE" + ); + let old = decision(signer, &e.previous_valid_response)?; + let rejected = decision(signer, &e.rejected_response)?; + for d in [&old, &rejected] { + ensure!( + d.tenant_id == e.tenant_id + && d.asset_id == e.asset_id + && d.agent_id == e.agent_id + && d.enrollment_generation == e.enrollment_generation + && d.assignment_version == e.assignment_version, + "REPAIR_IDENTITY_SCOPE" + ); + } + let unsent = SyncRequest::decode(e.uncommitted_request.as_slice())?; + let request = SyncRequest::decode(e.rejected_request.as_slice())?; + let unsent_id = Uuid::parse_str(&unsent.request_id)?; + ensure!( + !unsent_id.is_nil() + && unsent_id != rejected.request_id + && unsent_id != old.request_id + && old.decision_id != rejected.decision_id + && rejected.request_id.to_string() == request.request_id + && rejected.request_hash == hash(&e.rejected_request), + "REPAIR_REQUEST_BINDING" + ); + ensure!( + rejected.reason == FAULT + && !rejected.ready + && !rejected.helper_ready + && !rejected.attempts.iter().any(|a| a.unlock), + "REPAIR_DENIAL_ONLY" + ); + ensure!( + request.previous_decision_id.is_empty() + && request.previous_request_id == unsent.request_id + && unsent.previous_decision_id == old.decision_id.to_string() + && unsent.previous_request_id.is_empty(), + "REPAIR_EXACT_UNSENT_CHAIN" + ); + let helper = rejected + .helper_journal_id + .context("REPAIR_HELPER_REQUIRED")?; + ensure!( + old.journal_id == rejected.journal_id + && old.helper_journal_id == Some(helper) + && old.checkpoint < unsent.checkpoint + && unsent.checkpoint < rejected.checkpoint + && old.helper_checkpoint <= unsent.helper_checkpoint + && unsent.helper_checkpoint <= rejected.helper_checkpoint + && old.connection_generation <= rejected.connection_generation, + "REPAIR_MONOTONIC_CHAIN" + ); + for r in [&unsent, &request] { + ensure!( + r.journal_complete + && r.assignment_version == e.assignment_version + && r.journal_id == rejected.journal_id.to_string() + && r.helper_journal_id == helper.to_string(), + "REPAIR_REQUEST_SCOPE" + ); + } + ensure!( + request.checkpoint == rejected.checkpoint + && request.helper_checkpoint == rejected.helper_checkpoint + && request.helper_previous_decision_id == old.decision_id.to_string(), + "REPAIR_REQUEST_CHECKPOINT" + ); + let am = &e.agent_inventory["reconcile_meta"]; + let a = rows(am)?; + ensure!(a.len() == 1, "REPAIR_AGENT_META"); + let a = &a[0]; + ensure!( + integer(am, a, "id")? == 1 + && text(am, a, "journal_id")? == rejected.journal_id.to_string() + && integer(am, a, "checkpoint")? == rejected.checkpoint + && integer(am, a, "high_sequence")? >= rejected.checkpoint + && integer(am, a, "generation")? == rejected.connection_generation + && text(am, a, "last_decision")? == rejected.decision_id.to_string() + && text(am, a, "helper_id")? == helper.to_string() + && integer(am, a, "helper_checkpoint")? == rejected.helper_checkpoint + && text(am, a, "fault")? == FAULT + && cell(am, a, "pending_request")? == &json!({"type":"null"}), + "REPAIR_AGENT_HEAD" + ); + let hm = &e.helper_inventory["helper_sync_meta"]; + let h = rows(hm)?; + ensure!(h.len() == 1, "REPAIR_HELPER_META"); + let h = &h[0]; + ensure!( + integer(hm, h, "id")? == 1 + && text(hm, h, "journal_id")? == helper.to_string() + && integer(hm, h, "checkpoint")? == rejected.helper_checkpoint + && text(hm, h, "last_decision")? == rejected.decision_id.to_string() + && text(hm, h, "agent_journal_id")? == rejected.journal_id.to_string() + && integer(hm, h, "agent_checkpoint")? == rejected.checkpoint + && text(hm, h, "fault")? == FAULT, + "REPAIR_HELPER_HEAD" + ); + let ar = &e.agent_inventory["reconcile_requests"]; + let x = row_by(ar, "id", &unsent.request_id)?; + blob(ar, x, "request", &e.uncommitted_request)?; + ensure!( + integer(ar, x, "generation")? <= rejected.connection_generation + && cell(ar, x, "response")? == &json!({"type":"null"}), + "REPAIR_UNSENT_LOCAL_RESPONSE" + ); + let y = row_by(ar, "id", &request.request_id)?; + blob(ar, y, "request", &e.rejected_request)?; + blob( + ar, + y, + "response", + &rmm_protocol::SyncResponse { + decision_bytes: e.rejected_response.decision_bytes.clone(), + signature: e.rejected_response.signature.clone(), + } + .encode_to_vec(), + )?; + ensure!( + integer(ar, y, "generation")? == rejected.connection_generation, + "REPAIR_LOCAL_GENERATION" + ); + let hr = &e.helper_inventory["helper_sync_decisions"]; + let y = row_by(hr, "decision_id", &rejected.decision_id.to_string())?; + blob(hr, y, "request", &e.rejected_request)?; + blob( + hr, + y, + "response", + &serde_json::to_vec(&e.rejected_response)?, + )?; + ensure!( + integer(hr, y, "checkpoint")? == rejected.helper_checkpoint, + "REPAIR_HELPER_CAPTURE" + ); + ensure!( + e.helper_inventory["consumed"].is_object() + && e.helper_inventory["execution_evidence"].is_object(), + "REPAIR_LEDGER_REQUIRED" + ); + Ok(Case { + old, + rejected, + unsent, + request, + }) +} + +pub async fn issue(state: &crate::State, e: Evidence) -> Result { + closed_policy()?; + issue_with_closed_policy(state, e).await +} +async fn issue_with_closed_policy(state: &crate::State, e: Evidence) -> Result { + let c = validate(&state.job_signer, &e)?; + let evidence_hash = inventory_sha256(&serde_json::to_value(&e)?)?; + let mut tx = rmm_persistence::tenant_tx(&state.pool, e.tenant_id).await?; + crate::agent_service::control_lock(&mut tx, e.tenant_id, e.agent_id).await?; + let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,revoked_at,sequence_regressed,sync_recovery_required,sync_ready,helper_sync_ready FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE") + .bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_DEVICE_MISSING")?; + ensure!( + device.get::("asset_id") == e.asset_id + && device.get::("enrollment_generation") + == i64::try_from(e.enrollment_generation)? + && device.get::("assignment_version") == i64::try_from(e.assignment_version)? + // Faulted journals may survive newer ClientHello channels. This + // historical receipt grants no lease; the next ordinary sync must + // still bind its current channel. A regressed generation or changed + // boot/identity is never repaired here. + && device.get::("connection_generation") + >= i64::try_from(c.rejected.connection_generation)? + && device.get::, _>("connection_boot_id") == Some(c.rejected.boot_id) + && device + .get::>, _>("revoked_at") + .is_none() + && ![ + "sequence_regressed", + "sync_recovery_required", + "sync_ready", + "helper_sync_ready" + ] + .iter() + .any(|k| device.get::(*k)), + "REPAIR_DEVICE_FENCED" + ); + for (d, s) in [ + (&c.old, &e.previous_valid_response), + (&c.rejected, &e.rejected_response), + ] { + let stored=sqlx::query("SELECT decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3") + .bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).fetch_optional(&mut *tx).await?.context("REPAIR_STORED_DECISION_MISSING")?; + ensure!( + stored.get::("decision_id") == d.decision_id + && stored.get::("request_hash") == d.request_hash + && stored.get::("journal_id") == d.journal_id + && stored.get::("checkpoint") == i64::try_from(d.checkpoint)? + && stored.get::("connection_generation") + == i64::try_from(d.connection_generation)? + && stored.get::, _>("payload") == s.decision_bytes + && stored.get::, _>("signature") == s.signature, + "REPAIR_STORED_DECISION_MISMATCH" + ); + } + let committed:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3)") + .bind(e.tenant_id).bind(e.agent_id).bind(Uuid::parse_str(&c.unsent.request_id)?).fetch_one(&mut *tx).await?; + ensure!(!committed, "REPAIR_REQUEST_WAS_COMMITTED"); + let head=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE") + .bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_HEAD_MISSING")?; + let previous:Option=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type=$2 AND data->>'evidence_sha256'=$3 AND data->>'agent_id'=$4 ORDER BY created_at DESC LIMIT 1") + .bind(e.tenant_id).bind(EVENT).bind(&evidence_hash).bind(e.agent_id.to_string()).fetch_optional(&mut *tx).await?; + if let Some(previous) = previous { + let signed: SignedReceipt = serde_json::from_value(previous["receipt"].clone())?; + let receipt: Receipt = serde_json::from_slice(&signed.payload)?; + let key = PKey::public_key_from_raw_bytes(&state.job_signer.public_key()?, Id::ED25519)?; + ensure!( + signed.signature.len() == 64 + && Verifier::new_without_digest(&key)? + .verify_oneshot(&signed.signature, &signed.payload)? + && receipt.valid_profile(Utc::now()) + && receipt.key_id == state.job_signer.key_id + && receipt.tenant_id == e.tenant_id + && receipt.asset_id == e.asset_id + && receipt.agent_id == e.agent_id + && receipt.rejected_decision_id == c.rejected.decision_id + && receipt.rejected_request_sha256 == hash(&e.rejected_request) + && receipt.uncommitted_request_sha256 == hash(&e.uncommitted_request) + && receipt.agent_inventory_sha256 == inventory_sha256(&e.agent_inventory)? + && receipt.helper_inventory_sha256 == inventory_sha256(&e.helper_inventory)? + && head_matches(&head, &c.rejected)?, + "REPAIR_REPLAY_EXPIRED_OR_HEAD_CHANGED" + ); + tx.commit().await?; + return Ok(signed); + } + ensure!( + head_matches(&head, &c.old)?, + "REPAIR_AUTHORITATIVE_HEAD_CHANGED" + ); + let now = Utc::now(); + let receipt = Receipt { + domain: "rmm.sync.unsent-request-repair".into(), + schema_version: 1, + receipt_id: Uuid::new_v4(), + tenant_id: e.tenant_id, + asset_id: e.asset_id, + agent_id: e.agent_id, + enrollment_generation: e.enrollment_generation, + assignment_version: e.assignment_version, + issued_at: now, + valid_until: now + Duration::minutes(10), + key_id: state.job_signer.key_id.clone(), + fault: FAULT.into(), + agent_journal_id: c.rejected.journal_id, + agent_checkpoint: c.rejected.checkpoint, + helper_journal_id: c + .rejected + .helper_journal_id + .context("REPAIR_HELPER_REQUIRED")?, + helper_checkpoint: c.rejected.helper_checkpoint, + rejected_decision_id: c.rejected.decision_id, + rejected_decision_sha256: hash(&e.rejected_response.decision_bytes), + rejected_request_id: c.rejected.request_id, + rejected_request_sha256: hash(&e.rejected_request), + uncommitted_request_id: Uuid::parse_str(&c.unsent.request_id)?, + uncommitted_request_sha256: hash(&e.uncommitted_request), + previous_valid_decision_id: c.old.decision_id, + previous_valid_decision_sha256: hash(&e.previous_valid_response.decision_bytes), + agent_inventory_sha256: inventory_sha256(&e.agent_inventory)?, + helper_inventory_sha256: inventory_sha256(&e.helper_inventory)?, + helper_ledger_sha256: helper_ledger_sha256(&e.helper_inventory)?, + ready: false, + helper_ready: false, + unlock: false, + }; + ensure!(receipt.valid_profile(now), "REPAIR_RECEIPT_PROFILE"); + let payload = serde_json::to_vec(&receipt)?; + let signed = SignedReceipt { + signature: state.job_signer.sign(&payload)?, + payload, + }; + ensure!(sqlx::query("UPDATE agent_sync_state SET checkpoint=$3,decision_id=$4,helper_journal_id=$5,helper_checkpoint=$6 WHERE tenant_id=$1 AND agent_id=$2 AND decision_id=$7") + .bind(e.tenant_id).bind(e.agent_id).bind(i64::try_from(c.request.checkpoint)?).bind(c.rejected.decision_id) + .bind(c.rejected.helper_journal_id).bind(i64::try_from(c.rejected.helper_checkpoint)?).bind(c.old.decision_id) + .execute(&mut *tx).await?.rows_affected()==1,"REPAIR_HEAD_CONCURRENT_CHANGE"); + sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,$4,$5)") + .bind(e.tenant_id).bind(receipt.receipt_id).bind("development-bootstrap:unsent-sync-repair").bind(EVENT) + .bind(json!({"agent_id":e.agent_id,"evidence_sha256":evidence_hash,"receipt_sha256":hash(&signed.payload),"receipt":signed,"ready":false,"helper_ready":false,"unlock":false})) + .execute(&mut *tx).await?; + tx.commit().await?; + Ok(signed) +} +fn head_matches(head: &sqlx::postgres::PgRow, d: &ReconcileDecisionPayload) -> Result { + Ok(head.get::("journal_id") == d.journal_id + && head.get::("checkpoint") == i64::try_from(d.checkpoint)? + && head.get::("decision_id") == d.decision_id + && head.get::, _>("helper_journal_id") == d.helper_journal_id + && head.get::("helper_checkpoint") == i64::try_from(d.helper_checkpoint)?) +} + +#[cfg(test)] +mod tests { + use super::*; + fn table(fields: Vec<(&str, Value)>) -> Value { + json!({"columns":fields.iter().map(|(k,_)|k).collect::>(),"rows":[fields.iter().map(|(_,v)|v).collect::>() ]}) + } + fn t(v: impl ToString) -> Value { + json!({"type":"text","value":v.to_string()}) + } + fn n(v: u64) -> Value { + json!({"type":"integer","value":v}) + } + fn b(v: &[u8]) -> Value { + json!({"type":"blob","sha256":hash(v),"length":v.len()}) + } + fn sign(s: &crate::job_signer::JobSigner, d: &ReconcileDecisionPayload) -> SignedDecision { + let decision_bytes = serde_json::to_vec(d).unwrap(); + SignedDecision { + signature: s.sign(&decision_bytes).unwrap(), + decision_bytes, + } + } + fn fixture() -> (crate::job_signer::JobSigner, Evidence, std::path::PathBuf) { + let path = std::path::PathBuf::from(format!("/tmp/rmm-repair-{}", Uuid::new_v4())); + let signer = crate::job_signer::JobSigner::load_or_create(&path.join("key.pem")).unwrap(); + let tenant = Uuid::new_v4(); + let asset = Uuid::new_v4(); + let agent = Uuid::new_v4(); + let journal = Uuid::new_v4(); + let helper = Uuid::new_v4(); + let now = Utc::now(); + let old = ReconcileDecisionPayload { + domain: "rmm.reconciliation-decision".into(), + schema_version: 1, + decision_id: Uuid::new_v4(), + request_id: Uuid::new_v4(), + request_hash: "ab".repeat(32), + tenant_id: tenant, + asset_id: asset, + agent_id: agent, + enrollment_generation: 1, + assignment_version: 1, + connection_generation: 3, + boot_id: Uuid::new_v4(), + journal_id: journal, + checkpoint: 10, + helper_journal_id: Some(helper), + helper_checkpoint: 5, + issued_at: now, + valid_until: now + Duration::seconds(60), + ready: false, + helper_ready: false, + reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(), + attempts: vec![], + acknowledged_message_ids: vec![], + key_id: signer.key_id.clone(), + }; + let x = SyncRequest { + request_id: Uuid::new_v4().to_string(), + journal_id: journal.to_string(), + checkpoint: 11, + previous_decision_id: old.decision_id.to_string(), + assignment_version: 1, + journal_complete: true, + helper_journal_id: helper.to_string(), + helper_checkpoint: 6, + helper_previous_decision_id: old.decision_id.to_string(), + ..Default::default() + }; + let y = SyncRequest { + request_id: Uuid::new_v4().to_string(), + checkpoint: 12, + previous_decision_id: String::new(), + previous_request_id: x.request_id.clone(), + helper_checkpoint: 7, + ..x.clone() + }; + let mut rejected = old.clone(); + rejected.decision_id = Uuid::new_v4(); + rejected.request_id = Uuid::parse_str(&y.request_id).unwrap(); + rejected.request_hash = hash(&y.encode_to_vec()); + rejected.checkpoint = 12; + rejected.helper_checkpoint = 7; + rejected.reason = FAULT.into(); + let response = sign(&signer, &rejected); + let request = y.encode_to_vec(); + let unsent = x.encode_to_vec(); + let proto = rmm_protocol::SyncResponse { + decision_bytes: response.decision_bytes.clone(), + signature: response.signature.clone(), + } + .encode_to_vec(); + let mut requests = table(vec![ + ("id", t(&x.request_id)), + ("request", b(&unsent)), + ("generation", n(3)), + ("response", json!({"type":"null"})), + ]); + requests["rows"].as_array_mut().unwrap().push(json!([ + t(&y.request_id), + b(&request), + n(3), + b(&proto) + ])); + let am = table(vec![ + ("id", n(1)), + ("journal_id", t(journal)), + ("checkpoint", n(12)), + ("high_sequence", n(13)), + ("generation", n(3)), + ("last_decision", t(rejected.decision_id)), + ("pending_request", json!({"type":"null"})), + ("helper_id", t(helper)), + ("helper_checkpoint", n(7)), + ("fault", t(FAULT)), + ]); + let hm = table(vec![ + ("id", n(1)), + ("journal_id", t(helper)), + ("checkpoint", n(7)), + ("last_decision", t(rejected.decision_id)), + ("agent_journal_id", t(journal)), + ("agent_checkpoint", n(12)), + ("fault", t(FAULT)), + ]); + let history = table(vec![ + ("decision_id", t(rejected.decision_id)), + ("checkpoint", n(7)), + ("response", b(&serde_json::to_vec(&response).unwrap())), + ("request", b(&request)), + ]); + let e = Evidence { + schema_version: 1, + tenant_id: tenant, + asset_id: asset, + agent_id: agent, + enrollment_generation: 1, + assignment_version: 1, + agent_inventory: json!({"reconcile_meta":am,"reconcile_requests":requests}), + helper_inventory: json!({"helper_sync_meta":hm,"helper_sync_decisions":history,"consumed":{"columns":[],"rows":[]},"execution_evidence":{"columns":[],"rows":[]}}), + uncommitted_request: unsent, + rejected_request: request, + rejected_response: response, + previous_valid_response: sign(&signer, &old), + }; + (signer, e, path) + } + #[test] + fn only_exact_signed_unsent_case_validates() { + let (s, e, p) = fixture(); + validate(&s, &e).unwrap(); + let mut other = e.clone(); + other.tenant_id = Uuid::new_v4(); + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + other.asset_id = Uuid::new_v4(); + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + other.assignment_version += 1; + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + other.rejected_response.signature[0] ^= 1; + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + other.agent_inventory["reconcile_requests"]["rows"][0][3] = b(b"committed response"); + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + other.helper_inventory["helper_sync_meta"]["rows"][0][2] = n(8); + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + other.agent_inventory["reconcile_meta"]["rows"][0][9] = t("OTHER_FAULT"); + assert!(validate(&s, &other).is_err()); + let mut other = e.clone(); + let mut x = SyncRequest::decode(other.uncommitted_request.as_slice()).unwrap(); + x.previous_decision_id = Uuid::new_v4().to_string(); + other.uncommitted_request = x.encode_to_vec(); + assert!(validate(&s, &other).is_err()); + fs::remove_dir_all(p).unwrap(); + } + #[test] + fn signed_ready_response_is_never_repair_authority() { + let (s, mut e, p) = fixture(); + let mut d: ReconcileDecisionPayload = + serde_json::from_slice(&e.rejected_response.decision_bytes).unwrap(); + d.ready = true; + e.rejected_response = sign(&s, &d); + assert!(validate(&s, &e).is_err()); + d.ready = false; + d.reason = "CONSISTENT".into(); + e.rejected_response = sign(&s, &d); + assert!(validate(&s, &e).is_err()); + fs::remove_dir_all(p).unwrap(); + } + /// Opt-in dedicated PostgreSQL only; the harness creates/destroys its own + /// network-isolated container and never reads deployment credentials. + #[tokio::test] + #[ignore = "requires the dedicated isolated PostgreSQL harness"] + async fn isolated_database_repair_and_fail_closed_paths() { + let admin_url = std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap(); + let app_url = std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap(); + let admin = sqlx::PgPool::connect(&admin_url).await.unwrap(); + let pool = sqlx::PgPool::connect(&app_url).await.unwrap(); + rmm_persistence::verify_application_role(&pool) + .await + .unwrap(); + let (s, e, p) = fixture(); + let c = validate(&s, &e).unwrap(); + sqlx::query("INSERT INTO tenants VALUES($1,'isolated repair test')") + .bind(e.tenant_id) + .execute(&admin) + .await + .unwrap(); + sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)") + .bind(e.tenant_id) + .bind(e.asset_id) + .execute(&admin) + .await + .unwrap(); + sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,assignment_version,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5,$6)") + .bind(e.tenant_id).bind(e.agent_id).bind(e.asset_id).bind(hash(e.agent_id.as_bytes())).bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(c.rejected.boot_id).execute(&admin).await.unwrap(); + sqlx::query("INSERT INTO agent_sync_state VALUES($1,$2,$3,$4,$5,$6,$7)") + .bind(e.tenant_id) + .bind(e.agent_id) + .bind(c.old.journal_id) + .bind(i64::try_from(c.old.checkpoint).unwrap()) + .bind(c.old.decision_id) + .bind(c.old.helper_journal_id) + .bind(i64::try_from(c.old.helper_checkpoint).unwrap()) + .execute(&admin) + .await + .unwrap(); + for (d, signed) in [ + (&c.old, &e.previous_valid_response), + (&c.rejected, &e.rejected_response), + ] { + sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)") + .bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).bind(d.decision_id).bind(&d.request_hash).bind(d.journal_id).bind(i64::try_from(d.checkpoint).unwrap()).bind(i64::try_from(d.connection_generation).unwrap()).bind(&signed.decision_bytes).bind(&signed.signature).execute(&admin).await.unwrap(); + } + let state = crate::State { + pool, + signer: None, + viewer_a: [0; 32], + viewer_b: [1; 32], + auth: std::sync::Arc::new(crate::auth::Auth::from_env("development").await.unwrap()), + job_signer: std::sync::Arc::new(s), + }; + let mut wrong = e.clone(); + wrong.tenant_id = Uuid::new_v4(); + assert!(issue_with_closed_policy(&state, wrong).await.is_err()); + sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2") + .bind(e.tenant_id) + .bind(c.rejected.request_id) + .execute(&admin) + .await + .unwrap(); + assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); + sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)") + .bind(e.tenant_id).bind(e.agent_id).bind(c.rejected.request_id).bind(c.rejected.decision_id) + .bind(&c.rejected.request_hash).bind(c.rejected.journal_id).bind(i64::try_from(c.rejected.checkpoint).unwrap()) + .bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(&e.rejected_response.decision_bytes) + .bind(&e.rejected_response.signature).execute(&admin).await.unwrap(); + sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint+1 WHERE agent_id=$1") + .bind(e.agent_id) + .execute(&admin) + .await + .unwrap(); + assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); + sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint-1 WHERE agent_id=$1") + .bind(e.agent_id) + .execute(&admin) + .await + .unwrap(); + sqlx::query("UPDATE agent_installations SET revoked_at=now() WHERE agent_id=$1") + .bind(e.agent_id) + .execute(&admin) + .await + .unwrap(); + assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); + sqlx::query("UPDATE agent_installations SET revoked_at=NULL WHERE agent_id=$1") + .bind(e.agent_id) + .execute(&admin) + .await + .unwrap(); + let x = Uuid::parse_str(&c.unsent.request_id).unwrap(); + sqlx::query("INSERT INTO sync_checkpoints SELECT tenant_id,agent_id,$2,$3,request_hash,journal_id,checkpoint,connection_generation,payload,signature,created_at FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$4") + .bind(e.tenant_id).bind(x).bind(Uuid::new_v4()).bind(c.old.request_id).execute(&admin).await.unwrap(); + assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); + sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2") + .bind(e.tenant_id) + .bind(x) + .execute(&admin) + .await + .unwrap(); + sqlx::query("UPDATE agent_installations SET connection_generation=$2 WHERE agent_id=$1") + .bind(e.agent_id) + .bind(i64::try_from(c.rejected.connection_generation).unwrap() - 1) + .execute(&admin) + .await + .unwrap(); + assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); + sqlx::query("UPDATE agent_installations SET connection_generation=$2,connection_boot_id=$3 WHERE agent_id=$1") + .bind(e.agent_id).bind(i64::try_from(c.rejected.connection_generation).unwrap()+1).bind(Uuid::new_v4()) + .execute(&admin).await.unwrap(); + assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); + sqlx::query("UPDATE agent_installations SET connection_boot_id=$2 WHERE agent_id=$1") + .bind(e.agent_id) + .bind(c.rejected.boot_id) + .execute(&admin) + .await + .unwrap(); + let first = issue_with_closed_policy(&state, e.clone()).await.unwrap(); + assert_eq!( + first, + issue_with_closed_policy(&state, e.clone()).await.unwrap() + ); + let r: Receipt = serde_json::from_slice(&first.payload).unwrap(); + assert!(r.valid_profile(Utc::now())); + assert!(!r.ready && !r.helper_ready && !r.unlock); + let head: Uuid = + sqlx::query_scalar("SELECT decision_id FROM agent_sync_state WHERE agent_id=$1") + .bind(e.agent_id) + .fetch_one(&admin) + .await + .unwrap(); + assert_eq!(head, c.rejected.decision_id); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1") + .bind(e.tenant_id) + .fetch_one(&admin) + .await + .unwrap(); + assert_eq!(count, 1); + let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM outbox)+(SELECT count(*) FROM execution_permits)").fetch_one(&admin).await.unwrap(); + assert_eq!(effects, 0); + let ready: bool = sqlx::query_scalar( + "SELECT sync_ready OR helper_sync_ready FROM agent_installations WHERE agent_id=$1", + ) + .bind(e.agent_id) + .fetch_one(&admin) + .await + .unwrap(); + assert!(!ready); + fs::remove_dir_all(p).unwrap(); + } +} diff --git a/crates/server/src/dev_delivery_faults.rs b/crates/server/src/dev_delivery_faults.rs new file mode 100644 index 0000000..c7644e7 --- /dev/null +++ b/crates/server/src/dev_delivery_faults.rs @@ -0,0 +1,570 @@ +//! Narrow dev/test wire faults after normal authorization/durable ingest. +//! Never creates jobs, grants or approvals. Claims and observations are bounded, +//! insert-only audit slots; a restart cannot arm the same fault a second time. +use anyhow::{Context, Result, ensure}; +use chrono::{DateTime, Duration, Utc}; +use prost::Message; +use rmm_domain::jobs::{Action, JobPayload, hash}; +use rmm_protocol::{Envelope, Payload}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sqlx::{PgPool, Postgres, Row, Transaction}; +use std::{fs, os::unix::fs::MetadataExt, path::PathBuf}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +enum Mode { + DuplicateOfferOnce, + DropResultAckOnce, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Config { + schema_version: u32, + fault_id: Uuid, + tenant_id: Uuid, + agent_id: Uuid, + job_id: Uuid, + attempt_id: Uuid, + plan_sha256: String, + job_payload_sha256: String, + mode: Mode, + transport: String, + issued_at: DateTime, + expires_at: DateTime, +} +impl Config { + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == 1 + && [ + self.fault_id, + self.tenant_id, + self.agent_id, + self.job_id, + self.attempt_id + ] + .iter() + .all(|id| !id.is_nil()) + && valid_hash(&self.plan_sha256) + && valid_hash(&self.job_payload_sha256) + && matches!(self.transport.as_str(), "wss" | "https") + && self.issued_at.timestamp() > 0 + && self.expires_at > self.issued_at + && self.expires_at - self.issued_at <= Duration::minutes(15), + "DEV_DELIVERY_FAULT_CONFIG" + ); + Ok(()) + } + fn scope(&self, tenant: Uuid, agent: Uuid, transport: &str) -> bool { + self.tenant_id == tenant && self.agent_id == agent && self.transport == transport + } + fn job(&self, bytes: &[u8], payload_hash: &str, plan_hash: &str) -> Result { + if payload_hash != self.job_payload_sha256 || plan_hash != self.plan_sha256 { + return Ok(false); + } + ensure!( + hash(bytes) == self.job_payload_sha256, + "DEV_FAULT_PAYLOAD_HASH" + ); + let job: JobPayload = serde_json::from_slice(bytes)?; + Ok(job.schema_version == 1 + && job.tenant_id == self.tenant_id + && job.agent_id == self.agent_id + && job.job_id == self.job_id + && job.attempt_id == self.attempt_id + && job.execution_identity == "agent_service" + && job.enrollment_generation > 0 + && job.assignment_version > 0 + && job.max_runtime_seconds > 0 + && job.max_runtime_seconds <= 60 + && job.max_output_bytes <= 65536 + && job.action.validate() + && matches!(job.action, Action::RunScript { .. }) + && self.expires_at <= job.start_before) + } +} + +fn valid_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && value.bytes().any(|b| b != b'0') +} +fn configured_path( + flag: Option<&str>, + path: Option<&str>, + profile: &str, + dev: &str, +) -> Result> { + match (flag, path) { + (None | Some("false"), None) => Ok(None), + (Some("true"), Some(path)) => { + ensure!( + profile == "development" && dev == "true", + "DEV_DELIVERY_FAULT_PROFILE" + ); + ensure!( + path == "/root-policy/delivery-fault.json", + "DEV_DELIVERY_FAULT_PATH" + ); + Ok(Some(PathBuf::from(path))) + } + _ => anyhow::bail!("DEV_DELIVERY_FAULT_EXPLICIT_OPT_IN"), + } +} +#[derive(Clone)] +struct Loaded { + config: Config, + config_sha256: String, +} +fn load() -> Result> { + let flag = std::env::var("RMM_DEV_DELIVERY_FAULTS").ok(); + let path = std::env::var("RMM_DEV_DELIVERY_FAULT_FILE").ok(); + let profile = std::env::var("RMM_PROFILE").unwrap_or_default(); + let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default(); + let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else { + return Ok(None); + }; + let parent = path.parent().context("DEV_FAULT_PARENT")?; + let directory = fs::symlink_metadata(parent)?; + let metadata = fs::symlink_metadata(&path)?; + ensure!( + directory.is_dir() + && directory.uid() == 10001 + && directory.mode() & 0o777 == 0o700 + && metadata.is_file() + && metadata.uid() == 10001 + && metadata.mode() & 0o777 == 0o600 + && metadata.nlink() == 1 + && metadata.len() <= 8192, + "DEV_FAULT_PRIVATE_CONFIG" + ); + // /root-policy is the existing operator-provisioned read-only bind mount. + let bytes = fs::read(&path)?; + ensure!(bytes.len() <= 8192, "DEV_FAULT_CONFIG_SIZE"); + let config: Config = serde_json::from_slice(&bytes)?; + config.validate()?; + if Utc::now() < config.issued_at || Utc::now() >= config.expires_at { + return Ok(None); + } + Ok(Some(Loaded { + config, + config_sha256: hash(&bytes), + })) +} +pub(crate) fn validate_environment() -> Result<()> { + load().map(|_| ()) +} + +fn slot_id(fault: Uuid, slot: &str) -> Uuid { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(format!("rmm.dev-delivery-fault.v1:{fault}:{slot}")); + Uuid::from_slice(&digest[..16]).expect("fixed digest prefix") +} +fn envelope_hash(envelope: &Envelope) -> String { + let mut normalized = envelope.clone(); + normalized.connection_generation = 0; + hash(&normalized.encode_to_vec()) +} +fn data(loaded: &Loaded, envelope: &Envelope, detail: &str) -> Value { + let config = &loaded.config; + json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true, + "fault_id":config.fault_id,"mode":config.mode,"transport":config.transport, + "agent_id":config.agent_id,"job_id":config.job_id,"attempt_id":config.attempt_id, + "plan_sha256":config.plan_sha256,"job_payload_sha256":config.job_payload_sha256, + "config_sha256":loaded.config_sha256,"message_id":envelope.message_id, + "normalized_envelope_sha256":envelope_hash(envelope),"detail":detail}) +} +async fn transaction<'a>(pool: &'a PgPool, loaded: &Loaded) -> Result> { + let mut tx = rmm_persistence::tenant_tx(pool, loaded.config.tenant_id).await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))") + .bind(format!("dev-delivery-fault:{}", loaded.config.fault_id)) + .execute(&mut *tx) + .await?; + Ok(tx) +} +async fn get_slot( + tx: &mut Transaction<'_, Postgres>, + loaded: &Loaded, + slot: &str, +) -> Result> { + let value: Option = sqlx::query_scalar("SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_delivery_fault_adapter'") + .bind(slot_id(loaded.config.fault_id, slot)).fetch_optional(&mut **tx).await?; + if let Some(value) = &value { + ensure!( + value.get("config_sha256") == Some(&json!(loaded.config_sha256)), + "DEV_FAULT_ID_REUSED_WITH_CHANGED_CONFIG" + ); + } + Ok(value) +} +async fn insert_slot( + tx: &mut Transaction<'_, Postgres>, + loaded: &Loaded, + slot: &str, + value: Value, +) -> Result { + if get_slot(tx, loaded, slot).await?.is_some() { + return Ok(false); + } + let rows = sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_delivery_fault_adapter',$3,$4) ON CONFLICT(tenant_id,event_id) DO NOTHING") + .bind(loaded.config.tenant_id).bind(slot_id(loaded.config.fault_id, slot)) + .bind(format!("dev_delivery_fault.{slot}")).bind(value).execute(&mut **tx).await?.rows_affected(); + Ok(rows == 1) +} +async fn authorized_job( + tx: &mut Transaction<'_, Postgres>, + loaded: &Loaded, + message: Option<&Envelope>, +) -> Result { + let c = &loaded.config; + let row = sqlx::query("SELECT j.payload,j.payload_hash,p.plan_hash FROM jobs j JOIN operations o USING(tenant_id,operation_id) JOIN approval_requests r USING(tenant_id,operation_id) JOIN operation_plans p USING(tenant_id,plan_id) JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 AND r.approval_state='approved' AND a.testsimulation AND a.approval_origin='dev_test_browser_simulation' AND a.principal='simulated-testprincipal-a'") + .bind(c.job_id).bind(c.attempt_id).bind(c.agent_id).fetch_optional(&mut **tx).await?; + let Some(row) = row else { + return Ok(false); + }; + if !c.job( + &row.get::, _>("payload"), + &row.get::("payload_hash"), + &row.get::("plan_hash"), + )? { + return Ok(false); + } + if let Some(message) = message { + let receipt: Option = sqlx::query_scalar( + "SELECT payload_hash FROM job_receipts WHERE message_id=$1 AND agent_id=$2", + ) + .bind(Uuid::parse_str(&message.message_id)?) + .bind(c.agent_id) + .fetch_optional(&mut **tx) + .await?; + ensure!( + receipt.as_deref() == Some(envelope_hash(message).as_str()), + "DEV_FAULT_DURABLE_RECEIPT_REQUIRED" + ); + } + Ok(true) +} + +pub(crate) struct DuplicateClaim { + loaded: Loaded, + envelope: Envelope, +} +/// Called only for work returned by normal postapproval `job_gateway::next`. +pub(crate) async fn duplicate_offer( + pool: &PgPool, + tenant: Uuid, + agent: Uuid, + envelope: &Envelope, + transport: &str, +) -> Result> { + let Some(Payload::JobOffer(offer)) = &envelope.payload else { + return Ok(None); + }; + let Some(loaded) = load()? else { + return Ok(None); + }; + let c = &loaded.config; + if c.mode != Mode::DuplicateOfferOnce + || !c.scope(tenant, agent, transport) + || offer.job_id != c.job_id.to_string() + || offer.attempt_id != c.attempt_id.to_string() + || !c.job( + &offer.payload_bytes, + &offer.job_payload_hash, + &offer.plan_hash, + )? + { + return Ok(None); + } + let mut tx = transaction(pool, &loaded).await?; + let claimed = authorized_job(&mut tx, &loaded, None).await? + && insert_slot( + &mut tx, + &loaded, + "claim", + data(&loaded, envelope, "duplicate_offer_reserved_before_send"), + ) + .await?; + tx.commit().await?; + Ok(claimed.then(|| DuplicateClaim { + loaded, + envelope: envelope.clone(), + })) +} +/// WSS calls after both sends complete. HTTP labels construction honestly; the +/// two later distinct postcommit JobAccepted slots prove client observation. +pub(crate) async fn duplicate_emitted(pool: &PgPool, claim: DuplicateClaim) -> Result<()> { + let mut tx = transaction(pool, &claim.loaded).await?; + ensure!( + get_slot(&mut tx, &claim.loaded, "claim").await?.is_some(), + "DEV_FAULT_CLAIM_REQUIRED" + ); + let detail = if claim.loaded.config.transport == "wss" { + "same_encoded_offer_sent_twice" + } else { + "same_encoded_offer_in_http_response_twice" + }; + insert_slot( + &mut tx, + &claim.loaded, + "emitted", + data(&claim.loaded, &claim.envelope, detail), + ) + .await?; + tx.commit().await?; + Ok(()) +} +/// Called AFTER durable ingest returned its concrete ACK. Return true only to +/// omit that single matching result ACK; every subsequent replay is ACKed. +pub(crate) async fn after_ingest( + pool: &PgPool, + tenant: Uuid, + agent: Uuid, + incoming: &Envelope, + reply: &Envelope, + transport: &str, +) -> Result { + let (job, attempt, is_result) = match &incoming.payload { + Some(Payload::JobAccepted(value)) => (&value.job_id, &value.attempt_id, false), + Some(Payload::JobResult(value)) if value.state == "succeeded" => { + (&value.job_id, &value.attempt_id, true) + } + _ => return Ok(false), + }; + let Some(loaded) = load()? else { + return Ok(false); + }; + let c = &loaded.config; + if !c.scope(tenant, agent, transport) + || job != &c.job_id.to_string() + || attempt != &c.attempt_id.to_string() + { + return Ok(false); + } + let Some(Payload::Ack(ack)) = &reply.payload else { + return Ok(false); + }; + ensure!( + ack.message_id == incoming.message_id, + "DEV_FAULT_CONCRETE_ACK_REQUIRED" + ); + let mut tx = transaction(pool, &loaded).await?; + if !authorized_job(&mut tx, &loaded, Some(incoming)).await? { + tx.commit().await?; + return Ok(false); + } + let claim = get_slot(&mut tx, &loaded, "claim").await?; + let mut omit = false; + if is_result && c.mode == Mode::DropResultAckOnce { + if let Some(first) = claim { + if first.get("message_id") == Some(&json!(incoming.message_id)) + && first.get("normalized_envelope_sha256") == Some(&json!(envelope_hash(incoming))) + { + insert_slot( + &mut tx, + &loaded, + "replay", + data( + &loaded, + incoming, + "same_original_result_replayed_after_ack_omission", + ), + ) + .await?; + } + } else { + omit = insert_slot( + &mut tx, + &loaded, + "claim", + data( + &loaded, + incoming, + "one_concrete_result_ack_omitted_after_durable_commit", + ), + ) + .await?; + } + } else if !is_result && c.mode == Mode::DuplicateOfferOnce && claim.is_some() { + if let Some(first) = get_slot(&mut tx, &loaded, "accepted_1").await? { + if first.get("message_id") != Some(&json!(incoming.message_id)) { + insert_slot( + &mut tx, + &loaded, + "accepted_2", + data(&loaded, incoming, "second_distinct_client_acceptance"), + ) + .await?; + } + } else { + insert_slot( + &mut tx, + &loaded, + "accepted_1", + data(&loaded, incoming, "first_client_acceptance"), + ) + .await?; + } + } + tx.commit().await?; + Ok(omit) +} + +#[cfg(test)] +mod tests { + use super::*; + fn config() -> Config { + let now = Utc::now(); + Config { + schema_version: 1, + fault_id: Uuid::new_v4(), + tenant_id: Uuid::new_v4(), + agent_id: Uuid::new_v4(), + job_id: Uuid::new_v4(), + attempt_id: Uuid::new_v4(), + plan_sha256: "a".repeat(64), + job_payload_sha256: "b".repeat(64), + mode: Mode::DuplicateOfferOnce, + transport: "wss".into(), + issued_at: now, + expires_at: now + Duration::minutes(10), + } + } + #[test] + fn opt_in_is_explicit_and_never_production() { + assert!( + configured_path(None, None, "production", "false") + .unwrap() + .is_none() + ); + assert!( + configured_path( + Some("true"), + Some("/root-policy/delivery-fault.json"), + "production", + "false" + ) + .is_err() + ); + assert!(configured_path(Some("true"), None, "development", "true").is_err()); + assert!( + configured_path( + None, + Some("/root-policy/delivery-fault.json"), + "development", + "true" + ) + .is_err() + ); + assert!(configured_path(Some("true"), Some("/etc/passwd"), "development", "true").is_err()); + assert!( + configured_path( + Some("true"), + Some("/root-policy/delivery-fault.json"), + "development", + "true" + ) + .unwrap() + .is_some() + ); + } + #[test] + fn narrow_schema_rejects_wildcards_nil_placeholders_and_long_lifetime() { + let c = config(); + c.validate().unwrap(); + let mut altered = c.clone(); + altered.expires_at = altered.issued_at + Duration::minutes(16); + assert!(altered.validate().is_err()); + altered = c.clone(); + altered.agent_id = Uuid::nil(); + assert!(altered.validate().is_err()); + altered = c.clone(); + altered.job_payload_sha256 = "0".repeat(64); + assert!(altered.validate().is_err()); + altered = c.clone(); + altered.transport = "*".into(); + assert!(altered.validate().is_err()); + let mut raw = serde_json::to_value(c).unwrap(); + raw["approve"] = json!(true); + assert!(serde_json::from_value::(raw).is_err()); + } + #[test] + fn audit_slots_are_stable_bounded_and_distinct() { + let id = Uuid::new_v4(); + assert_eq!(slot_id(id, "claim"), slot_id(id, "claim")); + assert_ne!(slot_id(id, "claim"), slot_id(id, "replay")); + assert_ne!(slot_id(id, "claim"), slot_id(Uuid::new_v4(), "claim")); + } + #[test] + fn exact_signed_job_scope_cannot_fault_root_other_targets_or_other_actions() { + let mut c = config(); + let mut job = JobPayload { + schema_version: 1, + tenant_id: c.tenant_id, + asset_id: Uuid::new_v4(), + agent_id: c.agent_id, + enrollment_generation: 1, + assignment_version: 1, + job_id: c.job_id, + attempt_id: c.attempt_id, + action: Action::RunScript { + interpreter: "posix_sh".into(), + script_utf8: "printf hello\\n".into(), + arguments: vec![], + expected_stdout: None, + }, + execution_identity: "agent_service".into(), + not_before: c.issued_at, + start_before: c.expires_at, + max_runtime_seconds: 60, + max_output_bytes: 4096, + }; + let mut bytes = serde_json::to_vec(&job).unwrap(); + c.job_payload_sha256 = hash(&bytes); + assert!( + c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256) + .unwrap() + ); + assert!(!c.scope(c.tenant_id, Uuid::new_v4(), "wss")); + assert!(!c.scope(c.tenant_id, c.agent_id, "https")); + assert!( + !c.job(&bytes, &c.job_payload_sha256, &"c".repeat(64)) + .unwrap() + ); + job.execution_identity = "root".into(); + bytes = serde_json::to_vec(&job).unwrap(); + c.job_payload_sha256 = hash(&bytes); + assert!( + !c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256) + .unwrap() + ); + job.execution_identity = "agent_service".into(); + job.action = Action::RefreshInventory { + scope: "linux.base".into(), + }; + bytes = serde_json::to_vec(&job).unwrap(); + c.job_payload_sha256 = hash(&bytes); + assert!( + !c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256) + .unwrap() + ); + } + #[test] + fn durable_replay_only_normalizes_connection_generation() { + let e = Envelope { + message_id: Uuid::new_v4().to_string(), + connection_generation: 3, + ..Default::default() + }; + let mut repeated = e.clone(); + repeated.connection_generation = 4; + assert_eq!(envelope_hash(&e), envelope_hash(&repeated)); + repeated.message_id = Uuid::new_v4().to_string(); + assert_ne!(envelope_hash(&e), envelope_hash(&repeated)); + } +} diff --git a/crates/server/src/gateway.rs b/crates/server/src/gateway.rs index 4ea3a26..1625f66 100644 --- a/crates/server/src/gateway.rs +++ b/crates/server/src/gateway.rs @@ -116,7 +116,10 @@ async fn receive(socket: &mut WebSocket) -> Result { Ok(Envelope::decode(bytes)?) } async fn send( + state: &crate::State, socket: &mut WebSocket, + tenant: Uuid, + agent: Uuid, e: &Envelope, payload: Payload, generation: u64, @@ -133,11 +136,22 @@ async fn send( sequence: e.sequence, payload: Some(payload), }; + let claim = + dev_delivery_faults::duplicate_offer(&state.pool, tenant, agent, &reply, "wss").await?; + let bytes = reply.encode_to_vec(); tokio::time::timeout( Duration::from_secs(10), - socket.send(Message::Binary(reply.encode_to_vec().into())), + socket.send(Message::Binary(bytes.clone().into())), ) .await??; + if let Some(claim) = claim { + tokio::time::timeout( + Duration::from_secs(10), + socket.send(Message::Binary(bytes.into())), + ) + .await??; + dev_delivery_faults::duplicate_emitted(&state.pool, claim).await?; + } Ok(()) } async fn channel( @@ -162,7 +176,7 @@ async fn channel( .send(Message::Binary(response.encode_to_vec().into())) .await?; if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? { - send(socket, &hello, payload, generation).await?; + send(state, socket, tenant, agent, &hello, payload, generation).await?; } let mut recent_messages = VecDeque::new(); let mut timer = tokio::time::interval(Duration::from_secs(1)); @@ -177,7 +191,7 @@ async fn channel( let mut tx=rmm_persistence::tenant_tx(&state.pool,tenant).await?; let revoked:bool=sqlx::query_scalar("SELECT revoked_at IS NOT NULL FROM agent_installations WHERE agent_id=$1").bind(agent).fetch_one(&mut *tx).await?; tx.commit().await?;ensure!(!revoked,"device revoked"); - if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(socket,&hello,payload,generation).await?;} + if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(state,socket,tenant,agent,&hello,payload,generation).await?;} continue; } }; @@ -198,11 +212,17 @@ async fn channel( "channel generation mismatch" ); let response = crate::agent_service::ingest(state, tenant, agent, &e, "wss").await?; - socket - .send(Message::Binary(response.encode_to_vec().into())) - .await?; + if !dev_delivery_faults::after_ingest(&state.pool, tenant, agent, &e, &response, "wss") + .await? + { + socket + .send(Message::Binary(response.encode_to_vec().into())) + .await?; + } if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? { - send(socket, &e, payload, generation).await?; + send(state, socket, tenant, agent, &e, payload, generation).await?; } } } +#[path = "dev_delivery_faults.rs"] +pub(crate) mod dev_delivery_faults; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index ad02b49..7a120c5 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -5,6 +5,7 @@ mod api; mod artifact_store; mod auth; mod catalog; +mod checkpoint_repair; mod controls; mod errors; mod gateway; @@ -68,6 +69,16 @@ async fn main() -> Result<()> { .await; } let profile = env::var("RMM_PROFILE").unwrap_or_default(); + gateway::dev_delivery_faults::validate_environment()?; + let repair = env::args().nth(1).as_deref() == Some("repair-unsent-sync-request"); + if repair { + ensure!(env::args().count() == 2, "REPAIR_NO_ARGUMENTS"); + checkpoint_repair::closed_policy()?; + ensure!( + std::path::Path::new(&env::var("RMM_JOB_KEY_FILE")?).is_file(), + "REPAIR_EXISTING_KEY_REQUIRED" + ); + } let dev = env::var("RMM_DEV_AUTH").as_deref() == Ok("true"); ensure!( matches!( @@ -126,6 +137,19 @@ async fn main() -> Result<()> { .auth .validate_separation(&state.viewer_a, &state.viewer_b)?; } + if repair { + use std::io::{Read, Write}; + let mut raw = Vec::new(); + std::io::stdin() + .take(32 * 1024 * 1024 + 1) + .read_to_end(&mut raw)?; + ensure!(raw.len() <= 32 * 1024 * 1024, "REPAIR_INPUT_SIZE"); + let evidence = serde_json::from_slice(&raw).context("REPAIR_EVIDENCE_JSON")?; + let receipt = checkpoint_repair::issue(&state, evidence).await?; + serde_json::to_writer(std::io::stdout().lock(), &receipt)?; + std::io::stdout().write_all(b"\n")?; + return Ok(()); + } if role.gateway() { scheduler::start(state.clone()); } diff --git a/deploy/cloud_apt_reconcile_upgrade.py b/deploy/cloud_apt_reconcile_upgrade.py index 2aef12c..71d4126 100644 --- a/deploy/cloud_apt_reconcile_upgrade.py +++ b/deploy/cloud_apt_reconcile_upgrade.py @@ -25,6 +25,7 @@ def upgrade(payload): base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote') ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper') provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json' + agent_binary=Path('/usr/local/libexec/rmm-agent-remote') dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf') full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n' 'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n' @@ -52,6 +53,7 @@ def upgrade(payload): guard() require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved') require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required') + require(sha(agent_binary.read_bytes())==payload['old_agent_sha256'],'reviewed permanent agent required') require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required') original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json' retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json' @@ -59,7 +61,9 @@ def upgrade(payload): sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required') require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected') raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True) + agent_raw=base64.b64decode(payload['agent_binary'],validate=True) require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts') + require(sha(agent_raw)==payload['agent_sha256'],'qualified sync-prevention agent') backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700) def create(path,data,mode=0o600): fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode) @@ -68,7 +72,9 @@ def upgrade(payload): def save(): next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json') save();staged=backup/'helper-next';create(staged,raw,0o755) + staged_agent=backup/'agent-next';create(staged_agent,agent_raw,0o755) run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)]) + run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged_agent)]) run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service']) fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB) try: @@ -77,11 +83,11 @@ def upgrade(payload): witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))} old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata') general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() - for p in (binary,provider,metadata,dropin): + for p in (binary,agent_binary,provider,metadata,dropin): target=backup/str(p).lstrip('/').replace('/','__') require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup') os.rename(p,target) - os.rename(staged,binary);create(provider,code,0o755) + os.rename(staged,binary);os.rename(staged_agent,agent_binary);create(provider,code,0o755) create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644) inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json'))) require(inspected==json.loads(general),'general policy unchanged') @@ -90,6 +96,7 @@ def upgrade(payload): (ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift') report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown, helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save() + report['agent_sha256']=sha(agent_raw);save() finally: fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd) run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service']) @@ -139,12 +146,16 @@ def main(): folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')] expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant] ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes() + agent_raw=(STATE/'agent-sync-prevention').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json') payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant, old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c', old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945', binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(), + agent_binary=base64.b64encode(agent_raw).decode(),agent_sha256=hashlib.sha256(agent_raw).hexdigest(),old_agent_sha256=installed['binary_sha256'], helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest()) report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report) + require(report['completed'] and report['agent_sha256']==payload['agent_sha256'],'qualified preserving installation') + m.write_private(cloud.ROOT/'devstate/remote-vm.json',dict(installed,binary_sha256=report['agent_sha256'])) print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')}) diff --git a/deploy/cloud_checkpoint_repair.py b/deploy/cloud_checkpoint_repair.py new file mode 100644 index 0000000..64b76d9 --- /dev/null +++ b/deploy/cloud_checkpoint_repair.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""One signed development control-chain repair, never a job or grant repair.""" +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import uuid + +import cloud_apt_reconcile_upgrade as upgrade +cloud=upgrade.cloud +require=cloud.require +STATE=cloud.ROOT/'devstate/checkpoint-chain-repair' + + +def prepared_json(path): + """The bounded full historical witness exceeds ordinary 1 MiB DTOs.""" + import stat + fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW) + with os.fdopen(fd,'r') as stream: + info=os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid==os.getuid() and + stat.S_IMODE(info.st_mode)==0o600 and info.st_nlink==1 and info.st_size<=45*1024*1024, + 'private bounded repair witness') + return json.load(stream) + + +def prepare_vm(payload): + import re + import shutil + import sqlite3 + os.umask(0o077) + ident=str(uuid.UUID(payload['id']));state=Path('/var/lib/rmm-agent-remote');helper=Path('/var/lib/rmm-helper') + require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'reviewed permanent agent') + require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'fixed permanent device') + base=Path('/var/lib/rmm-apt-transaction');result=json.loads((base/'active/sandbox-result.json').read_text()) + require(result['apt_exit_code']==0 and not result['reboot_required'],'ended native child required; never stop package manager') + root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident);root.mkdir(mode=0o700) + def create(path,data,mode=0o600): + fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode) + with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode) + tool=base64.b64decode(payload['tool'],validate=True) + require(hashlib.sha256(tool).hexdigest()==payload['tool_sha256'],'qualified repair tool') + create(root/'checkpoint-repair',tool,0o755) + subprocess.run(['/lib64/ld-linux-x86-64.so.2','--list',str(root/'checkpoint-repair')],capture_output=True,check=True,timeout=10) + binding=json.loads((state/'binding.json').read_text()) + config=json.loads(Path('/etc/rmm-remote/helper-general.json').read_text()) + pin=json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_text()) + require(all(binding[k]==config['binding'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')) and + binding['public_key_hex']==pin['public_key_hex'],'independent root trust binding') + create(root/'binding.json',json.dumps(binding).encode()) + units=('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service') + before={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode() for name in units} + subprocess.run(['systemctl','stop',*units],capture_output=True,check=True,timeout=25) + prepared=False;report=dict(id=ident,vm_dir=str(root),tool_sha256=payload['tool_sha256'],prepared=False,package_effects=False) + try: + paths=(state/'journal.sqlite',state/'consumption.sqlite',helper/'consumption.sqlite') + hashes={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths} + for index,p in enumerate(paths): + with sqlite3.connect(p.as_uri()+'?mode=ro',uri=True) as db: + require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','preserved database integrity') + create(root/('database-before-'+str(index)+'.sqlite'),p.read_bytes()) + process=subprocess.run([str(root/'checkpoint-repair'),'prepare',str(root/'binding.json'),str(root/'evidence.json')], + env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25) + create(root/'prepare.stderr',process.stderr) + require(all(hashlib.sha256(p.read_bytes()).hexdigest()==hashes[str(p)] for p in paths),'prepare must not change databases') + report.update(exit_code=process.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',process.stderr)],preserved_database_bytes=True) + if process.returncode==0: + raw=(root/'evidence.json').read_bytes();require(len(raw)<=32*1024*1024,'bounded evidence') + report.update(prepared=True,evidence_sha256=hashlib.sha256(raw).hexdigest(),evidence_b64=base64.b64encode(raw).decode()) + prepared=True + finally: + create(root/'report.json',json.dumps({k:v for k,v in report.items() if k!='evidence_b64'}).encode()) + if not prepared: + for unit in units: + if before[unit]=='active':subprocess.run(['systemctl','start',unit],capture_output=True,timeout=20) + return report + + +def issue_server(payload): + root=Path('/opt/rust-discovery') + raw=base64.b64decode(payload['evidence_b64'],validate=True) + require(hashlib.sha256(raw).hexdigest()==payload['evidence_sha256'],'original exact witness bytes') + cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), + 'exec','-T','api','/usr/local/bin/rmm-server','repair-unsent-sync-request'] + result=subprocess.run(cmd,input=raw,capture_output=True,timeout=30) + require(result.returncode==0,'signed control receipt issuance failed') + value=json.loads(result.stdout);require(set(value)=={'payload','signature'},'only signed control receipt output') + return value + + +def apply_vm(payload): + import re + os.umask(0o077) + ident=str(uuid.UUID(payload['id']));root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident) + require(hashlib.sha256((root/'evidence.json').read_bytes()).hexdigest()==payload['evidence_sha256'] and + hashlib.sha256((root/'checkpoint-repair').read_bytes()).hexdigest()==payload['tool_sha256'],'original repair witnesses/tool') + signed=json.dumps(payload['receipt'],separators=(',',':')).encode();path=root/'receipt.json' + if path.exists():require(path.read_bytes()==signed,'identical existing receipt only') + else: + fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + with os.fdopen(fd,'wb') as f:f.write(signed);f.flush();os.fsync(f.fileno()) + result=subprocess.run([str(root/'checkpoint-repair'),'apply',str(root/'binding.json'),str(root/'evidence.json'),str(path)], + env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25) + fd=os.open(root/'apply.stderr',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + with os.fdopen(fd,'wb') as f:f.write(result.stderr) + return dict(applied=result.returncode==0,exit_code=result.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',result.stderr)], + ready_lease_issued=False,jobs_created=False,grants_created=False) + + +def status_vm(payload): + import sqlite3 + result={} + for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'), + ('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')): + with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db: + tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0], + receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0, + integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok') + if name=='helper': + result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0], + unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0]) + result['services']={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode() + for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')} + return result + + +def main(): + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('prepare','issue','apply'));parser.add_argument('--id') + args=parser.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True);m=cloud.contract + if args.action=='prepare': + require(upgrade.apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact no-new-execution gate') + ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700) + tool=(upgrade.STATE/'checkpoint-repair').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json') + result=cloud.bootstrap.rpc('linux-vm.conf',prepare_vm,dict(id=ident,agent_id=installed['identity']['agent_id'],agent_sha256=installed['binary_sha256'], + tool_sha256=hashlib.sha256(tool).hexdigest(),tool=base64.b64encode(tool).decode())) + m.write_private(folder/'prepared.json',result) + print({k:result.get(k) for k in ('id','prepared','exit_code','error_codes','preserved_database_bytes')}) + require(result['prepared'],'stored witness preparation incomplete; no fault changes') + else: + ident=str(uuid.UUID(args.id));folder=STATE/ident;prepared=prepared_json(folder/'prepared.json');require(prepared['prepared'],'qualified original witnesses') + if args.action=='issue': + result=cloud.bootstrap.rpc('server.conf',issue_server,prepared);m.write_private(folder/'receipt.json',result) + print(dict(signed_control_receipt=True,execution_permission=False)) + else: + result=cloud.bootstrap.rpc('linux-vm.conf',apply_vm,dict(prepared,receipt=m.protected_json(folder/'receipt.json'))) + m.write_private(folder/'applied.json',result);print(result) + + +if __name__=='__main__': + try:main() + except Exception as error: + message=str(error) + if message.startswith('remote bootstrap incomplete: '):print(message) + else:print('CLOUD_CHECKPOINT_REPAIR_INCOMPLETE:'+type(error).__name__) + raise SystemExit(2) from None diff --git a/deploy/cloud_server_upgrade.py b/deploy/cloud_server_upgrade.py index 149a466..2947b25 100644 --- a/deploy/cloud_server_upgrade.py +++ b/deploy/cloud_server_upgrade.py @@ -124,7 +124,7 @@ def deploy(payload): def main(): parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id') - parser.add_argument('--preserve-known-unknown', action='store_true', help='Windows inventory migration only; preserve exact fenced APT evidence') + parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path') args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True) if args.action == 'stage': names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service', @@ -160,7 +160,7 @@ def main(): before['ledger']['result'] is None and before['ledger']['outcome']['error_code'] == 'DPKG_INCONSISTENT', 'preserve exact native unknown') closed = cloud.contract.protected_json(apt.STATE/'policy-closed.json') payload['closed_policy_sha256'] = closed['policy_sha256'] - require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'additive Windows migration only') + require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'retain existing additive schema') else: require(gate['unresolved'] == 0, 'unresolved cloud job; no server switch') if args.action == 'deploy': stream = io.BytesIO() @@ -177,7 +177,7 @@ def main(): gate_after = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id'])) current = cloud.Backend().call('jobs.get', dict(tenant_id=cloud.TENANT, job_id=job['job_id'])) require(before == after and gate_after == gate and current['state'] == 'unknown' and current['attempt_id'] == job['attempt_id'], - 'unknown evidence or execution artifacts changed during Windows migration') + 'unknown evidence or execution artifacts changed during control-path upgrade') cloud.contract.write_private(STATE/(ident+'.unknown-preserved.json'), dict(evidence=after, gate=gate_after, result=current)) result['unknown_attempt_preserved'] = True print(result) diff --git a/docs/implementation-status.md b/docs/implementation-status.md index f5752df..2c3fbdb 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -1,5 +1,50 @@ # Tatsächlicher Implementierungsstand +## Signierte Kontrollkettenreparatur auf Cloud-VM bestanden — 2026-11-13 + +Der qualifizierte Root-Bootstrap prüfte native gespeicherte Agent-/Helper- +Historien und fertigte begrenzte Zeugen an; alle drei SQLite-Dateien blieben +bei dieser Vorbereitung bytegleich. Agent und beide Helper kontrolliert gestoppt, +keine Paketmanagerwirkung unterbrochen. Der neue ARM64-Serverbuild und das +Deployment bestanden; exakte ursprüngliche APT-Evidenz und Zähler vor/nach +Serverwechsel gleich. DB/PKI/Geräteidentität nicht ersetzt. + +Server-CLI `repair-unsent-sync-request` prüfte ursprüngliche signierte Ablehnung, +noch vorhandenen gültigen Serverhead, fehlenden ungesendeten Vorgänger, +Geräte-/Boot-/Generationsbindung und geschlossene Rootpolicy. Bestehende +wirkungslose Ablehnung monoton zum gemeinsamen Kontrollhead übernommen; +separate kurzlebige Signatur `rmm.sync.unsent-request-repair` ausgestellt. +Kein neuer HTTP-Endpunkt, keine interaktive Freigabe vorgetäuscht. + +Native unabhängige Verifier akzeptierten ausschließlich die exakten Zeugen. +Danach tatsächlich **0 Agentfaults / 0 Helperfaults**, jeweils **eine dauerhaft +archivierte Reparaturquittung**, beide Integritätsprüfungen ok. Verbrauchsledger +weiter **9 / 1 unresolved**, Zentrale weiter **13 Jobs / 13 Outbox / 12 Grants / +1 unresolved**. Keine Ready-Lease, Jobs, Grants, Outcomeänderung oder neue +Paketwirkung. Alle drei Dienste bleiben bis zum nachfolgenden qualifizierten +Agent-/Readonlyhelperwechsel gestoppt. APT weiterhin unknown; die +Kontrollreparatur ist ausdrücklich keine Paketabnahme oder Wiederholungsfreigabe. + +Lokale Prüfungen dieser Quittungsstrecke: **1 CLI-Historienmutationstest, +34 Helper-Library- und 21 RootSync-Tests**; Server **3 gezielte Prüfungen mit +frischem isoliertem PostgreSQL**, einschließlicher verweigerter manipuliert/ +veraltet/falsch gebundener Zeugen und unveränderter Null-Jobartefakte. +Format/Clippy `-D warnings` und Releasebuild bestanden. Außerdem fünf gezielte +Guards für eng begrenzte Entwicklungs-Wirefaults geprüft; deren native +Duplicate-/Lost-ACK-Abnahme steht weiterhin aus. + +Ein erster lokaler Quittungsaufruf stoppte am normalen 1-MiB-DTO-Limit für die +vollständigen historischen Zeugen, noch vor Serverzugriff. Separater begrenzter +Private-Dateileser ergänzt (Owner, 0600, keine Symlinks/Hardlinks, maximal45MiB); +identische ursprüngliche Zeugen anschließend erfolgreich verwendet. Geschützt: +`devstate/checkpoint-chain-repair/2c5c3f06-a10e-40de-a02c-3c8009bc034f/` und +`devstate/server-upgrades/2f6144c6-fcd5-49cb-9634-0d6e0b898165*`. + +**MVP-Prüfung: weiterhin offen.** Nächster Schritt ist der ausschließlich +lesende, an den ursprünglichen APT-Auftrag gebundene Teilwirkungsnachweis und +seine Übernahme über den normalen signierten Sync, erst danach ein neuer +WebUI-Wiederherstellungsplan. Weitere Gates: `mvp-gates.md`. + ## Nicht persistierter Sync-Request als MVP-Blocker eingegrenzt — 2026-11-13 Vorprüfung des geplanten Readonly-APT-Helperupgrades stoppte **vor jeder diff --git a/docs/mvp-gates.md b/docs/mvp-gates.md index 08234c7..70f67ac 100644 --- a/docs/mvp-gates.md +++ b/docs/mvp-gates.md @@ -1,6 +1,6 @@ # Linux-MVP: Abnahmegates -Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.9. Die +Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.10. Die Bestandsaufnahme selbst hat keine Tests, Jobs oder Deployments ausgeführt; anschließende tatsächliche Nachweise sind unten ausdrücklich ergänzt. Einzelbelege und spätere Ergebnisse stehen im [Implementierungsstand](implementation-status.md). @@ -39,12 +39,12 @@ bestanden. Dauerhafte APT-Unknown-Evidenz unverändert, keine Jobs, Testservice gestoppt. M1 braucht weiterhin tatsächlichen WSS-Ausfall/Fallback; Gate 0 bleibt die nächste blockierende Verwaltungsstrecke. MVP insgesamt weiterhin offen. -**Aktueller Gate-0-Blocker:** Upgradevorprüfung fand in Agent und Sharedhelper -eine gespeicherte `CHECKPOINT_CHAIN_MISMATCH`-Sperre. Nicht serverseitig -persistierter Vorgänger plus verworfener bestätigter Anker ist anhand der -gespeicherten Kette eingegrenzt; Prävention lokal geprüft, nicht deployt. -Vor APT-Abgleich ist eine exakt signierte Kontrollquittung mit erhaltener -Historie nötig. Keine direkte Faultlöschung oder pauschale Startfreigabe. +**Gate-0-Teilerfolg:** Native signierte Kontrollkettenreparatur bestanden; +ursprüngliche Faults mit je einer Quittung dauerhaft archiviert, jetzt beide +faultfrei. Keine Outcome-/Verbrauchs-/Grantänderung: weiterhin 9/1 bzw. +13/13/12/1. Dienste für qualifizierten Agent-/Readonlyhelperwechsel gestoppt. +Nächster Schritt bleibt der gebundene APT-Teilwirkungsnachweis mit normalem +signiertem Sync; die Kontrollreparatur schließt das Unknown nicht selbst. Alle folgenden Schritte sind **geplant, nicht durch dieses Dokument ausgeführt**. Jeder Ausführungsjob braucht weiterhin seinen eigenen festen WebUI-genehmigten diff --git a/docs/project-decisions.md b/docs/project-decisions.md index 2a788d4..a22ecd9 100644 --- a/docs/project-decisions.md +++ b/docs/project-decisions.md @@ -1,6 +1,6 @@ # Verbindliche Projektentscheidungen -Stand: **2026-11-13**, Spezifikation **1.9** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung. +Stand: **2026-11-13**, Spezifikation **1.10** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung. ## Einzelner signierter APT-Wiederaufnahmeversuch (2026-11-13) @@ -113,3 +113,15 @@ Genehmigungsregeln gelten weiterhin. - VM-Daten: `credentials/linux-vm.conf`, Verzeichnis durch `.gitignore` ausgeschlossen, Modus 700 für `credentials/`, 600 für die Datei. Strikt als Daten parsen, kein `source`/`eval`; keine Secrets lesen/loggen/committen oder in LLM-Kontexte übernehmen. - Auch Linux-Dev hat keine Jobfreigabeabkürzung: erst `pending_approval`, dann WebUI-Genehmigung durch einen berechtigten Menschen oder ausschließlich die oben dokumentierte dev/test-Browsersimulation. Passives Standardinventar ist kein Job. Keine impliziten Reboots, Updates oder destruktiven Tests auf dem Ziel; ausdrücklich gesondert abstimmen, RMM-Jobs zusätzlich über ihren festen Plan in der WebUI genehmigen. - Kein Erfolg ohne realen Nachweis. Geplante Befehle und nicht ausgeführte Tests ausdrücklich so kennzeichnen; fehlende VM-Abnahme bleibt offen. + +## Eng begrenzte Reparatur eines ungesendeten Kontrollrequests + +Spezifikation 1.10 konkretisiert den existierenden Unknown-/Sync-Abgleich: +Root-Bootstrap ausschließlich im Entwicklungsprofil, vollständige originale +Zeugen, unabhängige lokale Signatur-/Inventarprüfung und separate kurzlebige +Kontrollquittung. Nur vorhandene wirkungslose signierte Ablehnung als gemeinsamer +monotoner Head; Faults und Quittung append-only archivieren. Keine Jobfreigabe, +Bereit-Lease, Änderung von Outcome oder Verbrauch, keine Journalrücksetzung. +Native Prüfung und Grenzen: `implementation-status.md`. Direkte API-/DB- +Jobfreigaben bleiben verboten; anschließende Paketwirkung benötigt neuen festen +WebUI-genehmigten Plan. diff --git a/tests/checkpoint_repair_database.py b/tests/checkpoint_repair_database.py new file mode 100644 index 0000000..533779f --- /dev/null +++ b/tests/checkpoint_repair_database.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Checkpoint repair against a fresh, isolated PostgreSQL; no deployed state.""" +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +NAME = "rmm-checkpoint-repair-" + uuid.uuid4().hex[:10] + + +def command(args, data=None, timeout=120): + result = subprocess.run(args, input=data, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError(result.stderr.decode(errors="replace")[-4000:]) + return result.stdout + + +def main(): + command(["docker", "run", "-d", "--network", "none", "--name", NAME, + "-e", "POSTGRES_HOST_AUTH_METHOD=trust", "postgres:17.6-bookworm"]) + try: + for _ in range(100): + logs = command(["docker", "logs", NAME]) + probe = subprocess.run(["docker", "exec", NAME, "pg_isready", "-U", "postgres"], + capture_output=True) + if b"init process complete" in logs and probe.returncode == 0: + break + time.sleep(.2) + else: + raise RuntimeError("ISOLATED_POSTGRES_NOT_READY") + + def sql(statement): + return command(["docker", "exec", "-i", NAME, "psql", "-X", "-qAt", + "-v", "ON_ERROR_STOP=1", "-U", "postgres"], statement.encode()) + + sql("CREATE ROLE rmm_app LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS;") + for path in sorted((ROOT / "migrations").glob("*.sql")): + sql(path.read_text()) + # Current source with a pre-Windows dependency manifest uses the cached + # Linux toolchain; Windows is outside this Linux checkpoint test. + manifests = tempfile.TemporaryDirectory(prefix="rmm-checkpoint-linux-manifest-") + manifest = Path(manifests.name) + for name in ("Cargo.toml", "Cargo.lock"): + (manifest / name).write_bytes(command([ + "git", "-C", str(ROOT), "show", "a5c2dce:" + name])) + output = command([ + "docker", "run", "--rm", "--network", "container:" + NAME, + "--read-only", "--tmpfs", "/tmp:rw,size=512m", + "-v", str(ROOT) + ":/src:ro", + "-v", str(manifest / "Cargo.toml") + ":/src/Cargo.toml:ro", + "-v", str(manifest / "Cargo.lock") + ":/src/Cargo.lock:ro", + "-v", str(ROOT / "devstate/windows-client/target-linux") + ":/target", + "-e", "CARGO_TARGET_DIR=/target", + "-e", "RMM_TEST_REPAIR_ADMIN_URL=postgres://postgres@127.0.0.1/postgres", + "-e", "RMM_TEST_REPAIR_APP_URL=postgres://rmm_app@127.0.0.1/postgres", + "--entrypoint", "cargo", "rust-discovery-dev-build:local", + "test", "--locked", "--offline", "-p", "rmm-server", + "checkpoint_repair", "--", "--include-ignored", "--nocapture"], timeout=300) + print(output.decode()) + print("PASS: isolated PostgreSQL repair, scope/revocation/stale-head/committed-request rejection, exact replay; zero jobs/outbox/permits and no readiness.") + manifests.cleanup() + finally: + subprocess.run(["docker", "rm", "-f", NAME], capture_output=True) + + +if __name__ == "__main__": + main()