//! Transactional local half of the human-reviewed, zero-job test incident. use anyhow::{Context, Result, ensure}; use chrono::{DateTime, Utc}; use prost::Message; use rmm_agent_core::{ auth::Binding, crypto::{DalekVerifier, Ed25519Verifier}, windows_sync_history::{FAULT, LAST, verify_local_history}, }; use rmm_domain::{checkpoint_repair::SignedReceipt, jobs::hash, windows_sync_history::Receipt}; use rmm_protocol::{Envelope, Payload}; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params, types::ValueRef}; use serde_json::Value; use sha2::{Digest, Sha256}; use rmm_agent_windows::echo_probe; use rmm_domain::windows_echo_probe::{ProbeReceipt, ProbeScope}; use uuid::Uuid; const ECHO_AUDIT: &str = "windows_echo_checkpoint_recovery"; const AUDIT: &str = "windows_human_checkpoint_recovery"; const AUDIT_SQL: &str = "CREATE TABLE windows_human_checkpoint_recovery(id INTEGER PRIMARY KEY CHECK(id=1),review_id TEXT NOT NULL,receipt_id TEXT NOT NULL,old_fault TEXT NOT NULL,original_receipt BLOB NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)"; struct ReviewFields { request_id: Uuid, receipt_id: Uuid, issued_at: DateTime, valid_until: DateTime, witness: Value, } pub struct Reviewed { receipt: ReviewFields, original: Vec, echo: Option<(ProbeScope, Vec)>, } impl Reviewed { fn signed( raw: &[u8], agent: &Binding, helper: &Binding, expected: &Value, ) -> Result { ensure!(raw.len() <= 65536, "recovery envelope size"); let envelope: SignedReceipt = serde_json::from_slice(raw)?; ensure!( envelope.payload.len() <= 16384 && envelope.signature.len() == 64, "recovery signature size" ); for b in [agent, helper] { let key = hex_key(&b.public_key_hex)?; ensure!( expected["verification_key_id"] == hash(&key) && expected["tenant_id"] == b.tenant_id.to_string() && expected["agent_id"] == b.agent_id.to_string() && expected["asset_id"] == b.asset_id.to_string() && expected["enrollment_generation"] == b.enrollment_generation && expected["assignment_version"] == b.assignment_version, "pinned independent role identity" ); ensure!( DalekVerifier.verify(&key, &envelope.payload, &envelope.signature)?, "recovery signature" ); } Ok(envelope) } pub fn verify( raw: Vec, agent: &Binding, helper: &Binding, expected: &Value, ) -> Result { let envelope = Self::signed(&raw, agent, helper, expected)?; let receipt: Receipt = serde_json::from_slice(&envelope.payload)?; // Historical validity is sufficient for looking up a previously committed receipt. ensure!( receipt.matches_reviewed_witness(expected, receipt.issued_at), "historical human review" ); Ok(Self { receipt: ReviewFields { request_id: receipt.request_id, receipt_id: receipt.receipt_id, issued_at: receipt.issued_at, valid_until: receipt.valid_until, witness: receipt.witness, }, original: raw, echo: None, }) } pub fn verify_echo( raw: Vec, agent: &Binding, helper: &Binding, expected: &Value, configured: &ProbeScope, ) -> Result { let envelope = Self::signed(&raw, agent, helper, expected)?; let r: ProbeReceipt = serde_json::from_slice(&envelope.payload)?; ensure!( r.matches_reviewed_witness(expected, r.issued_at) && &r.scope == configured, "simulated echo scope/receipt" ); Ok(Self { receipt: ReviewFields { request_id: r.request_id, receipt_id: r.receipt_id, issued_at: r.issued_at, valid_until: r.valid_until, witness: r.witness, }, original: raw, echo: Some((r.scope, r.job_payload_bytes)), }) } pub fn timely(&self, now: DateTime) -> Result<()> { ensure!( self.receipt.issued_at <= now && now < self.receipt.valid_until, "human receipt expired or premature" ); Ok(()) } } fn hex_key(s: &str) -> Result> { ensure!(s.len() == 64 && s.is_ascii(), "job pin shape"); (0..64) .step_by(2) .map(|i| u8::from_str_radix(&s[i..i + 2], 16).map_err(Into::into)) .collect() } fn include(hash: &mut Sha256, tag: u8, bytes: &[u8]) { hash.update([tag]); hash.update((bytes.len() as u64).to_le_bytes()); hash.update(bytes); } /// Compare the full logical contents and schema, excluding only the receipt and normalized fault. #[cfg(test)] fn inventory(db: &Connection) -> Result { inventory_for(db, false) } fn inventory_for(db: &Connection, echo: bool) -> Result { let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; ensure!(check == "ok", "journal integrity"); let mut statement = db.prepare( "SELECT type,name,tbl_name,sql FROM sqlite_schema WHERE tbl_name<>?1 ORDER BY type,name", )?; let mut rows = statement.query([AUDIT])?; let mut tables = Vec::new(); let mut digest = Sha256::new(); while let Some(row) = rows.next()? { let kind: String = row.get(0)?; let name: String = row.get(1)?; let table: String = row.get(2)?; if echo && (echo_probe::OBJECTS.contains(&name.as_str()) || name == ECHO_AUDIT) { continue; } let sql: Option = row.get(3)?; ensure!( matches!(kind.as_str(), "table" | "index" | "trigger") && !(kind == "trigger" && table == "reconcile_meta") && !sql .as_deref() .unwrap_or_default() .to_ascii_uppercase() .contains("VIRTUAL TABLE"), "unexpected journal schema program" ); for s in [ kind.as_str(), name.as_str(), table.as_str(), sql.as_deref().unwrap_or_default(), ] { include(&mut digest, 0, s.as_bytes()); } if kind == "table" { ensure!( tables.len() < 256 && !name.is_empty() && name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'), "unsupported journal table" ); tables.push(name); } } for table in tables { include(&mut digest, 1, table.as_bytes()); let mut q = db.prepare(&format!("SELECT rowid,* FROM \"{table}\" ORDER BY rowid"))?; let cols: Vec<_> = q.column_names().iter().map(|s| s.to_string()).collect(); let mut rows = q.query([])?; while let Some(row) = rows.next()? { include(&mut digest, 2, &[]); for (i, name) in cols.iter().enumerate() { let value = row.get_ref(i)?; if table == "reconcile_meta" && name == "fault" { ensure!( value == ValueRef::Null || value == ValueRef::Text(FAULT.as_bytes()), "foreign fault" ); include(&mut digest, 3, FAULT.as_bytes()); continue; } match value { ValueRef::Null => include(&mut digest, 4, &[]), ValueRef::Integer(v) => include(&mut digest, 5, &v.to_le_bytes()), ValueRef::Real(v) => include(&mut digest, 6, &v.to_bits().to_le_bytes()), ValueRef::Text(bytes) => include(&mut digest, 7, bytes), ValueRef::Blob(bytes) => include(&mut digest, 8, bytes), } } } } Ok(format!("{:x}", digest.finalize())) } pub struct LocalJournals<'a> { pub agent: &'a mut Connection, pub consumption: &'a Connection, pub helper: &'a Connection, pub agent_binding: &'a Binding, pub helper_binding: &'a Binding, } impl LocalJournals<'_> { /// Only one database changes. Both service locks must span this transaction on Windows. pub fn apply( &mut self, reviewed: &Reviewed, now: impl Fn() -> DateTime, ) -> Result<&'static str> { let tx = self .agent .transaction_with_behavior(TransactionBehavior::Immediate)?; let echo = reviewed.echo.is_some(); let audit = if echo { ECHO_AUDIT } else { AUDIT }; let ddl = AUDIT_SQL.replace(AUDIT, audit); let other = if echo { AUDIT } else { ECHO_AUDIT }; ensure!( !tx.query_row( "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?)", [other], |r| r.get::<_, bool>(0) )?, "another recovery mode already exists" ); let prior: Option = tx .query_row( "SELECT sql FROM sqlite_schema WHERE name=?1", [audit], |r| r.get(0), ) .optional()?; let applied = prior.is_some(); if let Some(schema) = prior { ensure!( schema == ddl && tx.query_row( "SELECT count(*) FROM sqlite_schema WHERE tbl_name=?1", [audit], |r| r.get::<_, usize>(0) )? == 1, "foreign archive" ); let identical:bool=tx.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {audit} WHERE id=1 AND review_id=? AND receipt_id=? AND old_fault=? AND original_receipt=?)"),params![reviewed.receipt.request_id.to_string(),reviewed.receipt.receipt_id.to_string(),FAULT,reviewed.original],|r|r.get(0))?; ensure!(identical, "another recovery already exists"); } let original = verify_local_history( &tx, self.consumption, self.helper, self.agent_binding, self.helper_binding, (!applied).then_some(FAULT), )?; ensure!( reviewed.receipt.witness["decisions"] == Value::Array(original), "receipt differs from local original decisions" ); let current:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM reconcile_meta WHERE id=1 AND journal_id=? AND helper_id=? AND last_decision=? AND checkpoint=6756 AND helper_checkpoint=1 AND pending_request IS NULL)", params![reviewed.receipt.witness["agent_journal_id"].as_str().context("agent journal")?,reviewed.receipt.witness["helper_journal_id"].as_str().context("helper journal")?,LAST],|r|r.get(0))?; ensure!(current, "unreviewed journal identities"); let count: u64 = self.helper .query_row("SELECT count(*) FROM helper_sync_rows", [], |r| r.get(0))?; ensure!(count == 0, "helper consumption rows"); for db in [&*tx, self.consumption, self.helper] { let n:i64=db.query_row("SELECT count(*) FROM sqlite_schema WHERE type='view' OR (type='table' AND instr(upper(sql),'VIRTUAL TABLE')>0)",[],|r|r.get(0))?; ensure!(n == 0, "unexpected journal program"); } let health_rows: i64 = tx.query_row("SELECT count(*) FROM update_health_inventory", [], |r| { r.get(0) })?; ensure!(health_rows == 0, "unreviewed update grant"); for sql in [ "SELECT id,payload FROM events", "SELECT id,payload FROM job_events", ] { let mut q = tx.prepare(sql)?; let mut rows = q.query([])?; while let Some(row) = rows.next()? { let id: String = row.get(0)?; let bytes = row.get_ref(1)?.as_blob()?; ensure!(bytes.len() <= rmm_domain::MAX_MESSAGE, "event size"); let e = Envelope::decode(bytes)?; ensure!( e.message_id == id && matches!(e.payload, Some(Payload::InventorySnapshot(_))), "unreviewed event" ); } } if applied { let saved: String = tx.query_row( &format!("SELECT inventory_sha256 FROM {audit} WHERE id=1"), [], |r| r.get(0), )?; if let Some((scope, bytes)) = &reviewed.echo { echo_probe::check(&tx, scope, bytes, &reviewed.original)?; } ensure!( inventory_for(&tx, echo)? == saved, "archive inventory changed" ); return Ok("already applied"); } reviewed.timely(now())?; let before = inventory_for(&tx, echo)?; if let Some((scope, bytes)) = &reviewed.echo { echo_probe::install(&tx, scope, bytes, &reviewed.original)?; } tx.execute(&ddl, [])?; tx.execute( &format!("INSERT INTO {audit} VALUES(1,?,?,?,?,?,?)"), params![ reviewed.receipt.request_id.to_string(), reviewed.receipt.receipt_id.to_string(), FAULT, reviewed.original, before, now().to_rfc3339() ], )?; ensure!(tx.execute("UPDATE reconcile_meta SET fault=NULL WHERE id=1 AND fault=?1 AND last_decision=?2 AND checkpoint=6756 AND generation=150 AND pending_request IS NULL",params![FAULT,LAST])?==1,"journal head changed"); let cleared: bool = tx.query_row( "SELECT fault IS NULL FROM reconcile_meta WHERE id=1", [], |r| r.get(0), )?; ensure!( cleared && inventory_for(&tx, echo)? == before, "unrelated journal data changed" ); reviewed.timely(now())?; tx.commit()?; Ok("review archived; normal signed sync required") } } #[cfg(test)] mod tests { use super::*; use chrono::Duration; use ed25519_dalek::{Signer, SigningKey}; use rmm_domain::{ checkpoint_repair::inventory_sha256, windows_sync_history::{RECEIPT_DOMAIN, case_witness}, }; use uuid::Uuid; mod data { include!(concat!( env!("CARGO_MANIFEST_DIR"), "/../agent-core/tests/support/windows_review_fixture.rs" )); } fn signed(r: &impl serde::Serialize, k: &SigningKey) -> Result> { let payload = serde_json::to_vec(r)?; Ok(serde_json::to_vec(&SignedReceipt { signature: k.sign(&payload).to_bytes().to_vec(), payload, })?) } #[test] fn scoped_echo_preserves_journals_and_rejects_other_offers() -> Result<()> { use rmm_domain::{ jobs::{Action, JobPayload, PlanPayload, PlannedJob, WindowsExecutionProfile}, windows_echo_probe as echo, }; use rmm_protocol::JobOffer; let (mut a, l, h, binding, key) = data::verified_pair_for(echo::TENANT, echo::ASSET)?; a.execute_batch("DROP TABLE job_attempts; CREATE TABLE job_attempts(job TEXT NOT NULL,attempt TEXT NOT NULL,offer BLOB NOT NULL,state TEXT NOT NULL,PRIMARY KEY(job,attempt))")?; let now = Utc::now(); let job = JobPayload { schema_version: 1, tenant_id: binding.tenant_id, asset_id: binding.asset_id, agent_id: binding.agent_id, enrollment_generation: 1, assignment_version: 1, job_id: Uuid::new_v4(), attempt_id: Uuid::new_v4(), action: Action::RunScript { interpreter: "powershell".into(), script_utf8: echo::SCRIPT.into(), arguments: vec![], expected_stdout: Some(echo::OUTPUT.into()), windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned), privileged_script_profile: None, }, execution_identity: "agent_service".into(), not_before: now, start_before: now + Duration::minutes(15), max_runtime_seconds: 60, max_output_bytes: 65536, }; let bytes = serde_json::to_vec(&job)?; let plan = PlanPayload { schema_version: 1, plan_id: Uuid::new_v4(), tenant_id: echo::TENANT, principal: format!("automation:{}", echo::TENANT), reason: "fixture".into(), jobs: vec![PlannedJob { payload: job.clone(), job_payload_hash: hash(&bytes), script_sha256: Some(hash(echo::SCRIPT.as_bytes())), }], max_parallel: 1, halt_after_failures: 1, recipe_references: vec![], artifact_references: vec![], }; let scope = ProbeScope::from_plan(&plan, Uuid::new_v4()).context("fixed Echo plan")?; let mut w = case_witness(); let (j, p): (String, String) = a.query_row( "SELECT journal_id,helper_id FROM reconcile_meta WHERE id=1", [], |r| Ok((r.get(0)?, r.get(1)?)), )?; w["verification_key_id"] = hash(&key.verifying_key().to_bytes()).into(); w["agent_journal_id"] = j.into(); w["helper_journal_id"] = p.into(); w["decisions"] = verify_local_history(&a, &l, &h, &binding, &binding, Some(FAULT))?.into(); let receipt = ProbeReceipt { domain: echo::DOMAIN.into(), schema_version: 1, request_id: Uuid::new_v4(), receipt_id: Uuid::new_v4(), witness: w.clone(), witness_sha256: inventory_sha256(&w)?, scope: scope.clone(), scope_sha256: scope.digest().context("scope digest")?, job_payload_bytes: bytes.clone(), key_id: hash(&key.verifying_key().to_bytes()), issued_at: now, valid_until: now + Duration::minutes(10), confirmed_by: echo::TEST_PRINCIPAL.into(), approval_origin: "dev_test_browser_simulation".into(), testsimulation: true, }; let raw = signed(&receipt, &key)?; assert!(Reviewed::verify(raw.clone(), &binding, &binding, &w).is_err()); let mut different = scope.clone(); different.attempt_id = Uuid::new_v4(); assert!(Reviewed::verify_echo(raw.clone(), &binding, &binding, &w, &different).is_err()); let reviewed = Reviewed::verify_echo(raw, &binding, &binding, &w, &scope)?; let original = inventory(&a)?; let local = inventory(&l)?; let helper = inventory(&h)?; let mut db = LocalJournals { agent: &mut a, consumption: &l, helper: &h, agent_binding: &binding, helper_binding: &binding, }; assert!(db.apply(&reviewed, || now + Duration::minutes(11)).is_err()); db.apply(&reviewed, || now)?; assert_eq!(inventory_for(db.agent, true)?, original); assert_eq!((inventory(&l)?, inventory(&h)?), (local, helper)); assert_eq!( db.apply(&reviewed, || now + Duration::days(1))?, "already applied" ); let offer = JobOffer { job_id: scope.job_id.to_string(), attempt_id: scope.attempt_id.to_string(), authorization_id: Uuid::new_v4().to_string(), plan_hash: scope.plan_hash.clone(), job_payload_hash: scope.job_payload_hash.clone(), payload_bytes: bytes, }; assert!(echo_probe::permits_offer(&scope, &offer, &job)); let store = |o: &JobOffer| { db.agent.execute( "INSERT INTO job_attempts VALUES(?,?,?,'accepted')", params![o.job_id, o.attempt_id, o.encode_to_vec()], ) }; for field in 0..4 { let mut bad = offer.clone(); match field { 0 => bad.attempt_id = Uuid::new_v4().to_string(), 1 => bad.plan_hash = "0".repeat(64), 2 => bad.payload_bytes.push(b' '), _ => bad.job_id = Uuid::new_v4().to_string(), } assert!(!echo_probe::permits_offer(&scope, &bad, &job)); assert!(store(&bad).is_err()); } assert_eq!(store(&offer)?, 1); assert!( db.agent .execute("UPDATE job_attempts SET offer=x'01'", []) .is_err() ); assert!( db.agent .execute("UPDATE windows_echo_probe_scope SET job='other'", []) .is_err() ); assert!( db.agent .execute("DELETE FROM windows_echo_probe_scope", []) .is_err() ); assert!(db.agent.execute("INSERT OR REPLACE INTO windows_echo_probe_scope SELECT * FROM windows_echo_probe_scope",[]).is_err()); assert!(db.apply(&reviewed, || now).is_err()); Ok(()) } #[test] fn archived_human_review_preserves_both_journals() -> Result<()> { let (mut a, l, h, binding, key) = data::verified_pair()?; let now = Utc::now(); let mut witness = case_witness(); let (journal, peer): (String, String) = a.query_row( "SELECT journal_id,helper_id FROM reconcile_meta WHERE id=1", [], |r| Ok((r.get(0)?, r.get(1)?)), )?; witness["agent_journal_id"] = journal.into(); witness["helper_journal_id"] = peer.into(); witness["verification_key_id"] = hash(&key.verifying_key().to_bytes()).into(); witness["tenant_id"] = binding.tenant_id.to_string().into(); witness["asset_id"] = binding.asset_id.to_string().into(); witness["decisions"] = verify_local_history(&a, &l, &h, &binding, &binding, Some(FAULT))?.into(); let r = Receipt { domain: RECEIPT_DOMAIN.into(), schema_version: 1, request_id: Uuid::new_v4(), receipt_id: Uuid::new_v4(), key_id: hash(&key.verifying_key().to_bytes()), issued_at: now, valid_until: now + Duration::minutes(10), witness: witness.clone(), witness_sha256: inventory_sha256(&witness)?, confirmed_by: format!("oidc:{}", "a".repeat(64)), approval_origin: "human_webui".into(), testsimulation: false, ready: false, helper_ready: false, unlock: false, }; let raw = signed(&r, &key)?; let ok = Reviewed::verify(raw.clone(), &binding, &binding, &witness)?; for e in 0..7 { let mut bad = r.clone(); match e { 0 => bad.testsimulation = true, 1 => bad.ready = true, 2 => bad.confirmed_by = "simulated-testprincipal-a".into(), 3 => bad.approval_origin = "dev_test_browser_simulation".into(), 4 => bad.unlock = true, 5 => bad.witness["local_head_checkpoint"] = 6755.into(), _ => bad.domain = "rmm.reconciliation-decision".into(), } assert!(Reviewed::verify(signed(&bad, &key)?, &binding, &binding, &witness).is_err()); } let foreign = SigningKey::from_bytes(&[7; 32]); assert!(Reviewed::verify(signed(&r, &foreign)?, &binding, &binding, &witness).is_err()); let mut different = binding.clone(); different.public_key_hex = "00".repeat(32); assert!(Reviewed::verify(raw.clone(), &binding, &different, &witness).is_err()); let prior = inventory(&a)?; let local_before = inventory(&l)?; let helper_before = inventory(&h)?; let mut db = LocalJournals { agent: &mut a, consumption: &l, helper: &h, agent_binding: &binding, helper_binding: &binding, }; assert!(db.apply(&ok, || now + Duration::minutes(10)).is_err()); db.agent.execute_batch("CREATE TRIGGER unexpected AFTER UPDATE ON reconcile_meta BEGIN DELETE FROM job_attempts; END;")?; assert!(db.apply(&ok, || now).is_err()); db.agent.execute_batch("DROP TRIGGER unexpected")?; db.agent .execute("INSERT INTO job_attempts VALUES('test')", [])?; assert!(db.apply(&ok, || now).is_err()); db.agent.execute("DELETE FROM job_attempts", [])?; assert_eq!( db.apply(&ok, || now)?, "review archived; normal signed sync required" ); assert_eq!(inventory(db.agent)?, prior); assert_eq!(inventory(&l)?, local_before); assert_eq!(inventory(&h)?, helper_before); let (fault, bytes): (String, Vec) = db.agent.query_row( "SELECT old_fault,original_receipt FROM windows_human_checkpoint_recovery", [], |q| Ok((q.get(0)?, q.get(1)?)), )?; assert_eq!(fault, FAULT); assert_eq!(bytes, raw); let status: Option = db.agent .query_row("SELECT fault FROM reconcile_meta WHERE id=1", [], |q| { q.get(0) })?; assert!(status.is_none()); assert_eq!( db.apply(&ok, || now + Duration::days(1))?, "already applied" ); db.agent.execute_batch("CREATE TABLE unrelated(n)")?; assert!(db.apply(&ok, || now).is_err()); db.agent.execute_batch("DROP TABLE unrelated")?; let mut other = r.clone(); other.receipt_id = Uuid::new_v4(); let other = Reviewed::verify(signed(&other, &key)?, &binding, &binding, &witness)?; assert!(db.apply(&other, || now).is_err()); db.agent .execute("INSERT INTO job_attempts VALUES('later')", [])?; assert!(db.apply(&ok, || now).is_err()); Ok(()) } }