Files
rust-discovery/crates/agent-windows/examples/windows_reviewed_recovery/local.rs
T
2026-11-15 23:52:02 +01:00

642 lines
26 KiB
Rust

//! Transactional local half of the human-reviewed, zero-job test incident.
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Utc};
use prost::Message;
use rmm_agent_core::{
auth::Binding,
crypto::{DalekVerifier, Ed25519Verifier},
windows_sync_history::{FAULT, LAST, verify_local_history},
};
use rmm_domain::{checkpoint_repair::SignedReceipt, jobs::hash, windows_sync_history::Receipt};
use rmm_protocol::{Envelope, Payload};
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params, types::ValueRef};
use serde_json::Value;
use sha2::{Digest, Sha256};
use rmm_agent_windows::echo_probe;
use rmm_domain::windows_echo_probe::{ProbeReceipt, ProbeScope};
use uuid::Uuid;
const ECHO_AUDIT: &str = "windows_echo_checkpoint_recovery";
const AUDIT: &str = "windows_human_checkpoint_recovery";
const AUDIT_SQL: &str = "CREATE TABLE windows_human_checkpoint_recovery(id INTEGER PRIMARY KEY CHECK(id=1),review_id TEXT NOT NULL,receipt_id TEXT NOT NULL,old_fault TEXT NOT NULL,original_receipt BLOB NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)";
struct ReviewFields {
request_id: Uuid,
receipt_id: Uuid,
issued_at: DateTime<Utc>,
valid_until: DateTime<Utc>,
witness: Value,
}
pub struct Reviewed {
receipt: ReviewFields,
original: Vec<u8>,
echo: Option<(ProbeScope, Vec<u8>)>,
}
impl Reviewed {
fn signed(
raw: &[u8],
agent: &Binding,
helper: &Binding,
expected: &Value,
) -> Result<SignedReceipt> {
ensure!(raw.len() <= 65536, "recovery envelope size");
let envelope: SignedReceipt = serde_json::from_slice(raw)?;
ensure!(
envelope.payload.len() <= 16384 && envelope.signature.len() == 64,
"recovery signature size"
);
for b in [agent, helper] {
let key = hex_key(&b.public_key_hex)?;
ensure!(
expected["verification_key_id"] == hash(&key)
&& expected["tenant_id"] == b.tenant_id.to_string()
&& expected["agent_id"] == b.agent_id.to_string()
&& expected["asset_id"] == b.asset_id.to_string()
&& expected["enrollment_generation"] == b.enrollment_generation
&& expected["assignment_version"] == b.assignment_version,
"pinned independent role identity"
);
ensure!(
DalekVerifier.verify(&key, &envelope.payload, &envelope.signature)?,
"recovery signature"
);
}
Ok(envelope)
}
pub fn verify(
raw: Vec<u8>,
agent: &Binding,
helper: &Binding,
expected: &Value,
) -> Result<Self> {
let envelope = Self::signed(&raw, agent, helper, expected)?;
let receipt: Receipt = serde_json::from_slice(&envelope.payload)?;
// Historical validity is sufficient for looking up a previously committed receipt.
ensure!(
receipt.matches_reviewed_witness(expected, receipt.issued_at),
"historical human review"
);
Ok(Self {
receipt: ReviewFields {
request_id: receipt.request_id,
receipt_id: receipt.receipt_id,
issued_at: receipt.issued_at,
valid_until: receipt.valid_until,
witness: receipt.witness,
},
original: raw,
echo: None,
})
}
pub fn verify_echo(
raw: Vec<u8>,
agent: &Binding,
helper: &Binding,
expected: &Value,
configured: &ProbeScope,
) -> Result<Self> {
let envelope = Self::signed(&raw, agent, helper, expected)?;
let r: ProbeReceipt = serde_json::from_slice(&envelope.payload)?;
ensure!(
r.matches_reviewed_witness(expected, r.issued_at) && &r.scope == configured,
"simulated echo scope/receipt"
);
Ok(Self {
receipt: ReviewFields {
request_id: r.request_id,
receipt_id: r.receipt_id,
issued_at: r.issued_at,
valid_until: r.valid_until,
witness: r.witness,
},
original: raw,
echo: Some((r.scope, r.job_payload_bytes)),
})
}
pub fn timely(&self, now: DateTime<Utc>) -> Result<()> {
ensure!(
self.receipt.issued_at <= now && now < self.receipt.valid_until,
"human receipt expired or premature"
);
Ok(())
}
}
fn hex_key(s: &str) -> Result<Vec<u8>> {
ensure!(s.len() == 64 && s.is_ascii(), "job pin shape");
(0..64)
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).map_err(Into::into))
.collect()
}
fn include(hash: &mut Sha256, tag: u8, bytes: &[u8]) {
hash.update([tag]);
hash.update((bytes.len() as u64).to_le_bytes());
hash.update(bytes);
}
/// Compare the full logical contents and schema, excluding only the receipt and normalized fault.
#[cfg(test)]
fn inventory(db: &Connection) -> Result<String> {
inventory_for(db, false)
}
fn inventory_for(db: &Connection, echo: bool) -> Result<String> {
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "journal integrity");
let mut statement = db.prepare(
"SELECT type,name,tbl_name,sql FROM sqlite_schema WHERE tbl_name<>?1 ORDER BY type,name",
)?;
let mut rows = statement.query([AUDIT])?;
let mut tables = Vec::new();
let mut digest = Sha256::new();
while let Some(row) = rows.next()? {
let kind: String = row.get(0)?;
let name: String = row.get(1)?;
let table: String = row.get(2)?;
if echo && (echo_probe::OBJECTS.contains(&name.as_str()) || name == ECHO_AUDIT) {
continue;
}
let sql: Option<String> = row.get(3)?;
ensure!(
matches!(kind.as_str(), "table" | "index" | "trigger")
&& !(kind == "trigger" && table == "reconcile_meta")
&& !sql
.as_deref()
.unwrap_or_default()
.to_ascii_uppercase()
.contains("VIRTUAL TABLE"),
"unexpected journal schema program"
);
for s in [
kind.as_str(),
name.as_str(),
table.as_str(),
sql.as_deref().unwrap_or_default(),
] {
include(&mut digest, 0, s.as_bytes());
}
if kind == "table" {
ensure!(
tables.len() < 256
&& !name.is_empty()
&& name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
"unsupported journal table"
);
tables.push(name);
}
}
for table in tables {
include(&mut digest, 1, table.as_bytes());
let mut q = db.prepare(&format!("SELECT rowid,* FROM \"{table}\" ORDER BY rowid"))?;
let cols: Vec<_> = q.column_names().iter().map(|s| s.to_string()).collect();
let mut rows = q.query([])?;
while let Some(row) = rows.next()? {
include(&mut digest, 2, &[]);
for (i, name) in cols.iter().enumerate() {
let value = row.get_ref(i)?;
if table == "reconcile_meta" && name == "fault" {
ensure!(
value == ValueRef::Null || value == ValueRef::Text(FAULT.as_bytes()),
"foreign fault"
);
include(&mut digest, 3, FAULT.as_bytes());
continue;
}
match value {
ValueRef::Null => include(&mut digest, 4, &[]),
ValueRef::Integer(v) => include(&mut digest, 5, &v.to_le_bytes()),
ValueRef::Real(v) => include(&mut digest, 6, &v.to_bits().to_le_bytes()),
ValueRef::Text(bytes) => include(&mut digest, 7, bytes),
ValueRef::Blob(bytes) => include(&mut digest, 8, bytes),
}
}
}
}
Ok(format!("{:x}", digest.finalize()))
}
pub struct LocalJournals<'a> {
pub agent: &'a mut Connection,
pub consumption: &'a Connection,
pub helper: &'a Connection,
pub agent_binding: &'a Binding,
pub helper_binding: &'a Binding,
}
impl LocalJournals<'_> {
/// Only one database changes. Both service locks must span this transaction on Windows.
pub fn apply(
&mut self,
reviewed: &Reviewed,
now: impl Fn() -> DateTime<Utc>,
) -> Result<&'static str> {
let tx = self
.agent
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let echo = reviewed.echo.is_some();
let audit = if echo { ECHO_AUDIT } else { AUDIT };
let ddl = AUDIT_SQL.replace(AUDIT, audit);
let other = if echo { AUDIT } else { ECHO_AUDIT };
ensure!(
!tx.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?)",
[other],
|r| r.get::<_, bool>(0)
)?,
"another recovery mode already exists"
);
let prior: Option<String> = tx
.query_row(
"SELECT sql FROM sqlite_schema WHERE name=?1",
[audit],
|r| r.get(0),
)
.optional()?;
let applied = prior.is_some();
if let Some(schema) = prior {
ensure!(
schema == ddl
&& tx.query_row(
"SELECT count(*) FROM sqlite_schema WHERE tbl_name=?1",
[audit],
|r| r.get::<_, usize>(0)
)? == 1,
"foreign archive"
);
let identical:bool=tx.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {audit} WHERE id=1 AND review_id=? AND receipt_id=? AND old_fault=? AND original_receipt=?)"),params![reviewed.receipt.request_id.to_string(),reviewed.receipt.receipt_id.to_string(),FAULT,reviewed.original],|r|r.get(0))?;
ensure!(identical, "another recovery already exists");
}
let original = verify_local_history(
&tx,
self.consumption,
self.helper,
self.agent_binding,
self.helper_binding,
(!applied).then_some(FAULT),
)?;
ensure!(
reviewed.receipt.witness["decisions"] == Value::Array(original),
"receipt differs from local original decisions"
);
let current:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM reconcile_meta WHERE id=1 AND journal_id=? AND helper_id=? AND last_decision=? AND checkpoint=6756 AND helper_checkpoint=1 AND pending_request IS NULL)", params![reviewed.receipt.witness["agent_journal_id"].as_str().context("agent journal")?,reviewed.receipt.witness["helper_journal_id"].as_str().context("helper journal")?,LAST],|r|r.get(0))?;
ensure!(current, "unreviewed journal identities");
let count: u64 =
self.helper
.query_row("SELECT count(*) FROM helper_sync_rows", [], |r| r.get(0))?;
ensure!(count == 0, "helper consumption rows");
for db in [&*tx, self.consumption, self.helper] {
let n:i64=db.query_row("SELECT count(*) FROM sqlite_schema WHERE type='view' OR (type='table' AND instr(upper(sql),'VIRTUAL TABLE')>0)",[],|r|r.get(0))?;
ensure!(n == 0, "unexpected journal program");
}
let health_rows: i64 =
tx.query_row("SELECT count(*) FROM update_health_inventory", [], |r| {
r.get(0)
})?;
ensure!(health_rows == 0, "unreviewed update grant");
for sql in [
"SELECT id,payload FROM events",
"SELECT id,payload FROM job_events",
] {
let mut q = tx.prepare(sql)?;
let mut rows = q.query([])?;
while let Some(row) = rows.next()? {
let id: String = row.get(0)?;
let bytes = row.get_ref(1)?.as_blob()?;
ensure!(bytes.len() <= rmm_domain::MAX_MESSAGE, "event size");
let e = Envelope::decode(bytes)?;
ensure!(
e.message_id == id && matches!(e.payload, Some(Payload::InventorySnapshot(_))),
"unreviewed event"
);
}
}
if applied {
let saved: String = tx.query_row(
&format!("SELECT inventory_sha256 FROM {audit} WHERE id=1"),
[],
|r| r.get(0),
)?;
if let Some((scope, bytes)) = &reviewed.echo {
echo_probe::check(&tx, scope, bytes, &reviewed.original)?;
}
ensure!(
inventory_for(&tx, echo)? == saved,
"archive inventory changed"
);
return Ok("already applied");
}
reviewed.timely(now())?;
let before = inventory_for(&tx, echo)?;
if let Some((scope, bytes)) = &reviewed.echo {
echo_probe::install(&tx, scope, bytes, &reviewed.original)?;
}
tx.execute(&ddl, [])?;
tx.execute(
&format!("INSERT INTO {audit} VALUES(1,?,?,?,?,?,?)"),
params![
reviewed.receipt.request_id.to_string(),
reviewed.receipt.receipt_id.to_string(),
FAULT,
reviewed.original,
before,
now().to_rfc3339()
],
)?;
ensure!(tx.execute("UPDATE reconcile_meta SET fault=NULL WHERE id=1 AND fault=?1 AND last_decision=?2 AND checkpoint=6756 AND generation=150 AND pending_request IS NULL",params![FAULT,LAST])?==1,"journal head changed");
let cleared: bool = tx.query_row(
"SELECT fault IS NULL FROM reconcile_meta WHERE id=1",
[],
|r| r.get(0),
)?;
ensure!(
cleared && inventory_for(&tx, echo)? == before,
"unrelated journal data changed"
);
reviewed.timely(now())?;
tx.commit()?;
Ok("review archived; normal signed sync required")
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::Duration;
use ed25519_dalek::{Signer, SigningKey};
use rmm_domain::{
checkpoint_repair::inventory_sha256,
windows_sync_history::{RECEIPT_DOMAIN, case_witness},
};
use uuid::Uuid;
mod data {
include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../agent-core/tests/support/windows_review_fixture.rs"
));
}
fn signed(r: &impl serde::Serialize, k: &SigningKey) -> Result<Vec<u8>> {
let payload = serde_json::to_vec(r)?;
Ok(serde_json::to_vec(&SignedReceipt {
signature: k.sign(&payload).to_bytes().to_vec(),
payload,
})?)
}
#[test]
fn scoped_echo_preserves_journals_and_rejects_other_offers() -> Result<()> {
use rmm_domain::{
jobs::{Action, JobPayload, PlanPayload, PlannedJob, WindowsExecutionProfile},
windows_echo_probe as echo,
};
use rmm_protocol::JobOffer;
let (mut a, l, h, binding, key) = data::verified_pair_for(echo::TENANT, echo::ASSET)?;
a.execute_batch("DROP TABLE job_attempts; CREATE TABLE job_attempts(job TEXT NOT NULL,attempt TEXT NOT NULL,offer BLOB NOT NULL,state TEXT NOT NULL,PRIMARY KEY(job,attempt))")?;
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RunScript {
interpreter: "powershell".into(),
script_utf8: echo::SCRIPT.into(),
arguments: vec![],
expected_stdout: Some(echo::OUTPUT.into()),
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
privileged_script_profile: None,
},
execution_identity: "agent_service".into(),
not_before: now,
start_before: now + Duration::minutes(15),
max_runtime_seconds: 60,
max_output_bytes: 65536,
};
let bytes = serde_json::to_vec(&job)?;
let plan = PlanPayload {
schema_version: 1,
plan_id: Uuid::new_v4(),
tenant_id: echo::TENANT,
principal: format!("automation:{}", echo::TENANT),
reason: "fixture".into(),
jobs: vec![PlannedJob {
payload: job.clone(),
job_payload_hash: hash(&bytes),
script_sha256: Some(hash(echo::SCRIPT.as_bytes())),
}],
max_parallel: 1,
halt_after_failures: 1,
recipe_references: vec![],
artifact_references: vec![],
};
let scope = ProbeScope::from_plan(&plan, Uuid::new_v4()).context("fixed Echo plan")?;
let mut w = case_witness();
let (j, p): (String, String) = a.query_row(
"SELECT journal_id,helper_id FROM reconcile_meta WHERE id=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
w["verification_key_id"] = hash(&key.verifying_key().to_bytes()).into();
w["agent_journal_id"] = j.into();
w["helper_journal_id"] = p.into();
w["decisions"] = verify_local_history(&a, &l, &h, &binding, &binding, Some(FAULT))?.into();
let receipt = ProbeReceipt {
domain: echo::DOMAIN.into(),
schema_version: 1,
request_id: Uuid::new_v4(),
receipt_id: Uuid::new_v4(),
witness: w.clone(),
witness_sha256: inventory_sha256(&w)?,
scope: scope.clone(),
scope_sha256: scope.digest().context("scope digest")?,
job_payload_bytes: bytes.clone(),
key_id: hash(&key.verifying_key().to_bytes()),
issued_at: now,
valid_until: now + Duration::minutes(10),
confirmed_by: echo::TEST_PRINCIPAL.into(),
approval_origin: "dev_test_browser_simulation".into(),
testsimulation: true,
};
let raw = signed(&receipt, &key)?;
assert!(Reviewed::verify(raw.clone(), &binding, &binding, &w).is_err());
let mut different = scope.clone();
different.attempt_id = Uuid::new_v4();
assert!(Reviewed::verify_echo(raw.clone(), &binding, &binding, &w, &different).is_err());
let reviewed = Reviewed::verify_echo(raw, &binding, &binding, &w, &scope)?;
let original = inventory(&a)?;
let local = inventory(&l)?;
let helper = inventory(&h)?;
let mut db = LocalJournals {
agent: &mut a,
consumption: &l,
helper: &h,
agent_binding: &binding,
helper_binding: &binding,
};
assert!(db.apply(&reviewed, || now + Duration::minutes(11)).is_err());
db.apply(&reviewed, || now)?;
assert_eq!(inventory_for(db.agent, true)?, original);
assert_eq!((inventory(&l)?, inventory(&h)?), (local, helper));
assert_eq!(
db.apply(&reviewed, || now + Duration::days(1))?,
"already applied"
);
let offer = JobOffer {
job_id: scope.job_id.to_string(),
attempt_id: scope.attempt_id.to_string(),
authorization_id: Uuid::new_v4().to_string(),
plan_hash: scope.plan_hash.clone(),
job_payload_hash: scope.job_payload_hash.clone(),
payload_bytes: bytes,
};
assert!(echo_probe::permits_offer(&scope, &offer, &job));
let store = |o: &JobOffer| {
db.agent.execute(
"INSERT INTO job_attempts VALUES(?,?,?,'accepted')",
params![o.job_id, o.attempt_id, o.encode_to_vec()],
)
};
for field in 0..4 {
let mut bad = offer.clone();
match field {
0 => bad.attempt_id = Uuid::new_v4().to_string(),
1 => bad.plan_hash = "0".repeat(64),
2 => bad.payload_bytes.push(b' '),
_ => bad.job_id = Uuid::new_v4().to_string(),
}
assert!(!echo_probe::permits_offer(&scope, &bad, &job));
assert!(store(&bad).is_err());
}
assert_eq!(store(&offer)?, 1);
assert!(
db.agent
.execute("UPDATE job_attempts SET offer=x'01'", [])
.is_err()
);
assert!(
db.agent
.execute("UPDATE windows_echo_probe_scope SET job='other'", [])
.is_err()
);
assert!(
db.agent
.execute("DELETE FROM windows_echo_probe_scope", [])
.is_err()
);
assert!(db.agent.execute("INSERT OR REPLACE INTO windows_echo_probe_scope SELECT * FROM windows_echo_probe_scope",[]).is_err());
assert!(db.apply(&reviewed, || now).is_err());
Ok(())
}
#[test]
fn archived_human_review_preserves_both_journals() -> Result<()> {
let (mut a, l, h, binding, key) = data::verified_pair()?;
let now = Utc::now();
let mut witness = case_witness();
let (journal, peer): (String, String) = a.query_row(
"SELECT journal_id,helper_id FROM reconcile_meta WHERE id=1",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
witness["agent_journal_id"] = journal.into();
witness["helper_journal_id"] = peer.into();
witness["verification_key_id"] = hash(&key.verifying_key().to_bytes()).into();
witness["tenant_id"] = binding.tenant_id.to_string().into();
witness["asset_id"] = binding.asset_id.to_string().into();
witness["decisions"] =
verify_local_history(&a, &l, &h, &binding, &binding, Some(FAULT))?.into();
let r = Receipt {
domain: RECEIPT_DOMAIN.into(),
schema_version: 1,
request_id: Uuid::new_v4(),
receipt_id: Uuid::new_v4(),
key_id: hash(&key.verifying_key().to_bytes()),
issued_at: now,
valid_until: now + Duration::minutes(10),
witness: witness.clone(),
witness_sha256: inventory_sha256(&witness)?,
confirmed_by: format!("oidc:{}", "a".repeat(64)),
approval_origin: "human_webui".into(),
testsimulation: false,
ready: false,
helper_ready: false,
unlock: false,
};
let raw = signed(&r, &key)?;
let ok = Reviewed::verify(raw.clone(), &binding, &binding, &witness)?;
for e in 0..7 {
let mut bad = r.clone();
match e {
0 => bad.testsimulation = true,
1 => bad.ready = true,
2 => bad.confirmed_by = "simulated-testprincipal-a".into(),
3 => bad.approval_origin = "dev_test_browser_simulation".into(),
4 => bad.unlock = true,
5 => bad.witness["local_head_checkpoint"] = 6755.into(),
_ => bad.domain = "rmm.reconciliation-decision".into(),
}
assert!(Reviewed::verify(signed(&bad, &key)?, &binding, &binding, &witness).is_err());
}
let foreign = SigningKey::from_bytes(&[7; 32]);
assert!(Reviewed::verify(signed(&r, &foreign)?, &binding, &binding, &witness).is_err());
let mut different = binding.clone();
different.public_key_hex = "00".repeat(32);
assert!(Reviewed::verify(raw.clone(), &binding, &different, &witness).is_err());
let prior = inventory(&a)?;
let local_before = inventory(&l)?;
let helper_before = inventory(&h)?;
let mut db = LocalJournals {
agent: &mut a,
consumption: &l,
helper: &h,
agent_binding: &binding,
helper_binding: &binding,
};
assert!(db.apply(&ok, || now + Duration::minutes(10)).is_err());
db.agent.execute_batch("CREATE TRIGGER unexpected AFTER UPDATE ON reconcile_meta BEGIN DELETE FROM job_attempts; END;")?;
assert!(db.apply(&ok, || now).is_err());
db.agent.execute_batch("DROP TRIGGER unexpected")?;
db.agent
.execute("INSERT INTO job_attempts VALUES('test')", [])?;
assert!(db.apply(&ok, || now).is_err());
db.agent.execute("DELETE FROM job_attempts", [])?;
assert_eq!(
db.apply(&ok, || now)?,
"review archived; normal signed sync required"
);
assert_eq!(inventory(db.agent)?, prior);
assert_eq!(inventory(&l)?, local_before);
assert_eq!(inventory(&h)?, helper_before);
let (fault, bytes): (String, Vec<u8>) = db.agent.query_row(
"SELECT old_fault,original_receipt FROM windows_human_checkpoint_recovery",
[],
|q| Ok((q.get(0)?, q.get(1)?)),
)?;
assert_eq!(fault, FAULT);
assert_eq!(bytes, raw);
let status: Option<String> =
db.agent
.query_row("SELECT fault FROM reconcile_meta WHERE id=1", [], |q| {
q.get(0)
})?;
assert!(status.is_none());
assert_eq!(
db.apply(&ok, || now + Duration::days(1))?,
"already applied"
);
db.agent.execute_batch("CREATE TABLE unrelated(n)")?;
assert!(db.apply(&ok, || now).is_err());
db.agent.execute_batch("DROP TABLE unrelated")?;
let mut other = r.clone();
other.receipt_id = Uuid::new_v4();
let other = Reviewed::verify(signed(&other, &key)?, &binding, &binding, &witness)?;
assert!(db.apply(&other, || now).is_err());
db.agent
.execute("INSERT INTO job_attempts VALUES('later')", [])?;
assert!(db.apply(&ok, || now).is_err());
Ok(())
}
}