fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-13 17:43:38 +01:00
parent 998f16fde4
commit 359233c64d
21 changed files with 3030 additions and 30 deletions
@@ -0,0 +1,497 @@
//! Narrow, root-only development bootstrap. No job execution or ready lease.
use anyhow::{Context, Result, ensure};
use fs2::FileExt;
use prost::Message;
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rmm_privileged_helper::{Binding, checkpoint_repair as repair};
use rmm_protocol::{Envelope, Payload, SyncRequest, SyncResponse};
use rusqlite::{Connection, OpenFlags, OptionalExtension};
use std::{
fs::{self, OpenOptions},
io::Write,
os::unix::fs::{MetadataExt, OpenOptionsExt},
path::Path,
};
const AGENT: &str = "/var/lib/rmm-agent-remote";
const HELPER: &str = "/var/lib/rmm-helper";
fn read_private(path: &Path, owner: u32, limit: usize) -> Result<Vec<u8>> {
rmm_privileged_helper::private_path(path, owner, false)?;
let m = fs::metadata(path)?;
ensure!(m.len() <= limit as u64, "REPAIR_FILE_SIZE");
Ok(fs::read(path)?)
}
fn open_existing(path: &Path, owner: u32) -> Result<Connection> {
rmm_privileged_helper::private_path(path, owner, false)?;
// Require the existing DELETE journal mode. This bootstrap never opens WAL
// state or creates privileged sidecars in the unprivileged agent directory.
for suffix in ["-wal", "-shm", "-journal"] {
ensure!(
!Path::new(&format!("{}{suffix}", path.display())).try_exists()?,
"REPAIR_DATABASE_SIDECAR_PRESENT"
);
}
let db = Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
let mode: String = db.query_row("PRAGMA journal_mode", [], |v| v.get(0))?;
ensure!(mode == "delete", "REPAIR_DELETE_JOURNAL_REQUIRED");
db.execute_batch(
"PRAGMA synchronous=FULL; PRAGMA foreign_keys=OFF; PRAGMA trusted_schema=OFF;",
)?;
Ok(db)
}
fn signed(raw: &[u8]) -> Result<SignedDecision> {
let r = SyncResponse::decode(raw)?;
ensure!(r.encode_to_vec() == raw, "REPAIR_NONCANONICAL_RESPONSE");
Ok(SignedDecision {
decision_bytes: r.decision_bytes,
signature: r.signature,
})
}
fn check_response(
binding: &Binding,
request: &SyncRequest,
envelope: &Envelope,
response: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
let d = repair::verify_historical_decision(binding, response)?;
ensure!(
d.request_id.to_string() == request.request_id
&& d.request_hash == hash(&request.encode_to_vec())
&& d.journal_id.to_string() == request.journal_id
&& d.checkpoint == request.checkpoint
&& d.connection_generation == envelope.connection_generation
&& d.boot_id.to_string() == envelope.boot_id
&& d.helper_journal_id
.map(|id| id.to_string())
.unwrap_or_default()
== request.helper_journal_id
&& d.helper_checkpoint == request.helper_checkpoint,
"REPAIR_AGENT_SIGNED_REQUEST_BINDING"
);
Ok(d)
}
fn agent_history(db: &Connection, binding: &Binding) -> Result<Vec<(SyncRequest, SignedDecision)>> {
let mut s = db.prepare(
"SELECT request,envelope,generation,response FROM reconcile_requests ORDER BY rowid",
)?;
let rows = s
.query_map([], |v| {
Ok((
v.get::<_, Vec<u8>>(0)?,
v.get::<_, Vec<u8>>(1)?,
v.get::<_, u64>(2)?,
v.get::<_, Option<Vec<u8>>>(3)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
ensure!(rows.len() <= 100000, "REPAIR_AGENT_HISTORY_LIMIT");
let mut result = vec![];
for (raw, outer, generation, response) in rows {
ensure!(
raw.len() <= rmm_domain::MAX_MESSAGE && outer.len() <= rmm_domain::MAX_MESSAGE,
"REPAIR_REQUEST_SIZE"
);
let request = SyncRequest::decode(raw.as_slice())?;
let envelope = Envelope::decode(outer.as_slice())?;
ensure!(
request.encode_to_vec() == raw
&& envelope.encode_to_vec() == outer
&& envelope.payload == Some(Payload::SyncRequest(request.clone()))
&& envelope.agent_id == binding.agent_id.to_string()
&& envelope.enrollment_generation == binding.enrollment_generation
&& envelope.connection_generation == generation
&& envelope.sequence == request.checkpoint
&& request.assignment_version == binding.assignment_version,
"REPAIR_AGENT_DURABLE_REQUEST_BINDING"
);
if let Some(response) = response {
let response = signed(&response)?;
check_response(binding, &request, &envelope, &response)?;
result.push((request, response));
}
}
let mut offers = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
for row in offers.query_map([], |v| {
Ok((
v.get::<_, String>(0)?,
v.get::<_, String>(1)?,
v.get::<_, Vec<u8>>(2)?,
))
})? {
let (job, attempt, bytes) = row?;
let offer = rmm_protocol::JobOffer::decode(bytes.as_slice())?;
let payload = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job
&& offer.attempt_id == attempt
&& payload.job_id.to_string() == job
&& payload.attempt_id.to_string() == attempt
&& offer.job_payload_hash == hash(&offer.payload_bytes),
"REPAIR_AGENT_OFFER_BINDING"
);
}
Ok(result)
}
fn combined_inventory(agent: &Connection, consumption: &Connection) -> Result<serde_json::Value> {
let mut value = repair::inventory(agent, "reconcile_meta")?;
value
.as_object_mut()
.context("REPAIR_AGENT_INVENTORY")?
.insert(
"agent_consumption".into(),
repair::inventory(consumption, "")?,
);
Ok(value)
}
fn evidence(
agent: &Connection,
consumption: &Connection,
helper: &Connection,
binding: &Binding,
) -> Result<Evidence> {
rmm_privileged_helper::reconcile::verify_checkpoint_repair_history(helper, binding, None)?;
let history = agent_history(agent, binding)?;
let (head, fault, pending): (String, Option<String>, Option<String>) = agent.query_row(
"SELECT last_decision,fault,pending_request FROM reconcile_meta WHERE id=1",
[],
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?)),
)?;
ensure!(
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH") && pending.is_none(),
"REPAIR_AGENT_EXACT_FAULT_REQUIRED"
);
let (y, denial) = history
.iter()
.find(|(_, r)| {
serde_json::from_slice::<ReconcileDecisionPayload>(&r.decision_bytes)
.is_ok_and(|d| d.decision_id.to_string() == head)
})
.context("REPAIR_REJECTION_MISSING")?;
let raw_x: Vec<u8> = agent.query_row(
"SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL",
[&y.previous_request_id],
|v| v.get(0),
)?;
let x = SyncRequest::decode(raw_x.as_slice())?;
let (_, a) = history
.iter()
.find(|(_, r)| {
serde_json::from_slice::<ReconcileDecisionPayload>(&r.decision_bytes)
.is_ok_and(|d| d.decision_id.to_string() == x.previous_decision_id)
})
.context("REPAIR_PREVIOUS_VALID_MISSING")?;
Ok(Evidence {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: binding.enrollment_generation,
assignment_version: binding.assignment_version,
agent_inventory: combined_inventory(agent, consumption)?,
helper_inventory: repair::inventory(helper, "helper_sync_meta")?,
uncommitted_request: raw_x,
rejected_request: y.encode_to_vec(),
rejected_response: denial.clone(),
previous_valid_response: a.clone(),
})
}
fn verify_agent(
agent: &Connection,
consumption: &Connection,
binding: &Binding,
signed: &SignedReceipt,
e: &Evidence,
) -> Result<()> {
let r = repair::verify_receipt(binding, signed)?;
let _ = agent_history(agent, binding)?;
ensure!(
e.schema_version == 1
&& e.tenant_id == binding.tenant_id
&& e.asset_id == binding.asset_id
&& e.agent_id == binding.agent_id
&& e.enrollment_generation == binding.enrollment_generation
&& e.assignment_version == binding.assignment_version,
"REPAIR_EVIDENCE_IDENTITY"
);
let actual = combined_inventory(agent, consumption)?;
ensure!(
actual == e.agent_inventory && inventory_sha256(&actual)? == r.agent_inventory_sha256,
"REPAIR_AGENT_INVENTORY"
);
let (journal,checkpoint,high,head,pending,helperid,helpercp,fault):(String,u64,u64,String,Option<String>,String,u64,Option<String>)=agent.query_row("SELECT journal_id,checkpoint,high_sequence,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|v|Ok((v.get(0)?,v.get(1)?,v.get(2)?,v.get(3)?,v.get(4)?,v.get(5)?,v.get(6)?,v.get(7)?)))?;
let counter: u64 = agent.query_row("SELECT seq FROM counter WHERE id=1", [], |v| v.get(0))?;
ensure!(
journal == r.agent_journal_id.to_string()
&& checkpoint == r.agent_checkpoint
&& counter >= high
&& counter >= checkpoint
&& head == r.rejected_decision_id.to_string()
&& pending.is_none()
&& helperid == r.helper_journal_id.to_string()
&& helpercp == r.helper_checkpoint
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|| (fault.is_none() && repair::already_applied(agent, signed)?)),
"REPAIR_AGENT_HEAD"
);
let y = SyncRequest::decode(e.rejected_request.as_slice())?;
let x = SyncRequest::decode(e.uncommitted_request.as_slice())?;
let stored_y: (Vec<u8>, Vec<u8>) = agent.query_row(
"SELECT request,response FROM reconcile_requests WHERE id=?1",
[&y.request_id],
|v| Ok((v.get(0)?, v.get(1)?)),
)?;
let stored_x: Option<Vec<u8>> = agent
.query_row(
"SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL",
[&x.request_id],
|v| v.get(0),
)
.optional()?;
let d = repair::verify_historical_decision(binding, &e.rejected_response)?;
let a = repair::verify_historical_decision(binding, &e.previous_valid_response)?;
ensure!(
stored_y.0 == e.rejected_request
&& signed_response_eq(&stored_y.1, &e.rejected_response)?
&& stored_x.as_deref() == Some(e.uncommitted_request.as_slice())
&& d.decision_id == r.rejected_decision_id
&& hash(&e.rejected_response.decision_bytes) == r.rejected_decision_sha256
&& d.reason == r.fault
&& !d.ready
&& !d.helper_ready
&& !d.attempts.iter().any(|v| v.unlock)
&& a.decision_id == r.previous_valid_decision_id
&& hash(&e.previous_valid_response.decision_bytes) == r.previous_valid_decision_sha256
&& hash(&e.rejected_request) == r.rejected_request_sha256
&& y.request_id == r.rejected_request_id.to_string()
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
&& x.request_id == r.uncommitted_request_id.to_string()
&& x.previous_decision_id == a.decision_id.to_string()
&& x.helper_previous_decision_id == a.decision_id.to_string()
&& y.previous_decision_id.is_empty()
&& y.previous_request_id == x.request_id
&& y.helper_previous_decision_id == a.decision_id.to_string(),
"REPAIR_AGENT_EXACT_UNSENT_CHAIN"
);
Ok(())
}
fn signed_response_eq(raw: &[u8], expected: &SignedDecision) -> Result<bool> {
Ok(signed(raw)? == *expected)
}
fn main() -> Result<()> {
ensure!(
unsafe { libc::geteuid() } == 0,
"REPAIR_ROOT_BOOTSTRAP_ONLY"
);
ensure!(
std::env::var("RMM_CHECKPOINT_REPAIR_PROFILE").as_deref() == Ok("development"),
"REPAIR_EXPLICIT_DEVELOPMENT_PROFILE_REQUIRED"
);
ensure!(
rmm_privileged_helper::clock_trusted(),
"REPAIR_CLOCK_UNTRUSTED"
);
let args = std::env::args_os().skip(1).collect::<Vec<_>>();
ensure!(
matches!(args.len(), 3 | 4),
"usage: rmm-checkpoint-repair prepare BINDING_FILE EVIDENCE_FILE | apply BINDING_FILE EVIDENCE_FILE RECEIPT_FILE"
);
let binding = rmm_privileged_helper::read_binding(Path::new(&args[1]), 0)?;
let owner = fs::metadata(AGENT)?.uid();
ensure!(owner != 0, "REPAIR_UNPRIVILEGED_AGENT_REQUIRED");
rmm_privileged_helper::private_path(Path::new(AGENT), owner, true)?;
rmm_privileged_helper::private_path(Path::new(HELPER), 0, true)?;
let lock_path = Path::new(AGENT).join("agent.lock");
rmm_privileged_helper::private_path(&lock_path, owner, false)?;
let lock = OpenOptions::new()
.read(true)
.write(true)
.custom_flags(libc::O_NOFOLLOW)
.open(lock_path)?;
lock.try_lock_exclusive()?;
let _guard = rmm_privileged_helper::reconcile::ExecutorGuard::acquire(Path::new(HELPER), 0)?;
let mut agent = open_existing(&Path::new(AGENT).join("journal.sqlite"), owner)?;
let consumption = open_existing(&Path::new(AGENT).join("consumption.sqlite"), owner)?;
let mut helper = open_existing(&Path::new(HELPER).join("consumption.sqlite"), 0)?;
let mode = args[0].to_str().context("REPAIR_MODE")?;
match (mode, args.len()) {
("prepare", 3) => {
let e = evidence(&agent, &consumption, &helper, &binding)?;
let bytes = serde_json::to_vec(&e)?;
ensure!(bytes.len() <= 32 * 1024 * 1024, "REPAIR_EVIDENCE_SIZE");
let mut f = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(&args[2])?;
f.write_all(&bytes)?;
f.sync_all()?;
println!("checkpoint repair evidence prepared; no state changed");
}
("apply", 4) => {
let e: Evidence =
serde_json::from_slice(&read_private(Path::new(&args[2]), 0, 32 * 1024 * 1024)?)?;
let signed: SignedReceipt =
serde_json::from_slice(&read_private(Path::new(&args[3]), 0, 65536)?)?;
verify_agent(&agent, &consumption, &binding, &signed, &e)?;
repair::verify_helper(&helper, &binding, &signed, &e)?;
let r = repair::verify_receipt(&binding, &signed)?;
// Every witness is verified before either durable fault archive is written.
// A stopped bootstrap can replay the identical receipt after interruption.
repair::commit_fault_repair(
&mut helper,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)?;
repair::commit_fault_repair(
&mut agent,
"reconcile_meta",
&signed,
&r.agent_inventory_sha256,
Some(("agent_consumption", repair::inventory(&consumption, "")?)),
)?;
println!(
"exact checkpoint faults archived and repaired; no execution readiness granted"
);
}
_ => anyhow::bail!("REPAIR_UNKNOWN_MODE"),
}
drop(helper);
drop(consumption);
drop(agent);
// DELETE journaling leaves no root-owned WAL/SHM or journal behind.
for name in ["journal.sqlite", "consumption.sqlite"] {
for suffix in ["-wal", "-shm", "-journal"] {
ensure!(
!Path::new(AGENT)
.join(format!("{name}{suffix}"))
.try_exists()?,
"REPAIR_UNEXPECTED_AGENT_SIDECAR"
);
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::Utc;
use openssl::{pkey::PKey, sign::Signer};
use uuid::Uuid;
#[test]
fn durable_signed_request_history_rejects_mutation_before_any_repair() {
let key = PKey::generate_ed25519().unwrap();
let id = Uuid::new_v4();
let boot = Uuid::new_v4();
let binding = Binding {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: key
.raw_public_key()
.unwrap()
.iter()
.map(|b| format!("{b:02x}"))
.collect(),
};
let q = SyncRequest {
request_id: Uuid::new_v4().to_string(),
journal_id: id.to_string(),
checkpoint: 10,
assignment_version: 1,
journal_complete: true,
helper_journal_id: id.to_string(),
helper_checkpoint: 20,
..Default::default()
};
let e = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: id.to_string(),
enrollment_generation: 1,
connection_generation: 2,
boot_id: boot.to_string(),
sequence: 10,
observed_at: Utc::now().timestamp_millis(),
payload: Some(Payload::SyncRequest(q.clone())),
};
let now = Utc::now();
let d = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::parse_str(&q.request_id).unwrap(),
request_hash: hash(&q.encode_to_vec()),
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
connection_generation: 2,
boot_id: boot,
journal_id: id,
checkpoint: 10,
helper_journal_id: Some(id),
helper_checkpoint: 20,
issued_at: now,
valid_until: now + chrono::Duration::seconds(60),
ready: false,
helper_ready: false,
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
attempts: vec![],
acknowledged_message_ids: vec![],
key_id: hash(&key.raw_public_key().unwrap()),
};
let bytes = serde_json::to_vec(&d).unwrap();
let signature = Signer::new_without_digest(&key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap();
let response = SyncResponse {
decision_bytes: bytes,
signature,
};
let db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE reconcile_requests(id TEXT,request BLOB,envelope BLOB,generation INTEGER,response BLOB);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB);").unwrap();
db.execute(
"INSERT INTO reconcile_requests VALUES(?1,?2,?3,2,?4)",
rusqlite::params![
q.request_id,
q.encode_to_vec(),
e.encode_to_vec(),
response.encode_to_vec()
],
)
.unwrap();
assert_eq!(agent_history(&db, &binding).unwrap().len(), 1);
let mut bad = response.clone();
bad.signature[0] ^= 1;
db.execute(
"UPDATE reconcile_requests SET response=?1",
[bad.encode_to_vec()],
)
.unwrap();
assert!(agent_history(&db, &binding).is_err());
db.execute(
"UPDATE reconcile_requests SET response=?1",
[response.encode_to_vec()],
)
.unwrap();
let mut changed = e;
changed.sequence = 11;
db.execute(
"UPDATE reconcile_requests SET envelope=?1",
[changed.encode_to_vec()],
)
.unwrap();
assert!(agent_history(&db, &binding).is_err());
}
}