fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-13 17:43:38 +01:00
parent 998f16fde4
commit 359233c64d
21 changed files with 3030 additions and 30 deletions
+16 -1
View File
@@ -1,6 +1,6 @@
# Rust RMM: Architektur und Implementierungsspezifikation
Version 1.9 · 2026-11-13 · Sprache: Deutsch
Version 1.10 · 2026-11-13 · Sprache: Deutsch
**Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam.
@@ -66,6 +66,21 @@ Deploymentdaten, offene Konfiguration und reale Nachweise:
### 0.1 Auftrag an die implementierende KI
**Technische Präzisierung Version 1.10 — wirkungslose Kontrollkettenreparatur:**
Ein nachweislich nicht serverseitig persistierter Sync-Request darf im expliziten
Entwicklungsprofil über ein eng begrenztes Root-Bootstrapwerkzeug abgeglichen
werden. Server und beide lokalen Verifier prüfen die vollständigen erhaltenen
Journale, ursprünglichen Signaturen, exakten Geräte-/Boot-/Checkpointbindungen
und den unveränderten offenen Verbrauch. Nur die bereits gespeicherte signierte
Ablehnung wird zum monotonen gemeinsamen Kontrollhead; eine kurzlebige,
domänengetrennte Signatur bindet sämtliche Zeugen. Lokale alte Faults und
Quittung werden dauerhaft archiviert. Keine Bereitschaft, Genehmigung, Jobs,
Grants, Verbräuche oder Outcomes entstehen oder ändern sich dadurch. Kein
manuelles Löschen einer Sperre, kein Journal-/Identitätsreset. Erst der normale
signierte Sync und eine unabhängig gebundene Nachprüfung dürfen anschließend
ein unbekanntes Ausführungsergebnis klassifizieren. Dies konkretisiert den
bestehenden Abgleichvertrag und ist keine Ausnahme von der WebUI-Jobgenehmigung.
**Präzisierung Version 1.9, 2026-11-13 — expliziter neuer APT-Testversuch:**
Ein bekannter Fehler vor der vertrauenswürdigen Hookübergabe darf nach belegtem
unverändertem vollständigem Paketbestand durch genau einen neuen festen,
+1
View File
@@ -3,6 +3,7 @@ name = "rmm-agent"
version.workspace = true
edition.workspace = true
license.workspace = true
default-run = "rmm-agent"
[dependencies]
anyhow.workspace = true
chrono.workspace = true
@@ -0,0 +1,497 @@
//! Narrow, root-only development bootstrap. No job execution or ready lease.
use anyhow::{Context, Result, ensure};
use fs2::FileExt;
use prost::Message;
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rmm_privileged_helper::{Binding, checkpoint_repair as repair};
use rmm_protocol::{Envelope, Payload, SyncRequest, SyncResponse};
use rusqlite::{Connection, OpenFlags, OptionalExtension};
use std::{
fs::{self, OpenOptions},
io::Write,
os::unix::fs::{MetadataExt, OpenOptionsExt},
path::Path,
};
const AGENT: &str = "/var/lib/rmm-agent-remote";
const HELPER: &str = "/var/lib/rmm-helper";
fn read_private(path: &Path, owner: u32, limit: usize) -> Result<Vec<u8>> {
rmm_privileged_helper::private_path(path, owner, false)?;
let m = fs::metadata(path)?;
ensure!(m.len() <= limit as u64, "REPAIR_FILE_SIZE");
Ok(fs::read(path)?)
}
fn open_existing(path: &Path, owner: u32) -> Result<Connection> {
rmm_privileged_helper::private_path(path, owner, false)?;
// Require the existing DELETE journal mode. This bootstrap never opens WAL
// state or creates privileged sidecars in the unprivileged agent directory.
for suffix in ["-wal", "-shm", "-journal"] {
ensure!(
!Path::new(&format!("{}{suffix}", path.display())).try_exists()?,
"REPAIR_DATABASE_SIDECAR_PRESENT"
);
}
let db = Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
let mode: String = db.query_row("PRAGMA journal_mode", [], |v| v.get(0))?;
ensure!(mode == "delete", "REPAIR_DELETE_JOURNAL_REQUIRED");
db.execute_batch(
"PRAGMA synchronous=FULL; PRAGMA foreign_keys=OFF; PRAGMA trusted_schema=OFF;",
)?;
Ok(db)
}
fn signed(raw: &[u8]) -> Result<SignedDecision> {
let r = SyncResponse::decode(raw)?;
ensure!(r.encode_to_vec() == raw, "REPAIR_NONCANONICAL_RESPONSE");
Ok(SignedDecision {
decision_bytes: r.decision_bytes,
signature: r.signature,
})
}
fn check_response(
binding: &Binding,
request: &SyncRequest,
envelope: &Envelope,
response: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
let d = repair::verify_historical_decision(binding, response)?;
ensure!(
d.request_id.to_string() == request.request_id
&& d.request_hash == hash(&request.encode_to_vec())
&& d.journal_id.to_string() == request.journal_id
&& d.checkpoint == request.checkpoint
&& d.connection_generation == envelope.connection_generation
&& d.boot_id.to_string() == envelope.boot_id
&& d.helper_journal_id
.map(|id| id.to_string())
.unwrap_or_default()
== request.helper_journal_id
&& d.helper_checkpoint == request.helper_checkpoint,
"REPAIR_AGENT_SIGNED_REQUEST_BINDING"
);
Ok(d)
}
fn agent_history(db: &Connection, binding: &Binding) -> Result<Vec<(SyncRequest, SignedDecision)>> {
let mut s = db.prepare(
"SELECT request,envelope,generation,response FROM reconcile_requests ORDER BY rowid",
)?;
let rows = s
.query_map([], |v| {
Ok((
v.get::<_, Vec<u8>>(0)?,
v.get::<_, Vec<u8>>(1)?,
v.get::<_, u64>(2)?,
v.get::<_, Option<Vec<u8>>>(3)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
ensure!(rows.len() <= 100000, "REPAIR_AGENT_HISTORY_LIMIT");
let mut result = vec![];
for (raw, outer, generation, response) in rows {
ensure!(
raw.len() <= rmm_domain::MAX_MESSAGE && outer.len() <= rmm_domain::MAX_MESSAGE,
"REPAIR_REQUEST_SIZE"
);
let request = SyncRequest::decode(raw.as_slice())?;
let envelope = Envelope::decode(outer.as_slice())?;
ensure!(
request.encode_to_vec() == raw
&& envelope.encode_to_vec() == outer
&& envelope.payload == Some(Payload::SyncRequest(request.clone()))
&& envelope.agent_id == binding.agent_id.to_string()
&& envelope.enrollment_generation == binding.enrollment_generation
&& envelope.connection_generation == generation
&& envelope.sequence == request.checkpoint
&& request.assignment_version == binding.assignment_version,
"REPAIR_AGENT_DURABLE_REQUEST_BINDING"
);
if let Some(response) = response {
let response = signed(&response)?;
check_response(binding, &request, &envelope, &response)?;
result.push((request, response));
}
}
let mut offers = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
for row in offers.query_map([], |v| {
Ok((
v.get::<_, String>(0)?,
v.get::<_, String>(1)?,
v.get::<_, Vec<u8>>(2)?,
))
})? {
let (job, attempt, bytes) = row?;
let offer = rmm_protocol::JobOffer::decode(bytes.as_slice())?;
let payload = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job
&& offer.attempt_id == attempt
&& payload.job_id.to_string() == job
&& payload.attempt_id.to_string() == attempt
&& offer.job_payload_hash == hash(&offer.payload_bytes),
"REPAIR_AGENT_OFFER_BINDING"
);
}
Ok(result)
}
fn combined_inventory(agent: &Connection, consumption: &Connection) -> Result<serde_json::Value> {
let mut value = repair::inventory(agent, "reconcile_meta")?;
value
.as_object_mut()
.context("REPAIR_AGENT_INVENTORY")?
.insert(
"agent_consumption".into(),
repair::inventory(consumption, "")?,
);
Ok(value)
}
fn evidence(
agent: &Connection,
consumption: &Connection,
helper: &Connection,
binding: &Binding,
) -> Result<Evidence> {
rmm_privileged_helper::reconcile::verify_checkpoint_repair_history(helper, binding, None)?;
let history = agent_history(agent, binding)?;
let (head, fault, pending): (String, Option<String>, Option<String>) = agent.query_row(
"SELECT last_decision,fault,pending_request FROM reconcile_meta WHERE id=1",
[],
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?)),
)?;
ensure!(
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH") && pending.is_none(),
"REPAIR_AGENT_EXACT_FAULT_REQUIRED"
);
let (y, denial) = history
.iter()
.find(|(_, r)| {
serde_json::from_slice::<ReconcileDecisionPayload>(&r.decision_bytes)
.is_ok_and(|d| d.decision_id.to_string() == head)
})
.context("REPAIR_REJECTION_MISSING")?;
let raw_x: Vec<u8> = agent.query_row(
"SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL",
[&y.previous_request_id],
|v| v.get(0),
)?;
let x = SyncRequest::decode(raw_x.as_slice())?;
let (_, a) = history
.iter()
.find(|(_, r)| {
serde_json::from_slice::<ReconcileDecisionPayload>(&r.decision_bytes)
.is_ok_and(|d| d.decision_id.to_string() == x.previous_decision_id)
})
.context("REPAIR_PREVIOUS_VALID_MISSING")?;
Ok(Evidence {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: binding.enrollment_generation,
assignment_version: binding.assignment_version,
agent_inventory: combined_inventory(agent, consumption)?,
helper_inventory: repair::inventory(helper, "helper_sync_meta")?,
uncommitted_request: raw_x,
rejected_request: y.encode_to_vec(),
rejected_response: denial.clone(),
previous_valid_response: a.clone(),
})
}
fn verify_agent(
agent: &Connection,
consumption: &Connection,
binding: &Binding,
signed: &SignedReceipt,
e: &Evidence,
) -> Result<()> {
let r = repair::verify_receipt(binding, signed)?;
let _ = agent_history(agent, binding)?;
ensure!(
e.schema_version == 1
&& e.tenant_id == binding.tenant_id
&& e.asset_id == binding.asset_id
&& e.agent_id == binding.agent_id
&& e.enrollment_generation == binding.enrollment_generation
&& e.assignment_version == binding.assignment_version,
"REPAIR_EVIDENCE_IDENTITY"
);
let actual = combined_inventory(agent, consumption)?;
ensure!(
actual == e.agent_inventory && inventory_sha256(&actual)? == r.agent_inventory_sha256,
"REPAIR_AGENT_INVENTORY"
);
let (journal,checkpoint,high,head,pending,helperid,helpercp,fault):(String,u64,u64,String,Option<String>,String,u64,Option<String>)=agent.query_row("SELECT journal_id,checkpoint,high_sequence,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|v|Ok((v.get(0)?,v.get(1)?,v.get(2)?,v.get(3)?,v.get(4)?,v.get(5)?,v.get(6)?,v.get(7)?)))?;
let counter: u64 = agent.query_row("SELECT seq FROM counter WHERE id=1", [], |v| v.get(0))?;
ensure!(
journal == r.agent_journal_id.to_string()
&& checkpoint == r.agent_checkpoint
&& counter >= high
&& counter >= checkpoint
&& head == r.rejected_decision_id.to_string()
&& pending.is_none()
&& helperid == r.helper_journal_id.to_string()
&& helpercp == r.helper_checkpoint
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|| (fault.is_none() && repair::already_applied(agent, signed)?)),
"REPAIR_AGENT_HEAD"
);
let y = SyncRequest::decode(e.rejected_request.as_slice())?;
let x = SyncRequest::decode(e.uncommitted_request.as_slice())?;
let stored_y: (Vec<u8>, Vec<u8>) = agent.query_row(
"SELECT request,response FROM reconcile_requests WHERE id=?1",
[&y.request_id],
|v| Ok((v.get(0)?, v.get(1)?)),
)?;
let stored_x: Option<Vec<u8>> = agent
.query_row(
"SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL",
[&x.request_id],
|v| v.get(0),
)
.optional()?;
let d = repair::verify_historical_decision(binding, &e.rejected_response)?;
let a = repair::verify_historical_decision(binding, &e.previous_valid_response)?;
ensure!(
stored_y.0 == e.rejected_request
&& signed_response_eq(&stored_y.1, &e.rejected_response)?
&& stored_x.as_deref() == Some(e.uncommitted_request.as_slice())
&& d.decision_id == r.rejected_decision_id
&& hash(&e.rejected_response.decision_bytes) == r.rejected_decision_sha256
&& d.reason == r.fault
&& !d.ready
&& !d.helper_ready
&& !d.attempts.iter().any(|v| v.unlock)
&& a.decision_id == r.previous_valid_decision_id
&& hash(&e.previous_valid_response.decision_bytes) == r.previous_valid_decision_sha256
&& hash(&e.rejected_request) == r.rejected_request_sha256
&& y.request_id == r.rejected_request_id.to_string()
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
&& x.request_id == r.uncommitted_request_id.to_string()
&& x.previous_decision_id == a.decision_id.to_string()
&& x.helper_previous_decision_id == a.decision_id.to_string()
&& y.previous_decision_id.is_empty()
&& y.previous_request_id == x.request_id
&& y.helper_previous_decision_id == a.decision_id.to_string(),
"REPAIR_AGENT_EXACT_UNSENT_CHAIN"
);
Ok(())
}
fn signed_response_eq(raw: &[u8], expected: &SignedDecision) -> Result<bool> {
Ok(signed(raw)? == *expected)
}
fn main() -> Result<()> {
ensure!(
unsafe { libc::geteuid() } == 0,
"REPAIR_ROOT_BOOTSTRAP_ONLY"
);
ensure!(
std::env::var("RMM_CHECKPOINT_REPAIR_PROFILE").as_deref() == Ok("development"),
"REPAIR_EXPLICIT_DEVELOPMENT_PROFILE_REQUIRED"
);
ensure!(
rmm_privileged_helper::clock_trusted(),
"REPAIR_CLOCK_UNTRUSTED"
);
let args = std::env::args_os().skip(1).collect::<Vec<_>>();
ensure!(
matches!(args.len(), 3 | 4),
"usage: rmm-checkpoint-repair prepare BINDING_FILE EVIDENCE_FILE | apply BINDING_FILE EVIDENCE_FILE RECEIPT_FILE"
);
let binding = rmm_privileged_helper::read_binding(Path::new(&args[1]), 0)?;
let owner = fs::metadata(AGENT)?.uid();
ensure!(owner != 0, "REPAIR_UNPRIVILEGED_AGENT_REQUIRED");
rmm_privileged_helper::private_path(Path::new(AGENT), owner, true)?;
rmm_privileged_helper::private_path(Path::new(HELPER), 0, true)?;
let lock_path = Path::new(AGENT).join("agent.lock");
rmm_privileged_helper::private_path(&lock_path, owner, false)?;
let lock = OpenOptions::new()
.read(true)
.write(true)
.custom_flags(libc::O_NOFOLLOW)
.open(lock_path)?;
lock.try_lock_exclusive()?;
let _guard = rmm_privileged_helper::reconcile::ExecutorGuard::acquire(Path::new(HELPER), 0)?;
let mut agent = open_existing(&Path::new(AGENT).join("journal.sqlite"), owner)?;
let consumption = open_existing(&Path::new(AGENT).join("consumption.sqlite"), owner)?;
let mut helper = open_existing(&Path::new(HELPER).join("consumption.sqlite"), 0)?;
let mode = args[0].to_str().context("REPAIR_MODE")?;
match (mode, args.len()) {
("prepare", 3) => {
let e = evidence(&agent, &consumption, &helper, &binding)?;
let bytes = serde_json::to_vec(&e)?;
ensure!(bytes.len() <= 32 * 1024 * 1024, "REPAIR_EVIDENCE_SIZE");
let mut f = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(&args[2])?;
f.write_all(&bytes)?;
f.sync_all()?;
println!("checkpoint repair evidence prepared; no state changed");
}
("apply", 4) => {
let e: Evidence =
serde_json::from_slice(&read_private(Path::new(&args[2]), 0, 32 * 1024 * 1024)?)?;
let signed: SignedReceipt =
serde_json::from_slice(&read_private(Path::new(&args[3]), 0, 65536)?)?;
verify_agent(&agent, &consumption, &binding, &signed, &e)?;
repair::verify_helper(&helper, &binding, &signed, &e)?;
let r = repair::verify_receipt(&binding, &signed)?;
// Every witness is verified before either durable fault archive is written.
// A stopped bootstrap can replay the identical receipt after interruption.
repair::commit_fault_repair(
&mut helper,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)?;
repair::commit_fault_repair(
&mut agent,
"reconcile_meta",
&signed,
&r.agent_inventory_sha256,
Some(("agent_consumption", repair::inventory(&consumption, "")?)),
)?;
println!(
"exact checkpoint faults archived and repaired; no execution readiness granted"
);
}
_ => anyhow::bail!("REPAIR_UNKNOWN_MODE"),
}
drop(helper);
drop(consumption);
drop(agent);
// DELETE journaling leaves no root-owned WAL/SHM or journal behind.
for name in ["journal.sqlite", "consumption.sqlite"] {
for suffix in ["-wal", "-shm", "-journal"] {
ensure!(
!Path::new(AGENT)
.join(format!("{name}{suffix}"))
.try_exists()?,
"REPAIR_UNEXPECTED_AGENT_SIDECAR"
);
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::Utc;
use openssl::{pkey::PKey, sign::Signer};
use uuid::Uuid;
#[test]
fn durable_signed_request_history_rejects_mutation_before_any_repair() {
let key = PKey::generate_ed25519().unwrap();
let id = Uuid::new_v4();
let boot = Uuid::new_v4();
let binding = Binding {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: key
.raw_public_key()
.unwrap()
.iter()
.map(|b| format!("{b:02x}"))
.collect(),
};
let q = SyncRequest {
request_id: Uuid::new_v4().to_string(),
journal_id: id.to_string(),
checkpoint: 10,
assignment_version: 1,
journal_complete: true,
helper_journal_id: id.to_string(),
helper_checkpoint: 20,
..Default::default()
};
let e = Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: id.to_string(),
enrollment_generation: 1,
connection_generation: 2,
boot_id: boot.to_string(),
sequence: 10,
observed_at: Utc::now().timestamp_millis(),
payload: Some(Payload::SyncRequest(q.clone())),
};
let now = Utc::now();
let d = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::parse_str(&q.request_id).unwrap(),
request_hash: hash(&q.encode_to_vec()),
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
connection_generation: 2,
boot_id: boot,
journal_id: id,
checkpoint: 10,
helper_journal_id: Some(id),
helper_checkpoint: 20,
issued_at: now,
valid_until: now + chrono::Duration::seconds(60),
ready: false,
helper_ready: false,
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
attempts: vec![],
acknowledged_message_ids: vec![],
key_id: hash(&key.raw_public_key().unwrap()),
};
let bytes = serde_json::to_vec(&d).unwrap();
let signature = Signer::new_without_digest(&key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap();
let response = SyncResponse {
decision_bytes: bytes,
signature,
};
let db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE reconcile_requests(id TEXT,request BLOB,envelope BLOB,generation INTEGER,response BLOB);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB);").unwrap();
db.execute(
"INSERT INTO reconcile_requests VALUES(?1,?2,?3,2,?4)",
rusqlite::params![
q.request_id,
q.encode_to_vec(),
e.encode_to_vec(),
response.encode_to_vec()
],
)
.unwrap();
assert_eq!(agent_history(&db, &binding).unwrap().len(), 1);
let mut bad = response.clone();
bad.signature[0] ^= 1;
db.execute(
"UPDATE reconcile_requests SET response=?1",
[bad.encode_to_vec()],
)
.unwrap();
assert!(agent_history(&db, &binding).is_err());
db.execute(
"UPDATE reconcile_requests SET response=?1",
[response.encode_to_vec()],
)
.unwrap();
let mut changed = e;
changed.sequence = 11;
db.execute(
"UPDATE reconcile_requests SET envelope=?1",
[changed.encode_to_vec()],
)
.unwrap();
assert!(agent_history(&db, &binding).is_err());
}
}
+139
View File
@@ -0,0 +1,139 @@
//! Development bootstrap receipt for one proven unsent reconciliation request.
//! This is neither a job approval, an execution permit nor a ready lease.
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct SignedDecision {
pub decision_bytes: Vec<u8>,
pub signature: Vec<u8>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct SignedReceipt {
pub payload: Vec<u8>,
pub signature: Vec<u8>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Evidence {
pub schema_version: u32,
pub tenant_id: Uuid,
pub asset_id: Uuid,
pub agent_id: Uuid,
pub enrollment_generation: u64,
pub assignment_version: u64,
pub agent_inventory: Value,
pub helper_inventory: Value,
pub uncommitted_request: Vec<u8>,
pub rejected_request: Vec<u8>,
pub rejected_response: SignedDecision,
pub previous_valid_response: SignedDecision,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct Receipt {
pub domain: String,
pub schema_version: u32,
pub receipt_id: Uuid,
pub tenant_id: Uuid,
pub asset_id: Uuid,
pub agent_id: Uuid,
pub enrollment_generation: u64,
pub assignment_version: u64,
pub issued_at: DateTime<Utc>,
pub valid_until: DateTime<Utc>,
pub key_id: String,
pub fault: String,
pub agent_journal_id: Uuid,
pub agent_checkpoint: u64,
pub helper_journal_id: Uuid,
pub helper_checkpoint: u64,
pub rejected_decision_id: Uuid,
pub rejected_decision_sha256: String,
pub rejected_request_id: Uuid,
pub rejected_request_sha256: String,
pub uncommitted_request_id: Uuid,
pub uncommitted_request_sha256: String,
pub previous_valid_decision_id: Uuid,
pub previous_valid_decision_sha256: String,
pub agent_inventory_sha256: String,
pub helper_inventory_sha256: String,
pub helper_ledger_sha256: String,
pub ready: bool,
pub helper_ready: bool,
pub unlock: bool,
}
pub fn canonical_bytes(value: &Value) -> Result<Vec<u8>, serde_json::Error> {
fn ordered(v: Value) -> Value {
match v {
Value::Object(m) => {
let mut items = m.into_iter().collect::<Vec<_>>();
items.sort_by(|a, b| a.0.cmp(&b.0));
Value::Object(items.into_iter().map(|(k, v)| (k, ordered(v))).collect())
}
Value::Array(a) => Value::Array(a.into_iter().map(ordered).collect()),
other => other,
}
}
serde_json::to_vec(&ordered(value.clone()))
}
pub fn inventory_sha256(value: &Value) -> Result<String, serde_json::Error> {
Ok(crate::jobs::hash(&canonical_bytes(value)?))
}
pub fn helper_ledger_sha256(inventory: &Value) -> Result<String, serde_json::Error> {
inventory_sha256(
&serde_json::json!({"consumed":inventory["consumed"],"execution_evidence":inventory["execution_evidence"]}),
)
}
impl Receipt {
pub fn valid_profile(&self, now: DateTime<Utc>) -> bool {
let digest = |s: &str| {
s.len() == 64
&& s.bytes()
.all(|c| c.is_ascii_digit() || b"abcdef".contains(&c))
};
self.domain == "rmm.sync.unsent-request-repair"
&& self.schema_version == 1
&& !self.receipt_id.is_nil()
&& self.fault == "CHECKPOINT_CHAIN_MISMATCH"
&& !self.ready
&& !self.helper_ready
&& !self.unlock
&& self.issued_at <= now
&& now < self.valid_until
&& self.valid_until > self.issued_at
&& self.valid_until - self.issued_at <= chrono::Duration::minutes(15)
&& [
self.tenant_id,
self.asset_id,
self.agent_id,
self.agent_journal_id,
self.helper_journal_id,
self.rejected_decision_id,
self.rejected_request_id,
self.uncommitted_request_id,
self.previous_valid_decision_id,
]
.iter()
.all(|id| !id.is_nil())
&& self.enrollment_generation > 0
&& self.assignment_version > 0
&& self.agent_checkpoint > 0
&& self.helper_checkpoint > 0
&& [
&self.key_id,
&self.rejected_decision_sha256,
&self.rejected_request_sha256,
&self.uncommitted_request_sha256,
&self.previous_valid_decision_sha256,
&self.agent_inventory_sha256,
&self.helper_inventory_sha256,
&self.helper_ledger_sha256,
]
.iter()
.all(|s| digest(s))
}
}
+1
View File
@@ -1,5 +1,6 @@
use serde::{Deserialize, Serialize};
pub mod apt_transaction;
pub mod checkpoint_repair;
pub mod inventory_document;
pub mod jobs;
pub mod reconciliation;
@@ -0,0 +1,458 @@
//! Fixed bootstrap verification of one signed no-effect checkpoint repair.
use crate::Binding;
use anyhow::{Context, Result, ensure};
use chrono::Utc;
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rusqlite::{Connection, OptionalExtension, params, types::ValueRef};
use serde_json::{Value, json};
pub const AUDIT: &str = "checkpoint_repair_receipts";
pub fn verify_historical_decision(
binding: &Binding,
signed: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
ensure!(
signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64,
"REPAIR_DECISION_SIZE"
);
let raw = hex::decode(&binding.public_key_hex)?;
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?
.verify_oneshot(&signed.signature, &signed.decision_bytes)?,
"REPAIR_HISTORICAL_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
ensure!(
d.valid_profile(d.issued_at)
&& d.key_id == hash(&raw)
&& d.tenant_id == binding.tenant_id
&& d.asset_id == binding.asset_id
&& d.agent_id == binding.agent_id
&& d.enrollment_generation == binding.enrollment_generation
&& d.assignment_version == binding.assignment_version,
"REPAIR_HISTORICAL_IDENTITY"
);
Ok(d)
}
pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result<Receipt> {
ensure!(
signed.payload.len() <= 16384 && signed.signature.len() == 64,
"REPAIR_RECEIPT_SIZE"
);
let raw = hex::decode(&binding.public_key_hex)?;
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?,
"REPAIR_RECEIPT_SIGNATURE"
);
let r: Receipt = serde_json::from_slice(&signed.payload)?;
ensure!(
r.valid_profile(Utc::now())
&& r.key_id == hash(&raw)
&& r.tenant_id == binding.tenant_id
&& r.asset_id == binding.asset_id
&& r.agent_id == binding.agent_id
&& r.enrollment_generation == binding.enrollment_generation
&& r.assignment_version == binding.assignment_version,
"REPAIR_RECEIPT_BINDING"
);
Ok(r)
}
/// Full database inventory with typed values; byte fields are bounded hashes.
/// The append-only repair archive is excluded; only the repaired fault cell is
/// normalized for exact idempotent replay of the same signed receipt.
pub fn inventory(db: &Connection, meta: &str) -> Result<Value> {
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY");
let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?;
ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM");
let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?;
let names = q
.query_map([AUDIT], |r| r.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut tables = serde_json::Map::new();
let mut count = 0;
for name in names {
ensure!(
name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
"REPAIR_TABLE_NAME"
);
let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?;
let columns = s
.column_names()
.into_iter()
.map(str::to_owned)
.collect::<Vec<_>>();
let mut rows = vec![];
let mut cursor = s.query([])?;
while let Some(row) = cursor.next()? {
count += 1;
ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT");
let mut cells = vec![];
for (i, col) in columns.iter().enumerate() {
let cell = if name == meta && col == "fault" {
let v = row.get_ref(i)?;
ensure!(
matches!(v, ValueRef::Null)
|| matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")),
"REPAIR_DIFFERENT_FAULT"
);
json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"})
} else {
match row.get_ref(i)? {
ValueRef::Null => json!({"type":"null"}),
ValueRef::Integer(v) => json!({"type":"integer","value":v}),
ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"),
ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}),
ValueRef::Blob(v) => {
json!({"type":"blob","sha256":hash(v),"length":v.len()})
}
}
};
cells.push(cell);
}
rows.push(cells);
}
tables.insert(name, json!({"columns":columns,"rows":rows}));
}
Ok(Value::Object(tables))
}
pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result<bool> {
let exists: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
[AUDIT],
|r| r.get(0),
)?;
if !exists {
return Ok(false);
}
let r: Receipt = serde_json::from_slice(&signed.payload)?;
let old: Option<Vec<u8>> = db
.query_row(
"SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1",
[r.receipt_id.to_string()],
|v| v.get(0),
)
.optional()?;
if let Some(old) = old {
ensure!(
old == serde_json::to_vec(signed)?,
"REPAIR_RECEIPT_CONFLICT"
);
Ok(true)
} else {
Ok(false)
}
}
pub fn commit_fault_repair(
db: &mut Connection,
meta: &str,
signed: &SignedReceipt,
expected_hash: &str,
additional: Option<(&str, Value)>,
) -> Result<()> {
ensure!(
matches!(meta, "reconcile_meta" | "helper_sync_meta"),
"REPAIR_META_TABLE"
);
let r: Receipt = serde_json::from_slice(&signed.payload)?;
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let mut actual = inventory(&tx, meta)?;
if let Some((key, value)) = additional {
actual
.as_object_mut()
.context("REPAIR_INVENTORY_OBJECT")?
.insert(key.into(), value);
}
ensure!(
inventory_sha256(&actual)? == expected_hash,
"REPAIR_WITNESS_CHANGED"
);
let fault: Option<String> =
tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| {
v.get(0)
})?;
if already_applied(&tx, signed)? {
ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED");
tx.commit()?;
return Ok(());
}
ensure!(
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"),
"REPAIR_EXACT_FAULT_REQUIRED"
);
tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?;
tx.execute(
"INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)",
params![
r.receipt_id.to_string(),
serde_json::to_vec(signed)?,
fault,
expected_hash,
Utc::now().to_rfc3339()
],
)?;
ensure!(
tx.execute(
&format!(
"UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'"
),
[]
)? == 1,
"REPAIR_FAULT_CHANGED"
);
let mut after = inventory(&tx, meta)?;
if let Some(value) = actual.get("agent_consumption") {
after
.as_object_mut()
.context("REPAIR_INVENTORY_OBJECT")?
.insert("agent_consumption".into(), value.clone());
}
ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED");
tx.commit()?;
Ok(())
}
pub fn verify_helper(
db: &Connection,
binding: &Binding,
signed: &SignedReceipt,
e: &Evidence,
) -> Result<()> {
let r = verify_receipt(binding, signed)?;
ensure!(
e.schema_version == 1
&& e.tenant_id == binding.tenant_id
&& e.asset_id == binding.asset_id
&& e.agent_id == binding.agent_id
&& e.enrollment_generation == binding.enrollment_generation
&& e.assignment_version == binding.assignment_version,
"REPAIR_HELPER_EVIDENCE_IDENTITY"
);
crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?;
let inv = inventory(db, "helper_sync_meta")?;
ensure!(
inventory_sha256(&inv)? == r.helper_inventory_sha256
&& inv == e.helper_inventory
&& helper_ledger_sha256(&inv)? == r.helper_ledger_sha256,
"REPAIR_HELPER_INVENTORY"
);
let (journal, checkpoint, decision, fault): (String, u64, String, Option<String>) = db
.query_row(
"SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1",
[],
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)),
)?;
ensure!(
journal == r.helper_journal_id.to_string()
&& checkpoint == r.helper_checkpoint
&& decision == r.rejected_decision_id.to_string()
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|| (fault.is_none() && already_applied(db, signed)?)),
"REPAIR_HELPER_HEAD"
);
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
[decision],
|v| Ok((v.get(0)?, v.get(1)?)),
)?;
let stored: SignedDecision = serde_json::from_slice(&response)?;
ensure!(
stored == e.rejected_response
&& request == e.rejected_request
&& hash(&request) == r.rejected_request_sha256
&& hash(&stored.decision_bytes) == r.rejected_decision_sha256,
"REPAIR_HELPER_REJECTION"
);
let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?;
ensure!(
rejection.reason == r.fault
&& !rejection.ready
&& !rejection.helper_ready
&& !rejection.attempts.iter().any(|v| v.unlock),
"REPAIR_REJECTION_HAS_AUTHORITY"
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use openssl::{pkey::Private, sign::Signer};
use uuid::Uuid;
fn fixture() -> (Binding, PKey<Private>, Receipt) {
let key = PKey::generate_ed25519().unwrap();
let id = Uuid::new_v4();
let now = Utc::now();
let binding = Binding {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
};
let r = Receipt {
domain: "rmm.sync.unsent-request-repair".into(),
schema_version: 1,
receipt_id: Uuid::new_v4(),
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
issued_at: now,
valid_until: now + chrono::Duration::minutes(15),
key_id: hash(&key.raw_public_key().unwrap()),
fault: "CHECKPOINT_CHAIN_MISMATCH".into(),
agent_journal_id: id,
agent_checkpoint: 13,
helper_journal_id: id,
helper_checkpoint: 22,
rejected_decision_id: id,
rejected_decision_sha256: hash(b"denied"),
rejected_request_id: id,
rejected_request_sha256: hash(b"request"),
uncommitted_request_id: id,
uncommitted_request_sha256: hash(b"unsent"),
previous_valid_decision_id: id,
previous_valid_decision_sha256: hash(b"previous"),
agent_inventory_sha256: hash(b"agent"),
helper_inventory_sha256: hash(b"helper"),
helper_ledger_sha256: hash(b"ledger"),
ready: false,
helper_ready: false,
unlock: false,
};
(binding, key, r)
}
fn sign(key: &PKey<Private>, r: &Receipt) -> SignedReceipt {
let payload = serde_json::to_vec(r).unwrap();
let signature = Signer::new_without_digest(key)
.unwrap()
.sign_oneshot_to_vec(&payload)
.unwrap();
SignedReceipt { payload, signature }
}
#[test]
fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() {
let (binding, key, r) = fixture();
assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok());
let mut bad = sign(&key, &r);
bad.payload[0] ^= 1;
assert!(verify_receipt(&binding, &bad).is_err());
for changed in [0, 1, 2, 3, 4, 5] {
let mut bad = r.clone();
match changed {
0 => bad.ready = true,
1 => bad.helper_ready = true,
2 => bad.unlock = true,
3 => bad.agent_id = Uuid::new_v4(),
4 => bad.domain = "rmm.execution-permit".into(),
_ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16),
};
assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err());
}
let mut expired = r.clone();
expired.issued_at = Utc::now() - chrono::Duration::minutes(2);
expired.valid_until = Utc::now() - chrono::Duration::minutes(1);
assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err());
}
fn database() -> Connection {
let db = Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap();
db
}
#[test]
fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() {
let (_, key, mut r) = fixture();
let mut db = database();
let before = inventory(&db, "helper_sync_meta").unwrap();
r.helper_inventory_sha256 = inventory_sha256(&before).unwrap();
r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap();
let signed = sign(&key, &r);
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)
.unwrap();
assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before);
assert!(
db.query_row("SELECT result IS NULL FROM consumed", [], |v| v
.get::<_, bool>(0))
.unwrap()
);
assert_eq!(
db.query_row("SELECT outcome FROM execution_evidence", [], |v| v
.get::<_, Vec<u8>>(0))
.unwrap(),
b"unknown"
);
assert_eq!(
db.query_row(
"SELECT old_fault FROM checkpoint_repair_receipts",
[],
|v| v.get::<_, String>(0)
)
.unwrap(),
"CHECKPOINT_CHAIN_MISMATCH"
);
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None,
)
.unwrap();
assert_eq!(
db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v
.get::<_, u64>(0))
.unwrap(),
1
);
db.execute("UPDATE consumed SET result=x'01'", []).unwrap();
assert!(
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None
)
.is_err()
);
}
#[test]
fn changed_witness_or_other_fault_is_never_cleared() {
let (_, key, r) = fixture();
let mut db = database();
let signed = sign(&key, &r);
assert!(
commit_fault_repair(
&mut db,
"helper_sync_meta",
&signed,
&r.helper_inventory_sha256,
None
)
.is_err()
);
db.execute(
"UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'",
[],
)
.unwrap();
assert!(inventory(&db, "helper_sync_meta").is_err());
assert!(!already_applied(&db, &signed).unwrap());
let db = database();
db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap();
assert!(inventory(&db, "helper_sync_meta").is_err());
}
}
+1
View File
@@ -3,6 +3,7 @@
extern crate self as rmm_privileged_helper;
pub mod apt;
pub mod apt_reconcile;
pub mod checkpoint_repair;
pub mod control;
pub mod readiness;
pub mod reconcile;
+143
View File
@@ -681,6 +681,142 @@ fn validate_history(
}
Ok(())
}
/// Bootstrap-only historical verification; this never creates a lease.
pub fn verify_checkpoint_repair_history(
db: &Connection,
binding: &Binding,
links: Option<(
&rmm_domain::checkpoint_repair::Receipt,
&rmm_domain::checkpoint_repair::Evidence,
)>,
) -> Result<()> {
let meta = metadata(db)?;
validate_history(db, binding, &meta, meta.generation)?;
let mut statement = db.prepare(
"SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint",
)?;
let entries = statement
.query_map([], |r| {
Ok((
r.get::<_, Vec<u8>>(0)?,
r.get::<_, Vec<u8>>(1)?,
r.get::<_, u64>(2)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT");
let mut previous = None;
let mut previous_checkpoint = 0;
let mut previous_agent_checkpoint = 0;
for (signed, raw, checkpoint) in entries {
let response: SignedResponse = serde_json::from_slice(&signed)?;
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
let decision = verify(binding, &response, parsed.issued_at)?;
let request = RequestView::decode(&raw)?;
let captured: Vec<u8> = db.query_row(
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
[checkpoint],
|r| r.get(0),
)?;
let capture: Snapshot = serde_json::from_slice(&captured)?;
ensure!(
decision.helper_journal_id == Some(meta.journal_id)
&& decision.helper_checkpoint == checkpoint
&& request.helper_journal_id == meta.journal_id.to_string()
&& request.helper_checkpoint == checkpoint
&& request.assignment_version == binding.assignment_version
&& request.journal_complete
&& decision.request_hash == hash(&raw)
&& decision.request_id.to_string() == request.request_id
&& decision.journal_id.to_string() == request.journal_id
&& decision.checkpoint == request.checkpoint
&& capture.journal_id == meta.journal_id
&& capture.checkpoint == checkpoint
&& capture.boot_id == decision.boot_id
&& capture.captured_at <= decision.issued_at
&& capture.previous_decision_id == previous
&& checkpoint > previous_checkpoint
&& decision.checkpoint >= previous_agent_checkpoint
&& decision.connection_generation <= meta.generation
&& request.helper_previous_decision_id
== capture
.previous_decision_id
.map(|v| v.to_string())
.unwrap_or_default(),
"REPAIR_HELPER_HISTORY_BINDING"
);
previous = Some(decision.decision_id);
previous_checkpoint = checkpoint;
previous_agent_checkpoint = decision.checkpoint;
}
let _ = load_rows(db, binding, false)?;
if let Some((r, e)) = links {
verify_checkpoint_repair_links(db, binding, r, e)?;
}
Ok(())
}
fn verify_checkpoint_repair_links(
db: &Connection,
binding: &Binding,
r: &rmm_domain::checkpoint_repair::Receipt,
e: &rmm_domain::checkpoint_repair::Evidence,
) -> Result<()> {
let x = RequestView::decode(&e.uncommitted_request)?;
let y = RequestView::decode(&e.rejected_request)?;
let a: ReconcileDecisionPayload =
serde_json::from_slice(&e.previous_valid_response.decision_bytes)?;
let a = verify(
binding,
&SignedResponse {
decision_bytes: e.previous_valid_response.decision_bytes.clone(),
signature: e.previous_valid_response.signature.clone(),
},
a.issued_at,
)?;
let old: Vec<u8> = db.query_row(
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
[a.decision_id.to_string()],
|v| v.get(0),
)?;
let old: SignedResponse = serde_json::from_slice(&old)?;
ensure!(
old.decision_bytes == e.previous_valid_response.decision_bytes
&& old.signature == e.previous_valid_response.signature
&& a.decision_id == r.previous_valid_decision_id
&& hash(&old.decision_bytes) == r.previous_valid_decision_sha256
&& a.reason == "ATTEMPT_RECONCILIATION_REQUIRED"
&& !a.ready
&& !a.helper_ready
&& x.request_id == r.uncommitted_request_id.to_string()
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
&& y.request_id == r.rejected_request_id.to_string()
&& hash(&e.rejected_request) == r.rejected_request_sha256
&& x.previous_decision_id == a.decision_id.to_string()
&& x.previous_request_id.is_empty()
&& x.helper_previous_decision_id == a.decision_id.to_string()
&& y.helper_previous_decision_id == a.decision_id.to_string()
&& y.previous_decision_id.is_empty()
&& y.previous_request_id == x.request_id
&& x.journal_id == r.agent_journal_id.to_string()
&& y.journal_id == x.journal_id
&& a.journal_id == r.agent_journal_id
&& x.helper_journal_id == r.helper_journal_id.to_string()
&& y.helper_journal_id == x.helper_journal_id
&& a.helper_journal_id == Some(r.helper_journal_id)
&& a.checkpoint < x.checkpoint
&& x.checkpoint < y.checkpoint
&& y.checkpoint == r.agent_checkpoint
&& a.helper_checkpoint < x.helper_checkpoint
&& x.helper_checkpoint < y.helper_checkpoint
&& y.helper_checkpoint == r.helper_checkpoint
&& x.assignment_version == binding.assignment_version
&& y.assignment_version == binding.assignment_version
&& x.journal_complete
&& y.journal_complete,
"REPAIR_EXACT_UNSENT_CHAIN_REQUIRED"
);
Ok(())
}
fn persist_fault(db: &Connection, error: &anyhow::Error) {
let text = error.to_string();
if [
@@ -991,6 +1127,8 @@ struct RequestView {
helper_journal_id: String,
helper_checkpoint: u64,
helper_previous_decision_id: String,
previous_decision_id: String,
previous_request_id: String,
}
#[derive(Default)]
struct AttemptView {
@@ -1089,6 +1227,11 @@ impl RequestView {
if !s.is_empty() {
Uuid::parse_str(&s)?;
}
if tag == 4 {
r.previous_decision_id = s;
} else {
r.previous_request_id = s;
}
}
5 => r.assignment_version = number(value)?,
6 => {
@@ -39,6 +39,32 @@ struct Fixture {
boot: TestBoot,
agent_journal: Uuid,
}
#[test]
fn checkpoint_repair_verifies_complete_signed_history_and_detects_removed_predecessor() {
let mut f = Fixture::new();
let mut sync = f.sync();
let first = f.snapshot(&mut sync, 7);
let request = f.request(&first);
let d = f.decision(&first, &request, 7, false);
let signed = f.sign(&d);
sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot)
.unwrap();
f.clock.now += Duration::seconds(1);
let second = f.snapshot(&mut sync, 7);
let request = f.request(&second);
let d = f.decision(&second, &request, 7, false);
let signed = f.sign(&d);
sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot)
.unwrap();
verify_checkpoint_repair_history(&f.db, &f.binding, None).unwrap();
f.db.execute(
"DELETE FROM helper_sync_decisions WHERE checkpoint=?1",
[first.checkpoint],
)
.unwrap();
assert!(verify_checkpoint_repair_history(&f.db, &f.binding, None).is_err());
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.dir);
+47 -9
View File
@@ -38,6 +38,35 @@ fn response(messages: Vec<Envelope>) -> Response {
)
.into_response()
}
async fn append_work(
s: &crate::State,
tenant: Uuid,
agent: Uuid,
source: &Envelope,
payload: Payload,
generation: u64,
replies: &mut Vec<Envelope>,
) -> anyhow::Result<()> {
let reply = crate::agent_service::reply(source, payload, generation);
// Preserve the protocol's 16-envelope response limit. An offer that cannot
// fit twice is delivered normally without consuming the optional fault.
let claim = if replies.len() <= 14 {
crate::gateway::dev_delivery_faults::duplicate_offer(
&s.pool, tenant, agent, &reply, "https",
)
.await?
} else {
None
};
if claim.is_some() {
replies.push(reply.clone());
}
replies.push(reply);
if let Some(claim) = claim {
crate::gateway::dev_delivery_faults::duplicate_emitted(&s.pool, claim).await?;
}
Ok(())
}
pub async fn messages(
State(s): State<crate::State>,
Extension(peer): Extension<crate::Peer>,
@@ -83,13 +112,21 @@ pub async fn messages(
}
.map_err(|_| StatusCode::CONFLICT)?;
generation = reply.connection_generation;
replies.push(reply);
if !crate::gateway::dev_delivery_faults::after_ingest(&s.pool, t, a, &e, &reply, "https")
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
replies.push(reply);
}
}
if let Some(work) = crate::job_gateway::next(&s, t, a, generation)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
if replies.len() < 16
&& let Some(work) = crate::job_gateway::next(&s, t, a, generation)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
replies.push(crate::agent_service::reply(&last, work, generation));
append_work(&s, t, a, &last, work, generation, &mut replies)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
}
Ok(response(replies))
}
@@ -119,10 +156,11 @@ pub async fn poll(
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
return Ok(response(vec![
ack,
crate::agent_service::reply(&e, work, e.connection_generation),
]));
let mut replies = vec![ack];
append_work(&s, t, a, &e, work, e.connection_generation, &mut replies)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
return Ok(response(replies));
}
if tokio::time::Instant::now() >= deadline {
return Ok(response(vec![ack]));
+772
View File
@@ -0,0 +1,772 @@
//! Fixed development bootstrap for a proven unsent sync request. This promotes
//! an existing signed denial only; it grants no readiness or execution rights.
use anyhow::{Context, Result, ensure};
use chrono::{Duration, Utc};
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use prost::Message;
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rmm_protocol::SyncRequest;
use serde_json::{Value, json};
use sqlx::Row;
use std::{
fs,
os::unix::fs::{MetadataExt, PermissionsExt},
path::Path,
};
use uuid::Uuid;
const EVENT: &str = "unsent_request_repair_issued";
const FAULT: &str = "CHECKPOINT_CHAIN_MISMATCH";
pub fn closed_policy() -> Result<()> {
ensure!(
std::env::var("RMM_PROFILE").as_deref() == Ok("development")
&& std::env::var("RMM_DEV_AUTH").as_deref() == Ok("true")
&& std::env::var("RMM_SERVER_ROLE").as_deref() == Ok("api"),
"REPAIR_DEVELOPMENT_API_ONLY"
);
let path = std::env::var("RMM_DEV_ROOT_POLICY_FILE").context("REPAIR_POLICY_REQUIRED")?;
let m = fs::symlink_metadata(&path)?;
let parent = fs::symlink_metadata(Path::new(&path).parent().context("REPAIR_POLICY_PARENT")?)?;
let key = std::env::var("RMM_JOB_KEY_FILE").context("REPAIR_EXISTING_KEY_REQUIRED")?;
ensure!(
m.is_file()
&& m.len() <= 65536
&& m.permissions().mode() & 0o077 == 0
&& m.uid() == fs::metadata(key)?.uid()
&& parent.is_dir()
&& parent.permissions().mode() & 0o022 == 0,
"REPAIR_POLICY_PERMISSIONS"
);
let p: Value = serde_json::from_slice(&fs::read(path)?)?;
ensure!(
p["schema_version"] == 1
&& p["profile"] == "development"
&& p["expires_at"]
.as_str()
.and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
.is_some_and(|until| until <= Utc::now()),
"REPAIR_POLICY_MUST_BE_CLOSED"
);
Ok(())
}
fn cell<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a Value> {
let columns = table["columns"]
.as_array()
.context("REPAIR_INVENTORY_COLUMNS")?;
let i = columns
.iter()
.position(|v| v.as_str() == Some(column))
.context("REPAIR_INVENTORY_COLUMN")?;
row.as_array()
.and_then(|r| r.get(i))
.context("REPAIR_INVENTORY_CELL")
}
fn rows(table: &Value) -> Result<&Vec<Value>> {
table["rows"].as_array().context("REPAIR_INVENTORY_ROWS")
}
fn text<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a str> {
let v = cell(table, row, column)?;
ensure!(v["type"] == "text", "REPAIR_CELL_TYPE");
v["value"].as_str().context("REPAIR_CELL_TEXT")
}
fn integer(table: &Value, row: &Value, column: &str) -> Result<u64> {
let v = cell(table, row, column)?;
ensure!(v["type"] == "integer", "REPAIR_CELL_TYPE");
v["value"].as_u64().context("REPAIR_CELL_INTEGER")
}
fn blob(table: &Value, row: &Value, column: &str, bytes: &[u8]) -> Result<()> {
let v = cell(table, row, column)?;
ensure!(
v == &json!({"type":"blob","sha256":hash(bytes),"length":bytes.len()}),
"REPAIR_WITNESS_BLOB"
);
Ok(())
}
fn row_by<'a>(table: &'a Value, column: &str, id: &str) -> Result<&'a Value> {
let found = rows(table)?
.iter()
.filter(|row| text(table, row, column).ok() == Some(id))
.collect::<Vec<_>>();
ensure!(found.len() == 1, "REPAIR_WITNESS_ROW");
Ok(found[0])
}
fn decision(
signer: &crate::job_signer::JobSigner,
s: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
ensure!(
s.decision_bytes.len() <= 1024 * 1024 && s.signature.len() == 64,
"REPAIR_DECISION_SIZE"
);
let key = PKey::public_key_from_raw_bytes(&signer.public_key()?, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?.verify_oneshot(&s.signature, &s.decision_bytes)?,
"REPAIR_DECISION_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&s.decision_bytes)?;
ensure!(
d.valid_profile(d.issued_at) && d.key_id == signer.key_id,
"REPAIR_DECISION_PROFILE"
);
Ok(d)
}
struct Case {
old: ReconcileDecisionPayload,
rejected: ReconcileDecisionPayload,
unsent: SyncRequest,
request: SyncRequest,
}
fn validate(signer: &crate::job_signer::JobSigner, e: &Evidence) -> Result<Case> {
ensure!(
e.schema_version == 1
&& e.uncommitted_request.len() <= 1024 * 1024
&& e.rejected_request.len() <= 1024 * 1024,
"REPAIR_EVIDENCE_PROFILE"
);
let old = decision(signer, &e.previous_valid_response)?;
let rejected = decision(signer, &e.rejected_response)?;
for d in [&old, &rejected] {
ensure!(
d.tenant_id == e.tenant_id
&& d.asset_id == e.asset_id
&& d.agent_id == e.agent_id
&& d.enrollment_generation == e.enrollment_generation
&& d.assignment_version == e.assignment_version,
"REPAIR_IDENTITY_SCOPE"
);
}
let unsent = SyncRequest::decode(e.uncommitted_request.as_slice())?;
let request = SyncRequest::decode(e.rejected_request.as_slice())?;
let unsent_id = Uuid::parse_str(&unsent.request_id)?;
ensure!(
!unsent_id.is_nil()
&& unsent_id != rejected.request_id
&& unsent_id != old.request_id
&& old.decision_id != rejected.decision_id
&& rejected.request_id.to_string() == request.request_id
&& rejected.request_hash == hash(&e.rejected_request),
"REPAIR_REQUEST_BINDING"
);
ensure!(
rejected.reason == FAULT
&& !rejected.ready
&& !rejected.helper_ready
&& !rejected.attempts.iter().any(|a| a.unlock),
"REPAIR_DENIAL_ONLY"
);
ensure!(
request.previous_decision_id.is_empty()
&& request.previous_request_id == unsent.request_id
&& unsent.previous_decision_id == old.decision_id.to_string()
&& unsent.previous_request_id.is_empty(),
"REPAIR_EXACT_UNSENT_CHAIN"
);
let helper = rejected
.helper_journal_id
.context("REPAIR_HELPER_REQUIRED")?;
ensure!(
old.journal_id == rejected.journal_id
&& old.helper_journal_id == Some(helper)
&& old.checkpoint < unsent.checkpoint
&& unsent.checkpoint < rejected.checkpoint
&& old.helper_checkpoint <= unsent.helper_checkpoint
&& unsent.helper_checkpoint <= rejected.helper_checkpoint
&& old.connection_generation <= rejected.connection_generation,
"REPAIR_MONOTONIC_CHAIN"
);
for r in [&unsent, &request] {
ensure!(
r.journal_complete
&& r.assignment_version == e.assignment_version
&& r.journal_id == rejected.journal_id.to_string()
&& r.helper_journal_id == helper.to_string(),
"REPAIR_REQUEST_SCOPE"
);
}
ensure!(
request.checkpoint == rejected.checkpoint
&& request.helper_checkpoint == rejected.helper_checkpoint
&& request.helper_previous_decision_id == old.decision_id.to_string(),
"REPAIR_REQUEST_CHECKPOINT"
);
let am = &e.agent_inventory["reconcile_meta"];
let a = rows(am)?;
ensure!(a.len() == 1, "REPAIR_AGENT_META");
let a = &a[0];
ensure!(
integer(am, a, "id")? == 1
&& text(am, a, "journal_id")? == rejected.journal_id.to_string()
&& integer(am, a, "checkpoint")? == rejected.checkpoint
&& integer(am, a, "high_sequence")? >= rejected.checkpoint
&& integer(am, a, "generation")? == rejected.connection_generation
&& text(am, a, "last_decision")? == rejected.decision_id.to_string()
&& text(am, a, "helper_id")? == helper.to_string()
&& integer(am, a, "helper_checkpoint")? == rejected.helper_checkpoint
&& text(am, a, "fault")? == FAULT
&& cell(am, a, "pending_request")? == &json!({"type":"null"}),
"REPAIR_AGENT_HEAD"
);
let hm = &e.helper_inventory["helper_sync_meta"];
let h = rows(hm)?;
ensure!(h.len() == 1, "REPAIR_HELPER_META");
let h = &h[0];
ensure!(
integer(hm, h, "id")? == 1
&& text(hm, h, "journal_id")? == helper.to_string()
&& integer(hm, h, "checkpoint")? == rejected.helper_checkpoint
&& text(hm, h, "last_decision")? == rejected.decision_id.to_string()
&& text(hm, h, "agent_journal_id")? == rejected.journal_id.to_string()
&& integer(hm, h, "agent_checkpoint")? == rejected.checkpoint
&& text(hm, h, "fault")? == FAULT,
"REPAIR_HELPER_HEAD"
);
let ar = &e.agent_inventory["reconcile_requests"];
let x = row_by(ar, "id", &unsent.request_id)?;
blob(ar, x, "request", &e.uncommitted_request)?;
ensure!(
integer(ar, x, "generation")? <= rejected.connection_generation
&& cell(ar, x, "response")? == &json!({"type":"null"}),
"REPAIR_UNSENT_LOCAL_RESPONSE"
);
let y = row_by(ar, "id", &request.request_id)?;
blob(ar, y, "request", &e.rejected_request)?;
blob(
ar,
y,
"response",
&rmm_protocol::SyncResponse {
decision_bytes: e.rejected_response.decision_bytes.clone(),
signature: e.rejected_response.signature.clone(),
}
.encode_to_vec(),
)?;
ensure!(
integer(ar, y, "generation")? == rejected.connection_generation,
"REPAIR_LOCAL_GENERATION"
);
let hr = &e.helper_inventory["helper_sync_decisions"];
let y = row_by(hr, "decision_id", &rejected.decision_id.to_string())?;
blob(hr, y, "request", &e.rejected_request)?;
blob(
hr,
y,
"response",
&serde_json::to_vec(&e.rejected_response)?,
)?;
ensure!(
integer(hr, y, "checkpoint")? == rejected.helper_checkpoint,
"REPAIR_HELPER_CAPTURE"
);
ensure!(
e.helper_inventory["consumed"].is_object()
&& e.helper_inventory["execution_evidence"].is_object(),
"REPAIR_LEDGER_REQUIRED"
);
Ok(Case {
old,
rejected,
unsent,
request,
})
}
pub async fn issue(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
closed_policy()?;
issue_with_closed_policy(state, e).await
}
async fn issue_with_closed_policy(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
let c = validate(&state.job_signer, &e)?;
let evidence_hash = inventory_sha256(&serde_json::to_value(&e)?)?;
let mut tx = rmm_persistence::tenant_tx(&state.pool, e.tenant_id).await?;
crate::agent_service::control_lock(&mut tx, e.tenant_id, e.agent_id).await?;
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,revoked_at,sequence_regressed,sync_recovery_required,sync_ready,helper_sync_ready FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_DEVICE_MISSING")?;
ensure!(
device.get::<Uuid, _>("asset_id") == e.asset_id
&& device.get::<i64, _>("enrollment_generation")
== i64::try_from(e.enrollment_generation)?
&& device.get::<i64, _>("assignment_version") == i64::try_from(e.assignment_version)?
// Faulted journals may survive newer ClientHello channels. This
// historical receipt grants no lease; the next ordinary sync must
// still bind its current channel. A regressed generation or changed
// boot/identity is never repaired here.
&& device.get::<i64, _>("connection_generation")
>= i64::try_from(c.rejected.connection_generation)?
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(c.rejected.boot_id)
&& device
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
.is_none()
&& ![
"sequence_regressed",
"sync_recovery_required",
"sync_ready",
"helper_sync_ready"
]
.iter()
.any(|k| device.get::<bool, _>(*k)),
"REPAIR_DEVICE_FENCED"
);
for (d, s) in [
(&c.old, &e.previous_valid_response),
(&c.rejected, &e.rejected_response),
] {
let stored=sqlx::query("SELECT decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3")
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).fetch_optional(&mut *tx).await?.context("REPAIR_STORED_DECISION_MISSING")?;
ensure!(
stored.get::<Uuid, _>("decision_id") == d.decision_id
&& stored.get::<String, _>("request_hash") == d.request_hash
&& stored.get::<Uuid, _>("journal_id") == d.journal_id
&& stored.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
&& stored.get::<i64, _>("connection_generation")
== i64::try_from(d.connection_generation)?
&& stored.get::<Vec<u8>, _>("payload") == s.decision_bytes
&& stored.get::<Vec<u8>, _>("signature") == s.signature,
"REPAIR_STORED_DECISION_MISMATCH"
);
}
let committed:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3)")
.bind(e.tenant_id).bind(e.agent_id).bind(Uuid::parse_str(&c.unsent.request_id)?).fetch_one(&mut *tx).await?;
ensure!(!committed, "REPAIR_REQUEST_WAS_COMMITTED");
let head=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_HEAD_MISSING")?;
let previous:Option<Value>=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type=$2 AND data->>'evidence_sha256'=$3 AND data->>'agent_id'=$4 ORDER BY created_at DESC LIMIT 1")
.bind(e.tenant_id).bind(EVENT).bind(&evidence_hash).bind(e.agent_id.to_string()).fetch_optional(&mut *tx).await?;
if let Some(previous) = previous {
let signed: SignedReceipt = serde_json::from_value(previous["receipt"].clone())?;
let receipt: Receipt = serde_json::from_slice(&signed.payload)?;
let key = PKey::public_key_from_raw_bytes(&state.job_signer.public_key()?, Id::ED25519)?;
ensure!(
signed.signature.len() == 64
&& Verifier::new_without_digest(&key)?
.verify_oneshot(&signed.signature, &signed.payload)?
&& receipt.valid_profile(Utc::now())
&& receipt.key_id == state.job_signer.key_id
&& receipt.tenant_id == e.tenant_id
&& receipt.asset_id == e.asset_id
&& receipt.agent_id == e.agent_id
&& receipt.rejected_decision_id == c.rejected.decision_id
&& receipt.rejected_request_sha256 == hash(&e.rejected_request)
&& receipt.uncommitted_request_sha256 == hash(&e.uncommitted_request)
&& receipt.agent_inventory_sha256 == inventory_sha256(&e.agent_inventory)?
&& receipt.helper_inventory_sha256 == inventory_sha256(&e.helper_inventory)?
&& head_matches(&head, &c.rejected)?,
"REPAIR_REPLAY_EXPIRED_OR_HEAD_CHANGED"
);
tx.commit().await?;
return Ok(signed);
}
ensure!(
head_matches(&head, &c.old)?,
"REPAIR_AUTHORITATIVE_HEAD_CHANGED"
);
let now = Utc::now();
let receipt = Receipt {
domain: "rmm.sync.unsent-request-repair".into(),
schema_version: 1,
receipt_id: Uuid::new_v4(),
tenant_id: e.tenant_id,
asset_id: e.asset_id,
agent_id: e.agent_id,
enrollment_generation: e.enrollment_generation,
assignment_version: e.assignment_version,
issued_at: now,
valid_until: now + Duration::minutes(10),
key_id: state.job_signer.key_id.clone(),
fault: FAULT.into(),
agent_journal_id: c.rejected.journal_id,
agent_checkpoint: c.rejected.checkpoint,
helper_journal_id: c
.rejected
.helper_journal_id
.context("REPAIR_HELPER_REQUIRED")?,
helper_checkpoint: c.rejected.helper_checkpoint,
rejected_decision_id: c.rejected.decision_id,
rejected_decision_sha256: hash(&e.rejected_response.decision_bytes),
rejected_request_id: c.rejected.request_id,
rejected_request_sha256: hash(&e.rejected_request),
uncommitted_request_id: Uuid::parse_str(&c.unsent.request_id)?,
uncommitted_request_sha256: hash(&e.uncommitted_request),
previous_valid_decision_id: c.old.decision_id,
previous_valid_decision_sha256: hash(&e.previous_valid_response.decision_bytes),
agent_inventory_sha256: inventory_sha256(&e.agent_inventory)?,
helper_inventory_sha256: inventory_sha256(&e.helper_inventory)?,
helper_ledger_sha256: helper_ledger_sha256(&e.helper_inventory)?,
ready: false,
helper_ready: false,
unlock: false,
};
ensure!(receipt.valid_profile(now), "REPAIR_RECEIPT_PROFILE");
let payload = serde_json::to_vec(&receipt)?;
let signed = SignedReceipt {
signature: state.job_signer.sign(&payload)?,
payload,
};
ensure!(sqlx::query("UPDATE agent_sync_state SET checkpoint=$3,decision_id=$4,helper_journal_id=$5,helper_checkpoint=$6 WHERE tenant_id=$1 AND agent_id=$2 AND decision_id=$7")
.bind(e.tenant_id).bind(e.agent_id).bind(i64::try_from(c.request.checkpoint)?).bind(c.rejected.decision_id)
.bind(c.rejected.helper_journal_id).bind(i64::try_from(c.rejected.helper_checkpoint)?).bind(c.old.decision_id)
.execute(&mut *tx).await?.rows_affected()==1,"REPAIR_HEAD_CONCURRENT_CHANGE");
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,$4,$5)")
.bind(e.tenant_id).bind(receipt.receipt_id).bind("development-bootstrap:unsent-sync-repair").bind(EVENT)
.bind(json!({"agent_id":e.agent_id,"evidence_sha256":evidence_hash,"receipt_sha256":hash(&signed.payload),"receipt":signed,"ready":false,"helper_ready":false,"unlock":false}))
.execute(&mut *tx).await?;
tx.commit().await?;
Ok(signed)
}
fn head_matches(head: &sqlx::postgres::PgRow, d: &ReconcileDecisionPayload) -> Result<bool> {
Ok(head.get::<Uuid, _>("journal_id") == d.journal_id
&& head.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
&& head.get::<Uuid, _>("decision_id") == d.decision_id
&& head.get::<Option<Uuid>, _>("helper_journal_id") == d.helper_journal_id
&& head.get::<i64, _>("helper_checkpoint") == i64::try_from(d.helper_checkpoint)?)
}
#[cfg(test)]
mod tests {
use super::*;
fn table(fields: Vec<(&str, Value)>) -> Value {
json!({"columns":fields.iter().map(|(k,_)|k).collect::<Vec<_>>(),"rows":[fields.iter().map(|(_,v)|v).collect::<Vec<_>>() ]})
}
fn t(v: impl ToString) -> Value {
json!({"type":"text","value":v.to_string()})
}
fn n(v: u64) -> Value {
json!({"type":"integer","value":v})
}
fn b(v: &[u8]) -> Value {
json!({"type":"blob","sha256":hash(v),"length":v.len()})
}
fn sign(s: &crate::job_signer::JobSigner, d: &ReconcileDecisionPayload) -> SignedDecision {
let decision_bytes = serde_json::to_vec(d).unwrap();
SignedDecision {
signature: s.sign(&decision_bytes).unwrap(),
decision_bytes,
}
}
fn fixture() -> (crate::job_signer::JobSigner, Evidence, std::path::PathBuf) {
let path = std::path::PathBuf::from(format!("/tmp/rmm-repair-{}", Uuid::new_v4()));
let signer = crate::job_signer::JobSigner::load_or_create(&path.join("key.pem")).unwrap();
let tenant = Uuid::new_v4();
let asset = Uuid::new_v4();
let agent = Uuid::new_v4();
let journal = Uuid::new_v4();
let helper = Uuid::new_v4();
let now = Utc::now();
let old = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::new_v4(),
request_hash: "ab".repeat(32),
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
connection_generation: 3,
boot_id: Uuid::new_v4(),
journal_id: journal,
checkpoint: 10,
helper_journal_id: Some(helper),
helper_checkpoint: 5,
issued_at: now,
valid_until: now + Duration::seconds(60),
ready: false,
helper_ready: false,
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
attempts: vec![],
acknowledged_message_ids: vec![],
key_id: signer.key_id.clone(),
};
let x = SyncRequest {
request_id: Uuid::new_v4().to_string(),
journal_id: journal.to_string(),
checkpoint: 11,
previous_decision_id: old.decision_id.to_string(),
assignment_version: 1,
journal_complete: true,
helper_journal_id: helper.to_string(),
helper_checkpoint: 6,
helper_previous_decision_id: old.decision_id.to_string(),
..Default::default()
};
let y = SyncRequest {
request_id: Uuid::new_v4().to_string(),
checkpoint: 12,
previous_decision_id: String::new(),
previous_request_id: x.request_id.clone(),
helper_checkpoint: 7,
..x.clone()
};
let mut rejected = old.clone();
rejected.decision_id = Uuid::new_v4();
rejected.request_id = Uuid::parse_str(&y.request_id).unwrap();
rejected.request_hash = hash(&y.encode_to_vec());
rejected.checkpoint = 12;
rejected.helper_checkpoint = 7;
rejected.reason = FAULT.into();
let response = sign(&signer, &rejected);
let request = y.encode_to_vec();
let unsent = x.encode_to_vec();
let proto = rmm_protocol::SyncResponse {
decision_bytes: response.decision_bytes.clone(),
signature: response.signature.clone(),
}
.encode_to_vec();
let mut requests = table(vec![
("id", t(&x.request_id)),
("request", b(&unsent)),
("generation", n(3)),
("response", json!({"type":"null"})),
]);
requests["rows"].as_array_mut().unwrap().push(json!([
t(&y.request_id),
b(&request),
n(3),
b(&proto)
]));
let am = table(vec![
("id", n(1)),
("journal_id", t(journal)),
("checkpoint", n(12)),
("high_sequence", n(13)),
("generation", n(3)),
("last_decision", t(rejected.decision_id)),
("pending_request", json!({"type":"null"})),
("helper_id", t(helper)),
("helper_checkpoint", n(7)),
("fault", t(FAULT)),
]);
let hm = table(vec![
("id", n(1)),
("journal_id", t(helper)),
("checkpoint", n(7)),
("last_decision", t(rejected.decision_id)),
("agent_journal_id", t(journal)),
("agent_checkpoint", n(12)),
("fault", t(FAULT)),
]);
let history = table(vec![
("decision_id", t(rejected.decision_id)),
("checkpoint", n(7)),
("response", b(&serde_json::to_vec(&response).unwrap())),
("request", b(&request)),
]);
let e = Evidence {
schema_version: 1,
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
agent_inventory: json!({"reconcile_meta":am,"reconcile_requests":requests}),
helper_inventory: json!({"helper_sync_meta":hm,"helper_sync_decisions":history,"consumed":{"columns":[],"rows":[]},"execution_evidence":{"columns":[],"rows":[]}}),
uncommitted_request: unsent,
rejected_request: request,
rejected_response: response,
previous_valid_response: sign(&signer, &old),
};
(signer, e, path)
}
#[test]
fn only_exact_signed_unsent_case_validates() {
let (s, e, p) = fixture();
validate(&s, &e).unwrap();
let mut other = e.clone();
other.tenant_id = Uuid::new_v4();
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.asset_id = Uuid::new_v4();
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.assignment_version += 1;
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.rejected_response.signature[0] ^= 1;
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.agent_inventory["reconcile_requests"]["rows"][0][3] = b(b"committed response");
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.helper_inventory["helper_sync_meta"]["rows"][0][2] = n(8);
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.agent_inventory["reconcile_meta"]["rows"][0][9] = t("OTHER_FAULT");
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
let mut x = SyncRequest::decode(other.uncommitted_request.as_slice()).unwrap();
x.previous_decision_id = Uuid::new_v4().to_string();
other.uncommitted_request = x.encode_to_vec();
assert!(validate(&s, &other).is_err());
fs::remove_dir_all(p).unwrap();
}
#[test]
fn signed_ready_response_is_never_repair_authority() {
let (s, mut e, p) = fixture();
let mut d: ReconcileDecisionPayload =
serde_json::from_slice(&e.rejected_response.decision_bytes).unwrap();
d.ready = true;
e.rejected_response = sign(&s, &d);
assert!(validate(&s, &e).is_err());
d.ready = false;
d.reason = "CONSISTENT".into();
e.rejected_response = sign(&s, &d);
assert!(validate(&s, &e).is_err());
fs::remove_dir_all(p).unwrap();
}
/// Opt-in dedicated PostgreSQL only; the harness creates/destroys its own
/// network-isolated container and never reads deployment credentials.
#[tokio::test]
#[ignore = "requires the dedicated isolated PostgreSQL harness"]
async fn isolated_database_repair_and_fail_closed_paths() {
let admin_url = std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap();
let app_url = std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap();
let admin = sqlx::PgPool::connect(&admin_url).await.unwrap();
let pool = sqlx::PgPool::connect(&app_url).await.unwrap();
rmm_persistence::verify_application_role(&pool)
.await
.unwrap();
let (s, e, p) = fixture();
let c = validate(&s, &e).unwrap();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated repair test')")
.bind(e.tenant_id)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
.bind(e.tenant_id)
.bind(e.asset_id)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,assignment_version,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5,$6)")
.bind(e.tenant_id).bind(e.agent_id).bind(e.asset_id).bind(hash(e.agent_id.as_bytes())).bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(c.rejected.boot_id).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO agent_sync_state VALUES($1,$2,$3,$4,$5,$6,$7)")
.bind(e.tenant_id)
.bind(e.agent_id)
.bind(c.old.journal_id)
.bind(i64::try_from(c.old.checkpoint).unwrap())
.bind(c.old.decision_id)
.bind(c.old.helper_journal_id)
.bind(i64::try_from(c.old.helper_checkpoint).unwrap())
.execute(&admin)
.await
.unwrap();
for (d, signed) in [
(&c.old, &e.previous_valid_response),
(&c.rejected, &e.rejected_response),
] {
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).bind(d.decision_id).bind(&d.request_hash).bind(d.journal_id).bind(i64::try_from(d.checkpoint).unwrap()).bind(i64::try_from(d.connection_generation).unwrap()).bind(&signed.decision_bytes).bind(&signed.signature).execute(&admin).await.unwrap();
}
let state = crate::State {
pool,
signer: None,
viewer_a: [0; 32],
viewer_b: [1; 32],
auth: std::sync::Arc::new(crate::auth::Auth::from_env("development").await.unwrap()),
job_signer: std::sync::Arc::new(s),
};
let mut wrong = e.clone();
wrong.tenant_id = Uuid::new_v4();
assert!(issue_with_closed_policy(&state, wrong).await.is_err());
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
.bind(e.tenant_id)
.bind(c.rejected.request_id)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
.bind(e.tenant_id).bind(e.agent_id).bind(c.rejected.request_id).bind(c.rejected.decision_id)
.bind(&c.rejected.request_hash).bind(c.rejected.journal_id).bind(i64::try_from(c.rejected.checkpoint).unwrap())
.bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(&e.rejected_response.decision_bytes)
.bind(&e.rejected_response.signature).execute(&admin).await.unwrap();
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint+1 WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint-1 WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
sqlx::query("UPDATE agent_installations SET revoked_at=now() WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_installations SET revoked_at=NULL WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
let x = Uuid::parse_str(&c.unsent.request_id).unwrap();
sqlx::query("INSERT INTO sync_checkpoints SELECT tenant_id,agent_id,$2,$3,request_hash,journal_id,checkpoint,connection_generation,payload,signature,created_at FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$4")
.bind(e.tenant_id).bind(x).bind(Uuid::new_v4()).bind(c.old.request_id).execute(&admin).await.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
.bind(e.tenant_id)
.bind(x)
.execute(&admin)
.await
.unwrap();
sqlx::query("UPDATE agent_installations SET connection_generation=$2 WHERE agent_id=$1")
.bind(e.agent_id)
.bind(i64::try_from(c.rejected.connection_generation).unwrap() - 1)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_installations SET connection_generation=$2,connection_boot_id=$3 WHERE agent_id=$1")
.bind(e.agent_id).bind(i64::try_from(c.rejected.connection_generation).unwrap()+1).bind(Uuid::new_v4())
.execute(&admin).await.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_installations SET connection_boot_id=$2 WHERE agent_id=$1")
.bind(e.agent_id)
.bind(c.rejected.boot_id)
.execute(&admin)
.await
.unwrap();
let first = issue_with_closed_policy(&state, e.clone()).await.unwrap();
assert_eq!(
first,
issue_with_closed_policy(&state, e.clone()).await.unwrap()
);
let r: Receipt = serde_json::from_slice(&first.payload).unwrap();
assert!(r.valid_profile(Utc::now()));
assert!(!r.ready && !r.helper_ready && !r.unlock);
let head: Uuid =
sqlx::query_scalar("SELECT decision_id FROM agent_sync_state WHERE agent_id=$1")
.bind(e.agent_id)
.fetch_one(&admin)
.await
.unwrap();
assert_eq!(head, c.rejected.decision_id);
let count: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
.bind(e.tenant_id)
.fetch_one(&admin)
.await
.unwrap();
assert_eq!(count, 1);
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM outbox)+(SELECT count(*) FROM execution_permits)").fetch_one(&admin).await.unwrap();
assert_eq!(effects, 0);
let ready: bool = sqlx::query_scalar(
"SELECT sync_ready OR helper_sync_ready FROM agent_installations WHERE agent_id=$1",
)
.bind(e.agent_id)
.fetch_one(&admin)
.await
.unwrap();
assert!(!ready);
fs::remove_dir_all(p).unwrap();
}
}
+570
View File
@@ -0,0 +1,570 @@
//! Narrow dev/test wire faults after normal authorization/durable ingest.
//! Never creates jobs, grants or approvals. Claims and observations are bounded,
//! insert-only audit slots; a restart cannot arm the same fault a second time.
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Duration, Utc};
use prost::Message;
use rmm_domain::jobs::{Action, JobPayload, hash};
use rmm_protocol::{Envelope, Payload};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sqlx::{PgPool, Postgres, Row, Transaction};
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
use uuid::Uuid;
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum Mode {
DuplicateOfferOnce,
DropResultAckOnce,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Config {
schema_version: u32,
fault_id: Uuid,
tenant_id: Uuid,
agent_id: Uuid,
job_id: Uuid,
attempt_id: Uuid,
plan_sha256: String,
job_payload_sha256: String,
mode: Mode,
transport: String,
issued_at: DateTime<Utc>,
expires_at: DateTime<Utc>,
}
impl Config {
fn validate(&self) -> Result<()> {
ensure!(
self.schema_version == 1
&& [
self.fault_id,
self.tenant_id,
self.agent_id,
self.job_id,
self.attempt_id
]
.iter()
.all(|id| !id.is_nil())
&& valid_hash(&self.plan_sha256)
&& valid_hash(&self.job_payload_sha256)
&& matches!(self.transport.as_str(), "wss" | "https")
&& self.issued_at.timestamp() > 0
&& self.expires_at > self.issued_at
&& self.expires_at - self.issued_at <= Duration::minutes(15),
"DEV_DELIVERY_FAULT_CONFIG"
);
Ok(())
}
fn scope(&self, tenant: Uuid, agent: Uuid, transport: &str) -> bool {
self.tenant_id == tenant && self.agent_id == agent && self.transport == transport
}
fn job(&self, bytes: &[u8], payload_hash: &str, plan_hash: &str) -> Result<bool> {
if payload_hash != self.job_payload_sha256 || plan_hash != self.plan_sha256 {
return Ok(false);
}
ensure!(
hash(bytes) == self.job_payload_sha256,
"DEV_FAULT_PAYLOAD_HASH"
);
let job: JobPayload = serde_json::from_slice(bytes)?;
Ok(job.schema_version == 1
&& job.tenant_id == self.tenant_id
&& job.agent_id == self.agent_id
&& job.job_id == self.job_id
&& job.attempt_id == self.attempt_id
&& job.execution_identity == "agent_service"
&& job.enrollment_generation > 0
&& job.assignment_version > 0
&& job.max_runtime_seconds > 0
&& job.max_runtime_seconds <= 60
&& job.max_output_bytes <= 65536
&& job.action.validate()
&& matches!(job.action, Action::RunScript { .. })
&& self.expires_at <= job.start_before)
}
}
fn valid_hash(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
&& value.bytes().any(|b| b != b'0')
}
fn configured_path(
flag: Option<&str>,
path: Option<&str>,
profile: &str,
dev: &str,
) -> Result<Option<PathBuf>> {
match (flag, path) {
(None | Some("false"), None) => Ok(None),
(Some("true"), Some(path)) => {
ensure!(
profile == "development" && dev == "true",
"DEV_DELIVERY_FAULT_PROFILE"
);
ensure!(
path == "/root-policy/delivery-fault.json",
"DEV_DELIVERY_FAULT_PATH"
);
Ok(Some(PathBuf::from(path)))
}
_ => anyhow::bail!("DEV_DELIVERY_FAULT_EXPLICIT_OPT_IN"),
}
}
#[derive(Clone)]
struct Loaded {
config: Config,
config_sha256: String,
}
fn load() -> Result<Option<Loaded>> {
let flag = std::env::var("RMM_DEV_DELIVERY_FAULTS").ok();
let path = std::env::var("RMM_DEV_DELIVERY_FAULT_FILE").ok();
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
return Ok(None);
};
let parent = path.parent().context("DEV_FAULT_PARENT")?;
let directory = fs::symlink_metadata(parent)?;
let metadata = fs::symlink_metadata(&path)?;
ensure!(
directory.is_dir()
&& directory.uid() == 10001
&& directory.mode() & 0o777 == 0o700
&& metadata.is_file()
&& metadata.uid() == 10001
&& metadata.mode() & 0o777 == 0o600
&& metadata.nlink() == 1
&& metadata.len() <= 8192,
"DEV_FAULT_PRIVATE_CONFIG"
);
// /root-policy is the existing operator-provisioned read-only bind mount.
let bytes = fs::read(&path)?;
ensure!(bytes.len() <= 8192, "DEV_FAULT_CONFIG_SIZE");
let config: Config = serde_json::from_slice(&bytes)?;
config.validate()?;
if Utc::now() < config.issued_at || Utc::now() >= config.expires_at {
return Ok(None);
}
Ok(Some(Loaded {
config,
config_sha256: hash(&bytes),
}))
}
pub(crate) fn validate_environment() -> Result<()> {
load().map(|_| ())
}
fn slot_id(fault: Uuid, slot: &str) -> Uuid {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(format!("rmm.dev-delivery-fault.v1:{fault}:{slot}"));
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
}
fn envelope_hash(envelope: &Envelope) -> String {
let mut normalized = envelope.clone();
normalized.connection_generation = 0;
hash(&normalized.encode_to_vec())
}
fn data(loaded: &Loaded, envelope: &Envelope, detail: &str) -> Value {
let config = &loaded.config;
json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
"fault_id":config.fault_id,"mode":config.mode,"transport":config.transport,
"agent_id":config.agent_id,"job_id":config.job_id,"attempt_id":config.attempt_id,
"plan_sha256":config.plan_sha256,"job_payload_sha256":config.job_payload_sha256,
"config_sha256":loaded.config_sha256,"message_id":envelope.message_id,
"normalized_envelope_sha256":envelope_hash(envelope),"detail":detail})
}
async fn transaction<'a>(pool: &'a PgPool, loaded: &Loaded) -> Result<Transaction<'a, Postgres>> {
let mut tx = rmm_persistence::tenant_tx(pool, loaded.config.tenant_id).await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("dev-delivery-fault:{}", loaded.config.fault_id))
.execute(&mut *tx)
.await?;
Ok(tx)
}
async fn get_slot(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
slot: &str,
) -> Result<Option<Value>> {
let value: Option<Value> = sqlx::query_scalar("SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_delivery_fault_adapter'")
.bind(slot_id(loaded.config.fault_id, slot)).fetch_optional(&mut **tx).await?;
if let Some(value) = &value {
ensure!(
value.get("config_sha256") == Some(&json!(loaded.config_sha256)),
"DEV_FAULT_ID_REUSED_WITH_CHANGED_CONFIG"
);
}
Ok(value)
}
async fn insert_slot(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
slot: &str,
value: Value,
) -> Result<bool> {
if get_slot(tx, loaded, slot).await?.is_some() {
return Ok(false);
}
let rows = sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_delivery_fault_adapter',$3,$4) ON CONFLICT(tenant_id,event_id) DO NOTHING")
.bind(loaded.config.tenant_id).bind(slot_id(loaded.config.fault_id, slot))
.bind(format!("dev_delivery_fault.{slot}")).bind(value).execute(&mut **tx).await?.rows_affected();
Ok(rows == 1)
}
async fn authorized_job(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
message: Option<&Envelope>,
) -> Result<bool> {
let c = &loaded.config;
let row = sqlx::query("SELECT j.payload,j.payload_hash,p.plan_hash FROM jobs j JOIN operations o USING(tenant_id,operation_id) JOIN approval_requests r USING(tenant_id,operation_id) JOIN operation_plans p USING(tenant_id,plan_id) JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 AND r.approval_state='approved' AND a.testsimulation AND a.approval_origin='dev_test_browser_simulation' AND a.principal='simulated-testprincipal-a'")
.bind(c.job_id).bind(c.attempt_id).bind(c.agent_id).fetch_optional(&mut **tx).await?;
let Some(row) = row else {
return Ok(false);
};
if !c.job(
&row.get::<Vec<u8>, _>("payload"),
&row.get::<String, _>("payload_hash"),
&row.get::<String, _>("plan_hash"),
)? {
return Ok(false);
}
if let Some(message) = message {
let receipt: Option<String> = sqlx::query_scalar(
"SELECT payload_hash FROM job_receipts WHERE message_id=$1 AND agent_id=$2",
)
.bind(Uuid::parse_str(&message.message_id)?)
.bind(c.agent_id)
.fetch_optional(&mut **tx)
.await?;
ensure!(
receipt.as_deref() == Some(envelope_hash(message).as_str()),
"DEV_FAULT_DURABLE_RECEIPT_REQUIRED"
);
}
Ok(true)
}
pub(crate) struct DuplicateClaim {
loaded: Loaded,
envelope: Envelope,
}
/// Called only for work returned by normal postapproval `job_gateway::next`.
pub(crate) async fn duplicate_offer(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
envelope: &Envelope,
transport: &str,
) -> Result<Option<DuplicateClaim>> {
let Some(Payload::JobOffer(offer)) = &envelope.payload else {
return Ok(None);
};
let Some(loaded) = load()? else {
return Ok(None);
};
let c = &loaded.config;
if c.mode != Mode::DuplicateOfferOnce
|| !c.scope(tenant, agent, transport)
|| offer.job_id != c.job_id.to_string()
|| offer.attempt_id != c.attempt_id.to_string()
|| !c.job(
&offer.payload_bytes,
&offer.job_payload_hash,
&offer.plan_hash,
)?
{
return Ok(None);
}
let mut tx = transaction(pool, &loaded).await?;
let claimed = authorized_job(&mut tx, &loaded, None).await?
&& insert_slot(
&mut tx,
&loaded,
"claim",
data(&loaded, envelope, "duplicate_offer_reserved_before_send"),
)
.await?;
tx.commit().await?;
Ok(claimed.then(|| DuplicateClaim {
loaded,
envelope: envelope.clone(),
}))
}
/// WSS calls after both sends complete. HTTP labels construction honestly; the
/// two later distinct postcommit JobAccepted slots prove client observation.
pub(crate) async fn duplicate_emitted(pool: &PgPool, claim: DuplicateClaim) -> Result<()> {
let mut tx = transaction(pool, &claim.loaded).await?;
ensure!(
get_slot(&mut tx, &claim.loaded, "claim").await?.is_some(),
"DEV_FAULT_CLAIM_REQUIRED"
);
let detail = if claim.loaded.config.transport == "wss" {
"same_encoded_offer_sent_twice"
} else {
"same_encoded_offer_in_http_response_twice"
};
insert_slot(
&mut tx,
&claim.loaded,
"emitted",
data(&claim.loaded, &claim.envelope, detail),
)
.await?;
tx.commit().await?;
Ok(())
}
/// Called AFTER durable ingest returned its concrete ACK. Return true only to
/// omit that single matching result ACK; every subsequent replay is ACKed.
pub(crate) async fn after_ingest(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
incoming: &Envelope,
reply: &Envelope,
transport: &str,
) -> Result<bool> {
let (job, attempt, is_result) = match &incoming.payload {
Some(Payload::JobAccepted(value)) => (&value.job_id, &value.attempt_id, false),
Some(Payload::JobResult(value)) if value.state == "succeeded" => {
(&value.job_id, &value.attempt_id, true)
}
_ => return Ok(false),
};
let Some(loaded) = load()? else {
return Ok(false);
};
let c = &loaded.config;
if !c.scope(tenant, agent, transport)
|| job != &c.job_id.to_string()
|| attempt != &c.attempt_id.to_string()
{
return Ok(false);
}
let Some(Payload::Ack(ack)) = &reply.payload else {
return Ok(false);
};
ensure!(
ack.message_id == incoming.message_id,
"DEV_FAULT_CONCRETE_ACK_REQUIRED"
);
let mut tx = transaction(pool, &loaded).await?;
if !authorized_job(&mut tx, &loaded, Some(incoming)).await? {
tx.commit().await?;
return Ok(false);
}
let claim = get_slot(&mut tx, &loaded, "claim").await?;
let mut omit = false;
if is_result && c.mode == Mode::DropResultAckOnce {
if let Some(first) = claim {
if first.get("message_id") == Some(&json!(incoming.message_id))
&& first.get("normalized_envelope_sha256") == Some(&json!(envelope_hash(incoming)))
{
insert_slot(
&mut tx,
&loaded,
"replay",
data(
&loaded,
incoming,
"same_original_result_replayed_after_ack_omission",
),
)
.await?;
}
} else {
omit = insert_slot(
&mut tx,
&loaded,
"claim",
data(
&loaded,
incoming,
"one_concrete_result_ack_omitted_after_durable_commit",
),
)
.await?;
}
} else if !is_result && c.mode == Mode::DuplicateOfferOnce && claim.is_some() {
if let Some(first) = get_slot(&mut tx, &loaded, "accepted_1").await? {
if first.get("message_id") != Some(&json!(incoming.message_id)) {
insert_slot(
&mut tx,
&loaded,
"accepted_2",
data(&loaded, incoming, "second_distinct_client_acceptance"),
)
.await?;
}
} else {
insert_slot(
&mut tx,
&loaded,
"accepted_1",
data(&loaded, incoming, "first_client_acceptance"),
)
.await?;
}
}
tx.commit().await?;
Ok(omit)
}
#[cfg(test)]
mod tests {
use super::*;
fn config() -> Config {
let now = Utc::now();
Config {
schema_version: 1,
fault_id: Uuid::new_v4(),
tenant_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
plan_sha256: "a".repeat(64),
job_payload_sha256: "b".repeat(64),
mode: Mode::DuplicateOfferOnce,
transport: "wss".into(),
issued_at: now,
expires_at: now + Duration::minutes(10),
}
}
#[test]
fn opt_in_is_explicit_and_never_production() {
assert!(
configured_path(None, None, "production", "false")
.unwrap()
.is_none()
);
assert!(
configured_path(
Some("true"),
Some("/root-policy/delivery-fault.json"),
"production",
"false"
)
.is_err()
);
assert!(configured_path(Some("true"), None, "development", "true").is_err());
assert!(
configured_path(
None,
Some("/root-policy/delivery-fault.json"),
"development",
"true"
)
.is_err()
);
assert!(configured_path(Some("true"), Some("/etc/passwd"), "development", "true").is_err());
assert!(
configured_path(
Some("true"),
Some("/root-policy/delivery-fault.json"),
"development",
"true"
)
.unwrap()
.is_some()
);
}
#[test]
fn narrow_schema_rejects_wildcards_nil_placeholders_and_long_lifetime() {
let c = config();
c.validate().unwrap();
let mut altered = c.clone();
altered.expires_at = altered.issued_at + Duration::minutes(16);
assert!(altered.validate().is_err());
altered = c.clone();
altered.agent_id = Uuid::nil();
assert!(altered.validate().is_err());
altered = c.clone();
altered.job_payload_sha256 = "0".repeat(64);
assert!(altered.validate().is_err());
altered = c.clone();
altered.transport = "*".into();
assert!(altered.validate().is_err());
let mut raw = serde_json::to_value(c).unwrap();
raw["approve"] = json!(true);
assert!(serde_json::from_value::<Config>(raw).is_err());
}
#[test]
fn audit_slots_are_stable_bounded_and_distinct() {
let id = Uuid::new_v4();
assert_eq!(slot_id(id, "claim"), slot_id(id, "claim"));
assert_ne!(slot_id(id, "claim"), slot_id(id, "replay"));
assert_ne!(slot_id(id, "claim"), slot_id(Uuid::new_v4(), "claim"));
}
#[test]
fn exact_signed_job_scope_cannot_fault_root_other_targets_or_other_actions() {
let mut c = config();
let mut job = JobPayload {
schema_version: 1,
tenant_id: c.tenant_id,
asset_id: Uuid::new_v4(),
agent_id: c.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: c.job_id,
attempt_id: c.attempt_id,
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf hello\\n".into(),
arguments: vec![],
expected_stdout: None,
},
execution_identity: "agent_service".into(),
not_before: c.issued_at,
start_before: c.expires_at,
max_runtime_seconds: 60,
max_output_bytes: 4096,
};
let mut bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
assert!(!c.scope(c.tenant_id, Uuid::new_v4(), "wss"));
assert!(!c.scope(c.tenant_id, c.agent_id, "https"));
assert!(
!c.job(&bytes, &c.job_payload_sha256, &"c".repeat(64))
.unwrap()
);
job.execution_identity = "root".into();
bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
job.execution_identity = "agent_service".into();
job.action = Action::RefreshInventory {
scope: "linux.base".into(),
};
bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
}
#[test]
fn durable_replay_only_normalizes_connection_generation() {
let e = Envelope {
message_id: Uuid::new_v4().to_string(),
connection_generation: 3,
..Default::default()
};
let mut repeated = e.clone();
repeated.connection_generation = 4;
assert_eq!(envelope_hash(&e), envelope_hash(&repeated));
repeated.message_id = Uuid::new_v4().to_string();
assert_ne!(envelope_hash(&e), envelope_hash(&repeated));
}
}
+27 -7
View File
@@ -116,7 +116,10 @@ async fn receive(socket: &mut WebSocket) -> Result<Envelope> {
Ok(Envelope::decode(bytes)?)
}
async fn send(
state: &crate::State,
socket: &mut WebSocket,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
payload: Payload,
generation: u64,
@@ -133,11 +136,22 @@ async fn send(
sequence: e.sequence,
payload: Some(payload),
};
let claim =
dev_delivery_faults::duplicate_offer(&state.pool, tenant, agent, &reply, "wss").await?;
let bytes = reply.encode_to_vec();
tokio::time::timeout(
Duration::from_secs(10),
socket.send(Message::Binary(reply.encode_to_vec().into())),
socket.send(Message::Binary(bytes.clone().into())),
)
.await??;
if let Some(claim) = claim {
tokio::time::timeout(
Duration::from_secs(10),
socket.send(Message::Binary(bytes.into())),
)
.await??;
dev_delivery_faults::duplicate_emitted(&state.pool, claim).await?;
}
Ok(())
}
async fn channel(
@@ -162,7 +176,7 @@ async fn channel(
.send(Message::Binary(response.encode_to_vec().into()))
.await?;
if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? {
send(socket, &hello, payload, generation).await?;
send(state, socket, tenant, agent, &hello, payload, generation).await?;
}
let mut recent_messages = VecDeque::new();
let mut timer = tokio::time::interval(Duration::from_secs(1));
@@ -177,7 +191,7 @@ async fn channel(
let mut tx=rmm_persistence::tenant_tx(&state.pool,tenant).await?;
let revoked:bool=sqlx::query_scalar("SELECT revoked_at IS NOT NULL FROM agent_installations WHERE agent_id=$1").bind(agent).fetch_one(&mut *tx).await?;
tx.commit().await?;ensure!(!revoked,"device revoked");
if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(socket,&hello,payload,generation).await?;}
if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(state,socket,tenant,agent,&hello,payload,generation).await?;}
continue;
}
};
@@ -198,11 +212,17 @@ async fn channel(
"channel generation mismatch"
);
let response = crate::agent_service::ingest(state, tenant, agent, &e, "wss").await?;
socket
.send(Message::Binary(response.encode_to_vec().into()))
.await?;
if !dev_delivery_faults::after_ingest(&state.pool, tenant, agent, &e, &response, "wss")
.await?
{
socket
.send(Message::Binary(response.encode_to_vec().into()))
.await?;
}
if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? {
send(socket, &e, payload, generation).await?;
send(state, socket, tenant, agent, &e, payload, generation).await?;
}
}
}
#[path = "dev_delivery_faults.rs"]
pub(crate) mod dev_delivery_faults;
+24
View File
@@ -5,6 +5,7 @@ mod api;
mod artifact_store;
mod auth;
mod catalog;
mod checkpoint_repair;
mod controls;
mod errors;
mod gateway;
@@ -68,6 +69,16 @@ async fn main() -> Result<()> {
.await;
}
let profile = env::var("RMM_PROFILE").unwrap_or_default();
gateway::dev_delivery_faults::validate_environment()?;
let repair = env::args().nth(1).as_deref() == Some("repair-unsent-sync-request");
if repair {
ensure!(env::args().count() == 2, "REPAIR_NO_ARGUMENTS");
checkpoint_repair::closed_policy()?;
ensure!(
std::path::Path::new(&env::var("RMM_JOB_KEY_FILE")?).is_file(),
"REPAIR_EXISTING_KEY_REQUIRED"
);
}
let dev = env::var("RMM_DEV_AUTH").as_deref() == Ok("true");
ensure!(
matches!(
@@ -126,6 +137,19 @@ async fn main() -> Result<()> {
.auth
.validate_separation(&state.viewer_a, &state.viewer_b)?;
}
if repair {
use std::io::{Read, Write};
let mut raw = Vec::new();
std::io::stdin()
.take(32 * 1024 * 1024 + 1)
.read_to_end(&mut raw)?;
ensure!(raw.len() <= 32 * 1024 * 1024, "REPAIR_INPUT_SIZE");
let evidence = serde_json::from_slice(&raw).context("REPAIR_EVIDENCE_JSON")?;
let receipt = checkpoint_repair::issue(&state, evidence).await?;
serde_json::to_writer(std::io::stdout().lock(), &receipt)?;
std::io::stdout().write_all(b"\n")?;
return Ok(());
}
if role.gateway() {
scheduler::start(state.clone());
}
+13 -2
View File
@@ -25,6 +25,7 @@ def upgrade(payload):
base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote')
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper')
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json'
agent_binary=Path('/usr/local/libexec/rmm-agent-remote')
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
@@ -52,6 +53,7 @@ def upgrade(payload):
guard()
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved')
require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required')
require(sha(agent_binary.read_bytes())==payload['old_agent_sha256'],'reviewed permanent agent required')
require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required')
original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json'
retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json'
@@ -59,7 +61,9 @@ def upgrade(payload):
sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required')
require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected')
raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True)
agent_raw=base64.b64decode(payload['agent_binary'],validate=True)
require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts')
require(sha(agent_raw)==payload['agent_sha256'],'qualified sync-prevention agent')
backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700)
def create(path,data,mode=0o600):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
@@ -68,7 +72,9 @@ def upgrade(payload):
def save():
next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json')
save();staged=backup/'helper-next';create(staged,raw,0o755)
staged_agent=backup/'agent-next';create(staged_agent,agent_raw,0o755)
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged_agent)])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
try:
@@ -77,11 +83,11 @@ def upgrade(payload):
witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))}
old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata')
general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
for p in (binary,provider,metadata,dropin):
for p in (binary,agent_binary,provider,metadata,dropin):
target=backup/str(p).lstrip('/').replace('/','__')
require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup')
os.rename(p,target)
os.rename(staged,binary);create(provider,code,0o755)
os.rename(staged,binary);os.rename(staged_agent,agent_binary);create(provider,code,0o755)
create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644)
inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(inspected==json.loads(general),'general policy unchanged')
@@ -90,6 +96,7 @@ def upgrade(payload):
(ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift')
report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown,
helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save()
report['agent_sha256']=sha(agent_raw);save()
finally:
fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
@@ -139,12 +146,16 @@ def main():
folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant]
ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes()
agent_raw=(STATE/'agent-sync-prevention').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant,
old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c',
old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945',
binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(),
agent_binary=base64.b64encode(agent_raw).decode(),agent_sha256=hashlib.sha256(agent_raw).hexdigest(),old_agent_sha256=installed['binary_sha256'],
helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest())
report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report)
require(report['completed'] and report['agent_sha256']==payload['agent_sha256'],'qualified preserving installation')
m.write_private(cloud.ROOT/'devstate/remote-vm.json',dict(installed,binary_sha256=report['agent_sha256']))
print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')})
+158
View File
@@ -0,0 +1,158 @@
#!/usr/bin/env python3
"""One signed development control-chain repair, never a job or grant repair."""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import uuid
import cloud_apt_reconcile_upgrade as upgrade
cloud=upgrade.cloud
require=cloud.require
STATE=cloud.ROOT/'devstate/checkpoint-chain-repair'
def prepared_json(path):
"""The bounded full historical witness exceeds ordinary 1 MiB DTOs."""
import stat
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
with os.fdopen(fd,'r') as stream:
info=os.fstat(stream.fileno())
require(stat.S_ISREG(info.st_mode) and info.st_uid==os.getuid() and
stat.S_IMODE(info.st_mode)==0o600 and info.st_nlink==1 and info.st_size<=45*1024*1024,
'private bounded repair witness')
return json.load(stream)
def prepare_vm(payload):
import re
import shutil
import sqlite3
os.umask(0o077)
ident=str(uuid.UUID(payload['id']));state=Path('/var/lib/rmm-agent-remote');helper=Path('/var/lib/rmm-helper')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'reviewed permanent agent')
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'fixed permanent device')
base=Path('/var/lib/rmm-apt-transaction');result=json.loads((base/'active/sandbox-result.json').read_text())
require(result['apt_exit_code']==0 and not result['reboot_required'],'ended native child required; never stop package manager')
root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident);root.mkdir(mode=0o700)
def create(path,data,mode=0o600):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode)
tool=base64.b64decode(payload['tool'],validate=True)
require(hashlib.sha256(tool).hexdigest()==payload['tool_sha256'],'qualified repair tool')
create(root/'checkpoint-repair',tool,0o755)
subprocess.run(['/lib64/ld-linux-x86-64.so.2','--list',str(root/'checkpoint-repair')],capture_output=True,check=True,timeout=10)
binding=json.loads((state/'binding.json').read_text())
config=json.loads(Path('/etc/rmm-remote/helper-general.json').read_text())
pin=json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_text())
require(all(binding[k]==config['binding'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')) and
binding['public_key_hex']==pin['public_key_hex'],'independent root trust binding')
create(root/'binding.json',json.dumps(binding).encode())
units=('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')
before={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode() for name in units}
subprocess.run(['systemctl','stop',*units],capture_output=True,check=True,timeout=25)
prepared=False;report=dict(id=ident,vm_dir=str(root),tool_sha256=payload['tool_sha256'],prepared=False,package_effects=False)
try:
paths=(state/'journal.sqlite',state/'consumption.sqlite',helper/'consumption.sqlite')
hashes={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
for index,p in enumerate(paths):
with sqlite3.connect(p.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','preserved database integrity')
create(root/('database-before-'+str(index)+'.sqlite'),p.read_bytes())
process=subprocess.run([str(root/'checkpoint-repair'),'prepare',str(root/'binding.json'),str(root/'evidence.json')],
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25)
create(root/'prepare.stderr',process.stderr)
require(all(hashlib.sha256(p.read_bytes()).hexdigest()==hashes[str(p)] for p in paths),'prepare must not change databases')
report.update(exit_code=process.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',process.stderr)],preserved_database_bytes=True)
if process.returncode==0:
raw=(root/'evidence.json').read_bytes();require(len(raw)<=32*1024*1024,'bounded evidence')
report.update(prepared=True,evidence_sha256=hashlib.sha256(raw).hexdigest(),evidence_b64=base64.b64encode(raw).decode())
prepared=True
finally:
create(root/'report.json',json.dumps({k:v for k,v in report.items() if k!='evidence_b64'}).encode())
if not prepared:
for unit in units:
if before[unit]=='active':subprocess.run(['systemctl','start',unit],capture_output=True,timeout=20)
return report
def issue_server(payload):
root=Path('/opt/rust-discovery')
raw=base64.b64decode(payload['evidence_b64'],validate=True)
require(hashlib.sha256(raw).hexdigest()==payload['evidence_sha256'],'original exact witness bytes')
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','api','/usr/local/bin/rmm-server','repair-unsent-sync-request']
result=subprocess.run(cmd,input=raw,capture_output=True,timeout=30)
require(result.returncode==0,'signed control receipt issuance failed')
value=json.loads(result.stdout);require(set(value)=={'payload','signature'},'only signed control receipt output')
return value
def apply_vm(payload):
import re
os.umask(0o077)
ident=str(uuid.UUID(payload['id']));root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident)
require(hashlib.sha256((root/'evidence.json').read_bytes()).hexdigest()==payload['evidence_sha256'] and
hashlib.sha256((root/'checkpoint-repair').read_bytes()).hexdigest()==payload['tool_sha256'],'original repair witnesses/tool')
signed=json.dumps(payload['receipt'],separators=(',',':')).encode();path=root/'receipt.json'
if path.exists():require(path.read_bytes()==signed,'identical existing receipt only')
else:
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as f:f.write(signed);f.flush();os.fsync(f.fileno())
result=subprocess.run([str(root/'checkpoint-repair'),'apply',str(root/'binding.json'),str(root/'evidence.json'),str(path)],
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25)
fd=os.open(root/'apply.stderr',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as f:f.write(result.stderr)
return dict(applied=result.returncode==0,exit_code=result.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',result.stderr)],
ready_lease_issued=False,jobs_created=False,grants_created=False)
def status_vm(payload):
import sqlite3
result={}
for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'),
('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')):
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0],
receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0,
integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok')
if name=='helper':
result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
result['services']={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode()
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')}
return result
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('prepare','issue','apply'));parser.add_argument('--id')
args=parser.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True);m=cloud.contract
if args.action=='prepare':
require(upgrade.apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact no-new-execution gate')
ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700)
tool=(upgrade.STATE/'checkpoint-repair').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
result=cloud.bootstrap.rpc('linux-vm.conf',prepare_vm,dict(id=ident,agent_id=installed['identity']['agent_id'],agent_sha256=installed['binary_sha256'],
tool_sha256=hashlib.sha256(tool).hexdigest(),tool=base64.b64encode(tool).decode()))
m.write_private(folder/'prepared.json',result)
print({k:result.get(k) for k in ('id','prepared','exit_code','error_codes','preserved_database_bytes')})
require(result['prepared'],'stored witness preparation incomplete; no fault changes')
else:
ident=str(uuid.UUID(args.id));folder=STATE/ident;prepared=prepared_json(folder/'prepared.json');require(prepared['prepared'],'qualified original witnesses')
if args.action=='issue':
result=cloud.bootstrap.rpc('server.conf',issue_server,prepared);m.write_private(folder/'receipt.json',result)
print(dict(signed_control_receipt=True,execution_permission=False))
else:
result=cloud.bootstrap.rpc('linux-vm.conf',apply_vm,dict(prepared,receipt=m.protected_json(folder/'receipt.json')))
m.write_private(folder/'applied.json',result);print(result)
if __name__=='__main__':
try:main()
except Exception as error:
message=str(error)
if message.startswith('remote bootstrap incomplete: '):print(message)
else:print('CLOUD_CHECKPOINT_REPAIR_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+3 -3
View File
@@ -124,7 +124,7 @@ def deploy(payload):
def main():
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
parser.add_argument('--preserve-known-unknown', action='store_true', help='Windows inventory migration only; preserve exact fenced APT evidence')
parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path')
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
if args.action == 'stage':
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
@@ -160,7 +160,7 @@ def main():
before['ledger']['result'] is None and before['ledger']['outcome']['error_code'] == 'DPKG_INCONSISTENT', 'preserve exact native unknown')
closed = cloud.contract.protected_json(apt.STATE/'policy-closed.json')
payload['closed_policy_sha256'] = closed['policy_sha256']
require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'additive Windows migration only')
require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'retain existing additive schema')
else: require(gate['unresolved'] == 0, 'unresolved cloud job; no server switch')
if args.action == 'deploy':
stream = io.BytesIO()
@@ -177,7 +177,7 @@ def main():
gate_after = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
current = cloud.Backend().call('jobs.get', dict(tenant_id=cloud.TENANT, job_id=job['job_id']))
require(before == after and gate_after == gate and current['state'] == 'unknown' and current['attempt_id'] == job['attempt_id'],
'unknown evidence or execution artifacts changed during Windows migration')
'unknown evidence or execution artifacts changed during control-path upgrade')
cloud.contract.write_private(STATE/(ident+'.unknown-preserved.json'), dict(evidence=after, gate=gate_after, result=current))
result['unknown_attempt_preserved'] = True
print(result)
+45
View File
@@ -1,5 +1,50 @@
# Tatsächlicher Implementierungsstand
## Signierte Kontrollkettenreparatur auf Cloud-VM bestanden — 2026-11-13
Der qualifizierte Root-Bootstrap prüfte native gespeicherte Agent-/Helper-
Historien und fertigte begrenzte Zeugen an; alle drei SQLite-Dateien blieben
bei dieser Vorbereitung bytegleich. Agent und beide Helper kontrolliert gestoppt,
keine Paketmanagerwirkung unterbrochen. Der neue ARM64-Serverbuild und das
Deployment bestanden; exakte ursprüngliche APT-Evidenz und Zähler vor/nach
Serverwechsel gleich. DB/PKI/Geräteidentität nicht ersetzt.
Server-CLI `repair-unsent-sync-request` prüfte ursprüngliche signierte Ablehnung,
noch vorhandenen gültigen Serverhead, fehlenden ungesendeten Vorgänger,
Geräte-/Boot-/Generationsbindung und geschlossene Rootpolicy. Bestehende
wirkungslose Ablehnung monoton zum gemeinsamen Kontrollhead übernommen;
separate kurzlebige Signatur `rmm.sync.unsent-request-repair` ausgestellt.
Kein neuer HTTP-Endpunkt, keine interaktive Freigabe vorgetäuscht.
Native unabhängige Verifier akzeptierten ausschließlich die exakten Zeugen.
Danach tatsächlich **0 Agentfaults / 0 Helperfaults**, jeweils **eine dauerhaft
archivierte Reparaturquittung**, beide Integritätsprüfungen ok. Verbrauchsledger
weiter **9 / 1 unresolved**, Zentrale weiter **13 Jobs / 13 Outbox / 12 Grants /
1 unresolved**. Keine Ready-Lease, Jobs, Grants, Outcomeänderung oder neue
Paketwirkung. Alle drei Dienste bleiben bis zum nachfolgenden qualifizierten
Agent-/Readonlyhelperwechsel gestoppt. APT weiterhin unknown; die
Kontrollreparatur ist ausdrücklich keine Paketabnahme oder Wiederholungsfreigabe.
Lokale Prüfungen dieser Quittungsstrecke: **1 CLI-Historienmutationstest,
34 Helper-Library- und 21 RootSync-Tests**; Server **3 gezielte Prüfungen mit
frischem isoliertem PostgreSQL**, einschließlicher verweigerter manipuliert/
veraltet/falsch gebundener Zeugen und unveränderter Null-Jobartefakte.
Format/Clippy `-D warnings` und Releasebuild bestanden. Außerdem fünf gezielte
Guards für eng begrenzte Entwicklungs-Wirefaults geprüft; deren native
Duplicate-/Lost-ACK-Abnahme steht weiterhin aus.
Ein erster lokaler Quittungsaufruf stoppte am normalen 1-MiB-DTO-Limit für die
vollständigen historischen Zeugen, noch vor Serverzugriff. Separater begrenzter
Private-Dateileser ergänzt (Owner, 0600, keine Symlinks/Hardlinks, maximal45MiB);
identische ursprüngliche Zeugen anschließend erfolgreich verwendet. Geschützt:
`devstate/checkpoint-chain-repair/2c5c3f06-a10e-40de-a02c-3c8009bc034f/` und
`devstate/server-upgrades/2f6144c6-fcd5-49cb-9634-0d6e0b898165*`.
**MVP-Prüfung: weiterhin offen.** Nächster Schritt ist der ausschließlich
lesende, an den ursprünglichen APT-Auftrag gebundene Teilwirkungsnachweis und
seine Übernahme über den normalen signierten Sync, erst danach ein neuer
WebUI-Wiederherstellungsplan. Weitere Gates: `mvp-gates.md`.
## Nicht persistierter Sync-Request als MVP-Blocker eingegrenzt — 2026-11-13
Vorprüfung des geplanten Readonly-APT-Helperupgrades stoppte **vor jeder
+7 -7
View File
@@ -1,6 +1,6 @@
# Linux-MVP: Abnahmegates
Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.9. Die
Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.10. Die
Bestandsaufnahme selbst hat keine Tests, Jobs oder Deployments ausgeführt;
anschließende tatsächliche Nachweise sind unten ausdrücklich ergänzt. Einzelbelege und spätere
Ergebnisse stehen im [Implementierungsstand](implementation-status.md).
@@ -39,12 +39,12 @@ bestanden. Dauerhafte APT-Unknown-Evidenz unverändert, keine Jobs, Testservice
gestoppt. M1 braucht weiterhin tatsächlichen WSS-Ausfall/Fallback; Gate 0 bleibt
die nächste blockierende Verwaltungsstrecke. MVP insgesamt weiterhin offen.
**Aktueller Gate-0-Blocker:** Upgradevorprüfung fand in Agent und Sharedhelper
eine gespeicherte `CHECKPOINT_CHAIN_MISMATCH`-Sperre. Nicht serverseitig
persistierter Vorgänger plus verworfener bestätigter Anker ist anhand der
gespeicherten Kette eingegrenzt; Prävention lokal geprüft, nicht deployt.
Vor APT-Abgleich ist eine exakt signierte Kontrollquittung mit erhaltener
Historie nötig. Keine direkte Faultlöschung oder pauschale Startfreigabe.
**Gate-0-Teilerfolg:** Native signierte Kontrollkettenreparatur bestanden;
ursprüngliche Faults mit je einer Quittung dauerhaft archiviert, jetzt beide
faultfrei. Keine Outcome-/Verbrauchs-/Grantänderung: weiterhin 9/1 bzw.
13/13/12/1. Dienste für qualifizierten Agent-/Readonlyhelperwechsel gestoppt.
Nächster Schritt bleibt der gebundene APT-Teilwirkungsnachweis mit normalem
signiertem Sync; die Kontrollreparatur schließt das Unknown nicht selbst.
Alle folgenden Schritte sind **geplant, nicht durch dieses Dokument ausgeführt**.
Jeder Ausführungsjob braucht weiterhin seinen eigenen festen WebUI-genehmigten
+13 -1
View File
@@ -1,6 +1,6 @@
# Verbindliche Projektentscheidungen
Stand: **2026-11-13**, Spezifikation **1.9** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
Stand: **2026-11-13**, Spezifikation **1.10** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
## Einzelner signierter APT-Wiederaufnahmeversuch (2026-11-13)
@@ -113,3 +113,15 @@ Genehmigungsregeln gelten weiterhin.
- VM-Daten: `credentials/linux-vm.conf`, Verzeichnis durch `.gitignore` ausgeschlossen, Modus 700 für `credentials/`, 600 für die Datei. Strikt als Daten parsen, kein `source`/`eval`; keine Secrets lesen/loggen/committen oder in LLM-Kontexte übernehmen.
- Auch Linux-Dev hat keine Jobfreigabeabkürzung: erst `pending_approval`, dann WebUI-Genehmigung durch einen berechtigten Menschen oder ausschließlich die oben dokumentierte dev/test-Browsersimulation. Passives Standardinventar ist kein Job. Keine impliziten Reboots, Updates oder destruktiven Tests auf dem Ziel; ausdrücklich gesondert abstimmen, RMM-Jobs zusätzlich über ihren festen Plan in der WebUI genehmigen.
- Kein Erfolg ohne realen Nachweis. Geplante Befehle und nicht ausgeführte Tests ausdrücklich so kennzeichnen; fehlende VM-Abnahme bleibt offen.
## Eng begrenzte Reparatur eines ungesendeten Kontrollrequests
Spezifikation 1.10 konkretisiert den existierenden Unknown-/Sync-Abgleich:
Root-Bootstrap ausschließlich im Entwicklungsprofil, vollständige originale
Zeugen, unabhängige lokale Signatur-/Inventarprüfung und separate kurzlebige
Kontrollquittung. Nur vorhandene wirkungslose signierte Ablehnung als gemeinsamer
monotoner Head; Faults und Quittung append-only archivieren. Keine Jobfreigabe,
Bereit-Lease, Änderung von Outcome oder Verbrauch, keine Journalrücksetzung.
Native Prüfung und Grenzen: `implementation-status.md`. Direkte API-/DB-
Jobfreigaben bleiben verboten; anschließende Paketwirkung benötigt neuen festen
WebUI-genehmigten Plan.
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Checkpoint repair against a fresh, isolated PostgreSQL; no deployed state."""
from pathlib import Path
import subprocess
import tempfile
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
NAME = "rmm-checkpoint-repair-" + uuid.uuid4().hex[:10]
def command(args, data=None, timeout=120):
result = subprocess.run(args, input=data, capture_output=True, timeout=timeout)
if result.returncode:
raise RuntimeError(result.stderr.decode(errors="replace")[-4000:])
return result.stdout
def main():
command(["docker", "run", "-d", "--network", "none", "--name", NAME,
"-e", "POSTGRES_HOST_AUTH_METHOD=trust", "postgres:17.6-bookworm"])
try:
for _ in range(100):
logs = command(["docker", "logs", NAME])
probe = subprocess.run(["docker", "exec", NAME, "pg_isready", "-U", "postgres"],
capture_output=True)
if b"init process complete" in logs and probe.returncode == 0:
break
time.sleep(.2)
else:
raise RuntimeError("ISOLATED_POSTGRES_NOT_READY")
def sql(statement):
return command(["docker", "exec", "-i", NAME, "psql", "-X", "-qAt",
"-v", "ON_ERROR_STOP=1", "-U", "postgres"], statement.encode())
sql("CREATE ROLE rmm_app LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS;")
for path in sorted((ROOT / "migrations").glob("*.sql")):
sql(path.read_text())
# Current source with a pre-Windows dependency manifest uses the cached
# Linux toolchain; Windows is outside this Linux checkpoint test.
manifests = tempfile.TemporaryDirectory(prefix="rmm-checkpoint-linux-manifest-")
manifest = Path(manifests.name)
for name in ("Cargo.toml", "Cargo.lock"):
(manifest / name).write_bytes(command([
"git", "-C", str(ROOT), "show", "a5c2dce:" + name]))
output = command([
"docker", "run", "--rm", "--network", "container:" + NAME,
"--read-only", "--tmpfs", "/tmp:rw,size=512m",
"-v", str(ROOT) + ":/src:ro",
"-v", str(manifest / "Cargo.toml") + ":/src/Cargo.toml:ro",
"-v", str(manifest / "Cargo.lock") + ":/src/Cargo.lock:ro",
"-v", str(ROOT / "devstate/windows-client/target-linux") + ":/target",
"-e", "CARGO_TARGET_DIR=/target",
"-e", "RMM_TEST_REPAIR_ADMIN_URL=postgres://postgres@127.0.0.1/postgres",
"-e", "RMM_TEST_REPAIR_APP_URL=postgres://rmm_app@127.0.0.1/postgres",
"--entrypoint", "cargo", "rust-discovery-dev-build:local",
"test", "--locked", "--offline", "-p", "rmm-server",
"checkpoint_repair", "--", "--include-ignored", "--nocapture"], timeout=300)
print(output.decode())
print("PASS: isolated PostgreSQL repair, scope/revocation/stale-head/committed-request rejection, exact replay; zero jobs/outbox/permits and no readiness.")
manifests.cleanup()
finally:
subprocess.run(["docker", "rm", "-f", NAME], capture_output=True)
if __name__ == "__main__":
main()