fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Rust RMM: Architektur und Implementierungsspezifikation
|
||||
|
||||
Version 1.9 · 2026-11-13 · Sprache: Deutsch
|
||||
Version 1.10 · 2026-11-13 · Sprache: Deutsch
|
||||
|
||||
**Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam.
|
||||
|
||||
@@ -66,6 +66,21 @@ Deploymentdaten, offene Konfiguration und reale Nachweise:
|
||||
|
||||
### 0.1 Auftrag an die implementierende KI
|
||||
|
||||
**Technische Präzisierung Version 1.10 — wirkungslose Kontrollkettenreparatur:**
|
||||
Ein nachweislich nicht serverseitig persistierter Sync-Request darf im expliziten
|
||||
Entwicklungsprofil über ein eng begrenztes Root-Bootstrapwerkzeug abgeglichen
|
||||
werden. Server und beide lokalen Verifier prüfen die vollständigen erhaltenen
|
||||
Journale, ursprünglichen Signaturen, exakten Geräte-/Boot-/Checkpointbindungen
|
||||
und den unveränderten offenen Verbrauch. Nur die bereits gespeicherte signierte
|
||||
Ablehnung wird zum monotonen gemeinsamen Kontrollhead; eine kurzlebige,
|
||||
domänengetrennte Signatur bindet sämtliche Zeugen. Lokale alte Faults und
|
||||
Quittung werden dauerhaft archiviert. Keine Bereitschaft, Genehmigung, Jobs,
|
||||
Grants, Verbräuche oder Outcomes entstehen oder ändern sich dadurch. Kein
|
||||
manuelles Löschen einer Sperre, kein Journal-/Identitätsreset. Erst der normale
|
||||
signierte Sync und eine unabhängig gebundene Nachprüfung dürfen anschließend
|
||||
ein unbekanntes Ausführungsergebnis klassifizieren. Dies konkretisiert den
|
||||
bestehenden Abgleichvertrag und ist keine Ausnahme von der WebUI-Jobgenehmigung.
|
||||
|
||||
**Präzisierung Version 1.9, 2026-11-13 — expliziter neuer APT-Testversuch:**
|
||||
Ein bekannter Fehler vor der vertrauenswürdigen Hookübergabe darf nach belegtem
|
||||
unverändertem vollständigem Paketbestand durch genau einen neuen festen,
|
||||
|
||||
@@ -3,6 +3,7 @@ name = "rmm-agent"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
default-run = "rmm-agent"
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
//! Narrow, root-only development bootstrap. No job execution or ready lease.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use fs2::FileExt;
|
||||
use prost::Message;
|
||||
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
|
||||
use rmm_privileged_helper::{Binding, checkpoint_repair as repair};
|
||||
use rmm_protocol::{Envelope, Payload, SyncRequest, SyncResponse};
|
||||
use rusqlite::{Connection, OpenFlags, OptionalExtension};
|
||||
use std::{
|
||||
fs::{self, OpenOptions},
|
||||
io::Write,
|
||||
os::unix::fs::{MetadataExt, OpenOptionsExt},
|
||||
path::Path,
|
||||
};
|
||||
const AGENT: &str = "/var/lib/rmm-agent-remote";
|
||||
const HELPER: &str = "/var/lib/rmm-helper";
|
||||
fn read_private(path: &Path, owner: u32, limit: usize) -> Result<Vec<u8>> {
|
||||
rmm_privileged_helper::private_path(path, owner, false)?;
|
||||
let m = fs::metadata(path)?;
|
||||
ensure!(m.len() <= limit as u64, "REPAIR_FILE_SIZE");
|
||||
Ok(fs::read(path)?)
|
||||
}
|
||||
fn open_existing(path: &Path, owner: u32) -> Result<Connection> {
|
||||
rmm_privileged_helper::private_path(path, owner, false)?;
|
||||
// Require the existing DELETE journal mode. This bootstrap never opens WAL
|
||||
// state or creates privileged sidecars in the unprivileged agent directory.
|
||||
for suffix in ["-wal", "-shm", "-journal"] {
|
||||
ensure!(
|
||||
!Path::new(&format!("{}{suffix}", path.display())).try_exists()?,
|
||||
"REPAIR_DATABASE_SIDECAR_PRESENT"
|
||||
);
|
||||
}
|
||||
let db = Connection::open_with_flags(
|
||||
path,
|
||||
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX,
|
||||
)?;
|
||||
let mode: String = db.query_row("PRAGMA journal_mode", [], |v| v.get(0))?;
|
||||
ensure!(mode == "delete", "REPAIR_DELETE_JOURNAL_REQUIRED");
|
||||
db.execute_batch(
|
||||
"PRAGMA synchronous=FULL; PRAGMA foreign_keys=OFF; PRAGMA trusted_schema=OFF;",
|
||||
)?;
|
||||
Ok(db)
|
||||
}
|
||||
fn signed(raw: &[u8]) -> Result<SignedDecision> {
|
||||
let r = SyncResponse::decode(raw)?;
|
||||
ensure!(r.encode_to_vec() == raw, "REPAIR_NONCANONICAL_RESPONSE");
|
||||
Ok(SignedDecision {
|
||||
decision_bytes: r.decision_bytes,
|
||||
signature: r.signature,
|
||||
})
|
||||
}
|
||||
fn check_response(
|
||||
binding: &Binding,
|
||||
request: &SyncRequest,
|
||||
envelope: &Envelope,
|
||||
response: &SignedDecision,
|
||||
) -> Result<ReconcileDecisionPayload> {
|
||||
let d = repair::verify_historical_decision(binding, response)?;
|
||||
ensure!(
|
||||
d.request_id.to_string() == request.request_id
|
||||
&& d.request_hash == hash(&request.encode_to_vec())
|
||||
&& d.journal_id.to_string() == request.journal_id
|
||||
&& d.checkpoint == request.checkpoint
|
||||
&& d.connection_generation == envelope.connection_generation
|
||||
&& d.boot_id.to_string() == envelope.boot_id
|
||||
&& d.helper_journal_id
|
||||
.map(|id| id.to_string())
|
||||
.unwrap_or_default()
|
||||
== request.helper_journal_id
|
||||
&& d.helper_checkpoint == request.helper_checkpoint,
|
||||
"REPAIR_AGENT_SIGNED_REQUEST_BINDING"
|
||||
);
|
||||
Ok(d)
|
||||
}
|
||||
fn agent_history(db: &Connection, binding: &Binding) -> Result<Vec<(SyncRequest, SignedDecision)>> {
|
||||
let mut s = db.prepare(
|
||||
"SELECT request,envelope,generation,response FROM reconcile_requests ORDER BY rowid",
|
||||
)?;
|
||||
let rows = s
|
||||
.query_map([], |v| {
|
||||
Ok((
|
||||
v.get::<_, Vec<u8>>(0)?,
|
||||
v.get::<_, Vec<u8>>(1)?,
|
||||
v.get::<_, u64>(2)?,
|
||||
v.get::<_, Option<Vec<u8>>>(3)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ensure!(rows.len() <= 100000, "REPAIR_AGENT_HISTORY_LIMIT");
|
||||
let mut result = vec![];
|
||||
for (raw, outer, generation, response) in rows {
|
||||
ensure!(
|
||||
raw.len() <= rmm_domain::MAX_MESSAGE && outer.len() <= rmm_domain::MAX_MESSAGE,
|
||||
"REPAIR_REQUEST_SIZE"
|
||||
);
|
||||
let request = SyncRequest::decode(raw.as_slice())?;
|
||||
let envelope = Envelope::decode(outer.as_slice())?;
|
||||
ensure!(
|
||||
request.encode_to_vec() == raw
|
||||
&& envelope.encode_to_vec() == outer
|
||||
&& envelope.payload == Some(Payload::SyncRequest(request.clone()))
|
||||
&& envelope.agent_id == binding.agent_id.to_string()
|
||||
&& envelope.enrollment_generation == binding.enrollment_generation
|
||||
&& envelope.connection_generation == generation
|
||||
&& envelope.sequence == request.checkpoint
|
||||
&& request.assignment_version == binding.assignment_version,
|
||||
"REPAIR_AGENT_DURABLE_REQUEST_BINDING"
|
||||
);
|
||||
if let Some(response) = response {
|
||||
let response = signed(&response)?;
|
||||
check_response(binding, &request, &envelope, &response)?;
|
||||
result.push((request, response));
|
||||
}
|
||||
}
|
||||
let mut offers = db.prepare("SELECT job,attempt,offer FROM job_attempts")?;
|
||||
for row in offers.query_map([], |v| {
|
||||
Ok((
|
||||
v.get::<_, String>(0)?,
|
||||
v.get::<_, String>(1)?,
|
||||
v.get::<_, Vec<u8>>(2)?,
|
||||
))
|
||||
})? {
|
||||
let (job, attempt, bytes) = row?;
|
||||
let offer = rmm_protocol::JobOffer::decode(bytes.as_slice())?;
|
||||
let payload = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job
|
||||
&& offer.attempt_id == attempt
|
||||
&& payload.job_id.to_string() == job
|
||||
&& payload.attempt_id.to_string() == attempt
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes),
|
||||
"REPAIR_AGENT_OFFER_BINDING"
|
||||
);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
fn combined_inventory(agent: &Connection, consumption: &Connection) -> Result<serde_json::Value> {
|
||||
let mut value = repair::inventory(agent, "reconcile_meta")?;
|
||||
value
|
||||
.as_object_mut()
|
||||
.context("REPAIR_AGENT_INVENTORY")?
|
||||
.insert(
|
||||
"agent_consumption".into(),
|
||||
repair::inventory(consumption, "")?,
|
||||
);
|
||||
Ok(value)
|
||||
}
|
||||
fn evidence(
|
||||
agent: &Connection,
|
||||
consumption: &Connection,
|
||||
helper: &Connection,
|
||||
binding: &Binding,
|
||||
) -> Result<Evidence> {
|
||||
rmm_privileged_helper::reconcile::verify_checkpoint_repair_history(helper, binding, None)?;
|
||||
let history = agent_history(agent, binding)?;
|
||||
let (head, fault, pending): (String, Option<String>, Option<String>) = agent.query_row(
|
||||
"SELECT last_decision,fault,pending_request FROM reconcile_meta WHERE id=1",
|
||||
[],
|
||||
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?)),
|
||||
)?;
|
||||
ensure!(
|
||||
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH") && pending.is_none(),
|
||||
"REPAIR_AGENT_EXACT_FAULT_REQUIRED"
|
||||
);
|
||||
let (y, denial) = history
|
||||
.iter()
|
||||
.find(|(_, r)| {
|
||||
serde_json::from_slice::<ReconcileDecisionPayload>(&r.decision_bytes)
|
||||
.is_ok_and(|d| d.decision_id.to_string() == head)
|
||||
})
|
||||
.context("REPAIR_REJECTION_MISSING")?;
|
||||
let raw_x: Vec<u8> = agent.query_row(
|
||||
"SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL",
|
||||
[&y.previous_request_id],
|
||||
|v| v.get(0),
|
||||
)?;
|
||||
let x = SyncRequest::decode(raw_x.as_slice())?;
|
||||
let (_, a) = history
|
||||
.iter()
|
||||
.find(|(_, r)| {
|
||||
serde_json::from_slice::<ReconcileDecisionPayload>(&r.decision_bytes)
|
||||
.is_ok_and(|d| d.decision_id.to_string() == x.previous_decision_id)
|
||||
})
|
||||
.context("REPAIR_PREVIOUS_VALID_MISSING")?;
|
||||
Ok(Evidence {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: binding.enrollment_generation,
|
||||
assignment_version: binding.assignment_version,
|
||||
agent_inventory: combined_inventory(agent, consumption)?,
|
||||
helper_inventory: repair::inventory(helper, "helper_sync_meta")?,
|
||||
uncommitted_request: raw_x,
|
||||
rejected_request: y.encode_to_vec(),
|
||||
rejected_response: denial.clone(),
|
||||
previous_valid_response: a.clone(),
|
||||
})
|
||||
}
|
||||
fn verify_agent(
|
||||
agent: &Connection,
|
||||
consumption: &Connection,
|
||||
binding: &Binding,
|
||||
signed: &SignedReceipt,
|
||||
e: &Evidence,
|
||||
) -> Result<()> {
|
||||
let r = repair::verify_receipt(binding, signed)?;
|
||||
let _ = agent_history(agent, binding)?;
|
||||
ensure!(
|
||||
e.schema_version == 1
|
||||
&& e.tenant_id == binding.tenant_id
|
||||
&& e.asset_id == binding.asset_id
|
||||
&& e.agent_id == binding.agent_id
|
||||
&& e.enrollment_generation == binding.enrollment_generation
|
||||
&& e.assignment_version == binding.assignment_version,
|
||||
"REPAIR_EVIDENCE_IDENTITY"
|
||||
);
|
||||
let actual = combined_inventory(agent, consumption)?;
|
||||
ensure!(
|
||||
actual == e.agent_inventory && inventory_sha256(&actual)? == r.agent_inventory_sha256,
|
||||
"REPAIR_AGENT_INVENTORY"
|
||||
);
|
||||
let (journal,checkpoint,high,head,pending,helperid,helpercp,fault):(String,u64,u64,String,Option<String>,String,u64,Option<String>)=agent.query_row("SELECT journal_id,checkpoint,high_sequence,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta WHERE id=1",[],|v|Ok((v.get(0)?,v.get(1)?,v.get(2)?,v.get(3)?,v.get(4)?,v.get(5)?,v.get(6)?,v.get(7)?)))?;
|
||||
let counter: u64 = agent.query_row("SELECT seq FROM counter WHERE id=1", [], |v| v.get(0))?;
|
||||
ensure!(
|
||||
journal == r.agent_journal_id.to_string()
|
||||
&& checkpoint == r.agent_checkpoint
|
||||
&& counter >= high
|
||||
&& counter >= checkpoint
|
||||
&& head == r.rejected_decision_id.to_string()
|
||||
&& pending.is_none()
|
||||
&& helperid == r.helper_journal_id.to_string()
|
||||
&& helpercp == r.helper_checkpoint
|
||||
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|
||||
|| (fault.is_none() && repair::already_applied(agent, signed)?)),
|
||||
"REPAIR_AGENT_HEAD"
|
||||
);
|
||||
let y = SyncRequest::decode(e.rejected_request.as_slice())?;
|
||||
let x = SyncRequest::decode(e.uncommitted_request.as_slice())?;
|
||||
let stored_y: (Vec<u8>, Vec<u8>) = agent.query_row(
|
||||
"SELECT request,response FROM reconcile_requests WHERE id=?1",
|
||||
[&y.request_id],
|
||||
|v| Ok((v.get(0)?, v.get(1)?)),
|
||||
)?;
|
||||
let stored_x: Option<Vec<u8>> = agent
|
||||
.query_row(
|
||||
"SELECT request FROM reconcile_requests WHERE id=?1 AND response IS NULL",
|
||||
[&x.request_id],
|
||||
|v| v.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
let d = repair::verify_historical_decision(binding, &e.rejected_response)?;
|
||||
let a = repair::verify_historical_decision(binding, &e.previous_valid_response)?;
|
||||
ensure!(
|
||||
stored_y.0 == e.rejected_request
|
||||
&& signed_response_eq(&stored_y.1, &e.rejected_response)?
|
||||
&& stored_x.as_deref() == Some(e.uncommitted_request.as_slice())
|
||||
&& d.decision_id == r.rejected_decision_id
|
||||
&& hash(&e.rejected_response.decision_bytes) == r.rejected_decision_sha256
|
||||
&& d.reason == r.fault
|
||||
&& !d.ready
|
||||
&& !d.helper_ready
|
||||
&& !d.attempts.iter().any(|v| v.unlock)
|
||||
&& a.decision_id == r.previous_valid_decision_id
|
||||
&& hash(&e.previous_valid_response.decision_bytes) == r.previous_valid_decision_sha256
|
||||
&& hash(&e.rejected_request) == r.rejected_request_sha256
|
||||
&& y.request_id == r.rejected_request_id.to_string()
|
||||
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
|
||||
&& x.request_id == r.uncommitted_request_id.to_string()
|
||||
&& x.previous_decision_id == a.decision_id.to_string()
|
||||
&& x.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.previous_decision_id.is_empty()
|
||||
&& y.previous_request_id == x.request_id
|
||||
&& y.helper_previous_decision_id == a.decision_id.to_string(),
|
||||
"REPAIR_AGENT_EXACT_UNSENT_CHAIN"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn signed_response_eq(raw: &[u8], expected: &SignedDecision) -> Result<bool> {
|
||||
Ok(signed(raw)? == *expected)
|
||||
}
|
||||
fn main() -> Result<()> {
|
||||
ensure!(
|
||||
unsafe { libc::geteuid() } == 0,
|
||||
"REPAIR_ROOT_BOOTSTRAP_ONLY"
|
||||
);
|
||||
ensure!(
|
||||
std::env::var("RMM_CHECKPOINT_REPAIR_PROFILE").as_deref() == Ok("development"),
|
||||
"REPAIR_EXPLICIT_DEVELOPMENT_PROFILE_REQUIRED"
|
||||
);
|
||||
ensure!(
|
||||
rmm_privileged_helper::clock_trusted(),
|
||||
"REPAIR_CLOCK_UNTRUSTED"
|
||||
);
|
||||
let args = std::env::args_os().skip(1).collect::<Vec<_>>();
|
||||
ensure!(
|
||||
matches!(args.len(), 3 | 4),
|
||||
"usage: rmm-checkpoint-repair prepare BINDING_FILE EVIDENCE_FILE | apply BINDING_FILE EVIDENCE_FILE RECEIPT_FILE"
|
||||
);
|
||||
let binding = rmm_privileged_helper::read_binding(Path::new(&args[1]), 0)?;
|
||||
let owner = fs::metadata(AGENT)?.uid();
|
||||
ensure!(owner != 0, "REPAIR_UNPRIVILEGED_AGENT_REQUIRED");
|
||||
rmm_privileged_helper::private_path(Path::new(AGENT), owner, true)?;
|
||||
rmm_privileged_helper::private_path(Path::new(HELPER), 0, true)?;
|
||||
let lock_path = Path::new(AGENT).join("agent.lock");
|
||||
rmm_privileged_helper::private_path(&lock_path, owner, false)?;
|
||||
let lock = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(lock_path)?;
|
||||
lock.try_lock_exclusive()?;
|
||||
let _guard = rmm_privileged_helper::reconcile::ExecutorGuard::acquire(Path::new(HELPER), 0)?;
|
||||
let mut agent = open_existing(&Path::new(AGENT).join("journal.sqlite"), owner)?;
|
||||
let consumption = open_existing(&Path::new(AGENT).join("consumption.sqlite"), owner)?;
|
||||
let mut helper = open_existing(&Path::new(HELPER).join("consumption.sqlite"), 0)?;
|
||||
let mode = args[0].to_str().context("REPAIR_MODE")?;
|
||||
match (mode, args.len()) {
|
||||
("prepare", 3) => {
|
||||
let e = evidence(&agent, &consumption, &helper, &binding)?;
|
||||
let bytes = serde_json::to_vec(&e)?;
|
||||
ensure!(bytes.len() <= 32 * 1024 * 1024, "REPAIR_EVIDENCE_SIZE");
|
||||
let mut f = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(&args[2])?;
|
||||
f.write_all(&bytes)?;
|
||||
f.sync_all()?;
|
||||
println!("checkpoint repair evidence prepared; no state changed");
|
||||
}
|
||||
("apply", 4) => {
|
||||
let e: Evidence =
|
||||
serde_json::from_slice(&read_private(Path::new(&args[2]), 0, 32 * 1024 * 1024)?)?;
|
||||
let signed: SignedReceipt =
|
||||
serde_json::from_slice(&read_private(Path::new(&args[3]), 0, 65536)?)?;
|
||||
verify_agent(&agent, &consumption, &binding, &signed, &e)?;
|
||||
repair::verify_helper(&helper, &binding, &signed, &e)?;
|
||||
let r = repair::verify_receipt(&binding, &signed)?;
|
||||
// Every witness is verified before either durable fault archive is written.
|
||||
// A stopped bootstrap can replay the identical receipt after interruption.
|
||||
repair::commit_fault_repair(
|
||||
&mut helper,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)?;
|
||||
repair::commit_fault_repair(
|
||||
&mut agent,
|
||||
"reconcile_meta",
|
||||
&signed,
|
||||
&r.agent_inventory_sha256,
|
||||
Some(("agent_consumption", repair::inventory(&consumption, "")?)),
|
||||
)?;
|
||||
println!(
|
||||
"exact checkpoint faults archived and repaired; no execution readiness granted"
|
||||
);
|
||||
}
|
||||
_ => anyhow::bail!("REPAIR_UNKNOWN_MODE"),
|
||||
}
|
||||
drop(helper);
|
||||
drop(consumption);
|
||||
drop(agent);
|
||||
// DELETE journaling leaves no root-owned WAL/SHM or journal behind.
|
||||
for name in ["journal.sqlite", "consumption.sqlite"] {
|
||||
for suffix in ["-wal", "-shm", "-journal"] {
|
||||
ensure!(
|
||||
!Path::new(AGENT)
|
||||
.join(format!("{name}{suffix}"))
|
||||
.try_exists()?,
|
||||
"REPAIR_UNEXPECTED_AGENT_SIDECAR"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Utc;
|
||||
use openssl::{pkey::PKey, sign::Signer};
|
||||
use uuid::Uuid;
|
||||
#[test]
|
||||
fn durable_signed_request_history_rejects_mutation_before_any_repair() {
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let id = Uuid::new_v4();
|
||||
let boot = Uuid::new_v4();
|
||||
let binding = Binding {
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: key
|
||||
.raw_public_key()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect(),
|
||||
};
|
||||
let q = SyncRequest {
|
||||
request_id: Uuid::new_v4().to_string(),
|
||||
journal_id: id.to_string(),
|
||||
checkpoint: 10,
|
||||
assignment_version: 1,
|
||||
journal_complete: true,
|
||||
helper_journal_id: id.to_string(),
|
||||
helper_checkpoint: 20,
|
||||
..Default::default()
|
||||
};
|
||||
let e = Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: id.to_string(),
|
||||
enrollment_generation: 1,
|
||||
connection_generation: 2,
|
||||
boot_id: boot.to_string(),
|
||||
sequence: 10,
|
||||
observed_at: Utc::now().timestamp_millis(),
|
||||
payload: Some(Payload::SyncRequest(q.clone())),
|
||||
};
|
||||
let now = Utc::now();
|
||||
let d = ReconcileDecisionPayload {
|
||||
domain: "rmm.reconciliation-decision".into(),
|
||||
schema_version: 1,
|
||||
decision_id: Uuid::new_v4(),
|
||||
request_id: Uuid::parse_str(&q.request_id).unwrap(),
|
||||
request_hash: hash(&q.encode_to_vec()),
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
connection_generation: 2,
|
||||
boot_id: boot,
|
||||
journal_id: id,
|
||||
checkpoint: 10,
|
||||
helper_journal_id: Some(id),
|
||||
helper_checkpoint: 20,
|
||||
issued_at: now,
|
||||
valid_until: now + chrono::Duration::seconds(60),
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
|
||||
attempts: vec![],
|
||||
acknowledged_message_ids: vec![],
|
||||
key_id: hash(&key.raw_public_key().unwrap()),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&d).unwrap();
|
||||
let signature = Signer::new_without_digest(&key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&bytes)
|
||||
.unwrap();
|
||||
let response = SyncResponse {
|
||||
decision_bytes: bytes,
|
||||
signature,
|
||||
};
|
||||
let db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE reconcile_requests(id TEXT,request BLOB,envelope BLOB,generation INTEGER,response BLOB);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB);").unwrap();
|
||||
db.execute(
|
||||
"INSERT INTO reconcile_requests VALUES(?1,?2,?3,2,?4)",
|
||||
rusqlite::params![
|
||||
q.request_id,
|
||||
q.encode_to_vec(),
|
||||
e.encode_to_vec(),
|
||||
response.encode_to_vec()
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(agent_history(&db, &binding).unwrap().len(), 1);
|
||||
let mut bad = response.clone();
|
||||
bad.signature[0] ^= 1;
|
||||
db.execute(
|
||||
"UPDATE reconcile_requests SET response=?1",
|
||||
[bad.encode_to_vec()],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(agent_history(&db, &binding).is_err());
|
||||
db.execute(
|
||||
"UPDATE reconcile_requests SET response=?1",
|
||||
[response.encode_to_vec()],
|
||||
)
|
||||
.unwrap();
|
||||
let mut changed = e;
|
||||
changed.sequence = 11;
|
||||
db.execute(
|
||||
"UPDATE reconcile_requests SET envelope=?1",
|
||||
[changed.encode_to_vec()],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(agent_history(&db, &binding).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
//! Development bootstrap receipt for one proven unsent reconciliation request.
|
||||
//! This is neither a job approval, an execution permit nor a ready lease.
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SignedDecision {
|
||||
pub decision_bytes: Vec<u8>,
|
||||
pub signature: Vec<u8>,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SignedReceipt {
|
||||
pub payload: Vec<u8>,
|
||||
pub signature: Vec<u8>,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Evidence {
|
||||
pub schema_version: u32,
|
||||
pub tenant_id: Uuid,
|
||||
pub asset_id: Uuid,
|
||||
pub agent_id: Uuid,
|
||||
pub enrollment_generation: u64,
|
||||
pub assignment_version: u64,
|
||||
pub agent_inventory: Value,
|
||||
pub helper_inventory: Value,
|
||||
pub uncommitted_request: Vec<u8>,
|
||||
pub rejected_request: Vec<u8>,
|
||||
pub rejected_response: SignedDecision,
|
||||
pub previous_valid_response: SignedDecision,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Receipt {
|
||||
pub domain: String,
|
||||
pub schema_version: u32,
|
||||
pub receipt_id: Uuid,
|
||||
pub tenant_id: Uuid,
|
||||
pub asset_id: Uuid,
|
||||
pub agent_id: Uuid,
|
||||
pub enrollment_generation: u64,
|
||||
pub assignment_version: u64,
|
||||
pub issued_at: DateTime<Utc>,
|
||||
pub valid_until: DateTime<Utc>,
|
||||
pub key_id: String,
|
||||
pub fault: String,
|
||||
pub agent_journal_id: Uuid,
|
||||
pub agent_checkpoint: u64,
|
||||
pub helper_journal_id: Uuid,
|
||||
pub helper_checkpoint: u64,
|
||||
pub rejected_decision_id: Uuid,
|
||||
pub rejected_decision_sha256: String,
|
||||
pub rejected_request_id: Uuid,
|
||||
pub rejected_request_sha256: String,
|
||||
pub uncommitted_request_id: Uuid,
|
||||
pub uncommitted_request_sha256: String,
|
||||
pub previous_valid_decision_id: Uuid,
|
||||
pub previous_valid_decision_sha256: String,
|
||||
pub agent_inventory_sha256: String,
|
||||
pub helper_inventory_sha256: String,
|
||||
pub helper_ledger_sha256: String,
|
||||
pub ready: bool,
|
||||
pub helper_ready: bool,
|
||||
pub unlock: bool,
|
||||
}
|
||||
pub fn canonical_bytes(value: &Value) -> Result<Vec<u8>, serde_json::Error> {
|
||||
fn ordered(v: Value) -> Value {
|
||||
match v {
|
||||
Value::Object(m) => {
|
||||
let mut items = m.into_iter().collect::<Vec<_>>();
|
||||
items.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
Value::Object(items.into_iter().map(|(k, v)| (k, ordered(v))).collect())
|
||||
}
|
||||
Value::Array(a) => Value::Array(a.into_iter().map(ordered).collect()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
serde_json::to_vec(&ordered(value.clone()))
|
||||
}
|
||||
pub fn inventory_sha256(value: &Value) -> Result<String, serde_json::Error> {
|
||||
Ok(crate::jobs::hash(&canonical_bytes(value)?))
|
||||
}
|
||||
pub fn helper_ledger_sha256(inventory: &Value) -> Result<String, serde_json::Error> {
|
||||
inventory_sha256(
|
||||
&serde_json::json!({"consumed":inventory["consumed"],"execution_evidence":inventory["execution_evidence"]}),
|
||||
)
|
||||
}
|
||||
impl Receipt {
|
||||
pub fn valid_profile(&self, now: DateTime<Utc>) -> bool {
|
||||
let digest = |s: &str| {
|
||||
s.len() == 64
|
||||
&& s.bytes()
|
||||
.all(|c| c.is_ascii_digit() || b"abcdef".contains(&c))
|
||||
};
|
||||
self.domain == "rmm.sync.unsent-request-repair"
|
||||
&& self.schema_version == 1
|
||||
&& !self.receipt_id.is_nil()
|
||||
&& self.fault == "CHECKPOINT_CHAIN_MISMATCH"
|
||||
&& !self.ready
|
||||
&& !self.helper_ready
|
||||
&& !self.unlock
|
||||
&& self.issued_at <= now
|
||||
&& now < self.valid_until
|
||||
&& self.valid_until > self.issued_at
|
||||
&& self.valid_until - self.issued_at <= chrono::Duration::minutes(15)
|
||||
&& [
|
||||
self.tenant_id,
|
||||
self.asset_id,
|
||||
self.agent_id,
|
||||
self.agent_journal_id,
|
||||
self.helper_journal_id,
|
||||
self.rejected_decision_id,
|
||||
self.rejected_request_id,
|
||||
self.uncommitted_request_id,
|
||||
self.previous_valid_decision_id,
|
||||
]
|
||||
.iter()
|
||||
.all(|id| !id.is_nil())
|
||||
&& self.enrollment_generation > 0
|
||||
&& self.assignment_version > 0
|
||||
&& self.agent_checkpoint > 0
|
||||
&& self.helper_checkpoint > 0
|
||||
&& [
|
||||
&self.key_id,
|
||||
&self.rejected_decision_sha256,
|
||||
&self.rejected_request_sha256,
|
||||
&self.uncommitted_request_sha256,
|
||||
&self.previous_valid_decision_sha256,
|
||||
&self.agent_inventory_sha256,
|
||||
&self.helper_inventory_sha256,
|
||||
&self.helper_ledger_sha256,
|
||||
]
|
||||
.iter()
|
||||
.all(|s| digest(s))
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub mod apt_transaction;
|
||||
pub mod checkpoint_repair;
|
||||
pub mod inventory_document;
|
||||
pub mod jobs;
|
||||
pub mod reconciliation;
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
//! Fixed bootstrap verification of one signed no-effect checkpoint repair.
|
||||
use crate::Binding;
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use openssl::{
|
||||
pkey::{Id, PKey},
|
||||
sign::Verifier,
|
||||
};
|
||||
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
|
||||
use rusqlite::{Connection, OptionalExtension, params, types::ValueRef};
|
||||
use serde_json::{Value, json};
|
||||
pub const AUDIT: &str = "checkpoint_repair_receipts";
|
||||
pub fn verify_historical_decision(
|
||||
binding: &Binding,
|
||||
signed: &SignedDecision,
|
||||
) -> Result<ReconcileDecisionPayload> {
|
||||
ensure!(
|
||||
signed.decision_bytes.len() <= 512 * 1024 && signed.signature.len() == 64,
|
||||
"REPAIR_DECISION_SIZE"
|
||||
);
|
||||
let raw = hex::decode(&binding.public_key_hex)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?
|
||||
.verify_oneshot(&signed.signature, &signed.decision_bytes)?,
|
||||
"REPAIR_HISTORICAL_SIGNATURE"
|
||||
);
|
||||
let d: ReconcileDecisionPayload = serde_json::from_slice(&signed.decision_bytes)?;
|
||||
ensure!(
|
||||
d.valid_profile(d.issued_at)
|
||||
&& d.key_id == hash(&raw)
|
||||
&& d.tenant_id == binding.tenant_id
|
||||
&& d.asset_id == binding.asset_id
|
||||
&& d.agent_id == binding.agent_id
|
||||
&& d.enrollment_generation == binding.enrollment_generation
|
||||
&& d.assignment_version == binding.assignment_version,
|
||||
"REPAIR_HISTORICAL_IDENTITY"
|
||||
);
|
||||
Ok(d)
|
||||
}
|
||||
pub fn verify_receipt(binding: &Binding, signed: &SignedReceipt) -> Result<Receipt> {
|
||||
ensure!(
|
||||
signed.payload.len() <= 16384 && signed.signature.len() == 64,
|
||||
"REPAIR_RECEIPT_SIZE"
|
||||
);
|
||||
let raw = hex::decode(&binding.public_key_hex)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&raw, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?.verify_oneshot(&signed.signature, &signed.payload)?,
|
||||
"REPAIR_RECEIPT_SIGNATURE"
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
ensure!(
|
||||
r.valid_profile(Utc::now())
|
||||
&& r.key_id == hash(&raw)
|
||||
&& r.tenant_id == binding.tenant_id
|
||||
&& r.asset_id == binding.asset_id
|
||||
&& r.agent_id == binding.agent_id
|
||||
&& r.enrollment_generation == binding.enrollment_generation
|
||||
&& r.assignment_version == binding.assignment_version,
|
||||
"REPAIR_RECEIPT_BINDING"
|
||||
);
|
||||
Ok(r)
|
||||
}
|
||||
/// Full database inventory with typed values; byte fields are bounded hashes.
|
||||
/// The append-only repair archive is excluded; only the repaired fault cell is
|
||||
/// normalized for exact idempotent replay of the same signed receipt.
|
||||
pub fn inventory(db: &Connection, meta: &str) -> Result<Value> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "REPAIR_DATABASE_INTEGRITY");
|
||||
let triggers:bool=db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') OR (type='table' AND upper(sql) LIKE '%VIRTUAL TABLE%'))",[],|r|r.get(0))?;
|
||||
ensure!(!triggers, "REPAIR_UNEXPECTED_DATABASE_PROGRAM");
|
||||
let mut q=db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name<>?1 ORDER BY name")?;
|
||||
let names = q
|
||||
.query_map([AUDIT], |r| r.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let mut tables = serde_json::Map::new();
|
||||
let mut count = 0;
|
||||
for name in names {
|
||||
ensure!(
|
||||
name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_'),
|
||||
"REPAIR_TABLE_NAME"
|
||||
);
|
||||
let mut s = db.prepare(&format!("SELECT * FROM \"{name}\" ORDER BY rowid"))?;
|
||||
let columns = s
|
||||
.column_names()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
let mut rows = vec![];
|
||||
let mut cursor = s.query([])?;
|
||||
while let Some(row) = cursor.next()? {
|
||||
count += 1;
|
||||
ensure!(count <= 200000, "REPAIR_INVENTORY_LIMIT");
|
||||
let mut cells = vec![];
|
||||
for (i, col) in columns.iter().enumerate() {
|
||||
let cell = if name == meta && col == "fault" {
|
||||
let v = row.get_ref(i)?;
|
||||
ensure!(
|
||||
matches!(v, ValueRef::Null)
|
||||
|| matches!(v, ValueRef::Text(b"CHECKPOINT_CHAIN_MISMATCH")),
|
||||
"REPAIR_DIFFERENT_FAULT"
|
||||
);
|
||||
json!({"type":"text","value":"CHECKPOINT_CHAIN_MISMATCH"})
|
||||
} else {
|
||||
match row.get_ref(i)? {
|
||||
ValueRef::Null => json!({"type":"null"}),
|
||||
ValueRef::Integer(v) => json!({"type":"integer","value":v}),
|
||||
ValueRef::Real(_) => anyhow::bail!("REPAIR_REAL_VALUE_UNSUPPORTED"),
|
||||
ValueRef::Text(v) => json!({"type":"text","value":std::str::from_utf8(v)?}),
|
||||
ValueRef::Blob(v) => {
|
||||
json!({"type":"blob","sha256":hash(v),"length":v.len()})
|
||||
}
|
||||
}
|
||||
};
|
||||
cells.push(cell);
|
||||
}
|
||||
rows.push(cells);
|
||||
}
|
||||
tables.insert(name, json!({"columns":columns,"rows":rows}));
|
||||
}
|
||||
Ok(Value::Object(tables))
|
||||
}
|
||||
pub fn already_applied(db: &Connection, signed: &SignedReceipt) -> Result<bool> {
|
||||
let exists: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
|
||||
[AUDIT],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if !exists {
|
||||
return Ok(false);
|
||||
}
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let old: Option<Vec<u8>> = db
|
||||
.query_row(
|
||||
"SELECT receipt FROM checkpoint_repair_receipts WHERE receipt_id=?1",
|
||||
[r.receipt_id.to_string()],
|
||||
|v| v.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some(old) = old {
|
||||
ensure!(
|
||||
old == serde_json::to_vec(signed)?,
|
||||
"REPAIR_RECEIPT_CONFLICT"
|
||||
);
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
pub fn commit_fault_repair(
|
||||
db: &mut Connection,
|
||||
meta: &str,
|
||||
signed: &SignedReceipt,
|
||||
expected_hash: &str,
|
||||
additional: Option<(&str, Value)>,
|
||||
) -> Result<()> {
|
||||
ensure!(
|
||||
matches!(meta, "reconcile_meta" | "helper_sync_meta"),
|
||||
"REPAIR_META_TABLE"
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let tx = db.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
let mut actual = inventory(&tx, meta)?;
|
||||
if let Some((key, value)) = additional {
|
||||
actual
|
||||
.as_object_mut()
|
||||
.context("REPAIR_INVENTORY_OBJECT")?
|
||||
.insert(key.into(), value);
|
||||
}
|
||||
ensure!(
|
||||
inventory_sha256(&actual)? == expected_hash,
|
||||
"REPAIR_WITNESS_CHANGED"
|
||||
);
|
||||
let fault: Option<String> =
|
||||
tx.query_row(&format!("SELECT fault FROM {meta} WHERE id=1"), [], |v| {
|
||||
v.get(0)
|
||||
})?;
|
||||
if already_applied(&tx, signed)? {
|
||||
ensure!(fault.is_none(), "REPAIR_REPLAY_FAULT_CHANGED");
|
||||
tx.commit()?;
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH"),
|
||||
"REPAIR_EXACT_FAULT_REQUIRED"
|
||||
);
|
||||
tx.execute_batch("CREATE TABLE IF NOT EXISTS checkpoint_repair_receipts(receipt_id TEXT PRIMARY KEY,receipt BLOB NOT NULL,old_fault TEXT NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)")?;
|
||||
tx.execute(
|
||||
"INSERT INTO checkpoint_repair_receipts VALUES(?1,?2,?3,?4,?5)",
|
||||
params![
|
||||
r.receipt_id.to_string(),
|
||||
serde_json::to_vec(signed)?,
|
||||
fault,
|
||||
expected_hash,
|
||||
Utc::now().to_rfc3339()
|
||||
],
|
||||
)?;
|
||||
ensure!(
|
||||
tx.execute(
|
||||
&format!(
|
||||
"UPDATE {meta} SET fault=NULL WHERE id=1 AND fault='CHECKPOINT_CHAIN_MISMATCH'"
|
||||
),
|
||||
[]
|
||||
)? == 1,
|
||||
"REPAIR_FAULT_CHANGED"
|
||||
);
|
||||
let mut after = inventory(&tx, meta)?;
|
||||
if let Some(value) = actual.get("agent_consumption") {
|
||||
after
|
||||
.as_object_mut()
|
||||
.context("REPAIR_INVENTORY_OBJECT")?
|
||||
.insert("agent_consumption".into(), value.clone());
|
||||
}
|
||||
ensure!(after == actual, "REPAIR_NONFAULT_STATE_CHANGED");
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn verify_helper(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
signed: &SignedReceipt,
|
||||
e: &Evidence,
|
||||
) -> Result<()> {
|
||||
let r = verify_receipt(binding, signed)?;
|
||||
ensure!(
|
||||
e.schema_version == 1
|
||||
&& e.tenant_id == binding.tenant_id
|
||||
&& e.asset_id == binding.asset_id
|
||||
&& e.agent_id == binding.agent_id
|
||||
&& e.enrollment_generation == binding.enrollment_generation
|
||||
&& e.assignment_version == binding.assignment_version,
|
||||
"REPAIR_HELPER_EVIDENCE_IDENTITY"
|
||||
);
|
||||
crate::reconcile::verify_checkpoint_repair_history(db, binding, Some((&r, e)))?;
|
||||
let inv = inventory(db, "helper_sync_meta")?;
|
||||
ensure!(
|
||||
inventory_sha256(&inv)? == r.helper_inventory_sha256
|
||||
&& inv == e.helper_inventory
|
||||
&& helper_ledger_sha256(&inv)? == r.helper_ledger_sha256,
|
||||
"REPAIR_HELPER_INVENTORY"
|
||||
);
|
||||
let (journal, checkpoint, decision, fault): (String, u64, String, Option<String>) = db
|
||||
.query_row(
|
||||
"SELECT journal_id,checkpoint,last_decision,fault FROM helper_sync_meta WHERE id=1",
|
||||
[],
|
||||
|v| Ok((v.get(0)?, v.get(1)?, v.get(2)?, v.get(3)?)),
|
||||
)?;
|
||||
ensure!(
|
||||
journal == r.helper_journal_id.to_string()
|
||||
&& checkpoint == r.helper_checkpoint
|
||||
&& decision == r.rejected_decision_id.to_string()
|
||||
&& (fault.as_deref() == Some("CHECKPOINT_CHAIN_MISMATCH")
|
||||
|| (fault.is_none() && already_applied(db, signed)?)),
|
||||
"REPAIR_HELPER_HEAD"
|
||||
);
|
||||
let (response, request): (Vec<u8>, Vec<u8>) = db.query_row(
|
||||
"SELECT response,request FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[decision],
|
||||
|v| Ok((v.get(0)?, v.get(1)?)),
|
||||
)?;
|
||||
let stored: SignedDecision = serde_json::from_slice(&response)?;
|
||||
ensure!(
|
||||
stored == e.rejected_response
|
||||
&& request == e.rejected_request
|
||||
&& hash(&request) == r.rejected_request_sha256
|
||||
&& hash(&stored.decision_bytes) == r.rejected_decision_sha256,
|
||||
"REPAIR_HELPER_REJECTION"
|
||||
);
|
||||
let rejection: ReconcileDecisionPayload = serde_json::from_slice(&stored.decision_bytes)?;
|
||||
ensure!(
|
||||
rejection.reason == r.fault
|
||||
&& !rejection.ready
|
||||
&& !rejection.helper_ready
|
||||
&& !rejection.attempts.iter().any(|v| v.unlock),
|
||||
"REPAIR_REJECTION_HAS_AUTHORITY"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use openssl::{pkey::Private, sign::Signer};
|
||||
use uuid::Uuid;
|
||||
fn fixture() -> (Binding, PKey<Private>, Receipt) {
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let id = Uuid::new_v4();
|
||||
let now = Utc::now();
|
||||
let binding = Binding {
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: hex::encode(key.raw_public_key().unwrap()),
|
||||
};
|
||||
let r = Receipt {
|
||||
domain: "rmm.sync.unsent-request-repair".into(),
|
||||
schema_version: 1,
|
||||
receipt_id: Uuid::new_v4(),
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
issued_at: now,
|
||||
valid_until: now + chrono::Duration::minutes(15),
|
||||
key_id: hash(&key.raw_public_key().unwrap()),
|
||||
fault: "CHECKPOINT_CHAIN_MISMATCH".into(),
|
||||
agent_journal_id: id,
|
||||
agent_checkpoint: 13,
|
||||
helper_journal_id: id,
|
||||
helper_checkpoint: 22,
|
||||
rejected_decision_id: id,
|
||||
rejected_decision_sha256: hash(b"denied"),
|
||||
rejected_request_id: id,
|
||||
rejected_request_sha256: hash(b"request"),
|
||||
uncommitted_request_id: id,
|
||||
uncommitted_request_sha256: hash(b"unsent"),
|
||||
previous_valid_decision_id: id,
|
||||
previous_valid_decision_sha256: hash(b"previous"),
|
||||
agent_inventory_sha256: hash(b"agent"),
|
||||
helper_inventory_sha256: hash(b"helper"),
|
||||
helper_ledger_sha256: hash(b"ledger"),
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
unlock: false,
|
||||
};
|
||||
(binding, key, r)
|
||||
}
|
||||
fn sign(key: &PKey<Private>, r: &Receipt) -> SignedReceipt {
|
||||
let payload = serde_json::to_vec(r).unwrap();
|
||||
let signature = Signer::new_without_digest(key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&payload)
|
||||
.unwrap();
|
||||
SignedReceipt { payload, signature }
|
||||
}
|
||||
#[test]
|
||||
fn only_exact_identity_signed_no_authority_short_lived_receipts_verify() {
|
||||
let (binding, key, r) = fixture();
|
||||
assert!(verify_receipt(&binding, &sign(&key, &r)).is_ok());
|
||||
let mut bad = sign(&key, &r);
|
||||
bad.payload[0] ^= 1;
|
||||
assert!(verify_receipt(&binding, &bad).is_err());
|
||||
for changed in [0, 1, 2, 3, 4, 5] {
|
||||
let mut bad = r.clone();
|
||||
match changed {
|
||||
0 => bad.ready = true,
|
||||
1 => bad.helper_ready = true,
|
||||
2 => bad.unlock = true,
|
||||
3 => bad.agent_id = Uuid::new_v4(),
|
||||
4 => bad.domain = "rmm.execution-permit".into(),
|
||||
_ => bad.valid_until = bad.issued_at + chrono::Duration::minutes(16),
|
||||
};
|
||||
assert!(verify_receipt(&binding, &sign(&key, &bad)).is_err());
|
||||
}
|
||||
let mut expired = r.clone();
|
||||
expired.issued_at = Utc::now() - chrono::Duration::minutes(2);
|
||||
expired.valid_until = Utc::now() - chrono::Duration::minutes(1);
|
||||
assert!(verify_receipt(&binding, &sign(&key, &expired)).is_err());
|
||||
}
|
||||
fn database() -> Connection {
|
||||
let db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE helper_sync_meta(id INTEGER PRIMARY KEY,fault TEXT);INSERT INTO helper_sync_meta VALUES(1,'CHECKPOINT_CHAIN_MISMATCH');CREATE TABLE consumed(grant_id TEXT PRIMARY KEY,result BLOB);INSERT INTO consumed VALUES('old-grant',NULL);CREATE TABLE execution_evidence(grant_id TEXT PRIMARY KEY,outcome BLOB);INSERT INTO execution_evidence VALUES('old-grant',x'756e6b6e6f776e');").unwrap();
|
||||
db
|
||||
}
|
||||
#[test]
|
||||
fn repair_archives_exact_fault_without_changing_unknown_or_consumption_and_replays() {
|
||||
let (_, key, mut r) = fixture();
|
||||
let mut db = database();
|
||||
let before = inventory(&db, "helper_sync_meta").unwrap();
|
||||
r.helper_inventory_sha256 = inventory_sha256(&before).unwrap();
|
||||
r.helper_ledger_sha256 = helper_ledger_sha256(&before).unwrap();
|
||||
let signed = sign(&key, &r);
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(inventory(&db, "helper_sync_meta").unwrap(), before);
|
||||
assert!(
|
||||
db.query_row("SELECT result IS NULL FROM consumed", [], |v| v
|
||||
.get::<_, bool>(0))
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT outcome FROM execution_evidence", [], |v| v
|
||||
.get::<_, Vec<u8>>(0))
|
||||
.unwrap(),
|
||||
b"unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row(
|
||||
"SELECT old_fault FROM checkpoint_repair_receipts",
|
||||
[],
|
||||
|v| v.get::<_, String>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
"CHECKPOINT_CHAIN_MISMATCH"
|
||||
);
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM checkpoint_repair_receipts", [], |v| v
|
||||
.get::<_, u64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
db.execute("UPDATE consumed SET result=x'01'", []).unwrap();
|
||||
assert!(
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn changed_witness_or_other_fault_is_never_cleared() {
|
||||
let (_, key, r) = fixture();
|
||||
let mut db = database();
|
||||
let signed = sign(&key, &r);
|
||||
assert!(
|
||||
commit_fault_repair(
|
||||
&mut db,
|
||||
"helper_sync_meta",
|
||||
&signed,
|
||||
&r.helper_inventory_sha256,
|
||||
None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
db.execute(
|
||||
"UPDATE helper_sync_meta SET fault='HELPER_CHECKPOINT_ROLLBACK'",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(inventory(&db, "helper_sync_meta").is_err());
|
||||
assert!(!already_applied(&db, &signed).unwrap());
|
||||
let db = database();
|
||||
db.execute_batch("CREATE TRIGGER hidden_effect AFTER UPDATE ON helper_sync_meta BEGIN UPDATE consumed SET result=x'01'; END;").unwrap();
|
||||
assert!(inventory(&db, "helper_sync_meta").is_err());
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
extern crate self as rmm_privileged_helper;
|
||||
pub mod apt;
|
||||
pub mod apt_reconcile;
|
||||
pub mod checkpoint_repair;
|
||||
pub mod control;
|
||||
pub mod readiness;
|
||||
pub mod reconcile;
|
||||
|
||||
@@ -681,6 +681,142 @@ fn validate_history(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Bootstrap-only historical verification; this never creates a lease.
|
||||
pub fn verify_checkpoint_repair_history(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
links: Option<(
|
||||
&rmm_domain::checkpoint_repair::Receipt,
|
||||
&rmm_domain::checkpoint_repair::Evidence,
|
||||
)>,
|
||||
) -> Result<()> {
|
||||
let meta = metadata(db)?;
|
||||
validate_history(db, binding, &meta, meta.generation)?;
|
||||
let mut statement = db.prepare(
|
||||
"SELECT response,request,checkpoint FROM helper_sync_decisions ORDER BY checkpoint",
|
||||
)?;
|
||||
let entries = statement
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, Vec<u8>>(0)?,
|
||||
r.get::<_, Vec<u8>>(1)?,
|
||||
r.get::<_, u64>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ensure!(entries.len() <= 100000, "REPAIR_HISTORY_LIMIT");
|
||||
let mut previous = None;
|
||||
let mut previous_checkpoint = 0;
|
||||
let mut previous_agent_checkpoint = 0;
|
||||
for (signed, raw, checkpoint) in entries {
|
||||
let response: SignedResponse = serde_json::from_slice(&signed)?;
|
||||
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
|
||||
let decision = verify(binding, &response, parsed.issued_at)?;
|
||||
let request = RequestView::decode(&raw)?;
|
||||
let captured: Vec<u8> = db.query_row(
|
||||
"SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?1",
|
||||
[checkpoint],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let capture: Snapshot = serde_json::from_slice(&captured)?;
|
||||
ensure!(
|
||||
decision.helper_journal_id == Some(meta.journal_id)
|
||||
&& decision.helper_checkpoint == checkpoint
|
||||
&& request.helper_journal_id == meta.journal_id.to_string()
|
||||
&& request.helper_checkpoint == checkpoint
|
||||
&& request.assignment_version == binding.assignment_version
|
||||
&& request.journal_complete
|
||||
&& decision.request_hash == hash(&raw)
|
||||
&& decision.request_id.to_string() == request.request_id
|
||||
&& decision.journal_id.to_string() == request.journal_id
|
||||
&& decision.checkpoint == request.checkpoint
|
||||
&& capture.journal_id == meta.journal_id
|
||||
&& capture.checkpoint == checkpoint
|
||||
&& capture.boot_id == decision.boot_id
|
||||
&& capture.captured_at <= decision.issued_at
|
||||
&& capture.previous_decision_id == previous
|
||||
&& checkpoint > previous_checkpoint
|
||||
&& decision.checkpoint >= previous_agent_checkpoint
|
||||
&& decision.connection_generation <= meta.generation
|
||||
&& request.helper_previous_decision_id
|
||||
== capture
|
||||
.previous_decision_id
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default(),
|
||||
"REPAIR_HELPER_HISTORY_BINDING"
|
||||
);
|
||||
previous = Some(decision.decision_id);
|
||||
previous_checkpoint = checkpoint;
|
||||
previous_agent_checkpoint = decision.checkpoint;
|
||||
}
|
||||
let _ = load_rows(db, binding, false)?;
|
||||
if let Some((r, e)) = links {
|
||||
verify_checkpoint_repair_links(db, binding, r, e)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn verify_checkpoint_repair_links(
|
||||
db: &Connection,
|
||||
binding: &Binding,
|
||||
r: &rmm_domain::checkpoint_repair::Receipt,
|
||||
e: &rmm_domain::checkpoint_repair::Evidence,
|
||||
) -> Result<()> {
|
||||
let x = RequestView::decode(&e.uncommitted_request)?;
|
||||
let y = RequestView::decode(&e.rejected_request)?;
|
||||
let a: ReconcileDecisionPayload =
|
||||
serde_json::from_slice(&e.previous_valid_response.decision_bytes)?;
|
||||
let a = verify(
|
||||
binding,
|
||||
&SignedResponse {
|
||||
decision_bytes: e.previous_valid_response.decision_bytes.clone(),
|
||||
signature: e.previous_valid_response.signature.clone(),
|
||||
},
|
||||
a.issued_at,
|
||||
)?;
|
||||
let old: Vec<u8> = db.query_row(
|
||||
"SELECT response FROM helper_sync_decisions WHERE decision_id=?1",
|
||||
[a.decision_id.to_string()],
|
||||
|v| v.get(0),
|
||||
)?;
|
||||
let old: SignedResponse = serde_json::from_slice(&old)?;
|
||||
ensure!(
|
||||
old.decision_bytes == e.previous_valid_response.decision_bytes
|
||||
&& old.signature == e.previous_valid_response.signature
|
||||
&& a.decision_id == r.previous_valid_decision_id
|
||||
&& hash(&old.decision_bytes) == r.previous_valid_decision_sha256
|
||||
&& a.reason == "ATTEMPT_RECONCILIATION_REQUIRED"
|
||||
&& !a.ready
|
||||
&& !a.helper_ready
|
||||
&& x.request_id == r.uncommitted_request_id.to_string()
|
||||
&& hash(&e.uncommitted_request) == r.uncommitted_request_sha256
|
||||
&& y.request_id == r.rejected_request_id.to_string()
|
||||
&& hash(&e.rejected_request) == r.rejected_request_sha256
|
||||
&& x.previous_decision_id == a.decision_id.to_string()
|
||||
&& x.previous_request_id.is_empty()
|
||||
&& x.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.helper_previous_decision_id == a.decision_id.to_string()
|
||||
&& y.previous_decision_id.is_empty()
|
||||
&& y.previous_request_id == x.request_id
|
||||
&& x.journal_id == r.agent_journal_id.to_string()
|
||||
&& y.journal_id == x.journal_id
|
||||
&& a.journal_id == r.agent_journal_id
|
||||
&& x.helper_journal_id == r.helper_journal_id.to_string()
|
||||
&& y.helper_journal_id == x.helper_journal_id
|
||||
&& a.helper_journal_id == Some(r.helper_journal_id)
|
||||
&& a.checkpoint < x.checkpoint
|
||||
&& x.checkpoint < y.checkpoint
|
||||
&& y.checkpoint == r.agent_checkpoint
|
||||
&& a.helper_checkpoint < x.helper_checkpoint
|
||||
&& x.helper_checkpoint < y.helper_checkpoint
|
||||
&& y.helper_checkpoint == r.helper_checkpoint
|
||||
&& x.assignment_version == binding.assignment_version
|
||||
&& y.assignment_version == binding.assignment_version
|
||||
&& x.journal_complete
|
||||
&& y.journal_complete,
|
||||
"REPAIR_EXACT_UNSENT_CHAIN_REQUIRED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn persist_fault(db: &Connection, error: &anyhow::Error) {
|
||||
let text = error.to_string();
|
||||
if [
|
||||
@@ -991,6 +1127,8 @@ struct RequestView {
|
||||
helper_journal_id: String,
|
||||
helper_checkpoint: u64,
|
||||
helper_previous_decision_id: String,
|
||||
previous_decision_id: String,
|
||||
previous_request_id: String,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct AttemptView {
|
||||
@@ -1089,6 +1227,11 @@ impl RequestView {
|
||||
if !s.is_empty() {
|
||||
Uuid::parse_str(&s)?;
|
||||
}
|
||||
if tag == 4 {
|
||||
r.previous_decision_id = s;
|
||||
} else {
|
||||
r.previous_request_id = s;
|
||||
}
|
||||
}
|
||||
5 => r.assignment_version = number(value)?,
|
||||
6 => {
|
||||
|
||||
@@ -39,6 +39,32 @@ struct Fixture {
|
||||
boot: TestBoot,
|
||||
agent_journal: Uuid,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_repair_verifies_complete_signed_history_and_detects_removed_predecessor() {
|
||||
let mut f = Fixture::new();
|
||||
let mut sync = f.sync();
|
||||
let first = f.snapshot(&mut sync, 7);
|
||||
let request = f.request(&first);
|
||||
let d = f.decision(&first, &request, 7, false);
|
||||
let signed = f.sign(&d);
|
||||
sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot)
|
||||
.unwrap();
|
||||
f.clock.now += Duration::seconds(1);
|
||||
let second = f.snapshot(&mut sync, 7);
|
||||
let request = f.request(&second);
|
||||
let d = f.decision(&second, &request, 7, false);
|
||||
let signed = f.sign(&d);
|
||||
sync.apply(&mut f.db, &f.guard, &signed, &request, &f.clock, &f.boot)
|
||||
.unwrap();
|
||||
verify_checkpoint_repair_history(&f.db, &f.binding, None).unwrap();
|
||||
f.db.execute(
|
||||
"DELETE FROM helper_sync_decisions WHERE checkpoint=?1",
|
||||
[first.checkpoint],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(verify_checkpoint_repair_history(&f.db, &f.binding, None).is_err());
|
||||
}
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.dir);
|
||||
|
||||
@@ -38,6 +38,35 @@ fn response(messages: Vec<Envelope>) -> Response {
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
async fn append_work(
|
||||
s: &crate::State,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
source: &Envelope,
|
||||
payload: Payload,
|
||||
generation: u64,
|
||||
replies: &mut Vec<Envelope>,
|
||||
) -> anyhow::Result<()> {
|
||||
let reply = crate::agent_service::reply(source, payload, generation);
|
||||
// Preserve the protocol's 16-envelope response limit. An offer that cannot
|
||||
// fit twice is delivered normally without consuming the optional fault.
|
||||
let claim = if replies.len() <= 14 {
|
||||
crate::gateway::dev_delivery_faults::duplicate_offer(
|
||||
&s.pool, tenant, agent, &reply, "https",
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if claim.is_some() {
|
||||
replies.push(reply.clone());
|
||||
}
|
||||
replies.push(reply);
|
||||
if let Some(claim) = claim {
|
||||
crate::gateway::dev_delivery_faults::duplicate_emitted(&s.pool, claim).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub async fn messages(
|
||||
State(s): State<crate::State>,
|
||||
Extension(peer): Extension<crate::Peer>,
|
||||
@@ -83,13 +112,21 @@ pub async fn messages(
|
||||
}
|
||||
.map_err(|_| StatusCode::CONFLICT)?;
|
||||
generation = reply.connection_generation;
|
||||
replies.push(reply);
|
||||
if !crate::gateway::dev_delivery_faults::after_ingest(&s.pool, t, a, &e, &reply, "https")
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
{
|
||||
replies.push(reply);
|
||||
}
|
||||
}
|
||||
if let Some(work) = crate::job_gateway::next(&s, t, a, generation)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
if replies.len() < 16
|
||||
&& let Some(work) = crate::job_gateway::next(&s, t, a, generation)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
{
|
||||
replies.push(crate::agent_service::reply(&last, work, generation));
|
||||
append_work(&s, t, a, &last, work, generation, &mut replies)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
|
||||
}
|
||||
Ok(response(replies))
|
||||
}
|
||||
@@ -119,10 +156,11 @@ pub async fn poll(
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
{
|
||||
return Ok(response(vec![
|
||||
ack,
|
||||
crate::agent_service::reply(&e, work, e.connection_generation),
|
||||
]));
|
||||
let mut replies = vec![ack];
|
||||
append_work(&s, t, a, &e, work, e.connection_generation, &mut replies)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
|
||||
return Ok(response(replies));
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Ok(response(vec![ack]));
|
||||
|
||||
@@ -0,0 +1,772 @@
|
||||
//! Fixed development bootstrap for a proven unsent sync request. This promotes
|
||||
//! an existing signed denial only; it grants no readiness or execution rights.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{Duration, Utc};
|
||||
use openssl::{
|
||||
pkey::{Id, PKey},
|
||||
sign::Verifier,
|
||||
};
|
||||
use prost::Message;
|
||||
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
|
||||
use rmm_protocol::SyncRequest;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::Row;
|
||||
use std::{
|
||||
fs,
|
||||
os::unix::fs::{MetadataExt, PermissionsExt},
|
||||
path::Path,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
const EVENT: &str = "unsent_request_repair_issued";
|
||||
const FAULT: &str = "CHECKPOINT_CHAIN_MISMATCH";
|
||||
|
||||
pub fn closed_policy() -> Result<()> {
|
||||
ensure!(
|
||||
std::env::var("RMM_PROFILE").as_deref() == Ok("development")
|
||||
&& std::env::var("RMM_DEV_AUTH").as_deref() == Ok("true")
|
||||
&& std::env::var("RMM_SERVER_ROLE").as_deref() == Ok("api"),
|
||||
"REPAIR_DEVELOPMENT_API_ONLY"
|
||||
);
|
||||
let path = std::env::var("RMM_DEV_ROOT_POLICY_FILE").context("REPAIR_POLICY_REQUIRED")?;
|
||||
let m = fs::symlink_metadata(&path)?;
|
||||
let parent = fs::symlink_metadata(Path::new(&path).parent().context("REPAIR_POLICY_PARENT")?)?;
|
||||
let key = std::env::var("RMM_JOB_KEY_FILE").context("REPAIR_EXISTING_KEY_REQUIRED")?;
|
||||
ensure!(
|
||||
m.is_file()
|
||||
&& m.len() <= 65536
|
||||
&& m.permissions().mode() & 0o077 == 0
|
||||
&& m.uid() == fs::metadata(key)?.uid()
|
||||
&& parent.is_dir()
|
||||
&& parent.permissions().mode() & 0o022 == 0,
|
||||
"REPAIR_POLICY_PERMISSIONS"
|
||||
);
|
||||
let p: Value = serde_json::from_slice(&fs::read(path)?)?;
|
||||
ensure!(
|
||||
p["schema_version"] == 1
|
||||
&& p["profile"] == "development"
|
||||
&& p["expires_at"]
|
||||
.as_str()
|
||||
.and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
|
||||
.is_some_and(|until| until <= Utc::now()),
|
||||
"REPAIR_POLICY_MUST_BE_CLOSED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cell<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a Value> {
|
||||
let columns = table["columns"]
|
||||
.as_array()
|
||||
.context("REPAIR_INVENTORY_COLUMNS")?;
|
||||
let i = columns
|
||||
.iter()
|
||||
.position(|v| v.as_str() == Some(column))
|
||||
.context("REPAIR_INVENTORY_COLUMN")?;
|
||||
row.as_array()
|
||||
.and_then(|r| r.get(i))
|
||||
.context("REPAIR_INVENTORY_CELL")
|
||||
}
|
||||
fn rows(table: &Value) -> Result<&Vec<Value>> {
|
||||
table["rows"].as_array().context("REPAIR_INVENTORY_ROWS")
|
||||
}
|
||||
fn text<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a str> {
|
||||
let v = cell(table, row, column)?;
|
||||
ensure!(v["type"] == "text", "REPAIR_CELL_TYPE");
|
||||
v["value"].as_str().context("REPAIR_CELL_TEXT")
|
||||
}
|
||||
fn integer(table: &Value, row: &Value, column: &str) -> Result<u64> {
|
||||
let v = cell(table, row, column)?;
|
||||
ensure!(v["type"] == "integer", "REPAIR_CELL_TYPE");
|
||||
v["value"].as_u64().context("REPAIR_CELL_INTEGER")
|
||||
}
|
||||
fn blob(table: &Value, row: &Value, column: &str, bytes: &[u8]) -> Result<()> {
|
||||
let v = cell(table, row, column)?;
|
||||
ensure!(
|
||||
v == &json!({"type":"blob","sha256":hash(bytes),"length":bytes.len()}),
|
||||
"REPAIR_WITNESS_BLOB"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn row_by<'a>(table: &'a Value, column: &str, id: &str) -> Result<&'a Value> {
|
||||
let found = rows(table)?
|
||||
.iter()
|
||||
.filter(|row| text(table, row, column).ok() == Some(id))
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(found.len() == 1, "REPAIR_WITNESS_ROW");
|
||||
Ok(found[0])
|
||||
}
|
||||
fn decision(
|
||||
signer: &crate::job_signer::JobSigner,
|
||||
s: &SignedDecision,
|
||||
) -> Result<ReconcileDecisionPayload> {
|
||||
ensure!(
|
||||
s.decision_bytes.len() <= 1024 * 1024 && s.signature.len() == 64,
|
||||
"REPAIR_DECISION_SIZE"
|
||||
);
|
||||
let key = PKey::public_key_from_raw_bytes(&signer.public_key()?, Id::ED25519)?;
|
||||
ensure!(
|
||||
Verifier::new_without_digest(&key)?.verify_oneshot(&s.signature, &s.decision_bytes)?,
|
||||
"REPAIR_DECISION_SIGNATURE"
|
||||
);
|
||||
let d: ReconcileDecisionPayload = serde_json::from_slice(&s.decision_bytes)?;
|
||||
ensure!(
|
||||
d.valid_profile(d.issued_at) && d.key_id == signer.key_id,
|
||||
"REPAIR_DECISION_PROFILE"
|
||||
);
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
struct Case {
|
||||
old: ReconcileDecisionPayload,
|
||||
rejected: ReconcileDecisionPayload,
|
||||
unsent: SyncRequest,
|
||||
request: SyncRequest,
|
||||
}
|
||||
fn validate(signer: &crate::job_signer::JobSigner, e: &Evidence) -> Result<Case> {
|
||||
ensure!(
|
||||
e.schema_version == 1
|
||||
&& e.uncommitted_request.len() <= 1024 * 1024
|
||||
&& e.rejected_request.len() <= 1024 * 1024,
|
||||
"REPAIR_EVIDENCE_PROFILE"
|
||||
);
|
||||
let old = decision(signer, &e.previous_valid_response)?;
|
||||
let rejected = decision(signer, &e.rejected_response)?;
|
||||
for d in [&old, &rejected] {
|
||||
ensure!(
|
||||
d.tenant_id == e.tenant_id
|
||||
&& d.asset_id == e.asset_id
|
||||
&& d.agent_id == e.agent_id
|
||||
&& d.enrollment_generation == e.enrollment_generation
|
||||
&& d.assignment_version == e.assignment_version,
|
||||
"REPAIR_IDENTITY_SCOPE"
|
||||
);
|
||||
}
|
||||
let unsent = SyncRequest::decode(e.uncommitted_request.as_slice())?;
|
||||
let request = SyncRequest::decode(e.rejected_request.as_slice())?;
|
||||
let unsent_id = Uuid::parse_str(&unsent.request_id)?;
|
||||
ensure!(
|
||||
!unsent_id.is_nil()
|
||||
&& unsent_id != rejected.request_id
|
||||
&& unsent_id != old.request_id
|
||||
&& old.decision_id != rejected.decision_id
|
||||
&& rejected.request_id.to_string() == request.request_id
|
||||
&& rejected.request_hash == hash(&e.rejected_request),
|
||||
"REPAIR_REQUEST_BINDING"
|
||||
);
|
||||
ensure!(
|
||||
rejected.reason == FAULT
|
||||
&& !rejected.ready
|
||||
&& !rejected.helper_ready
|
||||
&& !rejected.attempts.iter().any(|a| a.unlock),
|
||||
"REPAIR_DENIAL_ONLY"
|
||||
);
|
||||
ensure!(
|
||||
request.previous_decision_id.is_empty()
|
||||
&& request.previous_request_id == unsent.request_id
|
||||
&& unsent.previous_decision_id == old.decision_id.to_string()
|
||||
&& unsent.previous_request_id.is_empty(),
|
||||
"REPAIR_EXACT_UNSENT_CHAIN"
|
||||
);
|
||||
let helper = rejected
|
||||
.helper_journal_id
|
||||
.context("REPAIR_HELPER_REQUIRED")?;
|
||||
ensure!(
|
||||
old.journal_id == rejected.journal_id
|
||||
&& old.helper_journal_id == Some(helper)
|
||||
&& old.checkpoint < unsent.checkpoint
|
||||
&& unsent.checkpoint < rejected.checkpoint
|
||||
&& old.helper_checkpoint <= unsent.helper_checkpoint
|
||||
&& unsent.helper_checkpoint <= rejected.helper_checkpoint
|
||||
&& old.connection_generation <= rejected.connection_generation,
|
||||
"REPAIR_MONOTONIC_CHAIN"
|
||||
);
|
||||
for r in [&unsent, &request] {
|
||||
ensure!(
|
||||
r.journal_complete
|
||||
&& r.assignment_version == e.assignment_version
|
||||
&& r.journal_id == rejected.journal_id.to_string()
|
||||
&& r.helper_journal_id == helper.to_string(),
|
||||
"REPAIR_REQUEST_SCOPE"
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
request.checkpoint == rejected.checkpoint
|
||||
&& request.helper_checkpoint == rejected.helper_checkpoint
|
||||
&& request.helper_previous_decision_id == old.decision_id.to_string(),
|
||||
"REPAIR_REQUEST_CHECKPOINT"
|
||||
);
|
||||
let am = &e.agent_inventory["reconcile_meta"];
|
||||
let a = rows(am)?;
|
||||
ensure!(a.len() == 1, "REPAIR_AGENT_META");
|
||||
let a = &a[0];
|
||||
ensure!(
|
||||
integer(am, a, "id")? == 1
|
||||
&& text(am, a, "journal_id")? == rejected.journal_id.to_string()
|
||||
&& integer(am, a, "checkpoint")? == rejected.checkpoint
|
||||
&& integer(am, a, "high_sequence")? >= rejected.checkpoint
|
||||
&& integer(am, a, "generation")? == rejected.connection_generation
|
||||
&& text(am, a, "last_decision")? == rejected.decision_id.to_string()
|
||||
&& text(am, a, "helper_id")? == helper.to_string()
|
||||
&& integer(am, a, "helper_checkpoint")? == rejected.helper_checkpoint
|
||||
&& text(am, a, "fault")? == FAULT
|
||||
&& cell(am, a, "pending_request")? == &json!({"type":"null"}),
|
||||
"REPAIR_AGENT_HEAD"
|
||||
);
|
||||
let hm = &e.helper_inventory["helper_sync_meta"];
|
||||
let h = rows(hm)?;
|
||||
ensure!(h.len() == 1, "REPAIR_HELPER_META");
|
||||
let h = &h[0];
|
||||
ensure!(
|
||||
integer(hm, h, "id")? == 1
|
||||
&& text(hm, h, "journal_id")? == helper.to_string()
|
||||
&& integer(hm, h, "checkpoint")? == rejected.helper_checkpoint
|
||||
&& text(hm, h, "last_decision")? == rejected.decision_id.to_string()
|
||||
&& text(hm, h, "agent_journal_id")? == rejected.journal_id.to_string()
|
||||
&& integer(hm, h, "agent_checkpoint")? == rejected.checkpoint
|
||||
&& text(hm, h, "fault")? == FAULT,
|
||||
"REPAIR_HELPER_HEAD"
|
||||
);
|
||||
let ar = &e.agent_inventory["reconcile_requests"];
|
||||
let x = row_by(ar, "id", &unsent.request_id)?;
|
||||
blob(ar, x, "request", &e.uncommitted_request)?;
|
||||
ensure!(
|
||||
integer(ar, x, "generation")? <= rejected.connection_generation
|
||||
&& cell(ar, x, "response")? == &json!({"type":"null"}),
|
||||
"REPAIR_UNSENT_LOCAL_RESPONSE"
|
||||
);
|
||||
let y = row_by(ar, "id", &request.request_id)?;
|
||||
blob(ar, y, "request", &e.rejected_request)?;
|
||||
blob(
|
||||
ar,
|
||||
y,
|
||||
"response",
|
||||
&rmm_protocol::SyncResponse {
|
||||
decision_bytes: e.rejected_response.decision_bytes.clone(),
|
||||
signature: e.rejected_response.signature.clone(),
|
||||
}
|
||||
.encode_to_vec(),
|
||||
)?;
|
||||
ensure!(
|
||||
integer(ar, y, "generation")? == rejected.connection_generation,
|
||||
"REPAIR_LOCAL_GENERATION"
|
||||
);
|
||||
let hr = &e.helper_inventory["helper_sync_decisions"];
|
||||
let y = row_by(hr, "decision_id", &rejected.decision_id.to_string())?;
|
||||
blob(hr, y, "request", &e.rejected_request)?;
|
||||
blob(
|
||||
hr,
|
||||
y,
|
||||
"response",
|
||||
&serde_json::to_vec(&e.rejected_response)?,
|
||||
)?;
|
||||
ensure!(
|
||||
integer(hr, y, "checkpoint")? == rejected.helper_checkpoint,
|
||||
"REPAIR_HELPER_CAPTURE"
|
||||
);
|
||||
ensure!(
|
||||
e.helper_inventory["consumed"].is_object()
|
||||
&& e.helper_inventory["execution_evidence"].is_object(),
|
||||
"REPAIR_LEDGER_REQUIRED"
|
||||
);
|
||||
Ok(Case {
|
||||
old,
|
||||
rejected,
|
||||
unsent,
|
||||
request,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn issue(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
|
||||
closed_policy()?;
|
||||
issue_with_closed_policy(state, e).await
|
||||
}
|
||||
async fn issue_with_closed_policy(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
|
||||
let c = validate(&state.job_signer, &e)?;
|
||||
let evidence_hash = inventory_sha256(&serde_json::to_value(&e)?)?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&state.pool, e.tenant_id).await?;
|
||||
crate::agent_service::control_lock(&mut tx, e.tenant_id, e.agent_id).await?;
|
||||
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,revoked_at,sequence_regressed,sync_recovery_required,sync_ready,helper_sync_ready FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
|
||||
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_DEVICE_MISSING")?;
|
||||
ensure!(
|
||||
device.get::<Uuid, _>("asset_id") == e.asset_id
|
||||
&& device.get::<i64, _>("enrollment_generation")
|
||||
== i64::try_from(e.enrollment_generation)?
|
||||
&& device.get::<i64, _>("assignment_version") == i64::try_from(e.assignment_version)?
|
||||
// Faulted journals may survive newer ClientHello channels. This
|
||||
// historical receipt grants no lease; the next ordinary sync must
|
||||
// still bind its current channel. A regressed generation or changed
|
||||
// boot/identity is never repaired here.
|
||||
&& device.get::<i64, _>("connection_generation")
|
||||
>= i64::try_from(c.rejected.connection_generation)?
|
||||
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(c.rejected.boot_id)
|
||||
&& device
|
||||
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
|
||||
.is_none()
|
||||
&& ![
|
||||
"sequence_regressed",
|
||||
"sync_recovery_required",
|
||||
"sync_ready",
|
||||
"helper_sync_ready"
|
||||
]
|
||||
.iter()
|
||||
.any(|k| device.get::<bool, _>(*k)),
|
||||
"REPAIR_DEVICE_FENCED"
|
||||
);
|
||||
for (d, s) in [
|
||||
(&c.old, &e.previous_valid_response),
|
||||
(&c.rejected, &e.rejected_response),
|
||||
] {
|
||||
let stored=sqlx::query("SELECT decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3")
|
||||
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).fetch_optional(&mut *tx).await?.context("REPAIR_STORED_DECISION_MISSING")?;
|
||||
ensure!(
|
||||
stored.get::<Uuid, _>("decision_id") == d.decision_id
|
||||
&& stored.get::<String, _>("request_hash") == d.request_hash
|
||||
&& stored.get::<Uuid, _>("journal_id") == d.journal_id
|
||||
&& stored.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
|
||||
&& stored.get::<i64, _>("connection_generation")
|
||||
== i64::try_from(d.connection_generation)?
|
||||
&& stored.get::<Vec<u8>, _>("payload") == s.decision_bytes
|
||||
&& stored.get::<Vec<u8>, _>("signature") == s.signature,
|
||||
"REPAIR_STORED_DECISION_MISMATCH"
|
||||
);
|
||||
}
|
||||
let committed:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3)")
|
||||
.bind(e.tenant_id).bind(e.agent_id).bind(Uuid::parse_str(&c.unsent.request_id)?).fetch_one(&mut *tx).await?;
|
||||
ensure!(!committed, "REPAIR_REQUEST_WAS_COMMITTED");
|
||||
let head=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
|
||||
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_HEAD_MISSING")?;
|
||||
let previous:Option<Value>=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type=$2 AND data->>'evidence_sha256'=$3 AND data->>'agent_id'=$4 ORDER BY created_at DESC LIMIT 1")
|
||||
.bind(e.tenant_id).bind(EVENT).bind(&evidence_hash).bind(e.agent_id.to_string()).fetch_optional(&mut *tx).await?;
|
||||
if let Some(previous) = previous {
|
||||
let signed: SignedReceipt = serde_json::from_value(previous["receipt"].clone())?;
|
||||
let receipt: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
let key = PKey::public_key_from_raw_bytes(&state.job_signer.public_key()?, Id::ED25519)?;
|
||||
ensure!(
|
||||
signed.signature.len() == 64
|
||||
&& Verifier::new_without_digest(&key)?
|
||||
.verify_oneshot(&signed.signature, &signed.payload)?
|
||||
&& receipt.valid_profile(Utc::now())
|
||||
&& receipt.key_id == state.job_signer.key_id
|
||||
&& receipt.tenant_id == e.tenant_id
|
||||
&& receipt.asset_id == e.asset_id
|
||||
&& receipt.agent_id == e.agent_id
|
||||
&& receipt.rejected_decision_id == c.rejected.decision_id
|
||||
&& receipt.rejected_request_sha256 == hash(&e.rejected_request)
|
||||
&& receipt.uncommitted_request_sha256 == hash(&e.uncommitted_request)
|
||||
&& receipt.agent_inventory_sha256 == inventory_sha256(&e.agent_inventory)?
|
||||
&& receipt.helper_inventory_sha256 == inventory_sha256(&e.helper_inventory)?
|
||||
&& head_matches(&head, &c.rejected)?,
|
||||
"REPAIR_REPLAY_EXPIRED_OR_HEAD_CHANGED"
|
||||
);
|
||||
tx.commit().await?;
|
||||
return Ok(signed);
|
||||
}
|
||||
ensure!(
|
||||
head_matches(&head, &c.old)?,
|
||||
"REPAIR_AUTHORITATIVE_HEAD_CHANGED"
|
||||
);
|
||||
let now = Utc::now();
|
||||
let receipt = Receipt {
|
||||
domain: "rmm.sync.unsent-request-repair".into(),
|
||||
schema_version: 1,
|
||||
receipt_id: Uuid::new_v4(),
|
||||
tenant_id: e.tenant_id,
|
||||
asset_id: e.asset_id,
|
||||
agent_id: e.agent_id,
|
||||
enrollment_generation: e.enrollment_generation,
|
||||
assignment_version: e.assignment_version,
|
||||
issued_at: now,
|
||||
valid_until: now + Duration::minutes(10),
|
||||
key_id: state.job_signer.key_id.clone(),
|
||||
fault: FAULT.into(),
|
||||
agent_journal_id: c.rejected.journal_id,
|
||||
agent_checkpoint: c.rejected.checkpoint,
|
||||
helper_journal_id: c
|
||||
.rejected
|
||||
.helper_journal_id
|
||||
.context("REPAIR_HELPER_REQUIRED")?,
|
||||
helper_checkpoint: c.rejected.helper_checkpoint,
|
||||
rejected_decision_id: c.rejected.decision_id,
|
||||
rejected_decision_sha256: hash(&e.rejected_response.decision_bytes),
|
||||
rejected_request_id: c.rejected.request_id,
|
||||
rejected_request_sha256: hash(&e.rejected_request),
|
||||
uncommitted_request_id: Uuid::parse_str(&c.unsent.request_id)?,
|
||||
uncommitted_request_sha256: hash(&e.uncommitted_request),
|
||||
previous_valid_decision_id: c.old.decision_id,
|
||||
previous_valid_decision_sha256: hash(&e.previous_valid_response.decision_bytes),
|
||||
agent_inventory_sha256: inventory_sha256(&e.agent_inventory)?,
|
||||
helper_inventory_sha256: inventory_sha256(&e.helper_inventory)?,
|
||||
helper_ledger_sha256: helper_ledger_sha256(&e.helper_inventory)?,
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
unlock: false,
|
||||
};
|
||||
ensure!(receipt.valid_profile(now), "REPAIR_RECEIPT_PROFILE");
|
||||
let payload = serde_json::to_vec(&receipt)?;
|
||||
let signed = SignedReceipt {
|
||||
signature: state.job_signer.sign(&payload)?,
|
||||
payload,
|
||||
};
|
||||
ensure!(sqlx::query("UPDATE agent_sync_state SET checkpoint=$3,decision_id=$4,helper_journal_id=$5,helper_checkpoint=$6 WHERE tenant_id=$1 AND agent_id=$2 AND decision_id=$7")
|
||||
.bind(e.tenant_id).bind(e.agent_id).bind(i64::try_from(c.request.checkpoint)?).bind(c.rejected.decision_id)
|
||||
.bind(c.rejected.helper_journal_id).bind(i64::try_from(c.rejected.helper_checkpoint)?).bind(c.old.decision_id)
|
||||
.execute(&mut *tx).await?.rows_affected()==1,"REPAIR_HEAD_CONCURRENT_CHANGE");
|
||||
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,$4,$5)")
|
||||
.bind(e.tenant_id).bind(receipt.receipt_id).bind("development-bootstrap:unsent-sync-repair").bind(EVENT)
|
||||
.bind(json!({"agent_id":e.agent_id,"evidence_sha256":evidence_hash,"receipt_sha256":hash(&signed.payload),"receipt":signed,"ready":false,"helper_ready":false,"unlock":false}))
|
||||
.execute(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
Ok(signed)
|
||||
}
|
||||
fn head_matches(head: &sqlx::postgres::PgRow, d: &ReconcileDecisionPayload) -> Result<bool> {
|
||||
Ok(head.get::<Uuid, _>("journal_id") == d.journal_id
|
||||
&& head.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
|
||||
&& head.get::<Uuid, _>("decision_id") == d.decision_id
|
||||
&& head.get::<Option<Uuid>, _>("helper_journal_id") == d.helper_journal_id
|
||||
&& head.get::<i64, _>("helper_checkpoint") == i64::try_from(d.helper_checkpoint)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
fn table(fields: Vec<(&str, Value)>) -> Value {
|
||||
json!({"columns":fields.iter().map(|(k,_)|k).collect::<Vec<_>>(),"rows":[fields.iter().map(|(_,v)|v).collect::<Vec<_>>() ]})
|
||||
}
|
||||
fn t(v: impl ToString) -> Value {
|
||||
json!({"type":"text","value":v.to_string()})
|
||||
}
|
||||
fn n(v: u64) -> Value {
|
||||
json!({"type":"integer","value":v})
|
||||
}
|
||||
fn b(v: &[u8]) -> Value {
|
||||
json!({"type":"blob","sha256":hash(v),"length":v.len()})
|
||||
}
|
||||
fn sign(s: &crate::job_signer::JobSigner, d: &ReconcileDecisionPayload) -> SignedDecision {
|
||||
let decision_bytes = serde_json::to_vec(d).unwrap();
|
||||
SignedDecision {
|
||||
signature: s.sign(&decision_bytes).unwrap(),
|
||||
decision_bytes,
|
||||
}
|
||||
}
|
||||
fn fixture() -> (crate::job_signer::JobSigner, Evidence, std::path::PathBuf) {
|
||||
let path = std::path::PathBuf::from(format!("/tmp/rmm-repair-{}", Uuid::new_v4()));
|
||||
let signer = crate::job_signer::JobSigner::load_or_create(&path.join("key.pem")).unwrap();
|
||||
let tenant = Uuid::new_v4();
|
||||
let asset = Uuid::new_v4();
|
||||
let agent = Uuid::new_v4();
|
||||
let journal = Uuid::new_v4();
|
||||
let helper = Uuid::new_v4();
|
||||
let now = Utc::now();
|
||||
let old = ReconcileDecisionPayload {
|
||||
domain: "rmm.reconciliation-decision".into(),
|
||||
schema_version: 1,
|
||||
decision_id: Uuid::new_v4(),
|
||||
request_id: Uuid::new_v4(),
|
||||
request_hash: "ab".repeat(32),
|
||||
tenant_id: tenant,
|
||||
asset_id: asset,
|
||||
agent_id: agent,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
connection_generation: 3,
|
||||
boot_id: Uuid::new_v4(),
|
||||
journal_id: journal,
|
||||
checkpoint: 10,
|
||||
helper_journal_id: Some(helper),
|
||||
helper_checkpoint: 5,
|
||||
issued_at: now,
|
||||
valid_until: now + Duration::seconds(60),
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
|
||||
attempts: vec![],
|
||||
acknowledged_message_ids: vec![],
|
||||
key_id: signer.key_id.clone(),
|
||||
};
|
||||
let x = SyncRequest {
|
||||
request_id: Uuid::new_v4().to_string(),
|
||||
journal_id: journal.to_string(),
|
||||
checkpoint: 11,
|
||||
previous_decision_id: old.decision_id.to_string(),
|
||||
assignment_version: 1,
|
||||
journal_complete: true,
|
||||
helper_journal_id: helper.to_string(),
|
||||
helper_checkpoint: 6,
|
||||
helper_previous_decision_id: old.decision_id.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let y = SyncRequest {
|
||||
request_id: Uuid::new_v4().to_string(),
|
||||
checkpoint: 12,
|
||||
previous_decision_id: String::new(),
|
||||
previous_request_id: x.request_id.clone(),
|
||||
helper_checkpoint: 7,
|
||||
..x.clone()
|
||||
};
|
||||
let mut rejected = old.clone();
|
||||
rejected.decision_id = Uuid::new_v4();
|
||||
rejected.request_id = Uuid::parse_str(&y.request_id).unwrap();
|
||||
rejected.request_hash = hash(&y.encode_to_vec());
|
||||
rejected.checkpoint = 12;
|
||||
rejected.helper_checkpoint = 7;
|
||||
rejected.reason = FAULT.into();
|
||||
let response = sign(&signer, &rejected);
|
||||
let request = y.encode_to_vec();
|
||||
let unsent = x.encode_to_vec();
|
||||
let proto = rmm_protocol::SyncResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut requests = table(vec![
|
||||
("id", t(&x.request_id)),
|
||||
("request", b(&unsent)),
|
||||
("generation", n(3)),
|
||||
("response", json!({"type":"null"})),
|
||||
]);
|
||||
requests["rows"].as_array_mut().unwrap().push(json!([
|
||||
t(&y.request_id),
|
||||
b(&request),
|
||||
n(3),
|
||||
b(&proto)
|
||||
]));
|
||||
let am = table(vec![
|
||||
("id", n(1)),
|
||||
("journal_id", t(journal)),
|
||||
("checkpoint", n(12)),
|
||||
("high_sequence", n(13)),
|
||||
("generation", n(3)),
|
||||
("last_decision", t(rejected.decision_id)),
|
||||
("pending_request", json!({"type":"null"})),
|
||||
("helper_id", t(helper)),
|
||||
("helper_checkpoint", n(7)),
|
||||
("fault", t(FAULT)),
|
||||
]);
|
||||
let hm = table(vec![
|
||||
("id", n(1)),
|
||||
("journal_id", t(helper)),
|
||||
("checkpoint", n(7)),
|
||||
("last_decision", t(rejected.decision_id)),
|
||||
("agent_journal_id", t(journal)),
|
||||
("agent_checkpoint", n(12)),
|
||||
("fault", t(FAULT)),
|
||||
]);
|
||||
let history = table(vec![
|
||||
("decision_id", t(rejected.decision_id)),
|
||||
("checkpoint", n(7)),
|
||||
("response", b(&serde_json::to_vec(&response).unwrap())),
|
||||
("request", b(&request)),
|
||||
]);
|
||||
let e = Evidence {
|
||||
schema_version: 1,
|
||||
tenant_id: tenant,
|
||||
asset_id: asset,
|
||||
agent_id: agent,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
agent_inventory: json!({"reconcile_meta":am,"reconcile_requests":requests}),
|
||||
helper_inventory: json!({"helper_sync_meta":hm,"helper_sync_decisions":history,"consumed":{"columns":[],"rows":[]},"execution_evidence":{"columns":[],"rows":[]}}),
|
||||
uncommitted_request: unsent,
|
||||
rejected_request: request,
|
||||
rejected_response: response,
|
||||
previous_valid_response: sign(&signer, &old),
|
||||
};
|
||||
(signer, e, path)
|
||||
}
|
||||
#[test]
|
||||
fn only_exact_signed_unsent_case_validates() {
|
||||
let (s, e, p) = fixture();
|
||||
validate(&s, &e).unwrap();
|
||||
let mut other = e.clone();
|
||||
other.tenant_id = Uuid::new_v4();
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
other.asset_id = Uuid::new_v4();
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
other.assignment_version += 1;
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
other.rejected_response.signature[0] ^= 1;
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
other.agent_inventory["reconcile_requests"]["rows"][0][3] = b(b"committed response");
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
other.helper_inventory["helper_sync_meta"]["rows"][0][2] = n(8);
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
other.agent_inventory["reconcile_meta"]["rows"][0][9] = t("OTHER_FAULT");
|
||||
assert!(validate(&s, &other).is_err());
|
||||
let mut other = e.clone();
|
||||
let mut x = SyncRequest::decode(other.uncommitted_request.as_slice()).unwrap();
|
||||
x.previous_decision_id = Uuid::new_v4().to_string();
|
||||
other.uncommitted_request = x.encode_to_vec();
|
||||
assert!(validate(&s, &other).is_err());
|
||||
fs::remove_dir_all(p).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn signed_ready_response_is_never_repair_authority() {
|
||||
let (s, mut e, p) = fixture();
|
||||
let mut d: ReconcileDecisionPayload =
|
||||
serde_json::from_slice(&e.rejected_response.decision_bytes).unwrap();
|
||||
d.ready = true;
|
||||
e.rejected_response = sign(&s, &d);
|
||||
assert!(validate(&s, &e).is_err());
|
||||
d.ready = false;
|
||||
d.reason = "CONSISTENT".into();
|
||||
e.rejected_response = sign(&s, &d);
|
||||
assert!(validate(&s, &e).is_err());
|
||||
fs::remove_dir_all(p).unwrap();
|
||||
}
|
||||
/// Opt-in dedicated PostgreSQL only; the harness creates/destroys its own
|
||||
/// network-isolated container and never reads deployment credentials.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the dedicated isolated PostgreSQL harness"]
|
||||
async fn isolated_database_repair_and_fail_closed_paths() {
|
||||
let admin_url = std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap();
|
||||
let app_url = std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap();
|
||||
let admin = sqlx::PgPool::connect(&admin_url).await.unwrap();
|
||||
let pool = sqlx::PgPool::connect(&app_url).await.unwrap();
|
||||
rmm_persistence::verify_application_role(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let (s, e, p) = fixture();
|
||||
let c = validate(&s, &e).unwrap();
|
||||
sqlx::query("INSERT INTO tenants VALUES($1,'isolated repair test')")
|
||||
.bind(e.tenant_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
|
||||
.bind(e.tenant_id)
|
||||
.bind(e.asset_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,assignment_version,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5,$6)")
|
||||
.bind(e.tenant_id).bind(e.agent_id).bind(e.asset_id).bind(hash(e.agent_id.as_bytes())).bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(c.rejected.boot_id).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO agent_sync_state VALUES($1,$2,$3,$4,$5,$6,$7)")
|
||||
.bind(e.tenant_id)
|
||||
.bind(e.agent_id)
|
||||
.bind(c.old.journal_id)
|
||||
.bind(i64::try_from(c.old.checkpoint).unwrap())
|
||||
.bind(c.old.decision_id)
|
||||
.bind(c.old.helper_journal_id)
|
||||
.bind(i64::try_from(c.old.helper_checkpoint).unwrap())
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
for (d, signed) in [
|
||||
(&c.old, &e.previous_valid_response),
|
||||
(&c.rejected, &e.rejected_response),
|
||||
] {
|
||||
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
|
||||
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).bind(d.decision_id).bind(&d.request_hash).bind(d.journal_id).bind(i64::try_from(d.checkpoint).unwrap()).bind(i64::try_from(d.connection_generation).unwrap()).bind(&signed.decision_bytes).bind(&signed.signature).execute(&admin).await.unwrap();
|
||||
}
|
||||
let state = crate::State {
|
||||
pool,
|
||||
signer: None,
|
||||
viewer_a: [0; 32],
|
||||
viewer_b: [1; 32],
|
||||
auth: std::sync::Arc::new(crate::auth::Auth::from_env("development").await.unwrap()),
|
||||
job_signer: std::sync::Arc::new(s),
|
||||
};
|
||||
let mut wrong = e.clone();
|
||||
wrong.tenant_id = Uuid::new_v4();
|
||||
assert!(issue_with_closed_policy(&state, wrong).await.is_err());
|
||||
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
|
||||
.bind(e.tenant_id)
|
||||
.bind(c.rejected.request_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
||||
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
|
||||
.bind(e.tenant_id).bind(e.agent_id).bind(c.rejected.request_id).bind(c.rejected.decision_id)
|
||||
.bind(&c.rejected.request_hash).bind(c.rejected.journal_id).bind(i64::try_from(c.rejected.checkpoint).unwrap())
|
||||
.bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(&e.rejected_response.decision_bytes)
|
||||
.bind(&e.rejected_response.signature).execute(&admin).await.unwrap();
|
||||
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint+1 WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
||||
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint-1 WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE agent_installations SET revoked_at=now() WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
||||
sqlx::query("UPDATE agent_installations SET revoked_at=NULL WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let x = Uuid::parse_str(&c.unsent.request_id).unwrap();
|
||||
sqlx::query("INSERT INTO sync_checkpoints SELECT tenant_id,agent_id,$2,$3,request_hash,journal_id,checkpoint,connection_generation,payload,signature,created_at FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$4")
|
||||
.bind(e.tenant_id).bind(x).bind(Uuid::new_v4()).bind(c.old.request_id).execute(&admin).await.unwrap();
|
||||
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
||||
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
|
||||
.bind(e.tenant_id)
|
||||
.bind(x)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE agent_installations SET connection_generation=$2 WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.bind(i64::try_from(c.rejected.connection_generation).unwrap() - 1)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
||||
sqlx::query("UPDATE agent_installations SET connection_generation=$2,connection_boot_id=$3 WHERE agent_id=$1")
|
||||
.bind(e.agent_id).bind(i64::try_from(c.rejected.connection_generation).unwrap()+1).bind(Uuid::new_v4())
|
||||
.execute(&admin).await.unwrap();
|
||||
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
||||
sqlx::query("UPDATE agent_installations SET connection_boot_id=$2 WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.bind(c.rejected.boot_id)
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let first = issue_with_closed_policy(&state, e.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
first,
|
||||
issue_with_closed_policy(&state, e.clone()).await.unwrap()
|
||||
);
|
||||
let r: Receipt = serde_json::from_slice(&first.payload).unwrap();
|
||||
assert!(r.valid_profile(Utc::now()));
|
||||
assert!(!r.ready && !r.helper_ready && !r.unlock);
|
||||
let head: Uuid =
|
||||
sqlx::query_scalar("SELECT decision_id FROM agent_sync_state WHERE agent_id=$1")
|
||||
.bind(e.agent_id)
|
||||
.fetch_one(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(head, c.rejected.decision_id);
|
||||
let count: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
|
||||
.bind(e.tenant_id)
|
||||
.fetch_one(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM outbox)+(SELECT count(*) FROM execution_permits)").fetch_one(&admin).await.unwrap();
|
||||
assert_eq!(effects, 0);
|
||||
let ready: bool = sqlx::query_scalar(
|
||||
"SELECT sync_ready OR helper_sync_ready FROM agent_installations WHERE agent_id=$1",
|
||||
)
|
||||
.bind(e.agent_id)
|
||||
.fetch_one(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!ready);
|
||||
fs::remove_dir_all(p).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
//! Narrow dev/test wire faults after normal authorization/durable ingest.
|
||||
//! Never creates jobs, grants or approvals. Claims and observations are bounded,
|
||||
//! insert-only audit slots; a restart cannot arm the same fault a second time.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use prost::Message;
|
||||
use rmm_domain::jobs::{Action, JobPayload, hash};
|
||||
use rmm_protocol::{Envelope, Payload};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum Mode {
|
||||
DuplicateOfferOnce,
|
||||
DropResultAckOnce,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Config {
|
||||
schema_version: u32,
|
||||
fault_id: Uuid,
|
||||
tenant_id: Uuid,
|
||||
agent_id: Uuid,
|
||||
job_id: Uuid,
|
||||
attempt_id: Uuid,
|
||||
plan_sha256: String,
|
||||
job_payload_sha256: String,
|
||||
mode: Mode,
|
||||
transport: String,
|
||||
issued_at: DateTime<Utc>,
|
||||
expires_at: DateTime<Utc>,
|
||||
}
|
||||
impl Config {
|
||||
fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == 1
|
||||
&& [
|
||||
self.fault_id,
|
||||
self.tenant_id,
|
||||
self.agent_id,
|
||||
self.job_id,
|
||||
self.attempt_id
|
||||
]
|
||||
.iter()
|
||||
.all(|id| !id.is_nil())
|
||||
&& valid_hash(&self.plan_sha256)
|
||||
&& valid_hash(&self.job_payload_sha256)
|
||||
&& matches!(self.transport.as_str(), "wss" | "https")
|
||||
&& self.issued_at.timestamp() > 0
|
||||
&& self.expires_at > self.issued_at
|
||||
&& self.expires_at - self.issued_at <= Duration::minutes(15),
|
||||
"DEV_DELIVERY_FAULT_CONFIG"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn scope(&self, tenant: Uuid, agent: Uuid, transport: &str) -> bool {
|
||||
self.tenant_id == tenant && self.agent_id == agent && self.transport == transport
|
||||
}
|
||||
fn job(&self, bytes: &[u8], payload_hash: &str, plan_hash: &str) -> Result<bool> {
|
||||
if payload_hash != self.job_payload_sha256 || plan_hash != self.plan_sha256 {
|
||||
return Ok(false);
|
||||
}
|
||||
ensure!(
|
||||
hash(bytes) == self.job_payload_sha256,
|
||||
"DEV_FAULT_PAYLOAD_HASH"
|
||||
);
|
||||
let job: JobPayload = serde_json::from_slice(bytes)?;
|
||||
Ok(job.schema_version == 1
|
||||
&& job.tenant_id == self.tenant_id
|
||||
&& job.agent_id == self.agent_id
|
||||
&& job.job_id == self.job_id
|
||||
&& job.attempt_id == self.attempt_id
|
||||
&& job.execution_identity == "agent_service"
|
||||
&& job.enrollment_generation > 0
|
||||
&& job.assignment_version > 0
|
||||
&& job.max_runtime_seconds > 0
|
||||
&& job.max_runtime_seconds <= 60
|
||||
&& job.max_output_bytes <= 65536
|
||||
&& job.action.validate()
|
||||
&& matches!(job.action, Action::RunScript { .. })
|
||||
&& self.expires_at <= job.start_before)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_hash(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
&& value.bytes().any(|b| b != b'0')
|
||||
}
|
||||
fn configured_path(
|
||||
flag: Option<&str>,
|
||||
path: Option<&str>,
|
||||
profile: &str,
|
||||
dev: &str,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
match (flag, path) {
|
||||
(None | Some("false"), None) => Ok(None),
|
||||
(Some("true"), Some(path)) => {
|
||||
ensure!(
|
||||
profile == "development" && dev == "true",
|
||||
"DEV_DELIVERY_FAULT_PROFILE"
|
||||
);
|
||||
ensure!(
|
||||
path == "/root-policy/delivery-fault.json",
|
||||
"DEV_DELIVERY_FAULT_PATH"
|
||||
);
|
||||
Ok(Some(PathBuf::from(path)))
|
||||
}
|
||||
_ => anyhow::bail!("DEV_DELIVERY_FAULT_EXPLICIT_OPT_IN"),
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct Loaded {
|
||||
config: Config,
|
||||
config_sha256: String,
|
||||
}
|
||||
fn load() -> Result<Option<Loaded>> {
|
||||
let flag = std::env::var("RMM_DEV_DELIVERY_FAULTS").ok();
|
||||
let path = std::env::var("RMM_DEV_DELIVERY_FAULT_FILE").ok();
|
||||
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
|
||||
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
|
||||
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let parent = path.parent().context("DEV_FAULT_PARENT")?;
|
||||
let directory = fs::symlink_metadata(parent)?;
|
||||
let metadata = fs::symlink_metadata(&path)?;
|
||||
ensure!(
|
||||
directory.is_dir()
|
||||
&& directory.uid() == 10001
|
||||
&& directory.mode() & 0o777 == 0o700
|
||||
&& metadata.is_file()
|
||||
&& metadata.uid() == 10001
|
||||
&& metadata.mode() & 0o777 == 0o600
|
||||
&& metadata.nlink() == 1
|
||||
&& metadata.len() <= 8192,
|
||||
"DEV_FAULT_PRIVATE_CONFIG"
|
||||
);
|
||||
// /root-policy is the existing operator-provisioned read-only bind mount.
|
||||
let bytes = fs::read(&path)?;
|
||||
ensure!(bytes.len() <= 8192, "DEV_FAULT_CONFIG_SIZE");
|
||||
let config: Config = serde_json::from_slice(&bytes)?;
|
||||
config.validate()?;
|
||||
if Utc::now() < config.issued_at || Utc::now() >= config.expires_at {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(Loaded {
|
||||
config,
|
||||
config_sha256: hash(&bytes),
|
||||
}))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
load().map(|_| ())
|
||||
}
|
||||
|
||||
fn slot_id(fault: Uuid, slot: &str) -> Uuid {
|
||||
use sha2::{Digest, Sha256};
|
||||
let digest = Sha256::digest(format!("rmm.dev-delivery-fault.v1:{fault}:{slot}"));
|
||||
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
|
||||
}
|
||||
fn envelope_hash(envelope: &Envelope) -> String {
|
||||
let mut normalized = envelope.clone();
|
||||
normalized.connection_generation = 0;
|
||||
hash(&normalized.encode_to_vec())
|
||||
}
|
||||
fn data(loaded: &Loaded, envelope: &Envelope, detail: &str) -> Value {
|
||||
let config = &loaded.config;
|
||||
json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
|
||||
"fault_id":config.fault_id,"mode":config.mode,"transport":config.transport,
|
||||
"agent_id":config.agent_id,"job_id":config.job_id,"attempt_id":config.attempt_id,
|
||||
"plan_sha256":config.plan_sha256,"job_payload_sha256":config.job_payload_sha256,
|
||||
"config_sha256":loaded.config_sha256,"message_id":envelope.message_id,
|
||||
"normalized_envelope_sha256":envelope_hash(envelope),"detail":detail})
|
||||
}
|
||||
async fn transaction<'a>(pool: &'a PgPool, loaded: &Loaded) -> Result<Transaction<'a, Postgres>> {
|
||||
let mut tx = rmm_persistence::tenant_tx(pool, loaded.config.tenant_id).await?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
||||
.bind(format!("dev-delivery-fault:{}", loaded.config.fault_id))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
Ok(tx)
|
||||
}
|
||||
async fn get_slot(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
loaded: &Loaded,
|
||||
slot: &str,
|
||||
) -> Result<Option<Value>> {
|
||||
let value: Option<Value> = sqlx::query_scalar("SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_delivery_fault_adapter'")
|
||||
.bind(slot_id(loaded.config.fault_id, slot)).fetch_optional(&mut **tx).await?;
|
||||
if let Some(value) = &value {
|
||||
ensure!(
|
||||
value.get("config_sha256") == Some(&json!(loaded.config_sha256)),
|
||||
"DEV_FAULT_ID_REUSED_WITH_CHANGED_CONFIG"
|
||||
);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
async fn insert_slot(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
loaded: &Loaded,
|
||||
slot: &str,
|
||||
value: Value,
|
||||
) -> Result<bool> {
|
||||
if get_slot(tx, loaded, slot).await?.is_some() {
|
||||
return Ok(false);
|
||||
}
|
||||
let rows = sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_delivery_fault_adapter',$3,$4) ON CONFLICT(tenant_id,event_id) DO NOTHING")
|
||||
.bind(loaded.config.tenant_id).bind(slot_id(loaded.config.fault_id, slot))
|
||||
.bind(format!("dev_delivery_fault.{slot}")).bind(value).execute(&mut **tx).await?.rows_affected();
|
||||
Ok(rows == 1)
|
||||
}
|
||||
async fn authorized_job(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
loaded: &Loaded,
|
||||
message: Option<&Envelope>,
|
||||
) -> Result<bool> {
|
||||
let c = &loaded.config;
|
||||
let row = sqlx::query("SELECT j.payload,j.payload_hash,p.plan_hash FROM jobs j JOIN operations o USING(tenant_id,operation_id) JOIN approval_requests r USING(tenant_id,operation_id) JOIN operation_plans p USING(tenant_id,plan_id) JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 AND r.approval_state='approved' AND a.testsimulation AND a.approval_origin='dev_test_browser_simulation' AND a.principal='simulated-testprincipal-a'")
|
||||
.bind(c.job_id).bind(c.attempt_id).bind(c.agent_id).fetch_optional(&mut **tx).await?;
|
||||
let Some(row) = row else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !c.job(
|
||||
&row.get::<Vec<u8>, _>("payload"),
|
||||
&row.get::<String, _>("payload_hash"),
|
||||
&row.get::<String, _>("plan_hash"),
|
||||
)? {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(message) = message {
|
||||
let receipt: Option<String> = sqlx::query_scalar(
|
||||
"SELECT payload_hash FROM job_receipts WHERE message_id=$1 AND agent_id=$2",
|
||||
)
|
||||
.bind(Uuid::parse_str(&message.message_id)?)
|
||||
.bind(c.agent_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?;
|
||||
ensure!(
|
||||
receipt.as_deref() == Some(envelope_hash(message).as_str()),
|
||||
"DEV_FAULT_DURABLE_RECEIPT_REQUIRED"
|
||||
);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(crate) struct DuplicateClaim {
|
||||
loaded: Loaded,
|
||||
envelope: Envelope,
|
||||
}
|
||||
/// Called only for work returned by normal postapproval `job_gateway::next`.
|
||||
pub(crate) async fn duplicate_offer(
|
||||
pool: &PgPool,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
envelope: &Envelope,
|
||||
transport: &str,
|
||||
) -> Result<Option<DuplicateClaim>> {
|
||||
let Some(Payload::JobOffer(offer)) = &envelope.payload else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(loaded) = load()? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let c = &loaded.config;
|
||||
if c.mode != Mode::DuplicateOfferOnce
|
||||
|| !c.scope(tenant, agent, transport)
|
||||
|| offer.job_id != c.job_id.to_string()
|
||||
|| offer.attempt_id != c.attempt_id.to_string()
|
||||
|| !c.job(
|
||||
&offer.payload_bytes,
|
||||
&offer.job_payload_hash,
|
||||
&offer.plan_hash,
|
||||
)?
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let mut tx = transaction(pool, &loaded).await?;
|
||||
let claimed = authorized_job(&mut tx, &loaded, None).await?
|
||||
&& insert_slot(
|
||||
&mut tx,
|
||||
&loaded,
|
||||
"claim",
|
||||
data(&loaded, envelope, "duplicate_offer_reserved_before_send"),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok(claimed.then(|| DuplicateClaim {
|
||||
loaded,
|
||||
envelope: envelope.clone(),
|
||||
}))
|
||||
}
|
||||
/// WSS calls after both sends complete. HTTP labels construction honestly; the
|
||||
/// two later distinct postcommit JobAccepted slots prove client observation.
|
||||
pub(crate) async fn duplicate_emitted(pool: &PgPool, claim: DuplicateClaim) -> Result<()> {
|
||||
let mut tx = transaction(pool, &claim.loaded).await?;
|
||||
ensure!(
|
||||
get_slot(&mut tx, &claim.loaded, "claim").await?.is_some(),
|
||||
"DEV_FAULT_CLAIM_REQUIRED"
|
||||
);
|
||||
let detail = if claim.loaded.config.transport == "wss" {
|
||||
"same_encoded_offer_sent_twice"
|
||||
} else {
|
||||
"same_encoded_offer_in_http_response_twice"
|
||||
};
|
||||
insert_slot(
|
||||
&mut tx,
|
||||
&claim.loaded,
|
||||
"emitted",
|
||||
data(&claim.loaded, &claim.envelope, detail),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
/// Called AFTER durable ingest returned its concrete ACK. Return true only to
|
||||
/// omit that single matching result ACK; every subsequent replay is ACKed.
|
||||
pub(crate) async fn after_ingest(
|
||||
pool: &PgPool,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
incoming: &Envelope,
|
||||
reply: &Envelope,
|
||||
transport: &str,
|
||||
) -> Result<bool> {
|
||||
let (job, attempt, is_result) = match &incoming.payload {
|
||||
Some(Payload::JobAccepted(value)) => (&value.job_id, &value.attempt_id, false),
|
||||
Some(Payload::JobResult(value)) if value.state == "succeeded" => {
|
||||
(&value.job_id, &value.attempt_id, true)
|
||||
}
|
||||
_ => return Ok(false),
|
||||
};
|
||||
let Some(loaded) = load()? else {
|
||||
return Ok(false);
|
||||
};
|
||||
let c = &loaded.config;
|
||||
if !c.scope(tenant, agent, transport)
|
||||
|| job != &c.job_id.to_string()
|
||||
|| attempt != &c.attempt_id.to_string()
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(Payload::Ack(ack)) = &reply.payload else {
|
||||
return Ok(false);
|
||||
};
|
||||
ensure!(
|
||||
ack.message_id == incoming.message_id,
|
||||
"DEV_FAULT_CONCRETE_ACK_REQUIRED"
|
||||
);
|
||||
let mut tx = transaction(pool, &loaded).await?;
|
||||
if !authorized_job(&mut tx, &loaded, Some(incoming)).await? {
|
||||
tx.commit().await?;
|
||||
return Ok(false);
|
||||
}
|
||||
let claim = get_slot(&mut tx, &loaded, "claim").await?;
|
||||
let mut omit = false;
|
||||
if is_result && c.mode == Mode::DropResultAckOnce {
|
||||
if let Some(first) = claim {
|
||||
if first.get("message_id") == Some(&json!(incoming.message_id))
|
||||
&& first.get("normalized_envelope_sha256") == Some(&json!(envelope_hash(incoming)))
|
||||
{
|
||||
insert_slot(
|
||||
&mut tx,
|
||||
&loaded,
|
||||
"replay",
|
||||
data(
|
||||
&loaded,
|
||||
incoming,
|
||||
"same_original_result_replayed_after_ack_omission",
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
omit = insert_slot(
|
||||
&mut tx,
|
||||
&loaded,
|
||||
"claim",
|
||||
data(
|
||||
&loaded,
|
||||
incoming,
|
||||
"one_concrete_result_ack_omitted_after_durable_commit",
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
} else if !is_result && c.mode == Mode::DuplicateOfferOnce && claim.is_some() {
|
||||
if let Some(first) = get_slot(&mut tx, &loaded, "accepted_1").await? {
|
||||
if first.get("message_id") != Some(&json!(incoming.message_id)) {
|
||||
insert_slot(
|
||||
&mut tx,
|
||||
&loaded,
|
||||
"accepted_2",
|
||||
data(&loaded, incoming, "second_distinct_client_acceptance"),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
insert_slot(
|
||||
&mut tx,
|
||||
&loaded,
|
||||
"accepted_1",
|
||||
data(&loaded, incoming, "first_client_acceptance"),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(omit)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
fn config() -> Config {
|
||||
let now = Utc::now();
|
||||
Config {
|
||||
schema_version: 1,
|
||||
fault_id: Uuid::new_v4(),
|
||||
tenant_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
plan_sha256: "a".repeat(64),
|
||||
job_payload_sha256: "b".repeat(64),
|
||||
mode: Mode::DuplicateOfferOnce,
|
||||
transport: "wss".into(),
|
||||
issued_at: now,
|
||||
expires_at: now + Duration::minutes(10),
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn opt_in_is_explicit_and_never_production() {
|
||||
assert!(
|
||||
configured_path(None, None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
configured_path(
|
||||
Some("true"),
|
||||
Some("/root-policy/delivery-fault.json"),
|
||||
"production",
|
||||
"false"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(configured_path(Some("true"), None, "development", "true").is_err());
|
||||
assert!(
|
||||
configured_path(
|
||||
None,
|
||||
Some("/root-policy/delivery-fault.json"),
|
||||
"development",
|
||||
"true"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(configured_path(Some("true"), Some("/etc/passwd"), "development", "true").is_err());
|
||||
assert!(
|
||||
configured_path(
|
||||
Some("true"),
|
||||
Some("/root-policy/delivery-fault.json"),
|
||||
"development",
|
||||
"true"
|
||||
)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn narrow_schema_rejects_wildcards_nil_placeholders_and_long_lifetime() {
|
||||
let c = config();
|
||||
c.validate().unwrap();
|
||||
let mut altered = c.clone();
|
||||
altered.expires_at = altered.issued_at + Duration::minutes(16);
|
||||
assert!(altered.validate().is_err());
|
||||
altered = c.clone();
|
||||
altered.agent_id = Uuid::nil();
|
||||
assert!(altered.validate().is_err());
|
||||
altered = c.clone();
|
||||
altered.job_payload_sha256 = "0".repeat(64);
|
||||
assert!(altered.validate().is_err());
|
||||
altered = c.clone();
|
||||
altered.transport = "*".into();
|
||||
assert!(altered.validate().is_err());
|
||||
let mut raw = serde_json::to_value(c).unwrap();
|
||||
raw["approve"] = json!(true);
|
||||
assert!(serde_json::from_value::<Config>(raw).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn audit_slots_are_stable_bounded_and_distinct() {
|
||||
let id = Uuid::new_v4();
|
||||
assert_eq!(slot_id(id, "claim"), slot_id(id, "claim"));
|
||||
assert_ne!(slot_id(id, "claim"), slot_id(id, "replay"));
|
||||
assert_ne!(slot_id(id, "claim"), slot_id(Uuid::new_v4(), "claim"));
|
||||
}
|
||||
#[test]
|
||||
fn exact_signed_job_scope_cannot_fault_root_other_targets_or_other_actions() {
|
||||
let mut c = config();
|
||||
let mut job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: c.tenant_id,
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: c.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: c.job_id,
|
||||
attempt_id: c.attempt_id,
|
||||
action: Action::RunScript {
|
||||
interpreter: "posix_sh".into(),
|
||||
script_utf8: "printf hello\\n".into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: None,
|
||||
},
|
||||
execution_identity: "agent_service".into(),
|
||||
not_before: c.issued_at,
|
||||
start_before: c.expires_at,
|
||||
max_runtime_seconds: 60,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let mut bytes = serde_json::to_vec(&job).unwrap();
|
||||
c.job_payload_sha256 = hash(&bytes);
|
||||
assert!(
|
||||
c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
|
||||
.unwrap()
|
||||
);
|
||||
assert!(!c.scope(c.tenant_id, Uuid::new_v4(), "wss"));
|
||||
assert!(!c.scope(c.tenant_id, c.agent_id, "https"));
|
||||
assert!(
|
||||
!c.job(&bytes, &c.job_payload_sha256, &"c".repeat(64))
|
||||
.unwrap()
|
||||
);
|
||||
job.execution_identity = "root".into();
|
||||
bytes = serde_json::to_vec(&job).unwrap();
|
||||
c.job_payload_sha256 = hash(&bytes);
|
||||
assert!(
|
||||
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
|
||||
.unwrap()
|
||||
);
|
||||
job.execution_identity = "agent_service".into();
|
||||
job.action = Action::RefreshInventory {
|
||||
scope: "linux.base".into(),
|
||||
};
|
||||
bytes = serde_json::to_vec(&job).unwrap();
|
||||
c.job_payload_sha256 = hash(&bytes);
|
||||
assert!(
|
||||
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn durable_replay_only_normalizes_connection_generation() {
|
||||
let e = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
connection_generation: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut repeated = e.clone();
|
||||
repeated.connection_generation = 4;
|
||||
assert_eq!(envelope_hash(&e), envelope_hash(&repeated));
|
||||
repeated.message_id = Uuid::new_v4().to_string();
|
||||
assert_ne!(envelope_hash(&e), envelope_hash(&repeated));
|
||||
}
|
||||
}
|
||||
@@ -116,7 +116,10 @@ async fn receive(socket: &mut WebSocket) -> Result<Envelope> {
|
||||
Ok(Envelope::decode(bytes)?)
|
||||
}
|
||||
async fn send(
|
||||
state: &crate::State,
|
||||
socket: &mut WebSocket,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
e: &Envelope,
|
||||
payload: Payload,
|
||||
generation: u64,
|
||||
@@ -133,11 +136,22 @@ async fn send(
|
||||
sequence: e.sequence,
|
||||
payload: Some(payload),
|
||||
};
|
||||
let claim =
|
||||
dev_delivery_faults::duplicate_offer(&state.pool, tenant, agent, &reply, "wss").await?;
|
||||
let bytes = reply.encode_to_vec();
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
socket.send(Message::Binary(reply.encode_to_vec().into())),
|
||||
socket.send(Message::Binary(bytes.clone().into())),
|
||||
)
|
||||
.await??;
|
||||
if let Some(claim) = claim {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
socket.send(Message::Binary(bytes.into())),
|
||||
)
|
||||
.await??;
|
||||
dev_delivery_faults::duplicate_emitted(&state.pool, claim).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn channel(
|
||||
@@ -162,7 +176,7 @@ async fn channel(
|
||||
.send(Message::Binary(response.encode_to_vec().into()))
|
||||
.await?;
|
||||
if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? {
|
||||
send(socket, &hello, payload, generation).await?;
|
||||
send(state, socket, tenant, agent, &hello, payload, generation).await?;
|
||||
}
|
||||
let mut recent_messages = VecDeque::new();
|
||||
let mut timer = tokio::time::interval(Duration::from_secs(1));
|
||||
@@ -177,7 +191,7 @@ async fn channel(
|
||||
let mut tx=rmm_persistence::tenant_tx(&state.pool,tenant).await?;
|
||||
let revoked:bool=sqlx::query_scalar("SELECT revoked_at IS NOT NULL FROM agent_installations WHERE agent_id=$1").bind(agent).fetch_one(&mut *tx).await?;
|
||||
tx.commit().await?;ensure!(!revoked,"device revoked");
|
||||
if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(socket,&hello,payload,generation).await?;}
|
||||
if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(state,socket,tenant,agent,&hello,payload,generation).await?;}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -198,11 +212,17 @@ async fn channel(
|
||||
"channel generation mismatch"
|
||||
);
|
||||
let response = crate::agent_service::ingest(state, tenant, agent, &e, "wss").await?;
|
||||
socket
|
||||
.send(Message::Binary(response.encode_to_vec().into()))
|
||||
.await?;
|
||||
if !dev_delivery_faults::after_ingest(&state.pool, tenant, agent, &e, &response, "wss")
|
||||
.await?
|
||||
{
|
||||
socket
|
||||
.send(Message::Binary(response.encode_to_vec().into()))
|
||||
.await?;
|
||||
}
|
||||
if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? {
|
||||
send(socket, &e, payload, generation).await?;
|
||||
send(state, socket, tenant, agent, &e, payload, generation).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
#[path = "dev_delivery_faults.rs"]
|
||||
pub(crate) mod dev_delivery_faults;
|
||||
|
||||
@@ -5,6 +5,7 @@ mod api;
|
||||
mod artifact_store;
|
||||
mod auth;
|
||||
mod catalog;
|
||||
mod checkpoint_repair;
|
||||
mod controls;
|
||||
mod errors;
|
||||
mod gateway;
|
||||
@@ -68,6 +69,16 @@ async fn main() -> Result<()> {
|
||||
.await;
|
||||
}
|
||||
let profile = env::var("RMM_PROFILE").unwrap_or_default();
|
||||
gateway::dev_delivery_faults::validate_environment()?;
|
||||
let repair = env::args().nth(1).as_deref() == Some("repair-unsent-sync-request");
|
||||
if repair {
|
||||
ensure!(env::args().count() == 2, "REPAIR_NO_ARGUMENTS");
|
||||
checkpoint_repair::closed_policy()?;
|
||||
ensure!(
|
||||
std::path::Path::new(&env::var("RMM_JOB_KEY_FILE")?).is_file(),
|
||||
"REPAIR_EXISTING_KEY_REQUIRED"
|
||||
);
|
||||
}
|
||||
let dev = env::var("RMM_DEV_AUTH").as_deref() == Ok("true");
|
||||
ensure!(
|
||||
matches!(
|
||||
@@ -126,6 +137,19 @@ async fn main() -> Result<()> {
|
||||
.auth
|
||||
.validate_separation(&state.viewer_a, &state.viewer_b)?;
|
||||
}
|
||||
if repair {
|
||||
use std::io::{Read, Write};
|
||||
let mut raw = Vec::new();
|
||||
std::io::stdin()
|
||||
.take(32 * 1024 * 1024 + 1)
|
||||
.read_to_end(&mut raw)?;
|
||||
ensure!(raw.len() <= 32 * 1024 * 1024, "REPAIR_INPUT_SIZE");
|
||||
let evidence = serde_json::from_slice(&raw).context("REPAIR_EVIDENCE_JSON")?;
|
||||
let receipt = checkpoint_repair::issue(&state, evidence).await?;
|
||||
serde_json::to_writer(std::io::stdout().lock(), &receipt)?;
|
||||
std::io::stdout().write_all(b"\n")?;
|
||||
return Ok(());
|
||||
}
|
||||
if role.gateway() {
|
||||
scheduler::start(state.clone());
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ def upgrade(payload):
|
||||
base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote')
|
||||
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper')
|
||||
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json'
|
||||
agent_binary=Path('/usr/local/libexec/rmm-agent-remote')
|
||||
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
||||
full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
|
||||
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
|
||||
@@ -52,6 +53,7 @@ def upgrade(payload):
|
||||
guard()
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved')
|
||||
require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required')
|
||||
require(sha(agent_binary.read_bytes())==payload['old_agent_sha256'],'reviewed permanent agent required')
|
||||
require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required')
|
||||
original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json'
|
||||
retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json'
|
||||
@@ -59,7 +61,9 @@ def upgrade(payload):
|
||||
sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required')
|
||||
require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected')
|
||||
raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True)
|
||||
agent_raw=base64.b64decode(payload['agent_binary'],validate=True)
|
||||
require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts')
|
||||
require(sha(agent_raw)==payload['agent_sha256'],'qualified sync-prevention agent')
|
||||
backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700)
|
||||
def create(path,data,mode=0o600):
|
||||
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
|
||||
@@ -68,7 +72,9 @@ def upgrade(payload):
|
||||
def save():
|
||||
next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json')
|
||||
save();staged=backup/'helper-next';create(staged,raw,0o755)
|
||||
staged_agent=backup/'agent-next';create(staged_agent,agent_raw,0o755)
|
||||
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
|
||||
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged_agent)])
|
||||
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
|
||||
fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
try:
|
||||
@@ -77,11 +83,11 @@ def upgrade(payload):
|
||||
witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))}
|
||||
old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata')
|
||||
general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
|
||||
for p in (binary,provider,metadata,dropin):
|
||||
for p in (binary,agent_binary,provider,metadata,dropin):
|
||||
target=backup/str(p).lstrip('/').replace('/','__')
|
||||
require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup')
|
||||
os.rename(p,target)
|
||||
os.rename(staged,binary);create(provider,code,0o755)
|
||||
os.rename(staged,binary);os.rename(staged_agent,agent_binary);create(provider,code,0o755)
|
||||
create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644)
|
||||
inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
|
||||
require(inspected==json.loads(general),'general policy unchanged')
|
||||
@@ -90,6 +96,7 @@ def upgrade(payload):
|
||||
(ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift')
|
||||
report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown,
|
||||
helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save()
|
||||
report['agent_sha256']=sha(agent_raw);save()
|
||||
finally:
|
||||
fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
|
||||
run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
|
||||
@@ -139,12 +146,16 @@ def main():
|
||||
folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
|
||||
expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant]
|
||||
ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes()
|
||||
agent_raw=(STATE/'agent-sync-prevention').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant,
|
||||
old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c',
|
||||
old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945',
|
||||
binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(),
|
||||
agent_binary=base64.b64encode(agent_raw).decode(),agent_sha256=hashlib.sha256(agent_raw).hexdigest(),old_agent_sha256=installed['binary_sha256'],
|
||||
helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest())
|
||||
report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report)
|
||||
require(report['completed'] and report['agent_sha256']==payload['agent_sha256'],'qualified preserving installation')
|
||||
m.write_private(cloud.ROOT/'devstate/remote-vm.json',dict(installed,binary_sha256=report['agent_sha256']))
|
||||
print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')})
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One signed development control-chain repair, never a job or grant repair."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
import cloud_apt_reconcile_upgrade as upgrade
|
||||
cloud=upgrade.cloud
|
||||
require=cloud.require
|
||||
STATE=cloud.ROOT/'devstate/checkpoint-chain-repair'
|
||||
|
||||
|
||||
def prepared_json(path):
|
||||
"""The bounded full historical witness exceeds ordinary 1 MiB DTOs."""
|
||||
import stat
|
||||
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd,'r') as stream:
|
||||
info=os.fstat(stream.fileno())
|
||||
require(stat.S_ISREG(info.st_mode) and info.st_uid==os.getuid() and
|
||||
stat.S_IMODE(info.st_mode)==0o600 and info.st_nlink==1 and info.st_size<=45*1024*1024,
|
||||
'private bounded repair witness')
|
||||
return json.load(stream)
|
||||
|
||||
|
||||
def prepare_vm(payload):
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
os.umask(0o077)
|
||||
ident=str(uuid.UUID(payload['id']));state=Path('/var/lib/rmm-agent-remote');helper=Path('/var/lib/rmm-helper')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'reviewed permanent agent')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'fixed permanent device')
|
||||
base=Path('/var/lib/rmm-apt-transaction');result=json.loads((base/'active/sandbox-result.json').read_text())
|
||||
require(result['apt_exit_code']==0 and not result['reboot_required'],'ended native child required; never stop package manager')
|
||||
root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident);root.mkdir(mode=0o700)
|
||||
def create(path,data,mode=0o600):
|
||||
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
|
||||
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode)
|
||||
tool=base64.b64decode(payload['tool'],validate=True)
|
||||
require(hashlib.sha256(tool).hexdigest()==payload['tool_sha256'],'qualified repair tool')
|
||||
create(root/'checkpoint-repair',tool,0o755)
|
||||
subprocess.run(['/lib64/ld-linux-x86-64.so.2','--list',str(root/'checkpoint-repair')],capture_output=True,check=True,timeout=10)
|
||||
binding=json.loads((state/'binding.json').read_text())
|
||||
config=json.loads(Path('/etc/rmm-remote/helper-general.json').read_text())
|
||||
pin=json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_text())
|
||||
require(all(binding[k]==config['binding'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')) and
|
||||
binding['public_key_hex']==pin['public_key_hex'],'independent root trust binding')
|
||||
create(root/'binding.json',json.dumps(binding).encode())
|
||||
units=('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')
|
||||
before={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode() for name in units}
|
||||
subprocess.run(['systemctl','stop',*units],capture_output=True,check=True,timeout=25)
|
||||
prepared=False;report=dict(id=ident,vm_dir=str(root),tool_sha256=payload['tool_sha256'],prepared=False,package_effects=False)
|
||||
try:
|
||||
paths=(state/'journal.sqlite',state/'consumption.sqlite',helper/'consumption.sqlite')
|
||||
hashes={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
|
||||
for index,p in enumerate(paths):
|
||||
with sqlite3.connect(p.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','preserved database integrity')
|
||||
create(root/('database-before-'+str(index)+'.sqlite'),p.read_bytes())
|
||||
process=subprocess.run([str(root/'checkpoint-repair'),'prepare',str(root/'binding.json'),str(root/'evidence.json')],
|
||||
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25)
|
||||
create(root/'prepare.stderr',process.stderr)
|
||||
require(all(hashlib.sha256(p.read_bytes()).hexdigest()==hashes[str(p)] for p in paths),'prepare must not change databases')
|
||||
report.update(exit_code=process.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',process.stderr)],preserved_database_bytes=True)
|
||||
if process.returncode==0:
|
||||
raw=(root/'evidence.json').read_bytes();require(len(raw)<=32*1024*1024,'bounded evidence')
|
||||
report.update(prepared=True,evidence_sha256=hashlib.sha256(raw).hexdigest(),evidence_b64=base64.b64encode(raw).decode())
|
||||
prepared=True
|
||||
finally:
|
||||
create(root/'report.json',json.dumps({k:v for k,v in report.items() if k!='evidence_b64'}).encode())
|
||||
if not prepared:
|
||||
for unit in units:
|
||||
if before[unit]=='active':subprocess.run(['systemctl','start',unit],capture_output=True,timeout=20)
|
||||
return report
|
||||
|
||||
|
||||
def issue_server(payload):
|
||||
root=Path('/opt/rust-discovery')
|
||||
raw=base64.b64decode(payload['evidence_b64'],validate=True)
|
||||
require(hashlib.sha256(raw).hexdigest()==payload['evidence_sha256'],'original exact witness bytes')
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','api','/usr/local/bin/rmm-server','repair-unsent-sync-request']
|
||||
result=subprocess.run(cmd,input=raw,capture_output=True,timeout=30)
|
||||
require(result.returncode==0,'signed control receipt issuance failed')
|
||||
value=json.loads(result.stdout);require(set(value)=={'payload','signature'},'only signed control receipt output')
|
||||
return value
|
||||
|
||||
|
||||
def apply_vm(payload):
|
||||
import re
|
||||
os.umask(0o077)
|
||||
ident=str(uuid.UUID(payload['id']));root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident)
|
||||
require(hashlib.sha256((root/'evidence.json').read_bytes()).hexdigest()==payload['evidence_sha256'] and
|
||||
hashlib.sha256((root/'checkpoint-repair').read_bytes()).hexdigest()==payload['tool_sha256'],'original repair witnesses/tool')
|
||||
signed=json.dumps(payload['receipt'],separators=(',',':')).encode();path=root/'receipt.json'
|
||||
if path.exists():require(path.read_bytes()==signed,'identical existing receipt only')
|
||||
else:
|
||||
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'wb') as f:f.write(signed);f.flush();os.fsync(f.fileno())
|
||||
result=subprocess.run([str(root/'checkpoint-repair'),'apply',str(root/'binding.json'),str(root/'evidence.json'),str(path)],
|
||||
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25)
|
||||
fd=os.open(root/'apply.stderr',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'wb') as f:f.write(result.stderr)
|
||||
return dict(applied=result.returncode==0,exit_code=result.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',result.stderr)],
|
||||
ready_lease_issued=False,jobs_created=False,grants_created=False)
|
||||
|
||||
|
||||
def status_vm(payload):
|
||||
import sqlite3
|
||||
result={}
|
||||
for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'),
|
||||
('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')):
|
||||
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
|
||||
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0],
|
||||
receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0,
|
||||
integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok')
|
||||
if name=='helper':
|
||||
result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
||||
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
|
||||
result['services']={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode()
|
||||
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('prepare','issue','apply'));parser.add_argument('--id')
|
||||
args=parser.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True);m=cloud.contract
|
||||
if args.action=='prepare':
|
||||
require(upgrade.apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact no-new-execution gate')
|
||||
ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700)
|
||||
tool=(upgrade.STATE/'checkpoint-repair').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
result=cloud.bootstrap.rpc('linux-vm.conf',prepare_vm,dict(id=ident,agent_id=installed['identity']['agent_id'],agent_sha256=installed['binary_sha256'],
|
||||
tool_sha256=hashlib.sha256(tool).hexdigest(),tool=base64.b64encode(tool).decode()))
|
||||
m.write_private(folder/'prepared.json',result)
|
||||
print({k:result.get(k) for k in ('id','prepared','exit_code','error_codes','preserved_database_bytes')})
|
||||
require(result['prepared'],'stored witness preparation incomplete; no fault changes')
|
||||
else:
|
||||
ident=str(uuid.UUID(args.id));folder=STATE/ident;prepared=prepared_json(folder/'prepared.json');require(prepared['prepared'],'qualified original witnesses')
|
||||
if args.action=='issue':
|
||||
result=cloud.bootstrap.rpc('server.conf',issue_server,prepared);m.write_private(folder/'receipt.json',result)
|
||||
print(dict(signed_control_receipt=True,execution_permission=False))
|
||||
else:
|
||||
result=cloud.bootstrap.rpc('linux-vm.conf',apply_vm,dict(prepared,receipt=m.protected_json(folder/'receipt.json')))
|
||||
m.write_private(folder/'applied.json',result);print(result)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
message=str(error)
|
||||
if message.startswith('remote bootstrap incomplete: '):print(message)
|
||||
else:print('CLOUD_CHECKPOINT_REPAIR_INCOMPLETE:'+type(error).__name__)
|
||||
raise SystemExit(2) from None
|
||||
@@ -124,7 +124,7 @@ def deploy(payload):
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
|
||||
parser.add_argument('--preserve-known-unknown', action='store_true', help='Windows inventory migration only; preserve exact fenced APT evidence')
|
||||
parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path')
|
||||
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if args.action == 'stage':
|
||||
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
|
||||
@@ -160,7 +160,7 @@ def main():
|
||||
before['ledger']['result'] is None and before['ledger']['outcome']['error_code'] == 'DPKG_INCONSISTENT', 'preserve exact native unknown')
|
||||
closed = cloud.contract.protected_json(apt.STATE/'policy-closed.json')
|
||||
payload['closed_policy_sha256'] = closed['policy_sha256']
|
||||
require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'additive Windows migration only')
|
||||
require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'retain existing additive schema')
|
||||
else: require(gate['unresolved'] == 0, 'unresolved cloud job; no server switch')
|
||||
if args.action == 'deploy':
|
||||
stream = io.BytesIO()
|
||||
@@ -177,7 +177,7 @@ def main():
|
||||
gate_after = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
|
||||
current = cloud.Backend().call('jobs.get', dict(tenant_id=cloud.TENANT, job_id=job['job_id']))
|
||||
require(before == after and gate_after == gate and current['state'] == 'unknown' and current['attempt_id'] == job['attempt_id'],
|
||||
'unknown evidence or execution artifacts changed during Windows migration')
|
||||
'unknown evidence or execution artifacts changed during control-path upgrade')
|
||||
cloud.contract.write_private(STATE/(ident+'.unknown-preserved.json'), dict(evidence=after, gate=gate_after, result=current))
|
||||
result['unknown_attempt_preserved'] = True
|
||||
print(result)
|
||||
|
||||
@@ -1,5 +1,50 @@
|
||||
# Tatsächlicher Implementierungsstand
|
||||
|
||||
## Signierte Kontrollkettenreparatur auf Cloud-VM bestanden — 2026-11-13
|
||||
|
||||
Der qualifizierte Root-Bootstrap prüfte native gespeicherte Agent-/Helper-
|
||||
Historien und fertigte begrenzte Zeugen an; alle drei SQLite-Dateien blieben
|
||||
bei dieser Vorbereitung bytegleich. Agent und beide Helper kontrolliert gestoppt,
|
||||
keine Paketmanagerwirkung unterbrochen. Der neue ARM64-Serverbuild und das
|
||||
Deployment bestanden; exakte ursprüngliche APT-Evidenz und Zähler vor/nach
|
||||
Serverwechsel gleich. DB/PKI/Geräteidentität nicht ersetzt.
|
||||
|
||||
Server-CLI `repair-unsent-sync-request` prüfte ursprüngliche signierte Ablehnung,
|
||||
noch vorhandenen gültigen Serverhead, fehlenden ungesendeten Vorgänger,
|
||||
Geräte-/Boot-/Generationsbindung und geschlossene Rootpolicy. Bestehende
|
||||
wirkungslose Ablehnung monoton zum gemeinsamen Kontrollhead übernommen;
|
||||
separate kurzlebige Signatur `rmm.sync.unsent-request-repair` ausgestellt.
|
||||
Kein neuer HTTP-Endpunkt, keine interaktive Freigabe vorgetäuscht.
|
||||
|
||||
Native unabhängige Verifier akzeptierten ausschließlich die exakten Zeugen.
|
||||
Danach tatsächlich **0 Agentfaults / 0 Helperfaults**, jeweils **eine dauerhaft
|
||||
archivierte Reparaturquittung**, beide Integritätsprüfungen ok. Verbrauchsledger
|
||||
weiter **9 / 1 unresolved**, Zentrale weiter **13 Jobs / 13 Outbox / 12 Grants /
|
||||
1 unresolved**. Keine Ready-Lease, Jobs, Grants, Outcomeänderung oder neue
|
||||
Paketwirkung. Alle drei Dienste bleiben bis zum nachfolgenden qualifizierten
|
||||
Agent-/Readonlyhelperwechsel gestoppt. APT weiterhin unknown; die
|
||||
Kontrollreparatur ist ausdrücklich keine Paketabnahme oder Wiederholungsfreigabe.
|
||||
|
||||
Lokale Prüfungen dieser Quittungsstrecke: **1 CLI-Historienmutationstest,
|
||||
34 Helper-Library- und 21 RootSync-Tests**; Server **3 gezielte Prüfungen mit
|
||||
frischem isoliertem PostgreSQL**, einschließlicher verweigerter manipuliert/
|
||||
veraltet/falsch gebundener Zeugen und unveränderter Null-Jobartefakte.
|
||||
Format/Clippy `-D warnings` und Releasebuild bestanden. Außerdem fünf gezielte
|
||||
Guards für eng begrenzte Entwicklungs-Wirefaults geprüft; deren native
|
||||
Duplicate-/Lost-ACK-Abnahme steht weiterhin aus.
|
||||
|
||||
Ein erster lokaler Quittungsaufruf stoppte am normalen 1-MiB-DTO-Limit für die
|
||||
vollständigen historischen Zeugen, noch vor Serverzugriff. Separater begrenzter
|
||||
Private-Dateileser ergänzt (Owner, 0600, keine Symlinks/Hardlinks, maximal45MiB);
|
||||
identische ursprüngliche Zeugen anschließend erfolgreich verwendet. Geschützt:
|
||||
`devstate/checkpoint-chain-repair/2c5c3f06-a10e-40de-a02c-3c8009bc034f/` und
|
||||
`devstate/server-upgrades/2f6144c6-fcd5-49cb-9634-0d6e0b898165*`.
|
||||
|
||||
**MVP-Prüfung: weiterhin offen.** Nächster Schritt ist der ausschließlich
|
||||
lesende, an den ursprünglichen APT-Auftrag gebundene Teilwirkungsnachweis und
|
||||
seine Übernahme über den normalen signierten Sync, erst danach ein neuer
|
||||
WebUI-Wiederherstellungsplan. Weitere Gates: `mvp-gates.md`.
|
||||
|
||||
## Nicht persistierter Sync-Request als MVP-Blocker eingegrenzt — 2026-11-13
|
||||
|
||||
Vorprüfung des geplanten Readonly-APT-Helperupgrades stoppte **vor jeder
|
||||
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
# Linux-MVP: Abnahmegates
|
||||
|
||||
Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.9. Die
|
||||
Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.10. Die
|
||||
Bestandsaufnahme selbst hat keine Tests, Jobs oder Deployments ausgeführt;
|
||||
anschließende tatsächliche Nachweise sind unten ausdrücklich ergänzt. Einzelbelege und spätere
|
||||
Ergebnisse stehen im [Implementierungsstand](implementation-status.md).
|
||||
@@ -39,12 +39,12 @@ bestanden. Dauerhafte APT-Unknown-Evidenz unverändert, keine Jobs, Testservice
|
||||
gestoppt. M1 braucht weiterhin tatsächlichen WSS-Ausfall/Fallback; Gate 0 bleibt
|
||||
die nächste blockierende Verwaltungsstrecke. MVP insgesamt weiterhin offen.
|
||||
|
||||
**Aktueller Gate-0-Blocker:** Upgradevorprüfung fand in Agent und Sharedhelper
|
||||
eine gespeicherte `CHECKPOINT_CHAIN_MISMATCH`-Sperre. Nicht serverseitig
|
||||
persistierter Vorgänger plus verworfener bestätigter Anker ist anhand der
|
||||
gespeicherten Kette eingegrenzt; Prävention lokal geprüft, nicht deployt.
|
||||
Vor APT-Abgleich ist eine exakt signierte Kontrollquittung mit erhaltener
|
||||
Historie nötig. Keine direkte Faultlöschung oder pauschale Startfreigabe.
|
||||
**Gate-0-Teilerfolg:** Native signierte Kontrollkettenreparatur bestanden;
|
||||
ursprüngliche Faults mit je einer Quittung dauerhaft archiviert, jetzt beide
|
||||
faultfrei. Keine Outcome-/Verbrauchs-/Grantänderung: weiterhin 9/1 bzw.
|
||||
13/13/12/1. Dienste für qualifizierten Agent-/Readonlyhelperwechsel gestoppt.
|
||||
Nächster Schritt bleibt der gebundene APT-Teilwirkungsnachweis mit normalem
|
||||
signiertem Sync; die Kontrollreparatur schließt das Unknown nicht selbst.
|
||||
|
||||
Alle folgenden Schritte sind **geplant, nicht durch dieses Dokument ausgeführt**.
|
||||
Jeder Ausführungsjob braucht weiterhin seinen eigenen festen WebUI-genehmigten
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Verbindliche Projektentscheidungen
|
||||
|
||||
Stand: **2026-11-13**, Spezifikation **1.9** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
|
||||
Stand: **2026-11-13**, Spezifikation **1.10** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
|
||||
|
||||
## Einzelner signierter APT-Wiederaufnahmeversuch (2026-11-13)
|
||||
|
||||
@@ -113,3 +113,15 @@ Genehmigungsregeln gelten weiterhin.
|
||||
- VM-Daten: `credentials/linux-vm.conf`, Verzeichnis durch `.gitignore` ausgeschlossen, Modus 700 für `credentials/`, 600 für die Datei. Strikt als Daten parsen, kein `source`/`eval`; keine Secrets lesen/loggen/committen oder in LLM-Kontexte übernehmen.
|
||||
- Auch Linux-Dev hat keine Jobfreigabeabkürzung: erst `pending_approval`, dann WebUI-Genehmigung durch einen berechtigten Menschen oder ausschließlich die oben dokumentierte dev/test-Browsersimulation. Passives Standardinventar ist kein Job. Keine impliziten Reboots, Updates oder destruktiven Tests auf dem Ziel; ausdrücklich gesondert abstimmen, RMM-Jobs zusätzlich über ihren festen Plan in der WebUI genehmigen.
|
||||
- Kein Erfolg ohne realen Nachweis. Geplante Befehle und nicht ausgeführte Tests ausdrücklich so kennzeichnen; fehlende VM-Abnahme bleibt offen.
|
||||
|
||||
## Eng begrenzte Reparatur eines ungesendeten Kontrollrequests
|
||||
|
||||
Spezifikation 1.10 konkretisiert den existierenden Unknown-/Sync-Abgleich:
|
||||
Root-Bootstrap ausschließlich im Entwicklungsprofil, vollständige originale
|
||||
Zeugen, unabhängige lokale Signatur-/Inventarprüfung und separate kurzlebige
|
||||
Kontrollquittung. Nur vorhandene wirkungslose signierte Ablehnung als gemeinsamer
|
||||
monotoner Head; Faults und Quittung append-only archivieren. Keine Jobfreigabe,
|
||||
Bereit-Lease, Änderung von Outcome oder Verbrauch, keine Journalrücksetzung.
|
||||
Native Prüfung und Grenzen: `implementation-status.md`. Direkte API-/DB-
|
||||
Jobfreigaben bleiben verboten; anschließende Paketwirkung benötigt neuen festen
|
||||
WebUI-genehmigten Plan.
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checkpoint repair against a fresh, isolated PostgreSQL; no deployed state."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = "rmm-checkpoint-repair-" + uuid.uuid4().hex[:10]
|
||||
|
||||
|
||||
def command(args, data=None, timeout=120):
|
||||
result = subprocess.run(args, input=data, capture_output=True, timeout=timeout)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr.decode(errors="replace")[-4000:])
|
||||
return result.stdout
|
||||
|
||||
|
||||
def main():
|
||||
command(["docker", "run", "-d", "--network", "none", "--name", NAME,
|
||||
"-e", "POSTGRES_HOST_AUTH_METHOD=trust", "postgres:17.6-bookworm"])
|
||||
try:
|
||||
for _ in range(100):
|
||||
logs = command(["docker", "logs", NAME])
|
||||
probe = subprocess.run(["docker", "exec", NAME, "pg_isready", "-U", "postgres"],
|
||||
capture_output=True)
|
||||
if b"init process complete" in logs and probe.returncode == 0:
|
||||
break
|
||||
time.sleep(.2)
|
||||
else:
|
||||
raise RuntimeError("ISOLATED_POSTGRES_NOT_READY")
|
||||
|
||||
def sql(statement):
|
||||
return command(["docker", "exec", "-i", NAME, "psql", "-X", "-qAt",
|
||||
"-v", "ON_ERROR_STOP=1", "-U", "postgres"], statement.encode())
|
||||
|
||||
sql("CREATE ROLE rmm_app LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS;")
|
||||
for path in sorted((ROOT / "migrations").glob("*.sql")):
|
||||
sql(path.read_text())
|
||||
# Current source with a pre-Windows dependency manifest uses the cached
|
||||
# Linux toolchain; Windows is outside this Linux checkpoint test.
|
||||
manifests = tempfile.TemporaryDirectory(prefix="rmm-checkpoint-linux-manifest-")
|
||||
manifest = Path(manifests.name)
|
||||
for name in ("Cargo.toml", "Cargo.lock"):
|
||||
(manifest / name).write_bytes(command([
|
||||
"git", "-C", str(ROOT), "show", "a5c2dce:" + name]))
|
||||
output = command([
|
||||
"docker", "run", "--rm", "--network", "container:" + NAME,
|
||||
"--read-only", "--tmpfs", "/tmp:rw,size=512m",
|
||||
"-v", str(ROOT) + ":/src:ro",
|
||||
"-v", str(manifest / "Cargo.toml") + ":/src/Cargo.toml:ro",
|
||||
"-v", str(manifest / "Cargo.lock") + ":/src/Cargo.lock:ro",
|
||||
"-v", str(ROOT / "devstate/windows-client/target-linux") + ":/target",
|
||||
"-e", "CARGO_TARGET_DIR=/target",
|
||||
"-e", "RMM_TEST_REPAIR_ADMIN_URL=postgres://postgres@127.0.0.1/postgres",
|
||||
"-e", "RMM_TEST_REPAIR_APP_URL=postgres://rmm_app@127.0.0.1/postgres",
|
||||
"--entrypoint", "cargo", "rust-discovery-dev-build:local",
|
||||
"test", "--locked", "--offline", "-p", "rmm-server",
|
||||
"checkpoint_repair", "--", "--include-ignored", "--nocapture"], timeout=300)
|
||||
print(output.decode())
|
||||
print("PASS: isolated PostgreSQL repair, scope/revocation/stale-head/committed-request rejection, exact replay; zero jobs/outbox/permits and no readiness.")
|
||||
manifests.cleanup()
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", "-f", NAME], capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user