773 lines
34 KiB
Rust
773 lines
34 KiB
Rust
//! Fixed development bootstrap for a proven unsent sync request. This promotes
|
|
//! an existing signed denial only; it grants no readiness or execution rights.
|
|
use anyhow::{Context, Result, ensure};
|
|
use chrono::{Duration, Utc};
|
|
use openssl::{
|
|
pkey::{Id, PKey},
|
|
sign::Verifier,
|
|
};
|
|
use prost::Message;
|
|
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
|
|
use rmm_protocol::SyncRequest;
|
|
use serde_json::{Value, json};
|
|
use sqlx::Row;
|
|
use std::{
|
|
fs,
|
|
os::unix::fs::{MetadataExt, PermissionsExt},
|
|
path::Path,
|
|
};
|
|
use uuid::Uuid;
|
|
|
|
const EVENT: &str = "unsent_request_repair_issued";
|
|
const FAULT: &str = "CHECKPOINT_CHAIN_MISMATCH";
|
|
|
|
pub fn closed_policy() -> Result<()> {
|
|
ensure!(
|
|
std::env::var("RMM_PROFILE").as_deref() == Ok("development")
|
|
&& std::env::var("RMM_DEV_AUTH").as_deref() == Ok("true")
|
|
&& std::env::var("RMM_SERVER_ROLE").as_deref() == Ok("api"),
|
|
"REPAIR_DEVELOPMENT_API_ONLY"
|
|
);
|
|
let path = std::env::var("RMM_DEV_ROOT_POLICY_FILE").context("REPAIR_POLICY_REQUIRED")?;
|
|
let m = fs::symlink_metadata(&path)?;
|
|
let parent = fs::symlink_metadata(Path::new(&path).parent().context("REPAIR_POLICY_PARENT")?)?;
|
|
let key = std::env::var("RMM_JOB_KEY_FILE").context("REPAIR_EXISTING_KEY_REQUIRED")?;
|
|
ensure!(
|
|
m.is_file()
|
|
&& m.len() <= 65536
|
|
&& m.permissions().mode() & 0o077 == 0
|
|
&& m.uid() == fs::metadata(key)?.uid()
|
|
&& parent.is_dir()
|
|
&& parent.permissions().mode() & 0o022 == 0,
|
|
"REPAIR_POLICY_PERMISSIONS"
|
|
);
|
|
let p: Value = serde_json::from_slice(&fs::read(path)?)?;
|
|
ensure!(
|
|
p["schema_version"] == 1
|
|
&& p["profile"] == "development"
|
|
&& p["expires_at"]
|
|
.as_str()
|
|
.and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
|
|
.is_some_and(|until| until <= Utc::now()),
|
|
"REPAIR_POLICY_MUST_BE_CLOSED"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn cell<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a Value> {
|
|
let columns = table["columns"]
|
|
.as_array()
|
|
.context("REPAIR_INVENTORY_COLUMNS")?;
|
|
let i = columns
|
|
.iter()
|
|
.position(|v| v.as_str() == Some(column))
|
|
.context("REPAIR_INVENTORY_COLUMN")?;
|
|
row.as_array()
|
|
.and_then(|r| r.get(i))
|
|
.context("REPAIR_INVENTORY_CELL")
|
|
}
|
|
fn rows(table: &Value) -> Result<&Vec<Value>> {
|
|
table["rows"].as_array().context("REPAIR_INVENTORY_ROWS")
|
|
}
|
|
fn text<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a str> {
|
|
let v = cell(table, row, column)?;
|
|
ensure!(v["type"] == "text", "REPAIR_CELL_TYPE");
|
|
v["value"].as_str().context("REPAIR_CELL_TEXT")
|
|
}
|
|
fn integer(table: &Value, row: &Value, column: &str) -> Result<u64> {
|
|
let v = cell(table, row, column)?;
|
|
ensure!(v["type"] == "integer", "REPAIR_CELL_TYPE");
|
|
v["value"].as_u64().context("REPAIR_CELL_INTEGER")
|
|
}
|
|
fn blob(table: &Value, row: &Value, column: &str, bytes: &[u8]) -> Result<()> {
|
|
let v = cell(table, row, column)?;
|
|
ensure!(
|
|
v == &json!({"type":"blob","sha256":hash(bytes),"length":bytes.len()}),
|
|
"REPAIR_WITNESS_BLOB"
|
|
);
|
|
Ok(())
|
|
}
|
|
fn row_by<'a>(table: &'a Value, column: &str, id: &str) -> Result<&'a Value> {
|
|
let found = rows(table)?
|
|
.iter()
|
|
.filter(|row| text(table, row, column).ok() == Some(id))
|
|
.collect::<Vec<_>>();
|
|
ensure!(found.len() == 1, "REPAIR_WITNESS_ROW");
|
|
Ok(found[0])
|
|
}
|
|
fn decision(
|
|
signer: &crate::job_signer::JobSigner,
|
|
s: &SignedDecision,
|
|
) -> Result<ReconcileDecisionPayload> {
|
|
ensure!(
|
|
s.decision_bytes.len() <= 1024 * 1024 && s.signature.len() == 64,
|
|
"REPAIR_DECISION_SIZE"
|
|
);
|
|
let key = PKey::public_key_from_raw_bytes(&signer.public_key()?, Id::ED25519)?;
|
|
ensure!(
|
|
Verifier::new_without_digest(&key)?.verify_oneshot(&s.signature, &s.decision_bytes)?,
|
|
"REPAIR_DECISION_SIGNATURE"
|
|
);
|
|
let d: ReconcileDecisionPayload = serde_json::from_slice(&s.decision_bytes)?;
|
|
ensure!(
|
|
d.valid_profile(d.issued_at) && d.key_id == signer.key_id,
|
|
"REPAIR_DECISION_PROFILE"
|
|
);
|
|
Ok(d)
|
|
}
|
|
|
|
struct Case {
|
|
old: ReconcileDecisionPayload,
|
|
rejected: ReconcileDecisionPayload,
|
|
unsent: SyncRequest,
|
|
request: SyncRequest,
|
|
}
|
|
fn validate(signer: &crate::job_signer::JobSigner, e: &Evidence) -> Result<Case> {
|
|
ensure!(
|
|
e.schema_version == 1
|
|
&& e.uncommitted_request.len() <= 1024 * 1024
|
|
&& e.rejected_request.len() <= 1024 * 1024,
|
|
"REPAIR_EVIDENCE_PROFILE"
|
|
);
|
|
let old = decision(signer, &e.previous_valid_response)?;
|
|
let rejected = decision(signer, &e.rejected_response)?;
|
|
for d in [&old, &rejected] {
|
|
ensure!(
|
|
d.tenant_id == e.tenant_id
|
|
&& d.asset_id == e.asset_id
|
|
&& d.agent_id == e.agent_id
|
|
&& d.enrollment_generation == e.enrollment_generation
|
|
&& d.assignment_version == e.assignment_version,
|
|
"REPAIR_IDENTITY_SCOPE"
|
|
);
|
|
}
|
|
let unsent = SyncRequest::decode(e.uncommitted_request.as_slice())?;
|
|
let request = SyncRequest::decode(e.rejected_request.as_slice())?;
|
|
let unsent_id = Uuid::parse_str(&unsent.request_id)?;
|
|
ensure!(
|
|
!unsent_id.is_nil()
|
|
&& unsent_id != rejected.request_id
|
|
&& unsent_id != old.request_id
|
|
&& old.decision_id != rejected.decision_id
|
|
&& rejected.request_id.to_string() == request.request_id
|
|
&& rejected.request_hash == hash(&e.rejected_request),
|
|
"REPAIR_REQUEST_BINDING"
|
|
);
|
|
ensure!(
|
|
rejected.reason == FAULT
|
|
&& !rejected.ready
|
|
&& !rejected.helper_ready
|
|
&& !rejected.attempts.iter().any(|a| a.unlock),
|
|
"REPAIR_DENIAL_ONLY"
|
|
);
|
|
ensure!(
|
|
request.previous_decision_id.is_empty()
|
|
&& request.previous_request_id == unsent.request_id
|
|
&& unsent.previous_decision_id == old.decision_id.to_string()
|
|
&& unsent.previous_request_id.is_empty(),
|
|
"REPAIR_EXACT_UNSENT_CHAIN"
|
|
);
|
|
let helper = rejected
|
|
.helper_journal_id
|
|
.context("REPAIR_HELPER_REQUIRED")?;
|
|
ensure!(
|
|
old.journal_id == rejected.journal_id
|
|
&& old.helper_journal_id == Some(helper)
|
|
&& old.checkpoint < unsent.checkpoint
|
|
&& unsent.checkpoint < rejected.checkpoint
|
|
&& old.helper_checkpoint <= unsent.helper_checkpoint
|
|
&& unsent.helper_checkpoint <= rejected.helper_checkpoint
|
|
&& old.connection_generation <= rejected.connection_generation,
|
|
"REPAIR_MONOTONIC_CHAIN"
|
|
);
|
|
for r in [&unsent, &request] {
|
|
ensure!(
|
|
r.journal_complete
|
|
&& r.assignment_version == e.assignment_version
|
|
&& r.journal_id == rejected.journal_id.to_string()
|
|
&& r.helper_journal_id == helper.to_string(),
|
|
"REPAIR_REQUEST_SCOPE"
|
|
);
|
|
}
|
|
ensure!(
|
|
request.checkpoint == rejected.checkpoint
|
|
&& request.helper_checkpoint == rejected.helper_checkpoint
|
|
&& request.helper_previous_decision_id == old.decision_id.to_string(),
|
|
"REPAIR_REQUEST_CHECKPOINT"
|
|
);
|
|
let am = &e.agent_inventory["reconcile_meta"];
|
|
let a = rows(am)?;
|
|
ensure!(a.len() == 1, "REPAIR_AGENT_META");
|
|
let a = &a[0];
|
|
ensure!(
|
|
integer(am, a, "id")? == 1
|
|
&& text(am, a, "journal_id")? == rejected.journal_id.to_string()
|
|
&& integer(am, a, "checkpoint")? == rejected.checkpoint
|
|
&& integer(am, a, "high_sequence")? >= rejected.checkpoint
|
|
&& integer(am, a, "generation")? == rejected.connection_generation
|
|
&& text(am, a, "last_decision")? == rejected.decision_id.to_string()
|
|
&& text(am, a, "helper_id")? == helper.to_string()
|
|
&& integer(am, a, "helper_checkpoint")? == rejected.helper_checkpoint
|
|
&& text(am, a, "fault")? == FAULT
|
|
&& cell(am, a, "pending_request")? == &json!({"type":"null"}),
|
|
"REPAIR_AGENT_HEAD"
|
|
);
|
|
let hm = &e.helper_inventory["helper_sync_meta"];
|
|
let h = rows(hm)?;
|
|
ensure!(h.len() == 1, "REPAIR_HELPER_META");
|
|
let h = &h[0];
|
|
ensure!(
|
|
integer(hm, h, "id")? == 1
|
|
&& text(hm, h, "journal_id")? == helper.to_string()
|
|
&& integer(hm, h, "checkpoint")? == rejected.helper_checkpoint
|
|
&& text(hm, h, "last_decision")? == rejected.decision_id.to_string()
|
|
&& text(hm, h, "agent_journal_id")? == rejected.journal_id.to_string()
|
|
&& integer(hm, h, "agent_checkpoint")? == rejected.checkpoint
|
|
&& text(hm, h, "fault")? == FAULT,
|
|
"REPAIR_HELPER_HEAD"
|
|
);
|
|
let ar = &e.agent_inventory["reconcile_requests"];
|
|
let x = row_by(ar, "id", &unsent.request_id)?;
|
|
blob(ar, x, "request", &e.uncommitted_request)?;
|
|
ensure!(
|
|
integer(ar, x, "generation")? <= rejected.connection_generation
|
|
&& cell(ar, x, "response")? == &json!({"type":"null"}),
|
|
"REPAIR_UNSENT_LOCAL_RESPONSE"
|
|
);
|
|
let y = row_by(ar, "id", &request.request_id)?;
|
|
blob(ar, y, "request", &e.rejected_request)?;
|
|
blob(
|
|
ar,
|
|
y,
|
|
"response",
|
|
&rmm_protocol::SyncResponse {
|
|
decision_bytes: e.rejected_response.decision_bytes.clone(),
|
|
signature: e.rejected_response.signature.clone(),
|
|
}
|
|
.encode_to_vec(),
|
|
)?;
|
|
ensure!(
|
|
integer(ar, y, "generation")? == rejected.connection_generation,
|
|
"REPAIR_LOCAL_GENERATION"
|
|
);
|
|
let hr = &e.helper_inventory["helper_sync_decisions"];
|
|
let y = row_by(hr, "decision_id", &rejected.decision_id.to_string())?;
|
|
blob(hr, y, "request", &e.rejected_request)?;
|
|
blob(
|
|
hr,
|
|
y,
|
|
"response",
|
|
&serde_json::to_vec(&e.rejected_response)?,
|
|
)?;
|
|
ensure!(
|
|
integer(hr, y, "checkpoint")? == rejected.helper_checkpoint,
|
|
"REPAIR_HELPER_CAPTURE"
|
|
);
|
|
ensure!(
|
|
e.helper_inventory["consumed"].is_object()
|
|
&& e.helper_inventory["execution_evidence"].is_object(),
|
|
"REPAIR_LEDGER_REQUIRED"
|
|
);
|
|
Ok(Case {
|
|
old,
|
|
rejected,
|
|
unsent,
|
|
request,
|
|
})
|
|
}
|
|
|
|
pub async fn issue(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
|
|
closed_policy()?;
|
|
issue_with_closed_policy(state, e).await
|
|
}
|
|
async fn issue_with_closed_policy(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
|
|
let c = validate(&state.job_signer, &e)?;
|
|
let evidence_hash = inventory_sha256(&serde_json::to_value(&e)?)?;
|
|
let mut tx = rmm_persistence::tenant_tx(&state.pool, e.tenant_id).await?;
|
|
crate::agent_service::control_lock(&mut tx, e.tenant_id, e.agent_id).await?;
|
|
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,revoked_at,sequence_regressed,sync_recovery_required,sync_ready,helper_sync_ready FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
|
|
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_DEVICE_MISSING")?;
|
|
ensure!(
|
|
device.get::<Uuid, _>("asset_id") == e.asset_id
|
|
&& device.get::<i64, _>("enrollment_generation")
|
|
== i64::try_from(e.enrollment_generation)?
|
|
&& device.get::<i64, _>("assignment_version") == i64::try_from(e.assignment_version)?
|
|
// Faulted journals may survive newer ClientHello channels. This
|
|
// historical receipt grants no lease; the next ordinary sync must
|
|
// still bind its current channel. A regressed generation or changed
|
|
// boot/identity is never repaired here.
|
|
&& device.get::<i64, _>("connection_generation")
|
|
>= i64::try_from(c.rejected.connection_generation)?
|
|
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(c.rejected.boot_id)
|
|
&& device
|
|
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
|
|
.is_none()
|
|
&& ![
|
|
"sequence_regressed",
|
|
"sync_recovery_required",
|
|
"sync_ready",
|
|
"helper_sync_ready"
|
|
]
|
|
.iter()
|
|
.any(|k| device.get::<bool, _>(*k)),
|
|
"REPAIR_DEVICE_FENCED"
|
|
);
|
|
for (d, s) in [
|
|
(&c.old, &e.previous_valid_response),
|
|
(&c.rejected, &e.rejected_response),
|
|
] {
|
|
let stored=sqlx::query("SELECT decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3")
|
|
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).fetch_optional(&mut *tx).await?.context("REPAIR_STORED_DECISION_MISSING")?;
|
|
ensure!(
|
|
stored.get::<Uuid, _>("decision_id") == d.decision_id
|
|
&& stored.get::<String, _>("request_hash") == d.request_hash
|
|
&& stored.get::<Uuid, _>("journal_id") == d.journal_id
|
|
&& stored.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
|
|
&& stored.get::<i64, _>("connection_generation")
|
|
== i64::try_from(d.connection_generation)?
|
|
&& stored.get::<Vec<u8>, _>("payload") == s.decision_bytes
|
|
&& stored.get::<Vec<u8>, _>("signature") == s.signature,
|
|
"REPAIR_STORED_DECISION_MISMATCH"
|
|
);
|
|
}
|
|
let committed:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3)")
|
|
.bind(e.tenant_id).bind(e.agent_id).bind(Uuid::parse_str(&c.unsent.request_id)?).fetch_one(&mut *tx).await?;
|
|
ensure!(!committed, "REPAIR_REQUEST_WAS_COMMITTED");
|
|
let head=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
|
|
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_HEAD_MISSING")?;
|
|
let previous:Option<Value>=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type=$2 AND data->>'evidence_sha256'=$3 AND data->>'agent_id'=$4 ORDER BY created_at DESC LIMIT 1")
|
|
.bind(e.tenant_id).bind(EVENT).bind(&evidence_hash).bind(e.agent_id.to_string()).fetch_optional(&mut *tx).await?;
|
|
if let Some(previous) = previous {
|
|
let signed: SignedReceipt = serde_json::from_value(previous["receipt"].clone())?;
|
|
let receipt: Receipt = serde_json::from_slice(&signed.payload)?;
|
|
let key = PKey::public_key_from_raw_bytes(&state.job_signer.public_key()?, Id::ED25519)?;
|
|
ensure!(
|
|
signed.signature.len() == 64
|
|
&& Verifier::new_without_digest(&key)?
|
|
.verify_oneshot(&signed.signature, &signed.payload)?
|
|
&& receipt.valid_profile(Utc::now())
|
|
&& receipt.key_id == state.job_signer.key_id
|
|
&& receipt.tenant_id == e.tenant_id
|
|
&& receipt.asset_id == e.asset_id
|
|
&& receipt.agent_id == e.agent_id
|
|
&& receipt.rejected_decision_id == c.rejected.decision_id
|
|
&& receipt.rejected_request_sha256 == hash(&e.rejected_request)
|
|
&& receipt.uncommitted_request_sha256 == hash(&e.uncommitted_request)
|
|
&& receipt.agent_inventory_sha256 == inventory_sha256(&e.agent_inventory)?
|
|
&& receipt.helper_inventory_sha256 == inventory_sha256(&e.helper_inventory)?
|
|
&& head_matches(&head, &c.rejected)?,
|
|
"REPAIR_REPLAY_EXPIRED_OR_HEAD_CHANGED"
|
|
);
|
|
tx.commit().await?;
|
|
return Ok(signed);
|
|
}
|
|
ensure!(
|
|
head_matches(&head, &c.old)?,
|
|
"REPAIR_AUTHORITATIVE_HEAD_CHANGED"
|
|
);
|
|
let now = Utc::now();
|
|
let receipt = Receipt {
|
|
domain: "rmm.sync.unsent-request-repair".into(),
|
|
schema_version: 1,
|
|
receipt_id: Uuid::new_v4(),
|
|
tenant_id: e.tenant_id,
|
|
asset_id: e.asset_id,
|
|
agent_id: e.agent_id,
|
|
enrollment_generation: e.enrollment_generation,
|
|
assignment_version: e.assignment_version,
|
|
issued_at: now,
|
|
valid_until: now + Duration::minutes(10),
|
|
key_id: state.job_signer.key_id.clone(),
|
|
fault: FAULT.into(),
|
|
agent_journal_id: c.rejected.journal_id,
|
|
agent_checkpoint: c.rejected.checkpoint,
|
|
helper_journal_id: c
|
|
.rejected
|
|
.helper_journal_id
|
|
.context("REPAIR_HELPER_REQUIRED")?,
|
|
helper_checkpoint: c.rejected.helper_checkpoint,
|
|
rejected_decision_id: c.rejected.decision_id,
|
|
rejected_decision_sha256: hash(&e.rejected_response.decision_bytes),
|
|
rejected_request_id: c.rejected.request_id,
|
|
rejected_request_sha256: hash(&e.rejected_request),
|
|
uncommitted_request_id: Uuid::parse_str(&c.unsent.request_id)?,
|
|
uncommitted_request_sha256: hash(&e.uncommitted_request),
|
|
previous_valid_decision_id: c.old.decision_id,
|
|
previous_valid_decision_sha256: hash(&e.previous_valid_response.decision_bytes),
|
|
agent_inventory_sha256: inventory_sha256(&e.agent_inventory)?,
|
|
helper_inventory_sha256: inventory_sha256(&e.helper_inventory)?,
|
|
helper_ledger_sha256: helper_ledger_sha256(&e.helper_inventory)?,
|
|
ready: false,
|
|
helper_ready: false,
|
|
unlock: false,
|
|
};
|
|
ensure!(receipt.valid_profile(now), "REPAIR_RECEIPT_PROFILE");
|
|
let payload = serde_json::to_vec(&receipt)?;
|
|
let signed = SignedReceipt {
|
|
signature: state.job_signer.sign(&payload)?,
|
|
payload,
|
|
};
|
|
ensure!(sqlx::query("UPDATE agent_sync_state SET checkpoint=$3,decision_id=$4,helper_journal_id=$5,helper_checkpoint=$6 WHERE tenant_id=$1 AND agent_id=$2 AND decision_id=$7")
|
|
.bind(e.tenant_id).bind(e.agent_id).bind(i64::try_from(c.request.checkpoint)?).bind(c.rejected.decision_id)
|
|
.bind(c.rejected.helper_journal_id).bind(i64::try_from(c.rejected.helper_checkpoint)?).bind(c.old.decision_id)
|
|
.execute(&mut *tx).await?.rows_affected()==1,"REPAIR_HEAD_CONCURRENT_CHANGE");
|
|
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,$4,$5)")
|
|
.bind(e.tenant_id).bind(receipt.receipt_id).bind("development-bootstrap:unsent-sync-repair").bind(EVENT)
|
|
.bind(json!({"agent_id":e.agent_id,"evidence_sha256":evidence_hash,"receipt_sha256":hash(&signed.payload),"receipt":signed,"ready":false,"helper_ready":false,"unlock":false}))
|
|
.execute(&mut *tx).await?;
|
|
tx.commit().await?;
|
|
Ok(signed)
|
|
}
|
|
fn head_matches(head: &sqlx::postgres::PgRow, d: &ReconcileDecisionPayload) -> Result<bool> {
|
|
Ok(head.get::<Uuid, _>("journal_id") == d.journal_id
|
|
&& head.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
|
|
&& head.get::<Uuid, _>("decision_id") == d.decision_id
|
|
&& head.get::<Option<Uuid>, _>("helper_journal_id") == d.helper_journal_id
|
|
&& head.get::<i64, _>("helper_checkpoint") == i64::try_from(d.helper_checkpoint)?)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
fn table(fields: Vec<(&str, Value)>) -> Value {
|
|
json!({"columns":fields.iter().map(|(k,_)|k).collect::<Vec<_>>(),"rows":[fields.iter().map(|(_,v)|v).collect::<Vec<_>>() ]})
|
|
}
|
|
fn t(v: impl ToString) -> Value {
|
|
json!({"type":"text","value":v.to_string()})
|
|
}
|
|
fn n(v: u64) -> Value {
|
|
json!({"type":"integer","value":v})
|
|
}
|
|
fn b(v: &[u8]) -> Value {
|
|
json!({"type":"blob","sha256":hash(v),"length":v.len()})
|
|
}
|
|
fn sign(s: &crate::job_signer::JobSigner, d: &ReconcileDecisionPayload) -> SignedDecision {
|
|
let decision_bytes = serde_json::to_vec(d).unwrap();
|
|
SignedDecision {
|
|
signature: s.sign(&decision_bytes).unwrap(),
|
|
decision_bytes,
|
|
}
|
|
}
|
|
fn fixture() -> (crate::job_signer::JobSigner, Evidence, std::path::PathBuf) {
|
|
let path = std::path::PathBuf::from(format!("/tmp/rmm-repair-{}", Uuid::new_v4()));
|
|
let signer = crate::job_signer::JobSigner::load_or_create(&path.join("key.pem")).unwrap();
|
|
let tenant = Uuid::new_v4();
|
|
let asset = Uuid::new_v4();
|
|
let agent = Uuid::new_v4();
|
|
let journal = Uuid::new_v4();
|
|
let helper = Uuid::new_v4();
|
|
let now = Utc::now();
|
|
let old = ReconcileDecisionPayload {
|
|
domain: "rmm.reconciliation-decision".into(),
|
|
schema_version: 1,
|
|
decision_id: Uuid::new_v4(),
|
|
request_id: Uuid::new_v4(),
|
|
request_hash: "ab".repeat(32),
|
|
tenant_id: tenant,
|
|
asset_id: asset,
|
|
agent_id: agent,
|
|
enrollment_generation: 1,
|
|
assignment_version: 1,
|
|
connection_generation: 3,
|
|
boot_id: Uuid::new_v4(),
|
|
journal_id: journal,
|
|
checkpoint: 10,
|
|
helper_journal_id: Some(helper),
|
|
helper_checkpoint: 5,
|
|
issued_at: now,
|
|
valid_until: now + Duration::seconds(60),
|
|
ready: false,
|
|
helper_ready: false,
|
|
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
|
|
attempts: vec![],
|
|
acknowledged_message_ids: vec![],
|
|
key_id: signer.key_id.clone(),
|
|
};
|
|
let x = SyncRequest {
|
|
request_id: Uuid::new_v4().to_string(),
|
|
journal_id: journal.to_string(),
|
|
checkpoint: 11,
|
|
previous_decision_id: old.decision_id.to_string(),
|
|
assignment_version: 1,
|
|
journal_complete: true,
|
|
helper_journal_id: helper.to_string(),
|
|
helper_checkpoint: 6,
|
|
helper_previous_decision_id: old.decision_id.to_string(),
|
|
..Default::default()
|
|
};
|
|
let y = SyncRequest {
|
|
request_id: Uuid::new_v4().to_string(),
|
|
checkpoint: 12,
|
|
previous_decision_id: String::new(),
|
|
previous_request_id: x.request_id.clone(),
|
|
helper_checkpoint: 7,
|
|
..x.clone()
|
|
};
|
|
let mut rejected = old.clone();
|
|
rejected.decision_id = Uuid::new_v4();
|
|
rejected.request_id = Uuid::parse_str(&y.request_id).unwrap();
|
|
rejected.request_hash = hash(&y.encode_to_vec());
|
|
rejected.checkpoint = 12;
|
|
rejected.helper_checkpoint = 7;
|
|
rejected.reason = FAULT.into();
|
|
let response = sign(&signer, &rejected);
|
|
let request = y.encode_to_vec();
|
|
let unsent = x.encode_to_vec();
|
|
let proto = rmm_protocol::SyncResponse {
|
|
decision_bytes: response.decision_bytes.clone(),
|
|
signature: response.signature.clone(),
|
|
}
|
|
.encode_to_vec();
|
|
let mut requests = table(vec![
|
|
("id", t(&x.request_id)),
|
|
("request", b(&unsent)),
|
|
("generation", n(3)),
|
|
("response", json!({"type":"null"})),
|
|
]);
|
|
requests["rows"].as_array_mut().unwrap().push(json!([
|
|
t(&y.request_id),
|
|
b(&request),
|
|
n(3),
|
|
b(&proto)
|
|
]));
|
|
let am = table(vec![
|
|
("id", n(1)),
|
|
("journal_id", t(journal)),
|
|
("checkpoint", n(12)),
|
|
("high_sequence", n(13)),
|
|
("generation", n(3)),
|
|
("last_decision", t(rejected.decision_id)),
|
|
("pending_request", json!({"type":"null"})),
|
|
("helper_id", t(helper)),
|
|
("helper_checkpoint", n(7)),
|
|
("fault", t(FAULT)),
|
|
]);
|
|
let hm = table(vec![
|
|
("id", n(1)),
|
|
("journal_id", t(helper)),
|
|
("checkpoint", n(7)),
|
|
("last_decision", t(rejected.decision_id)),
|
|
("agent_journal_id", t(journal)),
|
|
("agent_checkpoint", n(12)),
|
|
("fault", t(FAULT)),
|
|
]);
|
|
let history = table(vec![
|
|
("decision_id", t(rejected.decision_id)),
|
|
("checkpoint", n(7)),
|
|
("response", b(&serde_json::to_vec(&response).unwrap())),
|
|
("request", b(&request)),
|
|
]);
|
|
let e = Evidence {
|
|
schema_version: 1,
|
|
tenant_id: tenant,
|
|
asset_id: asset,
|
|
agent_id: agent,
|
|
enrollment_generation: 1,
|
|
assignment_version: 1,
|
|
agent_inventory: json!({"reconcile_meta":am,"reconcile_requests":requests}),
|
|
helper_inventory: json!({"helper_sync_meta":hm,"helper_sync_decisions":history,"consumed":{"columns":[],"rows":[]},"execution_evidence":{"columns":[],"rows":[]}}),
|
|
uncommitted_request: unsent,
|
|
rejected_request: request,
|
|
rejected_response: response,
|
|
previous_valid_response: sign(&signer, &old),
|
|
};
|
|
(signer, e, path)
|
|
}
|
|
#[test]
|
|
fn only_exact_signed_unsent_case_validates() {
|
|
let (s, e, p) = fixture();
|
|
validate(&s, &e).unwrap();
|
|
let mut other = e.clone();
|
|
other.tenant_id = Uuid::new_v4();
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
other.asset_id = Uuid::new_v4();
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
other.assignment_version += 1;
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
other.rejected_response.signature[0] ^= 1;
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
other.agent_inventory["reconcile_requests"]["rows"][0][3] = b(b"committed response");
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
other.helper_inventory["helper_sync_meta"]["rows"][0][2] = n(8);
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
other.agent_inventory["reconcile_meta"]["rows"][0][9] = t("OTHER_FAULT");
|
|
assert!(validate(&s, &other).is_err());
|
|
let mut other = e.clone();
|
|
let mut x = SyncRequest::decode(other.uncommitted_request.as_slice()).unwrap();
|
|
x.previous_decision_id = Uuid::new_v4().to_string();
|
|
other.uncommitted_request = x.encode_to_vec();
|
|
assert!(validate(&s, &other).is_err());
|
|
fs::remove_dir_all(p).unwrap();
|
|
}
|
|
#[test]
|
|
fn signed_ready_response_is_never_repair_authority() {
|
|
let (s, mut e, p) = fixture();
|
|
let mut d: ReconcileDecisionPayload =
|
|
serde_json::from_slice(&e.rejected_response.decision_bytes).unwrap();
|
|
d.ready = true;
|
|
e.rejected_response = sign(&s, &d);
|
|
assert!(validate(&s, &e).is_err());
|
|
d.ready = false;
|
|
d.reason = "CONSISTENT".into();
|
|
e.rejected_response = sign(&s, &d);
|
|
assert!(validate(&s, &e).is_err());
|
|
fs::remove_dir_all(p).unwrap();
|
|
}
|
|
/// Opt-in dedicated PostgreSQL only; the harness creates/destroys its own
|
|
/// network-isolated container and never reads deployment credentials.
|
|
#[tokio::test]
|
|
#[ignore = "requires the dedicated isolated PostgreSQL harness"]
|
|
async fn isolated_database_repair_and_fail_closed_paths() {
|
|
let admin_url = std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap();
|
|
let app_url = std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap();
|
|
let admin = sqlx::PgPool::connect(&admin_url).await.unwrap();
|
|
let pool = sqlx::PgPool::connect(&app_url).await.unwrap();
|
|
rmm_persistence::verify_application_role(&pool)
|
|
.await
|
|
.unwrap();
|
|
let (s, e, p) = fixture();
|
|
let c = validate(&s, &e).unwrap();
|
|
sqlx::query("INSERT INTO tenants VALUES($1,'isolated repair test')")
|
|
.bind(e.tenant_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
|
|
.bind(e.tenant_id)
|
|
.bind(e.asset_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,assignment_version,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5,$6)")
|
|
.bind(e.tenant_id).bind(e.agent_id).bind(e.asset_id).bind(hash(e.agent_id.as_bytes())).bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(c.rejected.boot_id).execute(&admin).await.unwrap();
|
|
sqlx::query("INSERT INTO agent_sync_state VALUES($1,$2,$3,$4,$5,$6,$7)")
|
|
.bind(e.tenant_id)
|
|
.bind(e.agent_id)
|
|
.bind(c.old.journal_id)
|
|
.bind(i64::try_from(c.old.checkpoint).unwrap())
|
|
.bind(c.old.decision_id)
|
|
.bind(c.old.helper_journal_id)
|
|
.bind(i64::try_from(c.old.helper_checkpoint).unwrap())
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
for (d, signed) in [
|
|
(&c.old, &e.previous_valid_response),
|
|
(&c.rejected, &e.rejected_response),
|
|
] {
|
|
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
|
|
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).bind(d.decision_id).bind(&d.request_hash).bind(d.journal_id).bind(i64::try_from(d.checkpoint).unwrap()).bind(i64::try_from(d.connection_generation).unwrap()).bind(&signed.decision_bytes).bind(&signed.signature).execute(&admin).await.unwrap();
|
|
}
|
|
let state = crate::State {
|
|
pool,
|
|
signer: None,
|
|
viewer_a: [0; 32],
|
|
viewer_b: [1; 32],
|
|
auth: std::sync::Arc::new(crate::auth::Auth::from_env("development").await.unwrap()),
|
|
job_signer: std::sync::Arc::new(s),
|
|
};
|
|
let mut wrong = e.clone();
|
|
wrong.tenant_id = Uuid::new_v4();
|
|
assert!(issue_with_closed_policy(&state, wrong).await.is_err());
|
|
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
|
|
.bind(e.tenant_id)
|
|
.bind(c.rejected.request_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
|
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
|
|
.bind(e.tenant_id).bind(e.agent_id).bind(c.rejected.request_id).bind(c.rejected.decision_id)
|
|
.bind(&c.rejected.request_hash).bind(c.rejected.journal_id).bind(i64::try_from(c.rejected.checkpoint).unwrap())
|
|
.bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(&e.rejected_response.decision_bytes)
|
|
.bind(&e.rejected_response.signature).execute(&admin).await.unwrap();
|
|
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint+1 WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
|
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint-1 WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("UPDATE agent_installations SET revoked_at=now() WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
|
sqlx::query("UPDATE agent_installations SET revoked_at=NULL WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
let x = Uuid::parse_str(&c.unsent.request_id).unwrap();
|
|
sqlx::query("INSERT INTO sync_checkpoints SELECT tenant_id,agent_id,$2,$3,request_hash,journal_id,checkpoint,connection_generation,payload,signature,created_at FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$4")
|
|
.bind(e.tenant_id).bind(x).bind(Uuid::new_v4()).bind(c.old.request_id).execute(&admin).await.unwrap();
|
|
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
|
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
|
|
.bind(e.tenant_id)
|
|
.bind(x)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("UPDATE agent_installations SET connection_generation=$2 WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.bind(i64::try_from(c.rejected.connection_generation).unwrap() - 1)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
|
sqlx::query("UPDATE agent_installations SET connection_generation=$2,connection_boot_id=$3 WHERE agent_id=$1")
|
|
.bind(e.agent_id).bind(i64::try_from(c.rejected.connection_generation).unwrap()+1).bind(Uuid::new_v4())
|
|
.execute(&admin).await.unwrap();
|
|
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
|
|
sqlx::query("UPDATE agent_installations SET connection_boot_id=$2 WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.bind(c.rejected.boot_id)
|
|
.execute(&admin)
|
|
.await
|
|
.unwrap();
|
|
let first = issue_with_closed_policy(&state, e.clone()).await.unwrap();
|
|
assert_eq!(
|
|
first,
|
|
issue_with_closed_policy(&state, e.clone()).await.unwrap()
|
|
);
|
|
let r: Receipt = serde_json::from_slice(&first.payload).unwrap();
|
|
assert!(r.valid_profile(Utc::now()));
|
|
assert!(!r.ready && !r.helper_ready && !r.unlock);
|
|
let head: Uuid =
|
|
sqlx::query_scalar("SELECT decision_id FROM agent_sync_state WHERE agent_id=$1")
|
|
.bind(e.agent_id)
|
|
.fetch_one(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(head, c.rejected.decision_id);
|
|
let count: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
|
|
.bind(e.tenant_id)
|
|
.fetch_one(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(count, 1);
|
|
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM outbox)+(SELECT count(*) FROM execution_permits)").fetch_one(&admin).await.unwrap();
|
|
assert_eq!(effects, 0);
|
|
let ready: bool = sqlx::query_scalar(
|
|
"SELECT sync_ready OR helper_sync_ready FROM agent_installations WHERE agent_id=$1",
|
|
)
|
|
.bind(e.agent_id)
|
|
.fetch_one(&admin)
|
|
.await
|
|
.unwrap();
|
|
assert!(!ready);
|
|
fs::remove_dir_all(p).unwrap();
|
|
}
|
|
}
|