fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-13 17:43:38 +01:00
parent 998f16fde4
commit 359233c64d
21 changed files with 3030 additions and 30 deletions
+47 -9
View File
@@ -38,6 +38,35 @@ fn response(messages: Vec<Envelope>) -> Response {
)
.into_response()
}
async fn append_work(
s: &crate::State,
tenant: Uuid,
agent: Uuid,
source: &Envelope,
payload: Payload,
generation: u64,
replies: &mut Vec<Envelope>,
) -> anyhow::Result<()> {
let reply = crate::agent_service::reply(source, payload, generation);
// Preserve the protocol's 16-envelope response limit. An offer that cannot
// fit twice is delivered normally without consuming the optional fault.
let claim = if replies.len() <= 14 {
crate::gateway::dev_delivery_faults::duplicate_offer(
&s.pool, tenant, agent, &reply, "https",
)
.await?
} else {
None
};
if claim.is_some() {
replies.push(reply.clone());
}
replies.push(reply);
if let Some(claim) = claim {
crate::gateway::dev_delivery_faults::duplicate_emitted(&s.pool, claim).await?;
}
Ok(())
}
pub async fn messages(
State(s): State<crate::State>,
Extension(peer): Extension<crate::Peer>,
@@ -83,13 +112,21 @@ pub async fn messages(
}
.map_err(|_| StatusCode::CONFLICT)?;
generation = reply.connection_generation;
replies.push(reply);
if !crate::gateway::dev_delivery_faults::after_ingest(&s.pool, t, a, &e, &reply, "https")
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
replies.push(reply);
}
}
if let Some(work) = crate::job_gateway::next(&s, t, a, generation)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
if replies.len() < 16
&& let Some(work) = crate::job_gateway::next(&s, t, a, generation)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
replies.push(crate::agent_service::reply(&last, work, generation));
append_work(&s, t, a, &last, work, generation, &mut replies)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
}
Ok(response(replies))
}
@@ -119,10 +156,11 @@ pub async fn poll(
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
{
return Ok(response(vec![
ack,
crate::agent_service::reply(&e, work, e.connection_generation),
]));
let mut replies = vec![ack];
append_work(&s, t, a, &e, work, e.connection_generation, &mut replies)
.await
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
return Ok(response(replies));
}
if tokio::time::Instant::now() >= deadline {
return Ok(response(vec![ack]));
+772
View File
@@ -0,0 +1,772 @@
//! Fixed development bootstrap for a proven unsent sync request. This promotes
//! an existing signed denial only; it grants no readiness or execution rights.
use anyhow::{Context, Result, ensure};
use chrono::{Duration, Utc};
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use prost::Message;
use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload};
use rmm_protocol::SyncRequest;
use serde_json::{Value, json};
use sqlx::Row;
use std::{
fs,
os::unix::fs::{MetadataExt, PermissionsExt},
path::Path,
};
use uuid::Uuid;
const EVENT: &str = "unsent_request_repair_issued";
const FAULT: &str = "CHECKPOINT_CHAIN_MISMATCH";
pub fn closed_policy() -> Result<()> {
ensure!(
std::env::var("RMM_PROFILE").as_deref() == Ok("development")
&& std::env::var("RMM_DEV_AUTH").as_deref() == Ok("true")
&& std::env::var("RMM_SERVER_ROLE").as_deref() == Ok("api"),
"REPAIR_DEVELOPMENT_API_ONLY"
);
let path = std::env::var("RMM_DEV_ROOT_POLICY_FILE").context("REPAIR_POLICY_REQUIRED")?;
let m = fs::symlink_metadata(&path)?;
let parent = fs::symlink_metadata(Path::new(&path).parent().context("REPAIR_POLICY_PARENT")?)?;
let key = std::env::var("RMM_JOB_KEY_FILE").context("REPAIR_EXISTING_KEY_REQUIRED")?;
ensure!(
m.is_file()
&& m.len() <= 65536
&& m.permissions().mode() & 0o077 == 0
&& m.uid() == fs::metadata(key)?.uid()
&& parent.is_dir()
&& parent.permissions().mode() & 0o022 == 0,
"REPAIR_POLICY_PERMISSIONS"
);
let p: Value = serde_json::from_slice(&fs::read(path)?)?;
ensure!(
p["schema_version"] == 1
&& p["profile"] == "development"
&& p["expires_at"]
.as_str()
.and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
.is_some_and(|until| until <= Utc::now()),
"REPAIR_POLICY_MUST_BE_CLOSED"
);
Ok(())
}
fn cell<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a Value> {
let columns = table["columns"]
.as_array()
.context("REPAIR_INVENTORY_COLUMNS")?;
let i = columns
.iter()
.position(|v| v.as_str() == Some(column))
.context("REPAIR_INVENTORY_COLUMN")?;
row.as_array()
.and_then(|r| r.get(i))
.context("REPAIR_INVENTORY_CELL")
}
fn rows(table: &Value) -> Result<&Vec<Value>> {
table["rows"].as_array().context("REPAIR_INVENTORY_ROWS")
}
fn text<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a str> {
let v = cell(table, row, column)?;
ensure!(v["type"] == "text", "REPAIR_CELL_TYPE");
v["value"].as_str().context("REPAIR_CELL_TEXT")
}
fn integer(table: &Value, row: &Value, column: &str) -> Result<u64> {
let v = cell(table, row, column)?;
ensure!(v["type"] == "integer", "REPAIR_CELL_TYPE");
v["value"].as_u64().context("REPAIR_CELL_INTEGER")
}
fn blob(table: &Value, row: &Value, column: &str, bytes: &[u8]) -> Result<()> {
let v = cell(table, row, column)?;
ensure!(
v == &json!({"type":"blob","sha256":hash(bytes),"length":bytes.len()}),
"REPAIR_WITNESS_BLOB"
);
Ok(())
}
fn row_by<'a>(table: &'a Value, column: &str, id: &str) -> Result<&'a Value> {
let found = rows(table)?
.iter()
.filter(|row| text(table, row, column).ok() == Some(id))
.collect::<Vec<_>>();
ensure!(found.len() == 1, "REPAIR_WITNESS_ROW");
Ok(found[0])
}
fn decision(
signer: &crate::job_signer::JobSigner,
s: &SignedDecision,
) -> Result<ReconcileDecisionPayload> {
ensure!(
s.decision_bytes.len() <= 1024 * 1024 && s.signature.len() == 64,
"REPAIR_DECISION_SIZE"
);
let key = PKey::public_key_from_raw_bytes(&signer.public_key()?, Id::ED25519)?;
ensure!(
Verifier::new_without_digest(&key)?.verify_oneshot(&s.signature, &s.decision_bytes)?,
"REPAIR_DECISION_SIGNATURE"
);
let d: ReconcileDecisionPayload = serde_json::from_slice(&s.decision_bytes)?;
ensure!(
d.valid_profile(d.issued_at) && d.key_id == signer.key_id,
"REPAIR_DECISION_PROFILE"
);
Ok(d)
}
struct Case {
old: ReconcileDecisionPayload,
rejected: ReconcileDecisionPayload,
unsent: SyncRequest,
request: SyncRequest,
}
fn validate(signer: &crate::job_signer::JobSigner, e: &Evidence) -> Result<Case> {
ensure!(
e.schema_version == 1
&& e.uncommitted_request.len() <= 1024 * 1024
&& e.rejected_request.len() <= 1024 * 1024,
"REPAIR_EVIDENCE_PROFILE"
);
let old = decision(signer, &e.previous_valid_response)?;
let rejected = decision(signer, &e.rejected_response)?;
for d in [&old, &rejected] {
ensure!(
d.tenant_id == e.tenant_id
&& d.asset_id == e.asset_id
&& d.agent_id == e.agent_id
&& d.enrollment_generation == e.enrollment_generation
&& d.assignment_version == e.assignment_version,
"REPAIR_IDENTITY_SCOPE"
);
}
let unsent = SyncRequest::decode(e.uncommitted_request.as_slice())?;
let request = SyncRequest::decode(e.rejected_request.as_slice())?;
let unsent_id = Uuid::parse_str(&unsent.request_id)?;
ensure!(
!unsent_id.is_nil()
&& unsent_id != rejected.request_id
&& unsent_id != old.request_id
&& old.decision_id != rejected.decision_id
&& rejected.request_id.to_string() == request.request_id
&& rejected.request_hash == hash(&e.rejected_request),
"REPAIR_REQUEST_BINDING"
);
ensure!(
rejected.reason == FAULT
&& !rejected.ready
&& !rejected.helper_ready
&& !rejected.attempts.iter().any(|a| a.unlock),
"REPAIR_DENIAL_ONLY"
);
ensure!(
request.previous_decision_id.is_empty()
&& request.previous_request_id == unsent.request_id
&& unsent.previous_decision_id == old.decision_id.to_string()
&& unsent.previous_request_id.is_empty(),
"REPAIR_EXACT_UNSENT_CHAIN"
);
let helper = rejected
.helper_journal_id
.context("REPAIR_HELPER_REQUIRED")?;
ensure!(
old.journal_id == rejected.journal_id
&& old.helper_journal_id == Some(helper)
&& old.checkpoint < unsent.checkpoint
&& unsent.checkpoint < rejected.checkpoint
&& old.helper_checkpoint <= unsent.helper_checkpoint
&& unsent.helper_checkpoint <= rejected.helper_checkpoint
&& old.connection_generation <= rejected.connection_generation,
"REPAIR_MONOTONIC_CHAIN"
);
for r in [&unsent, &request] {
ensure!(
r.journal_complete
&& r.assignment_version == e.assignment_version
&& r.journal_id == rejected.journal_id.to_string()
&& r.helper_journal_id == helper.to_string(),
"REPAIR_REQUEST_SCOPE"
);
}
ensure!(
request.checkpoint == rejected.checkpoint
&& request.helper_checkpoint == rejected.helper_checkpoint
&& request.helper_previous_decision_id == old.decision_id.to_string(),
"REPAIR_REQUEST_CHECKPOINT"
);
let am = &e.agent_inventory["reconcile_meta"];
let a = rows(am)?;
ensure!(a.len() == 1, "REPAIR_AGENT_META");
let a = &a[0];
ensure!(
integer(am, a, "id")? == 1
&& text(am, a, "journal_id")? == rejected.journal_id.to_string()
&& integer(am, a, "checkpoint")? == rejected.checkpoint
&& integer(am, a, "high_sequence")? >= rejected.checkpoint
&& integer(am, a, "generation")? == rejected.connection_generation
&& text(am, a, "last_decision")? == rejected.decision_id.to_string()
&& text(am, a, "helper_id")? == helper.to_string()
&& integer(am, a, "helper_checkpoint")? == rejected.helper_checkpoint
&& text(am, a, "fault")? == FAULT
&& cell(am, a, "pending_request")? == &json!({"type":"null"}),
"REPAIR_AGENT_HEAD"
);
let hm = &e.helper_inventory["helper_sync_meta"];
let h = rows(hm)?;
ensure!(h.len() == 1, "REPAIR_HELPER_META");
let h = &h[0];
ensure!(
integer(hm, h, "id")? == 1
&& text(hm, h, "journal_id")? == helper.to_string()
&& integer(hm, h, "checkpoint")? == rejected.helper_checkpoint
&& text(hm, h, "last_decision")? == rejected.decision_id.to_string()
&& text(hm, h, "agent_journal_id")? == rejected.journal_id.to_string()
&& integer(hm, h, "agent_checkpoint")? == rejected.checkpoint
&& text(hm, h, "fault")? == FAULT,
"REPAIR_HELPER_HEAD"
);
let ar = &e.agent_inventory["reconcile_requests"];
let x = row_by(ar, "id", &unsent.request_id)?;
blob(ar, x, "request", &e.uncommitted_request)?;
ensure!(
integer(ar, x, "generation")? <= rejected.connection_generation
&& cell(ar, x, "response")? == &json!({"type":"null"}),
"REPAIR_UNSENT_LOCAL_RESPONSE"
);
let y = row_by(ar, "id", &request.request_id)?;
blob(ar, y, "request", &e.rejected_request)?;
blob(
ar,
y,
"response",
&rmm_protocol::SyncResponse {
decision_bytes: e.rejected_response.decision_bytes.clone(),
signature: e.rejected_response.signature.clone(),
}
.encode_to_vec(),
)?;
ensure!(
integer(ar, y, "generation")? == rejected.connection_generation,
"REPAIR_LOCAL_GENERATION"
);
let hr = &e.helper_inventory["helper_sync_decisions"];
let y = row_by(hr, "decision_id", &rejected.decision_id.to_string())?;
blob(hr, y, "request", &e.rejected_request)?;
blob(
hr,
y,
"response",
&serde_json::to_vec(&e.rejected_response)?,
)?;
ensure!(
integer(hr, y, "checkpoint")? == rejected.helper_checkpoint,
"REPAIR_HELPER_CAPTURE"
);
ensure!(
e.helper_inventory["consumed"].is_object()
&& e.helper_inventory["execution_evidence"].is_object(),
"REPAIR_LEDGER_REQUIRED"
);
Ok(Case {
old,
rejected,
unsent,
request,
})
}
pub async fn issue(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
closed_policy()?;
issue_with_closed_policy(state, e).await
}
async fn issue_with_closed_policy(state: &crate::State, e: Evidence) -> Result<SignedReceipt> {
let c = validate(&state.job_signer, &e)?;
let evidence_hash = inventory_sha256(&serde_json::to_value(&e)?)?;
let mut tx = rmm_persistence::tenant_tx(&state.pool, e.tenant_id).await?;
crate::agent_service::control_lock(&mut tx, e.tenant_id, e.agent_id).await?;
let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,revoked_at,sequence_regressed,sync_recovery_required,sync_ready,helper_sync_ready FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_DEVICE_MISSING")?;
ensure!(
device.get::<Uuid, _>("asset_id") == e.asset_id
&& device.get::<i64, _>("enrollment_generation")
== i64::try_from(e.enrollment_generation)?
&& device.get::<i64, _>("assignment_version") == i64::try_from(e.assignment_version)?
// Faulted journals may survive newer ClientHello channels. This
// historical receipt grants no lease; the next ordinary sync must
// still bind its current channel. A regressed generation or changed
// boot/identity is never repaired here.
&& device.get::<i64, _>("connection_generation")
>= i64::try_from(c.rejected.connection_generation)?
&& device.get::<Option<Uuid>, _>("connection_boot_id") == Some(c.rejected.boot_id)
&& device
.get::<Option<chrono::DateTime<Utc>>, _>("revoked_at")
.is_none()
&& ![
"sequence_regressed",
"sync_recovery_required",
"sync_ready",
"helper_sync_ready"
]
.iter()
.any(|k| device.get::<bool, _>(*k)),
"REPAIR_DEVICE_FENCED"
);
for (d, s) in [
(&c.old, &e.previous_valid_response),
(&c.rejected, &e.rejected_response),
] {
let stored=sqlx::query("SELECT decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3")
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).fetch_optional(&mut *tx).await?.context("REPAIR_STORED_DECISION_MISSING")?;
ensure!(
stored.get::<Uuid, _>("decision_id") == d.decision_id
&& stored.get::<String, _>("request_hash") == d.request_hash
&& stored.get::<Uuid, _>("journal_id") == d.journal_id
&& stored.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
&& stored.get::<i64, _>("connection_generation")
== i64::try_from(d.connection_generation)?
&& stored.get::<Vec<u8>, _>("payload") == s.decision_bytes
&& stored.get::<Vec<u8>, _>("signature") == s.signature,
"REPAIR_STORED_DECISION_MISMATCH"
);
}
let committed:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3)")
.bind(e.tenant_id).bind(e.agent_id).bind(Uuid::parse_str(&c.unsent.request_id)?).fetch_one(&mut *tx).await?;
ensure!(!committed, "REPAIR_REQUEST_WAS_COMMITTED");
let head=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
.bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_HEAD_MISSING")?;
let previous:Option<Value>=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type=$2 AND data->>'evidence_sha256'=$3 AND data->>'agent_id'=$4 ORDER BY created_at DESC LIMIT 1")
.bind(e.tenant_id).bind(EVENT).bind(&evidence_hash).bind(e.agent_id.to_string()).fetch_optional(&mut *tx).await?;
if let Some(previous) = previous {
let signed: SignedReceipt = serde_json::from_value(previous["receipt"].clone())?;
let receipt: Receipt = serde_json::from_slice(&signed.payload)?;
let key = PKey::public_key_from_raw_bytes(&state.job_signer.public_key()?, Id::ED25519)?;
ensure!(
signed.signature.len() == 64
&& Verifier::new_without_digest(&key)?
.verify_oneshot(&signed.signature, &signed.payload)?
&& receipt.valid_profile(Utc::now())
&& receipt.key_id == state.job_signer.key_id
&& receipt.tenant_id == e.tenant_id
&& receipt.asset_id == e.asset_id
&& receipt.agent_id == e.agent_id
&& receipt.rejected_decision_id == c.rejected.decision_id
&& receipt.rejected_request_sha256 == hash(&e.rejected_request)
&& receipt.uncommitted_request_sha256 == hash(&e.uncommitted_request)
&& receipt.agent_inventory_sha256 == inventory_sha256(&e.agent_inventory)?
&& receipt.helper_inventory_sha256 == inventory_sha256(&e.helper_inventory)?
&& head_matches(&head, &c.rejected)?,
"REPAIR_REPLAY_EXPIRED_OR_HEAD_CHANGED"
);
tx.commit().await?;
return Ok(signed);
}
ensure!(
head_matches(&head, &c.old)?,
"REPAIR_AUTHORITATIVE_HEAD_CHANGED"
);
let now = Utc::now();
let receipt = Receipt {
domain: "rmm.sync.unsent-request-repair".into(),
schema_version: 1,
receipt_id: Uuid::new_v4(),
tenant_id: e.tenant_id,
asset_id: e.asset_id,
agent_id: e.agent_id,
enrollment_generation: e.enrollment_generation,
assignment_version: e.assignment_version,
issued_at: now,
valid_until: now + Duration::minutes(10),
key_id: state.job_signer.key_id.clone(),
fault: FAULT.into(),
agent_journal_id: c.rejected.journal_id,
agent_checkpoint: c.rejected.checkpoint,
helper_journal_id: c
.rejected
.helper_journal_id
.context("REPAIR_HELPER_REQUIRED")?,
helper_checkpoint: c.rejected.helper_checkpoint,
rejected_decision_id: c.rejected.decision_id,
rejected_decision_sha256: hash(&e.rejected_response.decision_bytes),
rejected_request_id: c.rejected.request_id,
rejected_request_sha256: hash(&e.rejected_request),
uncommitted_request_id: Uuid::parse_str(&c.unsent.request_id)?,
uncommitted_request_sha256: hash(&e.uncommitted_request),
previous_valid_decision_id: c.old.decision_id,
previous_valid_decision_sha256: hash(&e.previous_valid_response.decision_bytes),
agent_inventory_sha256: inventory_sha256(&e.agent_inventory)?,
helper_inventory_sha256: inventory_sha256(&e.helper_inventory)?,
helper_ledger_sha256: helper_ledger_sha256(&e.helper_inventory)?,
ready: false,
helper_ready: false,
unlock: false,
};
ensure!(receipt.valid_profile(now), "REPAIR_RECEIPT_PROFILE");
let payload = serde_json::to_vec(&receipt)?;
let signed = SignedReceipt {
signature: state.job_signer.sign(&payload)?,
payload,
};
ensure!(sqlx::query("UPDATE agent_sync_state SET checkpoint=$3,decision_id=$4,helper_journal_id=$5,helper_checkpoint=$6 WHERE tenant_id=$1 AND agent_id=$2 AND decision_id=$7")
.bind(e.tenant_id).bind(e.agent_id).bind(i64::try_from(c.request.checkpoint)?).bind(c.rejected.decision_id)
.bind(c.rejected.helper_journal_id).bind(i64::try_from(c.rejected.helper_checkpoint)?).bind(c.old.decision_id)
.execute(&mut *tx).await?.rows_affected()==1,"REPAIR_HEAD_CONCURRENT_CHANGE");
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,$4,$5)")
.bind(e.tenant_id).bind(receipt.receipt_id).bind("development-bootstrap:unsent-sync-repair").bind(EVENT)
.bind(json!({"agent_id":e.agent_id,"evidence_sha256":evidence_hash,"receipt_sha256":hash(&signed.payload),"receipt":signed,"ready":false,"helper_ready":false,"unlock":false}))
.execute(&mut *tx).await?;
tx.commit().await?;
Ok(signed)
}
fn head_matches(head: &sqlx::postgres::PgRow, d: &ReconcileDecisionPayload) -> Result<bool> {
Ok(head.get::<Uuid, _>("journal_id") == d.journal_id
&& head.get::<i64, _>("checkpoint") == i64::try_from(d.checkpoint)?
&& head.get::<Uuid, _>("decision_id") == d.decision_id
&& head.get::<Option<Uuid>, _>("helper_journal_id") == d.helper_journal_id
&& head.get::<i64, _>("helper_checkpoint") == i64::try_from(d.helper_checkpoint)?)
}
#[cfg(test)]
mod tests {
use super::*;
fn table(fields: Vec<(&str, Value)>) -> Value {
json!({"columns":fields.iter().map(|(k,_)|k).collect::<Vec<_>>(),"rows":[fields.iter().map(|(_,v)|v).collect::<Vec<_>>() ]})
}
fn t(v: impl ToString) -> Value {
json!({"type":"text","value":v.to_string()})
}
fn n(v: u64) -> Value {
json!({"type":"integer","value":v})
}
fn b(v: &[u8]) -> Value {
json!({"type":"blob","sha256":hash(v),"length":v.len()})
}
fn sign(s: &crate::job_signer::JobSigner, d: &ReconcileDecisionPayload) -> SignedDecision {
let decision_bytes = serde_json::to_vec(d).unwrap();
SignedDecision {
signature: s.sign(&decision_bytes).unwrap(),
decision_bytes,
}
}
fn fixture() -> (crate::job_signer::JobSigner, Evidence, std::path::PathBuf) {
let path = std::path::PathBuf::from(format!("/tmp/rmm-repair-{}", Uuid::new_v4()));
let signer = crate::job_signer::JobSigner::load_or_create(&path.join("key.pem")).unwrap();
let tenant = Uuid::new_v4();
let asset = Uuid::new_v4();
let agent = Uuid::new_v4();
let journal = Uuid::new_v4();
let helper = Uuid::new_v4();
let now = Utc::now();
let old = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::new_v4(),
request_hash: "ab".repeat(32),
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
connection_generation: 3,
boot_id: Uuid::new_v4(),
journal_id: journal,
checkpoint: 10,
helper_journal_id: Some(helper),
helper_checkpoint: 5,
issued_at: now,
valid_until: now + Duration::seconds(60),
ready: false,
helper_ready: false,
reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(),
attempts: vec![],
acknowledged_message_ids: vec![],
key_id: signer.key_id.clone(),
};
let x = SyncRequest {
request_id: Uuid::new_v4().to_string(),
journal_id: journal.to_string(),
checkpoint: 11,
previous_decision_id: old.decision_id.to_string(),
assignment_version: 1,
journal_complete: true,
helper_journal_id: helper.to_string(),
helper_checkpoint: 6,
helper_previous_decision_id: old.decision_id.to_string(),
..Default::default()
};
let y = SyncRequest {
request_id: Uuid::new_v4().to_string(),
checkpoint: 12,
previous_decision_id: String::new(),
previous_request_id: x.request_id.clone(),
helper_checkpoint: 7,
..x.clone()
};
let mut rejected = old.clone();
rejected.decision_id = Uuid::new_v4();
rejected.request_id = Uuid::parse_str(&y.request_id).unwrap();
rejected.request_hash = hash(&y.encode_to_vec());
rejected.checkpoint = 12;
rejected.helper_checkpoint = 7;
rejected.reason = FAULT.into();
let response = sign(&signer, &rejected);
let request = y.encode_to_vec();
let unsent = x.encode_to_vec();
let proto = rmm_protocol::SyncResponse {
decision_bytes: response.decision_bytes.clone(),
signature: response.signature.clone(),
}
.encode_to_vec();
let mut requests = table(vec![
("id", t(&x.request_id)),
("request", b(&unsent)),
("generation", n(3)),
("response", json!({"type":"null"})),
]);
requests["rows"].as_array_mut().unwrap().push(json!([
t(&y.request_id),
b(&request),
n(3),
b(&proto)
]));
let am = table(vec![
("id", n(1)),
("journal_id", t(journal)),
("checkpoint", n(12)),
("high_sequence", n(13)),
("generation", n(3)),
("last_decision", t(rejected.decision_id)),
("pending_request", json!({"type":"null"})),
("helper_id", t(helper)),
("helper_checkpoint", n(7)),
("fault", t(FAULT)),
]);
let hm = table(vec![
("id", n(1)),
("journal_id", t(helper)),
("checkpoint", n(7)),
("last_decision", t(rejected.decision_id)),
("agent_journal_id", t(journal)),
("agent_checkpoint", n(12)),
("fault", t(FAULT)),
]);
let history = table(vec![
("decision_id", t(rejected.decision_id)),
("checkpoint", n(7)),
("response", b(&serde_json::to_vec(&response).unwrap())),
("request", b(&request)),
]);
let e = Evidence {
schema_version: 1,
tenant_id: tenant,
asset_id: asset,
agent_id: agent,
enrollment_generation: 1,
assignment_version: 1,
agent_inventory: json!({"reconcile_meta":am,"reconcile_requests":requests}),
helper_inventory: json!({"helper_sync_meta":hm,"helper_sync_decisions":history,"consumed":{"columns":[],"rows":[]},"execution_evidence":{"columns":[],"rows":[]}}),
uncommitted_request: unsent,
rejected_request: request,
rejected_response: response,
previous_valid_response: sign(&signer, &old),
};
(signer, e, path)
}
#[test]
fn only_exact_signed_unsent_case_validates() {
let (s, e, p) = fixture();
validate(&s, &e).unwrap();
let mut other = e.clone();
other.tenant_id = Uuid::new_v4();
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.asset_id = Uuid::new_v4();
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.assignment_version += 1;
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.rejected_response.signature[0] ^= 1;
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.agent_inventory["reconcile_requests"]["rows"][0][3] = b(b"committed response");
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.helper_inventory["helper_sync_meta"]["rows"][0][2] = n(8);
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
other.agent_inventory["reconcile_meta"]["rows"][0][9] = t("OTHER_FAULT");
assert!(validate(&s, &other).is_err());
let mut other = e.clone();
let mut x = SyncRequest::decode(other.uncommitted_request.as_slice()).unwrap();
x.previous_decision_id = Uuid::new_v4().to_string();
other.uncommitted_request = x.encode_to_vec();
assert!(validate(&s, &other).is_err());
fs::remove_dir_all(p).unwrap();
}
#[test]
fn signed_ready_response_is_never_repair_authority() {
let (s, mut e, p) = fixture();
let mut d: ReconcileDecisionPayload =
serde_json::from_slice(&e.rejected_response.decision_bytes).unwrap();
d.ready = true;
e.rejected_response = sign(&s, &d);
assert!(validate(&s, &e).is_err());
d.ready = false;
d.reason = "CONSISTENT".into();
e.rejected_response = sign(&s, &d);
assert!(validate(&s, &e).is_err());
fs::remove_dir_all(p).unwrap();
}
/// Opt-in dedicated PostgreSQL only; the harness creates/destroys its own
/// network-isolated container and never reads deployment credentials.
#[tokio::test]
#[ignore = "requires the dedicated isolated PostgreSQL harness"]
async fn isolated_database_repair_and_fail_closed_paths() {
let admin_url = std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap();
let app_url = std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap();
let admin = sqlx::PgPool::connect(&admin_url).await.unwrap();
let pool = sqlx::PgPool::connect(&app_url).await.unwrap();
rmm_persistence::verify_application_role(&pool)
.await
.unwrap();
let (s, e, p) = fixture();
let c = validate(&s, &e).unwrap();
sqlx::query("INSERT INTO tenants VALUES($1,'isolated repair test')")
.bind(e.tenant_id)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)")
.bind(e.tenant_id)
.bind(e.asset_id)
.execute(&admin)
.await
.unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,assignment_version,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5,$6)")
.bind(e.tenant_id).bind(e.agent_id).bind(e.asset_id).bind(hash(e.agent_id.as_bytes())).bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(c.rejected.boot_id).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO agent_sync_state VALUES($1,$2,$3,$4,$5,$6,$7)")
.bind(e.tenant_id)
.bind(e.agent_id)
.bind(c.old.journal_id)
.bind(i64::try_from(c.old.checkpoint).unwrap())
.bind(c.old.decision_id)
.bind(c.old.helper_journal_id)
.bind(i64::try_from(c.old.helper_checkpoint).unwrap())
.execute(&admin)
.await
.unwrap();
for (d, signed) in [
(&c.old, &e.previous_valid_response),
(&c.rejected, &e.rejected_response),
] {
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
.bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).bind(d.decision_id).bind(&d.request_hash).bind(d.journal_id).bind(i64::try_from(d.checkpoint).unwrap()).bind(i64::try_from(d.connection_generation).unwrap()).bind(&signed.decision_bytes).bind(&signed.signature).execute(&admin).await.unwrap();
}
let state = crate::State {
pool,
signer: None,
viewer_a: [0; 32],
viewer_b: [1; 32],
auth: std::sync::Arc::new(crate::auth::Auth::from_env("development").await.unwrap()),
job_signer: std::sync::Arc::new(s),
};
let mut wrong = e.clone();
wrong.tenant_id = Uuid::new_v4();
assert!(issue_with_closed_policy(&state, wrong).await.is_err());
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
.bind(e.tenant_id)
.bind(c.rejected.request_id)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)")
.bind(e.tenant_id).bind(e.agent_id).bind(c.rejected.request_id).bind(c.rejected.decision_id)
.bind(&c.rejected.request_hash).bind(c.rejected.journal_id).bind(i64::try_from(c.rejected.checkpoint).unwrap())
.bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(&e.rejected_response.decision_bytes)
.bind(&e.rejected_response.signature).execute(&admin).await.unwrap();
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint+1 WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint-1 WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
sqlx::query("UPDATE agent_installations SET revoked_at=now() WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_installations SET revoked_at=NULL WHERE agent_id=$1")
.bind(e.agent_id)
.execute(&admin)
.await
.unwrap();
let x = Uuid::parse_str(&c.unsent.request_id).unwrap();
sqlx::query("INSERT INTO sync_checkpoints SELECT tenant_id,agent_id,$2,$3,request_hash,journal_id,checkpoint,connection_generation,payload,signature,created_at FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$4")
.bind(e.tenant_id).bind(x).bind(Uuid::new_v4()).bind(c.old.request_id).execute(&admin).await.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2")
.bind(e.tenant_id)
.bind(x)
.execute(&admin)
.await
.unwrap();
sqlx::query("UPDATE agent_installations SET connection_generation=$2 WHERE agent_id=$1")
.bind(e.agent_id)
.bind(i64::try_from(c.rejected.connection_generation).unwrap() - 1)
.execute(&admin)
.await
.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_installations SET connection_generation=$2,connection_boot_id=$3 WHERE agent_id=$1")
.bind(e.agent_id).bind(i64::try_from(c.rejected.connection_generation).unwrap()+1).bind(Uuid::new_v4())
.execute(&admin).await.unwrap();
assert!(issue_with_closed_policy(&state, e.clone()).await.is_err());
sqlx::query("UPDATE agent_installations SET connection_boot_id=$2 WHERE agent_id=$1")
.bind(e.agent_id)
.bind(c.rejected.boot_id)
.execute(&admin)
.await
.unwrap();
let first = issue_with_closed_policy(&state, e.clone()).await.unwrap();
assert_eq!(
first,
issue_with_closed_policy(&state, e.clone()).await.unwrap()
);
let r: Receipt = serde_json::from_slice(&first.payload).unwrap();
assert!(r.valid_profile(Utc::now()));
assert!(!r.ready && !r.helper_ready && !r.unlock);
let head: Uuid =
sqlx::query_scalar("SELECT decision_id FROM agent_sync_state WHERE agent_id=$1")
.bind(e.agent_id)
.fetch_one(&admin)
.await
.unwrap();
assert_eq!(head, c.rejected.decision_id);
let count: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1")
.bind(e.tenant_id)
.fetch_one(&admin)
.await
.unwrap();
assert_eq!(count, 1);
let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM outbox)+(SELECT count(*) FROM execution_permits)").fetch_one(&admin).await.unwrap();
assert_eq!(effects, 0);
let ready: bool = sqlx::query_scalar(
"SELECT sync_ready OR helper_sync_ready FROM agent_installations WHERE agent_id=$1",
)
.bind(e.agent_id)
.fetch_one(&admin)
.await
.unwrap();
assert!(!ready);
fs::remove_dir_all(p).unwrap();
}
}
+570
View File
@@ -0,0 +1,570 @@
//! Narrow dev/test wire faults after normal authorization/durable ingest.
//! Never creates jobs, grants or approvals. Claims and observations are bounded,
//! insert-only audit slots; a restart cannot arm the same fault a second time.
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Duration, Utc};
use prost::Message;
use rmm_domain::jobs::{Action, JobPayload, hash};
use rmm_protocol::{Envelope, Payload};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sqlx::{PgPool, Postgres, Row, Transaction};
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
use uuid::Uuid;
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum Mode {
DuplicateOfferOnce,
DropResultAckOnce,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Config {
schema_version: u32,
fault_id: Uuid,
tenant_id: Uuid,
agent_id: Uuid,
job_id: Uuid,
attempt_id: Uuid,
plan_sha256: String,
job_payload_sha256: String,
mode: Mode,
transport: String,
issued_at: DateTime<Utc>,
expires_at: DateTime<Utc>,
}
impl Config {
fn validate(&self) -> Result<()> {
ensure!(
self.schema_version == 1
&& [
self.fault_id,
self.tenant_id,
self.agent_id,
self.job_id,
self.attempt_id
]
.iter()
.all(|id| !id.is_nil())
&& valid_hash(&self.plan_sha256)
&& valid_hash(&self.job_payload_sha256)
&& matches!(self.transport.as_str(), "wss" | "https")
&& self.issued_at.timestamp() > 0
&& self.expires_at > self.issued_at
&& self.expires_at - self.issued_at <= Duration::minutes(15),
"DEV_DELIVERY_FAULT_CONFIG"
);
Ok(())
}
fn scope(&self, tenant: Uuid, agent: Uuid, transport: &str) -> bool {
self.tenant_id == tenant && self.agent_id == agent && self.transport == transport
}
fn job(&self, bytes: &[u8], payload_hash: &str, plan_hash: &str) -> Result<bool> {
if payload_hash != self.job_payload_sha256 || plan_hash != self.plan_sha256 {
return Ok(false);
}
ensure!(
hash(bytes) == self.job_payload_sha256,
"DEV_FAULT_PAYLOAD_HASH"
);
let job: JobPayload = serde_json::from_slice(bytes)?;
Ok(job.schema_version == 1
&& job.tenant_id == self.tenant_id
&& job.agent_id == self.agent_id
&& job.job_id == self.job_id
&& job.attempt_id == self.attempt_id
&& job.execution_identity == "agent_service"
&& job.enrollment_generation > 0
&& job.assignment_version > 0
&& job.max_runtime_seconds > 0
&& job.max_runtime_seconds <= 60
&& job.max_output_bytes <= 65536
&& job.action.validate()
&& matches!(job.action, Action::RunScript { .. })
&& self.expires_at <= job.start_before)
}
}
fn valid_hash(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
&& value.bytes().any(|b| b != b'0')
}
fn configured_path(
flag: Option<&str>,
path: Option<&str>,
profile: &str,
dev: &str,
) -> Result<Option<PathBuf>> {
match (flag, path) {
(None | Some("false"), None) => Ok(None),
(Some("true"), Some(path)) => {
ensure!(
profile == "development" && dev == "true",
"DEV_DELIVERY_FAULT_PROFILE"
);
ensure!(
path == "/root-policy/delivery-fault.json",
"DEV_DELIVERY_FAULT_PATH"
);
Ok(Some(PathBuf::from(path)))
}
_ => anyhow::bail!("DEV_DELIVERY_FAULT_EXPLICIT_OPT_IN"),
}
}
#[derive(Clone)]
struct Loaded {
config: Config,
config_sha256: String,
}
fn load() -> Result<Option<Loaded>> {
let flag = std::env::var("RMM_DEV_DELIVERY_FAULTS").ok();
let path = std::env::var("RMM_DEV_DELIVERY_FAULT_FILE").ok();
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
return Ok(None);
};
let parent = path.parent().context("DEV_FAULT_PARENT")?;
let directory = fs::symlink_metadata(parent)?;
let metadata = fs::symlink_metadata(&path)?;
ensure!(
directory.is_dir()
&& directory.uid() == 10001
&& directory.mode() & 0o777 == 0o700
&& metadata.is_file()
&& metadata.uid() == 10001
&& metadata.mode() & 0o777 == 0o600
&& metadata.nlink() == 1
&& metadata.len() <= 8192,
"DEV_FAULT_PRIVATE_CONFIG"
);
// /root-policy is the existing operator-provisioned read-only bind mount.
let bytes = fs::read(&path)?;
ensure!(bytes.len() <= 8192, "DEV_FAULT_CONFIG_SIZE");
let config: Config = serde_json::from_slice(&bytes)?;
config.validate()?;
if Utc::now() < config.issued_at || Utc::now() >= config.expires_at {
return Ok(None);
}
Ok(Some(Loaded {
config,
config_sha256: hash(&bytes),
}))
}
pub(crate) fn validate_environment() -> Result<()> {
load().map(|_| ())
}
fn slot_id(fault: Uuid, slot: &str) -> Uuid {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(format!("rmm.dev-delivery-fault.v1:{fault}:{slot}"));
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
}
fn envelope_hash(envelope: &Envelope) -> String {
let mut normalized = envelope.clone();
normalized.connection_generation = 0;
hash(&normalized.encode_to_vec())
}
fn data(loaded: &Loaded, envelope: &Envelope, detail: &str) -> Value {
let config = &loaded.config;
json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
"fault_id":config.fault_id,"mode":config.mode,"transport":config.transport,
"agent_id":config.agent_id,"job_id":config.job_id,"attempt_id":config.attempt_id,
"plan_sha256":config.plan_sha256,"job_payload_sha256":config.job_payload_sha256,
"config_sha256":loaded.config_sha256,"message_id":envelope.message_id,
"normalized_envelope_sha256":envelope_hash(envelope),"detail":detail})
}
async fn transaction<'a>(pool: &'a PgPool, loaded: &Loaded) -> Result<Transaction<'a, Postgres>> {
let mut tx = rmm_persistence::tenant_tx(pool, loaded.config.tenant_id).await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
.bind(format!("dev-delivery-fault:{}", loaded.config.fault_id))
.execute(&mut *tx)
.await?;
Ok(tx)
}
async fn get_slot(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
slot: &str,
) -> Result<Option<Value>> {
let value: Option<Value> = sqlx::query_scalar("SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_delivery_fault_adapter'")
.bind(slot_id(loaded.config.fault_id, slot)).fetch_optional(&mut **tx).await?;
if let Some(value) = &value {
ensure!(
value.get("config_sha256") == Some(&json!(loaded.config_sha256)),
"DEV_FAULT_ID_REUSED_WITH_CHANGED_CONFIG"
);
}
Ok(value)
}
async fn insert_slot(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
slot: &str,
value: Value,
) -> Result<bool> {
if get_slot(tx, loaded, slot).await?.is_some() {
return Ok(false);
}
let rows = sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_delivery_fault_adapter',$3,$4) ON CONFLICT(tenant_id,event_id) DO NOTHING")
.bind(loaded.config.tenant_id).bind(slot_id(loaded.config.fault_id, slot))
.bind(format!("dev_delivery_fault.{slot}")).bind(value).execute(&mut **tx).await?.rows_affected();
Ok(rows == 1)
}
async fn authorized_job(
tx: &mut Transaction<'_, Postgres>,
loaded: &Loaded,
message: Option<&Envelope>,
) -> Result<bool> {
let c = &loaded.config;
let row = sqlx::query("SELECT j.payload,j.payload_hash,p.plan_hash FROM jobs j JOIN operations o USING(tenant_id,operation_id) JOIN approval_requests r USING(tenant_id,operation_id) JOIN operation_plans p USING(tenant_id,plan_id) JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE j.job_id=$1 AND j.attempt_id=$2 AND j.agent_id=$3 AND r.approval_state='approved' AND a.testsimulation AND a.approval_origin='dev_test_browser_simulation' AND a.principal='simulated-testprincipal-a'")
.bind(c.job_id).bind(c.attempt_id).bind(c.agent_id).fetch_optional(&mut **tx).await?;
let Some(row) = row else {
return Ok(false);
};
if !c.job(
&row.get::<Vec<u8>, _>("payload"),
&row.get::<String, _>("payload_hash"),
&row.get::<String, _>("plan_hash"),
)? {
return Ok(false);
}
if let Some(message) = message {
let receipt: Option<String> = sqlx::query_scalar(
"SELECT payload_hash FROM job_receipts WHERE message_id=$1 AND agent_id=$2",
)
.bind(Uuid::parse_str(&message.message_id)?)
.bind(c.agent_id)
.fetch_optional(&mut **tx)
.await?;
ensure!(
receipt.as_deref() == Some(envelope_hash(message).as_str()),
"DEV_FAULT_DURABLE_RECEIPT_REQUIRED"
);
}
Ok(true)
}
pub(crate) struct DuplicateClaim {
loaded: Loaded,
envelope: Envelope,
}
/// Called only for work returned by normal postapproval `job_gateway::next`.
pub(crate) async fn duplicate_offer(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
envelope: &Envelope,
transport: &str,
) -> Result<Option<DuplicateClaim>> {
let Some(Payload::JobOffer(offer)) = &envelope.payload else {
return Ok(None);
};
let Some(loaded) = load()? else {
return Ok(None);
};
let c = &loaded.config;
if c.mode != Mode::DuplicateOfferOnce
|| !c.scope(tenant, agent, transport)
|| offer.job_id != c.job_id.to_string()
|| offer.attempt_id != c.attempt_id.to_string()
|| !c.job(
&offer.payload_bytes,
&offer.job_payload_hash,
&offer.plan_hash,
)?
{
return Ok(None);
}
let mut tx = transaction(pool, &loaded).await?;
let claimed = authorized_job(&mut tx, &loaded, None).await?
&& insert_slot(
&mut tx,
&loaded,
"claim",
data(&loaded, envelope, "duplicate_offer_reserved_before_send"),
)
.await?;
tx.commit().await?;
Ok(claimed.then(|| DuplicateClaim {
loaded,
envelope: envelope.clone(),
}))
}
/// WSS calls after both sends complete. HTTP labels construction honestly; the
/// two later distinct postcommit JobAccepted slots prove client observation.
pub(crate) async fn duplicate_emitted(pool: &PgPool, claim: DuplicateClaim) -> Result<()> {
let mut tx = transaction(pool, &claim.loaded).await?;
ensure!(
get_slot(&mut tx, &claim.loaded, "claim").await?.is_some(),
"DEV_FAULT_CLAIM_REQUIRED"
);
let detail = if claim.loaded.config.transport == "wss" {
"same_encoded_offer_sent_twice"
} else {
"same_encoded_offer_in_http_response_twice"
};
insert_slot(
&mut tx,
&claim.loaded,
"emitted",
data(&claim.loaded, &claim.envelope, detail),
)
.await?;
tx.commit().await?;
Ok(())
}
/// Called AFTER durable ingest returned its concrete ACK. Return true only to
/// omit that single matching result ACK; every subsequent replay is ACKed.
pub(crate) async fn after_ingest(
pool: &PgPool,
tenant: Uuid,
agent: Uuid,
incoming: &Envelope,
reply: &Envelope,
transport: &str,
) -> Result<bool> {
let (job, attempt, is_result) = match &incoming.payload {
Some(Payload::JobAccepted(value)) => (&value.job_id, &value.attempt_id, false),
Some(Payload::JobResult(value)) if value.state == "succeeded" => {
(&value.job_id, &value.attempt_id, true)
}
_ => return Ok(false),
};
let Some(loaded) = load()? else {
return Ok(false);
};
let c = &loaded.config;
if !c.scope(tenant, agent, transport)
|| job != &c.job_id.to_string()
|| attempt != &c.attempt_id.to_string()
{
return Ok(false);
}
let Some(Payload::Ack(ack)) = &reply.payload else {
return Ok(false);
};
ensure!(
ack.message_id == incoming.message_id,
"DEV_FAULT_CONCRETE_ACK_REQUIRED"
);
let mut tx = transaction(pool, &loaded).await?;
if !authorized_job(&mut tx, &loaded, Some(incoming)).await? {
tx.commit().await?;
return Ok(false);
}
let claim = get_slot(&mut tx, &loaded, "claim").await?;
let mut omit = false;
if is_result && c.mode == Mode::DropResultAckOnce {
if let Some(first) = claim {
if first.get("message_id") == Some(&json!(incoming.message_id))
&& first.get("normalized_envelope_sha256") == Some(&json!(envelope_hash(incoming)))
{
insert_slot(
&mut tx,
&loaded,
"replay",
data(
&loaded,
incoming,
"same_original_result_replayed_after_ack_omission",
),
)
.await?;
}
} else {
omit = insert_slot(
&mut tx,
&loaded,
"claim",
data(
&loaded,
incoming,
"one_concrete_result_ack_omitted_after_durable_commit",
),
)
.await?;
}
} else if !is_result && c.mode == Mode::DuplicateOfferOnce && claim.is_some() {
if let Some(first) = get_slot(&mut tx, &loaded, "accepted_1").await? {
if first.get("message_id") != Some(&json!(incoming.message_id)) {
insert_slot(
&mut tx,
&loaded,
"accepted_2",
data(&loaded, incoming, "second_distinct_client_acceptance"),
)
.await?;
}
} else {
insert_slot(
&mut tx,
&loaded,
"accepted_1",
data(&loaded, incoming, "first_client_acceptance"),
)
.await?;
}
}
tx.commit().await?;
Ok(omit)
}
#[cfg(test)]
mod tests {
use super::*;
fn config() -> Config {
let now = Utc::now();
Config {
schema_version: 1,
fault_id: Uuid::new_v4(),
tenant_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
plan_sha256: "a".repeat(64),
job_payload_sha256: "b".repeat(64),
mode: Mode::DuplicateOfferOnce,
transport: "wss".into(),
issued_at: now,
expires_at: now + Duration::minutes(10),
}
}
#[test]
fn opt_in_is_explicit_and_never_production() {
assert!(
configured_path(None, None, "production", "false")
.unwrap()
.is_none()
);
assert!(
configured_path(
Some("true"),
Some("/root-policy/delivery-fault.json"),
"production",
"false"
)
.is_err()
);
assert!(configured_path(Some("true"), None, "development", "true").is_err());
assert!(
configured_path(
None,
Some("/root-policy/delivery-fault.json"),
"development",
"true"
)
.is_err()
);
assert!(configured_path(Some("true"), Some("/etc/passwd"), "development", "true").is_err());
assert!(
configured_path(
Some("true"),
Some("/root-policy/delivery-fault.json"),
"development",
"true"
)
.unwrap()
.is_some()
);
}
#[test]
fn narrow_schema_rejects_wildcards_nil_placeholders_and_long_lifetime() {
let c = config();
c.validate().unwrap();
let mut altered = c.clone();
altered.expires_at = altered.issued_at + Duration::minutes(16);
assert!(altered.validate().is_err());
altered = c.clone();
altered.agent_id = Uuid::nil();
assert!(altered.validate().is_err());
altered = c.clone();
altered.job_payload_sha256 = "0".repeat(64);
assert!(altered.validate().is_err());
altered = c.clone();
altered.transport = "*".into();
assert!(altered.validate().is_err());
let mut raw = serde_json::to_value(c).unwrap();
raw["approve"] = json!(true);
assert!(serde_json::from_value::<Config>(raw).is_err());
}
#[test]
fn audit_slots_are_stable_bounded_and_distinct() {
let id = Uuid::new_v4();
assert_eq!(slot_id(id, "claim"), slot_id(id, "claim"));
assert_ne!(slot_id(id, "claim"), slot_id(id, "replay"));
assert_ne!(slot_id(id, "claim"), slot_id(Uuid::new_v4(), "claim"));
}
#[test]
fn exact_signed_job_scope_cannot_fault_root_other_targets_or_other_actions() {
let mut c = config();
let mut job = JobPayload {
schema_version: 1,
tenant_id: c.tenant_id,
asset_id: Uuid::new_v4(),
agent_id: c.agent_id,
enrollment_generation: 1,
assignment_version: 1,
job_id: c.job_id,
attempt_id: c.attempt_id,
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "printf hello\\n".into(),
arguments: vec![],
expected_stdout: None,
},
execution_identity: "agent_service".into(),
not_before: c.issued_at,
start_before: c.expires_at,
max_runtime_seconds: 60,
max_output_bytes: 4096,
};
let mut bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
assert!(!c.scope(c.tenant_id, Uuid::new_v4(), "wss"));
assert!(!c.scope(c.tenant_id, c.agent_id, "https"));
assert!(
!c.job(&bytes, &c.job_payload_sha256, &"c".repeat(64))
.unwrap()
);
job.execution_identity = "root".into();
bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
job.execution_identity = "agent_service".into();
job.action = Action::RefreshInventory {
scope: "linux.base".into(),
};
bytes = serde_json::to_vec(&job).unwrap();
c.job_payload_sha256 = hash(&bytes);
assert!(
!c.job(&bytes, &c.job_payload_sha256, &c.plan_sha256)
.unwrap()
);
}
#[test]
fn durable_replay_only_normalizes_connection_generation() {
let e = Envelope {
message_id: Uuid::new_v4().to_string(),
connection_generation: 3,
..Default::default()
};
let mut repeated = e.clone();
repeated.connection_generation = 4;
assert_eq!(envelope_hash(&e), envelope_hash(&repeated));
repeated.message_id = Uuid::new_v4().to_string();
assert_ne!(envelope_hash(&e), envelope_hash(&repeated));
}
}
+27 -7
View File
@@ -116,7 +116,10 @@ async fn receive(socket: &mut WebSocket) -> Result<Envelope> {
Ok(Envelope::decode(bytes)?)
}
async fn send(
state: &crate::State,
socket: &mut WebSocket,
tenant: Uuid,
agent: Uuid,
e: &Envelope,
payload: Payload,
generation: u64,
@@ -133,11 +136,22 @@ async fn send(
sequence: e.sequence,
payload: Some(payload),
};
let claim =
dev_delivery_faults::duplicate_offer(&state.pool, tenant, agent, &reply, "wss").await?;
let bytes = reply.encode_to_vec();
tokio::time::timeout(
Duration::from_secs(10),
socket.send(Message::Binary(reply.encode_to_vec().into())),
socket.send(Message::Binary(bytes.clone().into())),
)
.await??;
if let Some(claim) = claim {
tokio::time::timeout(
Duration::from_secs(10),
socket.send(Message::Binary(bytes.into())),
)
.await??;
dev_delivery_faults::duplicate_emitted(&state.pool, claim).await?;
}
Ok(())
}
async fn channel(
@@ -162,7 +176,7 @@ async fn channel(
.send(Message::Binary(response.encode_to_vec().into()))
.await?;
if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? {
send(socket, &hello, payload, generation).await?;
send(state, socket, tenant, agent, &hello, payload, generation).await?;
}
let mut recent_messages = VecDeque::new();
let mut timer = tokio::time::interval(Duration::from_secs(1));
@@ -177,7 +191,7 @@ async fn channel(
let mut tx=rmm_persistence::tenant_tx(&state.pool,tenant).await?;
let revoked:bool=sqlx::query_scalar("SELECT revoked_at IS NOT NULL FROM agent_installations WHERE agent_id=$1").bind(agent).fetch_one(&mut *tx).await?;
tx.commit().await?;ensure!(!revoked,"device revoked");
if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(socket,&hello,payload,generation).await?;}
if let Some(payload)=crate::job_gateway::next(state,tenant,agent,generation).await?{send(state,socket,tenant,agent,&hello,payload,generation).await?;}
continue;
}
};
@@ -198,11 +212,17 @@ async fn channel(
"channel generation mismatch"
);
let response = crate::agent_service::ingest(state, tenant, agent, &e, "wss").await?;
socket
.send(Message::Binary(response.encode_to_vec().into()))
.await?;
if !dev_delivery_faults::after_ingest(&state.pool, tenant, agent, &e, &response, "wss")
.await?
{
socket
.send(Message::Binary(response.encode_to_vec().into()))
.await?;
}
if let Some(payload) = crate::job_gateway::next(state, tenant, agent, generation).await? {
send(socket, &e, payload, generation).await?;
send(state, socket, tenant, agent, &e, payload, generation).await?;
}
}
}
#[path = "dev_delivery_faults.rs"]
pub(crate) mod dev_delivery_faults;
+24
View File
@@ -5,6 +5,7 @@ mod api;
mod artifact_store;
mod auth;
mod catalog;
mod checkpoint_repair;
mod controls;
mod errors;
mod gateway;
@@ -68,6 +69,16 @@ async fn main() -> Result<()> {
.await;
}
let profile = env::var("RMM_PROFILE").unwrap_or_default();
gateway::dev_delivery_faults::validate_environment()?;
let repair = env::args().nth(1).as_deref() == Some("repair-unsent-sync-request");
if repair {
ensure!(env::args().count() == 2, "REPAIR_NO_ARGUMENTS");
checkpoint_repair::closed_policy()?;
ensure!(
std::path::Path::new(&env::var("RMM_JOB_KEY_FILE")?).is_file(),
"REPAIR_EXISTING_KEY_REQUIRED"
);
}
let dev = env::var("RMM_DEV_AUTH").as_deref() == Ok("true");
ensure!(
matches!(
@@ -126,6 +137,19 @@ async fn main() -> Result<()> {
.auth
.validate_separation(&state.viewer_a, &state.viewer_b)?;
}
if repair {
use std::io::{Read, Write};
let mut raw = Vec::new();
std::io::stdin()
.take(32 * 1024 * 1024 + 1)
.read_to_end(&mut raw)?;
ensure!(raw.len() <= 32 * 1024 * 1024, "REPAIR_INPUT_SIZE");
let evidence = serde_json::from_slice(&raw).context("REPAIR_EVIDENCE_JSON")?;
let receipt = checkpoint_repair::issue(&state, evidence).await?;
serde_json::to_writer(std::io::stdout().lock(), &receipt)?;
std::io::stdout().write_all(b"\n")?;
return Ok(());
}
if role.gateway() {
scheduler::start(state.clone());
}