//! Fixed development bootstrap for a proven unsent sync request. This promotes //! an existing signed denial only; it grants no readiness or execution rights. use anyhow::{Context, Result, ensure}; use chrono::{Duration, Utc}; use openssl::{ pkey::{Id, PKey}, sign::Verifier, }; use prost::Message; use rmm_domain::{checkpoint_repair::*, jobs::hash, reconciliation::ReconcileDecisionPayload}; use rmm_protocol::SyncRequest; use serde_json::{Value, json}; use sqlx::Row; use std::{ fs, os::unix::fs::{MetadataExt, PermissionsExt}, path::Path, }; use uuid::Uuid; const EVENT: &str = "unsent_request_repair_issued"; const FAULT: &str = "CHECKPOINT_CHAIN_MISMATCH"; pub fn closed_policy() -> Result<()> { ensure!( std::env::var("RMM_PROFILE").as_deref() == Ok("development") && std::env::var("RMM_DEV_AUTH").as_deref() == Ok("true") && std::env::var("RMM_SERVER_ROLE").as_deref() == Ok("api"), "REPAIR_DEVELOPMENT_API_ONLY" ); let path = std::env::var("RMM_DEV_ROOT_POLICY_FILE").context("REPAIR_POLICY_REQUIRED")?; let m = fs::symlink_metadata(&path)?; let parent = fs::symlink_metadata(Path::new(&path).parent().context("REPAIR_POLICY_PARENT")?)?; let key = std::env::var("RMM_JOB_KEY_FILE").context("REPAIR_EXISTING_KEY_REQUIRED")?; ensure!( m.is_file() && m.len() <= 65536 && m.permissions().mode() & 0o077 == 0 && m.uid() == fs::metadata(key)?.uid() && parent.is_dir() && parent.permissions().mode() & 0o022 == 0, "REPAIR_POLICY_PERMISSIONS" ); let p: Value = serde_json::from_slice(&fs::read(path)?)?; ensure!( p["schema_version"] == 1 && p["profile"] == "development" && p["expires_at"] .as_str() .and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok()) .is_some_and(|until| until <= Utc::now()), "REPAIR_POLICY_MUST_BE_CLOSED" ); Ok(()) } fn cell<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a Value> { let columns = table["columns"] .as_array() .context("REPAIR_INVENTORY_COLUMNS")?; let i = columns .iter() .position(|v| v.as_str() == Some(column)) .context("REPAIR_INVENTORY_COLUMN")?; row.as_array() .and_then(|r| r.get(i)) .context("REPAIR_INVENTORY_CELL") } fn rows(table: &Value) -> Result<&Vec> { table["rows"].as_array().context("REPAIR_INVENTORY_ROWS") } fn text<'a>(table: &'a Value, row: &'a Value, column: &str) -> Result<&'a str> { let v = cell(table, row, column)?; ensure!(v["type"] == "text", "REPAIR_CELL_TYPE"); v["value"].as_str().context("REPAIR_CELL_TEXT") } fn integer(table: &Value, row: &Value, column: &str) -> Result { let v = cell(table, row, column)?; ensure!(v["type"] == "integer", "REPAIR_CELL_TYPE"); v["value"].as_u64().context("REPAIR_CELL_INTEGER") } fn blob(table: &Value, row: &Value, column: &str, bytes: &[u8]) -> Result<()> { let v = cell(table, row, column)?; ensure!( v == &json!({"type":"blob","sha256":hash(bytes),"length":bytes.len()}), "REPAIR_WITNESS_BLOB" ); Ok(()) } fn row_by<'a>(table: &'a Value, column: &str, id: &str) -> Result<&'a Value> { let found = rows(table)? .iter() .filter(|row| text(table, row, column).ok() == Some(id)) .collect::>(); ensure!(found.len() == 1, "REPAIR_WITNESS_ROW"); Ok(found[0]) } fn decision( signer: &crate::job_signer::JobSigner, s: &SignedDecision, ) -> Result { ensure!( s.decision_bytes.len() <= 1024 * 1024 && s.signature.len() == 64, "REPAIR_DECISION_SIZE" ); let key = PKey::public_key_from_raw_bytes(&signer.public_key()?, Id::ED25519)?; ensure!( Verifier::new_without_digest(&key)?.verify_oneshot(&s.signature, &s.decision_bytes)?, "REPAIR_DECISION_SIGNATURE" ); let d: ReconcileDecisionPayload = serde_json::from_slice(&s.decision_bytes)?; ensure!( d.valid_profile(d.issued_at) && d.key_id == signer.key_id, "REPAIR_DECISION_PROFILE" ); Ok(d) } struct Case { old: ReconcileDecisionPayload, rejected: ReconcileDecisionPayload, unsent: SyncRequest, request: SyncRequest, } fn validate(signer: &crate::job_signer::JobSigner, e: &Evidence) -> Result { ensure!( e.schema_version == 1 && e.uncommitted_request.len() <= 1024 * 1024 && e.rejected_request.len() <= 1024 * 1024, "REPAIR_EVIDENCE_PROFILE" ); let old = decision(signer, &e.previous_valid_response)?; let rejected = decision(signer, &e.rejected_response)?; for d in [&old, &rejected] { ensure!( d.tenant_id == e.tenant_id && d.asset_id == e.asset_id && d.agent_id == e.agent_id && d.enrollment_generation == e.enrollment_generation && d.assignment_version == e.assignment_version, "REPAIR_IDENTITY_SCOPE" ); } let unsent = SyncRequest::decode(e.uncommitted_request.as_slice())?; let request = SyncRequest::decode(e.rejected_request.as_slice())?; let unsent_id = Uuid::parse_str(&unsent.request_id)?; ensure!( !unsent_id.is_nil() && unsent_id != rejected.request_id && unsent_id != old.request_id && old.decision_id != rejected.decision_id && rejected.request_id.to_string() == request.request_id && rejected.request_hash == hash(&e.rejected_request), "REPAIR_REQUEST_BINDING" ); ensure!( rejected.reason == FAULT && !rejected.ready && !rejected.helper_ready && !rejected.attempts.iter().any(|a| a.unlock), "REPAIR_DENIAL_ONLY" ); ensure!( request.previous_decision_id.is_empty() && request.previous_request_id == unsent.request_id && unsent.previous_decision_id == old.decision_id.to_string() && unsent.previous_request_id.is_empty(), "REPAIR_EXACT_UNSENT_CHAIN" ); let helper = rejected .helper_journal_id .context("REPAIR_HELPER_REQUIRED")?; ensure!( old.journal_id == rejected.journal_id && old.helper_journal_id == Some(helper) && old.checkpoint < unsent.checkpoint && unsent.checkpoint < rejected.checkpoint && old.helper_checkpoint <= unsent.helper_checkpoint && unsent.helper_checkpoint <= rejected.helper_checkpoint && old.connection_generation <= rejected.connection_generation, "REPAIR_MONOTONIC_CHAIN" ); for r in [&unsent, &request] { ensure!( r.journal_complete && r.assignment_version == e.assignment_version && r.journal_id == rejected.journal_id.to_string() && r.helper_journal_id == helper.to_string(), "REPAIR_REQUEST_SCOPE" ); } ensure!( request.checkpoint == rejected.checkpoint && request.helper_checkpoint == rejected.helper_checkpoint && request.helper_previous_decision_id == old.decision_id.to_string(), "REPAIR_REQUEST_CHECKPOINT" ); let am = &e.agent_inventory["reconcile_meta"]; let a = rows(am)?; ensure!(a.len() == 1, "REPAIR_AGENT_META"); let a = &a[0]; ensure!( integer(am, a, "id")? == 1 && text(am, a, "journal_id")? == rejected.journal_id.to_string() && integer(am, a, "checkpoint")? == rejected.checkpoint && integer(am, a, "high_sequence")? >= rejected.checkpoint && integer(am, a, "generation")? == rejected.connection_generation && text(am, a, "last_decision")? == rejected.decision_id.to_string() && text(am, a, "helper_id")? == helper.to_string() && integer(am, a, "helper_checkpoint")? == rejected.helper_checkpoint && text(am, a, "fault")? == FAULT && cell(am, a, "pending_request")? == &json!({"type":"null"}), "REPAIR_AGENT_HEAD" ); let hm = &e.helper_inventory["helper_sync_meta"]; let h = rows(hm)?; ensure!(h.len() == 1, "REPAIR_HELPER_META"); let h = &h[0]; ensure!( integer(hm, h, "id")? == 1 && text(hm, h, "journal_id")? == helper.to_string() && integer(hm, h, "checkpoint")? == rejected.helper_checkpoint && text(hm, h, "last_decision")? == rejected.decision_id.to_string() && text(hm, h, "agent_journal_id")? == rejected.journal_id.to_string() && integer(hm, h, "agent_checkpoint")? == rejected.checkpoint && text(hm, h, "fault")? == FAULT, "REPAIR_HELPER_HEAD" ); let ar = &e.agent_inventory["reconcile_requests"]; let x = row_by(ar, "id", &unsent.request_id)?; blob(ar, x, "request", &e.uncommitted_request)?; ensure!( integer(ar, x, "generation")? <= rejected.connection_generation && cell(ar, x, "response")? == &json!({"type":"null"}), "REPAIR_UNSENT_LOCAL_RESPONSE" ); let y = row_by(ar, "id", &request.request_id)?; blob(ar, y, "request", &e.rejected_request)?; blob( ar, y, "response", &rmm_protocol::SyncResponse { decision_bytes: e.rejected_response.decision_bytes.clone(), signature: e.rejected_response.signature.clone(), } .encode_to_vec(), )?; ensure!( integer(ar, y, "generation")? == rejected.connection_generation, "REPAIR_LOCAL_GENERATION" ); let hr = &e.helper_inventory["helper_sync_decisions"]; let y = row_by(hr, "decision_id", &rejected.decision_id.to_string())?; blob(hr, y, "request", &e.rejected_request)?; blob( hr, y, "response", &serde_json::to_vec(&e.rejected_response)?, )?; ensure!( integer(hr, y, "checkpoint")? == rejected.helper_checkpoint, "REPAIR_HELPER_CAPTURE" ); ensure!( e.helper_inventory["consumed"].is_object() && e.helper_inventory["execution_evidence"].is_object(), "REPAIR_LEDGER_REQUIRED" ); Ok(Case { old, rejected, unsent, request, }) } pub async fn issue(state: &crate::State, e: Evidence) -> Result { closed_policy()?; issue_with_closed_policy(state, e).await } async fn issue_with_closed_policy(state: &crate::State, e: Evidence) -> Result { let c = validate(&state.job_signer, &e)?; let evidence_hash = inventory_sha256(&serde_json::to_value(&e)?)?; let mut tx = rmm_persistence::tenant_tx(&state.pool, e.tenant_id).await?; crate::agent_service::control_lock(&mut tx, e.tenant_id, e.agent_id).await?; let device=sqlx::query("SELECT asset_id,enrollment_generation,assignment_version,connection_generation,connection_boot_id,revoked_at,sequence_regressed,sync_recovery_required,sync_ready,helper_sync_ready FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE") .bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_DEVICE_MISSING")?; ensure!( device.get::("asset_id") == e.asset_id && device.get::("enrollment_generation") == i64::try_from(e.enrollment_generation)? && device.get::("assignment_version") == i64::try_from(e.assignment_version)? // Faulted journals may survive newer ClientHello channels. This // historical receipt grants no lease; the next ordinary sync must // still bind its current channel. A regressed generation or changed // boot/identity is never repaired here. && device.get::("connection_generation") >= i64::try_from(c.rejected.connection_generation)? && device.get::, _>("connection_boot_id") == Some(c.rejected.boot_id) && device .get::>, _>("revoked_at") .is_none() && ![ "sequence_regressed", "sync_recovery_required", "sync_ready", "helper_sync_ready" ] .iter() .any(|k| device.get::(*k)), "REPAIR_DEVICE_FENCED" ); for (d, s) in [ (&c.old, &e.previous_valid_response), (&c.rejected, &e.rejected_response), ] { let stored=sqlx::query("SELECT decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3") .bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).fetch_optional(&mut *tx).await?.context("REPAIR_STORED_DECISION_MISSING")?; ensure!( stored.get::("decision_id") == d.decision_id && stored.get::("request_hash") == d.request_hash && stored.get::("journal_id") == d.journal_id && stored.get::("checkpoint") == i64::try_from(d.checkpoint)? && stored.get::("connection_generation") == i64::try_from(d.connection_generation)? && stored.get::, _>("payload") == s.decision_bytes && stored.get::, _>("signature") == s.signature, "REPAIR_STORED_DECISION_MISMATCH" ); } let committed:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sync_checkpoints WHERE tenant_id=$1 AND agent_id=$2 AND request_id=$3)") .bind(e.tenant_id).bind(e.agent_id).bind(Uuid::parse_str(&c.unsent.request_id)?).fetch_one(&mut *tx).await?; ensure!(!committed, "REPAIR_REQUEST_WAS_COMMITTED"); let head=sqlx::query("SELECT journal_id,checkpoint,decision_id,helper_journal_id,helper_checkpoint FROM agent_sync_state WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE") .bind(e.tenant_id).bind(e.agent_id).fetch_optional(&mut *tx).await?.context("REPAIR_HEAD_MISSING")?; let previous:Option=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type=$2 AND data->>'evidence_sha256'=$3 AND data->>'agent_id'=$4 ORDER BY created_at DESC LIMIT 1") .bind(e.tenant_id).bind(EVENT).bind(&evidence_hash).bind(e.agent_id.to_string()).fetch_optional(&mut *tx).await?; if let Some(previous) = previous { let signed: SignedReceipt = serde_json::from_value(previous["receipt"].clone())?; let receipt: Receipt = serde_json::from_slice(&signed.payload)?; let key = PKey::public_key_from_raw_bytes(&state.job_signer.public_key()?, Id::ED25519)?; ensure!( signed.signature.len() == 64 && Verifier::new_without_digest(&key)? .verify_oneshot(&signed.signature, &signed.payload)? && receipt.valid_profile(Utc::now()) && receipt.key_id == state.job_signer.key_id && receipt.tenant_id == e.tenant_id && receipt.asset_id == e.asset_id && receipt.agent_id == e.agent_id && receipt.rejected_decision_id == c.rejected.decision_id && receipt.rejected_request_sha256 == hash(&e.rejected_request) && receipt.uncommitted_request_sha256 == hash(&e.uncommitted_request) && receipt.agent_inventory_sha256 == inventory_sha256(&e.agent_inventory)? && receipt.helper_inventory_sha256 == inventory_sha256(&e.helper_inventory)? && head_matches(&head, &c.rejected)?, "REPAIR_REPLAY_EXPIRED_OR_HEAD_CHANGED" ); tx.commit().await?; return Ok(signed); } ensure!( head_matches(&head, &c.old)?, "REPAIR_AUTHORITATIVE_HEAD_CHANGED" ); let now = Utc::now(); let receipt = Receipt { domain: "rmm.sync.unsent-request-repair".into(), schema_version: 1, receipt_id: Uuid::new_v4(), tenant_id: e.tenant_id, asset_id: e.asset_id, agent_id: e.agent_id, enrollment_generation: e.enrollment_generation, assignment_version: e.assignment_version, issued_at: now, valid_until: now + Duration::minutes(10), key_id: state.job_signer.key_id.clone(), fault: FAULT.into(), agent_journal_id: c.rejected.journal_id, agent_checkpoint: c.rejected.checkpoint, helper_journal_id: c .rejected .helper_journal_id .context("REPAIR_HELPER_REQUIRED")?, helper_checkpoint: c.rejected.helper_checkpoint, rejected_decision_id: c.rejected.decision_id, rejected_decision_sha256: hash(&e.rejected_response.decision_bytes), rejected_request_id: c.rejected.request_id, rejected_request_sha256: hash(&e.rejected_request), uncommitted_request_id: Uuid::parse_str(&c.unsent.request_id)?, uncommitted_request_sha256: hash(&e.uncommitted_request), previous_valid_decision_id: c.old.decision_id, previous_valid_decision_sha256: hash(&e.previous_valid_response.decision_bytes), agent_inventory_sha256: inventory_sha256(&e.agent_inventory)?, helper_inventory_sha256: inventory_sha256(&e.helper_inventory)?, helper_ledger_sha256: helper_ledger_sha256(&e.helper_inventory)?, ready: false, helper_ready: false, unlock: false, }; ensure!(receipt.valid_profile(now), "REPAIR_RECEIPT_PROFILE"); let payload = serde_json::to_vec(&receipt)?; let signed = SignedReceipt { signature: state.job_signer.sign(&payload)?, payload, }; ensure!(sqlx::query("UPDATE agent_sync_state SET checkpoint=$3,decision_id=$4,helper_journal_id=$5,helper_checkpoint=$6 WHERE tenant_id=$1 AND agent_id=$2 AND decision_id=$7") .bind(e.tenant_id).bind(e.agent_id).bind(i64::try_from(c.request.checkpoint)?).bind(c.rejected.decision_id) .bind(c.rejected.helper_journal_id).bind(i64::try_from(c.rejected.helper_checkpoint)?).bind(c.old.decision_id) .execute(&mut *tx).await?.rows_affected()==1,"REPAIR_HEAD_CONCURRENT_CHANGE"); sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,$4,$5)") .bind(e.tenant_id).bind(receipt.receipt_id).bind("development-bootstrap:unsent-sync-repair").bind(EVENT) .bind(json!({"agent_id":e.agent_id,"evidence_sha256":evidence_hash,"receipt_sha256":hash(&signed.payload),"receipt":signed,"ready":false,"helper_ready":false,"unlock":false})) .execute(&mut *tx).await?; tx.commit().await?; Ok(signed) } fn head_matches(head: &sqlx::postgres::PgRow, d: &ReconcileDecisionPayload) -> Result { Ok(head.get::("journal_id") == d.journal_id && head.get::("checkpoint") == i64::try_from(d.checkpoint)? && head.get::("decision_id") == d.decision_id && head.get::, _>("helper_journal_id") == d.helper_journal_id && head.get::("helper_checkpoint") == i64::try_from(d.helper_checkpoint)?) } #[cfg(test)] mod tests { use super::*; fn table(fields: Vec<(&str, Value)>) -> Value { json!({"columns":fields.iter().map(|(k,_)|k).collect::>(),"rows":[fields.iter().map(|(_,v)|v).collect::>() ]}) } fn t(v: impl ToString) -> Value { json!({"type":"text","value":v.to_string()}) } fn n(v: u64) -> Value { json!({"type":"integer","value":v}) } fn b(v: &[u8]) -> Value { json!({"type":"blob","sha256":hash(v),"length":v.len()}) } fn sign(s: &crate::job_signer::JobSigner, d: &ReconcileDecisionPayload) -> SignedDecision { let decision_bytes = serde_json::to_vec(d).unwrap(); SignedDecision { signature: s.sign(&decision_bytes).unwrap(), decision_bytes, } } fn fixture() -> (crate::job_signer::JobSigner, Evidence, std::path::PathBuf) { let path = std::path::PathBuf::from(format!("/tmp/rmm-repair-{}", Uuid::new_v4())); let signer = crate::job_signer::JobSigner::load_or_create(&path.join("key.pem")).unwrap(); let tenant = Uuid::new_v4(); let asset = Uuid::new_v4(); let agent = Uuid::new_v4(); let journal = Uuid::new_v4(); let helper = Uuid::new_v4(); let now = Utc::now(); let old = ReconcileDecisionPayload { domain: "rmm.reconciliation-decision".into(), schema_version: 1, decision_id: Uuid::new_v4(), request_id: Uuid::new_v4(), request_hash: "ab".repeat(32), tenant_id: tenant, asset_id: asset, agent_id: agent, enrollment_generation: 1, assignment_version: 1, connection_generation: 3, boot_id: Uuid::new_v4(), journal_id: journal, checkpoint: 10, helper_journal_id: Some(helper), helper_checkpoint: 5, issued_at: now, valid_until: now + Duration::seconds(60), ready: false, helper_ready: false, reason: "ATTEMPT_RECONCILIATION_REQUIRED".into(), attempts: vec![], acknowledged_message_ids: vec![], key_id: signer.key_id.clone(), }; let x = SyncRequest { request_id: Uuid::new_v4().to_string(), journal_id: journal.to_string(), checkpoint: 11, previous_decision_id: old.decision_id.to_string(), assignment_version: 1, journal_complete: true, helper_journal_id: helper.to_string(), helper_checkpoint: 6, helper_previous_decision_id: old.decision_id.to_string(), ..Default::default() }; let y = SyncRequest { request_id: Uuid::new_v4().to_string(), checkpoint: 12, previous_decision_id: String::new(), previous_request_id: x.request_id.clone(), helper_checkpoint: 7, ..x.clone() }; let mut rejected = old.clone(); rejected.decision_id = Uuid::new_v4(); rejected.request_id = Uuid::parse_str(&y.request_id).unwrap(); rejected.request_hash = hash(&y.encode_to_vec()); rejected.checkpoint = 12; rejected.helper_checkpoint = 7; rejected.reason = FAULT.into(); let response = sign(&signer, &rejected); let request = y.encode_to_vec(); let unsent = x.encode_to_vec(); let proto = rmm_protocol::SyncResponse { decision_bytes: response.decision_bytes.clone(), signature: response.signature.clone(), } .encode_to_vec(); let mut requests = table(vec![ ("id", t(&x.request_id)), ("request", b(&unsent)), ("generation", n(3)), ("response", json!({"type":"null"})), ]); requests["rows"].as_array_mut().unwrap().push(json!([ t(&y.request_id), b(&request), n(3), b(&proto) ])); let am = table(vec![ ("id", n(1)), ("journal_id", t(journal)), ("checkpoint", n(12)), ("high_sequence", n(13)), ("generation", n(3)), ("last_decision", t(rejected.decision_id)), ("pending_request", json!({"type":"null"})), ("helper_id", t(helper)), ("helper_checkpoint", n(7)), ("fault", t(FAULT)), ]); let hm = table(vec![ ("id", n(1)), ("journal_id", t(helper)), ("checkpoint", n(7)), ("last_decision", t(rejected.decision_id)), ("agent_journal_id", t(journal)), ("agent_checkpoint", n(12)), ("fault", t(FAULT)), ]); let history = table(vec![ ("decision_id", t(rejected.decision_id)), ("checkpoint", n(7)), ("response", b(&serde_json::to_vec(&response).unwrap())), ("request", b(&request)), ]); let e = Evidence { schema_version: 1, tenant_id: tenant, asset_id: asset, agent_id: agent, enrollment_generation: 1, assignment_version: 1, agent_inventory: json!({"reconcile_meta":am,"reconcile_requests":requests}), helper_inventory: json!({"helper_sync_meta":hm,"helper_sync_decisions":history,"consumed":{"columns":[],"rows":[]},"execution_evidence":{"columns":[],"rows":[]}}), uncommitted_request: unsent, rejected_request: request, rejected_response: response, previous_valid_response: sign(&signer, &old), }; (signer, e, path) } #[test] fn only_exact_signed_unsent_case_validates() { let (s, e, p) = fixture(); validate(&s, &e).unwrap(); let mut other = e.clone(); other.tenant_id = Uuid::new_v4(); assert!(validate(&s, &other).is_err()); let mut other = e.clone(); other.asset_id = Uuid::new_v4(); assert!(validate(&s, &other).is_err()); let mut other = e.clone(); other.assignment_version += 1; assert!(validate(&s, &other).is_err()); let mut other = e.clone(); other.rejected_response.signature[0] ^= 1; assert!(validate(&s, &other).is_err()); let mut other = e.clone(); other.agent_inventory["reconcile_requests"]["rows"][0][3] = b(b"committed response"); assert!(validate(&s, &other).is_err()); let mut other = e.clone(); other.helper_inventory["helper_sync_meta"]["rows"][0][2] = n(8); assert!(validate(&s, &other).is_err()); let mut other = e.clone(); other.agent_inventory["reconcile_meta"]["rows"][0][9] = t("OTHER_FAULT"); assert!(validate(&s, &other).is_err()); let mut other = e.clone(); let mut x = SyncRequest::decode(other.uncommitted_request.as_slice()).unwrap(); x.previous_decision_id = Uuid::new_v4().to_string(); other.uncommitted_request = x.encode_to_vec(); assert!(validate(&s, &other).is_err()); fs::remove_dir_all(p).unwrap(); } #[test] fn signed_ready_response_is_never_repair_authority() { let (s, mut e, p) = fixture(); let mut d: ReconcileDecisionPayload = serde_json::from_slice(&e.rejected_response.decision_bytes).unwrap(); d.ready = true; e.rejected_response = sign(&s, &d); assert!(validate(&s, &e).is_err()); d.ready = false; d.reason = "CONSISTENT".into(); e.rejected_response = sign(&s, &d); assert!(validate(&s, &e).is_err()); fs::remove_dir_all(p).unwrap(); } /// Opt-in dedicated PostgreSQL only; the harness creates/destroys its own /// network-isolated container and never reads deployment credentials. #[tokio::test] #[ignore = "requires the dedicated isolated PostgreSQL harness"] async fn isolated_database_repair_and_fail_closed_paths() { let admin_url = std::env::var("RMM_TEST_REPAIR_ADMIN_URL").unwrap(); let app_url = std::env::var("RMM_TEST_REPAIR_APP_URL").unwrap(); let admin = sqlx::PgPool::connect(&admin_url).await.unwrap(); let pool = sqlx::PgPool::connect(&app_url).await.unwrap(); rmm_persistence::verify_application_role(&pool) .await .unwrap(); let (s, e, p) = fixture(); let c = validate(&s, &e).unwrap(); sqlx::query("INSERT INTO tenants VALUES($1,'isolated repair test')") .bind(e.tenant_id) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO assets(tenant_id,asset_id) VALUES($1,$2)") .bind(e.tenant_id) .bind(e.asset_id) .execute(&admin) .await .unwrap(); sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,assignment_version,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5,$6)") .bind(e.tenant_id).bind(e.agent_id).bind(e.asset_id).bind(hash(e.agent_id.as_bytes())).bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(c.rejected.boot_id).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO agent_sync_state VALUES($1,$2,$3,$4,$5,$6,$7)") .bind(e.tenant_id) .bind(e.agent_id) .bind(c.old.journal_id) .bind(i64::try_from(c.old.checkpoint).unwrap()) .bind(c.old.decision_id) .bind(c.old.helper_journal_id) .bind(i64::try_from(c.old.helper_checkpoint).unwrap()) .execute(&admin) .await .unwrap(); for (d, signed) in [ (&c.old, &e.previous_valid_response), (&c.rejected, &e.rejected_response), ] { sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)") .bind(e.tenant_id).bind(e.agent_id).bind(d.request_id).bind(d.decision_id).bind(&d.request_hash).bind(d.journal_id).bind(i64::try_from(d.checkpoint).unwrap()).bind(i64::try_from(d.connection_generation).unwrap()).bind(&signed.decision_bytes).bind(&signed.signature).execute(&admin).await.unwrap(); } let state = crate::State { pool, signer: None, viewer_a: [0; 32], viewer_b: [1; 32], auth: std::sync::Arc::new(crate::auth::Auth::from_env("development").await.unwrap()), job_signer: std::sync::Arc::new(s), }; let mut wrong = e.clone(); wrong.tenant_id = Uuid::new_v4(); assert!(issue_with_closed_policy(&state, wrong).await.is_err()); sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2") .bind(e.tenant_id) .bind(c.rejected.request_id) .execute(&admin) .await .unwrap(); assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); sqlx::query("INSERT INTO sync_checkpoints(tenant_id,agent_id,request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,payload,signature) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)") .bind(e.tenant_id).bind(e.agent_id).bind(c.rejected.request_id).bind(c.rejected.decision_id) .bind(&c.rejected.request_hash).bind(c.rejected.journal_id).bind(i64::try_from(c.rejected.checkpoint).unwrap()) .bind(i64::try_from(c.rejected.connection_generation).unwrap()).bind(&e.rejected_response.decision_bytes) .bind(&e.rejected_response.signature).execute(&admin).await.unwrap(); sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint+1 WHERE agent_id=$1") .bind(e.agent_id) .execute(&admin) .await .unwrap(); assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); sqlx::query("UPDATE agent_sync_state SET checkpoint=checkpoint-1 WHERE agent_id=$1") .bind(e.agent_id) .execute(&admin) .await .unwrap(); sqlx::query("UPDATE agent_installations SET revoked_at=now() WHERE agent_id=$1") .bind(e.agent_id) .execute(&admin) .await .unwrap(); assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); sqlx::query("UPDATE agent_installations SET revoked_at=NULL WHERE agent_id=$1") .bind(e.agent_id) .execute(&admin) .await .unwrap(); let x = Uuid::parse_str(&c.unsent.request_id).unwrap(); sqlx::query("INSERT INTO sync_checkpoints SELECT tenant_id,agent_id,$2,$3,request_hash,journal_id,checkpoint,connection_generation,payload,signature,created_at FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$4") .bind(e.tenant_id).bind(x).bind(Uuid::new_v4()).bind(c.old.request_id).execute(&admin).await.unwrap(); assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); sqlx::query("DELETE FROM sync_checkpoints WHERE tenant_id=$1 AND request_id=$2") .bind(e.tenant_id) .bind(x) .execute(&admin) .await .unwrap(); sqlx::query("UPDATE agent_installations SET connection_generation=$2 WHERE agent_id=$1") .bind(e.agent_id) .bind(i64::try_from(c.rejected.connection_generation).unwrap() - 1) .execute(&admin) .await .unwrap(); assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); sqlx::query("UPDATE agent_installations SET connection_generation=$2,connection_boot_id=$3 WHERE agent_id=$1") .bind(e.agent_id).bind(i64::try_from(c.rejected.connection_generation).unwrap()+1).bind(Uuid::new_v4()) .execute(&admin).await.unwrap(); assert!(issue_with_closed_policy(&state, e.clone()).await.is_err()); sqlx::query("UPDATE agent_installations SET connection_boot_id=$2 WHERE agent_id=$1") .bind(e.agent_id) .bind(c.rejected.boot_id) .execute(&admin) .await .unwrap(); let first = issue_with_closed_policy(&state, e.clone()).await.unwrap(); assert_eq!( first, issue_with_closed_policy(&state, e.clone()).await.unwrap() ); let r: Receipt = serde_json::from_slice(&first.payload).unwrap(); assert!(r.valid_profile(Utc::now())); assert!(!r.ready && !r.helper_ready && !r.unlock); let head: Uuid = sqlx::query_scalar("SELECT decision_id FROM agent_sync_state WHERE agent_id=$1") .bind(e.agent_id) .fetch_one(&admin) .await .unwrap(); assert_eq!(head, c.rejected.decision_id); let count: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_events WHERE tenant_id=$1") .bind(e.tenant_id) .fetch_one(&admin) .await .unwrap(); assert_eq!(count, 1); let effects:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM outbox)+(SELECT count(*) FROM execution_permits)").fetch_one(&admin).await.unwrap(); assert_eq!(effects, 0); let ready: bool = sqlx::query_scalar( "SELECT sync_ready OR helper_sync_ready FROM agent_installations WHERE agent_id=$1", ) .bind(e.agent_id) .fetch_one(&admin) .await .unwrap(); assert!(!ready); fs::remove_dir_all(p).unwrap(); } }