Permit proven closed-control fresh sync and retry only precommit clock reads
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-16 22:29:45 +01:00
parent 2092596936
commit 052456f133
13 changed files with 1349 additions and 39 deletions
@@ -0,0 +1,180 @@
//! Fixed read-only fixture exporter / in-memory control contract. No executor.
use anyhow::{Result, ensure};
#[cfg(windows)]
use serde_json::Value;
#[cfg(windows)]
fn export() -> Result<Value> {
use prost::Message;
use rusqlite::{Connection, OpenFlags, ToSql, types::ValueRef};
use serde_json::json;
use std::{ffi::c_void, os::windows::fs::MetadataExt, path::Path};
#[link(name = "advapi32")]
unsafe extern "system" {
fn OpenProcessToken(p: *mut c_void, a: u32, t: *mut *mut c_void) -> i32;
fn GetTokenInformation(t: *mut c_void, c: u32, v: *mut c_void, n: u32, r: *mut u32) -> i32;
}
#[link(name = "kernel32")]
unsafe extern "system" {
fn GetCurrentProcess() -> *mut c_void;
fn CloseHandle(h: *mut c_void) -> i32;
}
let mut token = std::ptr::null_mut();
ensure!(
unsafe { OpenProcessToken(GetCurrentProcess(), 8, &mut token) } != 0,
"FIXTURE_ADMIN_TOKEN"
);
let mut elevated = 0u32;
let mut size = 0;
let good =
unsafe { GetTokenInformation(token, 20, (&mut elevated as *mut u32).cast(), 4, &mut size) };
unsafe { CloseHandle(token) };
ensure!(good != 0 && elevated != 0, "FIXTURE_ADMIN_REQUIRED");
fn path(p: &str) -> Result<()> {
for a in Path::new(p).ancestors() {
ensure!(
a.symlink_metadata()?.file_attributes() & 0x400 == 0,
"FIXTURE_NO_REPARSE"
);
}
Ok(())
}
fn open(p: &str) -> Result<Connection> {
path(p)?;
let d = Connection::open_with_flags(
p,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
d.busy_timeout(std::time::Duration::from_millis(500))?;
d.execute_batch("BEGIN")?;
Ok(d)
}
fn rows(db: &Connection, sql: &str, args: &[&dyn ToSql]) -> Result<Value> {
let mut q = db.prepare(sql)?;
let n = q.column_count();
let mut r = q.query(args)?;
let mut out = vec![];
while let Some(row) = r.next()? {
ensure!(out.len() < 8, "FIXTURE_ROW_BOUND");
let mut cols = vec![];
for i in 0..n {
cols.push(match row.get_ref(i)? {
ValueRef::Null => json!(["null"]),
ValueRef::Integer(v) => json!(["integer", v]),
ValueRef::Text(v) => json!(["text", std::str::from_utf8(v)?]),
ValueRef::Blob(v) => {
ensure!(v.len() <= 1024 * 1024, "FIXTURE_BLOB_BOUND");
json!(["blob", v])
}
_ => anyhow::bail!("FIXTURE_CELL_TYPE"),
});
}
out.push(cols);
}
Ok(json!(out))
}
const A: &str = r"C:\ProgramData\RustDiscovery\state\telemetry.sqlite";
const C: &str = r"C:\ProgramData\RustDiscovery\state\consumption.sqlite";
const H: &str = r"C:\ProgramData\RustDiscoverySystem\state\ledger.sqlite";
let a = open(A)?;
let c = open(C)?;
let h = open(H)?;
let head_a:String=a.query_row("SELECT last_decision FROM reconcile_meta WHERE id=1 AND fault IS NULL AND pending_request IS NULL",[],|r|r.get(0))?;
let (head_h, hcp): (String, u64) = h.query_row(
"SELECT last_decision,checkpoint FROM helper_sync_meta WHERE id=1 AND fault IS NULL",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
ensure!(
head_a == "7ea5dbda-2b61-4bb2-9c3c-3dbb4037e33a"
&& head_h == "7245d5c4-e078-4c76-a55f-d4afbca405ba",
"FIXTURE_EXACT_OBSERVED_HEADS"
);
let mut ids = vec![];
for id in [&head_a, &head_h] {
let raw: Vec<u8> = c.query_row(
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
[id],
|r| r.get(0),
)?;
let response = rmm_protocol::SyncResponse::decode(raw.as_slice())?;
let d: rmm_domain::reconciliation::ReconcileDecisionPayload =
serde_json::from_slice(&response.decision_bytes)?;
ensure!(d.decision_id.to_string() == *id, "FIXTURE_ACK_ID");
ids.push(d.request_id.to_string());
}
let binding_path = r"C:\ProgramData\RustDiscovery\binding.json";
path(binding_path)?;
let raw = std::fs::read(binding_path)?;
ensure!(raw.len() <= 8192, "FIXTURE_PUBLIC_BINDING_BOUND");
let binding: rmm_agent_core::auth::Binding = serde_json::from_slice(&raw)?;
ensure!(
binding.agent_id.to_string() == "63ea15b2-0643-403d-b5b4-cba1288dbea3"
&& binding.asset_id.to_string() == "a176520e-8a91-4e93-a78b-8e7620f430e0",
"FIXTURE_EXACT_TEST_GUEST"
);
let resolution = "SELECT grant_id,role,receipt,receipt_sha256,derived_outcome FROM windows_system_nonstart_221_resolution LIMIT 9";
let value = json!({"schema_version":1,"read_only":true,"database_writes":false,"private_keys_read":false,"captured_at_utc":chrono::Utc::now(),"binding":binding,
"agent":{
"meta":rows(&a,"SELECT id,journal_id,checkpoint,high_sequence,generation,last_decision,pending_request,helper_id,helper_checkpoint,fault FROM reconcile_meta",&[])?,
"counter":rows(&a,"SELECT id,seq FROM counter",&[])?,
"requests":rows(&a,"SELECT id,request,envelope,generation,response FROM reconcile_requests WHERE id IN (?1,?2) ORDER BY rowid",&[&ids[0],&ids[1]])?,
"jobs":rows(&a,"SELECT job,attempt,offer,state,revision,grant_id FROM job_attempts ORDER BY job LIMIT 9",&[])?,
"outcomes":rows(&a,"SELECT job,attempt,outcome,decision_id FROM reconcile_outcomes ORDER BY job LIMIT 9",&[])?,
"results":rows(&a,"SELECT id,job,attempt,revision,envelope FROM reconcile_results ORDER BY rowid LIMIT 9",&[])?,
"events":rows(&a,"SELECT id,payload FROM job_events ORDER BY rowid LIMIT 9",&[])?,
"legacy_count":a.query_row("SELECT count(*) FROM events",[],|r|r.get::<_,u64>(0))?,
"resolution":rows(&a,resolution,&[])?},
"ledger":{
"consumed":rows(&c,"SELECT grant_id,agent,generation,job,attempt,result FROM consumed ORDER BY grant_id LIMIT 9",&[])?,
"evidence":rows(&c,"SELECT grant_id,start,outcome FROM execution_evidence ORDER BY grant_id LIMIT 9",&[])?,
"acks":rows(&c,"SELECT decision_id,response FROM reconcile_consumption WHERE decision_id IN (?1,?2)",&[&head_a,&head_h])?,
"resolution":rows(&c,resolution,&[])?},
"helper":{
"meta":rows(&h,"SELECT journal_id,checkpoint,generation,last_decision,agent_journal_id,agent_checkpoint,fault FROM helper_sync_meta WHERE id=1",&[])?,
"capture":rows(&h,"SELECT checkpoint,snapshot,fingerprint,captured_at FROM helper_sync_captures WHERE checkpoint=?1",&[&hcp])?,
"decision":rows(&h,"SELECT decision_id,checkpoint,response,request,fingerprint FROM helper_sync_decisions WHERE decision_id=?1",&[&head_h])?}});
a.execute_batch("ROLLBACK")?;
c.execute_batch("ROLLBACK")?;
h.execute_batch("ROLLBACK")?;
ensure!(
a.query_row(
"SELECT last_decision FROM reconcile_meta WHERE id=1",
[],
|r| r.get::<_, String>(0)
)? == head_a
&& h.query_row(
"SELECT last_decision FROM helper_sync_meta WHERE id=1",
[],
|r| r.get::<_, String>(0)
)? == head_h,
"FIXTURE_AUTHORITATIVE_HEAD_CHANGED"
);
ensure!(
serde_json::to_vec(&value)?.len() <= 4 * 1024 * 1024,
"FIXTURE_TOTAL_BOUND"
);
Ok(value)
}
fn main() -> Result<()> {
let args = std::env::args().skip(1).collect::<Vec<_>>();
if args == ["--export"] {
#[cfg(windows)]
{
println!("{}", export()?);
return Ok(());
}
#[cfg(not(windows))]
anyhow::bail!("FIXTURE_EXPORT_WINDOWS_ONLY");
}
ensure!(
args.len() == 2 && args[0] == "--contract",
"FIXTURE_FIXED_ARGUMENTS"
);
let raw = std::fs::read(&args[1])?;
ensure!(raw.len() <= 4 * 1024 * 1024, "FIXTURE_TOTAL_BOUND");
println!(
"{}",
rmm_agent_core::reconcile::closed_control_contract(&raw)?
);
Ok(())
}
+114 -2
View File
@@ -168,6 +168,36 @@ struct Lease {
until: DateTime<Utc>,
fingerprint: String,
}
/// Proof of failure at the very first clock read, before BEGIN or any
/// durable helper mutation. Private construction prevents remote assertions.
#[derive(Debug)]
pub struct ApplyPrecommitClockError {
cause: anyhow::Error,
}
impl std::fmt::Display for ApplyPrecommitClockError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("HELPER_SYNC_PRECOMMIT_CLOCK")
}
}
impl std::error::Error for ApplyPrecommitClockError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(self.cause.as_ref())
}
}
/// Retry authority for this control call only, never for Execute. Trust/key,
/// generation, signature, binding and later storage failures are not eligible.
pub fn retryable_precommit_clock(error: &anyhow::Error) -> bool {
error
.downcast_ref::<ApplyPrecommitClockError>()
.is_some_and(|typed| {
typed.cause.chain().any(|e| {
matches!(
e.to_string().as_str(),
"TIME_STALE_RESPONSE" | "TIME_RUNTIME_NO_EVIDENCE" | "TIME_NATIVE_NO_EVIDENCE"
)
})
})
}
pub struct HelperSync<V = DefaultVerifier> {
binding: Binding,
verifier: V,
@@ -598,7 +628,9 @@ impl<V: Ed25519Verifier> HelperSync<V> {
let result = self.apply_inner(db, guard, response, raw_request, clock, boot);
if let Err(error) = &result {
self.lease = None;
persist_fault(db, error);
if error.downcast_ref::<ApplyPrecommitClockError>().is_none() {
persist_fault(db, error);
}
}
result
}
@@ -612,7 +644,9 @@ impl<V: Ed25519Verifier> HelperSync<V> {
boot: &impl BootSource,
) -> Result<Applied> {
guard.check(db)?;
let now = clock.now()?;
let now = clock
.now()
.map_err(|cause| ApplyPrecommitClockError { cause })?;
let capture = self
.capture
.as_ref()
@@ -2005,3 +2039,81 @@ pub(crate) fn nonstart221_memory_rows(
None,
)
}
/// Memory-only companion to the selected actual control regression. Verifies
/// the new retry type is possible only before durable writes, retaining the
/// exact capture and never creating a lease or retrying a signature failure.
#[doc(hidden)]
pub fn precommit_clock_contract(binding: Binding, snapshot: Snapshot) -> Result<()> {
struct Guard;
impl ExecutorLedgerGuard for Guard {
fn check(&self, _db: &Connection) -> Result<()> {
Ok(())
}
}
struct Time(DateTime<Utc>);
impl Clock for Time {
fn now(&self) -> Result<DateTime<Utc>> {
Ok(self.0)
}
}
struct Missing(&'static str);
impl Clock for Missing {
fn now(&self) -> Result<DateTime<Utc>> {
anyhow::bail!(self.0)
}
}
struct Boot(Uuid);
impl BootSource for Boot {
fn current_boot(&self) -> Result<Uuid> {
Ok(self.0)
}
}
let clock = Time(snapshot.captured_at);
let boot = Boot(snapshot.boot_id);
let raw = serde_json::to_vec(&snapshot)?;
let mut sync = HelperSync::with_verifier(binding, crate::crypto::DalekVerifier, &clock)?;
sync.capture = Some(Capture {
snapshot,
fingerprint: "unchanged".into(),
generation: 1,
});
let mut db = Connection::open_in_memory()?;
initialize(&db, &sync.binding)?;
let response = SignedResponse {
decision_bytes: vec![],
signature: vec![],
};
let before = db.total_changes();
for (code, eligible) in [
("TIME_STALE_RESPONSE", true),
("TIME_RUNTIME_GENERATION", false),
] {
let error = sync
.apply(&mut db, &Guard, &response, &[], &Missing(code), &boot)
.err()
.context("CONTROL_EXPECTED_CLOCK_OR_SIGNATURE_ERROR")?;
ensure!(
retryable_precommit_clock(&error) == eligible
&& db.total_changes() == before
&& serde_json::to_vec(
&sync
.capture
.as_ref()
.context("CONTROL_CAPTURE_LOST")?
.snapshot
)? == raw
&& sync.lease.is_none(),
"CONTROL_PRECOMMIT_CLOCK_CLASSIFICATION"
);
}
let error = sync
.apply(&mut db, &Guard, &response, &[], &clock, &boot)
.err()
.context("CONTROL_EXPECTED_CLOCK_OR_SIGNATURE_ERROR")?;
ensure!(
!retryable_precommit_clock(&error) && sync.lease.is_none(),
"CONTROL_SIGNATURE_ERROR_NOT_RETRYABLE"
);
Ok(())
}
+614 -5
View File
@@ -239,6 +239,314 @@ impl<V: Ed25519Verifier> ClientSync<V> {
gate.helper = None;
Ok(())
}
/// Historical authentication only. Never apply an old response or create a
/// lease: the returned bytes prove a previously durable local control step.
fn stored_control(
&self,
db: &Connection,
ledger: &Connection,
id: Uuid,
boot: Uuid,
) -> Result<(SyncRequest, ReconcileDecisionPayload)> {
let ack: Vec<u8> = ledger.query_row(
"SELECT response FROM reconcile_consumption WHERE decision_id=?1",
[id.to_string()],
|r| r.get(0),
)?;
ensure!(
ack.len() <= rmm_domain::MAX_MESSAGE,
"CLOSED_CONTROL_ACK_BOUND"
);
let response = SyncResponse::decode(ack.as_slice())?;
ensure!(
response.encode_to_vec() == ack,
"CLOSED_CONTROL_CANONICAL_ACK"
);
let parsed: ReconcileDecisionPayload = serde_json::from_slice(&response.decision_bytes)?;
let (raw, envelope, generation, saved): (Vec<u8>, Vec<u8>, u64, Vec<u8>) = db.query_row(
"SELECT request,envelope,generation,response FROM reconcile_requests WHERE id=?1",
[parsed.request_id.to_string()],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)?;
ensure!(
raw.len() <= rmm_domain::MAX_MESSAGE
&& envelope.len() <= rmm_domain::MAX_MESSAGE
&& saved == ack,
"CLOSED_CONTROL_ORIGINAL_BYTES"
);
let request = SyncRequest::decode(raw.as_slice())?;
let env = Envelope::decode(envelope.as_slice())?;
ensure!(
raw == request.encode_to_vec()
&& env.encode_to_vec() == envelope
&& env.payload == Some(Payload::SyncRequest(request.clone()))
&& rmm_protocol::valid_envelope(
&env,
&self.binding.agent_id.to_string(),
generation
)
&& env.boot_id == boot.to_string()
&& env.sequence == request.checkpoint,
"CLOSED_CONTROL_ORIGINAL_ENVELOPE"
);
let decision = verify_response_with(
&self.verifier,
&self.binding,
&request,
&Channel {
connection_generation: generation,
boot_id: boot,
},
&response,
parsed.issued_at,
)?;
ensure!(
decision.decision_id == id
&& decision.boot_id == boot
&& !recovery_reason(&decision.reason),
"CLOSED_CONTROL_SIGNED_ORIGINAL"
);
Ok((request, decision))
}
/// Admit only a new, ordinary online reconciliation of an exact missing
/// helper ACK with already definitive, identical outcomes. The caller owns
/// agent.lock, has no live executor/deferred work, and obtained `helper`
/// from the serialized, independently authenticated current-boot helper.
/// This predicate is read-only; fresh server/own-clock/helper ACK gates are
/// unchanged and no flag or authority from either old response is reused.
fn closed_control_continuation(
&self,
db: &Connection,
ledger: &Connection,
channel: &Channel,
helper: &HelperSnapshot,
meta: &Meta,
now: DateTime<Utc>,
) -> Result<()> {
ensure!(
self.helper_identity == HelperIdentity::System
&& self.pending.is_none()
&& self.gate.is_none()
&& meta.fault.is_none()
&& meta.pending_request.is_none(),
"CLOSED_CONTROL_IDLE_ONLY"
);
let a = meta
.decision
.context("CLOSED_CONTROL_AGENT_HEAD_REQUIRED")?;
let h = helper
.previous_decision_id
.context("CLOSED_CONTROL_HELPER_HEAD_REQUIRED")?;
ensure!(
a != h
&& Some(helper.journal_id) == meta.helper_id
&& helper.checkpoint >= meta.helper_checkpoint
&& pending_ids(db)?.is_empty(),
"CLOSED_CONTROL_CURRENT_HEADS"
);
let legacy: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='events')",
[],
|r| r.get(0),
)?;
ensure!(
!legacy
|| db.query_row("SELECT count(*) FROM events", [], |r| r.get::<_, u64>(0))? == 0,
"CLOSED_CONTROL_NO_PENDING_EVENTS"
);
let (original, decision) = self.stored_control(db, ledger, a, channel.boot_id)?;
let (_, previous) = self.stored_control(db, ledger, h, channel.boot_id)?;
ensure!(
decision.journal_id == meta.journal_id
&& decision.checkpoint == u64::try_from(meta.checkpoint)?
&& original.journal_id == meta.journal_id.to_string()
&& original.checkpoint == decision.checkpoint
&& original.previous_decision_id == h.to_string()
&& original.helper_previous_decision_id == h.to_string()
&& original.previous_request_id.is_empty()
&& original.journal_complete
&& original.durable_message_ids.is_empty()
&& decision.issued_at <= now
&& decision.connection_generation <= channel.connection_generation
&& previous.connection_generation <= decision.connection_generation
&& previous.checkpoint < decision.checkpoint
&& previous.journal_id == decision.journal_id
&& previous.helper_journal_id == Some(helper.journal_id)
&& decision.helper_journal_id == Some(helper.journal_id)
&& original.helper_journal_id == helper.journal_id.to_string()
&& previous.helper_checkpoint < decision.helper_checkpoint
&& decision.helper_checkpoint == meta.helper_checkpoint
&& helper.checkpoint >= decision.helper_checkpoint,
"CLOSED_CONTROL_EXACT_PARTIAL_EDGE"
);
let current = attempts(db, ledger, &self.binding)?;
ensure!(
current.len() == original.attempts.len()
&& current.len() == decision.attempts.len()
&& current.len() <= 256,
"CLOSED_CONTROL_EXACT_ATTEMPT_SET"
);
let expected_consumed = current
.iter()
.filter(|r| r.consumption == "consumed")
.count() as u64;
let ledger_counts: (u64, u64, u64) = ledger.query_row(
"SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM execution_evidence),(SELECT count(*) FROM consumed WHERE result IS NULL)",
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
ensure!(
ledger_counts == (expected_consumed, expected_consumed, 0),
"CLOSED_CONTROL_EXACT_COMPLETE_LEDGER"
);
for row in &current {
ensure!(
terminal(&row.state) && row.update_handoff_bytes.is_empty(),
"CLOSED_CONTROL_DEFINITIVE_ONLY"
);
let old = original
.attempts
.iter()
.find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id)
.context("CLOSED_CONTROL_ORIGINAL_ATTEMPT")?;
let signed = decision
.attempts
.iter()
.find(|r| {
r.job_id.to_string() == row.job_id && r.attempt_id.to_string() == row.attempt_id
})
.context("CLOSED_CONTROL_SIGNED_ATTEMPT")?;
ensure!(
old.state == row.state
&& old.job_payload_hash == row.job_payload_hash
&& old.grant_id == row.grant_id
&& old.consumption == row.consumption
&& old.original_boot_id == row.original_boot_id
&& old.outcome_bytes == row.outcome_bytes
&& old.outcome_sha256 == row.outcome_sha256
&& old.update_handoff_bytes.is_empty()
&& row.revision == old.revision.max(signed.revision)
&& signed.outcome_state == row.state
&& signed.update_inflight.is_none(),
"CLOSED_CONTROL_RESULT_UNCHANGED"
);
// An ungranted expiry has no fabricated outcome. Its exact
// definitive nonstart is already authenticated in the original
// request/decision; it can only permit a NEW online request.
if row.consumption == "unconsumed" {
let consumed: u64 = ledger.query_row(
"SELECT count(*) FROM consumed WHERE job=?1 AND attempt=?2",
params![row.job_id, row.attempt_id],
|r| r.get(0),
)?;
ensure!(
matches!(row.state.as_str(), "expired" | "cancelled")
&& row.grant_id.is_empty()
&& row.original_boot_id.is_empty()
&& row.outcome_bytes.is_empty()
&& row.outcome_sha256.is_empty()
&& signed.grant_id.is_none()
&& !signed.unlock
&& signed.reason == "DEFINITIVE_NONSTART"
&& signed.previous_outcome_sha256.is_none()
&& signed.outcome_sha256.is_none()
&& signed.outcome_bytes.is_empty()
&& consumed == 0
&& !helper
.attempts
.iter()
.any(|h| h.job_id == row.job_id && h.attempt_id == row.attempt_id),
"CLOSED_CONTROL_SIGNED_UNGRANTED_NONSTART"
);
continue;
}
ensure!(
!row.outcome_bytes.is_empty()
&& signed.grant_id.map(|id| id.to_string()).as_deref()
== Some(row.grant_id.as_str())
&& hash(&row.outcome_bytes) == row.outcome_sha256
&& signed.outcome_sha256.as_deref() == Some(row.outcome_sha256.as_str())
&& (signed.outcome_bytes.is_empty()
|| signed.outcome_bytes == row.outcome_bytes),
"CLOSED_CONTROL_SIGNED_CANONICAL_RESULT"
);
let outcome: ReconciledOutcome = serde_json::from_slice(&row.outcome_bytes)?;
ensure!(
canonical_outcome_bytes(&outcome)? == row.outcome_bytes
&& outcome.state == row.state,
"CLOSED_CONTROL_CANONICAL_RESULT"
);
match row.consumption.as_str() {
"consumed" => {
let (start,finished,last):(Vec<u8>,Vec<u8>,Vec<u8>)=ledger.query_row("SELECT e.start,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1 AND c.job=?2 AND c.attempt=?3 AND c.agent=?4 AND c.generation=?5",params![row.grant_id,row.job_id,row.attempt_id,self.binding.agent_id.to_string(),self.binding.enrollment_generation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
let saved: serde_json::Value = serde_json::from_slice(&start)?;
let request: crate::auth::Request =
serde_json::from_value(saved["request"].clone())?;
let permit: rmm_domain::jobs::ExecutionPermitPayload =
serde_json::from_slice(&request.permit_bytes)?;
let (job, permit) =
self.binding
.verify_with(&request, permit.not_before, &self.verifier)?;
ensure!(
job.job_id.to_string() == row.job_id
&& job.attempt_id.to_string() == row.attempt_id
&& hash(&request.job_bytes) == row.job_payload_hash
&& permit.grant_id.to_string() == row.grant_id
&& !matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. })
&& canonical_outcome_bytes(&serde_json::from_slice::<
ReconciledOutcome,
>(&finished)?)?
== row.outcome_bytes,
"CLOSED_CONTROL_ORIGINAL_CONSUMPTION"
);
if crate::windows_system_nonstart::effective_outcome(
ledger,
Some(&self.binding),
permit.grant_id,
)?
.is_none()
{
ensure!(
canonical_outcome_bytes(&serde_json::from_slice::<ReconciledOutcome>(
&last
)?)? == row.outcome_bytes,
"CLOSED_CONTROL_ORIGINAL_RESULT_CONFLICT"
);
}
if job.execution_identity == self.helper_identity.as_str() {
let r = helper
.attempts
.iter()
.find(|r| r.job_id == row.job_id && r.attempt_id == row.attempt_id)
.context("CLOSED_CONTROL_HELPER_ATTEMPT_REQUIRED")?;
ensure!(
r.state == row.state
&& r.grant_id == row.grant_id
&& r.job_payload_hash == row.job_payload_hash
&& r.consumption == "consumed"
&& r.outcome_bytes == row.outcome_bytes
&& r.outcome_sha256 == row.outcome_sha256
&& r.original_boot_id == row.original_boot_id
&& r.revision <= row.revision
&& r.update_handoff_bytes.is_empty(),
"CLOSED_CONTROL_HELPER_RESULT_UNCHANGED"
);
}
}
_ => anyhow::bail!("CLOSED_CONTROL_UNCERTAIN_CONSUMPTION"),
}
}
for row in &helper.attempts {
ensure!(
terminal(&row.state)
&& row.update_handoff_bytes.is_empty()
&& current.iter().any(|r| r.job_id == row.job_id
&& r.attempt_id == row.attempt_id
&& r.grant_id == row.grant_id
&& r.outcome_bytes == row.outcome_bytes),
"CLOSED_CONTROL_NO_EXTRA_HELPER_ATTEMPT"
);
}
Ok(())
}
pub fn prepare(
&mut self,
db: &mut Connection,
@@ -247,7 +555,21 @@ impl<V: Ed25519Verifier> ClientSync<V> {
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
) -> Result<PreparedSync> {
let result = self.prepare_inner(db, ledger, channel, helper, clock);
self.prepare_with_closed_control_idle(db, ledger, channel, helper, clock, false)
}
/// The OS actor may attest quiescence only after its execution/deferred
/// workers and protected intent store are all settled. Ordinary same-head
/// synchronization does not need this attestation, including while busy.
pub fn prepare_with_closed_control_idle(
&mut self,
db: &mut Connection,
ledger: &Connection,
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
closed_control_idle: bool,
) -> Result<PreparedSync> {
let result = self.prepare_inner(db, ledger, channel, helper, clock, closed_control_idle);
if let Err(error) = &result {
let text = error.to_string();
if [
@@ -275,6 +597,7 @@ impl<V: Ed25519Verifier> ClientSync<V> {
channel: Channel,
helper: Option<&HelperSnapshot>,
clock: &impl SyncClock,
closed_control_idle: bool,
) -> Result<PreparedSync> {
self.close();
let now = clock.now()?;
@@ -309,10 +632,11 @@ impl<V: Ed25519Verifier> ClientSync<V> {
// A locally committed Windows decision is not proof of its
// application by SYSTEM. Never advance either checkpoint
// from an older helper head, including after process loss.
ensure!(
meta.decision.is_some() && h.previous_decision_id == meta.decision,
"SYSTEM_HELPER_APPLY_UNCONFIRMED"
);
if meta.decision.is_none() || h.previous_decision_id != meta.decision {
ensure!(closed_control_idle, "SYSTEM_HELPER_APPLY_UNCONFIRMED");
self.closed_control_continuation(&tx, ledger, &channel, h, &meta, now)
.context("SYSTEM_HELPER_APPLY_UNCONFIRMED")?;
}
}
}
} else {
@@ -1557,3 +1881,288 @@ fn validate_attempt(
);
Ok(Some(bytes))
}
/// Operator regression only: imports the bounded, selected actual public
/// fixture into two IN-MEMORY databases. No production path/network/executor.
#[doc(hidden)]
pub fn closed_control_contract(raw: &[u8]) -> Result<serde_json::Value> {
use rusqlite::types::Value as SqlValue;
use serde_json::{Value, json};
ensure!(raw.len() <= 4 * 1024 * 1024, "CONTROL_FIXTURE_BOUND");
let f: Value = serde_json::from_slice(raw)?;
ensure!(
f["schema_version"] == 1
&& f["read_only"] == true
&& f["database_writes"] == false
&& f["private_keys_read"] == false,
"CONTROL_FIXTURE_SOURCE"
);
let binding: Binding = serde_json::from_value(f["binding"].clone())?;
let mut db = Connection::open_in_memory()?;
let ledger = Connection::open_in_memory()?;
initialize(&db)?;
crate::ledger::initialize_ledger(&ledger)?;
db.execute_batch("DELETE FROM reconcile_meta; CREATE TABLE counter(id INTEGER PRIMARY KEY,seq INTEGER);CREATE TABLE job_attempts(job TEXT,attempt TEXT,offer BLOB,state TEXT,revision INTEGER,grant_id TEXT,PRIMARY KEY(job,attempt));CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB);CREATE TABLE events(id TEXT PRIMARY KEY,payload BLOB)")?;
ledger.execute_batch(
"CREATE TABLE reconcile_consumption(decision_id TEXT PRIMARY KEY,response BLOB)",
)?;
const RESOLUTION: &str = "CREATE TABLE windows_system_nonstart_221_resolution(grant_id TEXT PRIMARY KEY,role TEXT,receipt BLOB,receipt_sha256 TEXT,derived_outcome BLOB)";
db.execute_batch(RESOLUTION)?;
ledger.execute_batch(RESOLUTION)?;
fn cell(v: &Value) -> Result<SqlValue> {
let a = v.as_array().context("CONTROL_FIXTURE_CELL")?;
Ok(match a.first().and_then(Value::as_str) {
Some("null") if a.len() == 1 => SqlValue::Null,
Some("integer") if a.len() == 2 => {
SqlValue::Integer(a[1].as_i64().context("CONTROL_FIXTURE_INTEGER")?)
}
Some("text") if a.len() == 2 => {
SqlValue::Text(a[1].as_str().context("CONTROL_FIXTURE_TEXT")?.into())
}
Some("blob") if a.len() == 2 => SqlValue::Blob(serde_json::from_value(a[1].clone())?),
_ => anyhow::bail!("CONTROL_FIXTURE_CELL_TYPE"),
})
}
fn import(db: &Connection, rows: &Value, sql: &str) -> Result<()> {
let rows = rows.as_array().context("CONTROL_FIXTURE_ROWS")?;
ensure!(rows.len() <= 8, "CONTROL_FIXTURE_ROW_BOUND");
for row in rows {
let values = row
.as_array()
.context("CONTROL_FIXTURE_ROW")?
.iter()
.map(cell)
.collect::<Result<Vec<_>>>()?;
db.execute(sql, rusqlite::params_from_iter(values))?;
}
Ok(())
}
for (key, sql) in [
(
"meta",
"INSERT INTO reconcile_meta VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",
),
("counter", "INSERT INTO counter VALUES(?1,?2)"),
(
"requests",
"INSERT INTO reconcile_requests VALUES(?1,?2,?3,?4,?5)",
),
("jobs", "INSERT INTO job_attempts VALUES(?1,?2,?3,?4,?5,?6)"),
(
"outcomes",
"INSERT INTO reconcile_outcomes VALUES(?1,?2,?3,?4)",
),
(
"results",
"INSERT INTO reconcile_results VALUES(?1,?2,?3,?4,?5)",
),
("events", "INSERT INTO job_events VALUES(?1,?2)"),
(
"resolution",
"INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)",
),
] {
import(&db, &f["agent"][key], sql)?;
}
ensure!(
f["agent"]["legacy_count"] == 0,
"CONTROL_FIXTURE_PENDING_LEGACY"
);
for (key, sql) in [
("consumed", "INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,?6)"),
(
"evidence",
"INSERT INTO execution_evidence VALUES(?1,?2,?3)",
),
("acks", "INSERT INTO reconcile_consumption VALUES(?1,?2)"),
(
"resolution",
"INSERT INTO windows_system_nonstart_221_resolution VALUES(?1,?2,?3,?4,?5)",
),
] {
import(&ledger, &f["ledger"][key], sql)?;
}
let captures = f["helper"]["capture"]
.as_array()
.context("CONTROL_FIXTURE_CAPTURE")?;
ensure!(captures.len() == 1, "CONTROL_FIXTURE_CAPTURE_COUNT");
let SqlValue::Blob(raw_snapshot) = cell(&captures[0][1])? else {
anyhow::bail!("CONTROL_FIXTURE_CAPTURE_BYTES")
};
let snapshot: crate::helper_reconcile::Snapshot = serde_json::from_slice(&raw_snapshot)?;
let m = f["helper"]["meta"]
.as_array()
.context("CONTROL_FIXTURE_HELPER_META")?;
ensure!(m.len() == 1, "CONTROL_FIXTURE_HELPER_META_COUNT");
ensure!(
cell(&m[0][0])? == SqlValue::Text(snapshot.journal_id.to_string())
&& cell(&m[0][1])? == SqlValue::Integer(i64::try_from(snapshot.checkpoint)?)
&& cell(&m[0][3])?
== SqlValue::Text(
snapshot
.previous_decision_id
.context("CONTROL_FIXTURE_HELPER_HEAD")?
.to_string()
)
&& cell(&m[0][6])? == SqlValue::Null,
"CONTROL_FIXTURE_ACTUAL_HELPER_CAPTURE"
);
crate::helper_reconcile::precommit_clock_contract(binding.clone(), snapshot.clone())?;
let helper = HelperSnapshot {
journal_id: snapshot.journal_id,
checkpoint: snapshot.checkpoint,
previous_decision_id: snapshot.previous_decision_id,
attempts: snapshot
.attempts
.iter()
.map(|a| SyncAttempt {
job_id: a.job_id.to_string(),
attempt_id: a.attempt_id.to_string(),
job_payload_hash: a.job_payload_hash.clone(),
grant_id: a.grant_id.to_string(),
consumption: a.consumption.clone(),
state: a.state.clone(),
revision: a.revision,
original_boot_id: a.original_boot_id.to_string(),
outcome_sha256: a.outcome_sha256.clone(),
outcome_bytes: a.outcome_bytes.clone(),
update_handoff_bytes: a.update_handoff_bytes.clone(),
..Default::default()
})
.collect(),
};
let meta = metadata(&db)?;
let helper_generation = match cell(&m[0][2])? {
SqlValue::Integer(v) => u64::try_from(v)?,
_ => anyhow::bail!("CONTROL_FIXTURE_HELPER_GENERATION"),
};
let channel = Channel {
connection_generation: meta
.generation
.max(helper_generation)
.checked_add(1)
.context("CONTROL_FIXTURE_GENERATION_OVERFLOW")?,
boot_id: snapshot.boot_id,
};
let now: DateTime<Utc> = serde_json::from_value(f["captured_at_utc"].clone())?;
let client = ClientSync::with_verifier(binding.clone(), crate::crypto::DalekVerifier)
.with_helper_identity(HelperIdentity::System);
let before = (db.total_changes(), ledger.total_changes());
client.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)?;
ensure!(
(db.total_changes(), ledger.total_changes()) == before,
"CONTROL_ADMISSION_WROTE_HISTORY"
);
ledger.execute(
"INSERT INTO consumed VALUES(?1,?2,?3,?4,?5,NULL)",
params![
Uuid::from_u128(2).to_string(),
binding.agent_id.to_string(),
binding.enrollment_generation,
Uuid::from_u128(3).to_string(),
Uuid::from_u128(4).to_string()
],
)?;
ensure!(
client
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
.is_err(),
"CONTROL_HIDDEN_OPEN_GRANT_NOT_REJECTED"
);
ledger.execute(
"DELETE FROM consumed WHERE grant_id=?1",
[Uuid::from_u128(2).to_string()],
)?;
let a = meta.decision.context("CONTROL_FIXTURE_A")?;
let (request, _) = client.stored_control(&db, &ledger, a, channel.boot_id)?;
let (original_raw, original_env, original_response): (Vec<u8>, Vec<u8>, Vec<u8>) = db
.query_row(
"SELECT request,envelope,response FROM reconcile_requests WHERE id=?1",
[request.request_id.clone()],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)?;
let mut changed = request.clone();
changed.helper_previous_decision_id = Uuid::from_u128(1).to_string();
let mut env = Envelope::decode(original_env.as_slice())?;
env.payload = Some(Payload::SyncRequest(changed.clone()));
db.execute(
"UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3",
params![
changed.encode_to_vec(),
env.encode_to_vec(),
request.request_id
],
)?;
ensure!(
client
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
.is_err(),
"CONTROL_MUTATED_A_NOT_REJECTED"
);
db.execute(
"UPDATE reconcile_requests SET request=?1,envelope=?2 WHERE id=?3",
params![original_raw, original_env, request.request_id],
)?;
let mut changed = SyncResponse::decode(original_response.as_slice())?;
changed.signature[0] ^= 1;
let bytes = changed.encode_to_vec();
db.execute(
"UPDATE reconcile_requests SET response=?1 WHERE id=?2",
params![bytes, request.request_id],
)?;
ledger.execute(
"UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2",
params![bytes, a.to_string()],
)?;
ensure!(
client
.closed_control_continuation(&db, &ledger, &channel, &helper, &meta, now)
.is_err(),
"CONTROL_MUTATED_SERVER_NOT_REJECTED"
);
db.execute(
"UPDATE reconcile_requests SET response=?1 WHERE id=?2",
params![original_response, request.request_id],
)?;
ledger.execute(
"UPDATE reconcile_consumption SET response=?1 WHERE decision_id=?2",
params![original_response, a.to_string()],
)?;
struct CurrentClock(DateTime<Utc>);
impl SyncClock for CurrentClock {
fn now(&self) -> Result<DateTime<Utc>> {
Ok(self.0)
}
}
let clock = CurrentClock(now);
let mut client = ClientSync::with_verifier(binding, crate::crypto::DalekVerifier)
.with_helper_identity(HelperIdentity::System);
ensure!(
client
.prepare(&mut db, &ledger, channel.clone(), Some(&helper), &clock)
.is_err(),
"CONTROL_IDLE_ATTESTATION_REQUIRED"
);
let fresh = client.prepare_with_closed_control_idle(
&mut db,
&ledger,
channel.clone(),
Some(&helper),
&clock,
true,
)?;
ensure!(
fresh.request.previous_decision_id == a.to_string()
&& fresh.request.helper_previous_decision_id
== helper.previous_decision_id.unwrap().to_string()
&& fresh.envelope.connection_generation == channel.connection_generation
&& fresh.envelope.observed_at == now.timestamp_millis()
&& !client.can_start(&channel, false, &clock)
&& !client.can_start(&channel, true, &clock)
&& client.gate_observation().is_none(),
"CONTROL_FRESH_REQUEST_MUST_NOT_AUTHORIZE"
);
Ok(
json!({"actual_closed_partial_edge_accepted":true,"mutated_agent_request_rejected":true,"mutated_server_signature_and_ack_rejected":true,"admission_read_only":true,"hidden_open_grant_rejected":true,"explicit_idle_attestation_required":true,"typed_precommit_clock_keeps_capture_without_writes":true,"fresh_request_keeps_both_heads":true,"old_authority_reused":false,"new_ready_or_unlock":false,"product_databases_opened":false,"effects":false}),
)
}
+51 -9
View File
@@ -922,8 +922,54 @@ impl SystemHelper {
);
Ok(())
}
fn wait_apply_clock(&self,issued:chrono::DateTime<chrono::Utc>,until:chrono::DateTime<chrono::Utc>)->Result<()> {
crate::job_readiness::wait_for_helper_apply_clock(&self.clock,issued,until)
// One authenticated control call, same original bytes/capture/deadline.
// Only a typed clock failure proven before BEGIN can be retried.
fn apply_control_with_clock(
&mut self,
response: &rmm_agent_core::helper_reconcile::SignedResponse,
request: &[u8],
issued: chrono::DateTime<chrono::Utc>,
until: chrono::DateTime<chrono::Utc>,
) -> Result<rmm_agent_core::helper_reconcile::Applied> {
let began = Instant::now();
loop {
ensure!(
began.elapsed() < Duration::from_secs(3),
"SYSTEM_APPLY_CLOCK_WAIT_EXPIRED"
);
match crate::job_readiness::response_window(
issued,
until,
began.elapsed(),
self.clock.reading().ok().map(|r| (r.own_wall, r.upper)),
) {
crate::job_readiness::ResponseWindow::WaitForClock => {
std::thread::sleep(Duration::from_millis(20));
continue;
}
crate::job_readiness::ResponseWindow::Expired => {
anyhow::bail!("SYSTEM_APPLY_ORIGINAL_DEADLINE_EXPIRED")
}
crate::job_readiness::ResponseWindow::WithinWindow => {}
}
self.sync()?;
match self.sync.as_mut().unwrap().apply(
&mut self.db,
&self.guard,
response,
request,
&self.clock,
&self.boot,
) {
Ok(value) => return Ok(value),
Err(error)
if rmm_agent_core::helper_reconcile::retryable_precommit_clock(&error) =>
{
std::thread::sleep(Duration::from_millis(20))
}
Err(error) => return Err(error),
}
}
}
/// Passive, bounded static denial metadata only. Never feeds a gate.
fn record_apply_error(&mut self,error:&anyhow::Error)->Result<()> {
@@ -1025,15 +1071,11 @@ impl SystemHelper {
// Same authenticated original call, BEFORE any application.
// The independent time pipe continues on its own thread.
// No signature/deadline/lease is relaxed and no Execute retry.
self.wait_apply_clock(decision.issued_at,decision.valid_until)?;
self.sync()?;
let value = self.sync.as_mut().unwrap().apply(
&mut self.db,
&self.guard,
let value = self.apply_control_with_clock(
response,
request_bytes,
&self.clock,
&self.boot,
decision.issued_at,
decision.valid_until,
)?;
let no_start_anchor = if value.ready && self.settings.enable_effects {
let guard = DenialGuard {
+25 -1
View File
@@ -1448,13 +1448,37 @@ impl Agent {
helper: Option<&reconcile::HelperSnapshot>,
) -> Result<()> {
let context = self.channel(generation)?;
let closed_control_idle = {
#[cfg(feature = "system-helper")]
{
self.running.is_none()
&& !self.prepared
&& self.deferred_sync.is_none()
&& self.deferred_permit.is_none()
&& self.helper_control.is_none()
&& self.helper_probe.is_none()
&& helper.is_some()
&& (if Path::new(rmm_agent_windows::helper_runtime::intent::DB).exists() {
rmm_agent_windows::helper_runtime::intent::Store::open(false)
.and_then(|store| store.pending())
.is_ok_and(|pending| pending.is_empty())
} else {
helper.is_some_and(|snapshot| snapshot.attempts.is_empty())
})
}
#[cfg(not(feature = "system-helper"))]
{
false
}
};
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
let prepared = sync.prepare(
let prepared = sync.prepare_with_closed_control_idle(
self.journal.connection_mut(),
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
context,
helper,
&self.clock,
closed_control_idle,
);
#[cfg(feature = "system-helper")]
if helper.is_some()