Avoid repeated full journal scans and fast ticks in Windows idle
This commit is contained in:
@@ -205,6 +205,7 @@ pub struct HelperSync<V = DefaultVerifier> {
|
||||
born: DateTime<Utc>,
|
||||
capture: Option<Capture>,
|
||||
lease: Option<Lease>,
|
||||
integrity: Option<crate::ledger::IntegrityMonitor>,
|
||||
}
|
||||
impl HelperSync<DefaultVerifier> {
|
||||
/// No persisted ready bit can open a newly started helper.
|
||||
@@ -222,6 +223,7 @@ impl<V: Ed25519Verifier> HelperSync<V> {
|
||||
born: clock.now()?,
|
||||
capture: None,
|
||||
lease: None,
|
||||
integrity: None,
|
||||
})
|
||||
}
|
||||
/// Select exactly one protected helper execution identity, closing leases.
|
||||
@@ -230,6 +232,12 @@ impl<V: Ed25519Verifier> HelperSync<V> {
|
||||
self.identity = identity;
|
||||
self
|
||||
}
|
||||
/// Use with one protected SQLite connection retained for this instance.
|
||||
/// The native owner must run a full check before any new host effect.
|
||||
pub fn with_periodic_idle_integrity_checks(mut self) -> Self {
|
||||
self.integrity = Some(crate::ledger::IntegrityMonitor::default());
|
||||
self
|
||||
}
|
||||
pub fn close(&mut self) {
|
||||
self.lease = None;
|
||||
self.capture = None;
|
||||
@@ -523,7 +531,12 @@ impl<V: Ed25519Verifier> HelperSync<V> {
|
||||
);
|
||||
let boot_id = boot.current_boot()?;
|
||||
ensure!(!boot_id.is_nil(), "BOOT_ID_REQUIRED");
|
||||
initialize(db, &self.binding)?;
|
||||
if let Some(integrity) = &mut self.integrity {
|
||||
integrity.check(db, false, "ROOT_LEDGER_INTEGRITY")?;
|
||||
initialize_schema(db, &self.binding)?;
|
||||
} else {
|
||||
initialize(db, &self.binding)?;
|
||||
}
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let meta = metadata(&tx)?;
|
||||
ensure!(meta.fault.is_none(), "HELPER_MANUAL_RECOVERY_REQUIRED");
|
||||
@@ -1031,8 +1044,11 @@ struct Meta {
|
||||
agent_checkpoint: u64,
|
||||
}
|
||||
fn initialize(db: &Connection, binding: &Binding) -> Result<()> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "ROOT_LEDGER_INTEGRITY");
|
||||
crate::ledger::check_integrity(db, "ROOT_LEDGER_INTEGRITY")?;
|
||||
initialize_schema(db, binding)
|
||||
}
|
||||
|
||||
fn initialize_schema(db: &Connection, binding: &Binding) -> Result<()> {
|
||||
db.execute_batch("PRAGMA synchronous=FULL;
|
||||
CREATE TABLE IF NOT EXISTS helper_sync_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,binding_sha256 TEXT NOT NULL,last_decision TEXT,fault TEXT,agent_journal_id TEXT,agent_checkpoint INTEGER NOT NULL DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS helper_sync_rows(grant_id TEXT PRIMARY KEY,revision INTEGER NOT NULL,fingerprint TEXT NOT NULL);
|
||||
|
||||
@@ -7,7 +7,8 @@ use rmm_domain::{
|
||||
jobs::hash,
|
||||
reconciliation::{ReconciledOutcome, canonical_outcome_bytes},
|
||||
};
|
||||
use rusqlite::{OptionalExtension, TransactionBehavior, params};
|
||||
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
||||
use std::time::{Duration, Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Finite 1 GiB bound for durable agent/helper history. Unexpected page sizes
|
||||
@@ -29,11 +30,48 @@ pub fn initialize_ledger(db: &rusqlite::Connection) -> Result<()> {
|
||||
CREATE TABLE IF NOT EXISTS consumed(grant_id TEXT PRIMARY KEY, agent TEXT NOT NULL, generation INTEGER NOT NULL,
|
||||
job TEXT NOT NULL, attempt TEXT NOT NULL, result BLOB, UNIQUE(agent,generation,job,attempt));
|
||||
CREATE TABLE IF NOT EXISTS execution_evidence(grant_id TEXT PRIMARY KEY, start BLOB NOT NULL, outcome BLOB);")?;
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "JOURNAL_INTEGRITY");
|
||||
check_integrity(db, "JOURNAL_INTEGRITY")
|
||||
}
|
||||
|
||||
pub fn check_integrity(db: &Connection, error_code: &'static str) -> Result<()> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |row| row.get(0))?;
|
||||
ensure!(check == "ok", "{error_code}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full integrity scans for an owner holding one protected, long-lived SQLite
|
||||
/// connection. External commits and schema changes require another scan even
|
||||
/// when the interval has not elapsed. No cached result authorizes an effect.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct IntegrityMonitor {
|
||||
checked: Option<(u64, u64, Instant)>,
|
||||
}
|
||||
|
||||
impl IntegrityMonitor {
|
||||
pub(crate) fn check(
|
||||
&mut self,
|
||||
db: &Connection,
|
||||
required: bool,
|
||||
error_code: &'static str,
|
||||
) -> Result<()> {
|
||||
let data: u64 = db.query_row("PRAGMA data_version", [], |row| row.get(0))?;
|
||||
let schema: u64 = db.query_row("PRAGMA schema_version", [], |row| row.get(0))?;
|
||||
if !required
|
||||
&& self.checked.is_some_and(|(previous_data, previous_schema, checked)| {
|
||||
data == previous_data
|
||||
&& schema == previous_schema
|
||||
&& checked.elapsed() < Duration::from_secs(300)
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
self.checked = None;
|
||||
check_integrity(db, error_code)?;
|
||||
self.checked = Some((data, schema, Instant::now()));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Before a human WebUI closure replaces the unresolved local outcome, keep the
|
||||
/// exact original outcome bytes once. A differing second copy fails closed.
|
||||
pub(crate) fn retain_closure_original(
|
||||
@@ -379,3 +417,34 @@ pub fn start_witness_bytes(
|
||||
"reauthentication_required":true,"postconditions_passed":false});
|
||||
Ok(serde_json::to_vec(&evidence)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod integrity_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn idle_integrity_reuses_the_check_but_reaudits_external_changes_and_expiry() {
|
||||
let path = std::env::temp_dir().join(format!("rmm-integrity-{}.sqlite", Uuid::new_v4()));
|
||||
{
|
||||
let db = Connection::open(&path).unwrap();
|
||||
db.execute_batch("CREATE TABLE sample(id INTEGER PRIMARY KEY);").unwrap();
|
||||
let mut monitor = IntegrityMonitor::default();
|
||||
monitor.check(&db, false, "JOURNAL_INTEGRITY").unwrap();
|
||||
let first = monitor.checked.unwrap();
|
||||
db.execute("INSERT INTO sample VALUES (1)", []).unwrap();
|
||||
monitor.check(&db, false, "JOURNAL_INTEGRITY").unwrap();
|
||||
assert_eq!(monitor.checked, Some(first));
|
||||
|
||||
let peer = Connection::open(&path).unwrap();
|
||||
peer.execute("INSERT INTO sample VALUES (2)", []).unwrap();
|
||||
monitor.check(&db, false, "JOURNAL_INTEGRITY").unwrap();
|
||||
let changed = monitor.checked.unwrap();
|
||||
assert_ne!(changed.0, first.0);
|
||||
|
||||
monitor.checked = Some((changed.0, changed.1, changed.2 - Duration::from_secs(300)));
|
||||
monitor.check(&db, false, "JOURNAL_INTEGRITY").unwrap();
|
||||
assert!(monitor.checked.unwrap().2 >= changed.2);
|
||||
}
|
||||
std::fs::remove_file(path).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ pub struct ClientSync<V = DefaultVerifier> {
|
||||
verifier: V,
|
||||
helper_identity: HelperIdentity,
|
||||
pending: Option<PreparedSync>,
|
||||
integrity: Option<crate::ledger::IntegrityMonitor>,
|
||||
channel: Option<Channel>,
|
||||
gate: Option<Gate>,
|
||||
}
|
||||
@@ -117,6 +118,7 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
verifier,
|
||||
helper_identity: HelperIdentity::Root,
|
||||
pending: None,
|
||||
integrity: None,
|
||||
channel: None,
|
||||
gate: None,
|
||||
}
|
||||
@@ -127,6 +129,11 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
self.helper_identity = identity;
|
||||
self
|
||||
}
|
||||
/// Use with one protected SQLite connection retained for this instance.
|
||||
pub fn with_periodic_idle_integrity_checks(mut self) -> Self {
|
||||
self.integrity = Some(crate::ledger::IntegrityMonitor::default());
|
||||
self
|
||||
}
|
||||
/// Invoke on every authenticated/fenced ServerHello or transport loss.
|
||||
pub fn close(&mut self) {
|
||||
self.gate = None;
|
||||
@@ -606,7 +613,17 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
channel.connection_generation > 0 && !channel.boot_id.is_nil(),
|
||||
"UNAUTHENTICATED_CHANNEL"
|
||||
);
|
||||
initialize(db)?;
|
||||
if let Some(integrity) = &mut self.integrity {
|
||||
let work_pending: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
integrity.check(db, work_pending, "JOURNAL_INTEGRITY")?;
|
||||
initialize_schema(db)?;
|
||||
} else {
|
||||
initialize(db)?;
|
||||
}
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let meta = metadata(&tx)?;
|
||||
ensure!(meta.fault.is_none(), "SYNC_MANUAL_RECOVERY_REQUIRED");
|
||||
@@ -1147,8 +1164,11 @@ impl<V: Ed25519Verifier> ClientSync<V> {
|
||||
}
|
||||
|
||||
pub fn initialize(db: &Connection) -> Result<()> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "JOURNAL_INTEGRITY");
|
||||
crate::ledger::check_integrity(db, "JOURNAL_INTEGRITY")?;
|
||||
initialize_schema(db)
|
||||
}
|
||||
|
||||
fn initialize_schema(db: &Connection) -> Result<()> {
|
||||
db.execute_batch("PRAGMA synchronous=FULL;
|
||||
CREATE TABLE IF NOT EXISTS reconcile_meta(id INTEGER PRIMARY KEY CHECK(id=1),journal_id TEXT NOT NULL,checkpoint INTEGER NOT NULL DEFAULT 0,high_sequence INTEGER NOT NULL DEFAULT 0,generation INTEGER NOT NULL DEFAULT 0,last_decision TEXT,pending_request TEXT,helper_id TEXT,helper_checkpoint INTEGER NOT NULL DEFAULT 0,fault TEXT);
|
||||
CREATE TABLE IF NOT EXISTS reconcile_requests(id TEXT PRIMARY KEY,request BLOB NOT NULL,envelope BLOB NOT NULL,generation INTEGER NOT NULL,response BLOB);
|
||||
|
||||
@@ -903,9 +903,6 @@ impl SystemHelper {
|
||||
RuntimeClock::enrolled_windows_update_helper(settings.binding.clone(), settings.helper.clone())
|
||||
} else { RuntimeClock::new(rmm_time_attestation::Role::SystemHelper) }.for_generation(0);
|
||||
let boot = WindowsBoot::capture()?;
|
||||
let sync = HelperSync::with_verifier(settings.binding.clone(), DalekVerifier, &clock)
|
||||
.ok()
|
||||
.map(|s| s.with_helper_identity(HelperIdentity::System));
|
||||
Ok(Self {
|
||||
settings,
|
||||
pin,
|
||||
@@ -914,7 +911,8 @@ impl SystemHelper {
|
||||
db,
|
||||
clock,
|
||||
boot,
|
||||
sync,
|
||||
// The initial connection creates the reconciler using its own qualified time.
|
||||
sync: None,
|
||||
last_diagnostic: None,
|
||||
last_apply_diagnostic: None,
|
||||
unsettled_msi: None,
|
||||
@@ -1008,14 +1006,17 @@ impl SystemHelper {
|
||||
}
|
||||
fn sync(&mut self) -> Result<&mut HelperSync<DalekVerifier>> {
|
||||
if self.sync.is_none() {
|
||||
self.sync = Some(
|
||||
HelperSync::with_verifier(
|
||||
self.settings.binding.clone(),
|
||||
DalekVerifier,
|
||||
&self.clock,
|
||||
)?
|
||||
.with_helper_identity(HelperIdentity::System),
|
||||
);
|
||||
let sync = HelperSync::with_verifier(
|
||||
self.settings.binding.clone(),
|
||||
DalekVerifier,
|
||||
&self.clock,
|
||||
)?
|
||||
.with_helper_identity(HelperIdentity::System);
|
||||
self.sync = Some(if self.settings.managed_scripts() {
|
||||
sync.with_periodic_idle_integrity_checks()
|
||||
} else {
|
||||
sync
|
||||
});
|
||||
}
|
||||
self.sync.as_mut().context("SYSTEM_SYNC_CLOSED")
|
||||
}
|
||||
@@ -1290,6 +1291,9 @@ impl SystemHelper {
|
||||
}
|
||||
ensure!(self.settings.enable_effects, "SYSTEM_EFFECTS_DISABLED");
|
||||
verified_window(&self.settings.binding, request, &self.clock)?;
|
||||
if self.settings.managed_scripts() {
|
||||
ledger::check_integrity(&self.db, "ROOT_LEDGER_INTEGRITY")?;
|
||||
}
|
||||
self.sync()?;
|
||||
ensure!(
|
||||
self.sync.as_ref().unwrap().can_start(
|
||||
|
||||
@@ -1291,8 +1291,8 @@ impl Agent {
|
||||
&& !self.helper_apply_unconfirmed
|
||||
&& !self.prepared
|
||||
&& self.running.is_none()
|
||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||
&& self.helper_probe_at.elapsed() >= Duration::from_secs(30)
|
||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||
{
|
||||
let agent = self.identity.agent_id.clone();
|
||||
let asset = self.identity.asset_id.clone();
|
||||
@@ -2663,8 +2663,9 @@ impl Agent {
|
||||
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
||||
let mut previous_cpu = None;
|
||||
let mut heartbeat_after = Duration::from_secs(30);
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let active_tick = Duration::from_millis(20);
|
||||
let idle_tick = Duration::from_millis(250);
|
||||
let mut next_tick = tokio::time::Instant::now();
|
||||
let mut readiness_at = Instant::now() - Duration::from_secs(5);
|
||||
#[cfg(feature = "system-helper")]
|
||||
let mut update_control_at = Instant::now() - Duration::from_secs(2);
|
||||
@@ -2675,9 +2676,19 @@ impl Agent {
|
||||
running.cancel.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
let control_pending = self.running.is_some()
|
||||
|| self.prepared
|
||||
|| self.deferred_sync.is_some()
|
||||
|| self.deferred_permit.is_some()
|
||||
|| self.helper_control_pending();
|
||||
#[cfg(feature = "system-helper")]
|
||||
let control_pending = control_pending || self.helper_probe.is_some();
|
||||
let cadence = if control_pending { active_tick } else { idle_tick };
|
||||
next_tick = next_tick.min(tokio::time::Instant::now() + cadence);
|
||||
tokio::select! {
|
||||
Some(completion)=self.completed_rx.recv()=>{self.completed(generation,completion)?;needs_sync=true;},
|
||||
_=tick.tick()=>{
|
||||
_=tokio::time::sleep_until(next_tick)=>{
|
||||
next_tick = tokio::time::Instant::now() + cadence;
|
||||
if !stop.load(Ordering::SeqCst) && self.poll_sync_apply(generation,true).is_err(){self.close_gate();needs_sync=true;}
|
||||
#[cfg(feature = "system-helper")]
|
||||
if !stop.load(Ordering::SeqCst) && self.poll_helper(&channel,generation).await.is_err() {self.close_gate();needs_sync=self.sync.is_some();}
|
||||
@@ -2709,7 +2720,9 @@ impl Agent {
|
||||
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
||||
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if Instant::now() >= permit_after && !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
||||
if Instant::now() >= permit_after && !needs_sync && !self.prepared && self.config.enable_jobs
|
||||
&& self.journal.connection().prepare_cached("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state='accepted' AND cancel_requested=0)")?.query_row([], |row| row.get::<_, bool>(0))?
|
||||
&& self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
||||
// A transiently stale clock skips this tick; it must not end the session.
|
||||
if let Ok(now)=self.clock.now() && let Some(offer)=jobs::next_at(self.journal.connection(),now)? {
|
||||
let job=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.job(&offer.payload_bytes)?;
|
||||
@@ -3078,7 +3091,19 @@ async fn run_service_binding(path: &Path, stop: &'static AtomicBool, release_gua
|
||||
ensure!(rmm_agent_windows::echo_probe::protected_scope()? == *scope, "WINDOWS_ECHO_CONFIGURATION_CHANGED");
|
||||
}
|
||||
}
|
||||
let sync=if config.enable_jobs && (!config.windows_echo_probe_only || config.echo_scope.is_some()) {binding.clone().map(|binding|reconcile::ClientSync::new(binding).with_helper_identity(reconcile::HelperIdentity::System))} else {None};
|
||||
let sync = if config.enable_jobs && (!config.windows_echo_probe_only || config.echo_scope.is_some()) {
|
||||
binding.clone().map(|binding| {
|
||||
let sync = reconcile::ClientSync::new(binding)
|
||||
.with_helper_identity(reconcile::HelperIdentity::System);
|
||||
if config.localservice_time_policy.is_some() {
|
||||
sync.with_periodic_idle_integrity_checks()
|
||||
} else {
|
||||
sync
|
||||
}
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (completed_tx,completed_rx)=tokio::sync::mpsc::channel(1);
|
||||
let mut agent = Agent {
|
||||
config,
|
||||
|
||||
Reference in New Issue
Block a user