3304 lines
141 KiB
Rust
3304 lines
141 KiB
Rust
use anyhow::{Context, Result, ensure};
|
|
use chrono::Utc;
|
|
use prost::Message;
|
|
use rcgen::{CertificateParams, DnType, KeyPair};
|
|
use rmm_agent_core::{
|
|
auth::{Binding, Outcome, Request},
|
|
jobs, ledger,
|
|
reconcile::{self, SyncClock},
|
|
};
|
|
use rmm_agent_windows::certificate_rotation::{ProtectedKeyFiles, RotationStore};
|
|
use rmm_agent_windows::{CAPABILITIES, job_binding::readonly_binding, journal::Journal, native};
|
|
use rmm_domain::jobs::{Action, ExecutionPermitPayload};
|
|
use rmm_protocol::*;
|
|
use rusqlite::{Connection, OptionalExtension};
|
|
use serde::Deserialize;
|
|
use sha2::{Digest, Sha256};
|
|
use std::{
|
|
fs,
|
|
path::{Path, PathBuf},
|
|
sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
|
|
},
|
|
time::{Duration, Instant},
|
|
};
|
|
use uuid::Uuid;
|
|
|
|
#[derive(Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
struct Config {
|
|
server_name: String,
|
|
connect_address: std::net::SocketAddr,
|
|
ca_file: PathBuf,
|
|
state_directory: PathBuf,
|
|
enrollment_token_file: PathBuf,
|
|
transport: String,
|
|
#[serde(default)]
|
|
job_binding_file: Option<PathBuf>,
|
|
#[serde(default)]
|
|
localservice_time_policy: Option<PathBuf>,
|
|
#[serde(default)]
|
|
windows_update_helper: bool,
|
|
// Explicit protected opt-in only; this flag never qualifies a clock or pins.
|
|
#[serde(default)]
|
|
enable_jobs: bool,
|
|
#[serde(default)]
|
|
windows_echo_probe_only: bool,
|
|
#[serde(skip)]
|
|
echo_scope: Option<rmm_domain::windows_echo_probe::ProbeScope>,
|
|
#[serde(
|
|
default,
|
|
deserialize_with = "rmm_domain::jobs::deserialize_windows_execution_profile"
|
|
)]
|
|
windows_execution_profile: Option<rmm_domain::jobs::WindowsExecutionProfile>,
|
|
}
|
|
/// Local TLS maintenance evidence only. It never restores readiness; after
|
|
/// renewal both roles must pass their ordinary fresh signed synchronization.
|
|
#[derive(serde::Serialize, Deserialize, PartialEq, Eq)]
|
|
#[serde(deny_unknown_fields)]
|
|
struct RotationIdleAdmission {
|
|
schema_version: u32,
|
|
agent_id: String,
|
|
binding_sha256: String,
|
|
boot_id: String,
|
|
journal_file_id: [u32; 3],
|
|
ledger_file_id: [u32; 3],
|
|
history_sha256: String,
|
|
helper_pin_sha256: Option<String>,
|
|
helper_anchor_sha256: Option<String>,
|
|
intent_file_id: Option<[u32; 3]>,
|
|
}
|
|
fn rotation_file_id(path: &Path) -> Result<[u32; 3]> {
|
|
use std::os::windows::io::AsRawHandle;
|
|
use windows_sys::Win32::Storage::FileSystem::{
|
|
BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle,
|
|
};
|
|
native::private_path(path, false)?;
|
|
let file = fs::File::open(path)?;
|
|
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
|
|
// SAFETY: live file handle and correctly sized writable native structure.
|
|
ensure!(
|
|
unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0
|
|
&& info.nNumberOfLinks == 1,
|
|
"ROTATION_FILE_IDENTITY"
|
|
);
|
|
Ok([
|
|
info.dwVolumeSerialNumber,
|
|
info.nFileIndexHigh,
|
|
info.nFileIndexLow,
|
|
])
|
|
}
|
|
impl Config {
|
|
fn profile_enabled(&self, profile: rmm_domain::jobs::WindowsExecutionProfile) -> bool {
|
|
self.enable_jobs
|
|
&& (self.job_binding_file.is_some() || self.localservice_time_policy.is_some())
|
|
&& cfg!(feature = "system-helper")
|
|
&& self.windows_execution_profile == Some(profile)
|
|
}
|
|
fn read(path: &Path) -> Result<Self> {
|
|
let raw = protected_read(path, 32768)?;
|
|
let config: Self = serde_json::from_slice(&raw)?;
|
|
if config.windows_update_helper {
|
|
ensure!(config.localservice_time_policy.is_some() && cfg!(feature = "system-helper"), "ENROLLED_WINDOWS_UPDATE_HELPER_REQUIRED");
|
|
}
|
|
if let Some(policy) = &config.localservice_time_policy {
|
|
let parent = path.parent().context("CONFIG_PARENT")?;
|
|
ensure!(policy == &parent.join("job-time.json")
|
|
&& config.job_binding_file.is_none()
|
|
&& config.profile_enabled(rmm_domain::jobs::WindowsExecutionProfile::PowerShell51RemoteSigned)
|
|
&& !config.windows_echo_probe_only, "INVALID_ENROLLED_LOCALSERVICE_CONFIGURATION");
|
|
}
|
|
if config.windows_echo_probe_only {
|
|
ensure!(
|
|
config.profile_enabled(
|
|
rmm_domain::jobs::WindowsExecutionProfile::PowerShell51RemoteSigned
|
|
) && config.server_name == "discovery.ease-technologies.de"
|
|
&& config.state_directory == Path::new(r"C:\ProgramData\RustDiscovery\state"),
|
|
"WINDOWS_ECHO_CONFIGURATION"
|
|
);
|
|
}
|
|
if let Some(profile) = config.windows_execution_profile {
|
|
ensure!(
|
|
config.profile_enabled(profile),
|
|
"WINDOWS_PROFILE_REQUIRES_JOB_CONFIGURATION"
|
|
);
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
use rmm_agent_windows::helper_ipc::PinnedFile;
|
|
use std::io::Read;
|
|
// Verify the exact bytes used for the opt-in under the existing
|
|
// administrator-owned, service-read-only file and directory ACLs.
|
|
let readers = [native::service_sid()?];
|
|
let mut parent = PinnedFile::open(
|
|
path.parent().context("CONFIG_PARENT")?,
|
|
true,
|
|
&readers,
|
|
&[],
|
|
false,
|
|
)?;
|
|
let mut pin = PinnedFile::open(path, false, &readers, &[], false)?;
|
|
let mut verified = Vec::new();
|
|
(&mut pin.file).take(32769).read_to_end(&mut verified)?;
|
|
ensure!(verified == raw, "WINDOWS_PROFILE_CONFIGURATION_CHANGED");
|
|
pin.check()?;
|
|
parent.check()?;
|
|
}
|
|
}
|
|
Ok(config)
|
|
}
|
|
}
|
|
fn create_private(path: &Path, bytes: &[u8]) -> Result<()> {
|
|
use std::io::Write;
|
|
let mut file = fs::OpenOptions::new()
|
|
.write(true)
|
|
.create_new(true)
|
|
.open(path)?;
|
|
file.write_all(bytes)?;
|
|
file.sync_all()?;
|
|
native::private_path(path, false)
|
|
}
|
|
fn protected_read(path: &Path, max: usize) -> Result<Vec<u8>> {
|
|
native::private_path(path, false)?;
|
|
ensure!(
|
|
fs::metadata(path)?.len() <= max as u64,
|
|
"PRIVATE_FILE_LIMIT"
|
|
);
|
|
Ok(fs::read(path)?)
|
|
}
|
|
async fn enrollment(config: &Config, ca: &[u8]) -> Result<EnrollmentResponse> {
|
|
let state = &config.state_directory;
|
|
let identity = state.join("identity.json");
|
|
if identity.try_exists()? {
|
|
return Ok(serde_json::from_slice(&protected_read(&identity, 32768)?)?);
|
|
}
|
|
let key_path = state.join("device.key");
|
|
let key = if key_path.try_exists()? {
|
|
KeyPair::from_pem(std::str::from_utf8(&protected_read(&key_path, 16384)?)?)?
|
|
} else {
|
|
let key = KeyPair::generate()?;
|
|
create_private(&key_path, key.serialize_pem().as_bytes())?;
|
|
key
|
|
};
|
|
let mut parameters = CertificateParams::default();
|
|
parameters
|
|
.distinguished_name
|
|
.push(DnType::CommonName, "new-windows-device");
|
|
let csr = parameters.serialize_request(&key)?.pem()?;
|
|
let token = String::from_utf8(protected_read(&config.enrollment_token_file, 128)?)?
|
|
.trim()
|
|
.to_owned();
|
|
ensure!(
|
|
token.len() == 64 && token.bytes().all(|b| b.is_ascii_hexdigit()),
|
|
"ENROLLMENT_TOKEN_FORMAT"
|
|
);
|
|
let client = reqwest::Client::builder()
|
|
.https_only(true)
|
|
.tls_built_in_root_certs(false)
|
|
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
|
.resolve(&config.server_name, config.connect_address)
|
|
.no_proxy()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.timeout(Duration::from_secs(20))
|
|
.build()?;
|
|
let response = client
|
|
.post(format!(
|
|
"https://{}:{}/agent/v1/enroll",
|
|
config.server_name,
|
|
config.connect_address.port()
|
|
))
|
|
.json(&EnrollmentRequest {
|
|
token,
|
|
csr_pem: csr,
|
|
})
|
|
.send()
|
|
.await?;
|
|
let bytes = crate::https_channel::bounded_body(response, 32768, "application/json").await?;
|
|
let identity: EnrollmentResponse = serde_json::from_slice(&bytes)?;
|
|
Uuid::parse_str(&identity.agent_id)?;
|
|
Uuid::parse_str(&identity.asset_id)?;
|
|
// The server consumes the token once; a lost response is an explicit recovery
|
|
// gate. No automatic re-enrollment or replacement identity is attempted.
|
|
create_private(
|
|
&state.join("device.pem"),
|
|
identity.certificate_pem.as_bytes(),
|
|
)?;
|
|
create_private(
|
|
&state.join("identity.json"),
|
|
&serde_json::to_vec(&identity)?,
|
|
)?;
|
|
fs::remove_file(&config.enrollment_token_file)?;
|
|
Ok(identity)
|
|
}
|
|
#[derive(Clone)]
|
|
struct JobClock {
|
|
#[cfg(feature = "system-helper")]
|
|
native: rmm_agent_windows::signed_clock::BoundClock,
|
|
enabled: bool,
|
|
}
|
|
impl JobClock {
|
|
fn new(enabled: bool) -> Self {
|
|
Self {
|
|
#[cfg(feature = "system-helper")]
|
|
native: rmm_agent_windows::signed_clock::RuntimeClock::new(
|
|
rmm_time_attestation::Role::Agent,
|
|
)
|
|
.for_generation(0),
|
|
enabled,
|
|
}
|
|
}
|
|
fn for_generation(&self, generation: u64) -> Self {
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
Self {
|
|
native: self.native.for_generation(generation),
|
|
enabled: self.enabled,
|
|
}
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
{
|
|
let _ = generation;
|
|
self.clone()
|
|
}
|
|
}
|
|
}
|
|
impl SyncClock for JobClock {
|
|
fn now(&self) -> Result<chrono::DateTime<Utc>> {
|
|
Ok(self.reading()?.own_wall)
|
|
}
|
|
fn reading(&self) -> Result<reconcile::ClockReading> {
|
|
ensure!(self.enabled, "WINDOWS_JOBS_NOT_ENABLED");
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
self.native.reading()
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
{
|
|
anyhow::bail!("WINDOWS_SIGNED_TIME_NOT_CONFIGURED")
|
|
}
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
struct TimeRefresh {
|
|
generation: u64,
|
|
stop: Arc<AtomicBool>,
|
|
clock: rmm_agent_windows::signed_clock::RuntimeClock,
|
|
helper: Option<rmm_agent_windows::helper_runtime::Client>,
|
|
tasks: Vec<tokio::task::JoinHandle<()>>,
|
|
samples: Vec<Arc<std::sync::OnceLock<VerifiedTimeSample>>>,
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
impl Drop for TimeRefresh {
|
|
fn drop(&mut self) {
|
|
self.stop.store(true, Ordering::SeqCst);
|
|
let _ = self.clock.close_generation(self.generation);
|
|
for task in &self.tasks {
|
|
task.abort();
|
|
}
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
struct VerifiedTimeSample {
|
|
request: rmm_time_attestation::Request,
|
|
response_hash: String,
|
|
observed_at: chrono::DateTime<Utc>,
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
impl VerifiedTimeSample {
|
|
fn new(request: rmm_time_attestation::Request, response_hash: String) -> Self {
|
|
Self {
|
|
request,
|
|
response_hash,
|
|
observed_at: Utc::now(),
|
|
}
|
|
}
|
|
fn persist(&self, state: &Path) -> Result<()> {
|
|
rmm_agent_windows::helper_runtime::record_verified_time(
|
|
state,
|
|
&self.request,
|
|
&self.response_hash,
|
|
self.observed_at,
|
|
)
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
const TIME_TRANSPORT: &str = "TIME_TRANSPORT_UNAVAILABLE";
|
|
#[cfg(feature = "system-helper")]
|
|
/// Transport loss or a busy SYSTEM helper never invalidates held evidence.
|
|
fn transient_refresh(error: &anyhow::Error) -> bool {
|
|
error
|
|
.chain()
|
|
.any(|cause| matches!(cause.to_string().as_str(), TIME_TRANSPORT | "HELPER_TIME_BUSY"))
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
/// Operator diagnostics only: last refresh failure per role, written on change.
|
|
#[derive(Default)]
|
|
struct RefreshReport(Option<String>);
|
|
#[cfg(feature = "system-helper")]
|
|
impl RefreshReport {
|
|
fn record(&mut self, state: &Path, role: &str, result: &Result<()>) {
|
|
let current = result.as_ref().err().map(|e| {
|
|
let mut text = format!("{e:#}");
|
|
text.truncate(512);
|
|
text
|
|
});
|
|
if current == self.0 {
|
|
return;
|
|
}
|
|
let value = serde_json::json!({"schema_version":1,"role":role,"observed_at_utc":Utc::now(),"error":current});
|
|
let _ = std::fs::write(state.join(format!("time-refresh-{role}.json")), value.to_string());
|
|
self.0 = current;
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
async fn time_response(
|
|
client: &reqwest::Client,
|
|
address: std::net::SocketAddr,
|
|
request: &rmm_time_attestation::Request,
|
|
) -> Result<Vec<u8>> {
|
|
request.validate()?;
|
|
ensure!(
|
|
crate::https_channel::server_name()? == "discovery.ease-technologies.de",
|
|
"TIME_FIXED_SERVER"
|
|
);
|
|
tokio::time::timeout(Duration::from_millis(650), async {
|
|
let response = client
|
|
.post(format!(
|
|
"https://discovery.ease-technologies.de:{}/agent/v1/time-attestation",
|
|
address.port()
|
|
))
|
|
.timeout(Duration::from_millis(650))
|
|
.json(request)
|
|
.send()
|
|
.await?;
|
|
crate::https_channel::bounded_body(response, 16384, "application/json").await
|
|
})
|
|
.await
|
|
.context("TIME_HTTP_DEADLINE")?
|
|
}
|
|
#[derive(Default)]
|
|
struct EffectFence {
|
|
generation: AtomicU64,
|
|
helper_generation: AtomicU64,
|
|
deadline: AtomicI64,
|
|
}
|
|
impl EffectFence {
|
|
fn close(&self) {
|
|
self.generation.store(0, Ordering::SeqCst);
|
|
self.helper_generation.store(0, Ordering::SeqCst);
|
|
}
|
|
fn allows(&self, generation: u64, clock: &impl SyncClock) -> bool {
|
|
self.allows_identity(generation, false, clock)
|
|
}
|
|
fn allows_identity(&self, generation: u64, helper: bool, clock: &impl SyncClock) -> bool {
|
|
(!helper || self.helper_generation.load(Ordering::SeqCst) == generation)
|
|
&& self.generation.load(Ordering::SeqCst) == generation
|
|
&& generation > 0
|
|
&& clock.reading().is_ok_and(|reading| {
|
|
reading.own_wall.timestamp_millis() < self.deadline.load(Ordering::SeqCst)
|
|
&& reading.upper.timestamp_millis() < self.deadline.load(Ordering::SeqCst)
|
|
})
|
|
}
|
|
}
|
|
enum ExecutorResult {
|
|
Outcome(Outcome),
|
|
#[cfg(feature = "system-helper")]
|
|
UpdatePending(Option<rmm_agent_windows::update_admission::Accepted>),
|
|
}
|
|
impl From<Outcome> for ExecutorResult {
|
|
fn from(value: Outcome) -> Self {
|
|
Self::Outcome(value)
|
|
}
|
|
}
|
|
fn interrupted_executor(grant: &ExecutionPermitPayload) -> ExecutorResult {
|
|
#[cfg(not(feature="system-helper"))]
|
|
let _=grant;
|
|
#[cfg(feature = "system-helper")]
|
|
if grant.action == "update_agent" {
|
|
return ExecutorResult::UpdatePending(None);
|
|
}
|
|
Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED").into()
|
|
}
|
|
type Completion = (JobOffer, ExecutionPermitPayload, ExecutorResult);
|
|
#[cfg(feature = "system-helper")]
|
|
fn needs_bound_outcome(
|
|
db: &Connection,
|
|
bound: &rmm_agent_windows::helper_runtime::BoundOutcome,
|
|
) -> Result<bool> {
|
|
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
|
|
// Archiving is durable retention only: no ACK, event deletion or unlock.
|
|
reconcile::archive_events(db)?;
|
|
let (state, revision, grant): (String, u64, String) = db.query_row(
|
|
"SELECT state,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
rusqlite::params![bound.job_id.to_string(), bound.attempt_id.to_string()],
|
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
|
)?;
|
|
let bytes:Option<Vec<u8>>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",rusqlite::params![bound.job_id.to_string(),bound.attempt_id.to_string()],|r|r.get(0)).optional()?;
|
|
if state == bound.outcome.state
|
|
&& grant == bound.grant_id.to_string()
|
|
&& let Some(bytes) = bytes
|
|
{
|
|
let envelope = Envelope::decode(bytes.as_slice())?;
|
|
if let Some(Payload::JobResult(r)) = envelope.payload
|
|
&& r.revision == revision
|
|
&& r.grant_id == grant
|
|
{
|
|
let saved = ReconciledOutcome {
|
|
state: r.state,
|
|
stdout: r.stdout,
|
|
stderr: r.stderr,
|
|
exit_code: r.exit_code,
|
|
output_truncated: r.output_truncated,
|
|
reboot_required: r.reboot_required,
|
|
evidence: serde_json::from_str(&r.evidence_json)?,
|
|
error_code: r.error_code,
|
|
};
|
|
let actual: ReconciledOutcome =
|
|
serde_json::from_value(serde_json::to_value(&bound.outcome)?)?;
|
|
if canonical_outcome_bytes(&saved)? == canonical_outcome_bytes(&actual)? {
|
|
return Ok(false);
|
|
}
|
|
}
|
|
}
|
|
Ok(true)
|
|
}
|
|
struct DeferredSync {
|
|
generation: u64,
|
|
response: SyncResponse,
|
|
issued_at: chrono::DateTime<Utc>,
|
|
valid_until: chrono::DateTime<Utc>,
|
|
received_at: Instant,
|
|
applied: Option<reconcile::Applied>,
|
|
}
|
|
fn clock_temporarily_unavailable(error: &anyhow::Error) -> bool {
|
|
error
|
|
.chain()
|
|
.any(|cause| cause.to_string().starts_with("TIME_"))
|
|
}
|
|
struct Running {
|
|
offer: JobOffer,
|
|
grant: ExecutionPermitPayload,
|
|
cancel: Arc<AtomicBool>,
|
|
worker: tokio::task::JoinHandle<()>,
|
|
}
|
|
fn verified_window(
|
|
binding: &Binding,
|
|
request: &Request,
|
|
clock: &impl SyncClock,
|
|
) -> Result<(rmm_domain::jobs::JobPayload, ExecutionPermitPayload)> {
|
|
let now = clock.now()?;
|
|
// The entire qualified <=2s clock interval must lie inside the grant window.
|
|
let result = binding.verify(request, now - chrono::Duration::seconds(2))?;
|
|
binding.verify(request, now + chrono::Duration::seconds(2))?;
|
|
Ok(result)
|
|
}
|
|
fn import_legacy_inventory(journal: &mut Journal) -> Result<()> {
|
|
let original = {
|
|
let mut q = journal
|
|
.connection()
|
|
.prepare("SELECT id,payload FROM events ORDER BY rowid")?;
|
|
q.query_map([], |r| {
|
|
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
|
|
})?
|
|
.collect::<rusqlite::Result<Vec<_>>>()?
|
|
};
|
|
let tx = journal.connection_mut().transaction()?;
|
|
for (id, bytes) in original {
|
|
let envelope = Envelope::decode(bytes.as_slice())?;
|
|
ensure!(
|
|
envelope.message_id == id
|
|
&& matches!(envelope.payload, Some(Payload::InventorySnapshot(_))),
|
|
"LEGACY_INVENTORY_BINDING"
|
|
);
|
|
let old: Option<Vec<u8>> = tx
|
|
.query_row("SELECT payload FROM job_events WHERE id=?1", [&id], |r| {
|
|
r.get(0)
|
|
})
|
|
.optional()?;
|
|
ensure!(
|
|
old.as_ref().is_none_or(|old| old == &bytes),
|
|
"LEGACY_INVENTORY_CONFLICT"
|
|
);
|
|
// Copy original bytes/ID, never decode/re-encode or discard pending telemetry.
|
|
tx.execute(
|
|
"INSERT OR IGNORE INTO job_events(id,payload) VALUES(?1,?2)",
|
|
rusqlite::params![id, bytes],
|
|
)?;
|
|
}
|
|
// Core ACK/Sync deletion and the old queue advance in the SAME transaction.
|
|
tx.execute_batch("CREATE TRIGGER IF NOT EXISTS windows_legacy_inventory_ack AFTER DELETE ON job_events BEGIN DELETE FROM events WHERE id=OLD.id AND payload=OLD.payload; END;")?;
|
|
tx.commit()?;
|
|
Ok(())
|
|
}
|
|
fn open_ledger(directory: &Path, binding: &Binding) -> Result<(Connection, fs::File)> {
|
|
use std::os::windows::fs::OpenOptionsExt;
|
|
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
|
let path = directory.join("consumption.sqlite");
|
|
let file = fs::OpenOptions::new()
|
|
.create(true)
|
|
.truncate(false)
|
|
.read(true)
|
|
.write(true)
|
|
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
|
|
.open(&path)?;
|
|
native::private_path(&path, false)?;
|
|
let db = Connection::open(&path)?;
|
|
ledger::initialize_ledger(&db)?;
|
|
db.execute_batch("CREATE TABLE IF NOT EXISTS windows_binding(identity TEXT NOT NULL);")?;
|
|
let identity = format!(
|
|
"{}:{}:{}:{}:{}",
|
|
binding.tenant_id,
|
|
binding.asset_id,
|
|
binding.agent_id,
|
|
binding.enrollment_generation,
|
|
binding.assignment_version
|
|
);
|
|
let old: Option<String> = db
|
|
.query_row("SELECT identity FROM windows_binding", [], |r| r.get(0))
|
|
.optional()?;
|
|
if let Some(old) = old {
|
|
ensure!(old == identity, "LEDGER_IDENTITY_MISMATCH");
|
|
} else {
|
|
db.execute("INSERT INTO windows_binding VALUES(?1)", [identity])?;
|
|
}
|
|
Ok((db, file))
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
type MirrorCompletion = (
|
|
Connection,
|
|
Result<(
|
|
rmm_agent_windows::helper_runtime::ControlSnapshot,
|
|
Vec<rmm_agent_windows::helper_runtime::BoundOutcome>,
|
|
)>,
|
|
);
|
|
#[cfg(feature = "system-helper")]
|
|
enum HelperControl {
|
|
Snapshot(
|
|
u64,
|
|
tokio::task::JoinHandle<Result<rmm_agent_windows::helper_runtime::ControlSnapshot>>,
|
|
),
|
|
Apply(
|
|
u64,
|
|
tokio::task::JoinHandle<Result<rmm_agent_windows::helper_runtime::ControlSnapshot>>,
|
|
),
|
|
Mirror(u64, tokio::task::JoinHandle<MirrorCompletion>),
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
impl HelperControl {
|
|
fn finished(&self) -> bool {
|
|
match self {
|
|
Self::Snapshot(_, t) | Self::Apply(_, t) => t.is_finished(),
|
|
Self::Mirror(_, t) => t.is_finished(),
|
|
}
|
|
}
|
|
fn abort(self) -> Option<Self> {
|
|
match self {
|
|
Self::Snapshot(_, t) => {
|
|
t.abort();
|
|
None
|
|
}
|
|
// Apply may already have committed in the helper. Retain its
|
|
// bounded worker until settled; never lose the only reply owner.
|
|
apply @ Self::Apply(..) => Some(apply),
|
|
mirror @ Self::Mirror(..) => Some(mirror),
|
|
}
|
|
}
|
|
}
|
|
struct Agent {
|
|
config: Config,
|
|
reported_version: String,
|
|
release_job_guard: Option<fs::File>,
|
|
identity: EnrollmentResponse,
|
|
ca: Vec<u8>,
|
|
journal: Journal,
|
|
certificates: RotationStore,
|
|
key_files: ProtectedKeyFiles,
|
|
boot: String,
|
|
instance: String,
|
|
started: Instant,
|
|
binding: Option<Binding>,
|
|
ledger: Option<Connection>,
|
|
_ledger_file: Option<fs::File>,
|
|
clock: JobClock,
|
|
#[cfg(feature = "system-helper")]
|
|
time_refresh: Option<TimeRefresh>,
|
|
sync: Option<reconcile::ClientSync>,
|
|
prepared: bool,
|
|
deferred_sync: Option<DeferredSync>,
|
|
#[cfg(feature = "system-helper")]
|
|
update_pending: bool,
|
|
#[cfg(feature="system-helper")]
|
|
update_observer:crate::update_observation_runtime::Observer,
|
|
#[cfg(feature = "system-helper")]
|
|
update_health: Option<tokio::task::JoinHandle<Result<()>>>,
|
|
#[cfg(feature = "system-helper")]
|
|
update_inventory_head: Option<(u64, String)>,
|
|
#[cfg(feature = "system-helper")]
|
|
prepared_request: Option<Vec<u8>>,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_apply_unconfirmed: bool,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_lane: Arc<tokio::sync::Semaphore>,
|
|
#[cfg(feature = "system-helper")]
|
|
helper: Option<rmm_agent_windows::helper_runtime::Client>,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_probe:
|
|
Option<tokio::task::JoinHandle<Result<Option<rmm_agent_windows::helper_runtime::Client>>>>,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_probe_at: Instant,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_control: Option<HelperControl>,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_anchor: Option<rmm_agent_core::helper_reconcile::NoStartAnchor>,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_checked: bool,
|
|
#[cfg(feature = "system-helper")]
|
|
helper_refused: bool,
|
|
fence: Arc<EffectFence>,
|
|
completed_tx: tokio::sync::mpsc::Sender<Completion>,
|
|
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
|
running: Option<Running>,
|
|
deferred_permit: Option<(u64, ExecutionPermit)>,
|
|
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
|
|
#[cfg(feature = "system-helper")]
|
|
update_prefetch: Option<(u64, String, String, tokio::task::JoinHandle<Result<()>>)>,
|
|
#[cfg(feature = "system-helper")]
|
|
update_cached: Option<(u64, String, String)>,
|
|
}
|
|
impl Agent {
|
|
fn rotation_without_effects(&self) -> Result<()> {
|
|
ensure!(
|
|
self.running.is_none() && self.deferred_permit.is_none() && !self.prepared,
|
|
"ROTATION_BUSY"
|
|
);
|
|
let idle: bool = self.journal.connection().query_row("SELECT NOT EXISTS(SELECT 1 FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired'))", [], |r| r.get(0))?;
|
|
ensure!(idle, "ROTATION_EFFECT_UNRESOLVED");
|
|
if let Some(ledger) = &self.ledger {
|
|
let resolved: bool = ledger.query_row("SELECT NOT EXISTS(SELECT 1 FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL OR e.start IS NULL OR e.outcome IS NULL)", [], |r| r.get(0))?;
|
|
ensure!(resolved, "ROTATION_CONSUMPTION_UNRESOLVED");
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
ensure!(
|
|
self.helper_control.is_none()
|
|
&& self.helper_probe.is_none()
|
|
&& !self.helper_apply_unconfirmed,
|
|
"ROTATION_HELPER_CONTROL_PENDING"
|
|
);
|
|
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).try_exists()? {
|
|
let intents = rmm_agent_windows::helper_runtime::intent::Store::open(false)?;
|
|
ensure!(
|
|
intents.pending()?.is_empty(),
|
|
"ROTATION_SYSTEM_DELEGATION_PENDING"
|
|
);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
fn rotation_admission(
|
|
&self,
|
|
generation: Option<u64>,
|
|
retained: Option<&[u8]>,
|
|
) -> Result<Vec<u8>> {
|
|
self.rotation_without_effects()?;
|
|
let old: Option<RotationIdleAdmission> =
|
|
retained.map(serde_json::from_slice).transpose()?;
|
|
if let Some(generation) = generation {
|
|
#[cfg(feature = "system-helper")]
|
|
let helper_present = self.helper.is_some();
|
|
#[cfg(not(feature = "system-helper"))]
|
|
let helper_present = false;
|
|
ensure!(
|
|
self.ready_identity(generation, helper_present),
|
|
"ROTATION_SIGNED_IDLE_REQUIRED"
|
|
);
|
|
} else {
|
|
ensure!(
|
|
self.fence.generation.load(Ordering::SeqCst) == 0 && old.is_some(),
|
|
"ROTATION_PENDING_IDLE_ADMISSION_REQUIRED"
|
|
);
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
let (helper_pin_sha256, helper_anchor_sha256, intent_file_id) = {
|
|
use rmm_agent_windows::helper_runtime::{Client, intent};
|
|
let helper = Client::discover(
|
|
&self.identity.agent_id,
|
|
&self.identity.asset_id,
|
|
self.binding.as_ref(),
|
|
self.helper_lane.clone(),
|
|
)?;
|
|
let pin = helper.as_ref().map(|h| {
|
|
format!(
|
|
"{:x}",
|
|
Sha256::digest(
|
|
format!("{}:{}", h.pin_hash(), h.helper_binary_hash()).as_bytes()
|
|
)
|
|
)
|
|
});
|
|
let anchor = if helper.is_some() {
|
|
if generation.is_some() {
|
|
Some(format!(
|
|
"{:x}",
|
|
Sha256::digest(serde_json::to_vec(
|
|
self.helper_anchor
|
|
.as_ref()
|
|
.context("ROTATION_HELPER_ANCHOR_REQUIRED")?
|
|
)?)
|
|
))
|
|
} else {
|
|
let saved = old
|
|
.as_ref()
|
|
.and_then(|a| a.helper_anchor_sha256.clone())
|
|
.context("ROTATION_RETAINED_HELPER_ANCHOR_REQUIRED")?;
|
|
ensure!(
|
|
saved.len() == 64 && saved.bytes().all(|b| b.is_ascii_hexdigit()),
|
|
"ROTATION_RETAINED_HELPER_ANCHOR_INVALID"
|
|
);
|
|
Some(saved)
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
let file = Path::new(intent::DB)
|
|
.try_exists()?
|
|
.then(|| rotation_file_id(Path::new(intent::DB)))
|
|
.transpose()?;
|
|
(pin, anchor, file)
|
|
};
|
|
#[cfg(not(feature = "system-helper"))]
|
|
let (helper_pin_sha256, helper_anchor_sha256, intent_file_id) = (None, None, None);
|
|
let admission = RotationIdleAdmission {
|
|
schema_version: 1,
|
|
agent_id: self.identity.agent_id.clone(),
|
|
binding_sha256: format!(
|
|
"{:x}",
|
|
Sha256::digest(serde_json::to_vec(
|
|
self.binding.as_ref().context("ROTATION_BINDING_REQUIRED")?
|
|
)?)
|
|
),
|
|
boot_id: self.boot.clone(),
|
|
journal_file_id: rotation_file_id(
|
|
&self.config.state_directory.join("telemetry.sqlite"),
|
|
)?,
|
|
ledger_file_id: rotation_file_id(
|
|
&self.config.state_directory.join("consumption.sqlite"),
|
|
)?,
|
|
history_sha256: rmm_agent_windows::certificate_rotation::idle_history_sha256(
|
|
self.journal.connection(),
|
|
self.ledger.as_ref().context("ROTATION_LEDGER_REQUIRED")?,
|
|
)?,
|
|
helper_pin_sha256,
|
|
helper_anchor_sha256,
|
|
intent_file_id,
|
|
};
|
|
ensure!(
|
|
old.as_ref().is_none_or(|saved| saved == &admission),
|
|
"ROTATION_IDLE_STATE_CHANGED"
|
|
);
|
|
Ok(serde_json::to_vec(&admission)?)
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
fn start_time_refresh(
|
|
&mut self,
|
|
generation: u64,
|
|
identity: &crate::certificate_rotation::DeviceIdentity,
|
|
) -> Result<()> {
|
|
ensure!(self.time_refresh.is_none(), "TIME_REFRESH_ALREADY_RUNNING");
|
|
use rmm_agent_windows::signed_clock::{CONFIG_PATH, SamplingMode};
|
|
// Presence opts into sampling only. Each role opens and verifies the
|
|
// protected profile and signed response itself before accepting time.
|
|
let Some(mode) = SamplingMode::configured(self.config.enable_jobs, Path::new(CONFIG_PATH))?
|
|
else {
|
|
return Ok(());
|
|
};
|
|
let client = crate::https_channel::mtls_client(
|
|
self.config.connect_address,
|
|
&self.ca,
|
|
&identity.certificate_pem,
|
|
&identity.private_key_pem,
|
|
)?;
|
|
let helper = if self.config.localservice_time_policy.is_some() && !self.config.windows_update_helper {
|
|
None
|
|
} else { rmm_agent_windows::helper_runtime::Client::discover(
|
|
&self.identity.agent_id,
|
|
&self.identity.asset_id,
|
|
self.binding.as_ref(),
|
|
self.helper_lane.clone(),
|
|
)? };
|
|
let clock = self.clock.native.source();
|
|
let stop = Arc::new(AtomicBool::new(false));
|
|
let mut tasks = Vec::new();
|
|
let own_clock = clock.clone();
|
|
let own_stop = stop.clone();
|
|
let own_http = client.clone();
|
|
let address = self.config.connect_address;
|
|
let own_state = self.config.state_directory.clone();
|
|
let helper_state = self.config.state_directory.clone();
|
|
// Only the connection owner publishes observations. Dropping it also
|
|
// discards unfinished samples, so an old worker cannot overwrite a newer report.
|
|
let sample_slot =
|
|
|| (mode == SamplingMode::Passive).then(|| Arc::new(std::sync::OnceLock::new()));
|
|
let own_receipt = sample_slot();
|
|
let helper_receipt = helper.as_ref().and_then(|_| sample_slot());
|
|
let samples = own_receipt
|
|
.iter()
|
|
.chain(helper_receipt.iter())
|
|
.cloned()
|
|
.collect();
|
|
let installed_localservice = self.config.localservice_time_policy.is_some();
|
|
tasks.push(tokio::spawn(async move {
|
|
let mut own_report = RefreshReport::default();
|
|
while !own_stop.load(Ordering::SeqCst) {
|
|
let result = async {
|
|
if !own_clock.configured_for(generation) {
|
|
own_clock.open_connection(generation).await?;
|
|
}
|
|
let request = own_clock.begin(generation)?;
|
|
let bytes = time_response(&own_http, address, &request)
|
|
.await
|
|
.map_err(|e| e.context(TIME_TRANSPORT))?;
|
|
ensure!(!own_stop.load(Ordering::SeqCst), "TIME_CONNECTION_CLOSED");
|
|
own_clock.complete(generation, &bytes)?;
|
|
if let Some(slot) = own_receipt.as_ref().filter(|s| s.get().is_none()) {
|
|
let hash = format!("{:x}", Sha256::digest(&bytes));
|
|
let _ = slot.set(VerifiedTimeSample::new(request, hash));
|
|
}
|
|
Ok::<(), anyhow::Error>(())
|
|
}
|
|
.await;
|
|
// A transport failure keeps the still-valid held evidence and the
|
|
// next begin supersedes the unanswered challenge. Any other failure
|
|
// closes so the next round reopens a fresh verifier.
|
|
if let Err(error) = &result
|
|
&& !transient_refresh(error)
|
|
{
|
|
let _ = own_clock.close_generation(generation);
|
|
}
|
|
own_report.record(&own_state, "agent", &result);
|
|
let interval = if installed_localservice && result.is_ok() {
|
|
Duration::from_secs(5)
|
|
} else {
|
|
mode.interval(result.is_ok())
|
|
};
|
|
tokio::time::sleep(interval).await;
|
|
}
|
|
let _ = own_clock.close_generation(generation);
|
|
}));
|
|
if let Some(time_helper) = helper.clone() {
|
|
let helper_stop = stop.clone();
|
|
tasks.push(tokio::spawn(async move {
|
|
let mut helper_report = RefreshReport::default();
|
|
while !helper_stop.load(Ordering::SeqCst) {
|
|
let result = async {
|
|
let request = time_helper.time_begin(generation).await?;
|
|
let bytes = time_response(&client, address, &request)
|
|
.await
|
|
.map_err(|e| e.context(TIME_TRANSPORT))?;
|
|
ensure!(
|
|
!helper_stop.load(Ordering::SeqCst),
|
|
"TIME_CONNECTION_CLOSED"
|
|
);
|
|
let pending = helper_receipt.as_ref().filter(|s| s.get().is_none());
|
|
let report =
|
|
pending.map(|slot| (slot, format!("{:x}", Sha256::digest(&bytes))));
|
|
time_helper.time_complete(generation, bytes).await?;
|
|
if let Some((slot, hash)) = report {
|
|
let _ = slot.set(VerifiedTimeSample::new(request, hash));
|
|
}
|
|
Ok::<(), anyhow::Error>(())
|
|
}
|
|
.await;
|
|
// HELPER_TIME_BUSY during a SYSTEM run or a transport failure keeps
|
|
// the helper's held evidence; other failures close and reopen.
|
|
if let Err(error) = &result
|
|
&& !transient_refresh(error)
|
|
{
|
|
let _ = time_helper.time_close(generation).await;
|
|
}
|
|
helper_report.record(&helper_state, "system_helper", &result);
|
|
tokio::time::sleep(mode.interval(result.is_ok())).await;
|
|
}
|
|
let _ = time_helper.time_close(generation).await;
|
|
}));
|
|
}
|
|
self.time_refresh = Some(TimeRefresh {
|
|
generation,
|
|
stop,
|
|
clock,
|
|
helper,
|
|
tasks,
|
|
samples,
|
|
});
|
|
Ok(())
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
async fn stop_time_refresh(&mut self) {
|
|
let Some(refresh) = self.time_refresh.take() else {
|
|
return;
|
|
};
|
|
refresh.stop.store(true, Ordering::SeqCst);
|
|
let _ = refresh.clock.close_generation(refresh.generation);
|
|
for task in &refresh.tasks {
|
|
task.abort();
|
|
}
|
|
if let Some(helper) = &refresh.helper {
|
|
let until = Instant::now() + Duration::from_millis(1200);
|
|
loop {
|
|
if helper.time_close(refresh.generation).await.is_ok() {
|
|
break;
|
|
}
|
|
if Instant::now() >= until {
|
|
// No successful remote fence is asserted after a lost or
|
|
// busy call; local effects are already closed independently.
|
|
eprintln!("HELPER_TIME_CLOSE_UNCONFIRMED");
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
}
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
async fn drain_helper_apply(&mut self, until: Instant) {
|
|
// Keep the same bounded original worker and independent time lane
|
|
// alive until its acknowledgement settles. This never opens a gate,
|
|
// confirms a new generation, retries Apply or authorizes an effect.
|
|
let Some(control) = self.helper_control.take() else {
|
|
return;
|
|
};
|
|
match control {
|
|
HelperControl::Apply(_, task) => {
|
|
if let Ok(Ok(Ok(_snapshot))) =
|
|
tokio::time::timeout(until.saturating_duration_since(Instant::now()), task)
|
|
.await
|
|
{
|
|
self.helper_apply_unconfirmed = false;
|
|
}
|
|
}
|
|
mirror @ HelperControl::Mirror(..) => {
|
|
self.helper_control = Some(mirror);
|
|
}
|
|
HelperControl::Snapshot(_, task) => {
|
|
task.abort();
|
|
}
|
|
}
|
|
}
|
|
fn close_gate(&mut self) {
|
|
self.fence.close();
|
|
#[cfg(feature = "system-helper")]
|
|
if let Some(running) = &self.running
|
|
&& running.grant.execution_identity == "system"
|
|
{
|
|
running.cancel.store(true, Ordering::SeqCst);
|
|
let denied = (|| -> Result<()> {
|
|
let mut store = rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
|
let intent = store.by_grant(running.grant.grant_id)?;
|
|
ensure!(
|
|
intent.permit()? == running.grant,
|
|
"SYSTEM_CANCEL_GRANT_BINDING"
|
|
);
|
|
store.deny(&intent.request, Uuid::parse_str(&self.boot)?)
|
|
})();
|
|
if denied.is_err() {
|
|
self.helper_refused = true;
|
|
eprintln!("SYSTEM_CANCEL_PERSISTENCE_FAILED");
|
|
}
|
|
}
|
|
if let Some(sync) = &mut self.sync {
|
|
sync.close();
|
|
}
|
|
self.prepared = false;
|
|
self.deferred_sync = None;
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
self.prepared_request = None;
|
|
self.helper_anchor = None;
|
|
if let Some(control) = self.helper_control.take() {
|
|
self.helper_control = control.abort();
|
|
}
|
|
}
|
|
self.deferred_permit = None;
|
|
}
|
|
fn helper_control_pending(&self) -> bool {
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
self.helper_control.is_some()
|
|
|| self.helper_apply_unconfirmed
|
|
|| self.helper_probe.is_some()
|
|
|| !self.helper_checked
|
|
|| self.helper_refused
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
{
|
|
false
|
|
}
|
|
}
|
|
fn helper_control_busy_for_sync(&self) -> bool {
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
self.helper_control.is_some()
|
|
|| self.helper_probe.is_some()
|
|
|| !self.helper_checked
|
|
|| self.helper_refused
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
{
|
|
false
|
|
}
|
|
}
|
|
fn readiness_observation(
|
|
&self,
|
|
generation: u64,
|
|
needs_sync: bool,
|
|
) -> Result<rmm_agent_windows::job_readiness::Observation> {
|
|
use rmm_agent_windows::job_readiness::{Counts, Observation, clock_code};
|
|
let clock = self.clock.reading();
|
|
let counts=self.journal.connection().query_row("SELECT COALESCE(sum(state='accepted'),0),COALESCE(sum(state='requesting'),0),COALESCE(sum(state='running'),0),COALESCE(sum(state='verifying'),0),COALESCE(sum(state='unknown'),0),COALESCE(sum(state='awaiting_reboot'),0) FROM job_attempts",[],|r|Ok(Counts{accepted:r.get(0)?,requesting:r.get(1)?,running:r.get(2)?,verifying:r.get(3)?,unknown:r.get(4)?,awaiting_reboot:r.get(5)?,events:0,legacy_events:0}))?;
|
|
let mut counts = counts;
|
|
counts.events =
|
|
self.journal
|
|
.connection()
|
|
.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
|
counts.legacy_events = self.journal.count()?;
|
|
let gate = self.sync.as_ref().and_then(|s| s.gate_observation());
|
|
#[cfg(feature = "system-helper")]
|
|
let (helper_control, unconfirmed) = (
|
|
match &self.helper_control {
|
|
Some(HelperControl::Snapshot(..)) => "snapshot",
|
|
Some(HelperControl::Mirror(..)) => "mirror",
|
|
Some(HelperControl::Apply(..)) => "apply",
|
|
None if self.helper_probe.is_some() => "probe",
|
|
None if self.helper_refused => "refused",
|
|
None if !self.helper_checked => "not_checked",
|
|
None => "idle",
|
|
},
|
|
self.helper_apply_unconfirmed,
|
|
);
|
|
#[cfg(not(feature = "system-helper"))]
|
|
let (helper_control, unconfirmed) = ("not_configured", false);
|
|
Ok(Observation {
|
|
schema_version: 1,
|
|
observed_at: Utc::now(),
|
|
agent_id: self.identity.agent_id.clone(),
|
|
boot_id: self.boot.clone(),
|
|
connection_generation: generation,
|
|
ready: self.ready(generation),
|
|
helper_ready: self.ready_identity(generation, true),
|
|
clock_qualified: clock.is_ok(),
|
|
clock_failure: clock.as_ref().err().map_or("", clock_code),
|
|
needs_sync,
|
|
prepared: self.prepared,
|
|
deferred_sync: self.deferred_sync.as_ref().map_or("none", |pending| {
|
|
if pending.applied.is_some() {
|
|
"local_consumption"
|
|
} else {
|
|
"fresh_response_clock"
|
|
}
|
|
}),
|
|
deferred_sync_wait_ms: self.deferred_sync.as_ref().map_or(0, |p| {
|
|
p.received_at
|
|
.elapsed()
|
|
.as_millis()
|
|
.min(u128::from(u64::MAX)) as u64
|
|
}),
|
|
helper_control,
|
|
helper_apply_unconfirmed: unconfirmed,
|
|
signed_gate_present: gate.is_some(),
|
|
signed_ready: gate.is_some_and(|g| g.signed_ready),
|
|
local_applied: gate.is_some_and(|g| g.local_applied),
|
|
helper_ack_pending: gate.is_some_and(|g| g.helper_ack_pending),
|
|
signed_valid_until: self.sync.as_ref().and_then(|s| s.validity_deadline()),
|
|
counts,
|
|
})
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
async fn poll_helper(
|
|
&mut self,
|
|
channel: &crate::wire_channel::Handle,
|
|
generation: u64,
|
|
) -> Result<()> {
|
|
if self
|
|
.helper_control
|
|
.as_ref()
|
|
.is_some_and(HelperControl::finished)
|
|
{
|
|
match self
|
|
.helper_control
|
|
.take()
|
|
.context("HELPER_CONTROL_REQUIRED")?
|
|
{
|
|
HelperControl::Snapshot(expected, task) => {
|
|
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
|
let snapshot = task.await??;
|
|
let helper = self
|
|
.helper
|
|
.clone()
|
|
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
|
let observed_at = self.clock.now()?;
|
|
let mut ledger = self.ledger.take().context("LEDGER_REQUIRED")?;
|
|
self.helper_control = Some(HelperControl::Mirror(
|
|
generation,
|
|
tokio::task::spawn_blocking(move || {
|
|
let result = helper
|
|
.mirror(&snapshot.raw, &mut ledger, observed_at)
|
|
.map(|unknowns| (snapshot, unknowns));
|
|
(ledger, result)
|
|
}),
|
|
));
|
|
}
|
|
HelperControl::Mirror(expected, task) => {
|
|
// Always restore ownership, including after reconnect. A
|
|
// stale generation may discard its snapshot, never its DB.
|
|
let (ledger, result) = task.await?;
|
|
self.ledger = Some(ledger);
|
|
let (snapshot, unknowns) = result?;
|
|
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
|
self.update_pending =
|
|
rmm_agent_windows::helper_runtime::intent::Store::open(false)?
|
|
.pending()?
|
|
.iter()
|
|
.any(|intent| {
|
|
self.binding.as_ref().is_some_and(|binding| {
|
|
binding.job(&intent.request.job_bytes).is_ok_and(|job| {
|
|
matches!(job.action, Action::UpdateAgent { .. })
|
|
})
|
|
})
|
|
});
|
|
}
|
|
for bound in unknowns {
|
|
if needs_bound_outcome(self.journal.connection(), &bound)? {
|
|
let bytes: Vec<u8> = self.journal.connection().query_row(
|
|
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
rusqlite::params![
|
|
bound.job_id.to_string(),
|
|
bound.attempt_id.to_string()
|
|
],
|
|
|r| r.get(0),
|
|
)?;
|
|
self.record(
|
|
generation,
|
|
&JobOffer::decode(bytes.as_slice())?,
|
|
&bound.grant_id.to_string(),
|
|
bound.outcome,
|
|
)?;
|
|
}
|
|
}
|
|
if expected != generation {
|
|
self.close_gate();
|
|
return Ok(());
|
|
}
|
|
self.send_sync(
|
|
channel,
|
|
generation,
|
|
Some(&rmm_agent_windows::helper_runtime::map_snapshot(
|
|
&snapshot.raw,
|
|
)),
|
|
)
|
|
.await?;
|
|
}
|
|
HelperControl::Apply(expected, task) => {
|
|
let snapshot = task.await??;
|
|
if expected != generation {
|
|
// A drained old reply is retained evidence only. A new
|
|
// generation requires a fresh snapshot and signed Sync.
|
|
self.helper_apply_unconfirmed = false;
|
|
self.close_gate();
|
|
return Ok(());
|
|
}
|
|
self.helper_anchor = snapshot.no_start_anchor;
|
|
let mapped = rmm_agent_windows::helper_runtime::map_snapshot(&snapshot.raw);
|
|
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
|
if mapped
|
|
.attempts
|
|
.iter()
|
|
.any(|a| !a.update_handoff_bytes.is_empty())
|
|
{
|
|
sync.confirm_update_helper(&mapped)?;
|
|
} else {
|
|
sync.confirm_helper(&mapped)?;
|
|
}
|
|
self.helper_apply_unconfirmed = false;
|
|
self.open_fence(generation)?;
|
|
}
|
|
}
|
|
}
|
|
if self
|
|
.helper_probe
|
|
.as_ref()
|
|
.is_some_and(|task| task.is_finished())
|
|
{
|
|
let result = self
|
|
.helper_probe
|
|
.take()
|
|
.context("HELPER_PROBE_REQUIRED")?
|
|
.await?;
|
|
self.helper_probe_at = Instant::now();
|
|
self.helper_checked = true;
|
|
match result {
|
|
Ok(client) => {
|
|
self.helper_refused = false;
|
|
if self.helper.as_ref().map(|c| c.pin_hash())
|
|
!= client.as_ref().map(|c| c.pin_hash())
|
|
{
|
|
self.close_gate();
|
|
}
|
|
self.helper = client;
|
|
}
|
|
Err(error) => {
|
|
self.helper_refused = true;
|
|
self.helper = None;
|
|
self.close_gate();
|
|
return Err(error);
|
|
}
|
|
}
|
|
}
|
|
if self.helper_probe.is_none()
|
|
&& self.helper_control.is_none()
|
|
&& !self.helper_apply_unconfirmed
|
|
&& !self.prepared
|
|
&& self.running.is_none()
|
|
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
|
&& self.helper_probe_at.elapsed() >= Duration::from_secs(30)
|
|
{
|
|
let agent = self.identity.agent_id.clone();
|
|
let asset = self.identity.asset_id.clone();
|
|
let binding = self.binding.clone();
|
|
let lane = self.helper_lane.clone();
|
|
let state = self.config.state_directory.clone();
|
|
self.helper_probe = Some(tokio::spawn(async move {
|
|
rmm_agent_windows::helper_runtime::Client::refresh(
|
|
agent, asset, binding, lane, state,
|
|
)
|
|
.await
|
|
}));
|
|
}
|
|
Ok(())
|
|
}
|
|
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
|
|
Ok(reconcile::Channel {
|
|
connection_generation: generation,
|
|
boot_id: Uuid::parse_str(&self.boot)?,
|
|
})
|
|
}
|
|
fn ready(&self, generation: u64) -> bool {
|
|
self.ready_identity(generation, false)
|
|
}
|
|
fn ready_identity(&self, generation: u64, helper: bool) -> bool {
|
|
#[cfg(feature = "system-helper")]
|
|
if helper
|
|
&& !self.helper_anchor.as_ref().is_some_and(|a| {
|
|
a.connection_generation() == generation && a.boot_id().to_string() == self.boot
|
|
})
|
|
{
|
|
return false;
|
|
}
|
|
self.config.enable_jobs
|
|
&& self.deferred_sync.is_none()
|
|
&& !self.helper_control_pending()
|
|
&& self.channel(generation).is_ok_and(|channel| {
|
|
self.sync
|
|
.as_ref()
|
|
.is_some_and(|sync| sync.can_start(&channel, helper, &self.clock))
|
|
})
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
async fn poll_update_health(&mut self, generation: u64) -> Result<bool> {
|
|
if self
|
|
.update_health
|
|
.as_ref()
|
|
.is_some_and(|task| task.is_finished())
|
|
{
|
|
let task = self.update_health.take().context("UPDATE_HEALTH_TASK")?;
|
|
task.await??;
|
|
return Ok(true);
|
|
}
|
|
if !self.update_pending
|
|
|| self.update_health.is_some()
|
|
|| self.prepared
|
|
|| self.helper_control_pending()
|
|
|| self.clock.reading().is_err()
|
|
{
|
|
return Ok(false);
|
|
}
|
|
let binding = self
|
|
.binding
|
|
.as_ref()
|
|
.context("UPDATE_BINDING_REQUIRED")?
|
|
.clone();
|
|
let Some(proof) = rmm_agent_core::update_health_inventory::current_proof(
|
|
self.journal.connection(),
|
|
&binding,
|
|
)?
|
|
else {
|
|
return Ok(true);
|
|
};
|
|
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
|
|
let event = decoded
|
|
.events
|
|
.last()
|
|
.map(|raw| serde_json::from_slice::<rmm_updater_handoff::Event>(raw))
|
|
.transpose()?;
|
|
if !event.is_some_and(|event| {
|
|
matches!(
|
|
event.evidence,
|
|
rmm_updater_handoff::Evidence::ServicesStarted { .. }
|
|
| rmm_updater_handoff::Evidence::RollbackServicesStarted { .. }
|
|
)
|
|
}) {
|
|
return Ok(true);
|
|
}
|
|
let inventory_key = (generation, proof.head_sha256.clone());
|
|
if self.update_inventory_head.as_ref() != Some(&inventory_key)
|
|
&& self.queue_inventory(generation)?
|
|
{
|
|
self.update_inventory_head = Some(inventory_key);
|
|
}
|
|
let Some(context) = crate::agent_update_health::context(
|
|
self.journal.connection(),
|
|
&binding,
|
|
&proof,
|
|
self.clock.now()?,
|
|
)?
|
|
else {
|
|
return Ok(true);
|
|
};
|
|
let handoff = context.handoff().clone();
|
|
let key = context.health_public_key();
|
|
let identity = self.certificates.current(
|
|
self.journal.connection(),
|
|
&self.key_files,
|
|
Utc::now().timestamp(),
|
|
)?;
|
|
let client = crate::https_channel::mtls_client(
|
|
self.config.connect_address,
|
|
&self.ca,
|
|
&identity.certificate_pem,
|
|
&identity.private_key_pem,
|
|
)?;
|
|
let helper = self.helper.clone().context("UPDATE_HELPER_REQUIRED")?;
|
|
let address = self.config.connect_address;
|
|
self.update_health = Some(tokio::spawn(async move {
|
|
let receipt = crate::agent_update_health::request(&client, address, &context).await?;
|
|
helper.submit_update_health(&handoff, key, receipt).await
|
|
}));
|
|
Ok(false)
|
|
}
|
|
fn reserve_release_gate(&mut self) -> Result<bool> {
|
|
let Some(policy) = &self.config.localservice_time_policy else { return Ok(true) };
|
|
if self.release_job_guard.is_some() { return Ok(true) }
|
|
use std::os::windows::fs::OpenOptionsExt;
|
|
use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ;
|
|
let path = policy.with_file_name("release-job.lock");
|
|
rmm_agent_windows::job_binding::check_admin_readonly(&path)?;
|
|
match fs::OpenOptions::new().read(true).share_mode(FILE_SHARE_READ).open(path) {
|
|
Ok(file) => { self.release_job_guard = Some(file); Ok(true) }
|
|
Err(error) if error.raw_os_error() == Some(32) => Ok(false),
|
|
Err(error) => Err(error.into()),
|
|
}
|
|
}
|
|
fn release_idle_gate(&mut self) -> Result<()> {
|
|
if self.release_job_guard.is_some() && self.running.is_none() && self.deferred_permit.is_none() {
|
|
let idle: bool = self.journal.connection().query_row(
|
|
"SELECT NOT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying','awaiting_reboot'))",
|
|
[], |row| row.get(0))?;
|
|
if idle { self.release_job_guard.take(); }
|
|
}
|
|
Ok(())
|
|
}
|
|
fn supported_job(&self, job: &rmm_domain::jobs::JobPayload) -> bool {
|
|
if self.config.localservice_time_policy.is_some() && job.execution_identity != "agent_service"
|
|
&& !(self.config.windows_update_helper && job.execution_identity == "system" && job.action.windows_update_or_identity_probe()) {
|
|
return false;
|
|
}
|
|
if self.config.windows_echo_probe_only
|
|
&& !self
|
|
.config
|
|
.echo_scope
|
|
.as_ref()
|
|
.is_some_and(|s| s.matches_job(job))
|
|
{
|
|
return false;
|
|
}
|
|
if native::require_local_service().is_err() {
|
|
return false;
|
|
}
|
|
if rmm_agent_windows::actions::supported(job) {
|
|
if let rmm_domain::jobs::Action::RunScript {
|
|
windows_execution_profile: Some(profile),
|
|
..
|
|
} = &job.action
|
|
{
|
|
return self.binding.is_some() && self.config.profile_enabled(*profile);
|
|
}
|
|
return true;
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
(job.action.windows_msi().is_none()
|
|
|| (self.config.state_directory
|
|
== Path::new(rmm_agent_windows::msi_cache::AGENT_STATE)
|
|
&& rmm_agent_windows::msi_cache::stage_configured()))
|
|
&& Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file()
|
|
&& self
|
|
.helper
|
|
.as_ref()
|
|
.is_some_and(|helper| helper.supports_system(job))
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
{
|
|
false
|
|
}
|
|
}
|
|
fn scope_permits(&self, offer: &JobOffer, job: &rmm_domain::jobs::JobPayload) -> bool {
|
|
!self.config.windows_echo_probe_only
|
|
|| self.config.echo_scope.as_ref().is_some_and(|scope| {
|
|
rmm_agent_windows::echo_probe::permits_offer(scope, offer, job)
|
|
})
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
async fn prefetch_update(
|
|
&mut self,
|
|
generation: u64,
|
|
offer: &JobOffer,
|
|
job: &rmm_domain::jobs::JobPayload,
|
|
) -> Result<bool> {
|
|
let Action::UpdateAgent { update } = &job.action else {
|
|
return Ok(true);
|
|
};
|
|
ensure!(
|
|
update.validate()
|
|
&& update.manifest().map_err(anyhow::Error::msg)?.platform
|
|
== rmm_agent_windows::release_platform::WIRE,
|
|
"UPDATE_PREFETCH_JOB"
|
|
);
|
|
let key = (generation, offer.job_id.clone(), offer.attempt_id.clone());
|
|
if self.update_cached.as_ref() == Some(&key) {
|
|
return Ok(true);
|
|
}
|
|
if self.update_prefetch.as_ref().is_some_and(|(g, j, a, _)| {
|
|
(*g, j.as_str(), a.as_str())
|
|
!= (generation, offer.job_id.as_str(), offer.attempt_id.as_str())
|
|
}) && let Some((_, _, _, task)) = self.update_prefetch.take() {
|
|
task.abort();
|
|
}
|
|
if self
|
|
.update_prefetch
|
|
.as_ref()
|
|
.is_some_and(|(_, _, _, task)| task.is_finished())
|
|
{
|
|
let (_, _, _, task) = self
|
|
.update_prefetch
|
|
.take()
|
|
.context("UPDATE_PREFETCH_TASK")?;
|
|
task.await??;
|
|
self.update_cached = Some(key);
|
|
return Ok(true);
|
|
}
|
|
if self.update_prefetch.is_none() {
|
|
ensure!(
|
|
rmm_agent_windows::update_cache::configured(),
|
|
"UPDATE_PREFETCH_NOT_PROVISIONED"
|
|
);
|
|
let address = self.config.connect_address;
|
|
let identity = self.certificates.current(
|
|
self.journal.connection(),
|
|
&self.key_files,
|
|
Utc::now().timestamp(),
|
|
)?;
|
|
let client = crate::https_channel::mtls_client(
|
|
address,
|
|
&self.ca,
|
|
&identity.certificate_pem,
|
|
&identity.private_key_pem,
|
|
)?;
|
|
let base = format!(
|
|
"https://{}:{}",
|
|
crate::https_channel::server_name()?,
|
|
address.port()
|
|
);
|
|
let guard_job = job.clone();
|
|
let clock = self.clock.clone();
|
|
let fence = self.fence.clone();
|
|
let job = job.clone();
|
|
let manifest = update.manifest().map_err(anyhow::Error::msg)?;
|
|
let task = tokio::spawn(crate::update_download::prefetch(
|
|
client,
|
|
base,
|
|
job,
|
|
manifest,
|
|
move || {
|
|
ensure!(
|
|
fence.allows_identity(generation, true, &clock)
|
|
&& clock.now()? < guard_job.start_before,
|
|
"UPDATE_PREFETCH_FENCED"
|
|
);
|
|
Ok(())
|
|
},
|
|
));
|
|
self.update_prefetch = Some((
|
|
generation,
|
|
offer.job_id.clone(),
|
|
offer.attempt_id.clone(),
|
|
task,
|
|
));
|
|
}
|
|
Ok(false)
|
|
}
|
|
fn capabilities(&self) -> Vec<String> {
|
|
let mut caps: Vec<String> = CAPABILITIES.iter().map(|c| c.to_string()).collect();
|
|
#[cfg(target_env = "gnu")]
|
|
caps.push("agent.release.windows.gnu/1".into());
|
|
// queue_update_observation emits the passive update status in every profile.
|
|
caps.push("agent.update/1".into());
|
|
if self.config.windows_echo_probe_only {
|
|
caps.push("windows.echo-probe/1".into());
|
|
}
|
|
if self.config.enable_jobs
|
|
&& cfg!(feature = "system-helper")
|
|
&& self.binding.is_some()
|
|
&& native::require_local_service().is_ok()
|
|
{
|
|
caps.push("rmm.sync.v1".into());
|
|
if !self.config.windows_echo_probe_only {
|
|
caps.push("refresh_inventory".into());
|
|
}
|
|
if rmm_agent_windows::actions::powershell().is_ok() {
|
|
caps.extend(["run_script", "script.powershell/1"].map(str::to_string));
|
|
if self.config.profile_enabled(
|
|
rmm_domain::jobs::WindowsExecutionProfile::PowerShell51RemoteSigned,
|
|
) {
|
|
caps.push(rmm_agent_windows::actions::REMOTE_SIGNED_PROFILE_CAPABILITY.into());
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
if !self.config.windows_echo_probe_only
|
|
&& let Some(helper) = &self.helper
|
|
{
|
|
caps.extend(helper.qualified_capabilities());
|
|
if helper.update_observer_configured()
|
|
&& self.update_observer.cached().is_some_and(|r|r.observation.predecessor.as_ref().is_some_and(|p|p.helper_sha256==helper.helper_binary_hash())) {
|
|
for cap in self.update_observer.capabilities(){if !caps.contains(&cap){caps.push(cap);}}
|
|
}
|
|
}
|
|
}
|
|
caps
|
|
}
|
|
fn record(
|
|
&mut self,
|
|
generation: u64,
|
|
offer: &JobOffer,
|
|
grant: &str,
|
|
outcome: Outcome,
|
|
) -> Result<()> {
|
|
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
|
|
let tx = self.journal.connection_mut().transaction()?;
|
|
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
|
envelope.payload = Some(Payload::JobResult(jobs::result(
|
|
offer, revision, grant, outcome,
|
|
)));
|
|
jobs::event(&tx, &envelope)?;
|
|
tx.commit()?;
|
|
Ok(())
|
|
}
|
|
fn recover_interrupted(&mut self) -> Result<()> {
|
|
let Some(binding) = self.binding.clone() else {
|
|
return Ok(());
|
|
};
|
|
#[cfg(feature = "system-helper")]
|
|
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
|
let mut intents = rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
|
let pending = intents.pending()?;
|
|
if let Some(intent) = pending.first() {
|
|
let p = intent.permit()?;
|
|
if matches!(
|
|
binding.job(&intent.request.job_bytes)?.action,
|
|
Action::UpdateAgent { .. }
|
|
) {
|
|
self.update_pending = true;
|
|
return Ok(());
|
|
}
|
|
let incompatible:bool=self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown','awaiting_reboot') AND NOT(job=?1 AND attempt=?2))",rusqlite::params![p.job_id.to_string(),p.attempt_id.to_string()],|r|r.get(0))?;
|
|
ensure!(!incompatible, "SYSTEM_PENDING_INTENT_CONFLICT");
|
|
binding.verify_historical_for_denial_with(
|
|
&intent.request,
|
|
reconcile::HelperIdentity::System,
|
|
&rmm_agent_core::crypto::DalekVerifier,
|
|
)?;
|
|
let bytes: Vec<u8> = self.journal.connection().query_row(
|
|
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
rusqlite::params![p.job_id.to_string(), p.attempt_id.to_string()],
|
|
|r| r.get(0),
|
|
)?;
|
|
let offer = JobOffer::decode(bytes.as_slice())?;
|
|
ensure!(
|
|
offer.payload_bytes == intent.request.job_bytes
|
|
&& offer.plan_hash == intent.request.plan_hash
|
|
&& offer.authorization_id == intent.request.authorization_id.to_string(),
|
|
"SYSTEM_PENDING_INTENT_BINDING"
|
|
);
|
|
// Even prepared-only intents are never executed after restart.
|
|
// Snapshot/explicit bound denial recovery is the only route.
|
|
intents.check()?;
|
|
let ledger = self.ledger.as_ref().context("LEDGER_REQUIRED")?;
|
|
let consumed: bool = ledger.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM consumed WHERE grant_id=?1)",
|
|
[p.grant_id.to_string()],
|
|
|r| r.get(0),
|
|
)?;
|
|
let actual = if consumed {
|
|
jobs::recover_interrupted_outcome(
|
|
ledger,
|
|
&binding,
|
|
&offer,
|
|
&p.grant_id.to_string(),
|
|
)?
|
|
} else {
|
|
None
|
|
};
|
|
if let Some(actual) = &actual {
|
|
let bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
|
job_id: p.job_id,
|
|
attempt_id: p.attempt_id,
|
|
grant_id: p.grant_id,
|
|
outcome: actual.clone(),
|
|
};
|
|
if !needs_bound_outcome(self.journal.connection(), &bound)? {
|
|
return Ok(());
|
|
}
|
|
}
|
|
self.record(
|
|
0,
|
|
&offer,
|
|
&p.grant_id.to_string(),
|
|
actual.unwrap_or_else(|| {
|
|
Outcome::unknown("SYSTEM_DELEGATION_INTERRUPTED_REQUIRES_HELPER_EVIDENCE")
|
|
}),
|
|
)?;
|
|
return Ok(());
|
|
}
|
|
}
|
|
let interrupted = jobs::interrupted(
|
|
self.journal.connection(),
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
&binding,
|
|
)?;
|
|
for (offer, grant) in interrupted {
|
|
if grant.is_empty() {
|
|
jobs::interrupted_request(self.journal.connection(), &offer)?;
|
|
} else {
|
|
let outcome = jobs::recover_interrupted_outcome(
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
&binding,
|
|
&offer,
|
|
&grant,
|
|
)?
|
|
.unwrap_or_else(|| Outcome::unknown("AGENT_PROCESS_INTERRUPTED"));
|
|
// Replay original durable evidence only. Normal signed Sync,
|
|
// never startup, releases the previously consumed attempt.
|
|
self.record(0, &offer, &grant, outcome)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
async fn replay(
|
|
&mut self,
|
|
channel: &crate::wire_channel::Handle,
|
|
generation: u64,
|
|
) -> Result<()> {
|
|
let events = jobs::events(self.journal.connection())?;
|
|
let passive = self.journal.events()?;
|
|
tokio::time::timeout(Duration::from_secs(2), async {
|
|
for mut event in events.into_iter().chain(passive).take(8) {
|
|
event.connection_generation = generation;
|
|
channel.send(&event).await?;
|
|
}
|
|
Ok::<_, anyhow::Error>(())
|
|
})
|
|
.await??;
|
|
Ok(())
|
|
}
|
|
async fn start_sync(
|
|
&mut self,
|
|
channel: &crate::wire_channel::Handle,
|
|
generation: u64,
|
|
) -> Result<()> {
|
|
self.fence.close();
|
|
ensure!(
|
|
!self.helper_control_busy_for_sync(),
|
|
"HELPER_CONTROL_NOT_SETTLED"
|
|
);
|
|
#[cfg(feature = "system-helper")]
|
|
if let Some(helper) = self.helper.clone() {
|
|
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
|
let boot = self.channel(generation)?.boot_id;
|
|
let instance = Uuid::parse_str(&self.instance)?;
|
|
let clock = self.clock.clone();
|
|
self.helper_control = Some(HelperControl::Snapshot(
|
|
generation,
|
|
tokio::spawn(async move {
|
|
let denial_resume = tokio::task::spawn_blocking(move || -> Result<_> {
|
|
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
|
use rmm_agent_windows::helper_runtime::intent::{Phase, Store};
|
|
let mut store = Store::open(true)?;
|
|
if let Some(intent) = store.pending()?.first() {
|
|
let (_, phase) = store.load(intent.id)?;
|
|
if phase == Phase::KnownNoDispatch
|
|
&& intent.agent_instance != instance
|
|
&& store.claim_denial_resume(
|
|
intent.id,
|
|
instance,
|
|
boot,
|
|
clock.now()?,
|
|
)?
|
|
{
|
|
return Ok(Some(intent.clone()));
|
|
}
|
|
}
|
|
}
|
|
Ok(None)
|
|
})
|
|
.await??;
|
|
if let Some(intent) = denial_resume {
|
|
let _denial = helper.resume_denial(intent).await;
|
|
}
|
|
helper.snapshot(generation, boot).await
|
|
}),
|
|
));
|
|
return Ok(());
|
|
}
|
|
self.send_sync(channel, generation, None).await
|
|
}
|
|
async fn send_sync(
|
|
&mut self,
|
|
channel: &crate::wire_channel::Handle,
|
|
generation: u64,
|
|
helper: Option<&reconcile::HelperSnapshot>,
|
|
) -> Result<()> {
|
|
let context = self.channel(generation)?;
|
|
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
|
let prepared = sync.prepare(
|
|
self.journal.connection_mut(),
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
context,
|
|
helper,
|
|
&self.clock,
|
|
);
|
|
#[cfg(feature = "system-helper")]
|
|
if helper.is_some()
|
|
&& prepared
|
|
.as_ref()
|
|
.is_err_and(|error| error.to_string() == "SYSTEM_HELPER_APPLY_UNCONFIRMED")
|
|
{
|
|
// This authenticated readback already established why another
|
|
// capture cannot advance this channel.
|
|
self.helper_apply_unconfirmed = true;
|
|
}
|
|
let prepared = prepared?;
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
self.prepared_request = Some(prepared.request.encode_to_vec());
|
|
}
|
|
self.prepared = true;
|
|
channel.send(&prepared.envelope).await?;
|
|
Ok(())
|
|
}
|
|
fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
|
ensure!(self.prepared, "UNSOLICITED_SYNC_RESPONSE");
|
|
if let Some(pending) = &self.deferred_sync {
|
|
ensure!(
|
|
pending.generation == generation && pending.response == response,
|
|
"DEFERRED_SYNC_RESPONSE_CHANGED"
|
|
);
|
|
return Ok(());
|
|
}
|
|
// Authenticate exact bytes/request/channel BEFORE retaining a reply.
|
|
// Declared issued_at is used only for historical signature/profile
|
|
// verification here. The regular fresh clock gates below still decide
|
|
// whether either state-application phase may run now.
|
|
ensure!(
|
|
response.decision_bytes.len() <= 512 * 1024 && response.signature.len() == 64,
|
|
"SYNC_RESPONSE_LIMIT"
|
|
);
|
|
let decision: rmm_domain::reconciliation::ReconcileDecisionPayload =
|
|
serde_json::from_slice(&response.decision_bytes)?;
|
|
let raw:Vec<u8>=self.journal.connection().query_row("SELECT request FROM reconcile_requests WHERE id=(SELECT pending_request FROM reconcile_meta WHERE id=1)",[],|r|r.get(0))?;
|
|
ensure!(raw.len() <= rmm_domain::MAX_MESSAGE, "SYNC_REQUEST_LIMIT");
|
|
let request = SyncRequest::decode(raw.as_slice())?;
|
|
ensure!(request.encode_to_vec() == raw, "SYNC_REQUEST_CANONICAL");
|
|
reconcile::verify_response_with(
|
|
&rmm_agent_core::crypto::DalekVerifier,
|
|
self.binding.as_ref().context("JOB_BINDING_REQUIRED")?,
|
|
&request,
|
|
&self.channel(generation)?,
|
|
&response,
|
|
decision.issued_at,
|
|
)?;
|
|
self.deferred_sync = Some(DeferredSync {
|
|
generation,
|
|
response,
|
|
issued_at: decision.issued_at,
|
|
valid_until: decision.valid_until,
|
|
received_at: Instant::now(),
|
|
applied: None,
|
|
});
|
|
self.poll_sync_apply(generation, true)
|
|
}
|
|
fn poll_sync_apply(&mut self, generation: u64, open_gate: bool) -> Result<()> {
|
|
let Some(mut pending) = self.deferred_sync.take() else {
|
|
return Ok(());
|
|
};
|
|
ensure!(pending.generation == generation, "DEFERRED_SYNC_GENERATION");
|
|
use rmm_agent_windows::job_readiness::{ResponseWindow, response_window};
|
|
match response_window(
|
|
pending.issued_at,
|
|
pending.valid_until,
|
|
pending.received_at.elapsed(),
|
|
self.clock.reading().ok().map(|r| (r.own_wall, r.upper)),
|
|
) {
|
|
ResponseWindow::WaitForClock => {
|
|
self.deferred_sync = Some(pending);
|
|
return Ok(());
|
|
}
|
|
ResponseWindow::Expired => anyhow::bail!("DEFERRED_SYNC_RESPONSE_EXPIRED"),
|
|
ResponseWindow::WithinWindow => {}
|
|
}
|
|
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
|
if pending.applied.is_none() {
|
|
match sync.apply(
|
|
self.journal.connection_mut(),
|
|
&pending.response,
|
|
&self.clock,
|
|
) {
|
|
Ok(applied) => {
|
|
#[cfg(feature = "system-helper")]
|
|
if applied.helper_instruction.is_some() {
|
|
self.helper_apply_unconfirmed = true;
|
|
}
|
|
pending.applied = Some(applied);
|
|
}
|
|
Err(error) if clock_temporarily_unavailable(&error) => {
|
|
self.deferred_sync = Some(pending);
|
|
return Ok(());
|
|
}
|
|
Err(error) => return Err(error),
|
|
}
|
|
}
|
|
match sync.apply_consumption(
|
|
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
|
&self.clock,
|
|
) {
|
|
Ok(()) => {}
|
|
Err(error) if clock_temporarily_unavailable(&error) => {
|
|
self.deferred_sync = Some(pending);
|
|
return Ok(());
|
|
}
|
|
Err(error) => return Err(error),
|
|
}
|
|
let applied = pending
|
|
.applied
|
|
.take()
|
|
.context("DEFERRED_SYNC_APPLIED_REQUIRED")?;
|
|
self.prepared = false;
|
|
if let Some(instruction) = applied.helper_instruction {
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
|
let helper = self
|
|
.helper
|
|
.clone()
|
|
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
|
let request = self
|
|
.prepared_request
|
|
.take()
|
|
.context("ORIGINAL_SYNC_REQUEST_REQUIRED")?;
|
|
let until = sync.validity_deadline().context("SIGNED_LEASE_REQUIRED")?;
|
|
let clock = self.clock.clone();
|
|
let boot = self.channel(generation)?.boot_id;
|
|
self.helper_control = Some(HelperControl::Apply(
|
|
generation,
|
|
tokio::spawn(async move {
|
|
helper
|
|
.apply(&instruction.response, request, boot, until, &clock)
|
|
.await
|
|
}),
|
|
));
|
|
return Ok(());
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
{
|
|
let _ = instruction;
|
|
anyhow::bail!("WINDOWS_HELPER_NOT_CONFIGURED");
|
|
}
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
self.prepared_request = None;
|
|
}
|
|
if open_gate {
|
|
self.open_fence(generation)
|
|
} else {
|
|
self.fence.close();
|
|
Ok(())
|
|
}
|
|
}
|
|
fn open_fence(&self, generation: u64) -> Result<()> {
|
|
if self.ready(generation) {
|
|
let deadline = self
|
|
.sync
|
|
.as_ref()
|
|
.and_then(|sync| sync.validity_deadline())
|
|
.context("SIGNED_LEASE_REQUIRED")?;
|
|
self.fence
|
|
.deadline
|
|
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
|
self.fence.generation.store(generation, Ordering::SeqCst);
|
|
self.fence.helper_generation.store(
|
|
if self.ready_identity(generation, true) {
|
|
generation
|
|
} else {
|
|
0
|
|
},
|
|
Ordering::SeqCst,
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
fn completed(&mut self, generation: u64, (offer, grant, result): Completion) -> Result<()> {
|
|
let running = self
|
|
.running
|
|
.as_ref()
|
|
.context("UNSOLICITED_EXECUTOR_COMPLETION")?;
|
|
ensure!(
|
|
running.offer.job_id == offer.job_id
|
|
&& running.offer.attempt_id == offer.attempt_id
|
|
&& running.grant == grant,
|
|
"EXECUTOR_COMPLETION_BINDING"
|
|
);
|
|
#[cfg(feature = "system-helper")]
|
|
if let ExecutorResult::UpdatePending(accepted) = result {
|
|
ensure!(
|
|
grant.execution_identity == "system" && grant.action == "update_agent",
|
|
"UPDATE_COMPLETION_SCOPE"
|
|
);
|
|
if let Some(accepted) = accepted {
|
|
let intent = rmm_agent_windows::helper_runtime::intent::Store::open(false)?
|
|
.by_grant(grant.grant_id)?;
|
|
accepted.matches(&intent.request, grant.grant_id)?;
|
|
let mut envelope = self.envelope(
|
|
generation,
|
|
Payload::JobStateChanged(JobStateChanged::default()),
|
|
)?;
|
|
let tx = self.journal.connection_mut().transaction()?;
|
|
let revision =
|
|
jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
|
|
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
revision,
|
|
state: "running".into(),
|
|
grant_id: grant.grant_id.to_string(),
|
|
}));
|
|
jobs::event(&tx, &envelope)?;
|
|
tx.commit()?;
|
|
}
|
|
// Neither a typed handoff ACK nor its loss is an Outcome. Import
|
|
// only actual original helper evidence in the next normal Sync.
|
|
self.update_pending = true;
|
|
self.running = None;
|
|
self.close_gate();
|
|
return Ok(());
|
|
}
|
|
#[cfg(not(feature="system-helper"))]
|
|
let ExecutorResult::Outcome(outcome)=result;
|
|
#[cfg(feature="system-helper")]
|
|
let outcome = match result {
|
|
ExecutorResult::Outcome(outcome) => outcome,
|
|
#[cfg(feature = "system-helper")]
|
|
ExecutorResult::UpdatePending(_) => unreachable!("handled update completion"),
|
|
};
|
|
if grant.execution_identity == "system" {
|
|
// No local consumption or definitive result from an IPC outcome.
|
|
// Import only a fresh raw helper snapshot before the next Sync.
|
|
self.record(
|
|
generation,
|
|
&offer,
|
|
&grant.grant_id.to_string(),
|
|
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"),
|
|
)?;
|
|
self.running = None;
|
|
self.close_gate();
|
|
return Ok(());
|
|
}
|
|
ledger::persist_outcome(
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
&grant,
|
|
&outcome,
|
|
)?;
|
|
self.record(
|
|
generation,
|
|
&offer,
|
|
&grant.grant_id.to_string(),
|
|
outcome.clone(),
|
|
)?;
|
|
ledger::finish(
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
&grant,
|
|
&outcome,
|
|
)?;
|
|
self.running = None;
|
|
if grant.execution_identity == "system" {
|
|
self.close_gate();
|
|
}
|
|
Ok(())
|
|
}
|
|
fn permit(&mut self, generation: u64, permit: ExecutionPermit) -> Result<()> {
|
|
if let Some((old_generation, old)) = &self.deferred_permit {
|
|
ensure!(
|
|
*old_generation == generation && *old == permit,
|
|
"PENDING_PERMIT_CONFLICT"
|
|
);
|
|
return Ok(());
|
|
}
|
|
let offer = jobs::requesting(self.journal.connection())?.context("UNSOLICITED_PERMIT")?;
|
|
if !permit.denial_code.is_empty() {
|
|
jobs::denied_request(self.journal.connection(), &offer, &permit)?;
|
|
self.close_gate();
|
|
return Ok(());
|
|
}
|
|
if self.clock.reading().is_err() {
|
|
self.deferred_permit = Some((generation, permit));
|
|
return Ok(());
|
|
}
|
|
ensure!(
|
|
self.ready(generation) && self.running.is_none(),
|
|
"FENCED_SYNC_REQUIRED"
|
|
);
|
|
let binding = self
|
|
.binding
|
|
.as_ref()
|
|
.context("JOB_BINDING_REQUIRED")?
|
|
.clone();
|
|
let request = jobs::permit_request(&offer, &permit)?;
|
|
let now = match self.clock.now() {
|
|
Ok(now) => now,
|
|
Err(_) => {
|
|
self.deferred_permit = Some((generation, permit));
|
|
return Ok(());
|
|
}
|
|
};
|
|
// Authenticate before interpreting timing. A freshly issued grant may
|
|
// need two seconds before the full uncertainty interval is inside it.
|
|
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
|
|
if now - chrono::Duration::seconds(2) < authenticated.not_before {
|
|
self.deferred_permit = Some((generation, permit));
|
|
return Ok(());
|
|
}
|
|
let (job, grant) = verified_window(&binding, &request, &self.clock)?;
|
|
ensure!(
|
|
self.supported_job(&job) && self.scope_permits(&offer, &job),
|
|
"UNSUPPORTED_CAPABILITY"
|
|
);
|
|
let system = job.execution_identity == "system";
|
|
ensure!(
|
|
self.ready_identity(generation, system),
|
|
"SYSTEM_HELPER_SYNC_REQUIRED"
|
|
);
|
|
let cancelled: bool = self.journal.connection().query_row(
|
|
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
rusqlite::params![offer.job_id, offer.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
#[cfg(feature = "system-helper")]
|
|
if system {
|
|
use rmm_agent_windows::helper_runtime::intent::{Intent, Store};
|
|
let helper = self
|
|
.helper
|
|
.clone()
|
|
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
|
let intent = Intent {
|
|
schema_version: 1,
|
|
id: Uuid::new_v4(),
|
|
request: request.clone(),
|
|
anchor: self
|
|
.helper_anchor
|
|
.clone()
|
|
.context("SYSTEM_READY_ANCHOR_REQUIRED")?,
|
|
original_boot_id: Uuid::parse_str(&self.boot)?,
|
|
agent_instance: Uuid::parse_str(&self.instance)?,
|
|
connection_generation: generation,
|
|
recorded_at: self.clock.now()?,
|
|
policy_sha256: helper.pin_hash().into(),
|
|
helper_binary_sha256: helper.helper_binary_hash().into(),
|
|
};
|
|
let mut store = Store::open(true)?;
|
|
store.prepare(&intent)?;
|
|
if cancelled {
|
|
store.decide(intent.id, false)?;
|
|
}
|
|
let cancelled = Arc::new(AtomicBool::new(cancelled));
|
|
let worker_cancel = cancelled.clone();
|
|
let clock = self.clock.clone();
|
|
let fence = self.fence.clone();
|
|
let completed = self.completed_tx.clone();
|
|
let work_offer = offer.clone();
|
|
let work_grant = grant.clone();
|
|
let is_update = matches!(job.action, Action::UpdateAgent { .. });
|
|
if is_update {
|
|
self.update_pending = true;
|
|
}
|
|
let msi = job.action.windows_msi().cloned();
|
|
let address = self.config.connect_address;
|
|
let ca = self.ca.clone();
|
|
let tls_identity = self.certificates.current(
|
|
self.journal.connection(),
|
|
&self.key_files,
|
|
Utc::now().timestamp(),
|
|
)?;
|
|
let request_for_stage = request.clone();
|
|
let binding_for_stage = binding.clone();
|
|
let worker = tokio::spawn(async move {
|
|
let staged = if let Some(m) = msi {
|
|
let result = async {
|
|
ensure!(
|
|
!worker_cancel.load(Ordering::SeqCst)
|
|
&& fence.allows_identity(generation, true, &clock),
|
|
"MSI_DOWNLOAD_FENCED"
|
|
);
|
|
verified_window(&binding_for_stage, &request_for_stage, &clock)?;
|
|
let tls = crate::https_channel::mtls_client(
|
|
address,
|
|
&ca,
|
|
&tls_identity.certificate_pem,
|
|
&tls_identity.private_key_pem,
|
|
)?;
|
|
let response = tls
|
|
.get(format!(
|
|
"https://{}:{}/agent/v1/artifacts/{}?job_id={}&attempt_id={}",
|
|
crate::https_channel::server_name()?,
|
|
address.port(),
|
|
m.recipe.artifact.artifact_id,
|
|
work_grant.job_id,
|
|
work_grant.attempt_id
|
|
))
|
|
.send()
|
|
.await?;
|
|
ensure!(
|
|
response
|
|
.headers()
|
|
.get("x-artifact-sha256")
|
|
.and_then(|v| v.to_str().ok())
|
|
== Some(m.recipe.artifact.sha256.as_str()),
|
|
"MSI_DOWNLOAD_METADATA"
|
|
);
|
|
let bytes = crate::https_channel::bounded_body(
|
|
response,
|
|
m.recipe.artifact.bytes as usize,
|
|
"application/octet-stream",
|
|
)
|
|
.await?;
|
|
ensure!(
|
|
!worker_cancel.load(Ordering::SeqCst)
|
|
&& fence.allows_identity(generation, true, &clock),
|
|
"MSI_DOWNLOAD_FENCED"
|
|
);
|
|
verified_window(&binding_for_stage, &request_for_stage, &clock)?;
|
|
tokio::task::spawn_blocking(move || {
|
|
rmm_agent_windows::msi_cache::stage(&m, &bytes)
|
|
})
|
|
.await??;
|
|
Ok::<_, anyhow::Error>(())
|
|
}
|
|
.await;
|
|
result.is_ok()
|
|
} else {
|
|
true
|
|
};
|
|
let result = helper
|
|
.delegate(intent, move || {
|
|
ensure!(staged, "MSI_DOWNLOAD_NOT_READY");
|
|
ensure!(
|
|
!worker_cancel.load(Ordering::SeqCst),
|
|
"CANCELLED_BEFORE_DISPATCH"
|
|
);
|
|
ensure!(
|
|
fence.allows_identity(generation, true, &clock),
|
|
"FENCED_BEFORE_DISPATCH"
|
|
);
|
|
verified_window(&binding, &request, &clock)?;
|
|
Ok(())
|
|
})
|
|
.await;
|
|
let _ = completed
|
|
.send((
|
|
work_offer,
|
|
work_grant,
|
|
if is_update {
|
|
ExecutorResult::UpdatePending(result.ok().flatten())
|
|
} else {
|
|
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT").into()
|
|
},
|
|
))
|
|
.await;
|
|
});
|
|
self.running = Some(Running {
|
|
offer,
|
|
grant,
|
|
cancel: cancelled,
|
|
worker,
|
|
});
|
|
return Ok(());
|
|
}
|
|
let started_at = self.clock.now()?;
|
|
let ledger = self.ledger.as_mut().context("LEDGER_REQUIRED")?;
|
|
ledger::begin_execution(
|
|
ledger,
|
|
&grant,
|
|
&request,
|
|
Uuid::parse_str(&self.boot)?,
|
|
started_at,
|
|
)?;
|
|
if cancelled {
|
|
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
|
ledger::persist_outcome(
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
&grant,
|
|
&outcome,
|
|
)?;
|
|
self.record(
|
|
generation,
|
|
&offer,
|
|
&grant.grant_id.to_string(),
|
|
outcome.clone(),
|
|
)?;
|
|
ledger::finish(
|
|
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
|
&grant,
|
|
&outcome,
|
|
)?;
|
|
return Ok(());
|
|
}
|
|
let mut started = self.envelope(
|
|
generation,
|
|
Payload::JobStateChanged(JobStateChanged::default()),
|
|
)?;
|
|
let tx = self.journal.connection_mut().transaction()?;
|
|
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
|
|
started.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
revision,
|
|
state: "running".into(),
|
|
grant_id: grant.grant_id.to_string(),
|
|
}));
|
|
jobs::event(&tx, &started)?;
|
|
tx.commit()?;
|
|
let cancelled = Arc::new(AtomicBool::new(false));
|
|
let worker_cancel = cancelled.clone();
|
|
let clock = self.clock.clone();
|
|
let fence = self.fence.clone();
|
|
let completed = self.completed_tx.clone();
|
|
let directory = self.config.state_directory.join("jobs");
|
|
let work_offer = offer.clone();
|
|
let work_grant = grant.clone();
|
|
let original_boot = Uuid::parse_str(&self.boot)?;
|
|
let echo_scope = self.config.echo_scope.clone();
|
|
let worker = tokio::task::spawn_blocking(move || {
|
|
let mut outcome = (|| -> Result<Outcome> {
|
|
if !fence.allows(generation, &clock)
|
|
|| verified_window(&binding, &request, &clock).is_err()
|
|
{
|
|
return Ok(jobs::no_start(
|
|
"failed",
|
|
"START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT",
|
|
));
|
|
}
|
|
if let Some(scope) = &echo_scope {
|
|
ensure!(
|
|
rmm_agent_windows::echo_probe::permits_offer(scope, &work_offer, &job),
|
|
"WINDOWS_ECHO_DISPATCH_SCOPE"
|
|
);
|
|
}
|
|
let prepared = match rmm_agent_windows::actions::PreparedAction::prepare(
|
|
&job,
|
|
&directory,
|
|
work_grant.grant_id,
|
|
&request,
|
|
original_boot,
|
|
) {
|
|
Ok(prepared) => prepared,
|
|
Err(_) => {
|
|
return Ok(jobs::no_start(
|
|
"failed",
|
|
if matches!(job.action, Action::RunScript { .. }) {
|
|
"SCRIPT_PREPARATION_FAILED"
|
|
} else {
|
|
"ACTION_PREPARATION_FAILED"
|
|
},
|
|
));
|
|
}
|
|
};
|
|
// File creation, pool scheduling and fsync never extend a start window.
|
|
if worker_cancel.load(Ordering::SeqCst) {
|
|
return Ok(jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT"));
|
|
}
|
|
if !fence.allows(generation, &clock)
|
|
|| verified_window(&binding, &request, &clock).is_err()
|
|
{
|
|
return Ok(jobs::no_start(
|
|
"failed",
|
|
"START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT",
|
|
));
|
|
}
|
|
prepared.run(&job, &worker_cancel)
|
|
})()
|
|
.unwrap_or_else(|_| Outcome::unknown("WINDOWS_EXECUTOR_EFFECT_UNKNOWN"));
|
|
if outcome.state == "succeeded" && clock.reading().is_err() {
|
|
// Use this worker's original connection-bound clock, never a
|
|
// newer session's evidence when its retained outcome is drained.
|
|
outcome.state = "unknown".into();
|
|
outcome.error_code = "WINDOWS_FINAL_CLOCK_UNTRUSTED".into();
|
|
outcome.evidence["postconditions_passed"] = false.into();
|
|
}
|
|
let _ = completed.blocking_send((work_offer, work_grant, outcome.into()));
|
|
});
|
|
self.running = Some(Running {
|
|
offer,
|
|
grant,
|
|
cancel: cancelled,
|
|
worker,
|
|
});
|
|
Ok(())
|
|
}
|
|
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
|
|
Ok(Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: self.identity.agent_id.clone(),
|
|
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
|
connection_generation: generation,
|
|
boot_id: self.boot.clone(),
|
|
sequence: self.journal.next_sequence()?,
|
|
observed_at: Utc::now().timestamp_millis(),
|
|
payload: Some(payload),
|
|
})
|
|
}
|
|
fn queue_inventory(&mut self, generation: u64) -> Result<bool> {
|
|
if self.journal.count()? > 0 || !jobs::events(self.journal.connection())?.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
let data = serde_json::to_vec(&native::snapshot()?)?;
|
|
ensure!(data.len() <= 32768, "WINDOWS_INVENTORY_LIMIT");
|
|
let envelope = self.envelope(
|
|
generation,
|
|
Payload::InventorySnapshot(InventorySnapshot {
|
|
snapshot_id: Uuid::new_v4().to_string(),
|
|
scope: "windows.base".into(),
|
|
schema_version: 1,
|
|
chunk_index: 0,
|
|
chunk_count: 1,
|
|
sha256: Sha256::digest(&data).to_vec(),
|
|
data,
|
|
completeness: "partial".into(),
|
|
}),
|
|
)?;
|
|
if let Some(binding) = &self.binding {
|
|
jobs::event(self.journal.connection(), &envelope)?;
|
|
let initialized:bool=self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='reconcile_meta')",[],|r|r.get(0))?;
|
|
if initialized
|
|
&& self.journal.connection().query_row(
|
|
"SELECT generation FROM reconcile_meta WHERE id=1",
|
|
[],
|
|
|r| r.get::<_, u64>(0),
|
|
)? == generation
|
|
{
|
|
rmm_agent_core::update_health_inventory::retain_current(
|
|
self.journal.connection(),
|
|
&envelope,
|
|
binding,
|
|
)?;
|
|
}
|
|
} else {
|
|
self.journal.save(&envelope)?;
|
|
}
|
|
Ok(true)
|
|
}
|
|
fn queue_update_observation(&mut self, generation: u64) -> Result<()> {
|
|
if self.journal.count()? > 0 || !jobs::events(self.journal.connection())?.is_empty() {
|
|
return Ok(());
|
|
}
|
|
#[cfg(feature="system-helper")]
|
|
let observation=self.update_observer.observation(generation);
|
|
#[cfg(not(feature="system-helper"))]
|
|
let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured(
|
|
rmm_agent_windows::release_platform::WIRE,Utc::now());
|
|
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
|
|
let envelope = self.envelope(
|
|
generation,
|
|
Payload::InventorySnapshot(InventorySnapshot {
|
|
snapshot_id: Uuid::new_v4().to_string(),
|
|
scope: "agent.update".into(),
|
|
schema_version: 1,
|
|
chunk_index: 0,
|
|
chunk_count: 1,
|
|
sha256: Sha256::digest(&data).to_vec(),
|
|
data,
|
|
completeness: "partial".into(),
|
|
}),
|
|
)?;
|
|
if self.binding.is_some() {
|
|
jobs::event(self.journal.connection(), &envelope)
|
|
} else {
|
|
self.journal.save(&envelope)
|
|
}
|
|
}
|
|
async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
|
|
self.close_gate();
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
// Reconnect discards the old channel. ClientSync::prepare checks
|
|
// the authenticated helper head against the durable agent journal.
|
|
self.helper_apply_unconfirmed = false;
|
|
}
|
|
let result = self.session_inner(stop).await;
|
|
self.fence.close();
|
|
#[cfg(feature = "system-helper")]
|
|
{
|
|
let until = Instant::now() + Duration::from_secs(70);
|
|
// Complete the already authenticated original control phases while
|
|
// their finite deadline and independent time lane still permit it.
|
|
// No job is running when this control pipeline was prepared.
|
|
while self.running.is_none() && self.deferred_sync.is_some() && Instant::now() < until {
|
|
let generation = self
|
|
.deferred_sync
|
|
.as_ref()
|
|
.expect("checked pending control")
|
|
.generation;
|
|
if self.poll_sync_apply(generation, false).is_err() {
|
|
break;
|
|
}
|
|
if self.deferred_sync.is_some() {
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
}
|
|
self.close_gate();
|
|
self.drain_helper_apply(until).await;
|
|
self.stop_time_refresh().await;
|
|
}
|
|
#[cfg(not(feature = "system-helper"))]
|
|
self.close_gate();
|
|
result
|
|
}
|
|
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
|
// Server commit may have fenced Hello even if its HTTP response was
|
|
// lost. Recover the durable original CSR before requiring any channel.
|
|
if self.certificates.has_pending(self.journal.connection())? {
|
|
let admission = if self.config.enable_jobs {
|
|
let retained = self
|
|
.certificates
|
|
.pending_idle_admission(self.journal.connection())?
|
|
.context("ROTATION_PENDING_REQUIRES_PASSIVE_RECOVERY")?;
|
|
Some(self.rotation_admission(None, Some(&retained))?)
|
|
} else {
|
|
None
|
|
};
|
|
self.renew_certificate(admission.as_deref()).await?;
|
|
}
|
|
self.boot = native::boot_id().await?;
|
|
let advertised = self.capabilities();
|
|
let hello = self.envelope(
|
|
0,
|
|
Payload::ClientHello(ClientHello {
|
|
agent_version: self.reported_version.clone(),
|
|
capabilities: advertised.clone(),
|
|
}),
|
|
)?;
|
|
let identity = self.certificates.current(
|
|
self.journal.connection(),
|
|
&self.key_files,
|
|
Utc::now().timestamp(),
|
|
)?;
|
|
let mode = crate::wire_channel::Mode::parse(&self.config.transport)?;
|
|
let mut channel = crate::wire_channel::connect(
|
|
mode,
|
|
self.config.connect_address,
|
|
&self.ca,
|
|
&identity,
|
|
hello,
|
|
)
|
|
.await?;
|
|
let reply = tokio::time::timeout(Duration::from_secs(20), channel.receive()).await??;
|
|
ensure!(
|
|
reply.agent_id == self.identity.agent_id
|
|
&& reply.enrollment_generation
|
|
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation)
|
|
&& reply.protocol_major == 1
|
|
&& reply.protocol_minor == 0,
|
|
"SERVER_HELLO_BINDING"
|
|
);
|
|
let Some(Payload::ServerHello(server)) = reply.payload else {
|
|
anyhow::bail!("SERVER_HELLO_REQUIRED")
|
|
};
|
|
let generation = server.connection_generation;
|
|
ensure!(
|
|
generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32,
|
|
"SERVER_LIMITS"
|
|
);
|
|
self.clock = self.clock.for_generation(generation);
|
|
#[cfg(feature = "system-helper")]
|
|
if self.config.enable_jobs {
|
|
self.start_time_refresh(generation, &identity)?;
|
|
}
|
|
self.queue_inventory(generation)?;
|
|
self.queue_update_observation(generation)?;
|
|
self.replay(&channel, generation).await?;
|
|
let mut needs_sync = self.sync.is_some();
|
|
let mut permit_after = Instant::now();
|
|
if self.sync.is_some() && self.running.is_none() {
|
|
if self.start_sync(&channel, generation).await.is_err() {
|
|
self.close_gate();
|
|
} else {
|
|
needs_sync = false;
|
|
}
|
|
}
|
|
let mut sync_at = Instant::now();
|
|
let mut capabilities_at = Instant::now();
|
|
let mut replay = Instant::now() - Duration::from_secs(10);
|
|
let mut inventory = Instant::now();
|
|
let mut update_at = Instant::now();
|
|
let mut rotation_at = Instant::now();
|
|
let mut rotation_after = Duration::from_secs(30);
|
|
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
|
let mut heartbeat_after = Duration::from_secs(30);
|
|
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
|
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
|
let mut readiness_at = Instant::now() - Duration::from_secs(5);
|
|
#[cfg(feature = "system-helper")]
|
|
let mut update_control_at = Instant::now() - Duration::from_secs(2);
|
|
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
|
|
if stop.load(Ordering::SeqCst) {
|
|
self.close_gate();
|
|
if let Some(running) = &self.running {
|
|
running.cancel.store(true, Ordering::SeqCst);
|
|
}
|
|
}
|
|
tokio::select! {
|
|
Some(completion)=self.completed_rx.recv()=>{self.completed(generation,completion)?;needs_sync=true;},
|
|
_=tick.tick()=>{
|
|
if !stop.load(Ordering::SeqCst) && self.poll_sync_apply(generation,true).is_err(){self.close_gate();needs_sync=true;}
|
|
#[cfg(feature = "system-helper")]
|
|
if !stop.load(Ordering::SeqCst) && self.poll_helper(&channel,generation).await.is_err() {self.close_gate();needs_sync=self.sync.is_some();}
|
|
if !stop.load(Ordering::SeqCst) && let Some((pending_generation,permit))=self.deferred_permit.take() {
|
|
ensure!(pending_generation==generation,"PENDING_PERMIT_GENERATION");self.permit(generation,permit)?;
|
|
}
|
|
if self.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
|
|
if let Ok(completion)=self.completed_rx.try_recv(){self.completed(generation,completion)?;}
|
|
else {let running=self.running.as_ref().context("RUNNING_REQUIRED")?;self.completed(generation,(running.offer.clone(),running.grant.clone(),interrupted_executor(&running.grant)))?;}
|
|
needs_sync=true;
|
|
}
|
|
#[cfg(feature="system-helper")]
|
|
if !stop.load(Ordering::SeqCst) && update_control_at.elapsed()>=Duration::from_secs(2) {
|
|
update_control_at=Instant::now();
|
|
if !self.update_pending && !self.prepared && !self.helper_control_pending() && self.running.is_none() && self.ready(generation) {
|
|
let transport=(||->Result<_>{let helper=self.helper.clone().filter(|h|h.update_observer_configured()).context("UPDATE_OBSERVER_NOT_CONFIGURED")?;
|
|
let identity=self.certificates.current(self.journal.connection(),&self.key_files,Utc::now().timestamp())?;
|
|
Ok((crate::https_channel::mtls_client(self.config.connect_address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?,self.config.connect_address,helper))})();
|
|
self.update_observer.poll(generation,||transport).await;
|
|
}
|
|
match self.poll_update_health(generation).await {Ok(refresh)=>{needs_sync|=refresh;},Err(_)=>{self.close_gate();needs_sync=true;}}
|
|
}
|
|
let busy=self.running.is_some() || self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?;
|
|
#[cfg(feature="system-helper")]
|
|
let busy=(busy && !(self.update_pending && self.running.is_none())) || self.update_health.is_some();
|
|
if !stop.load(Ordering::SeqCst) && !busy {
|
|
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
|
if self.prepared && self.deferred_sync.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
|
if self.sync.is_some() && !self.prepared && !self.helper_control_busy_for_sync() && rmm_agent_windows::job_readiness::refresh_due(needs_sync,sync_at.elapsed(),self.sync.as_ref().and_then(|s|s.validity_deadline()),self.clock.reading().ok().map(|r|r.upper)) {
|
|
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
|
}
|
|
if Instant::now() >= permit_after && !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
|
// A transiently stale clock skips this tick; it must not end the session.
|
|
if let Ok(now)=self.clock.now() && let Some(offer)=jobs::next_at(self.journal.connection(),now)? {
|
|
let job=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.job(&offer.payload_bytes)?;
|
|
ensure!(self.scope_permits(&offer,&job),"WINDOWS_ECHO_OFFER_OUT_OF_SCOPE");
|
|
if now>=job.start_before {self.record(generation,&offer,"",jobs::no_start("expired","START_DEADLINE_EXPIRED"))?;needs_sync=true;}
|
|
else if now-chrono::Duration::seconds(2)>=job.not_before && self.ready_identity(generation,job.execution_identity=="system") {
|
|
#[cfg(feature="system-helper")]
|
|
let prefetched=self.prefetch_update(generation,&offer,&job).await?;
|
|
#[cfg(not(feature="system-helper"))]
|
|
let prefetched=true;
|
|
if prefetched && self.reserve_release_gate()? {
|
|
jobs::reserve_request(self.journal.connection(),&offer)?;
|
|
let request=self.envelope(generation,Payload::ExecutionPermitRequest(ExecutionPermitRequest{job_id:offer.job_id,attempt_id:offer.attempt_id,job_payload_hash:offer.job_payload_hash,assignment_version:job.assignment_version}))?;
|
|
channel.send(&request).await?;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
message=channel.receive()=>{
|
|
let message=message?;
|
|
ensure!(message.agent_id==self.identity.agent_id && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.protocol_major==1 && message.protocol_minor==0 && message.connection_generation==generation,"SERVER_ENVELOPE_BINDING");
|
|
match message.payload {
|
|
Some(Payload::Ack(ack))=>{
|
|
let pending:Option<Vec<u8>>=self.journal.connection().query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
|
|
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
|
|
self.journal.ack(&ack.message_id)?;jobs::ack(self.journal.connection_mut(),&ack.message_id)?;
|
|
},
|
|
Some(Payload::SyncResponse(response))=>{if self.apply_sync(generation,response).is_err(){self.close_gate();needs_sync=true;}sync_at=Instant::now();},
|
|
Some(Payload::JobOffer(offer))=>{
|
|
ensure!(self.config.enable_jobs,"UNSUPPORTED_EXECUTION_CAPABILITY");
|
|
let binding=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.clone();
|
|
let accepted=self.envelope(generation,Payload::JobAccepted(JobAccepted{job_id:offer.job_id.clone(),attempt_id:offer.attempt_id.clone(),job_payload_hash:offer.job_payload_hash.clone(),journal_proof:rmm_domain::jobs::hash(&offer.encode_to_vec())}))?;
|
|
let job=binding.job(&offer.payload_bytes)?;
|
|
let supported=self.supported_job(&job) && self.scope_permits(&offer,&job);
|
|
let tx=self.journal.connection_mut().transaction()?;
|
|
let fresh=jobs::accept(&tx,&binding,&offer,|_|supported)?;
|
|
jobs::event(&tx,&accepted)?;tx.commit()?;needs_sync|=fresh;
|
|
},
|
|
Some(Payload::ExecutionPermit(permit))=> {
|
|
if !permit.denial_code.is_empty() {
|
|
needs_sync = true;
|
|
permit_after = Instant::now() + Duration::from_secs(30);
|
|
}
|
|
self.permit(generation,permit)?;
|
|
},
|
|
Some(Payload::CancelRequest(cancel))=>{
|
|
let row:Option<(Vec<u8>,String,String)>=self.journal.connection().query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",rusqlite::params![cancel.job_id,cancel.attempt_id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?;
|
|
if let Some((bytes,state,grant))=row {
|
|
self.journal.connection().execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2",rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
|
if let Some(running)=&self.running && running.offer.job_id==cancel.job_id && running.offer.attempt_id==cancel.attempt_id {running.cancel.store(true,Ordering::SeqCst);}
|
|
#[cfg(feature="system-helper")]
|
|
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
|
let mut store=rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
|
if let Some(intent)=store.pending()?.first() && intent.permit()?.job_id.to_string()==cancel.job_id && intent.permit()?.attempt_id.to_string()==cancel.attempt_id {store.deny(&intent.request,Uuid::parse_str(&self.boot)?)?;store.cancel_before_dispatch(intent.id)?;}
|
|
}
|
|
if state=="accepted" {self.record(generation,&JobOffer::decode(bytes.as_slice())?,&grant,jobs::no_start("cancelled","CANCELLED_BEFORE_START"))?;needs_sync=true;}
|
|
}
|
|
let ack=self.envelope(generation,Payload::Ack(Ack{message_id:message.message_id}))?;channel.send(&ack).await?;
|
|
},
|
|
_=>anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE")
|
|
}
|
|
}
|
|
}
|
|
self.release_idle_gate()?;
|
|
if readiness_at.elapsed() >= Duration::from_secs(5) {
|
|
readiness_at = Instant::now();
|
|
if self
|
|
.readiness_worker
|
|
.as_ref()
|
|
.is_none_or(|worker| worker.is_finished())
|
|
{
|
|
self.readiness_worker.take();
|
|
if let Ok(observation) = self.readiness_observation(generation, needs_sync) {
|
|
let state = self.config.state_directory.clone();
|
|
self.readiness_worker = Some(tokio::task::spawn_blocking(move || {
|
|
rmm_agent_windows::job_readiness::persist(&state, &observation)
|
|
}));
|
|
}
|
|
}
|
|
}
|
|
if stop.load(Ordering::SeqCst) {
|
|
continue;
|
|
}
|
|
// Certificate maintenance cannot interrupt accepted, ambiguous or
|
|
// live work. Active rotation also retains its exact signed-idle
|
|
// context for a closed, effect-free retry after response loss.
|
|
if rotation_at.elapsed() >= rotation_after {
|
|
rotation_at = Instant::now();
|
|
rotation_after = Duration::from_secs(30);
|
|
let now = Utc::now().timestamp();
|
|
if self
|
|
.certificates
|
|
.due(self.journal.connection(), &self.key_files, now)?
|
|
{
|
|
let admission = if self.config.enable_jobs {
|
|
self.rotation_admission(Some(generation), None).map(Some)
|
|
} else {
|
|
self.rotation_without_effects().map(|()| None)
|
|
};
|
|
if let Ok(admission) = admission {
|
|
self.renew_certificate(admission.as_deref()).await?;
|
|
// No offers are processed during renewal. The server
|
|
// fences this channel; fresh Hello/Sync must qualify
|
|
// both roles again before any subsequent job starts.
|
|
return Ok(());
|
|
}
|
|
// Avoid phase-locking a due renewal to the 30s helper
|
|
// probe cadence. Busy work keeps its guard and continues.
|
|
rotation_after = Duration::from_secs(1);
|
|
}
|
|
}
|
|
if heartbeat.elapsed() >= heartbeat_after {
|
|
#[cfg(feature = "system-helper")]
|
|
if !self.config.enable_jobs
|
|
&& self.time_refresh.is_none()
|
|
&& self.start_time_refresh(generation, &identity).is_err()
|
|
{
|
|
// An absent or unreadable profile must never take passive
|
|
// inventory offline. Retry setup at the next heartbeat.
|
|
eprintln!("PASSIVE_SIGNED_TIME_UNAVAILABLE");
|
|
}
|
|
let payload = Payload::Heartbeat(Heartbeat {
|
|
instance_id: self.instance.clone(),
|
|
uptime_seconds: self.started.elapsed().as_secs(),
|
|
});
|
|
let h = self.envelope(generation, payload)?;
|
|
channel.send(&h).await?;
|
|
if channel.uses_https() {
|
|
let poll = self.envelope(
|
|
generation,
|
|
Payload::Heartbeat(Heartbeat {
|
|
instance_id: self.instance.clone(),
|
|
uptime_seconds: self.started.elapsed().as_secs(),
|
|
}),
|
|
)?;
|
|
channel.poll(poll).await?;
|
|
}
|
|
#[cfg(feature = "system-helper")]
|
|
if let Some(refresh) = &mut self.time_refresh {
|
|
let state = &self.config.state_directory;
|
|
refresh.samples.retain(|slot| {
|
|
slot.get()
|
|
.is_none_or(|sample| sample.persist(state).is_err())
|
|
});
|
|
}
|
|
heartbeat = Instant::now();
|
|
heartbeat_after = Duration::from_millis(rand::random_range(27000..33000));
|
|
}
|
|
if replay.elapsed() >= Duration::from_secs(5) {
|
|
self.replay(&channel, generation).await?;
|
|
replay = Instant::now();
|
|
}
|
|
if !stop.load(Ordering::SeqCst) && inventory.elapsed() >= Duration::from_secs(86400) {
|
|
self.queue_inventory(generation)?;
|
|
inventory = Instant::now();
|
|
}
|
|
if !stop.load(Ordering::SeqCst) && update_at.elapsed() >= Duration::from_secs(60) {
|
|
self.queue_update_observation(generation)?;
|
|
update_at = Instant::now();
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
async fn renew_certificate(&mut self, admission: Option<&[u8]>) -> Result<()> {
|
|
self.rotation_without_effects()?;
|
|
ensure!(
|
|
!self.config.enable_jobs || admission.is_some(),
|
|
"ROTATION_ACTIVE_IDLE_REQUIRED"
|
|
);
|
|
self.close_gate();
|
|
#[cfg(feature = "system-helper")]
|
|
self.stop_time_refresh().await;
|
|
if self.config.enable_jobs {
|
|
self.rotation_admission(None, admission)?;
|
|
}
|
|
let request = self.certificates.prepare_with_idle_admission(
|
|
self.journal.connection_mut(),
|
|
&self.key_files,
|
|
Utc::now().timestamp(),
|
|
admission,
|
|
)?;
|
|
let client = crate::https_channel::mtls_client(
|
|
self.config.connect_address,
|
|
&self.ca,
|
|
&request.identity.certificate_pem,
|
|
&request.identity.private_key_pem,
|
|
)?;
|
|
let response = client
|
|
.post(format!(
|
|
"https://{}:{}/agent/v1/certificates/renew",
|
|
crate::https_channel::server_name()?,
|
|
self.config.connect_address.port()
|
|
))
|
|
.json(&serde_json::json!({"csr_pem":request.csr_pem}))
|
|
.send()
|
|
.await?;
|
|
let bytes = crate::https_channel::bounded_body(response, 65536, "application/json").await?;
|
|
self.certificates.install(
|
|
self.journal.connection_mut(),
|
|
&self.key_files,
|
|
&request,
|
|
&bytes,
|
|
Utc::now().timestamp(),
|
|
)
|
|
}
|
|
}
|
|
fn installed_version() -> Result<String> {
|
|
let executable = std::env::current_exe()?;
|
|
let directory = executable.parent().context("AGENT_IMAGE_DIRECTORY")?;
|
|
let path = directory.join("manifest.json");
|
|
if !path.try_exists()? {
|
|
return Ok(env!("CARGO_PKG_VERSION").into());
|
|
}
|
|
rmm_agent_windows::job_binding::check_admin_readonly(&path)?;
|
|
let manifest: rmm_updater::ReleaseManifest =
|
|
serde_json::from_slice(&protected_read(&path, rmm_updater::MAX_MANIFEST_BYTES)?)?;
|
|
let expected = &manifest.artifacts[0];
|
|
let mut digest = Sha256::new();
|
|
let bytes = std::io::copy(&mut fs::File::open(executable)?, &mut digest)?;
|
|
ensure!(
|
|
manifest.platform == rmm_agent_windows::release_platform::VERIFY
|
|
&& expected.role == rmm_updater::ArtifactRole::Agent
|
|
&& expected.bytes == bytes
|
|
&& expected.sha256 == hex::encode(digest.finalize()),
|
|
"INSTALLED_AGENT_IMAGE_MISMATCH"
|
|
);
|
|
Ok(manifest.version)
|
|
}
|
|
|
|
fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
|
let arguments: Vec<String> = std::env::args().skip(1).collect();
|
|
ensure!(
|
|
arguments.len() == 3 && arguments[0] == "--service" && arguments[1] == "--config",
|
|
"SERVICE_ARGUMENTS"
|
|
);
|
|
let path = Path::new(&arguments[2]);
|
|
ensure!(path.is_absolute(), "ABSOLUTE_CONFIG_REQUIRED");
|
|
let config = Config::read(path)?;
|
|
native::require_local_service()?;
|
|
ensure!(
|
|
config.state_directory.is_absolute(),
|
|
"ABSOLUTE_STATE_REQUIRED"
|
|
);
|
|
native::private_path(&config.state_directory, true)?;
|
|
ensure!(
|
|
crate::https_channel::server_name()? == config.server_name,
|
|
"SERVER_NAME"
|
|
);
|
|
let ca = protected_read(&config.ca_file, 32768)?;
|
|
for child in fs::read_dir(&config.state_directory)? {
|
|
let child = child?;
|
|
native::private_path(&child.path(), child.file_type()?.is_dir())?;
|
|
}
|
|
let journal = Journal::open(&config.state_directory)?;
|
|
let runtime = tokio::runtime::Builder::new_multi_thread()
|
|
.enable_all()
|
|
.worker_threads(2)
|
|
.build()?;
|
|
runtime.block_on(async move {
|
|
let identity = enrollment(&config, &ca).await?;
|
|
let mut journal=journal;
|
|
journal.pin_identity(&identity.agent_id)?;
|
|
let boot = native::boot_id().await?;
|
|
let binding=config.job_binding_file.as_deref().map(readonly_binding).transpose()?;
|
|
#[cfg(feature = "system-helper")]
|
|
let binding = if let Some(policy) = &config.localservice_time_policy {
|
|
Some(rmm_agent_windows::signed_clock::enrolled_binding(policy, &identity)?)
|
|
} else { binding };
|
|
if let Some(binding)=&binding {
|
|
ensure!(binding.agent_id.to_string()==identity.agent_id && binding.asset_id.to_string()==identity.asset_id,"ENROLLMENT_JOB_BINDING_MISMATCH");
|
|
native::private_path(&config.state_directory.join("jobs"),true)?;
|
|
import_legacy_inventory(&mut journal)?;
|
|
}
|
|
ensure!(!config.enable_jobs || binding.is_some(),"ENABLED_JOBS_REQUIRE_BINDING");
|
|
let certificates=RotationStore::new(Uuid::parse_str(&identity.agent_id)?,binding.as_ref().map_or(1,|b|b.enrollment_generation),ca.clone())?;
|
|
let key_files=ProtectedKeyFiles::open(&config.state_directory)?;
|
|
certificates.initialize(journal.connection_mut(),&key_files,&protected_read(&config.state_directory.join("device.pem"),32768)?,Utc::now().timestamp())?;
|
|
// SQLite and sidecars inherit the same service-SID-only state ACL.
|
|
for child in fs::read_dir(&config.state_directory)? { let child=child?;native::private_path(&child.path(),child.file_type()?.is_dir())?; }
|
|
let (ledger,ledger_file)=match &binding {Some(binding)=>{let (db,file)=open_ledger(&config.state_directory,binding)?;(Some(db),Some(file))},None=>(None,None)};
|
|
let clock=JobClock::new(config.enable_jobs);
|
|
#[cfg(feature = "system-helper")]
|
|
let clock = if let Some(policy) = &config.localservice_time_policy {
|
|
let binding = binding.clone().context("JOB_BINDING_REQUIRED")?;
|
|
JobClock { enabled: true, native: rmm_agent_windows::signed_clock::RuntimeClock::enrolled_local_service(binding,policy.clone()).for_generation(0) }
|
|
} else { clock };
|
|
let mut config=config;
|
|
if config.enable_jobs {
|
|
config.echo_scope=rmm_agent_windows::echo_probe::installed(journal.connection(),binding.as_ref().context("JOB_BINDING_REQUIRED")?)?;
|
|
}
|
|
if config.enable_jobs && config.echo_scope.is_some() {ensure!(config.windows_echo_probe_only, "WINDOWS_ECHO_PROFILE_REQUIRED");}
|
|
if config.windows_echo_probe_only {
|
|
ensure!(identity.agent_id==rmm_domain::windows_echo_probe::AGENT.to_string()
|
|
&& identity.asset_id==rmm_domain::windows_echo_probe::ASSET.to_string()
|
|
&& boot==rmm_agent_core::windows_sync_history::BOOT,"WINDOWS_ECHO_DEVICE");
|
|
#[cfg(feature="system-helper")]
|
|
if let Some(scope)=&config.echo_scope {
|
|
ensure!(rmm_agent_windows::echo_probe::protected_scope()? == *scope, "WINDOWS_ECHO_CONFIGURATION_CHANGED");
|
|
}
|
|
}
|
|
let sync=if config.enable_jobs && (!config.windows_echo_probe_only || config.echo_scope.is_some()) {binding.clone().map(|binding|reconcile::ClientSync::new(binding).with_helper_identity(reconcile::HelperIdentity::System))} else {None};
|
|
let (completed_tx,completed_rx)=tokio::sync::mpsc::channel(1);
|
|
let mut agent = Agent {
|
|
config,
|
|
reported_version: installed_version()?,
|
|
release_job_guard: None,
|
|
identity,
|
|
ca,
|
|
journal,
|
|
certificates,
|
|
key_files,
|
|
boot,
|
|
instance: Uuid::new_v4().to_string(),
|
|
started: Instant::now(), binding, ledger, _ledger_file:ledger_file, clock, sync,
|
|
#[cfg(feature="system-helper")]
|
|
time_refresh:None,
|
|
#[cfg(feature="system-helper")]
|
|
prepared_request:None,
|
|
#[cfg(feature="system-helper")]
|
|
helper_apply_unconfirmed:false,
|
|
#[cfg(feature="system-helper")]
|
|
helper_lane:Arc::new(tokio::sync::Semaphore::new(1)),
|
|
#[cfg(feature="system-helper")]
|
|
helper:None,
|
|
#[cfg(feature="system-helper")]
|
|
helper_probe:None,
|
|
#[cfg(feature="system-helper")]
|
|
helper_probe_at:Instant::now()-Duration::from_secs(30),
|
|
#[cfg(feature="system-helper")]
|
|
helper_control:None,
|
|
#[cfg(feature="system-helper")]
|
|
helper_anchor:None,
|
|
#[cfg(feature="system-helper")]
|
|
helper_checked:false,
|
|
#[cfg(feature="system-helper")]
|
|
helper_refused:false,
|
|
prepared:false,deferred_sync:None,
|
|
#[cfg(feature="system-helper")]
|
|
update_pending:false,
|
|
#[cfg(feature="system-helper")]
|
|
update_observer:crate::update_observation_runtime::Observer::default(),
|
|
#[cfg(feature="system-helper")]
|
|
update_health:None,
|
|
#[cfg(feature="system-helper")]
|
|
update_inventory_head:None,
|
|
fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
|
|
#[cfg(feature="system-helper")]
|
|
update_prefetch:None,
|
|
#[cfg(feature="system-helper")]
|
|
update_cached:None,
|
|
};
|
|
agent.recover_interrupted()?;
|
|
let mut backoff = 1;
|
|
while !stop.load(Ordering::SeqCst) || agent.running.is_some() {
|
|
if stop.load(Ordering::SeqCst) {
|
|
agent.close_gate();
|
|
if let Some(running)=&agent.running {running.cancel.store(true,Ordering::SeqCst);}
|
|
if let Ok(completion)=agent.completed_rx.try_recv(){agent.completed(0,completion)?;}
|
|
if agent.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
|
|
let running=agent.running.as_ref().context("RUNNING_REQUIRED")?;
|
|
agent.completed(0,(running.offer.clone(),running.grant.clone(),interrupted_executor(&running.grant)))?;
|
|
}
|
|
if agent.running.is_some() {tokio::select!{Some(completion)=agent.completed_rx.recv()=>agent.completed(0,completion)?,_=tokio::time::sleep(Duration::from_millis(250))=>{}}}
|
|
continue;
|
|
}
|
|
let start = Instant::now();
|
|
if let Err(error) = agent.session(stop).await {
|
|
record_session_error(&agent.config.state_directory, &error);
|
|
}
|
|
if start.elapsed() > Duration::from_secs(60) {
|
|
backoff = 1;
|
|
}
|
|
let delay = crate::https_channel::retry_delay(backoff);
|
|
backoff = (backoff + 1).min(6);
|
|
let until = Instant::now() + delay;
|
|
while Instant::now() < until && (!stop.load(Ordering::SeqCst) || agent.running.is_some()) {
|
|
if stop.load(Ordering::SeqCst) {
|
|
agent.close_gate();if let Some(running)=&agent.running {running.cancel.store(true,Ordering::SeqCst);}
|
|
}
|
|
if let Ok(completion)=agent.completed_rx.try_recv(){agent.completed(0,completion)?;}
|
|
if agent.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
|
|
let running=agent.running.as_ref().context("RUNNING_REQUIRED")?;
|
|
agent.completed(0,(running.offer.clone(),running.grant.clone(),interrupted_executor(&running.grant)))?;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
}
|
|
}
|
|
Ok(())
|
|
})
|
|
}
|
|
pub fn main() -> Result<()> {
|
|
let args: Vec<String> = std::env::args().skip(1).collect();
|
|
if args.first().map(String::as_str) == Some(rmm_agent_windows::event_log_job::MODE) {
|
|
ensure!(args.len() == 2, "EVENT_LOG_WORKER_ARGUMENTS");
|
|
return rmm_agent_windows::event_log_job::worker(Path::new(&args[1]));
|
|
}
|
|
if args == ["--release-capabilities"] {
|
|
println!("enrolled-localservice/1");
|
|
return Ok(());
|
|
}
|
|
if args == ["--help"] {
|
|
println!(
|
|
"Development Windows service. --service --config ABSOLUTE_PATH. Jobs disabled by default; require qualified clock, provisioned signing binding, signed Sync and WebUI-authorized grant. No approval bypass."
|
|
);
|
|
return Ok(());
|
|
}
|
|
if args == ["--native-probe"] {
|
|
let runtime = tokio::runtime::Runtime::new()?;
|
|
let boot = runtime.block_on(native::boot_id())?;
|
|
let inventory = native::snapshot()?;
|
|
println!(
|
|
"{}",
|
|
serde_json::json!({"platform":"windows","architecture":std::env::consts::ARCH,"hostname_available":!inventory["hostname"].as_str().unwrap_or("").is_empty(),"system_volume_available":inventory["root_available_bytes"].as_u64().is_some(),"inodes":null,"boot_id":boot,"capabilities":CAPABILITIES,"jobs_supported":false})
|
|
);
|
|
return Ok(());
|
|
}
|
|
ensure!(
|
|
args.first().map(String::as_str) == Some("--service"),
|
|
"UNKNOWN_ARGUMENT"
|
|
);
|
|
ensure!(
|
|
args.len() == 3 && args[1] == "--config",
|
|
"SERVICE_ARGUMENTS"
|
|
);
|
|
let config = Config::read(Path::new(&args[2]))?;
|
|
// Shared transport reads its one DNS identity. Set it before SCM dispatch
|
|
// creates its service thread and before any Tokio/collector threads exist.
|
|
unsafe {
|
|
std::env::set_var("RMM_SERVER_NAME", &config.server_name);
|
|
}
|
|
ensure!(
|
|
crate::https_channel::server_name()? == config.server_name,
|
|
"SERVER_NAME"
|
|
);
|
|
rmm_agent_windows::service::dispatch(service_run)
|
|
}
|
|
|
|
/// Operator diagnostics only: the bounded error chain of the last session end
|
|
/// in the agent's own private state directory. Never authority or retry input.
|
|
fn record_session_error(state: &Path, error: &anyhow::Error) {
|
|
let mut text = format!("{error:#}");
|
|
text.truncate(1024);
|
|
let value = serde_json::json!({"schema_version":1,"observed_at_utc":Utc::now(),"error":text});
|
|
let _ = std::fs::write(state.join("session-last-error.json"), value.to_string());
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
#[cfg(feature = "system-helper")]
|
|
#[tokio::test]
|
|
async fn gate_close_retains_the_original_helper_apply_worker_without_replaying_it() {
|
|
let (sender, receiver) = tokio::sync::oneshot::channel::<()>();
|
|
let task = tokio::spawn(async move {
|
|
receiver.await?;
|
|
Err(anyhow::anyhow!("original fixture reply"))
|
|
});
|
|
let kept = HelperControl::Apply(7, task)
|
|
.abort()
|
|
.expect("Apply must remain owned");
|
|
assert!(!kept.finished());
|
|
sender.send(()).unwrap();
|
|
let HelperControl::Apply(generation, task) = kept else {
|
|
panic!("wrong retained operation")
|
|
};
|
|
assert_eq!(generation, 7);
|
|
assert_eq!(
|
|
task.await.unwrap().err().unwrap().to_string(),
|
|
"original fixture reply"
|
|
);
|
|
}
|
|
|
|
#[cfg(feature = "system-helper")]
|
|
#[test]
|
|
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
|
|
let path = std::env::temp_dir().join(format!("windows-bound-unknown-{}", Uuid::new_v4()));
|
|
fs::create_dir(&path).unwrap();
|
|
let journal = Journal::open(&path).unwrap();
|
|
let db = journal.connection();
|
|
let job = Uuid::new_v4();
|
|
let attempt = Uuid::new_v4();
|
|
let grant = Uuid::new_v4();
|
|
let offer = JobOffer {
|
|
job_id: job.to_string(),
|
|
attempt_id: attempt.to_string(),
|
|
..Default::default()
|
|
};
|
|
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'unknown',1,?4)",rusqlite::params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),grant.to_string()]).unwrap();
|
|
let original = Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
payload: Some(Payload::JobResult(jobs::result(
|
|
&offer,
|
|
1,
|
|
&grant.to_string(),
|
|
Outcome::unknown("GENERIC_REPLY_LOST"),
|
|
))),
|
|
..Default::default()
|
|
};
|
|
jobs::event(db, &original).unwrap();
|
|
let mut actual = Outcome::unknown("SYSTEM_PREVIOUS_EFFECT_UNRESOLVED");
|
|
actual.evidence = serde_json::json!({"retained_helper_evidence":true});
|
|
let bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
|
job_id: job,
|
|
attempt_id: attempt,
|
|
grant_id: grant,
|
|
outcome: actual.clone(),
|
|
};
|
|
// Represents a crash after actual raw-helper outcome persistence but
|
|
// before the corresponding journal write, even if mirror receipt exists.
|
|
assert!(needs_bound_outcome(db, &bound).unwrap());
|
|
assert_eq!(jobs::events(db).unwrap().len(), 1);
|
|
let revision = jobs::transition(db, &offer, "unknown", &grant.to_string()).unwrap();
|
|
let repaired = Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
payload: Some(Payload::JobResult(jobs::result(
|
|
&offer,
|
|
revision,
|
|
&grant.to_string(),
|
|
actual,
|
|
))),
|
|
..Default::default()
|
|
};
|
|
jobs::event(db, &repaired).unwrap();
|
|
assert!(!needs_bound_outcome(db, &bound).unwrap());
|
|
assert_eq!(jobs::events(db).unwrap().len(), 2); // retention is never ACK
|
|
let mut terminal = bound.outcome.clone();
|
|
terminal.state = "failed".into();
|
|
let terminal_bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
|
job_id: job,
|
|
attempt_id: attempt,
|
|
grant_id: grant,
|
|
outcome: terminal.clone(),
|
|
};
|
|
let revision = jobs::transition(db, &offer, "failed", &grant.to_string()).unwrap();
|
|
jobs::event(
|
|
db,
|
|
&Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
payload: Some(Payload::JobResult(jobs::result(
|
|
&offer,
|
|
revision,
|
|
&grant.to_string(),
|
|
terminal,
|
|
))),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.unwrap();
|
|
// Two restarts after the terminal journal committed but before a
|
|
// mirror receipt may never request terminal -> terminal transitions.
|
|
assert!(!needs_bound_outcome(db, &terminal_bound).unwrap());
|
|
assert!(!needs_bound_outcome(db, &terminal_bound).unwrap());
|
|
assert_eq!(jobs::events(db).unwrap().len(), 3);
|
|
drop(journal);
|
|
fs::remove_dir_all(path).unwrap();
|
|
}
|
|
#[test]
|
|
fn old_passive_configuration_cannot_enable_jobs_or_read_the_clock() {
|
|
let config: Config = serde_json::from_value(passive_test_config()).unwrap();
|
|
assert!(!config.enable_jobs);
|
|
assert!(config.job_binding_file.is_none());
|
|
assert!(config.windows_execution_profile.is_none());
|
|
let clock = JobClock::new(config.enable_jobs);
|
|
assert_eq!(
|
|
clock.now().unwrap_err().to_string(),
|
|
"WINDOWS_JOBS_NOT_ENABLED"
|
|
);
|
|
}
|
|
fn passive_test_config() -> serde_json::Value {
|
|
serde_json::json!({
|
|
"server_name":"example.invalid","connect_address":"127.0.0.1:443",
|
|
"ca_file":"C:/fixture/ca.pem","state_directory":"C:/fixture/state",
|
|
"enrollment_token_file":"C:/fixture/token","transport":"auto"
|
|
})
|
|
}
|
|
#[test]
|
|
fn remote_signed_requires_explicit_profile_jobs_and_binding() {
|
|
let profile = rmm_domain::jobs::WindowsExecutionProfile::PowerShell51RemoteSigned;
|
|
let mut config: Config = serde_json::from_value(passive_test_config()).unwrap();
|
|
config.enable_jobs = true;
|
|
config.job_binding_file = Some("C:/fixture/binding.json".into());
|
|
assert!(!config.profile_enabled(profile));
|
|
config.windows_execution_profile = Some(profile);
|
|
assert_eq!(
|
|
config.profile_enabled(profile),
|
|
cfg!(feature = "system-helper")
|
|
);
|
|
config.enable_jobs = false;
|
|
assert!(!config.profile_enabled(profile));
|
|
config.enable_jobs = true;
|
|
config.job_binding_file = None;
|
|
assert!(!config.profile_enabled(profile));
|
|
|
|
let mut value = passive_test_config();
|
|
value["windows_execution_profile"] = serde_json::json!("powershell_51_remote_signed");
|
|
assert_eq!(
|
|
serde_json::from_value::<Config>(value.clone())
|
|
.unwrap()
|
|
.windows_execution_profile,
|
|
Some(profile)
|
|
);
|
|
for invalid in [
|
|
serde_json::Value::Null,
|
|
serde_json::json!(true),
|
|
serde_json::json!("Bypass"),
|
|
] {
|
|
value["windows_execution_profile"] = invalid;
|
|
assert!(serde_json::from_value::<Config>(value.clone()).is_err());
|
|
}
|
|
}
|
|
#[test]
|
|
fn legacy_inventory_keeps_original_bytes_until_concrete_core_ack() {
|
|
let directory =
|
|
std::env::temp_dir().join(format!("windows-runtime-import-{}", Uuid::new_v4()));
|
|
fs::create_dir(&directory).unwrap();
|
|
let mut journal = Journal::open(&directory).unwrap();
|
|
journal.pin_identity("retained").unwrap();
|
|
let sequence = journal.next_sequence().unwrap();
|
|
let envelope = Envelope {
|
|
message_id: Uuid::new_v4().to_string(),
|
|
sequence,
|
|
payload: Some(Payload::InventorySnapshot(InventorySnapshot::default())),
|
|
..Default::default()
|
|
};
|
|
journal.save(&envelope).unwrap();
|
|
let mut original = envelope.encode_to_vec();
|
|
original.extend([0xf8, 0x07, 0x01]); // Unknown additive protobuf field must remain byte-exact.
|
|
journal
|
|
.connection()
|
|
.execute(
|
|
"UPDATE events SET payload=?1 WHERE id=?2",
|
|
rusqlite::params![&original, &envelope.message_id],
|
|
)
|
|
.unwrap();
|
|
import_legacy_inventory(&mut journal).unwrap();
|
|
for table in ["events", "job_events"] {
|
|
let bytes: Vec<u8> = journal
|
|
.connection()
|
|
.query_row(
|
|
&format!("SELECT payload FROM {table} WHERE id=?1"),
|
|
[&envelope.message_id],
|
|
|r| r.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(bytes, original);
|
|
}
|
|
jobs::ack(journal.connection_mut(), &Uuid::new_v4().to_string()).unwrap();
|
|
assert_eq!(journal.count().unwrap(), 1);
|
|
import_legacy_inventory(&mut journal).unwrap(); // Exact same-ID copy is idempotent.
|
|
assert_eq!(journal.next_sequence().unwrap(), sequence + 1);
|
|
jobs::ack(journal.connection_mut(), &envelope.message_id).unwrap();
|
|
assert_eq!(journal.count().unwrap(), 0);
|
|
assert!(jobs::events(journal.connection()).unwrap().is_empty());
|
|
drop(journal);
|
|
fs::remove_dir_all(directory).unwrap();
|
|
}
|
|
#[test]
|
|
fn worker_fence_requires_same_generation_and_entire_clock_interval() {
|
|
struct Clock(chrono::DateTime<Utc>);
|
|
impl SyncClock for Clock {
|
|
fn now(&self) -> Result<chrono::DateTime<Utc>> {
|
|
Ok(self.0)
|
|
}
|
|
}
|
|
let now = Utc::now();
|
|
let fence = EffectFence::default();
|
|
fence.deadline.store(
|
|
(now + chrono::Duration::seconds(3)).timestamp_millis(),
|
|
Ordering::SeqCst,
|
|
);
|
|
assert!(!fence.allows(1, &Clock(now)));
|
|
fence.generation.store(1, Ordering::SeqCst);
|
|
assert!(fence.allows(1, &Clock(now)));
|
|
assert!(!fence.allows_identity(1, true, &Clock(now)));
|
|
fence.helper_generation.store(1, Ordering::SeqCst);
|
|
assert!(fence.allows_identity(1, true, &Clock(now)));
|
|
assert!(!fence.allows(2, &Clock(now)));
|
|
assert!(fence.allows(1, &Clock(now + chrono::Duration::seconds(1))));
|
|
assert!(!fence.allows(1, &Clock(now + chrono::Duration::seconds(3))));
|
|
fence.close();
|
|
assert!(!fence.allows_identity(1, true, &Clock(now)));
|
|
assert!(!fence.allows(1, &Clock(now)));
|
|
}
|
|
}
|