Preserve Windows runtime state and qualify native guarded adapters
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -1,14 +1,25 @@
|
||||
use anyhow::{Result, ensure};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use prost::Message;
|
||||
use rcgen::{CertificateParams, DnType, KeyPair};
|
||||
use rmm_agent_core::{
|
||||
auth::{Binding, Outcome, Request},
|
||||
jobs, ledger,
|
||||
reconcile::{self, SyncClock},
|
||||
};
|
||||
use rmm_agent_windows::{CAPABILITIES, journal::Journal, native};
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload};
|
||||
use rmm_protocol::*;
|
||||
use rusqlite::{Connection, OptionalExtension};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
@@ -22,6 +33,11 @@ struct Config {
|
||||
state_directory: PathBuf,
|
||||
enrollment_token_file: PathBuf,
|
||||
transport: String,
|
||||
#[serde(default)]
|
||||
job_binding_file: Option<PathBuf>,
|
||||
// False until the independent LocalService clock qualification is complete.
|
||||
#[serde(default)]
|
||||
enable_jobs: bool,
|
||||
}
|
||||
fn create_private(path: &Path, bytes: &[u8]) -> Result<()> {
|
||||
use std::io::Write;
|
||||
@@ -105,6 +121,222 @@ async fn enrollment(config: &Config, ca: &[u8]) -> Result<EnrollmentResponse> {
|
||||
fs::remove_file(&config.enrollment_token_file)?;
|
||||
Ok(identity)
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct JobClock {
|
||||
native: Arc<rmm_agent_windows::clock::NativeClock>,
|
||||
enabled: bool,
|
||||
}
|
||||
impl SyncClock for JobClock {
|
||||
fn now(&self) -> Result<chrono::DateTime<Utc>> {
|
||||
ensure!(self.enabled, "WINDOWS_JOBS_NOT_ENABLED");
|
||||
self.native.now()
|
||||
}
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
generation: AtomicU64,
|
||||
deadline: AtomicI64,
|
||||
}
|
||||
impl EffectFence {
|
||||
fn close(&self) {
|
||||
self.generation.store(0, Ordering::SeqCst);
|
||||
}
|
||||
fn allows(&self, generation: u64, clock: &impl SyncClock) -> bool {
|
||||
self.generation.load(Ordering::SeqCst) == generation
|
||||
&& generation > 0
|
||||
&& clock.now().is_ok_and(|now| {
|
||||
(now + chrono::Duration::seconds(2)).timestamp_millis()
|
||||
< self.deadline.load(Ordering::SeqCst)
|
||||
})
|
||||
}
|
||||
}
|
||||
type Completion = (JobOffer, ExecutionPermitPayload, Outcome);
|
||||
struct Running {
|
||||
offer: JobOffer,
|
||||
grant: ExecutionPermitPayload,
|
||||
cancel: Arc<AtomicBool>,
|
||||
worker: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
fn verified_window(
|
||||
binding: &Binding,
|
||||
request: &Request,
|
||||
clock: &impl SyncClock,
|
||||
) -> Result<(rmm_domain::jobs::JobPayload, ExecutionPermitPayload)> {
|
||||
let now = clock.now()?;
|
||||
// The entire qualified <=2s clock interval must lie inside the grant window.
|
||||
let result = binding.verify(request, now - chrono::Duration::seconds(2))?;
|
||||
binding.verify(request, now + chrono::Duration::seconds(2))?;
|
||||
Ok(result)
|
||||
}
|
||||
fn readonly_binding(path: &Path) -> Result<Binding> {
|
||||
use std::ptr;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::LocalFree,
|
||||
Security::{
|
||||
self, ACCESS_ALLOWED_ACE, ACL,
|
||||
Authorization::{ConvertSidToStringSidW, GetNamedSecurityInfoW, SE_FILE_OBJECT},
|
||||
DACL_SECURITY_INFORMATION, OWNER_SECURITY_INFORMATION,
|
||||
},
|
||||
};
|
||||
unsafe fn sid(sid: Security::PSID) -> Result<String> {
|
||||
let mut raw = ptr::null_mut();
|
||||
ensure!(
|
||||
unsafe { ConvertSidToStringSidW(sid, &mut raw) } != 0,
|
||||
"BINDING_SID"
|
||||
);
|
||||
let mut len = 0;
|
||||
while unsafe { *raw.add(len) } != 0 {
|
||||
len += 1;
|
||||
ensure!(len < 256, "BINDING_SID_LIMIT");
|
||||
}
|
||||
let value = String::from_utf16(unsafe { std::slice::from_raw_parts(raw, len) })?;
|
||||
unsafe {
|
||||
LocalFree(raw.cast());
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
ensure!(path.is_absolute(), "ABSOLUTE_JOB_BINDING_REQUIRED");
|
||||
// The immediate configuration directory must also deny agent replacement.
|
||||
for (item, directory) in [
|
||||
(path, false),
|
||||
(path.parent().context("BINDING_PARENT")?, true),
|
||||
] {
|
||||
native::private_path(item, directory)?;
|
||||
let mut owner = ptr::null_mut();
|
||||
let mut dacl: *mut ACL = ptr::null_mut();
|
||||
let mut descriptor = ptr::null_mut();
|
||||
let status = unsafe {
|
||||
GetNamedSecurityInfoW(
|
||||
native::wide(item).as_ptr(),
|
||||
SE_FILE_OBJECT,
|
||||
OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION,
|
||||
&mut owner,
|
||||
ptr::null_mut(),
|
||||
&mut dacl,
|
||||
ptr::null_mut(),
|
||||
&mut descriptor,
|
||||
)
|
||||
};
|
||||
ensure!(status == 0, "BINDING_SECURITY");
|
||||
let check = (|| -> Result<()> {
|
||||
ensure!(!owner.is_null() && !dacl.is_null(), "BINDING_NULL_SECURITY");
|
||||
ensure!(
|
||||
matches!(unsafe { sid(owner) }?.as_str(), "S-1-5-18" | "S-1-5-32-544"),
|
||||
"BINDING_ADMIN_OWNER_REQUIRED"
|
||||
);
|
||||
for index in 0..unsafe { (*dacl).AceCount } {
|
||||
let mut raw = ptr::null_mut();
|
||||
ensure!(
|
||||
unsafe { Security::GetAce(dacl, u32::from(index), &mut raw) } != 0,
|
||||
"BINDING_ACE"
|
||||
);
|
||||
let ace = unsafe { &*raw.cast::<ACCESS_ALLOWED_ACE>() };
|
||||
ensure!(
|
||||
ace.Header.AceType == 0
|
||||
&& usize::from(ace.Header.AceSize)
|
||||
>= std::mem::size_of::<ACCESS_ALLOWED_ACE>(),
|
||||
"BINDING_ACE_TYPE"
|
||||
);
|
||||
let who = unsafe { sid(ptr::addr_of!(ace.SidStart).cast_mut().cast()) }?;
|
||||
if !matches!(who.as_str(), "S-1-5-18" | "S-1-5-32-544") {
|
||||
// Generic write/all, owner/DACL/delete and all file/directory mutation bits.
|
||||
ensure!(ace.Mask & 0x500d_0156 == 0, "BINDING_AGENT_WRITE_FORBIDDEN");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
unsafe {
|
||||
LocalFree(descriptor.cast());
|
||||
}
|
||||
check?;
|
||||
}
|
||||
let binding: Binding = serde_json::from_slice(&protected_read(path, 8192)?)?;
|
||||
ensure!(
|
||||
!binding.tenant_id.is_nil()
|
||||
&& !binding.agent_id.is_nil()
|
||||
&& !binding.asset_id.is_nil()
|
||||
&& binding.enrollment_generation > 0
|
||||
&& binding.assignment_version > 0
|
||||
&& binding.public_key_hex.len() == 64
|
||||
&& binding
|
||||
.public_key_hex
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_hexdigit()),
|
||||
"JOB_BINDING_PROFILE"
|
||||
);
|
||||
Ok(binding)
|
||||
}
|
||||
fn import_legacy_inventory(journal: &mut Journal) -> Result<()> {
|
||||
let original = {
|
||||
let mut q = journal
|
||||
.connection()
|
||||
.prepare("SELECT id,payload FROM events ORDER BY rowid")?;
|
||||
q.query_map([], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, Vec<u8>>(1)?))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let tx = journal.connection_mut().transaction()?;
|
||||
for (id, bytes) in original {
|
||||
let envelope = Envelope::decode(bytes.as_slice())?;
|
||||
ensure!(
|
||||
envelope.message_id == id
|
||||
&& matches!(envelope.payload, Some(Payload::InventorySnapshot(_))),
|
||||
"LEGACY_INVENTORY_BINDING"
|
||||
);
|
||||
let old: Option<Vec<u8>> = tx
|
||||
.query_row("SELECT payload FROM job_events WHERE id=?1", [&id], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.optional()?;
|
||||
ensure!(
|
||||
old.as_ref().is_none_or(|old| old == &bytes),
|
||||
"LEGACY_INVENTORY_CONFLICT"
|
||||
);
|
||||
// Copy original bytes/ID, never decode/re-encode or discard pending telemetry.
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO job_events(id,payload) VALUES(?1,?2)",
|
||||
rusqlite::params![id, bytes],
|
||||
)?;
|
||||
}
|
||||
// Core ACK/Sync deletion and the old queue advance in the SAME transaction.
|
||||
tx.execute_batch("CREATE TRIGGER IF NOT EXISTS windows_legacy_inventory_ack AFTER DELETE ON job_events BEGIN DELETE FROM events WHERE id=OLD.id AND payload=OLD.payload; END;")?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn open_ledger(directory: &Path, binding: &Binding) -> Result<(Connection, fs::File)> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
let path = directory.join("consumption.sqlite");
|
||||
let file = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
|
||||
.open(&path)?;
|
||||
native::private_path(&path, false)?;
|
||||
let db = Connection::open(&path)?;
|
||||
ledger::initialize_ledger(&db)?;
|
||||
db.execute_batch("CREATE TABLE IF NOT EXISTS windows_binding(identity TEXT NOT NULL);")?;
|
||||
let identity = format!(
|
||||
"{}:{}:{}:{}:{}",
|
||||
binding.tenant_id,
|
||||
binding.asset_id,
|
||||
binding.agent_id,
|
||||
binding.enrollment_generation,
|
||||
binding.assignment_version
|
||||
);
|
||||
let old: Option<String> = db
|
||||
.query_row("SELECT identity FROM windows_binding", [], |r| r.get(0))
|
||||
.optional()?;
|
||||
if let Some(old) = old {
|
||||
ensure!(old == identity, "LEDGER_IDENTITY_MISMATCH");
|
||||
} else {
|
||||
db.execute("INSERT INTO windows_binding VALUES(?1)", [identity])?;
|
||||
}
|
||||
Ok((db, file))
|
||||
}
|
||||
struct Agent {
|
||||
config: Config,
|
||||
identity: EnrollmentResponse,
|
||||
@@ -113,15 +345,347 @@ struct Agent {
|
||||
boot: String,
|
||||
instance: String,
|
||||
started: Instant,
|
||||
binding: Option<Binding>,
|
||||
ledger: Option<Connection>,
|
||||
_ledger_file: Option<fs::File>,
|
||||
clock: JobClock,
|
||||
sync: Option<reconcile::ClientSync>,
|
||||
prepared: bool,
|
||||
fence: Arc<EffectFence>,
|
||||
completed_tx: tokio::sync::mpsc::Sender<Completion>,
|
||||
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
||||
running: Option<Running>,
|
||||
deferred_permit: Option<(u64, ExecutionPermit)>,
|
||||
}
|
||||
impl Agent {
|
||||
fn close_gate(&mut self) {
|
||||
self.fence.close();
|
||||
if let Some(sync) = &mut self.sync {
|
||||
sync.close();
|
||||
}
|
||||
self.prepared = false;
|
||||
self.deferred_permit = None;
|
||||
}
|
||||
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
|
||||
Ok(reconcile::Channel {
|
||||
connection_generation: generation,
|
||||
boot_id: Uuid::parse_str(&self.boot)?,
|
||||
})
|
||||
}
|
||||
fn ready(&self, generation: u64) -> bool {
|
||||
self.config.enable_jobs
|
||||
&& self.channel(generation).is_ok_and(|channel| {
|
||||
self.sync
|
||||
.as_ref()
|
||||
.is_some_and(|sync| sync.can_start(&channel, false, &self.clock))
|
||||
})
|
||||
}
|
||||
fn capabilities(&self) -> Vec<String> {
|
||||
let mut caps: Vec<String> = CAPABILITIES.iter().map(|c| c.to_string()).collect();
|
||||
if self.config.enable_jobs
|
||||
&& self.binding.is_some()
|
||||
&& native::require_local_service().is_ok()
|
||||
&& self.clock.now().is_ok()
|
||||
{
|
||||
caps.extend(["rmm.sync.v1", "refresh_inventory"].map(str::to_string));
|
||||
if rmm_agent_windows::actions::powershell().is_ok() {
|
||||
caps.extend(["run_script", "script.powershell/1"].map(str::to_string));
|
||||
}
|
||||
}
|
||||
caps
|
||||
}
|
||||
fn record(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
outcome: Outcome,
|
||||
) -> Result<()> {
|
||||
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
|
||||
let tx = self.journal.connection_mut().transaction()?;
|
||||
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
||||
envelope.payload = Some(Payload::JobResult(jobs::result(
|
||||
offer, revision, grant, outcome,
|
||||
)));
|
||||
jobs::event(&tx, &envelope)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
fn recover_interrupted(&mut self) -> Result<()> {
|
||||
let Some(binding) = self.binding.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
let interrupted = jobs::interrupted(
|
||||
self.journal.connection(),
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&binding,
|
||||
)?;
|
||||
for (offer, grant) in interrupted {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(self.journal.connection(), &offer)?;
|
||||
} else {
|
||||
let outcome = jobs::recover_interrupted_outcome(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&binding,
|
||||
&offer,
|
||||
&grant,
|
||||
)?
|
||||
.unwrap_or_else(|| Outcome::unknown("AGENT_PROCESS_INTERRUPTED"));
|
||||
// Replay original durable evidence only. Normal signed Sync,
|
||||
// never startup, releases the previously consumed attempt.
|
||||
self.record(0, &offer, &grant, outcome)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn replay(
|
||||
&mut self,
|
||||
channel: &crate::wire_channel::Handle,
|
||||
generation: u64,
|
||||
) -> Result<()> {
|
||||
let events = jobs::events(self.journal.connection())?;
|
||||
let passive = self.journal.events()?;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
for mut event in events.into_iter().chain(passive).take(8) {
|
||||
event.connection_generation = generation;
|
||||
channel.send(&event).await?;
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
async fn start_sync(
|
||||
&mut self,
|
||||
channel: &crate::wire_channel::Handle,
|
||||
generation: u64,
|
||||
) -> Result<()> {
|
||||
self.fence.close();
|
||||
let context = self.channel(generation)?;
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
let prepared = sync.prepare(
|
||||
self.journal.connection_mut(),
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
context,
|
||||
None,
|
||||
&self.clock,
|
||||
)?;
|
||||
self.prepared = true;
|
||||
channel.send(&prepared.envelope).await?;
|
||||
Ok(())
|
||||
}
|
||||
fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
||||
ensure!(self.prepared, "UNSOLICITED_SYNC_RESPONSE");
|
||||
self.prepared = false;
|
||||
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
|
||||
let applied = sync.apply(self.journal.connection_mut(), &response, &self.clock)?;
|
||||
ensure!(
|
||||
applied.helper_instruction.is_none(),
|
||||
"WINDOWS_HELPER_NOT_CONFIGURED"
|
||||
);
|
||||
sync.apply_consumption(
|
||||
self.ledger.as_mut().context("LEDGER_REQUIRED")?,
|
||||
&self.clock,
|
||||
)?;
|
||||
if self.ready(generation) {
|
||||
let deadline = self
|
||||
.sync
|
||||
.as_ref()
|
||||
.and_then(|sync| sync.validity_deadline())
|
||||
.context("SIGNED_LEASE_REQUIRED")?;
|
||||
self.fence
|
||||
.deadline
|
||||
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
||||
self.fence.generation.store(generation, Ordering::SeqCst);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn completed(&mut self, generation: u64, (offer, grant, outcome): Completion) -> Result<()> {
|
||||
let running = self
|
||||
.running
|
||||
.as_ref()
|
||||
.context("UNSOLICITED_EXECUTOR_COMPLETION")?;
|
||||
ensure!(
|
||||
running.offer.job_id == offer.job_id
|
||||
&& running.offer.attempt_id == offer.attempt_id
|
||||
&& running.grant == grant,
|
||||
"EXECUTOR_COMPLETION_BINDING"
|
||||
);
|
||||
ledger::persist_outcome(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&grant,
|
||||
&outcome,
|
||||
)?;
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
&grant.grant_id.to_string(),
|
||||
outcome.clone(),
|
||||
)?;
|
||||
ledger::finish(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&grant,
|
||||
&outcome,
|
||||
)?;
|
||||
self.running = None;
|
||||
Ok(())
|
||||
}
|
||||
fn permit(&mut self, generation: u64, permit: ExecutionPermit) -> Result<()> {
|
||||
if let Some((old_generation, old)) = &self.deferred_permit {
|
||||
ensure!(
|
||||
*old_generation == generation && *old == permit,
|
||||
"PENDING_PERMIT_CONFLICT"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let offer = jobs::requesting(self.journal.connection())?.context("UNSOLICITED_PERMIT")?;
|
||||
if !permit.denial_code.is_empty() {
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
"",
|
||||
Outcome::unknown("PERMIT_DENIED_RECONCILIATION_REQUIRED"),
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
self.ready(generation) && self.running.is_none(),
|
||||
"FENCED_SYNC_REQUIRED"
|
||||
);
|
||||
let binding = self
|
||||
.binding
|
||||
.as_ref()
|
||||
.context("JOB_BINDING_REQUIRED")?
|
||||
.clone();
|
||||
let request = jobs::permit_request(&offer, &permit)?;
|
||||
let now = self.clock.now()?;
|
||||
// Authenticate before interpreting timing. A freshly issued grant may
|
||||
// need two seconds before the full uncertainty interval is inside it.
|
||||
let (_, authenticated) = binding.verify(&request, now + chrono::Duration::seconds(2))?;
|
||||
if now - chrono::Duration::seconds(2) < authenticated.not_before {
|
||||
self.deferred_permit = Some((generation, permit));
|
||||
return Ok(());
|
||||
}
|
||||
let (job, grant) = verified_window(&binding, &request, &self.clock)?;
|
||||
ensure!(
|
||||
rmm_agent_windows::actions::supported(&job),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
);
|
||||
let started_at = self.clock.now()?;
|
||||
let ledger = self.ledger.as_mut().context("LEDGER_REQUIRED")?;
|
||||
ledger::begin_execution(
|
||||
ledger,
|
||||
&grant,
|
||||
&request,
|
||||
Uuid::parse_str(&self.boot)?,
|
||||
started_at,
|
||||
)?;
|
||||
let cancelled: bool = self.journal.connection().query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if cancelled {
|
||||
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
||||
ledger::persist_outcome(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&grant,
|
||||
&outcome,
|
||||
)?;
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
&grant.grant_id.to_string(),
|
||||
outcome.clone(),
|
||||
)?;
|
||||
ledger::finish(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&grant,
|
||||
&outcome,
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut started = self.envelope(
|
||||
generation,
|
||||
Payload::JobStateChanged(JobStateChanged::default()),
|
||||
)?;
|
||||
let tx = self.journal.connection_mut().transaction()?;
|
||||
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
|
||||
started.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
||||
job_id: offer.job_id.clone(),
|
||||
attempt_id: offer.attempt_id.clone(),
|
||||
revision,
|
||||
state: "running".into(),
|
||||
grant_id: grant.grant_id.to_string(),
|
||||
}));
|
||||
jobs::event(&tx, &started)?;
|
||||
tx.commit()?;
|
||||
let cancelled = Arc::new(AtomicBool::new(false));
|
||||
let worker_cancel = cancelled.clone();
|
||||
let clock = self.clock.clone();
|
||||
let fence = self.fence.clone();
|
||||
let completed = self.completed_tx.clone();
|
||||
let directory = self.config.state_directory.join("jobs");
|
||||
let work_offer = offer.clone();
|
||||
let work_grant = grant.clone();
|
||||
let worker = tokio::task::spawn_blocking(move || {
|
||||
let outcome = (|| -> Result<Outcome> {
|
||||
if !fence.allows(generation, &clock)
|
||||
|| verified_window(&binding, &request, &clock).is_err()
|
||||
{
|
||||
return Ok(jobs::no_start(
|
||||
"failed",
|
||||
"START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT",
|
||||
));
|
||||
}
|
||||
let script = if matches!(job.action, Action::RunScript { .. }) {
|
||||
match rmm_agent_windows::actions::Script::prepare(
|
||||
&job,
|
||||
&directory,
|
||||
work_grant.grant_id,
|
||||
) {
|
||||
Ok(script) => Some(script),
|
||||
Err(_) => return Ok(jobs::no_start("failed", "SCRIPT_PREPARATION_FAILED")),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// File creation, pool scheduling and fsync never extend a start window.
|
||||
if worker_cancel.load(Ordering::SeqCst) {
|
||||
return Ok(jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT"));
|
||||
}
|
||||
if !fence.allows(generation, &clock)
|
||||
|| verified_window(&binding, &request, &clock).is_err()
|
||||
{
|
||||
return Ok(jobs::no_start(
|
||||
"failed",
|
||||
"START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT",
|
||||
));
|
||||
}
|
||||
if let Some(script) = script {
|
||||
script.run(&job, &worker_cancel)
|
||||
} else {
|
||||
rmm_agent_windows::actions::inventory(&job)
|
||||
}
|
||||
})()
|
||||
.unwrap_or_else(|_| Outcome::unknown("WINDOWS_EXECUTOR_EFFECT_UNKNOWN"));
|
||||
let _ = completed.blocking_send((work_offer, work_grant, outcome));
|
||||
});
|
||||
self.running = Some(Running {
|
||||
offer,
|
||||
grant,
|
||||
cancel: cancelled,
|
||||
worker,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
|
||||
Ok(Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: self.identity.agent_id.clone(),
|
||||
enrollment_generation: 1,
|
||||
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
connection_generation: generation,
|
||||
boot_id: self.boot.clone(),
|
||||
sequence: self.journal.next_sequence()?,
|
||||
@@ -130,7 +694,7 @@ impl Agent {
|
||||
})
|
||||
}
|
||||
fn queue_inventory(&mut self, generation: u64) -> Result<()> {
|
||||
if self.journal.count()? > 0 {
|
||||
if self.journal.count()? > 0 || !jobs::events(self.journal.connection())?.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let data = serde_json::to_vec(&native::snapshot()?)?;
|
||||
@@ -148,14 +712,26 @@ impl Agent {
|
||||
completeness: "partial".into(),
|
||||
}),
|
||||
)?;
|
||||
self.journal.save(&envelope)
|
||||
if self.binding.is_some() {
|
||||
jobs::event(self.journal.connection(), &envelope)
|
||||
} else {
|
||||
self.journal.save(&envelope)
|
||||
}
|
||||
}
|
||||
async fn session(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||
self.close_gate();
|
||||
let result = self.session_inner(stop).await;
|
||||
self.close_gate();
|
||||
result
|
||||
}
|
||||
async fn session_inner(&mut self, stop: &AtomicBool) -> Result<()> {
|
||||
self.boot = native::boot_id().await?;
|
||||
let advertised = self.capabilities();
|
||||
let hello = self.envelope(
|
||||
0,
|
||||
Payload::ClientHello(ClientHello {
|
||||
agent_version: env!("CARGO_PKG_VERSION").into(),
|
||||
capabilities: CAPABILITIES.iter().map(|v| v.to_string()).collect(),
|
||||
capabilities: advertised.clone(),
|
||||
}),
|
||||
)?;
|
||||
let identity = crate::certificate_rotation::DeviceIdentity {
|
||||
@@ -180,7 +756,8 @@ impl Agent {
|
||||
let reply = tokio::time::timeout(Duration::from_secs(20), channel.receive()).await??;
|
||||
ensure!(
|
||||
reply.agent_id == self.identity.agent_id
|
||||
&& reply.enrollment_generation == 1
|
||||
&& reply.enrollment_generation
|
||||
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation)
|
||||
&& reply.protocol_major == 1
|
||||
&& reply.protocol_minor == 0,
|
||||
"SERVER_HELLO_BINDING"
|
||||
@@ -194,26 +771,96 @@ impl Agent {
|
||||
"SERVER_LIMITS"
|
||||
);
|
||||
self.queue_inventory(generation)?;
|
||||
self.replay(&channel, generation).await?;
|
||||
let mut needs_sync = self.sync.is_some();
|
||||
if self.sync.is_some() && self.running.is_none() {
|
||||
if self.start_sync(&channel, generation).await.is_err() {
|
||||
self.close_gate();
|
||||
} else {
|
||||
needs_sync = false;
|
||||
}
|
||||
}
|
||||
let mut sync_at = Instant::now();
|
||||
let mut capabilities_at = Instant::now();
|
||||
let mut replay = Instant::now() - Duration::from_secs(10);
|
||||
let mut inventory = Instant::now();
|
||||
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
||||
let mut heartbeat_after = Duration::from_secs(30);
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(1));
|
||||
while !stop.load(Ordering::SeqCst) {
|
||||
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
self.close_gate();
|
||||
if let Some(running) = &self.running {
|
||||
running.cancel.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
tokio::select! {
|
||||
_=tick.tick()=>{},
|
||||
Some(completion)=self.completed_rx.recv()=>{self.completed(generation,completion)?;needs_sync=true;},
|
||||
_=tick.tick()=>{
|
||||
if !stop.load(Ordering::SeqCst) && let Some((pending_generation,permit))=self.deferred_permit.take() {
|
||||
ensure!(pending_generation==generation,"PENDING_PERMIT_GENERATION");self.permit(generation,permit)?;
|
||||
}
|
||||
if self.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
|
||||
if let Ok(completion)=self.completed_rx.try_recv(){self.completed(generation,completion)?;}
|
||||
else {let running=self.running.as_ref().context("RUNNING_REQUIRED")?;self.completed(generation,(running.offer.clone(),running.grant.clone(),Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED")))?;}
|
||||
needs_sync=true;
|
||||
}
|
||||
let busy=self.running.is_some() || self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?;
|
||||
if !stop.load(Ordering::SeqCst) && !busy {
|
||||
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
|
||||
if self.prepared && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.sync.is_some() && !self.prepared && (needs_sync || sync_at.elapsed()>=Duration::from_secs(40) || !self.ready(generation)) {
|
||||
if self.start_sync(&channel,generation).await.is_err(){self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if !needs_sync && !self.prepared && self.ready(generation) && jobs::events(self.journal.connection())?.is_empty() && self.journal.count()?==0 {
|
||||
let now=self.clock.now()?;
|
||||
if let Some(offer)=jobs::next_at(self.journal.connection(),now)? {
|
||||
let job=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.job(&offer.payload_bytes)?;
|
||||
if now>=job.start_before {self.record(generation,&offer,"",jobs::no_start("expired","START_DEADLINE_EXPIRED"))?;needs_sync=true;}
|
||||
else if now-chrono::Duration::seconds(2)>=job.not_before {
|
||||
jobs::reserve_request(self.journal.connection(),&offer)?;
|
||||
let request=self.envelope(generation,Payload::ExecutionPermitRequest(ExecutionPermitRequest{job_id:offer.job_id,attempt_id:offer.attempt_id,job_payload_hash:offer.job_payload_hash,assignment_version:job.assignment_version}))?;
|
||||
channel.send(&request).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
message=channel.receive()=>{
|
||||
let message=message?;
|
||||
ensure!(message.agent_id==self.identity.agent_id && message.enrollment_generation==1 && message.protocol_major==1 && message.protocol_minor==0 && message.connection_generation==generation,"SERVER_ENVELOPE_BINDING");
|
||||
ensure!(message.agent_id==self.identity.agent_id && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.protocol_major==1 && message.protocol_minor==0 && message.connection_generation==generation,"SERVER_ENVELOPE_BINDING");
|
||||
match message.payload {
|
||||
Some(Payload::Ack(ack))=>self.journal.ack(&ack.message_id)?,
|
||||
// There is deliberately no acceptance/start path. A stale
|
||||
// or malicious offer cannot activate an unimplemented job.
|
||||
Some(Payload::JobOffer(_)|Payload::ExecutionPermit(_))=>anyhow::bail!("UNSUPPORTED_EXECUTION_CAPABILITY"),
|
||||
Some(Payload::Ack(ack))=>{
|
||||
let pending:Option<Vec<u8>>=self.journal.connection().query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
|
||||
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
|
||||
self.journal.ack(&ack.message_id)?;jobs::ack(self.journal.connection_mut(),&ack.message_id)?;
|
||||
},
|
||||
Some(Payload::SyncResponse(response))=>{if self.apply_sync(generation,response).is_err(){self.close_gate();needs_sync=true;}sync_at=Instant::now();},
|
||||
Some(Payload::JobOffer(offer))=>{
|
||||
ensure!(self.config.enable_jobs,"UNSUPPORTED_EXECUTION_CAPABILITY");
|
||||
let binding=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.clone();
|
||||
let accepted=self.envelope(generation,Payload::JobAccepted(JobAccepted{job_id:offer.job_id.clone(),attempt_id:offer.attempt_id.clone(),job_payload_hash:offer.job_payload_hash.clone(),journal_proof:rmm_domain::jobs::hash(&offer.encode_to_vec())}))?;
|
||||
let tx=self.journal.connection_mut().transaction()?;
|
||||
let fresh=jobs::accept(&tx,&binding,&offer,rmm_agent_windows::actions::supported)?;
|
||||
jobs::event(&tx,&accepted)?;tx.commit()?;needs_sync|=fresh;
|
||||
},
|
||||
Some(Payload::ExecutionPermit(permit))=>self.permit(generation,permit)?,
|
||||
Some(Payload::CancelRequest(cancel))=>{
|
||||
let row:Option<(Vec<u8>,String,String)>=self.journal.connection().query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",rusqlite::params![cancel.job_id,cancel.attempt_id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?;
|
||||
if let Some((bytes,state,grant))=row {
|
||||
self.journal.connection().execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2",rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
||||
if let Some(running)=&self.running && running.offer.job_id==cancel.job_id && running.offer.attempt_id==cancel.attempt_id {running.cancel.store(true,Ordering::SeqCst);}
|
||||
if state=="accepted" {self.record(generation,&JobOffer::decode(bytes.as_slice())?,&grant,jobs::no_start("cancelled","CANCELLED_BEFORE_START"))?;needs_sync=true;}
|
||||
}
|
||||
let ack=self.envelope(generation,Payload::Ack(Ack{message_id:message.message_id}))?;channel.send(&ack).await?;
|
||||
},
|
||||
_=>anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE")
|
||||
}
|
||||
}
|
||||
}
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
continue;
|
||||
}
|
||||
if heartbeat.elapsed() >= heartbeat_after {
|
||||
let payload = Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
@@ -235,13 +882,10 @@ impl Agent {
|
||||
heartbeat_after = Duration::from_millis(rand::random_range(27000..33000));
|
||||
}
|
||||
if replay.elapsed() >= Duration::from_secs(5) {
|
||||
for mut event in self.journal.events()? {
|
||||
event.connection_generation = generation;
|
||||
channel.send(&event).await?;
|
||||
}
|
||||
self.replay(&channel, generation).await?;
|
||||
replay = Instant::now();
|
||||
}
|
||||
if inventory.elapsed() >= Duration::from_secs(86400) {
|
||||
if !stop.load(Ordering::SeqCst) && inventory.elapsed() >= Duration::from_secs(86400) {
|
||||
self.queue_inventory(generation)?;
|
||||
inventory = Instant::now();
|
||||
}
|
||||
@@ -258,6 +902,11 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
let path = Path::new(&arguments[2]);
|
||||
ensure!(path.is_absolute(), "ABSOLUTE_CONFIG_REQUIRED");
|
||||
let config: Config = serde_json::from_slice(&protected_read(path, 32768)?)?;
|
||||
native::require_local_service()?;
|
||||
ensure!(
|
||||
config.state_directory.is_absolute(),
|
||||
"ABSOLUTE_STATE_REQUIRED"
|
||||
);
|
||||
native::private_path(&config.state_directory, true)?;
|
||||
ensure!(
|
||||
crate::https_channel::server_name()? == config.server_name,
|
||||
@@ -266,7 +915,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
let ca = protected_read(&config.ca_file, 32768)?;
|
||||
for child in fs::read_dir(&config.state_directory)? {
|
||||
let child = child?;
|
||||
native::private_path(&child.path(), false)?;
|
||||
native::private_path(&child.path(), child.file_type()?.is_dir())?;
|
||||
}
|
||||
let journal = Journal::open(&config.state_directory)?;
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
@@ -275,8 +924,20 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
.build()?;
|
||||
runtime.block_on(async move {
|
||||
let identity = enrollment(&config, &ca).await?;
|
||||
let mut journal=journal;
|
||||
journal.pin_identity(&identity.agent_id)?;
|
||||
let boot = native::boot_id().await?;
|
||||
let binding=config.job_binding_file.as_deref().map(readonly_binding).transpose()?;
|
||||
if let Some(binding)=&binding {
|
||||
ensure!(binding.agent_id.to_string()==identity.agent_id && binding.asset_id.to_string()==identity.asset_id,"ENROLLMENT_JOB_BINDING_MISMATCH");
|
||||
native::private_path(&config.state_directory.join("jobs"),true)?;
|
||||
import_legacy_inventory(&mut journal)?;
|
||||
}
|
||||
ensure!(!config.enable_jobs || binding.is_some(),"ENABLED_JOBS_REQUIRE_BINDING");
|
||||
let (ledger,ledger_file)=match &binding {Some(binding)=>{let (db,file)=open_ledger(&config.state_directory,binding)?;(Some(db),Some(file))},None=>(None,None)};
|
||||
let clock=JobClock{native:Arc::new(rmm_agent_windows::clock::NativeClock::new()),enabled:config.enable_jobs};
|
||||
let sync=if config.enable_jobs {binding.clone().map(reconcile::ClientSync::new)} else {None};
|
||||
let (completed_tx,completed_rx)=tokio::sync::mpsc::channel(1);
|
||||
let mut agent = Agent {
|
||||
config,
|
||||
identity,
|
||||
@@ -284,10 +945,23 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
journal,
|
||||
boot,
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
started: Instant::now(),
|
||||
started: Instant::now(), binding, ledger, _ledger_file:ledger_file, clock, sync,
|
||||
prepared:false,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1;
|
||||
while !stop.load(Ordering::SeqCst) {
|
||||
while !stop.load(Ordering::SeqCst) || agent.running.is_some() {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
agent.close_gate();
|
||||
if let Some(running)=&agent.running {running.cancel.store(true,Ordering::SeqCst);}
|
||||
if let Ok(completion)=agent.completed_rx.try_recv(){agent.completed(0,completion)?;}
|
||||
if agent.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
|
||||
let running=agent.running.as_ref().context("RUNNING_REQUIRED")?;
|
||||
agent.completed(0,(running.offer.clone(),running.grant.clone(),Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED")))?;
|
||||
}
|
||||
if agent.running.is_some() {tokio::select!{Some(completion)=agent.completed_rx.recv()=>agent.completed(0,completion)?,_=tokio::time::sleep(Duration::from_millis(250))=>{}}}
|
||||
continue;
|
||||
}
|
||||
let start = Instant::now();
|
||||
let _ = agent.session(stop).await;
|
||||
if start.elapsed() > Duration::from_secs(60) {
|
||||
@@ -296,7 +970,15 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
let delay = crate::https_channel::retry_delay(backoff);
|
||||
backoff = (backoff + 1).min(6);
|
||||
let until = Instant::now() + delay;
|
||||
while !stop.load(Ordering::SeqCst) && Instant::now() < until {
|
||||
while Instant::now() < until && (!stop.load(Ordering::SeqCst) || agent.running.is_some()) {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
agent.close_gate();if let Some(running)=&agent.running {running.cancel.store(true,Ordering::SeqCst);}
|
||||
}
|
||||
if let Ok(completion)=agent.completed_rx.try_recv(){agent.completed(0,completion)?;}
|
||||
if agent.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
|
||||
let running=agent.running.as_ref().context("RUNNING_REQUIRED")?;
|
||||
agent.completed(0,(running.offer.clone(),running.grant.clone(),Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED")))?;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
@@ -307,7 +989,7 @@ pub fn main() -> Result<()> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
if args == ["--help"] {
|
||||
println!(
|
||||
"Development Windows passive service. --service --config ABSOLUTE_PATH. No job executor or approval bypass."
|
||||
"Development Windows service. --service --config ABSOLUTE_PATH. Jobs disabled by default; require qualified clock, provisioned signing binding, signed Sync and WebUI-authorized grant. No approval bypass."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -341,3 +1023,90 @@ pub fn main() -> Result<()> {
|
||||
);
|
||||
rmm_agent_windows::service::dispatch(service_run)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn old_passive_configuration_cannot_enable_jobs_or_read_the_clock() {
|
||||
let config:Config=serde_json::from_value(serde_json::json!({"server_name":"example.invalid","connect_address":"127.0.0.1:443","ca_file":"C:/fixture/ca.pem","state_directory":"C:/fixture/state","enrollment_token_file":"C:/fixture/token","transport":"auto"})).unwrap();
|
||||
assert!(!config.enable_jobs);
|
||||
assert!(config.job_binding_file.is_none());
|
||||
let clock = JobClock {
|
||||
native: Arc::new(rmm_agent_windows::clock::NativeClock::new()),
|
||||
enabled: config.enable_jobs,
|
||||
};
|
||||
assert_eq!(
|
||||
clock.now().unwrap_err().to_string(),
|
||||
"WINDOWS_JOBS_NOT_ENABLED"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn legacy_inventory_keeps_original_bytes_until_concrete_core_ack() {
|
||||
let directory =
|
||||
std::env::temp_dir().join(format!("windows-runtime-import-{}", Uuid::new_v4()));
|
||||
fs::create_dir(&directory).unwrap();
|
||||
let mut journal = Journal::open(&directory).unwrap();
|
||||
journal.pin_identity("retained").unwrap();
|
||||
let sequence = journal.next_sequence().unwrap();
|
||||
let envelope = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
sequence,
|
||||
payload: Some(Payload::InventorySnapshot(InventorySnapshot::default())),
|
||||
..Default::default()
|
||||
};
|
||||
journal.save(&envelope).unwrap();
|
||||
let mut original = envelope.encode_to_vec();
|
||||
original.extend([0xf8, 0x07, 0x01]); // Unknown additive protobuf field must remain byte-exact.
|
||||
journal
|
||||
.connection()
|
||||
.execute(
|
||||
"UPDATE events SET payload=?1 WHERE id=?2",
|
||||
rusqlite::params![&original, &envelope.message_id],
|
||||
)
|
||||
.unwrap();
|
||||
import_legacy_inventory(&mut journal).unwrap();
|
||||
for table in ["events", "job_events"] {
|
||||
let bytes: Vec<u8> = journal
|
||||
.connection()
|
||||
.query_row(
|
||||
&format!("SELECT payload FROM {table} WHERE id=?1"),
|
||||
[&envelope.message_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(bytes, original);
|
||||
}
|
||||
jobs::ack(journal.connection_mut(), &Uuid::new_v4().to_string()).unwrap();
|
||||
assert_eq!(journal.count().unwrap(), 1);
|
||||
import_legacy_inventory(&mut journal).unwrap(); // Exact same-ID copy is idempotent.
|
||||
assert_eq!(journal.next_sequence().unwrap(), sequence + 1);
|
||||
jobs::ack(journal.connection_mut(), &envelope.message_id).unwrap();
|
||||
assert_eq!(journal.count().unwrap(), 0);
|
||||
assert!(jobs::events(journal.connection()).unwrap().is_empty());
|
||||
drop(journal);
|
||||
fs::remove_dir_all(directory).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn worker_fence_requires_same_generation_and_entire_clock_interval() {
|
||||
struct Clock(chrono::DateTime<Utc>);
|
||||
impl SyncClock for Clock {
|
||||
fn now(&self) -> Result<chrono::DateTime<Utc>> {
|
||||
Ok(self.0)
|
||||
}
|
||||
}
|
||||
let now = Utc::now();
|
||||
let fence = EffectFence::default();
|
||||
fence.deadline.store(
|
||||
(now + chrono::Duration::seconds(3)).timestamp_millis(),
|
||||
Ordering::SeqCst,
|
||||
);
|
||||
assert!(!fence.allows(1, &Clock(now)));
|
||||
fence.generation.store(1, Ordering::SeqCst);
|
||||
assert!(fence.allows(1, &Clock(now)));
|
||||
assert!(!fence.allows(2, &Clock(now)));
|
||||
assert!(!fence.allows(1, &Clock(now + chrono::Duration::seconds(1))));
|
||||
fence.close();
|
||||
assert!(!fence.allows(1, &Clock(now)));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user