Integrate signed agent catalog, tenant enrollment and HTTPS installers

This commit is contained in:
2026-11-17 15:56:18 +01:00
parent 689f326460
commit 7fd17272d5
70 changed files with 12982 additions and 327 deletions
+375 -48
View File
@@ -386,7 +386,26 @@ impl EffectFence {
})
}
}
type Completion = (JobOffer, ExecutionPermitPayload, Outcome);
enum ExecutorResult {
Outcome(Outcome),
#[cfg(feature = "system-helper")]
UpdatePending(Option<rmm_agent_windows::update_admission::Accepted>),
}
impl From<Outcome> for ExecutorResult {
fn from(value: Outcome) -> Self {
Self::Outcome(value)
}
}
fn interrupted_executor(grant: &ExecutionPermitPayload) -> ExecutorResult {
#[cfg(not(feature="system-helper"))]
let _=grant;
#[cfg(feature = "system-helper")]
if grant.action == "update_agent" {
return ExecutorResult::UpdatePending(None);
}
Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED").into()
}
type Completion = (JobOffer, ExecutionPermitPayload, ExecutorResult);
#[cfg(feature = "system-helper")]
fn needs_bound_outcome(
db: &Connection,
@@ -573,6 +592,7 @@ impl HelperControl {
}
struct Agent {
config: Config,
reported_version: String,
identity: EnrollmentResponse,
ca: Vec<u8>,
journal: Journal,
@@ -591,6 +611,14 @@ struct Agent {
prepared: bool,
deferred_sync: Option<DeferredSync>,
#[cfg(feature = "system-helper")]
update_pending: bool,
#[cfg(feature="system-helper")]
update_observer:crate::update_observation_runtime::Observer,
#[cfg(feature = "system-helper")]
update_health: Option<tokio::task::JoinHandle<Result<()>>>,
#[cfg(feature = "system-helper")]
update_inventory_head: Option<(u64, String)>,
#[cfg(feature = "system-helper")]
prepared_request: Option<Vec<u8>>,
#[cfg(feature = "system-helper")]
helper_apply_unconfirmed: bool,
@@ -617,6 +645,10 @@ struct Agent {
running: Option<Running>,
deferred_permit: Option<(u64, ExecutionPermit)>,
readiness_worker: Option<tokio::task::JoinHandle<Result<()>>>,
#[cfg(feature = "system-helper")]
update_prefetch: Option<(u64, String, String, tokio::task::JoinHandle<Result<()>>)>,
#[cfg(feature = "system-helper")]
update_cached: Option<(u64, String, String)>,
}
impl Agent {
fn rotation_without_effects(&self) -> Result<()> {
@@ -1096,6 +1128,19 @@ impl Agent {
let (ledger, result) = task.await?;
self.ledger = Some(ledger);
let (snapshot, unknowns) = result?;
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
self.update_pending =
rmm_agent_windows::helper_runtime::intent::Store::open(false)?
.pending()?
.iter()
.any(|intent| {
self.binding.as_ref().is_some_and(|binding| {
binding.job(&intent.request.job_bytes).is_ok_and(|job| {
matches!(job.action, Action::UpdateAgent { .. })
})
})
});
}
for bound in unknowns {
if needs_bound_outcome(self.journal.connection(), &bound)? {
let bytes: Vec<u8> = self.journal.connection().query_row(
@@ -1232,6 +1277,88 @@ impl Agent {
.is_some_and(|sync| sync.can_start(&channel, helper, &self.clock))
})
}
#[cfg(feature = "system-helper")]
async fn poll_update_health(&mut self, generation: u64) -> Result<bool> {
if self
.update_health
.as_ref()
.is_some_and(|task| task.is_finished())
{
let task = self.update_health.take().context("UPDATE_HEALTH_TASK")?;
task.await??;
return Ok(true);
}
if !self.update_pending
|| self.update_health.is_some()
|| self.prepared
|| self.helper_control_pending()
|| self.clock.reading().is_err()
{
return Ok(false);
}
let binding = self
.binding
.as_ref()
.context("UPDATE_BINDING_REQUIRED")?
.clone();
let Some(proof) = rmm_agent_core::update_health_inventory::current_proof(
self.journal.connection(),
&binding,
)?
else {
return Ok(true);
};
let decoded = proof.decode().map_err(anyhow::Error::msg)?;
let event = decoded
.events
.last()
.map(|raw| serde_json::from_slice::<rmm_updater_handoff::Event>(raw))
.transpose()?;
if !event.is_some_and(|event| {
matches!(
event.evidence,
rmm_updater_handoff::Evidence::ServicesStarted { .. }
| rmm_updater_handoff::Evidence::RollbackServicesStarted { .. }
)
}) {
return Ok(true);
}
let inventory_key = (generation, proof.head_sha256.clone());
if self.update_inventory_head.as_ref() != Some(&inventory_key)
&& self.queue_inventory(generation)?
{
self.update_inventory_head = Some(inventory_key);
}
let Some(context) = crate::agent_update_health::context(
self.journal.connection(),
&binding,
&proof,
self.clock.now()?,
)?
else {
return Ok(true);
};
let handoff = context.handoff().clone();
let key = context.health_public_key();
let identity = self.certificates.current(
self.journal.connection(),
&self.key_files,
Utc::now().timestamp(),
)?;
let client = crate::https_channel::mtls_client(
self.config.connect_address,
&self.ca,
&identity.certificate_pem,
&identity.private_key_pem,
)?;
let helper = self.helper.clone().context("UPDATE_HELPER_REQUIRED")?;
let address = self.config.connect_address;
self.update_health = Some(tokio::spawn(async move {
let receipt = crate::agent_update_health::request(&client, address, &context).await?;
helper.submit_update_health(&handoff, key, receipt).await
}));
Ok(false)
}
fn supported_job(&self, job: &rmm_domain::jobs::JobPayload) -> bool {
if self.config.windows_echo_probe_only
&& !self
@@ -1278,8 +1405,99 @@ impl Agent {
rmm_agent_windows::echo_probe::permits_offer(scope, offer, job)
})
}
#[cfg(feature = "system-helper")]
async fn prefetch_update(
&mut self,
generation: u64,
offer: &JobOffer,
job: &rmm_domain::jobs::JobPayload,
) -> Result<bool> {
let Action::UpdateAgent { update } = &job.action else {
return Ok(true);
};
ensure!(
update.validate()
&& update.manifest().map_err(anyhow::Error::msg)?.platform
== rmm_agent_windows::release_platform::WIRE,
"UPDATE_PREFETCH_JOB"
);
let key = (generation, offer.job_id.clone(), offer.attempt_id.clone());
if self.update_cached.as_ref() == Some(&key) {
return Ok(true);
}
if self.update_prefetch.as_ref().is_some_and(|(g, j, a, _)| {
(*g, j.as_str(), a.as_str())
!= (generation, offer.job_id.as_str(), offer.attempt_id.as_str())
}) && let Some((_, _, _, task)) = self.update_prefetch.take() {
task.abort();
}
if self
.update_prefetch
.as_ref()
.is_some_and(|(_, _, _, task)| task.is_finished())
{
let (_, _, _, task) = self
.update_prefetch
.take()
.context("UPDATE_PREFETCH_TASK")?;
task.await??;
self.update_cached = Some(key);
return Ok(true);
}
if self.update_prefetch.is_none() {
ensure!(
rmm_agent_windows::update_cache::configured(),
"UPDATE_PREFETCH_NOT_PROVISIONED"
);
let address = self.config.connect_address;
let identity = self.certificates.current(
self.journal.connection(),
&self.key_files,
Utc::now().timestamp(),
)?;
let client = crate::https_channel::mtls_client(
address,
&self.ca,
&identity.certificate_pem,
&identity.private_key_pem,
)?;
let base = format!(
"https://{}:{}",
crate::https_channel::server_name()?,
address.port()
);
let guard_job = job.clone();
let clock = self.clock.clone();
let fence = self.fence.clone();
let job = job.clone();
let manifest = update.manifest().map_err(anyhow::Error::msg)?;
let task = tokio::spawn(crate::update_download::prefetch(
client,
base,
job,
manifest,
move || {
ensure!(
fence.allows_identity(generation, true, &clock)
&& clock.now()? < guard_job.start_before,
"UPDATE_PREFETCH_FENCED"
);
Ok(())
},
));
self.update_prefetch = Some((
generation,
offer.job_id.clone(),
offer.attempt_id.clone(),
task,
));
}
Ok(false)
}
fn capabilities(&self) -> Vec<String> {
let mut caps: Vec<String> = CAPABILITIES.iter().map(|c| c.to_string()).collect();
#[cfg(target_env = "gnu")]
caps.push("agent.release.windows.gnu/1".into());
// queue_update_observation emits the passive update status in every profile.
caps.push("agent.update/1".into());
if self.config.windows_echo_probe_only {
@@ -1307,6 +1525,10 @@ impl Agent {
&& let Some(helper) = &self.helper
{
caps.extend(helper.qualified_capabilities());
if helper.update_observer_configured()
&& self.update_observer.cached().is_some_and(|r|r.observation.predecessor.as_ref().is_some_and(|p|p.helper_sha256==helper.helper_binary_hash())) {
for cap in self.update_observer.capabilities(){if !caps.contains(&cap){caps.push(cap);}}
}
}
}
caps
@@ -1338,6 +1560,13 @@ impl Agent {
let pending = intents.pending()?;
if let Some(intent) = pending.first() {
let p = intent.permit()?;
if matches!(
binding.job(&intent.request.job_bytes)?.action,
Action::UpdateAgent { .. }
) {
self.update_pending = true;
return Ok(());
}
let incompatible:bool=self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown','awaiting_reboot') AND NOT(job=?1 AND attempt=?2))",rusqlite::params![p.job_id.to_string(),p.attempt_id.to_string()],|r|r.get(0))?;
ensure!(!incompatible, "SYSTEM_PENDING_INTENT_CONFLICT");
binding.verify_historical_for_denial_with(
@@ -1496,37 +1725,13 @@ impl Agent {
helper: Option<&reconcile::HelperSnapshot>,
) -> Result<()> {
let context = self.channel(generation)?;
let closed_control_idle = {
#[cfg(feature = "system-helper")]
{
self.running.is_none()
&& !self.prepared
&& self.deferred_sync.is_none()
&& self.deferred_permit.is_none()
&& self.helper_control.is_none()
&& self.helper_probe.is_none()
&& helper.is_some()
&& (if Path::new(rmm_agent_windows::helper_runtime::intent::DB).exists() {
rmm_agent_windows::helper_runtime::intent::Store::open(false)
.and_then(|store| store.pending())
.is_ok_and(|pending| pending.is_empty())
} else {
helper.is_some_and(|snapshot| snapshot.attempts.is_empty())
})
}
#[cfg(not(feature = "system-helper"))]
{
false
}
};
let sync = self.sync.as_mut().context("SYNC_NOT_CONFIGURED")?;
let prepared = sync.prepare_with_closed_control_idle(
let prepared = sync.prepare(
self.journal.connection_mut(),
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
context,
helper,
&self.clock,
closed_control_idle,
);
#[cfg(feature = "system-helper")]
if helper.is_some()
@@ -1708,7 +1913,7 @@ impl Agent {
}
Ok(())
}
fn completed(&mut self, generation: u64, (offer, grant, outcome): Completion) -> Result<()> {
fn completed(&mut self, generation: u64, (offer, grant, result): Completion) -> Result<()> {
let running = self
.running
.as_ref()
@@ -1719,6 +1924,48 @@ impl Agent {
&& running.grant == grant,
"EXECUTOR_COMPLETION_BINDING"
);
#[cfg(feature = "system-helper")]
if let ExecutorResult::UpdatePending(accepted) = result {
ensure!(
grant.execution_identity == "system" && grant.action == "update_agent",
"UPDATE_COMPLETION_SCOPE"
);
if let Some(accepted) = accepted {
let intent = rmm_agent_windows::helper_runtime::intent::Store::open(false)?
.by_grant(grant.grant_id)?;
accepted.matches(&intent.request, grant.grant_id)?;
let mut envelope = self.envelope(
generation,
Payload::JobStateChanged(JobStateChanged::default()),
)?;
let tx = self.journal.connection_mut().transaction()?;
let revision =
jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
revision,
state: "running".into(),
grant_id: grant.grant_id.to_string(),
}));
jobs::event(&tx, &envelope)?;
tx.commit()?;
}
// Neither a typed handoff ACK nor its loss is an Outcome. Import
// only actual original helper evidence in the next normal Sync.
self.update_pending = true;
self.running = None;
self.close_gate();
return Ok(());
}
#[cfg(not(feature="system-helper"))]
let ExecutorResult::Outcome(outcome)=result;
#[cfg(feature="system-helper")]
let outcome = match result {
ExecutorResult::Outcome(outcome) => outcome,
#[cfg(feature = "system-helper")]
ExecutorResult::UpdatePending(_) => unreachable!("handled update completion"),
};
if grant.execution_identity == "system" {
// No local consumption or definitive result from an IPC outcome.
// Import only a fresh raw helper snapshot before the next Sync.
@@ -1845,6 +2092,10 @@ impl Agent {
let completed = self.completed_tx.clone();
let work_offer = offer.clone();
let work_grant = grant.clone();
let is_update = matches!(job.action, Action::UpdateAgent { .. });
if is_update {
self.update_pending = true;
}
let msi = job.action.windows_msi().cloned();
let address = self.config.connect_address;
let ca = self.ca.clone();
@@ -1912,7 +2163,7 @@ impl Agent {
} else {
true
};
let _result = helper
let result = helper
.delegate(intent, move || {
ensure!(staged, "MSI_DOWNLOAD_NOT_READY");
ensure!(
@@ -1931,7 +2182,11 @@ impl Agent {
.send((
work_offer,
work_grant,
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"),
if is_update {
ExecutorResult::UpdatePending(result.ok().flatten())
} else {
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT").into()
},
))
.await;
});
@@ -2054,7 +2309,7 @@ impl Agent {
outcome.error_code = "WINDOWS_FINAL_CLOCK_UNTRUSTED".into();
outcome.evidence["postconditions_passed"] = false.into();
}
let _ = completed.blocking_send((work_offer, work_grant, outcome));
let _ = completed.blocking_send((work_offer, work_grant, outcome.into()));
});
self.running = Some(Running {
offer,
@@ -2078,9 +2333,9 @@ impl Agent {
payload: Some(payload),
})
}
fn queue_inventory(&mut self, generation: u64) -> Result<()> {
fn queue_inventory(&mut self, generation: u64) -> Result<bool> {
if self.journal.count()? > 0 || !jobs::events(self.journal.connection())?.is_empty() {
return Ok(());
return Ok(false);
}
let data = serde_json::to_vec(&native::snapshot()?)?;
ensure!(data.len() <= 32768, "WINDOWS_INVENTORY_LIMIT");
@@ -2097,22 +2352,36 @@ impl Agent {
completeness: "partial".into(),
}),
)?;
if self.binding.is_some() {
jobs::event(self.journal.connection(), &envelope)
if let Some(binding) = &self.binding {
jobs::event(self.journal.connection(), &envelope)?;
let initialized:bool=self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='reconcile_meta')",[],|r|r.get(0))?;
if initialized
&& self.journal.connection().query_row(
"SELECT generation FROM reconcile_meta WHERE id=1",
[],
|r| r.get::<_, u64>(0),
)? == generation
{
rmm_agent_core::update_health_inventory::retain_current(
self.journal.connection(),
&envelope,
binding,
)?;
}
} else {
self.journal.save(&envelope)
self.journal.save(&envelope)?;
}
Ok(true)
}
fn queue_update_observation(&mut self, generation: u64) -> Result<()> {
if self.journal.count()? > 0 || !jobs::events(self.journal.connection())?.is_empty() {
return Ok(());
}
// Schema support is passive only; no qualified updater/helper observer
// is configured by this source checkpoint, so apply remains closed.
#[cfg(feature="system-helper")]
let observation=self.update_observer.observation(generation);
#[cfg(not(feature="system-helper"))]
let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured(
rmm_domain::agent_update::Platform::WindowsX64,
Utc::now(),
);
rmm_agent_windows::release_platform::WIRE,Utc::now());
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
let envelope = self.envelope(
generation,
@@ -2190,7 +2459,7 @@ impl Agent {
let hello = self.envelope(
0,
Payload::ClientHello(ClientHello {
agent_version: env!("CARGO_PKG_VERSION").into(),
agent_version: self.reported_version.clone(),
capabilities: advertised.clone(),
}),
)?;
@@ -2254,6 +2523,8 @@ impl Agent {
let mut tick = tokio::time::interval(Duration::from_millis(20));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut readiness_at = Instant::now() - Duration::from_secs(5);
#[cfg(feature = "system-helper")]
let mut update_control_at = Instant::now() - Duration::from_secs(2);
while !stop.load(Ordering::SeqCst) || self.running.is_some() {
if stop.load(Ordering::SeqCst) {
self.close_gate();
@@ -2272,10 +2543,23 @@ impl Agent {
}
if self.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
if let Ok(completion)=self.completed_rx.try_recv(){self.completed(generation,completion)?;}
else {let running=self.running.as_ref().context("RUNNING_REQUIRED")?;self.completed(generation,(running.offer.clone(),running.grant.clone(),Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED")))?;}
else {let running=self.running.as_ref().context("RUNNING_REQUIRED")?;self.completed(generation,(running.offer.clone(),running.grant.clone(),interrupted_executor(&running.grant)))?;}
needs_sync=true;
}
#[cfg(feature="system-helper")]
if !stop.load(Ordering::SeqCst) && update_control_at.elapsed()>=Duration::from_secs(2) {
update_control_at=Instant::now();
if !self.update_pending && !self.prepared && !self.helper_control_pending() && self.running.is_none() && self.ready(generation) {
let transport=(||->Result<_>{let helper=self.helper.clone().filter(|h|h.update_observer_configured()).context("UPDATE_OBSERVER_NOT_CONFIGURED")?;
let identity=self.certificates.current(self.journal.connection(),&self.key_files,Utc::now().timestamp())?;
Ok((crate::https_channel::mtls_client(self.config.connect_address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?,self.config.connect_address,helper))})();
self.update_observer.poll(generation,||transport).await;
}
match self.poll_update_health(generation).await {Ok(refresh)=>{needs_sync|=refresh;},Err(_)=>{self.close_gate();needs_sync=true;}}
}
let busy=self.running.is_some() || self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?;
#[cfg(feature="system-helper")]
let busy=(busy && !(self.update_pending && self.running.is_none())) || self.update_health.is_some();
if !stop.load(Ordering::SeqCst) && !busy {
if capabilities_at.elapsed()>=Duration::from_secs(60) {capabilities_at=Instant::now();if advertised!=self.capabilities(){return Ok(());}}
if self.prepared && self.deferred_sync.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
@@ -2289,9 +2573,15 @@ impl Agent {
ensure!(self.scope_permits(&offer,&job),"WINDOWS_ECHO_OFFER_OUT_OF_SCOPE");
if now>=job.start_before {self.record(generation,&offer,"",jobs::no_start("expired","START_DEADLINE_EXPIRED"))?;needs_sync=true;}
else if now-chrono::Duration::seconds(2)>=job.not_before && self.ready_identity(generation,job.execution_identity=="system") {
jobs::reserve_request(self.journal.connection(),&offer)?;
let request=self.envelope(generation,Payload::ExecutionPermitRequest(ExecutionPermitRequest{job_id:offer.job_id,attempt_id:offer.attempt_id,job_payload_hash:offer.job_payload_hash,assignment_version:job.assignment_version}))?;
channel.send(&request).await?;
#[cfg(feature="system-helper")]
let prefetched=self.prefetch_update(generation,&offer,&job).await?;
#[cfg(not(feature="system-helper"))]
let prefetched=true;
if prefetched {
jobs::reserve_request(self.journal.connection(),&offer)?;
let request=self.envelope(generation,Payload::ExecutionPermitRequest(ExecutionPermitRequest{job_id:offer.job_id,attempt_id:offer.attempt_id,job_payload_hash:offer.job_payload_hash,assignment_version:job.assignment_version}))?;
channel.send(&request).await?;
}
}
}
}
@@ -2484,6 +2774,29 @@ impl Agent {
)
}
}
fn installed_version() -> Result<String> {
let executable = std::env::current_exe()?;
let directory = executable.parent().context("AGENT_IMAGE_DIRECTORY")?;
let path = directory.join("manifest.json");
if !path.try_exists()? {
return Ok(env!("CARGO_PKG_VERSION").into());
}
rmm_agent_windows::job_binding::check_admin_readonly(&path)?;
let manifest: rmm_updater::ReleaseManifest =
serde_json::from_slice(&protected_read(&path, rmm_updater::MAX_MANIFEST_BYTES)?)?;
let expected = &manifest.artifacts[0];
let mut digest = Sha256::new();
let bytes = std::io::copy(&mut fs::File::open(executable)?, &mut digest)?;
ensure!(
manifest.platform == rmm_agent_windows::release_platform::VERIFY
&& expected.role == rmm_updater::ArtifactRole::Agent
&& expected.bytes == bytes
&& expected.sha256 == hex::encode(digest.finalize()),
"INSTALLED_AGENT_IMAGE_MISMATCH"
);
Ok(manifest.version)
}
fn service_run(stop: &'static AtomicBool) -> Result<()> {
let arguments: Vec<String> = std::env::args().skip(1).collect();
ensure!(
@@ -2550,6 +2863,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
let (completed_tx,completed_rx)=tokio::sync::mpsc::channel(1);
let mut agent = Agent {
config,
reported_version: installed_version()?,
identity,
ca,
journal,
@@ -2580,7 +2894,20 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
helper_checked:false,
#[cfg(feature="system-helper")]
helper_refused:false,
prepared:false,deferred_sync:None,fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
prepared:false,deferred_sync:None,
#[cfg(feature="system-helper")]
update_pending:false,
#[cfg(feature="system-helper")]
update_observer:crate::update_observation_runtime::Observer::default(),
#[cfg(feature="system-helper")]
update_health:None,
#[cfg(feature="system-helper")]
update_inventory_head:None,
fence:Arc::new(EffectFence::default()),completed_tx,completed_rx,running:None,deferred_permit:None,readiness_worker:None,
#[cfg(feature="system-helper")]
update_prefetch:None,
#[cfg(feature="system-helper")]
update_cached:None,
};
agent.recover_interrupted()?;
let mut backoff = 1;
@@ -2591,7 +2918,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
if let Ok(completion)=agent.completed_rx.try_recv(){agent.completed(0,completion)?;}
if agent.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
let running=agent.running.as_ref().context("RUNNING_REQUIRED")?;
agent.completed(0,(running.offer.clone(),running.grant.clone(),Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED")))?;
agent.completed(0,(running.offer.clone(),running.grant.clone(),interrupted_executor(&running.grant)))?;
}
if agent.running.is_some() {tokio::select!{Some(completion)=agent.completed_rx.recv()=>agent.completed(0,completion)?,_=tokio::time::sleep(Duration::from_millis(250))=>{}}}
continue;
@@ -2613,7 +2940,7 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
if let Ok(completion)=agent.completed_rx.try_recv(){agent.completed(0,completion)?;}
if agent.running.as_ref().is_some_and(|r|r.worker.is_finished()) {
let running=agent.running.as_ref().context("RUNNING_REQUIRED")?;
agent.completed(0,(running.offer.clone(),running.grant.clone(),Outcome::unknown("EXECUTOR_THREAD_INTERRUPTED")))?;
agent.completed(0,(running.offer.clone(),running.grant.clone(),interrupted_executor(&running.grant)))?;
}
tokio::time::sleep(Duration::from_millis(250)).await;
}