feat: add approval bound full maintenance script console and executors
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -50,6 +50,7 @@ struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
||||
inventory_tx: tokio::sync::mpsc::Sender<Result<(Vec<u8>, i64)>>,
|
||||
inventory_rx: tokio::sync::mpsc::Receiver<Result<(Vec<u8>, i64)>>,
|
||||
inventory_inflight: bool,
|
||||
running_cancels: std::collections::HashMap<(String, String), Arc<AtomicBool>>,
|
||||
sync: Option<reconcile::ClientSync>,
|
||||
sync_clock: C,
|
||||
helpers: Option<helper_sync::Helpers>,
|
||||
@@ -61,6 +62,7 @@ struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
running_generation: AtomicU64,
|
||||
generation: AtomicU64,
|
||||
deadline: AtomicI64,
|
||||
root: AtomicBool,
|
||||
@@ -80,6 +82,14 @@ fn report_closed_gate(error: &anyhow::Error) {
|
||||
}
|
||||
impl EffectFence {
|
||||
fn close(&self) {
|
||||
self.running_generation.store(0, Ordering::SeqCst);
|
||||
self.close_starts();
|
||||
}
|
||||
// A signed Sync/start lease authorizes NEW starts. Its natural expiry
|
||||
// cannot retroactively stop an approved bounded process (spec22.3).
|
||||
// Running effects instead observe their exact cancellation and current
|
||||
// connection generation; transport/signed refusal closes both fences.
|
||||
fn close_starts(&self) {
|
||||
self.generation.store(0, Ordering::SeqCst);
|
||||
self.root.store(false, Ordering::SeqCst);
|
||||
}
|
||||
@@ -117,7 +127,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
})
|
||||
}
|
||||
async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
self.fence.close();
|
||||
self.fence.close_starts();
|
||||
let helpers = self.helpers.clone();
|
||||
let snapshot = tokio::task::spawn_blocking(move || match helpers {
|
||||
Some(h) => h.snapshot(generation),
|
||||
@@ -182,6 +192,9 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
.as_ref()
|
||||
.and_then(|s| s.validity_deadline())
|
||||
.context("SIGNED_LEASE_REQUIRED")?;
|
||||
self.fence
|
||||
.running_generation
|
||||
.store(generation, Ordering::SeqCst);
|
||||
self.fence
|
||||
.deadline
|
||||
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
||||
@@ -189,6 +202,9 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
.root
|
||||
.store(self.ready(generation, true), Ordering::SeqCst);
|
||||
self.fence.generation.store(generation, Ordering::SeqCst);
|
||||
} else {
|
||||
// A valid signed refusal is a fence too, not just parse/I/O errors.
|
||||
self.fence.close();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -322,6 +338,11 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}));
|
||||
jobs::event(&tx, &envelope)?;
|
||||
tx.commit()?;
|
||||
let cancellation = Arc::new(AtomicBool::new(false));
|
||||
self.running_cancels.insert(
|
||||
(offer.job_id.clone(), offer.attempt_id.clone()),
|
||||
cancellation.clone(),
|
||||
);
|
||||
let completed = self.completed_tx.clone();
|
||||
let clock = self.clock.clone();
|
||||
let fence = self.fence.clone();
|
||||
@@ -334,7 +355,16 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
{
|
||||
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
|
||||
} else if job.execution_identity == "root" {
|
||||
apt::invoke(&binding, &job, &request).unwrap_or_else(|_| {
|
||||
apt::invoke_with_control(&binding, &job, &request, || {
|
||||
ensure!(
|
||||
!cancellation.load(Ordering::SeqCst)
|
||||
&& fence.running_generation.load(Ordering::SeqCst) == generation
|
||||
&& clock.trusted(),
|
||||
"SCRIPT_RUNNING_CANCEL_OR_FENCE"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap_or_else(|_| {
|
||||
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
|
||||
})
|
||||
} else {
|
||||
@@ -537,6 +567,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?
|
||||
};
|
||||
if let Some((bytes,state,grant)) = row {
|
||||
if let Some(flag) = self.running_cancels.get(&(cancel.job_id.clone(),cancel.attempt_id.clone())) { flag.store(true,Ordering::SeqCst); }
|
||||
self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
||||
if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; }
|
||||
// Running script cancellation is not confirmed by an
|
||||
@@ -549,6 +580,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
}
|
||||
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
|
||||
self.running_cancels.remove(&(offer.job_id.clone(),offer.attempt_id.clone()));
|
||||
// First preserve the original completion without unlocking.
|
||||
// A startup can replay it if the journal transaction is lost.
|
||||
let state = outcome.state.clone();
|
||||
@@ -807,6 +839,7 @@ async fn main() -> Result<()> {
|
||||
inventory_tx,
|
||||
inventory_rx,
|
||||
inventory_inflight: false,
|
||||
running_cancels: std::collections::HashMap::new(),
|
||||
sync,
|
||||
sync_clock: reconcile::NativeClock::default(),
|
||||
helpers,
|
||||
|
||||
Reference in New Issue
Block a user