Run approved FreeBSD root jobs through independently pinned and synchronized local helper
Rust development slice / linux (push) Failing after 11s

This commit is contained in:
2026-11-20 08:27:13 +01:00
parent 2e3916b84b
commit 0b0790de71
17 changed files with 1037 additions and 61 deletions
+148 -50
View File
@@ -1,6 +1,6 @@
//! One worker per authenticated Python HTTPS channel, never a network listener.
//! Only the fixed unprivileged probe is executable. The shared Rust core owns
//! all approval/permit verification, receipt, consumption and sync transitions.
//! Executes approved service-account POSIX scripts or delegates approved root
//! scripts to the independently synchronized local helper.
use anyhow::{Context, Result, bail, ensure};
use chrono::Utc;
use prost::Message;
@@ -22,6 +22,11 @@ use std::{
unix::fs::{MetadataExt, OpenOptionsExt},
},
path::Path,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
thread::JoinHandle,
time::{Duration, Instant},
};
use uuid::Uuid;
@@ -31,9 +36,16 @@ mod tests;
fn supported(job: &JobPayload) -> bool {
job.execution_identity == "agent_service"
&& job.action.opnsense_kernel_probe()
&& job.max_runtime_seconds == 10
&& job.max_output_bytes == 1024
&& job.action.opnsense_service_script()
&& job
.action
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
}
struct Running {
offer: JobOffer,
grant: rmm_domain::jobs::ExecutionPermitPayload,
cancel: Arc<AtomicBool>,
task: JoinHandle<Outcome>,
}
struct Worker {
@@ -45,6 +57,17 @@ struct Worker {
ledger: Connection,
sync_due: Instant,
permit_after: Instant,
root_enabled: bool,
pending_sync: Option<SyncRequest>,
running: Option<Running>,
}
impl Drop for Worker {
fn drop(&mut self) {
if let Some(r) = self.running.take() {
r.cancel.store(true, Ordering::Release);
let _ = r.task.join();
}
}
}
impl Worker {
fn envelope(&self, payload: Payload) -> Result<Envelope> {
@@ -93,24 +116,43 @@ impl Worker {
Ok(())
}
fn next(&mut self) -> Result<Option<Envelope>> {
if self.running.as_ref().is_some_and(|r| r.task.is_finished()) {
let r = self.running.take().unwrap();
let o = r
.task
.join()
.unwrap_or_else(|_| Outcome::unknown("WORKER_INTERRUPTED"));
ledger::persist_outcome(&self.ledger, &r.grant, &o)?;
self.result(&r.offer, &r.grant.grant_id.to_string(), o.clone())?;
ledger::finish(&self.ledger, &r.grant, &o)?;
}
if let Some(mut e) = jobs::events(&self.db)?.into_iter().next() {
e.connection_generation = self.channel.connection_generation;
return Ok(Some(e));
}
// A refused or unqualified clock closes the effect gate, not telemetry.
if self.running.is_some() {
return Ok(None);
}
if Instant::now() >= self.sync_due {
self.sync_due = Instant::now() + Duration::from_secs(40);
if self.clock.now().is_err() {
self.sync.close();
return Ok(None);
}
let snapshot = if self.root_enabled {
Some(crate::root::snapshot(self.channel.connection_generation)?)
} else {
None
};
let p = self.sync.prepare(
&mut self.db,
&self.ledger,
self.channel.clone(),
None,
snapshot.as_ref(),
&self.clock,
)?;
self.pending_sync = Some(p.request);
return Ok(Some(p.envelope));
}
if self.sync.can_start(&self.channel, false, &self.clock)
@@ -126,7 +168,12 @@ impl Worker {
)?;
return self.next();
}
ensure!(supported(&job), "UNSUPPORTED_JOB");
let root = self.root_enabled && crate::root::supported(&job);
ensure!(supported(&job) || root, "UNSUPPORTED_JOB");
if !self.sync.can_start(&self.channel, root, &self.clock) {
self.sync_due = Instant::now();
return Ok(None);
}
jobs::reserve_request(&self.db, &offer)?;
return self
.envelope(Payload::ExecutionPermitRequest(ExecutionPermitRequest {
@@ -151,9 +198,16 @@ impl Worker {
match e.payload.context("EMPTY_REPLY")? {
Payload::Ack(a) => jobs::ack(&mut self.db, &a.message_id)?,
Payload::SyncResponse(s) => {
let q = self.pending_sync.take().context("UNEXPECTED_SYNC")?;
let applied = self.sync.apply(&mut self.db, &s, &self.clock)?;
ensure!(applied.helper_instruction.is_none(), "UNEXPECTED_HELPER");
self.sync.apply_consumption(&mut self.ledger, &self.clock)?;
if applied.helper_instruction.is_some() {
ensure!(self.root_enabled, "UNEXPECTED_HELPER");
let snapshot = crate::root::apply(&q, &s)?;
if self.sync.can_start(&self.channel, false, &self.clock) {
self.sync.confirm_helper(&snapshot)?;
}
}
}
Payload::JobOffer(o) => {
let receipt = self.envelope(Payload::JobAccepted(JobAccepted {
@@ -163,7 +217,9 @@ impl Worker {
journal_proof: hash(&o.encode_to_vec()),
}))?;
let tx = self.db.transaction()?;
let fresh = jobs::accept(&tx, &self.binding, &o, supported)?;
let fresh = jobs::accept(&tx, &self.binding, &o, |j| {
supported(j) || (self.root_enabled && crate::root::supported(j))
})?;
jobs::event(&tx, &receipt)?;
tx.commit()?;
if fresh {
@@ -172,6 +228,12 @@ impl Worker {
}
Payload::ExecutionPermit(p) => self.permit(p)?,
Payload::CancelRequest(c) => {
if let Some(r) = &self.running
&& r.offer.job_id == c.job_id
&& r.offer.attempt_id == c.attempt_id
{
r.cancel.store(true, Ordering::Release);
}
let offer:Option<Vec<u8>>=rusqlite::OptionalExtension::optional(self.db.query_row(
"UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2 AND state='accepted' RETURNING offer",
params![c.job_id,c.attempt_id],|r|r.get(0)))?;
@@ -197,9 +259,10 @@ impl Worker {
return Ok(());
}
let req = jobs::permit_request(&offer, &p)?;
let root = self.binding.job(&offer.payload_bytes)?.execution_identity == "root";
let verify = || -> Result<_> {
ensure!(
self.sync.can_start(&self.channel, false, &self.clock),
self.sync.can_start(&self.channel, root, &self.clock),
"SYNC_REQUIRED"
);
let (low, high) = self.clock.bounds()?;
@@ -221,14 +284,23 @@ impl Worker {
Err(e) => return Err(e),
}
};
ensure!(supported(&job), "UNSUPPORTED_JOB");
let Action::RunScript { script_utf8, .. } = &job.action else {
ensure!(
supported(&job) || (self.root_enabled && crate::root::supported(&job)),
"UNSUPPORTED_JOB"
);
let Action::RunScript {
script_utf8,
arguments,
expected_stdout,
..
} = &job.action
else {
bail!("SCRIPT_REQUIRED")
};
let script = ScriptRequest::new(
script_utf8.as_bytes().to_vec(),
Interpreter::PosixSh,
vec![],
arguments.clone(),
Duration::from_secs(job.max_runtime_seconds.into()),
job.max_output_bytes as usize,
)?;
@@ -252,46 +324,69 @@ impl Worker {
}));
jobs::event(&tx, &running)?;
tx.commit()?;
let mut first = true;
let result = run_script_controlled(&script, || {
if first {
ensure!(
self.sync.can_start(&self.channel, false, &self.clock),
"SYNC_EXPIRED"
);
let (lo, hi) = self.clock.bounds()?;
self.binding.verify(&req, lo)?;
self.binding.verify(&req, hi)?;
first = false;
} else {
self.clock.now()?;
let cancel = Arc::new(AtomicBool::new(false));
let flag = cancel.clone();
let deadline = self.sync.validity_deadline().context("SYNC_LEASE")?;
let binding = self.binding.clone();
let generation = self.channel.connection_generation;
let expected = expected_stdout.clone();
let seconds = job.max_runtime_seconds;
let digest = hash(script_utf8.as_bytes());
let clock = self.clock.clone();
let task = std::thread::spawn(move || {
if root {
return crate::root::execute(generation, req, &flag, seconds)
.unwrap_or_else(|_| Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN"));
}
Ok(())
});
let outcome = match result {
Ok(o) => {
let ok = o.code == Some(0)
&& !o.timed_out
&& !o.interrupted
&& !o.truncated
&& o.stdout == b"FreeBSD\n";
Outcome {
state: if ok { "succeeded" } else { "failed" }.into(),
stdout: o.stdout,
stderr: o.stderr,
output_truncated: o.truncated,
reboot_required: false,
exit_code: o.code,
evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()}}),
error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(),
let mut first = true;
let result = run_script_controlled(&script, || {
ensure!(!flag.load(Ordering::Acquire), "CANCEL_OR_FENCE");
if first {
let (lo, hi) = clock.bounds()?;
ensure!(hi < deadline, "SYNC_EXPIRED");
binding.verify(&req, lo)?;
binding.verify(&req, hi)?;
first = false;
} else {
clock.now()?;
}
Ok(())
});
match result {
Ok(o) => {
let ok = o.code == Some(0)
&& !o.timed_out
&& !o.interrupted
&& expected
.as_ref()
.is_none_or(|v| !o.truncated && o.stdout == v.as_bytes());
Outcome {
state: if ok {
"succeeded"
} else if o.interrupted || o.timed_out || o.code.is_none() {
"unknown"
} else {
"failed"
}
.into(),
stdout: o.stdout,
stderr: o.stderr,
output_truncated: o.truncated,
reboot_required: false,
exit_code: o.code,
evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()},"script_sha256":digest}),
error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(),
}
}
Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"),
}
Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"),
};
ledger::persist_outcome(&self.ledger, &grant, &outcome)?;
let saved = serde_json::to_vec(&outcome)?;
self.result(&offer, &id, outcome)?;
ledger::finish(&self.ledger, &grant, &serde_json::from_slice(&saved)?)?;
});
self.running = Some(Running {
offer,
grant,
task,
cancel,
});
Ok(())
}
}
@@ -368,6 +463,9 @@ pub fn main() -> Result<()> {
clock: NativeClock::default(),
sync_due: Instant::now(),
permit_after: Instant::now(),
pending_sync: None,
running: None,
root_enabled: env::var_os("DISCOVERY_ROOT_SESSION").is_some(),
};
w.recover()?;
let mut input = io::stdin().lock();