Run approved FreeBSD root jobs through independently pinned and synchronized local helper
Rust development slice / linux (push) Failing after 11s
Rust development slice / linux (push) Failing after 11s
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
//! One worker per authenticated Python HTTPS channel, never a network listener.
|
||||
//! Only the fixed unprivileged probe is executable. The shared Rust core owns
|
||||
//! all approval/permit verification, receipt, consumption and sync transitions.
|
||||
//! Executes approved service-account POSIX scripts or delegates approved root
|
||||
//! scripts to the independently synchronized local helper.
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use chrono::Utc;
|
||||
use prost::Message;
|
||||
@@ -22,6 +22,11 @@ use std::{
|
||||
unix::fs::{MetadataExt, OpenOptionsExt},
|
||||
},
|
||||
path::Path,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
thread::JoinHandle,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
@@ -31,9 +36,16 @@ mod tests;
|
||||
|
||||
fn supported(job: &JobPayload) -> bool {
|
||||
job.execution_identity == "agent_service"
|
||||
&& job.action.opnsense_kernel_probe()
|
||||
&& job.max_runtime_seconds == 10
|
||||
&& job.max_output_bytes == 1024
|
||||
&& job.action.opnsense_service_script()
|
||||
&& job
|
||||
.action
|
||||
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
|
||||
}
|
||||
struct Running {
|
||||
offer: JobOffer,
|
||||
grant: rmm_domain::jobs::ExecutionPermitPayload,
|
||||
cancel: Arc<AtomicBool>,
|
||||
task: JoinHandle<Outcome>,
|
||||
}
|
||||
|
||||
struct Worker {
|
||||
@@ -45,6 +57,17 @@ struct Worker {
|
||||
ledger: Connection,
|
||||
sync_due: Instant,
|
||||
permit_after: Instant,
|
||||
root_enabled: bool,
|
||||
pending_sync: Option<SyncRequest>,
|
||||
running: Option<Running>,
|
||||
}
|
||||
impl Drop for Worker {
|
||||
fn drop(&mut self) {
|
||||
if let Some(r) = self.running.take() {
|
||||
r.cancel.store(true, Ordering::Release);
|
||||
let _ = r.task.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Worker {
|
||||
fn envelope(&self, payload: Payload) -> Result<Envelope> {
|
||||
@@ -93,24 +116,43 @@ impl Worker {
|
||||
Ok(())
|
||||
}
|
||||
fn next(&mut self) -> Result<Option<Envelope>> {
|
||||
if self.running.as_ref().is_some_and(|r| r.task.is_finished()) {
|
||||
let r = self.running.take().unwrap();
|
||||
let o = r
|
||||
.task
|
||||
.join()
|
||||
.unwrap_or_else(|_| Outcome::unknown("WORKER_INTERRUPTED"));
|
||||
ledger::persist_outcome(&self.ledger, &r.grant, &o)?;
|
||||
self.result(&r.offer, &r.grant.grant_id.to_string(), o.clone())?;
|
||||
ledger::finish(&self.ledger, &r.grant, &o)?;
|
||||
}
|
||||
if let Some(mut e) = jobs::events(&self.db)?.into_iter().next() {
|
||||
e.connection_generation = self.channel.connection_generation;
|
||||
return Ok(Some(e));
|
||||
}
|
||||
// A refused or unqualified clock closes the effect gate, not telemetry.
|
||||
if self.running.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
if Instant::now() >= self.sync_due {
|
||||
self.sync_due = Instant::now() + Duration::from_secs(40);
|
||||
if self.clock.now().is_err() {
|
||||
self.sync.close();
|
||||
return Ok(None);
|
||||
}
|
||||
let snapshot = if self.root_enabled {
|
||||
Some(crate::root::snapshot(self.channel.connection_generation)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let p = self.sync.prepare(
|
||||
&mut self.db,
|
||||
&self.ledger,
|
||||
self.channel.clone(),
|
||||
None,
|
||||
snapshot.as_ref(),
|
||||
&self.clock,
|
||||
)?;
|
||||
self.pending_sync = Some(p.request);
|
||||
return Ok(Some(p.envelope));
|
||||
}
|
||||
if self.sync.can_start(&self.channel, false, &self.clock)
|
||||
@@ -126,7 +168,12 @@ impl Worker {
|
||||
)?;
|
||||
return self.next();
|
||||
}
|
||||
ensure!(supported(&job), "UNSUPPORTED_JOB");
|
||||
let root = self.root_enabled && crate::root::supported(&job);
|
||||
ensure!(supported(&job) || root, "UNSUPPORTED_JOB");
|
||||
if !self.sync.can_start(&self.channel, root, &self.clock) {
|
||||
self.sync_due = Instant::now();
|
||||
return Ok(None);
|
||||
}
|
||||
jobs::reserve_request(&self.db, &offer)?;
|
||||
return self
|
||||
.envelope(Payload::ExecutionPermitRequest(ExecutionPermitRequest {
|
||||
@@ -151,9 +198,16 @@ impl Worker {
|
||||
match e.payload.context("EMPTY_REPLY")? {
|
||||
Payload::Ack(a) => jobs::ack(&mut self.db, &a.message_id)?,
|
||||
Payload::SyncResponse(s) => {
|
||||
let q = self.pending_sync.take().context("UNEXPECTED_SYNC")?;
|
||||
let applied = self.sync.apply(&mut self.db, &s, &self.clock)?;
|
||||
ensure!(applied.helper_instruction.is_none(), "UNEXPECTED_HELPER");
|
||||
self.sync.apply_consumption(&mut self.ledger, &self.clock)?;
|
||||
if applied.helper_instruction.is_some() {
|
||||
ensure!(self.root_enabled, "UNEXPECTED_HELPER");
|
||||
let snapshot = crate::root::apply(&q, &s)?;
|
||||
if self.sync.can_start(&self.channel, false, &self.clock) {
|
||||
self.sync.confirm_helper(&snapshot)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Payload::JobOffer(o) => {
|
||||
let receipt = self.envelope(Payload::JobAccepted(JobAccepted {
|
||||
@@ -163,7 +217,9 @@ impl Worker {
|
||||
journal_proof: hash(&o.encode_to_vec()),
|
||||
}))?;
|
||||
let tx = self.db.transaction()?;
|
||||
let fresh = jobs::accept(&tx, &self.binding, &o, supported)?;
|
||||
let fresh = jobs::accept(&tx, &self.binding, &o, |j| {
|
||||
supported(j) || (self.root_enabled && crate::root::supported(j))
|
||||
})?;
|
||||
jobs::event(&tx, &receipt)?;
|
||||
tx.commit()?;
|
||||
if fresh {
|
||||
@@ -172,6 +228,12 @@ impl Worker {
|
||||
}
|
||||
Payload::ExecutionPermit(p) => self.permit(p)?,
|
||||
Payload::CancelRequest(c) => {
|
||||
if let Some(r) = &self.running
|
||||
&& r.offer.job_id == c.job_id
|
||||
&& r.offer.attempt_id == c.attempt_id
|
||||
{
|
||||
r.cancel.store(true, Ordering::Release);
|
||||
}
|
||||
let offer:Option<Vec<u8>>=rusqlite::OptionalExtension::optional(self.db.query_row(
|
||||
"UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2 AND state='accepted' RETURNING offer",
|
||||
params![c.job_id,c.attempt_id],|r|r.get(0)))?;
|
||||
@@ -197,9 +259,10 @@ impl Worker {
|
||||
return Ok(());
|
||||
}
|
||||
let req = jobs::permit_request(&offer, &p)?;
|
||||
let root = self.binding.job(&offer.payload_bytes)?.execution_identity == "root";
|
||||
let verify = || -> Result<_> {
|
||||
ensure!(
|
||||
self.sync.can_start(&self.channel, false, &self.clock),
|
||||
self.sync.can_start(&self.channel, root, &self.clock),
|
||||
"SYNC_REQUIRED"
|
||||
);
|
||||
let (low, high) = self.clock.bounds()?;
|
||||
@@ -221,14 +284,23 @@ impl Worker {
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
};
|
||||
ensure!(supported(&job), "UNSUPPORTED_JOB");
|
||||
let Action::RunScript { script_utf8, .. } = &job.action else {
|
||||
ensure!(
|
||||
supported(&job) || (self.root_enabled && crate::root::supported(&job)),
|
||||
"UNSUPPORTED_JOB"
|
||||
);
|
||||
let Action::RunScript {
|
||||
script_utf8,
|
||||
arguments,
|
||||
expected_stdout,
|
||||
..
|
||||
} = &job.action
|
||||
else {
|
||||
bail!("SCRIPT_REQUIRED")
|
||||
};
|
||||
let script = ScriptRequest::new(
|
||||
script_utf8.as_bytes().to_vec(),
|
||||
Interpreter::PosixSh,
|
||||
vec![],
|
||||
arguments.clone(),
|
||||
Duration::from_secs(job.max_runtime_seconds.into()),
|
||||
job.max_output_bytes as usize,
|
||||
)?;
|
||||
@@ -252,46 +324,69 @@ impl Worker {
|
||||
}));
|
||||
jobs::event(&tx, &running)?;
|
||||
tx.commit()?;
|
||||
let mut first = true;
|
||||
let result = run_script_controlled(&script, || {
|
||||
if first {
|
||||
ensure!(
|
||||
self.sync.can_start(&self.channel, false, &self.clock),
|
||||
"SYNC_EXPIRED"
|
||||
);
|
||||
let (lo, hi) = self.clock.bounds()?;
|
||||
self.binding.verify(&req, lo)?;
|
||||
self.binding.verify(&req, hi)?;
|
||||
first = false;
|
||||
} else {
|
||||
self.clock.now()?;
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let flag = cancel.clone();
|
||||
let deadline = self.sync.validity_deadline().context("SYNC_LEASE")?;
|
||||
let binding = self.binding.clone();
|
||||
let generation = self.channel.connection_generation;
|
||||
let expected = expected_stdout.clone();
|
||||
let seconds = job.max_runtime_seconds;
|
||||
let digest = hash(script_utf8.as_bytes());
|
||||
let clock = self.clock.clone();
|
||||
let task = std::thread::spawn(move || {
|
||||
if root {
|
||||
return crate::root::execute(generation, req, &flag, seconds)
|
||||
.unwrap_or_else(|_| Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN"));
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
let outcome = match result {
|
||||
Ok(o) => {
|
||||
let ok = o.code == Some(0)
|
||||
&& !o.timed_out
|
||||
&& !o.interrupted
|
||||
&& !o.truncated
|
||||
&& o.stdout == b"FreeBSD\n";
|
||||
Outcome {
|
||||
state: if ok { "succeeded" } else { "failed" }.into(),
|
||||
stdout: o.stdout,
|
||||
stderr: o.stderr,
|
||||
output_truncated: o.truncated,
|
||||
reboot_required: false,
|
||||
exit_code: o.code,
|
||||
evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()}}),
|
||||
error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(),
|
||||
let mut first = true;
|
||||
let result = run_script_controlled(&script, || {
|
||||
ensure!(!flag.load(Ordering::Acquire), "CANCEL_OR_FENCE");
|
||||
if first {
|
||||
let (lo, hi) = clock.bounds()?;
|
||||
ensure!(hi < deadline, "SYNC_EXPIRED");
|
||||
binding.verify(&req, lo)?;
|
||||
binding.verify(&req, hi)?;
|
||||
first = false;
|
||||
} else {
|
||||
clock.now()?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
match result {
|
||||
Ok(o) => {
|
||||
let ok = o.code == Some(0)
|
||||
&& !o.timed_out
|
||||
&& !o.interrupted
|
||||
&& expected
|
||||
.as_ref()
|
||||
.is_none_or(|v| !o.truncated && o.stdout == v.as_bytes());
|
||||
Outcome {
|
||||
state: if ok {
|
||||
"succeeded"
|
||||
} else if o.interrupted || o.timed_out || o.code.is_none() {
|
||||
"unknown"
|
||||
} else {
|
||||
"failed"
|
||||
}
|
||||
.into(),
|
||||
stdout: o.stdout,
|
||||
stderr: o.stderr,
|
||||
output_truncated: o.truncated,
|
||||
reboot_required: false,
|
||||
exit_code: o.code,
|
||||
evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()},"script_sha256":digest}),
|
||||
error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(),
|
||||
}
|
||||
}
|
||||
Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"),
|
||||
}
|
||||
Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"),
|
||||
};
|
||||
ledger::persist_outcome(&self.ledger, &grant, &outcome)?;
|
||||
let saved = serde_json::to_vec(&outcome)?;
|
||||
self.result(&offer, &id, outcome)?;
|
||||
ledger::finish(&self.ledger, &grant, &serde_json::from_slice(&saved)?)?;
|
||||
});
|
||||
self.running = Some(Running {
|
||||
offer,
|
||||
grant,
|
||||
task,
|
||||
cancel,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -368,6 +463,9 @@ pub fn main() -> Result<()> {
|
||||
clock: NativeClock::default(),
|
||||
sync_due: Instant::now(),
|
||||
permit_after: Instant::now(),
|
||||
pending_sync: None,
|
||||
running: None,
|
||||
root_enabled: env::var_os("DISCOVERY_ROOT_SESSION").is_some(),
|
||||
};
|
||||
w.recover()?;
|
||||
let mut input = io::stdin().lock();
|
||||
|
||||
Reference in New Issue
Block a user