497 lines
19 KiB
Rust
497 lines
19 KiB
Rust
//! One worker per authenticated Python HTTPS channel, never a network listener.
|
|
//! Executes approved service-account POSIX scripts or delegates approved root
|
|
//! scripts to the independently synchronized local helper.
|
|
use anyhow::{Context, Result, bail, ensure};
|
|
use chrono::Utc;
|
|
use prost::Message;
|
|
use rmm_agent_core::{
|
|
auth::{Binding, Outcome},
|
|
freebsd_clock::NativeClock,
|
|
jobs, ledger,
|
|
reconcile::{self, Channel, ClientSync, SyncClock},
|
|
};
|
|
use rmm_agent_linux::actions::{Interpreter, ScriptRequest, run_script_controlled};
|
|
use rmm_domain::jobs::{Action, JobPayload, hash};
|
|
use rmm_protocol::*;
|
|
use rusqlite::{Connection, params};
|
|
use std::{
|
|
env, fs,
|
|
io::{self, BufRead, Read, Write},
|
|
os::{
|
|
fd::AsRawFd,
|
|
unix::fs::{MetadataExt, OpenOptionsExt},
|
|
},
|
|
path::Path,
|
|
sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, Ordering},
|
|
},
|
|
thread::JoinHandle,
|
|
time::{Duration, Instant},
|
|
};
|
|
use uuid::Uuid;
|
|
|
|
#[cfg(test)]
|
|
mod tests;
|
|
|
|
fn supported(job: &JobPayload) -> bool {
|
|
job.execution_identity == "agent_service"
|
|
&& job.action.opnsense_service_script()
|
|
&& job
|
|
.action
|
|
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
|
|
}
|
|
struct Running {
|
|
offer: JobOffer,
|
|
grant: rmm_domain::jobs::ExecutionPermitPayload,
|
|
cancel: Arc<AtomicBool>,
|
|
task: JoinHandle<Outcome>,
|
|
}
|
|
|
|
struct Worker {
|
|
binding: Binding,
|
|
channel: Channel,
|
|
sync: ClientSync,
|
|
clock: NativeClock,
|
|
db: Connection,
|
|
ledger: Connection,
|
|
sync_due: Instant,
|
|
permit_after: Instant,
|
|
root_enabled: bool,
|
|
pending_sync: Option<SyncRequest>,
|
|
running: Option<Running>,
|
|
}
|
|
impl Drop for Worker {
|
|
fn drop(&mut self) {
|
|
if let Some(r) = self.running.take() {
|
|
r.cancel.store(true, Ordering::Release);
|
|
let _ = r.task.join();
|
|
}
|
|
}
|
|
}
|
|
impl Worker {
|
|
fn envelope(&self, payload: Payload) -> Result<Envelope> {
|
|
let seq = self.db.query_row(
|
|
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
Ok(Envelope {
|
|
protocol_major: 1,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: self.binding.agent_id.to_string(),
|
|
enrollment_generation: self.binding.enrollment_generation,
|
|
connection_generation: self.channel.connection_generation,
|
|
boot_id: self.channel.boot_id.to_string(),
|
|
sequence: seq,
|
|
observed_at: Utc::now().timestamp_millis(),
|
|
payload: Some(payload),
|
|
..Default::default()
|
|
})
|
|
}
|
|
fn result(&mut self, offer: &JobOffer, grant: &str, outcome: Outcome) -> Result<()> {
|
|
let mut e = self.envelope(Payload::JobResult(Default::default()))?;
|
|
let tx = self.db.transaction()?;
|
|
let rev = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
|
e.payload = Some(Payload::JobResult(jobs::result(offer, rev, grant, outcome)));
|
|
jobs::event(&tx, &e)?;
|
|
tx.commit()?;
|
|
self.sync_due = Instant::now();
|
|
Ok(())
|
|
}
|
|
fn recover(&mut self) -> Result<()> {
|
|
for (offer, grant) in jobs::interrupted(&self.db, &self.ledger, &self.binding)? {
|
|
if grant.is_empty() {
|
|
jobs::interrupted_request(&self.db, &offer)?;
|
|
} else {
|
|
let saved =
|
|
jobs::recover_interrupted_outcome(&self.ledger, &self.binding, &offer, &grant)?;
|
|
self.result(
|
|
&offer,
|
|
&grant,
|
|
saved.unwrap_or_else(|| Outcome::unknown("AGENT_PROCESS_INTERRUPTED")),
|
|
)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
fn next(&mut self) -> Result<Option<Envelope>> {
|
|
if self.running.as_ref().is_some_and(|r| r.task.is_finished()) {
|
|
let r = self.running.take().unwrap();
|
|
let o = r
|
|
.task
|
|
.join()
|
|
.unwrap_or_else(|_| Outcome::unknown("WORKER_INTERRUPTED"));
|
|
ledger::persist_outcome(&self.ledger, &r.grant, &o)?;
|
|
self.result(&r.offer, &r.grant.grant_id.to_string(), o.clone())?;
|
|
ledger::finish(&self.ledger, &r.grant, &o)?;
|
|
}
|
|
if let Some(mut e) = jobs::events(&self.db)?.into_iter().next() {
|
|
e.connection_generation = self.channel.connection_generation;
|
|
return Ok(Some(e));
|
|
}
|
|
// A refused or unqualified clock closes the effect gate, not telemetry.
|
|
if self.running.is_some() {
|
|
return Ok(None);
|
|
}
|
|
if Instant::now() >= self.sync_due {
|
|
self.sync_due = Instant::now() + Duration::from_secs(40);
|
|
if self.clock.now().is_err() {
|
|
self.sync.close();
|
|
return Ok(None);
|
|
}
|
|
let snapshot = if self.root_enabled {
|
|
Some(crate::root::snapshot(self.channel.connection_generation)?)
|
|
} else {
|
|
None
|
|
};
|
|
let p = self.sync.prepare(
|
|
&mut self.db,
|
|
&self.ledger,
|
|
self.channel.clone(),
|
|
snapshot.as_ref(),
|
|
&self.clock,
|
|
)?;
|
|
self.pending_sync = Some(p.request);
|
|
return Ok(Some(p.envelope));
|
|
}
|
|
if self.sync.can_start(&self.channel, false, &self.clock)
|
|
&& Instant::now() >= self.permit_after
|
|
&& let Some(offer) = jobs::next(&self.db)?
|
|
{
|
|
let job = self.binding.job(&offer.payload_bytes)?;
|
|
if self.clock.now()? >= job.start_before {
|
|
self.result(
|
|
&offer,
|
|
"",
|
|
jobs::no_start("expired", "START_DEADLINE_EXPIRED"),
|
|
)?;
|
|
return self.next();
|
|
}
|
|
let root = self.root_enabled && crate::root::supported(&job);
|
|
ensure!(supported(&job) || root, "UNSUPPORTED_JOB");
|
|
if !self.sync.can_start(&self.channel, root, &self.clock) {
|
|
self.sync_due = Instant::now();
|
|
return Ok(None);
|
|
}
|
|
jobs::reserve_request(&self.db, &offer)?;
|
|
return self
|
|
.envelope(Payload::ExecutionPermitRequest(ExecutionPermitRequest {
|
|
job_id: offer.job_id,
|
|
attempt_id: offer.attempt_id,
|
|
job_payload_hash: offer.job_payload_hash,
|
|
assignment_version: self.binding.assignment_version,
|
|
}))
|
|
.map(Some);
|
|
}
|
|
Ok(None)
|
|
}
|
|
fn receive(&mut self, e: Envelope) -> Result<()> {
|
|
ensure!(
|
|
valid_envelope(
|
|
&e,
|
|
&self.binding.agent_id.to_string(),
|
|
self.channel.connection_generation
|
|
) && e.enrollment_generation == self.binding.enrollment_generation,
|
|
"WRONG_CHANNEL"
|
|
);
|
|
match e.payload.context("EMPTY_REPLY")? {
|
|
Payload::Ack(a) => jobs::ack(&mut self.db, &a.message_id)?,
|
|
Payload::SyncResponse(s) => {
|
|
let q = self.pending_sync.take().context("UNEXPECTED_SYNC")?;
|
|
let applied = self.sync.apply(&mut self.db, &s, &self.clock)?;
|
|
self.sync.apply_consumption(&mut self.ledger, &self.clock)?;
|
|
if applied.helper_instruction.is_some() {
|
|
ensure!(self.root_enabled, "UNEXPECTED_HELPER");
|
|
let snapshot = crate::root::apply(&q, &s)?;
|
|
if self.sync.can_start(&self.channel, false, &self.clock) {
|
|
self.sync.confirm_helper(&snapshot)?;
|
|
}
|
|
}
|
|
}
|
|
Payload::JobOffer(o) => {
|
|
let receipt = self.envelope(Payload::JobAccepted(JobAccepted {
|
|
job_id: o.job_id.clone(),
|
|
attempt_id: o.attempt_id.clone(),
|
|
job_payload_hash: o.job_payload_hash.clone(),
|
|
journal_proof: hash(&o.encode_to_vec()),
|
|
}))?;
|
|
let tx = self.db.transaction()?;
|
|
let fresh = jobs::accept(&tx, &self.binding, &o, |j| {
|
|
supported(j) || (self.root_enabled && crate::root::supported(j))
|
|
})?;
|
|
jobs::event(&tx, &receipt)?;
|
|
tx.commit()?;
|
|
if fresh {
|
|
self.sync_due = Instant::now();
|
|
}
|
|
}
|
|
Payload::ExecutionPermit(p) => self.permit(p)?,
|
|
Payload::CancelRequest(c) => {
|
|
if let Some(r) = &self.running
|
|
&& r.offer.job_id == c.job_id
|
|
&& r.offer.attempt_id == c.attempt_id
|
|
{
|
|
r.cancel.store(true, Ordering::Release);
|
|
}
|
|
let offer:Option<Vec<u8>>=rusqlite::OptionalExtension::optional(self.db.query_row(
|
|
"UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2 AND state='accepted' RETURNING offer",
|
|
params![c.job_id,c.attempt_id],|r|r.get(0)))?;
|
|
if let Some(bytes) = offer {
|
|
self.result(
|
|
&JobOffer::decode(bytes.as_slice())?,
|
|
"",
|
|
jobs::no_start("cancelled", "CANCELLED_BEFORE_START"),
|
|
)?;
|
|
}
|
|
}
|
|
_ => bail!("UNEXPECTED_JOB_MESSAGE"),
|
|
}
|
|
Ok(())
|
|
}
|
|
fn permit(&mut self, p: ExecutionPermit) -> Result<()> {
|
|
let offer = jobs::requesting(&self.db)?.context("UNSOLICITED_PERMIT")?;
|
|
if !p.denial_code.is_empty() {
|
|
jobs::denied_request(&self.db, &offer, &p)?;
|
|
self.sync.close();
|
|
self.sync_due = Instant::now();
|
|
self.permit_after = Instant::now() + Duration::from_secs(30);
|
|
return Ok(());
|
|
}
|
|
let req = jobs::permit_request(&offer, &p)?;
|
|
let root = self.binding.job(&offer.payload_bytes)?.execution_identity == "root";
|
|
let verify = || -> Result<_> {
|
|
ensure!(
|
|
self.sync.can_start(&self.channel, root, &self.clock),
|
|
"SYNC_REQUIRED"
|
|
);
|
|
let (low, high) = self.clock.bounds()?;
|
|
self.binding.verify(&req, high)?;
|
|
self.binding.verify(&req, low)
|
|
};
|
|
// The signed not_before is the reservation instant. Wait within the
|
|
// qualified uncertainty interval before checking both ends at start.
|
|
let began = Instant::now();
|
|
let (job, grant) = loop {
|
|
match verify() {
|
|
Ok(x) => break x,
|
|
Err(err)
|
|
if err.to_string() == "PERMIT_TIME"
|
|
&& began.elapsed() < Duration::from_secs(6) =>
|
|
{
|
|
std::thread::sleep(Duration::from_millis(50));
|
|
}
|
|
Err(e) => return Err(e),
|
|
}
|
|
};
|
|
ensure!(
|
|
supported(&job) || (self.root_enabled && crate::root::supported(&job)),
|
|
"UNSUPPORTED_JOB"
|
|
);
|
|
let Action::RunScript {
|
|
script_utf8,
|
|
arguments,
|
|
expected_stdout,
|
|
..
|
|
} = &job.action
|
|
else {
|
|
bail!("SCRIPT_REQUIRED")
|
|
};
|
|
let script = ScriptRequest::new(
|
|
script_utf8.as_bytes().to_vec(),
|
|
Interpreter::PosixSh,
|
|
arguments.clone(),
|
|
Duration::from_secs(job.max_runtime_seconds.into()),
|
|
job.max_output_bytes as usize,
|
|
)?;
|
|
ledger::begin_execution(
|
|
&mut self.ledger,
|
|
&grant,
|
|
&req,
|
|
self.channel.boot_id,
|
|
self.clock.now()?,
|
|
)?;
|
|
let id = grant.grant_id.to_string();
|
|
let mut running = self.envelope(Payload::JobStateChanged(Default::default()))?;
|
|
let tx = self.db.transaction()?;
|
|
let revision = jobs::transition(&tx, &offer, "running", &id)?;
|
|
running.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
grant_id: id.clone(),
|
|
revision,
|
|
state: "running".into(),
|
|
}));
|
|
jobs::event(&tx, &running)?;
|
|
tx.commit()?;
|
|
let cancel = Arc::new(AtomicBool::new(false));
|
|
let flag = cancel.clone();
|
|
let deadline = self.sync.validity_deadline().context("SYNC_LEASE")?;
|
|
let binding = self.binding.clone();
|
|
let generation = self.channel.connection_generation;
|
|
let expected = expected_stdout.clone();
|
|
let seconds = job.max_runtime_seconds;
|
|
let digest = hash(script_utf8.as_bytes());
|
|
let clock = self.clock.clone();
|
|
let task = std::thread::spawn(move || {
|
|
if root {
|
|
return crate::root::execute(generation, req, &flag, seconds)
|
|
.unwrap_or_else(|_| Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN"));
|
|
}
|
|
let mut first = true;
|
|
let result = run_script_controlled(&script, || {
|
|
ensure!(!flag.load(Ordering::Acquire), "CANCEL_OR_FENCE");
|
|
if first {
|
|
let (lo, hi) = clock.bounds()?;
|
|
ensure!(hi < deadline, "SYNC_EXPIRED");
|
|
binding.verify(&req, lo)?;
|
|
binding.verify(&req, hi)?;
|
|
first = false;
|
|
} else {
|
|
clock.now()?;
|
|
}
|
|
Ok(())
|
|
});
|
|
match result {
|
|
Ok(o) => {
|
|
let ok = o.code == Some(0)
|
|
&& !o.timed_out
|
|
&& !o.interrupted
|
|
&& expected
|
|
.as_ref()
|
|
.is_none_or(|v| !o.truncated && o.stdout == v.as_bytes());
|
|
Outcome {
|
|
state: if ok {
|
|
"succeeded"
|
|
} else if o.interrupted || o.timed_out || o.code.is_none() {
|
|
"unknown"
|
|
} else {
|
|
"failed"
|
|
}
|
|
.into(),
|
|
stdout: o.stdout,
|
|
stderr: o.stderr,
|
|
output_truncated: o.truncated,
|
|
reboot_required: false,
|
|
exit_code: o.code,
|
|
evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()},"script_sha256":digest}),
|
|
error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(),
|
|
}
|
|
}
|
|
Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"),
|
|
}
|
|
});
|
|
self.running = Some(Running {
|
|
offer,
|
|
grant,
|
|
task,
|
|
cancel,
|
|
});
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
pub fn main() -> Result<()> {
|
|
let args = env::args().skip(1).collect::<Vec<_>>();
|
|
ensure!(args.len() == 4, "bound channel required");
|
|
let uid = unsafe { libc::geteuid() };
|
|
ensure!(
|
|
uid != 0 && uid == unsafe { libc::getuid() },
|
|
"SERVICE_ACCOUNT_REQUIRED"
|
|
);
|
|
unsafe { libc::umask(0o077) };
|
|
let state = Path::new("/var/db/discovery-agent");
|
|
let m = fs::symlink_metadata(state)?;
|
|
ensure!(
|
|
m.is_dir() && m.uid() == uid && m.mode() & 0o077 == 0,
|
|
"PRIVATE_STATE_REQUIRED"
|
|
);
|
|
let lock = fs::OpenOptions::new()
|
|
.write(true)
|
|
.create(true)
|
|
.truncate(false)
|
|
.mode(0o600)
|
|
.custom_flags(libc::O_NOFOLLOW)
|
|
.open(state.join("job-worker.lock"))?;
|
|
ensure!(
|
|
unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0,
|
|
"ACTIVE_JOB_WORKER"
|
|
);
|
|
let config = Path::new("/usr/local/etc/discovery-agent/job-binding.json");
|
|
for path in [config.parent().unwrap(), config] {
|
|
let m = fs::symlink_metadata(path)?;
|
|
ensure!(
|
|
!m.is_symlink() && m.uid() == 0 && m.mode() & 0o022 == 0,
|
|
"PROTECTED_BINDING_REQUIRED"
|
|
);
|
|
}
|
|
let binding: Binding = serde_json::from_slice(&fs::read(config)?)?;
|
|
ensure!(
|
|
args[0] == binding.agent_id.to_string() && args[1] == binding.asset_id.to_string(),
|
|
"WRONG_DEVICE"
|
|
);
|
|
let channel = Channel {
|
|
boot_id: args[2].parse()?,
|
|
connection_generation: args[3].parse()?,
|
|
};
|
|
ensure!(
|
|
channel.connection_generation > 0 && !channel.boot_id.is_nil(),
|
|
"CHANNEL_REQUIRED"
|
|
);
|
|
let db = Connection::open(state.join("channel.sqlite"))?;
|
|
ledger::bounded_journal_capacity(&db)?;
|
|
db.execute_batch("PRAGMA synchronous=FULL;
|
|
CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL);
|
|
INSERT OR IGNORE INTO counter SELECT id,n FROM sequence;
|
|
UPDATE counter SET seq=max(seq,(SELECT n FROM sequence WHERE id=1)) WHERE id=1;
|
|
CREATE TRIGGER IF NOT EXISTS telemetry_sequence AFTER UPDATE ON sequence WHEN NEW.n>(SELECT seq FROM counter WHERE id=1)
|
|
BEGIN UPDATE counter SET seq=NEW.n WHERE id=1; END;
|
|
CREATE TRIGGER IF NOT EXISTS job_sequence AFTER UPDATE ON counter WHEN NEW.seq>(SELECT n FROM sequence WHERE id=1)
|
|
BEGIN UPDATE sequence SET n=NEW.seq WHERE id=1; END;
|
|
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL,attempt TEXT NOT NULL,offer BLOB NOT NULL,state TEXT NOT NULL,
|
|
revision INTEGER NOT NULL DEFAULT 0,grant_id TEXT NOT NULL DEFAULT '',cancel_requested INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(job,attempt));
|
|
CREATE TABLE IF NOT EXISTS job_events(id TEXT PRIMARY KEY,payload BLOB NOT NULL);")?;
|
|
reconcile::initialize(&db)?;
|
|
let ledger = Connection::open(state.join("consumption.sqlite"))?;
|
|
ledger::initialize_ledger(&ledger)?;
|
|
let mut w = Worker {
|
|
sync: ClientSync::new(binding.clone()),
|
|
binding,
|
|
db,
|
|
ledger,
|
|
channel,
|
|
clock: NativeClock::default(),
|
|
sync_due: Instant::now(),
|
|
permit_after: Instant::now(),
|
|
pending_sync: None,
|
|
running: None,
|
|
root_enabled: env::var_os("DISCOVERY_ROOT_SESSION").is_some(),
|
|
};
|
|
w.recover()?;
|
|
let mut input = io::stdin().lock();
|
|
println!("ready");
|
|
io::stdout().flush()?;
|
|
loop {
|
|
let mut line = String::new();
|
|
let n = (&mut input).take(2_100_000).read_line(&mut line)?;
|
|
if n == 0 {
|
|
return Ok(());
|
|
}
|
|
ensure!(line.ends_with('\n'), "FRAME_LIMIT");
|
|
let line = line.trim_end();
|
|
if line == "next" {
|
|
println!(
|
|
"{}",
|
|
w.next()?
|
|
.map(|e| hex::encode(e.encode_to_vec()))
|
|
.unwrap_or_default()
|
|
);
|
|
} else {
|
|
let data = hex::decode(line)?;
|
|
w.receive(Envelope::decode(data.as_slice())?)?;
|
|
println!("ok");
|
|
}
|
|
io::stdout().flush()?;
|
|
}
|
|
}
|