//! One worker per authenticated Python HTTPS channel, never a network listener. //! Executes approved service-account POSIX scripts or delegates approved root //! scripts to the independently synchronized local helper. use anyhow::{Context, Result, bail, ensure}; use chrono::Utc; use prost::Message; use rmm_agent_core::{ auth::{Binding, Outcome}, freebsd_clock::NativeClock, jobs, ledger, reconcile::{self, Channel, ClientSync, SyncClock}, }; use rmm_agent_linux::actions::{Interpreter, ScriptRequest, run_script_controlled}; use rmm_domain::jobs::{Action, JobPayload, hash}; use rmm_protocol::*; use rusqlite::{Connection, params}; use std::{ env, fs, io::{self, BufRead, Read, Write}, os::{ fd::AsRawFd, unix::fs::{MetadataExt, OpenOptionsExt}, }, path::Path, sync::{ Arc, atomic::{AtomicBool, Ordering}, }, thread::JoinHandle, time::{Duration, Instant}, }; use uuid::Uuid; #[cfg(test)] mod tests; fn supported(job: &JobPayload) -> bool { job.execution_identity == "agent_service" && job.action.opnsense_service_script() && job .action .valid_job_limits(job.max_runtime_seconds, job.max_output_bytes) } struct Running { offer: JobOffer, grant: rmm_domain::jobs::ExecutionPermitPayload, cancel: Arc, task: JoinHandle, } struct Worker { binding: Binding, channel: Channel, sync: ClientSync, clock: NativeClock, db: Connection, ledger: Connection, sync_due: Instant, permit_after: Instant, root_enabled: bool, pending_sync: Option, running: Option, } impl Drop for Worker { fn drop(&mut self) { if let Some(r) = self.running.take() { r.cancel.store(true, Ordering::Release); let _ = r.task.join(); } } } impl Worker { fn envelope(&self, payload: Payload) -> Result { let seq = self.db.query_row( "UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq", [], |r| r.get(0), )?; Ok(Envelope { protocol_major: 1, message_id: Uuid::new_v4().to_string(), agent_id: self.binding.agent_id.to_string(), enrollment_generation: self.binding.enrollment_generation, connection_generation: self.channel.connection_generation, boot_id: self.channel.boot_id.to_string(), sequence: seq, observed_at: Utc::now().timestamp_millis(), payload: Some(payload), ..Default::default() }) } fn result(&mut self, offer: &JobOffer, grant: &str, outcome: Outcome) -> Result<()> { let mut e = self.envelope(Payload::JobResult(Default::default()))?; let tx = self.db.transaction()?; let rev = jobs::transition(&tx, offer, &outcome.state, grant)?; e.payload = Some(Payload::JobResult(jobs::result(offer, rev, grant, outcome))); jobs::event(&tx, &e)?; tx.commit()?; self.sync_due = Instant::now(); Ok(()) } fn recover(&mut self) -> Result<()> { for (offer, grant) in jobs::interrupted(&self.db, &self.ledger, &self.binding)? { if grant.is_empty() { jobs::interrupted_request(&self.db, &offer)?; } else { let saved = jobs::recover_interrupted_outcome(&self.ledger, &self.binding, &offer, &grant)?; self.result( &offer, &grant, saved.unwrap_or_else(|| Outcome::unknown("AGENT_PROCESS_INTERRUPTED")), )?; } } Ok(()) } fn next(&mut self) -> Result> { if self.running.as_ref().is_some_and(|r| r.task.is_finished()) { let r = self.running.take().unwrap(); let o = r .task .join() .unwrap_or_else(|_| Outcome::unknown("WORKER_INTERRUPTED")); ledger::persist_outcome(&self.ledger, &r.grant, &o)?; self.result(&r.offer, &r.grant.grant_id.to_string(), o.clone())?; ledger::finish(&self.ledger, &r.grant, &o)?; } if let Some(mut e) = jobs::events(&self.db)?.into_iter().next() { e.connection_generation = self.channel.connection_generation; return Ok(Some(e)); } // A refused or unqualified clock closes the effect gate, not telemetry. if self.running.is_some() { return Ok(None); } if Instant::now() >= self.sync_due { self.sync_due = Instant::now() + Duration::from_secs(40); if self.clock.now().is_err() { self.sync.close(); return Ok(None); } let snapshot = if self.root_enabled { Some(crate::root::snapshot(self.channel.connection_generation)?) } else { None }; let p = self.sync.prepare( &mut self.db, &self.ledger, self.channel.clone(), snapshot.as_ref(), &self.clock, )?; self.pending_sync = Some(p.request); return Ok(Some(p.envelope)); } if self.sync.can_start(&self.channel, false, &self.clock) && Instant::now() >= self.permit_after && let Some(offer) = jobs::next(&self.db)? { let job = self.binding.job(&offer.payload_bytes)?; if self.clock.now()? >= job.start_before { self.result( &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"), )?; return self.next(); } let root = self.root_enabled && crate::root::supported(&job); ensure!(supported(&job) || root, "UNSUPPORTED_JOB"); if !self.sync.can_start(&self.channel, root, &self.clock) { self.sync_due = Instant::now(); return Ok(None); } jobs::reserve_request(&self.db, &offer)?; return self .envelope(Payload::ExecutionPermitRequest(ExecutionPermitRequest { job_id: offer.job_id, attempt_id: offer.attempt_id, job_payload_hash: offer.job_payload_hash, assignment_version: self.binding.assignment_version, })) .map(Some); } Ok(None) } fn receive(&mut self, e: Envelope) -> Result<()> { ensure!( valid_envelope( &e, &self.binding.agent_id.to_string(), self.channel.connection_generation ) && e.enrollment_generation == self.binding.enrollment_generation, "WRONG_CHANNEL" ); match e.payload.context("EMPTY_REPLY")? { Payload::Ack(a) => jobs::ack(&mut self.db, &a.message_id)?, Payload::SyncResponse(s) => { let q = self.pending_sync.take().context("UNEXPECTED_SYNC")?; let applied = self.sync.apply(&mut self.db, &s, &self.clock)?; self.sync.apply_consumption(&mut self.ledger, &self.clock)?; if applied.helper_instruction.is_some() { ensure!(self.root_enabled, "UNEXPECTED_HELPER"); let snapshot = crate::root::apply(&q, &s)?; if self.sync.can_start(&self.channel, false, &self.clock) { self.sync.confirm_helper(&snapshot)?; } } } Payload::JobOffer(o) => { let receipt = self.envelope(Payload::JobAccepted(JobAccepted { job_id: o.job_id.clone(), attempt_id: o.attempt_id.clone(), job_payload_hash: o.job_payload_hash.clone(), journal_proof: hash(&o.encode_to_vec()), }))?; let tx = self.db.transaction()?; let fresh = jobs::accept(&tx, &self.binding, &o, |j| { supported(j) || (self.root_enabled && crate::root::supported(j)) })?; jobs::event(&tx, &receipt)?; tx.commit()?; if fresh { self.sync_due = Instant::now(); } } Payload::ExecutionPermit(p) => self.permit(p)?, Payload::CancelRequest(c) => { if let Some(r) = &self.running && r.offer.job_id == c.job_id && r.offer.attempt_id == c.attempt_id { r.cancel.store(true, Ordering::Release); } let offer:Option>=rusqlite::OptionalExtension::optional(self.db.query_row( "UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2 AND state='accepted' RETURNING offer", params![c.job_id,c.attempt_id],|r|r.get(0)))?; if let Some(bytes) = offer { self.result( &JobOffer::decode(bytes.as_slice())?, "", jobs::no_start("cancelled", "CANCELLED_BEFORE_START"), )?; } } _ => bail!("UNEXPECTED_JOB_MESSAGE"), } Ok(()) } fn permit(&mut self, p: ExecutionPermit) -> Result<()> { let offer = jobs::requesting(&self.db)?.context("UNSOLICITED_PERMIT")?; if !p.denial_code.is_empty() { jobs::denied_request(&self.db, &offer, &p)?; self.sync.close(); self.sync_due = Instant::now(); self.permit_after = Instant::now() + Duration::from_secs(30); return Ok(()); } let req = jobs::permit_request(&offer, &p)?; let root = self.binding.job(&offer.payload_bytes)?.execution_identity == "root"; let verify = || -> Result<_> { ensure!( self.sync.can_start(&self.channel, root, &self.clock), "SYNC_REQUIRED" ); let (low, high) = self.clock.bounds()?; self.binding.verify(&req, high)?; self.binding.verify(&req, low) }; // The signed not_before is the reservation instant. Wait within the // qualified uncertainty interval before checking both ends at start. let began = Instant::now(); let (job, grant) = loop { match verify() { Ok(x) => break x, Err(err) if err.to_string() == "PERMIT_TIME" && began.elapsed() < Duration::from_secs(6) => { std::thread::sleep(Duration::from_millis(50)); } Err(e) => return Err(e), } }; ensure!( supported(&job) || (self.root_enabled && crate::root::supported(&job)), "UNSUPPORTED_JOB" ); let Action::RunScript { script_utf8, arguments, expected_stdout, .. } = &job.action else { bail!("SCRIPT_REQUIRED") }; let script = ScriptRequest::new( script_utf8.as_bytes().to_vec(), Interpreter::PosixSh, arguments.clone(), Duration::from_secs(job.max_runtime_seconds.into()), job.max_output_bytes as usize, )?; ledger::begin_execution( &mut self.ledger, &grant, &req, self.channel.boot_id, self.clock.now()?, )?; let id = grant.grant_id.to_string(); let mut running = self.envelope(Payload::JobStateChanged(Default::default()))?; let tx = self.db.transaction()?; let revision = jobs::transition(&tx, &offer, "running", &id)?; running.payload = Some(Payload::JobStateChanged(JobStateChanged { job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(), grant_id: id.clone(), revision, state: "running".into(), })); jobs::event(&tx, &running)?; tx.commit()?; let cancel = Arc::new(AtomicBool::new(false)); let flag = cancel.clone(); let deadline = self.sync.validity_deadline().context("SYNC_LEASE")?; let binding = self.binding.clone(); let generation = self.channel.connection_generation; let expected = expected_stdout.clone(); let seconds = job.max_runtime_seconds; let digest = hash(script_utf8.as_bytes()); let clock = self.clock.clone(); let task = std::thread::spawn(move || { if root { return crate::root::execute(generation, req, &flag, seconds) .unwrap_or_else(|_| Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")); } let mut first = true; let result = run_script_controlled(&script, || { ensure!(!flag.load(Ordering::Acquire), "CANCEL_OR_FENCE"); if first { let (lo, hi) = clock.bounds()?; ensure!(hi < deadline, "SYNC_EXPIRED"); binding.verify(&req, lo)?; binding.verify(&req, hi)?; first = false; } else { clock.now()?; } Ok(()) }); match result { Ok(o) => { let ok = o.code == Some(0) && !o.timed_out && !o.interrupted && expected .as_ref() .is_none_or(|v| !o.truncated && o.stdout == v.as_bytes()); Outcome { state: if ok { "succeeded" } else if o.interrupted || o.timed_out || o.code.is_none() { "unknown" } else { "failed" } .into(), stdout: o.stdout, stderr: o.stderr, output_truncated: o.truncated, reboot_required: false, exit_code: o.code, evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()},"script_sha256":digest}), error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(), } } Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"), } }); self.running = Some(Running { offer, grant, task, cancel, }); Ok(()) } } pub fn main() -> Result<()> { let args = env::args().skip(1).collect::>(); ensure!(args.len() == 4, "bound channel required"); let uid = unsafe { libc::geteuid() }; ensure!( uid != 0 && uid == unsafe { libc::getuid() }, "SERVICE_ACCOUNT_REQUIRED" ); unsafe { libc::umask(0o077) }; let state = Path::new("/var/db/discovery-agent"); let m = fs::symlink_metadata(state)?; ensure!( m.is_dir() && m.uid() == uid && m.mode() & 0o077 == 0, "PRIVATE_STATE_REQUIRED" ); let lock = fs::OpenOptions::new() .write(true) .create(true) .truncate(false) .mode(0o600) .custom_flags(libc::O_NOFOLLOW) .open(state.join("job-worker.lock"))?; ensure!( unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0, "ACTIVE_JOB_WORKER" ); let config = Path::new("/usr/local/etc/discovery-agent/job-binding.json"); for path in [config.parent().unwrap(), config] { let m = fs::symlink_metadata(path)?; ensure!( !m.is_symlink() && m.uid() == 0 && m.mode() & 0o022 == 0, "PROTECTED_BINDING_REQUIRED" ); } let binding: Binding = serde_json::from_slice(&fs::read(config)?)?; ensure!( args[0] == binding.agent_id.to_string() && args[1] == binding.asset_id.to_string(), "WRONG_DEVICE" ); let channel = Channel { boot_id: args[2].parse()?, connection_generation: args[3].parse()?, }; ensure!( channel.connection_generation > 0 && !channel.boot_id.is_nil(), "CHANNEL_REQUIRED" ); let db = Connection::open(state.join("channel.sqlite"))?; ledger::bounded_journal_capacity(&db)?; db.execute_batch("PRAGMA synchronous=FULL; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter SELECT id,n FROM sequence; UPDATE counter SET seq=max(seq,(SELECT n FROM sequence WHERE id=1)) WHERE id=1; CREATE TRIGGER IF NOT EXISTS telemetry_sequence AFTER UPDATE ON sequence WHEN NEW.n>(SELECT seq FROM counter WHERE id=1) BEGIN UPDATE counter SET seq=NEW.n WHERE id=1; END; CREATE TRIGGER IF NOT EXISTS job_sequence AFTER UPDATE ON counter WHEN NEW.seq>(SELECT n FROM sequence WHERE id=1) BEGIN UPDATE sequence SET n=NEW.seq WHERE id=1; END; CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL,attempt TEXT NOT NULL,offer BLOB NOT NULL,state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0,grant_id TEXT NOT NULL DEFAULT '',cancel_requested INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(job,attempt)); CREATE TABLE IF NOT EXISTS job_events(id TEXT PRIMARY KEY,payload BLOB NOT NULL);")?; reconcile::initialize(&db)?; let ledger = Connection::open(state.join("consumption.sqlite"))?; ledger::initialize_ledger(&ledger)?; let mut w = Worker { sync: ClientSync::new(binding.clone()), binding, db, ledger, channel, clock: NativeClock::default(), sync_due: Instant::now(), permit_after: Instant::now(), pending_sync: None, running: None, root_enabled: env::var_os("DISCOVERY_ROOT_SESSION").is_some(), }; w.recover()?; let mut input = io::stdin().lock(); println!("ready"); io::stdout().flush()?; loop { let mut line = String::new(); let n = (&mut input).take(2_100_000).read_line(&mut line)?; if n == 0 { return Ok(()); } ensure!(line.ends_with('\n'), "FRAME_LIMIT"); let line = line.trim_end(); if line == "next" { println!( "{}", w.next()? .map(|e| hex::encode(e.encode_to_vec())) .unwrap_or_default() ); } else { let data = hex::decode(line)?; w.receive(Envelope::decode(data.as_slice())?)?; println!("ok"); } io::stdout().flush()?; } }