399 lines
15 KiB
Rust
399 lines
15 KiB
Rust
//! One worker per authenticated Python HTTPS channel, never a network listener.
|
|
//! Only the fixed unprivileged probe is executable. The shared Rust core owns
|
|
//! all approval/permit verification, receipt, consumption and sync transitions.
|
|
use anyhow::{Context, Result, bail, ensure};
|
|
use chrono::Utc;
|
|
use prost::Message;
|
|
use rmm_agent_core::{
|
|
auth::{Binding, Outcome},
|
|
freebsd_clock::NativeClock,
|
|
jobs, ledger,
|
|
reconcile::{self, Channel, ClientSync, SyncClock},
|
|
};
|
|
use rmm_agent_linux::actions::{Interpreter, ScriptRequest, run_script_controlled};
|
|
use rmm_domain::jobs::{Action, JobPayload, hash};
|
|
use rmm_protocol::*;
|
|
use rusqlite::{Connection, params};
|
|
use std::{
|
|
env, fs,
|
|
io::{self, BufRead, Read, Write},
|
|
os::{
|
|
fd::AsRawFd,
|
|
unix::fs::{MetadataExt, OpenOptionsExt},
|
|
},
|
|
path::Path,
|
|
time::{Duration, Instant},
|
|
};
|
|
use uuid::Uuid;
|
|
|
|
#[cfg(test)]
|
|
mod tests;
|
|
|
|
fn supported(job: &JobPayload) -> bool {
|
|
job.execution_identity == "agent_service"
|
|
&& job.action.opnsense_kernel_probe()
|
|
&& job.max_runtime_seconds == 10
|
|
&& job.max_output_bytes == 1024
|
|
}
|
|
|
|
struct Worker {
|
|
binding: Binding,
|
|
channel: Channel,
|
|
sync: ClientSync,
|
|
clock: NativeClock,
|
|
db: Connection,
|
|
ledger: Connection,
|
|
sync_due: Instant,
|
|
permit_after: Instant,
|
|
}
|
|
impl Worker {
|
|
fn envelope(&self, payload: Payload) -> Result<Envelope> {
|
|
let seq = self.db.query_row(
|
|
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
|
[],
|
|
|r| r.get(0),
|
|
)?;
|
|
Ok(Envelope {
|
|
protocol_major: 1,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: self.binding.agent_id.to_string(),
|
|
enrollment_generation: self.binding.enrollment_generation,
|
|
connection_generation: self.channel.connection_generation,
|
|
boot_id: self.channel.boot_id.to_string(),
|
|
sequence: seq,
|
|
observed_at: Utc::now().timestamp_millis(),
|
|
payload: Some(payload),
|
|
..Default::default()
|
|
})
|
|
}
|
|
fn result(&mut self, offer: &JobOffer, grant: &str, outcome: Outcome) -> Result<()> {
|
|
let mut e = self.envelope(Payload::JobResult(Default::default()))?;
|
|
let tx = self.db.transaction()?;
|
|
let rev = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
|
e.payload = Some(Payload::JobResult(jobs::result(offer, rev, grant, outcome)));
|
|
jobs::event(&tx, &e)?;
|
|
tx.commit()?;
|
|
self.sync_due = Instant::now();
|
|
Ok(())
|
|
}
|
|
fn recover(&mut self) -> Result<()> {
|
|
for (offer, grant) in jobs::interrupted(&self.db, &self.ledger, &self.binding)? {
|
|
if grant.is_empty() {
|
|
jobs::interrupted_request(&self.db, &offer)?;
|
|
} else {
|
|
let saved =
|
|
jobs::recover_interrupted_outcome(&self.ledger, &self.binding, &offer, &grant)?;
|
|
self.result(
|
|
&offer,
|
|
&grant,
|
|
saved.unwrap_or_else(|| Outcome::unknown("AGENT_PROCESS_INTERRUPTED")),
|
|
)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
fn next(&mut self) -> Result<Option<Envelope>> {
|
|
if let Some(mut e) = jobs::events(&self.db)?.into_iter().next() {
|
|
e.connection_generation = self.channel.connection_generation;
|
|
return Ok(Some(e));
|
|
}
|
|
// A refused or unqualified clock closes the effect gate, not telemetry.
|
|
if Instant::now() >= self.sync_due {
|
|
self.sync_due = Instant::now() + Duration::from_secs(40);
|
|
if self.clock.now().is_err() {
|
|
self.sync.close();
|
|
return Ok(None);
|
|
}
|
|
let p = self.sync.prepare(
|
|
&mut self.db,
|
|
&self.ledger,
|
|
self.channel.clone(),
|
|
None,
|
|
&self.clock,
|
|
)?;
|
|
return Ok(Some(p.envelope));
|
|
}
|
|
if self.sync.can_start(&self.channel, false, &self.clock)
|
|
&& Instant::now() >= self.permit_after
|
|
&& let Some(offer) = jobs::next(&self.db)?
|
|
{
|
|
let job = self.binding.job(&offer.payload_bytes)?;
|
|
if self.clock.now()? >= job.start_before {
|
|
self.result(
|
|
&offer,
|
|
"",
|
|
jobs::no_start("expired", "START_DEADLINE_EXPIRED"),
|
|
)?;
|
|
return self.next();
|
|
}
|
|
ensure!(supported(&job), "UNSUPPORTED_JOB");
|
|
jobs::reserve_request(&self.db, &offer)?;
|
|
return self
|
|
.envelope(Payload::ExecutionPermitRequest(ExecutionPermitRequest {
|
|
job_id: offer.job_id,
|
|
attempt_id: offer.attempt_id,
|
|
job_payload_hash: offer.job_payload_hash,
|
|
assignment_version: self.binding.assignment_version,
|
|
}))
|
|
.map(Some);
|
|
}
|
|
Ok(None)
|
|
}
|
|
fn receive(&mut self, e: Envelope) -> Result<()> {
|
|
ensure!(
|
|
valid_envelope(
|
|
&e,
|
|
&self.binding.agent_id.to_string(),
|
|
self.channel.connection_generation
|
|
) && e.enrollment_generation == self.binding.enrollment_generation,
|
|
"WRONG_CHANNEL"
|
|
);
|
|
match e.payload.context("EMPTY_REPLY")? {
|
|
Payload::Ack(a) => jobs::ack(&mut self.db, &a.message_id)?,
|
|
Payload::SyncResponse(s) => {
|
|
let applied = self.sync.apply(&mut self.db, &s, &self.clock)?;
|
|
ensure!(applied.helper_instruction.is_none(), "UNEXPECTED_HELPER");
|
|
self.sync.apply_consumption(&mut self.ledger, &self.clock)?;
|
|
}
|
|
Payload::JobOffer(o) => {
|
|
let receipt = self.envelope(Payload::JobAccepted(JobAccepted {
|
|
job_id: o.job_id.clone(),
|
|
attempt_id: o.attempt_id.clone(),
|
|
job_payload_hash: o.job_payload_hash.clone(),
|
|
journal_proof: hash(&o.encode_to_vec()),
|
|
}))?;
|
|
let tx = self.db.transaction()?;
|
|
let fresh = jobs::accept(&tx, &self.binding, &o, supported)?;
|
|
jobs::event(&tx, &receipt)?;
|
|
tx.commit()?;
|
|
if fresh {
|
|
self.sync_due = Instant::now();
|
|
}
|
|
}
|
|
Payload::ExecutionPermit(p) => self.permit(p)?,
|
|
Payload::CancelRequest(c) => {
|
|
let offer:Option<Vec<u8>>=rusqlite::OptionalExtension::optional(self.db.query_row(
|
|
"UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2 AND state='accepted' RETURNING offer",
|
|
params![c.job_id,c.attempt_id],|r|r.get(0)))?;
|
|
if let Some(bytes) = offer {
|
|
self.result(
|
|
&JobOffer::decode(bytes.as_slice())?,
|
|
"",
|
|
jobs::no_start("cancelled", "CANCELLED_BEFORE_START"),
|
|
)?;
|
|
}
|
|
}
|
|
_ => bail!("UNEXPECTED_JOB_MESSAGE"),
|
|
}
|
|
Ok(())
|
|
}
|
|
fn permit(&mut self, p: ExecutionPermit) -> Result<()> {
|
|
let offer = jobs::requesting(&self.db)?.context("UNSOLICITED_PERMIT")?;
|
|
if !p.denial_code.is_empty() {
|
|
jobs::denied_request(&self.db, &offer, &p)?;
|
|
self.sync.close();
|
|
self.sync_due = Instant::now();
|
|
self.permit_after = Instant::now() + Duration::from_secs(30);
|
|
return Ok(());
|
|
}
|
|
let req = jobs::permit_request(&offer, &p)?;
|
|
let verify = || -> Result<_> {
|
|
ensure!(
|
|
self.sync.can_start(&self.channel, false, &self.clock),
|
|
"SYNC_REQUIRED"
|
|
);
|
|
let (low, high) = self.clock.bounds()?;
|
|
self.binding.verify(&req, high)?;
|
|
self.binding.verify(&req, low)
|
|
};
|
|
// The signed not_before is the reservation instant. Wait within the
|
|
// qualified uncertainty interval before checking both ends at start.
|
|
let began = Instant::now();
|
|
let (job, grant) = loop {
|
|
match verify() {
|
|
Ok(x) => break x,
|
|
Err(err)
|
|
if err.to_string() == "PERMIT_TIME"
|
|
&& began.elapsed() < Duration::from_secs(6) =>
|
|
{
|
|
std::thread::sleep(Duration::from_millis(50));
|
|
}
|
|
Err(e) => return Err(e),
|
|
}
|
|
};
|
|
ensure!(supported(&job), "UNSUPPORTED_JOB");
|
|
let Action::RunScript { script_utf8, .. } = &job.action else {
|
|
bail!("SCRIPT_REQUIRED")
|
|
};
|
|
let script = ScriptRequest::new(
|
|
script_utf8.as_bytes().to_vec(),
|
|
Interpreter::PosixSh,
|
|
vec![],
|
|
Duration::from_secs(job.max_runtime_seconds.into()),
|
|
job.max_output_bytes as usize,
|
|
)?;
|
|
ledger::begin_execution(
|
|
&mut self.ledger,
|
|
&grant,
|
|
&req,
|
|
self.channel.boot_id,
|
|
self.clock.now()?,
|
|
)?;
|
|
let id = grant.grant_id.to_string();
|
|
let mut running = self.envelope(Payload::JobStateChanged(Default::default()))?;
|
|
let tx = self.db.transaction()?;
|
|
let revision = jobs::transition(&tx, &offer, "running", &id)?;
|
|
running.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
grant_id: id.clone(),
|
|
revision,
|
|
state: "running".into(),
|
|
}));
|
|
jobs::event(&tx, &running)?;
|
|
tx.commit()?;
|
|
let mut first = true;
|
|
let result = run_script_controlled(&script, || {
|
|
if first {
|
|
ensure!(
|
|
self.sync.can_start(&self.channel, false, &self.clock),
|
|
"SYNC_EXPIRED"
|
|
);
|
|
let (lo, hi) = self.clock.bounds()?;
|
|
self.binding.verify(&req, lo)?;
|
|
self.binding.verify(&req, hi)?;
|
|
first = false;
|
|
} else {
|
|
self.clock.now()?;
|
|
}
|
|
Ok(())
|
|
});
|
|
let outcome = match result {
|
|
Ok(o) => {
|
|
let ok = o.code == Some(0)
|
|
&& !o.timed_out
|
|
&& !o.interrupted
|
|
&& !o.truncated
|
|
&& o.stdout == b"FreeBSD\n";
|
|
Outcome {
|
|
state: if ok { "succeeded" } else { "failed" }.into(),
|
|
stdout: o.stdout,
|
|
stderr: o.stderr,
|
|
output_truncated: o.truncated,
|
|
reboot_required: false,
|
|
exit_code: o.code,
|
|
evidence: serde_json::json!({"postconditions_passed":ok,"effective_uid":unsafe{libc::geteuid()}}),
|
|
error_code: if ok { "" } else { "PROBE_POSTCONDITION_FAILED" }.into(),
|
|
}
|
|
}
|
|
Err(_) => Outcome::unknown("PROBE_EFFECT_UNCERTAIN"),
|
|
};
|
|
ledger::persist_outcome(&self.ledger, &grant, &outcome)?;
|
|
let saved = serde_json::to_vec(&outcome)?;
|
|
self.result(&offer, &id, outcome)?;
|
|
ledger::finish(&self.ledger, &grant, &serde_json::from_slice(&saved)?)?;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
pub fn main() -> Result<()> {
|
|
let args = env::args().skip(1).collect::<Vec<_>>();
|
|
ensure!(args.len() == 4, "bound channel required");
|
|
let uid = unsafe { libc::geteuid() };
|
|
ensure!(
|
|
uid != 0 && uid == unsafe { libc::getuid() },
|
|
"SERVICE_ACCOUNT_REQUIRED"
|
|
);
|
|
unsafe { libc::umask(0o077) };
|
|
let state = Path::new("/var/db/discovery-agent");
|
|
let m = fs::symlink_metadata(state)?;
|
|
ensure!(
|
|
m.is_dir() && m.uid() == uid && m.mode() & 0o077 == 0,
|
|
"PRIVATE_STATE_REQUIRED"
|
|
);
|
|
let lock = fs::OpenOptions::new()
|
|
.write(true)
|
|
.create(true)
|
|
.truncate(false)
|
|
.mode(0o600)
|
|
.custom_flags(libc::O_NOFOLLOW)
|
|
.open(state.join("job-worker.lock"))?;
|
|
ensure!(
|
|
unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0,
|
|
"ACTIVE_JOB_WORKER"
|
|
);
|
|
let config = Path::new("/usr/local/etc/discovery-agent/job-binding.json");
|
|
for path in [config.parent().unwrap(), config] {
|
|
let m = fs::symlink_metadata(path)?;
|
|
ensure!(
|
|
!m.is_symlink() && m.uid() == 0 && m.mode() & 0o022 == 0,
|
|
"PROTECTED_BINDING_REQUIRED"
|
|
);
|
|
}
|
|
let binding: Binding = serde_json::from_slice(&fs::read(config)?)?;
|
|
ensure!(
|
|
args[0] == binding.agent_id.to_string() && args[1] == binding.asset_id.to_string(),
|
|
"WRONG_DEVICE"
|
|
);
|
|
let channel = Channel {
|
|
boot_id: args[2].parse()?,
|
|
connection_generation: args[3].parse()?,
|
|
};
|
|
ensure!(
|
|
channel.connection_generation > 0 && !channel.boot_id.is_nil(),
|
|
"CHANNEL_REQUIRED"
|
|
);
|
|
let db = Connection::open(state.join("channel.sqlite"))?;
|
|
ledger::bounded_journal_capacity(&db)?;
|
|
db.execute_batch("PRAGMA synchronous=FULL;
|
|
CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL);
|
|
INSERT OR IGNORE INTO counter SELECT id,n FROM sequence;
|
|
UPDATE counter SET seq=max(seq,(SELECT n FROM sequence WHERE id=1)) WHERE id=1;
|
|
CREATE TRIGGER IF NOT EXISTS telemetry_sequence AFTER UPDATE ON sequence WHEN NEW.n>(SELECT seq FROM counter WHERE id=1)
|
|
BEGIN UPDATE counter SET seq=NEW.n WHERE id=1; END;
|
|
CREATE TRIGGER IF NOT EXISTS job_sequence AFTER UPDATE ON counter WHEN NEW.seq>(SELECT n FROM sequence WHERE id=1)
|
|
BEGIN UPDATE sequence SET n=NEW.seq WHERE id=1; END;
|
|
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL,attempt TEXT NOT NULL,offer BLOB NOT NULL,state TEXT NOT NULL,
|
|
revision INTEGER NOT NULL DEFAULT 0,grant_id TEXT NOT NULL DEFAULT '',cancel_requested INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(job,attempt));
|
|
CREATE TABLE IF NOT EXISTS job_events(id TEXT PRIMARY KEY,payload BLOB NOT NULL);")?;
|
|
reconcile::initialize(&db)?;
|
|
let ledger = Connection::open(state.join("consumption.sqlite"))?;
|
|
ledger::initialize_ledger(&ledger)?;
|
|
let mut w = Worker {
|
|
sync: ClientSync::new(binding.clone()),
|
|
binding,
|
|
db,
|
|
ledger,
|
|
channel,
|
|
clock: NativeClock::default(),
|
|
sync_due: Instant::now(),
|
|
permit_after: Instant::now(),
|
|
};
|
|
w.recover()?;
|
|
let mut input = io::stdin().lock();
|
|
println!("ready");
|
|
io::stdout().flush()?;
|
|
loop {
|
|
let mut line = String::new();
|
|
let n = (&mut input).take(2_100_000).read_line(&mut line)?;
|
|
if n == 0 {
|
|
return Ok(());
|
|
}
|
|
ensure!(line.ends_with('\n'), "FRAME_LIMIT");
|
|
let line = line.trim_end();
|
|
if line == "next" {
|
|
println!(
|
|
"{}",
|
|
w.next()?
|
|
.map(|e| hex::encode(e.encode_to_vec()))
|
|
.unwrap_or_default()
|
|
);
|
|
} else {
|
|
let data = hex::decode(line)?;
|
|
w.receive(Envelope::decode(data.as_slice())?)?;
|
|
println!("ok");
|
|
}
|
|
io::stdout().flush()?;
|
|
}
|
|
}
|