feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s

This commit is contained in:
BuTzZ
2026-11-12 22:06:25 +01:00
commit 6c3203d725
194 changed files with 49234 additions and 0 deletions
+317
View File
@@ -0,0 +1,317 @@
//! Dedicated APT route: no fallback to generic helper, shell or SSH. Readiness
//! is nonce/root-peer/identity/profile/stack pinned before advertising or use.
use anyhow::{Context, Result, ensure};
use rmm_domain::jobs::JobPayload;
use rmm_privileged_helper::{Binding, Outcome, Request, apt as provider, readiness};
use std::{
ffi::OsString,
path::{Path, PathBuf},
};
pub fn socket_for(
job: &JobPayload,
general: Option<OsString>,
apt: Option<OsString>,
) -> Result<PathBuf> {
select_socket(provider::is_apt_action(job), general, apt)
}
fn select_socket(
is_apt: bool,
general: Option<OsString>,
apt: Option<OsString>,
) -> Result<PathBuf> {
if is_apt {
let socket = PathBuf::from(apt.context("APT_HELPER_SOCKET_NOT_CONFIGURED")?);
ensure!(
socket == Path::new(readiness::SOCKET),
"APT_HELPER_SOCKET_PROFILE"
);
Ok(socket)
} else {
Ok(PathBuf::from(
general.context("GENERAL_HELPER_SOCKET_NOT_CONFIGURED")?,
))
}
}
fn status(binding: &Binding) -> Result<readiness::Response> {
let path =
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_HELPER_PIN_NOT_CONFIGURED")?;
let pin = readiness::Pin::read(Path::new(&path), binding)?;
let socket =
std::env::var_os("RMM_APT_HELPER_SOCKET").context("APT_HELPER_SOCKET_NOT_CONFIGURED")?;
readiness::probe(Path::new(&socket), binding, &pin)
}
pub fn capabilities_from(response: &readiness::Response) -> Vec<String> {
if response.profile != "apt_only" {
return vec![];
}
let mut result = vec![];
if response.fixture_acceptance_ready && !response.bootstrap_recipes.is_empty() {
result.push("apt_native_acceptance".into());
}
if !response.accepted_recipes.is_empty() {
result.push("ensure_package".into());
}
if response
.accepted_recipes
.iter()
.any(|r| r == provider::UPDATE_RECIPE)
{
result.push("apply_updates".into());
}
result
}
pub fn capabilities(binding: &Binding) -> Vec<String> {
status(binding)
.map(|s| {
let mut caps = capabilities_from(&s);
if !caps.is_empty() {
let roots: Vec<String> = caps.iter().map(|c| format!("root:{c}")).collect();
caps.extend(roots);
caps.extend([
"helper.root.v1".into(),
format!("helper.policy.sha256:{}", s.provider_stack_sha256),
"apt.manifest.v2".into(),
]);
if s.fixture_acceptance_ready && !s.bootstrap_recipes.is_empty() {
caps.push("apt.native-acceptance.v1".into());
}
if !s.accepted_recipes.is_empty() {
caps.push("apt.native-accepted.v1".into());
}
}
caps
})
.unwrap_or_default()
}
/// Called only by the periodic blocking inventory worker, never heartbeat or a
/// caller-selected live query. Failure retains honest unknown/not_configured.
pub fn planning_observation(binding: Option<&Binding>) -> serde_json::Value {
let result = (|| -> Result<readiness::PlanningResult> {
let binding = binding.context("APT_PLANNING_NOT_CONFIGURED")?;
let path =
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_PLANNING_NOT_CONFIGURED")?;
let pin = readiness::Pin::read(Path::new(&path), binding)?;
let socket =
std::env::var_os("RMM_APT_HELPER_SOCKET").context("APT_PLANNING_NOT_CONFIGURED")?;
readiness::planning(Path::new(&socket), binding, &pin)?
.planning
.context("APT_PLANNING_UNAVAILABLE")
})();
match result {
Ok(planning) => {
serde_json::json!({"scope":"linux.apt.planning", "source_id":"root-helper:apt_planning_v1",
"observed_at":planning.observed_at, "quality":"valid", "completeness":"complete", "truncated":false, "data":planning})
}
Err(_) => {
serde_json::json!({"scope":"linux.apt.planning", "source_id":"root-helper:apt_planning_v1",
"observed_at":chrono::Utc::now(), "quality":"unknown", "completeness":"unknown", "truncated":false,
"error_code":if binding.is_none() || std::env::var_os("RMM_APT_HELPER_SOCKET").is_none(){"APT_PLANNING_NOT_CONFIGURED"}else{"APT_PLANNING_UNAVAILABLE"}, "data":null})
}
}
}
fn recipe_ready(job: &JobPayload, response: &readiness::Response) -> Result<()> {
ensure!(
job.execution_identity == "root" && provider::is_apt_action(job),
"APT_TYPED_ROOT_ACTION_REQUIRED"
);
let action = serde_json::to_value(&job.action)?;
let manifest: provider::AptExecutionManifest = serde_json::from_value(
action
.get("apt_manifest")
.context("APT_MANIFEST_NOT_CONFIGURED")?
.clone(),
)?;
manifest.resolved_apt_transaction.validate_fixture()?;
let recipe = &manifest.resolved_apt_transaction.recipe_id;
if job.action.name() == "apt_native_acceptance" {
ensure!(
manifest.action == "apt-native-acceptance"
&& response.fixture_acceptance_ready
&& response.bootstrap_recipes.contains(recipe),
"APT_FIXTURE_READINESS_REQUIRED"
);
} else {
ensure!(
manifest.action == "apt-execute" && response.accepted_recipes.contains(recipe),
"APT_NATIVE_ACCEPTANCE_REQUIRED"
);
ensure!(
job.action.name() != "apply_updates" || recipe == provider::UPDATE_RECIPE,
"APT_UPDATE_RECIPE_SCOPE"
);
}
Ok(())
}
pub fn supported(binding: &Binding, job: &JobPayload) -> bool {
status(binding).and_then(|s| recipe_ready(job, &s)).is_ok()
}
pub fn invoke(binding: &Binding, job: &JobPayload, request: &Request) -> Result<Outcome> {
let socket = socket_for(
job,
std::env::var_os("RMM_HELPER_SOCKET"),
std::env::var_os("RMM_APT_HELPER_SOCKET"),
)?;
if provider::is_apt_action(job) {
recipe_ready(job, &status(binding)?)?;
}
rmm_privileged_helper::invoke(&socket, request, job.max_runtime_seconds)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn passive_planning_without_binding_is_honest_unknown_without_data() {
let observation = planning_observation(None);
assert_eq!(observation["scope"], "linux.apt.planning");
assert_eq!(observation["quality"], "unknown");
assert_eq!(observation["error_code"], "APT_PLANNING_NOT_CONFIGURED");
assert!(observation["data"].is_null());
assert!(observation.get("received_at").is_none());
}
#[test]
fn typed_actions_require_their_own_fixture_readiness_or_accepted_recipe() {
let id = uuid::Uuid::new_v4();
let now = chrono::Utc::now();
let tx = serde_json::json!({"schema_version":1,"recipe_id":provider::INSTALL_RECIPE,"recipe_version":"1","platform":"ubuntu-24.04-amd64",
"resolver_config_version":"rmm-apt-local-v1","no_reboot":true,"status_fingerprint":"a".repeat(64),"baseline_sha256":"b".repeat(64),
"packages":[{"name":"rmm-mvp-test","architecture":"all","old_version":null,"new_version":"1.0","origin":"rmm-test",
"sha256":rmm_domain::apt_transaction::INSTALL_SHA,"size":20672,
"archive_path":format!("/var/lib/rmm-apt-transaction/archives/{}.deb",rmm_domain::apt_transaction::INSTALL_SHA)}]});
let bytes = serde_json::to_vec(&tx).unwrap();
let manifest = serde_json::json!({"type":"rmm.apt.helper-action","schema_version":2,"action":"apt-native-acceptance","execution_profile":"dev-test",
"transaction_sha256":rmm_domain::jobs::hash(&bytes),"resolved_apt_transaction":tx,"transaction_bytes":bytes,
"provider_sha256":"a".repeat(64),"hook_sha256":"a".repeat(64),"apt_config_sha256":"a".repeat(64),"runtime_sha256":"a".repeat(64),"test_result_id":"fixture-test"});
let mut job:JobPayload=serde_json::from_value(serde_json::json!({"schema_version":1,"tenant_id":id,"asset_id":id,"agent_id":id,
"enrollment_generation":1,"assignment_version":1,"job_id":id,"attempt_id":id,"execution_identity":"root",
"not_before":now,"start_before":now+chrono::Duration::minutes(1),"max_runtime_seconds":300,"max_output_bytes":4096,
"action":{"action":"apt_native_acceptance","recipe_id":provider::INSTALL_RECIPE,"recipe_version":"1","apt_manifest":manifest}})).unwrap();
assert!(job.action.validate());
let identity = readiness::Identity {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
signing_key_id: "a".repeat(64),
};
let mut response = readiness::Response {
schema_version: 1,
challenge: id,
identity,
profile: "apt_only".into(),
provider_stack_sha256: "a".repeat(64),
fixture_acceptance_ready: true,
bootstrap_recipes: vec![provider::INSTALL_RECIPE.into()],
accepted_recipes: vec![],
ledger_device: 1,
ledger_inode: 2,
lock_device: 1,
lock_inode: 3,
planning: None,
};
recipe_ready(&job, &response).unwrap();
let mut normal = manifest;
normal["action"] = serde_json::json!("apt-execute");
job.action=serde_json::from_value(serde_json::json!({"action":"ensure_package","provider":"apt","package":"rmm-mvp-test","version":"1.0",
"architecture":"all","recipe_version":"1","installation_context":"system","recipe_id":provider::INSTALL_RECIPE,"apt_manifest":normal})).unwrap();
assert!(job.action.validate());
assert!(recipe_ready(&job, &response).is_err());
response.accepted_recipes = vec![provider::UPDATE_RECIPE.into()];
assert!(recipe_ready(&job, &response).is_err());
response.accepted_recipes = vec![provider::INSTALL_RECIPE.into()];
recipe_ready(&job, &response).unwrap();
}
#[test]
fn apt_route_cannot_fall_back_to_general_or_a_caller_path() {
assert!(select_socket(true, Some("/run/general.sock".into()), None).is_err());
assert!(
select_socket(
true,
Some("/run/general.sock".into()),
Some("/tmp/fake.sock".into())
)
.is_err()
);
assert_eq!(
select_socket(
true,
Some("/run/general.sock".into()),
Some(readiness::SOCKET.into())
)
.unwrap(),
Path::new(readiness::SOCKET)
);
}
#[test]
fn bootstrap_readiness_never_claims_package_acceptance() {
let id = readiness::Identity {
tenant_id: uuid::Uuid::new_v4(),
asset_id: uuid::Uuid::new_v4(),
agent_id: uuid::Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
signing_key_id: "a".repeat(64),
};
let mut response = readiness::Response {
schema_version: 1,
challenge: uuid::Uuid::new_v4(),
identity: id,
profile: "apt_only".into(),
provider_stack_sha256: "a".repeat(64),
fixture_acceptance_ready: true,
bootstrap_recipes: vec![provider::INSTALL_RECIPE.into()],
accepted_recipes: vec![],
ledger_device: 1,
ledger_inode: 2,
lock_device: 1,
lock_inode: 3,
planning: None,
};
assert_eq!(capabilities_from(&response), vec!["apt_native_acceptance"]);
response.accepted_recipes = vec![provider::INSTALL_RECIPE.into()];
assert!(!capabilities_from(&response).contains(&"apply_updates".into()));
response
.accepted_recipes
.push(provider::UPDATE_RECIPE.into());
assert!(capabilities_from(&response).contains(&"apply_updates".into()));
}
#[test]
fn generic_script_never_uses_apt_socket() {
let now = chrono::Utc::now();
let id = uuid::Uuid::new_v4();
let job = JobPayload {
schema_version: 1,
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
job_id: id,
attempt_id: id,
action: rmm_domain::jobs::Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "true".into(),
arguments: vec![],
expected_stdout: None,
},
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(1),
max_runtime_seconds: 300,
max_output_bytes: 4096,
};
assert!(socket_for(&job, None, Some(readiness::SOCKET.into())).is_err());
assert_eq!(
socket_for(
&job,
Some("/run/general.sock".into()),
Some(readiness::SOCKET.into())
)
.unwrap(),
Path::new("/run/general.sock")
);
}
}
+309
View File
@@ -0,0 +1,309 @@
//! mTLS-only, write-once trust bootstrap. Never replaces a pin from the helper
//! or converts a job-supplied key into authority.
use anyhow::{Result, ensure};
use rmm_privileged_helper::{Binding, SigningKey};
#[cfg(test)]
use std::fs;
use std::{net::SocketAddr, path::Path, time::Duration};
#[cfg(test)]
fn client(dir: &Path, address: SocketAddr, ca: &[u8]) -> Result<reqwest::Client> {
let mut identity = fs::read(dir.join("device.pem"))?;
identity.extend_from_slice(&fs::read(dir.join("device.key"))?);
Ok(reqwest::Client::builder()
.https_only(true)
.tls_built_in_root_certs(false)
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
.identity(reqwest::Identity::from_pem(&identity)?)
.resolve("rmm-dev.internal", address)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(15))
.build()?)
}
#[cfg(test)]
pub async fn provision(
dir: &Path,
address: SocketAddr,
ca: &[u8],
binding: &mut Binding,
) -> Result<()> {
download(dir, address, binding, client(dir, address, ca)?).await
}
pub async fn provision_current(
dir: &Path,
address: SocketAddr,
ca: &[u8],
binding: &mut Binding,
identity: &crate::certificate_rotation::DeviceIdentity,
) -> Result<()> {
let client = crate::https_channel::mtls_client(
address,
ca,
&identity.certificate_pem,
&identity.private_key_pem,
)?;
download(dir, address, binding, client).await
}
async fn download(
dir: &Path,
address: SocketAddr,
binding: &mut Binding,
client: reqwest::Client,
) -> Result<()> {
let mut response = client
.get(format!(
"https://rmm-dev.internal:{}/api/v1/job-signing-key",
address.port()
))
.timeout(Duration::from_secs(15))
.send()
.await?;
ensure!(
response.status() == reqwest::StatusCode::OK,
"SIGNING_KEY_AUTHENTICATED_ENDPOINT_REJECTED"
);
ensure!(
response.content_length().is_none_or(|n| n <= 4096),
"SIGNING_KEY_SIZE"
);
let mut bytes = vec![];
while let Some(chunk) = response.chunk().await? {
ensure!(bytes.len() + chunk.len() <= 4096, "SIGNING_KEY_SIZE");
bytes.extend_from_slice(&chunk);
}
let downloaded: SigningKey = serde_json::from_slice(&bytes)?;
pin(dir, binding, &downloaded)
}
fn pin(dir: &Path, binding: &mut Binding, downloaded: &SigningKey) -> Result<()> {
downloaded.validate()?;
ensure!(
binding.public_key_hex.is_empty() || binding.public_key_hex == downloaded.public_key_hex,
"PROVISIONED_KEY_MISMATCH"
);
let path = dir.join("job-signing-key.json");
if path.try_exists()? {
let cached = SigningKey::read(&path, unsafe { libc::geteuid() })?;
ensure!(
cached == *downloaded,
"SIGNING_KEY_ROTATION_REQUIRES_CONTROLLED_PROVISIONING"
);
} else {
crate::storage::create_secret(&path, &serde_json::to_vec(downloaded)?)?;
}
binding.public_key_hex = downloaded.public_key_hex.clone();
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn key() -> SigningKey {
let public = openssl::pkey::PKey::generate_ed25519()
.unwrap()
.raw_public_key()
.unwrap();
SigningKey {
key_id: rmm_domain::jobs::hash(&public),
algorithm: "Ed25519".into(),
schema_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
}
}
#[test]
fn bootstrap_never_rotates_existing_or_provisioned_trust() {
let path = std::env::temp_dir().join(format!("rmm-key-bootstrap-{}", uuid::Uuid::new_v4()));
crate::storage::prepare_dir(&path).unwrap();
let mut binding = Binding {
tenant_id: uuid::Uuid::new_v4(),
asset_id: uuid::Uuid::new_v4(),
agent_id: uuid::Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: String::new(),
};
let first = key();
pin(&path, &mut binding, &first).unwrap();
pin(&path, &mut binding, &first).unwrap();
assert!(pin(&path, &mut binding, &key()).is_err());
binding.public_key_hex.clear();
assert!(pin(&path, &mut binding, &key()).is_err());
assert_eq!(
SigningKey::read(&path.join("job-signing-key.json"), unsafe {
libc::geteuid()
})
.unwrap(),
first
);
let mut bad = first.clone();
bad.algorithm = "RSA".into();
assert!(bad.validate().is_err());
bad = first.clone();
bad.schema_version = 2;
assert!(bad.validate().is_err());
bad = first;
bad.key_id = "wrong".into();
assert!(bad.validate().is_err());
fs::remove_dir_all(path).unwrap();
}
fn certificate(
cn: &str,
issuer: Option<(
&openssl::x509::X509,
&openssl::pkey::PKey<openssl::pkey::Private>,
)>,
server: bool,
) -> (
openssl::x509::X509,
openssl::pkey::PKey<openssl::pkey::Private>,
) {
use openssl::{
asn1::Asn1Time,
bn::BigNum,
ec::{EcGroup, EcKey},
nid::Nid,
pkey::PKey,
x509::{
X509, X509NameBuilder,
extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName},
},
};
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap();
let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap();
let mut name = X509NameBuilder::new().unwrap();
name.append_entry_by_text("CN", cn).unwrap();
let name = name.build();
let mut builder = X509::builder().unwrap();
builder.set_version(2).unwrap();
builder
.set_serial_number(
&BigNum::from_u32(rand::random_range(1..u32::MAX))
.unwrap()
.to_asn1_integer()
.unwrap(),
)
.unwrap();
builder.set_subject_name(&name).unwrap();
builder
.set_issuer_name(issuer.map_or(name.as_ref(), |(ca, _)| ca.subject_name()))
.unwrap();
builder.set_pubkey(&key).unwrap();
builder
.set_not_before(&Asn1Time::days_from_now(0).unwrap())
.unwrap();
builder
.set_not_after(&Asn1Time::days_from_now(1).unwrap())
.unwrap();
if issuer.is_none() {
builder
.append_extension(BasicConstraints::new().critical().ca().build().unwrap())
.unwrap();
builder
.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap())
.unwrap();
} else {
builder
.append_extension(BasicConstraints::new().build().unwrap())
.unwrap();
let mut usage = ExtendedKeyUsage::new();
if server {
usage.server_auth();
} else {
usage.client_auth();
}
builder.append_extension(usage.build().unwrap()).unwrap();
if server {
let context = builder.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None);
let san = SubjectAlternativeName::new()
.dns(cn)
.build(&context)
.unwrap();
builder.append_extension(san).unwrap();
}
}
builder
.sign(
issuer.map_or(&key, |(_, k)| k),
openssl::hash::MessageDigest::sha256(),
)
.unwrap();
(builder.build(), key)
}
async fn tls_bootstrap(server_name: &str) -> bool {
use openssl::ssl::{SslAcceptor, SslMethod, SslVerifyMode};
use std::{
io::{Read, Write},
net::TcpListener,
};
let dir = std::env::temp_dir().join(format!("rmm-mtls-key-{}", uuid::Uuid::new_v4()));
crate::storage::prepare_dir(&dir).unwrap();
let (ca, ca_key) = certificate("test CA", None, false);
let (server, server_key) = certificate(server_name, Some((&ca, &ca_key)), true);
let (device, device_key) = certificate("enrolled device", Some((&ca, &ca_key)), false);
crate::storage::create_secret(&dir.join("device.pem"), &device.to_pem().unwrap()).unwrap();
crate::storage::create_secret(
&dir.join("device.key"),
&device_key.private_key_to_pem_pkcs8().unwrap(),
)
.unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let downloaded = key();
let response = serde_json::to_vec(&downloaded).unwrap();
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap();
acceptor.set_certificate(&server).unwrap();
acceptor.set_private_key(&server_key).unwrap();
acceptor.cert_store_mut().add_cert(ca.clone()).unwrap();
acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT);
let acceptor = acceptor.build();
let worker = std::thread::spawn(move || {
let (socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
if let Ok(mut tls) = acceptor.accept(socket) {
assert!(tls.ssl().peer_certificate().is_some());
let mut bytes = [0; 8192];
if let Ok(n) = tls.read(&mut bytes) {
assert!(
String::from_utf8_lossy(&bytes[..n])
.starts_with("GET /api/v1/job-signing-key HTTP/1.1")
);
let header = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
response.len()
);
let _ = tls.write_all(header.as_bytes());
let _ = tls.write_all(&response);
}
}
});
let mut binding = Binding {
tenant_id: uuid::Uuid::new_v4(),
asset_id: uuid::Uuid::new_v4(),
agent_id: uuid::Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: String::new(),
};
let result = provision(&dir, address, &ca.to_pem().unwrap(), &mut binding).await;
worker.join().unwrap();
if result.is_ok() {
assert_eq!(binding.public_key_hex, downloaded.public_key_hex);
} else {
assert!(!dir.join("job-signing-key.json").exists());
}
fs::remove_dir_all(dir).unwrap();
result.is_ok()
}
#[tokio::test]
async fn bootstrap_uses_real_mtls_and_dns_connect_override() {
assert!(tls_bootstrap("rmm-dev.internal").await);
}
#[tokio::test]
async fn bootstrap_rejects_wrong_server_name_without_creating_trust() {
assert!(!tls_bootstrap("wrong.internal").await);
}
}
+361
View File
@@ -0,0 +1,361 @@
//! Crash-safe device TLS rotation only; never rotates the helper's signing pin.
//! The process must hold the existing exclusive agent-state lock while using this
//! store. Install returns a mandatory reconnect/Sync signal, not effect authority.
use anyhow::{Context, Result, ensure};
use openssl::{
asn1::Asn1Time,
ec::{EcGroup, EcKey},
hash::MessageDigest,
nid::Nid,
pkey::PKey,
stack::Stack,
x509::{
X509, X509NameBuilder, X509PurposeId, X509Req, X509StoreContext, store::X509StoreBuilder,
verify::X509VerifyParam,
},
};
use serde::{Deserialize, Serialize};
use std::{
fs::{self, File, OpenOptions},
io::Write,
net::SocketAddr,
os::unix::fs::{OpenOptionsExt, PermissionsExt},
path::{Path, PathBuf},
};
use uuid::Uuid;
const DAY: i64 = 86400;
#[derive(Clone, Copy)]
pub struct RotationPolicy {
pub begin_after_seconds: i64,
pub renew_before_seconds: i64,
}
impl Default for RotationPolicy {
fn default() -> Self {
Self {
begin_after_seconds: 20 * DAY,
renew_before_seconds: 10 * DAY,
}
}
}
#[derive(Serialize, Deserialize, Clone)]
#[serde(deny_unknown_fields)]
pub struct Selector {
pub schema_version: u32,
pub agent_id: Uuid,
pub enrollment_generation: u64,
pub version: Uuid,
pub previous_version: Option<Uuid>,
pub previous_until: Option<i64>,
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Pending {
schema_version: u32,
version: Uuid,
base_version: Uuid,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct RenewalResponse {
agent_id: Uuid,
certificate_pem: String,
overlap_seconds: u32,
}
pub struct DeviceIdentity {
pub selector: Selector,
pub certificate_pem: Vec<u8>,
pub private_key_pem: Vec<u8>,
}
pub struct RotationOutcome {
pub needs_reconnect: bool,
pub enrollment_generation: u64,
}
pub struct RotationStore {
root: PathBuf,
ca: Vec<u8>,
agent: Uuid,
enrollment: u64,
}
fn private_dir(path: &Path) -> Result<()> {
if !path.exists() {
let mut builder = fs::DirBuilder::new();
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700).create(path)?;
}
let metadata = fs::symlink_metadata(path)?;
ensure!(
metadata.is_dir()
&& !metadata.file_type().is_symlink()
&& metadata.permissions().mode() & 0o777 == 0o700,
"ROTATION_DIRECTORY_NOT_PRIVATE"
);
Ok(())
}
fn secret(path: &Path) -> Result<Vec<u8>> {
let metadata = fs::symlink_metadata(path)?;
ensure!(
metadata.is_file()
&& !metadata.file_type().is_symlink()
&& metadata.permissions().mode() & 0o777 == 0o600
&& metadata.len() <= 65536,
"ROTATION_FILE_NOT_PRIVATE"
);
Ok(fs::read(path)?)
}
fn create(path: &Path, data: &[u8]) -> Result<()> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)?;
file.write_all(data)?;
file.sync_all()?;
Ok(())
}
fn atomic_json<T: Serialize>(root: &Path, name: &str, value: &T) -> Result<()> {
let temp = root.join(format!(".{}.tmp", Uuid::new_v4()));
create(&temp, &serde_json::to_vec(value)?)?;
fs::rename(temp, root.join(name))?;
File::open(root)?.sync_all()?;
Ok(())
}
impl RotationStore {
/// Bootstrap from the existing enrolled key/certificate, not a new identity.
/// Existing current.json is never overwritten by bootstrap input.
pub fn open(root: &Path, ca: &[u8], agent: Uuid, enrollment: u64) -> Result<Self> {
ensure!(enrollment > 0, "ROTATION_IDENTITY_INVALID");
private_dir(root)?;
private_dir(&root.join("versions"))?;
Ok(Self {
root: root.into(),
ca: ca.into(),
agent,
enrollment,
})
}
fn version(&self, id: Uuid) -> PathBuf {
self.root.join("versions").join(id.to_string())
}
pub fn initialize(&self, cert: &[u8], key: &[u8], now: i64) -> Result<()> {
ensure!(
!self.root.join("current.json").exists(),
"ROTATION_ALREADY_INITIALIZED"
);
self.verify(cert, key, now)?;
let version = Uuid::new_v4();
let dir = self.version(version);
private_dir(&dir)?;
create(&dir.join("device.key"), key)?;
create(&dir.join("device.pem"), cert)?;
File::open(&dir)?.sync_all()?;
File::open(self.root.join("versions"))?.sync_all()?;
atomic_json(
&self.root,
"current.json",
&Selector {
schema_version: 1,
agent_id: self.agent,
enrollment_generation: self.enrollment,
version,
previous_version: None,
previous_until: None,
},
)
}
pub fn current(&self, now: i64) -> Result<DeviceIdentity> {
let selector: Selector = serde_json::from_slice(&secret(&self.root.join("current.json"))?)?;
ensure!(
selector.schema_version == 1
&& selector.agent_id == self.agent
&& selector.enrollment_generation == self.enrollment,
"ROTATION_IDENTITY_CHANGED"
);
let dir = self.version(selector.version);
private_dir(&dir)?;
let certificate_pem = secret(&dir.join("device.pem"))?;
let private_key_pem = secret(&dir.join("device.key"))?;
self.verify(&certificate_pem, &private_key_pem, now)?;
Ok(DeviceIdentity {
selector,
certificate_pem,
private_key_pem,
})
}
/// Checks full chain, current validity, clientAuth purpose, key and exact CN.
/// X509 SSL_CLIENT purpose rejects server-only certificates and CA leaves.
fn verify(&self, pem: &[u8], key: &[u8], now: i64) -> Result<()> {
let chain = X509::stack_from_pem(pem)?;
let leaf = chain.first().context("ROTATION_CERTIFICATE_MISSING")?;
let key = PKey::private_key_from_pem(key)?;
ensure!(leaf.public_key()?.public_eq(&key), "ROTATION_KEY_MISMATCH");
let names = leaf
.subject_name()
.entries_by_nid(Nid::COMMONNAME)
.collect::<Vec<_>>();
ensure!(
names.len() == 1 && names[0].data().as_slice() == self.agent.to_string().as_bytes(),
"ROTATION_AGENT_MISMATCH"
);
let time = Asn1Time::from_unix(now)?;
ensure!(
leaf.not_before() <= time && leaf.not_after() > time,
"ROTATION_EXPIRED_OR_NOT_YET_VALID_RECOVERY_REQUIRED"
);
let mut builder = X509StoreBuilder::new()?;
for ca in X509::stack_from_pem(&self.ca)? {
builder.add_cert(ca)?;
}
let mut param = X509VerifyParam::new()?;
param.set_time(now);
param.set_purpose(X509PurposeId::SSL_CLIENT)?;
builder.set_param(&param)?;
let mut intermediates = Stack::new()?;
for cert in chain.iter().skip(1) {
intermediates.push(cert.clone())?;
}
let mut context = X509StoreContext::new()?;
ensure!(
context.init(&builder.build(), leaf, &intermediates, |c| c.verify_cert())?,
"ROTATION_CERTIFICATE_UNTRUSTED"
);
Ok(())
}
pub fn due(&self, policy: RotationPolicy, now: i64) -> Result<bool> {
ensure!(
policy.begin_after_seconds > 0 && policy.renew_before_seconds > 0,
"ROTATION_POLICY_INVALID"
);
let identity = self.current(now)?;
let cert = X509::from_pem(&identity.certificate_pem)?;
let time = Asn1Time::from_unix(now)?;
let age = cert.not_before().diff(&time)?;
let left = time.diff(cert.not_after())?;
Ok(
i64::from(age.days) * DAY + i64::from(age.secs) >= policy.begin_after_seconds
|| i64::from(left.days) * DAY + i64::from(left.secs) <= policy.renew_before_seconds
|| self.root.join("pending.json").exists(),
)
}
/// Persist both key and exact CSR before publishing the pending selector.
/// Orphaned versions after a crash are harmless and never selected.
fn pending(&self, current: &Selector) -> Result<Pending> {
let path = self.root.join("pending.json");
if path.exists() {
let pending: Pending = serde_json::from_slice(&secret(&path)?)?;
ensure!(
pending.schema_version == 1
&& (pending.base_version == current.version
|| pending.version == current.version),
"ROTATION_PENDING_CONFLICT"
);
return Ok(pending);
}
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
let mut name = X509NameBuilder::new()?;
name.append_entry_by_text("CN", &self.agent.to_string())?;
let mut csr = X509Req::builder()?;
csr.set_subject_name(&name.build())?;
csr.set_pubkey(&key)?;
csr.sign(&key, MessageDigest::sha256())?;
let version = Uuid::new_v4();
let dir = self.version(version);
private_dir(&dir)?;
create(&dir.join("device.key"), &key.private_key_to_pem_pkcs8()?)?;
create(&dir.join("request.pem"), &csr.build().to_pem()?)?;
File::open(dir)?.sync_all()?;
File::open(self.root.join("versions"))?.sync_all()?;
let pending = Pending {
schema_version: 1,
version,
base_version: current.version,
};
atomic_json(&self.root, "pending.json", &pending)?;
Ok(pending)
}
/// Retry after response loss uses the original key/CSR and old valid mTLS
/// identity. Expiry/revocation remains blocked; no automatic re-enrollment.
pub async fn renew(&self, address: SocketAddr, now: i64) -> Result<RotationOutcome> {
let old = self.current(now)?;
let pending = self.pending(&old.selector)?;
if pending.version == old.selector.version {
self.clear_pending()?;
return Ok(RotationOutcome {
needs_reconnect: true,
enrollment_generation: self.enrollment,
});
}
let dir = self.version(pending.version);
private_dir(&dir)?;
let key = secret(&dir.join("device.key"))?;
let csr = secret(&dir.join("request.pem"))?;
let parsed = X509Req::from_pem(&csr)?;
let public = parsed.public_key()?;
let private = PKey::private_key_from_pem(&key)?;
ensure!(
parsed.verify(&public)? && public.public_eq(&private),
"ROTATION_PENDING_KEY_MISMATCH"
);
let client = crate::https_channel::mtls_client(
address,
&self.ca,
&old.certificate_pem,
&old.private_key_pem,
)?;
let response = client
.post(format!(
"https://rmm-dev.internal:{}/agent/v1/certificates/renew",
address.port()
))
.json(&serde_json::json!({"csr_pem": String::from_utf8(csr)?}))
.send()
.await?;
let bytes = crate::https_channel::bounded_body(response, 65536, "application/json").await?;
let response: RenewalResponse = serde_json::from_slice(&bytes)?;
ensure!(
response.agent_id == self.agent && response.overlap_seconds == 300,
"ROTATION_RESPONSE_BINDING_INVALID"
);
// Never install a certificate which expired during the network request.
let verified_at = now.max(chrono::Utc::now().timestamp());
self.verify(response.certificate_pem.as_bytes(), &key, verified_at)?;
// A crash after writing a response but before selector commit retries the
// same CSR. A mismatching response must not overwrite staged evidence.
let cert_path = dir.join("device.pem");
if cert_path.exists() {
ensure!(
secret(&cert_path)? == response.certificate_pem.as_bytes(),
"ROTATION_RESPONSE_CHANGED"
);
} else {
create(&cert_path, response.certificate_pem.as_bytes())?;
}
File::open(&dir)?.sync_all()?;
atomic_json(
&self.root,
"current.json",
&Selector {
schema_version: 1,
agent_id: self.agent,
enrollment_generation: self.enrollment,
version: pending.version,
previous_version: Some(old.selector.version),
previous_until: Some(verified_at + 300),
},
)?;
// Old version is retained, but is never automatically selected on failure.
self.clear_pending()?;
Ok(RotationOutcome {
needs_reconnect: true,
enrollment_generation: self.enrollment,
})
}
fn clear_pending(&self) -> Result<()> {
fs::remove_file(self.root.join("pending.json"))?;
File::open(&self.root)?.sync_all()?;
Ok(())
}
}
+208
View File
@@ -0,0 +1,208 @@
use crate::reconcile;
use anyhow::{Context, Result, ensure};
use rmm_privileged_helper::{Binding, control, readiness, reconcile as root};
use rmm_protocol::{SyncAttempt, SyncRequest, SyncResponse};
use std::path::{Path, PathBuf};
use uuid::Uuid;
#[derive(Clone)]
struct Endpoint {
socket: PathBuf,
profile: String,
hash: String,
}
#[derive(Clone)]
pub struct Helpers {
binding: Binding,
endpoints: Vec<Endpoint>,
}
fn map(s: &root::Snapshot) -> reconcile::HelperSnapshot {
reconcile::HelperSnapshot {
journal_id: s.journal_id,
checkpoint: s.checkpoint,
previous_decision_id: s.previous_decision_id,
attempts: s
.attempts
.iter()
.map(|a| SyncAttempt {
job_id: a.job_id.to_string(),
attempt_id: a.attempt_id.to_string(),
job_payload_hash: a.job_payload_hash.clone(),
grant_id: a.grant_id.to_string(),
consumption: a.consumption.clone(),
state: a.state.clone(),
revision: a.revision,
original_boot_id: a.original_boot_id.to_string(),
outcome_sha256: a.outcome_sha256.clone(),
outcome_bytes: a.outcome_bytes.clone(),
..Default::default()
})
.collect(),
}
}
fn general(binding: &Binding) -> Result<control::Response> {
let socket = std::env::var_os("RMM_HELPER_SOCKET").context("NO_GENERAL_HELPER")?;
let path = std::env::var_os("RMM_HELPER_POLICY_FILE").context("GENERAL_POLICY_PIN_REQUIRED")?;
let pin = control::GeneralPin::read(Path::new(&path), binding)?;
control::general_probe(Path::new(&socket), binding, &pin)
}
impl Helpers {
pub fn configured(binding: &Binding) -> Result<Self> {
let mut endpoints = vec![];
if let Some(socket) = std::env::var_os("RMM_HELPER_SOCKET") {
let path = std::env::var_os("RMM_HELPER_POLICY_FILE")
.context("GENERAL_POLICY_PIN_REQUIRED")?;
let pin = control::GeneralPin::read(Path::new(&path), binding)?;
endpoints.push(Endpoint {
socket: socket.into(),
profile: "general".into(),
hash: pin.policy_sha256,
});
}
if let Some(socket) = std::env::var_os("RMM_APT_HELPER_SOCKET") {
ensure!(
Path::new(&socket) == Path::new(readiness::SOCKET),
"APT_HELPER_ROUTE"
);
let path =
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_PROOF_REQUIRED")?;
let pin = readiness::Pin::read(Path::new(&path), binding)?;
endpoints.push(Endpoint {
socket: socket.into(),
profile: "apt_only".into(),
hash: pin.provider_stack_sha256,
});
}
Ok(Self {
binding: binding.clone(),
endpoints,
})
}
fn validate(&self, e: &Endpoint, r: &control::Response) -> Result<()> {
ensure!(
r.profile == e.profile && r.policy_sha256 == e.hash,
"HELPER_PIN_PROFILE_MISMATCH"
);
Ok(())
}
pub fn snapshot(&self, generation: u64) -> Result<Option<reconcile::HelperSnapshot>> {
if self.endpoints.is_empty() {
return Ok(None);
}
for _ in 0..3 {
let mut snapshots = vec![];
let mut inodes = None;
for e in &self.endpoints {
let q = control::Request::Snapshot {
challenge: Uuid::new_v4(),
identity: readiness::Identity::from_binding(&self.binding)?,
generation,
};
let r = control::call(&e.socket, &self.binding, &q)?;
self.validate(e, &r)?;
let ids = (r.ledger_device, r.ledger_inode, r.lock_device, r.lock_inode);
if let Some(old) = inodes {
ensure!(old == ids, "HELPERS_MUST_SHARE_LEDGER_AND_LOCK");
}
inodes = Some(ids);
snapshots.push(r.snapshot.context("HELPER_SNAPSHOT_REQUIRED")?);
}
let first = &snapshots[0];
if snapshots.iter().all(|s| {
s.journal_id == first.journal_id
&& s.checkpoint == first.checkpoint
&& s.previous_decision_id == first.previous_decision_id
&& serde_json::to_vec(&s.attempts).ok()
== serde_json::to_vec(&first.attempts).ok()
}) {
return Ok(Some(map(first)));
}
}
anyhow::bail!("HELPER_CHECKPOINTS_DID_NOT_CONVERGE")
}
pub fn apply(
&self,
response: &SyncResponse,
request: &SyncRequest,
) -> Result<Option<reconcile::HelperSnapshot>> {
use prost::Message;
let mut applied: Option<root::Snapshot> = None;
for e in &self.endpoints {
let q = control::Request::Apply {
challenge: Uuid::new_v4(),
identity: readiness::Identity::from_binding(&self.binding)?,
response: root::SignedResponse {
decision_bytes: response.decision_bytes.clone(),
signature: response.signature.clone(),
},
request_bytes: request.encode_to_vec(),
};
let r = control::call(&e.socket, &self.binding, &q)?;
self.validate(e, &r)?;
let s = r.snapshot.context("APPLIED_HELPER_SNAPSHOT_REQUIRED")?;
if let Some(old) = &applied {
ensure!(
old.journal_id == s.journal_id
&& old.checkpoint == s.checkpoint
&& old.previous_decision_id == s.previous_decision_id,
"HELPER_APPLY_CHECKPOINT_MISMATCH"
);
}
applied = Some(s);
}
Ok(applied.as_ref().map(map))
}
}
pub fn general_capabilities(binding: &Binding) -> Vec<String> {
(|| -> Result<Vec<String>> {
let response = general(binding)?;
let policy = response.policy.context("NO_GENERAL_POLICY")?;
let mut caps = vec![
"helper.root.v1".into(),
format!("helper.policy.sha256:{}", response.policy_sha256),
];
let mut actions = vec![];
if policy.allow_root_scripts && rmm_agent_linux::actions::Interpreter::PosixSh.available() {
actions.push("run_script");
}
if !policy.allowed_services.is_empty() {
actions.extend(["set_service_state", "query_logs"]);
}
if policy.allow_reboot {
actions.push("reboot");
}
for action in actions {
caps.push(action.into());
caps.push(format!("root:{action}"));
}
Ok(caps)
})()
.unwrap_or_default()
}
pub fn general_supported(binding: &Binding, job: &rmm_domain::jobs::JobPayload) -> bool {
(|| -> Result<bool> {
let p = general(binding)?.policy.context("NO_GENERAL_POLICY")?;
Ok(match &job.action {
rmm_domain::jobs::Action::RunScript { interpreter, .. } => {
p.allow_root_scripts
&& match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
}
}
rmm_domain::jobs::Action::SetServiceState { unit, state } => {
p.allowed_services.contains(unit)
&& matches!(state.as_str(), "running" | "stopped")
&& job.max_runtime_seconds >= 180
}
rmm_domain::jobs::Action::QueryLogs { unit, .. } => {
p.allowed_services.contains(unit) && job.max_runtime_seconds >= 20
}
rmm_domain::jobs::Action::Reboot { .. } => {
p.allow_reboot && job.max_runtime_seconds >= 60
}
_ => false,
})
})()
.unwrap_or(false)
}
+326
View File
@@ -0,0 +1,326 @@
//! HTTPS alternative to WSS, using the *same* durable envelopes.
//!
//! The caller owns the journal, sequence allocation and retry policy. In particular,
//! HTTP success never removes a journal entry: only a returned concrete `Ack` may
//! be passed to the journal consumer. There is deliberately no offline send queue.
//! PollRequest has no cursor field in wire profile 1: its fresh heartbeat carries
//! the boot/sequence/connection context. Do not invent a JSON cursor or ACK range.
use anyhow::{Context, Result, ensure};
use prost::Message;
use reqwest::{Client, StatusCode};
use rmm_protocol::{Envelope, MessageBatch, Payload, PollRequest, ServerHello};
use std::{collections::VecDeque, net::SocketAddr, time::Duration};
pub const MAX_BODY: usize = 1024 * 1024;
pub const MAX_BATCH: usize = 16;
const MAX_INCOMING: usize = 32;
/// No environment proxy trust, redirects, alternate names or system CA fallback.
/// `address` is a parsed RMM_CONNECT_ADDR; it overrides TCP only, never SNI.
pub fn mtls_client(address: SocketAddr, ca: &[u8], cert: &[u8], key: &[u8]) -> Result<Client> {
ensure!(
address.port() != 0 && !address.ip().is_unspecified() && !address.ip().is_multicast(),
"INVALID_CONNECT_ADDRESS"
);
let mut identity = Vec::with_capacity(cert.len() + key.len() + 1);
identity.extend_from_slice(cert);
identity.push(b'\n');
identity.extend_from_slice(key);
let client = Client::builder()
.https_only(true)
.tls_built_in_root_certs(false)
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
.identity(reqwest::Identity::from_pem(&identity)?)
.resolve("rmm-dev.internal", address)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(35))
.pool_max_idle_per_host(2)
.build()?;
Ok(client)
}
/// Bounded response reading also applies to chunked bodies, before prost parsing.
pub async fn bounded_body(
mut response: reqwest::Response,
limit: usize,
media: &str,
) -> Result<Vec<u8>> {
ensure!(
response.status() == StatusCode::OK,
"HTTPS_STATUS_{}",
response.status().as_u16()
);
ensure!(
response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
== Some(media),
"HTTPS_CONTENT_TYPE"
);
ensure!(
!response
.headers()
.contains_key(reqwest::header::CONTENT_ENCODING),
"HTTPS_COMPRESSION_UNSUPPORTED"
);
ensure!(
response.content_length().is_none_or(|n| n <= limit as u64),
"HTTPS_BODY_LIMIT"
);
let mut bytes = Vec::new();
while let Some(chunk) = response.chunk().await? {
ensure!(
chunk.len() <= limit.saturating_sub(bytes.len()),
"HTTPS_BODY_LIMIT"
);
bytes.extend_from_slice(&chunk);
}
Ok(bytes)
}
/// Only an explicitly classified WebSocket upgrade refusal may select fallback.
/// TLS, authentication and ambiguous network failures must remain blocked/retried
/// with normal verification. Never infer this classification from an error string.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum WssFailure {
UpgradeBlocked,
TlsOrIdentity,
Authentication,
Other,
}
pub fn allow_https_fallback(failure: WssFailure, attempts: u32) -> bool {
failure == WssFailure::UpgradeBlocked && (2..=3).contains(&attempts)
}
pub fn retry_delay(attempt: u32) -> Duration {
let ceiling = (1_u64 << attempt.min(6)).min(60) * 1000;
Duration::from_millis(rand::random_range(0..=ceiling))
}
pub struct HttpsChannel {
client: Client,
origin: String,
agent: String,
enrollment: u64,
generation: u64,
incoming: VecDeque<Envelope>,
next_request: tokio::time::Instant,
fenced: bool,
}
impl HttpsChannel {
/// Performs ClientHello first. ServerHello remains in the receive queue so
/// the scheduler can close its effect gate and apply reconciliation normally.
pub async fn connect(client: Client, address: SocketAddr, hello: Envelope) -> Result<Self> {
ensure!(
hello.connection_generation == 0
&& matches!(hello.payload, Some(Payload::ClientHello(_))),
"CLIENT_HELLO_REQUIRED"
);
ensure!(
rmm_protocol::valid_envelope(&hello, &hello.agent_id, 0),
"INVALID_CLIENT_HELLO"
);
let mut channel = Self {
client,
origin: format!("https://rmm-dev.internal:{}", address.port()),
agent: hello.agent_id.clone(),
enrollment: hello.enrollment_generation,
generation: 0,
incoming: VecDeque::new(),
next_request: tokio::time::Instant::now(),
fenced: false,
};
let batch = MessageBatch {
protocol_major: 1,
messages: vec![hello],
};
let reply = channel
.request("messages", batch.encode_to_vec(), Duration::from_secs(15))
.await?;
let first = reply.messages.first().context("SERVER_HELLO_REQUIRED")?;
let Some(Payload::ServerHello(ServerHello {
connection_generation,
max_message_bytes,
})) = &first.payload
else {
anyhow::bail!("SERVER_HELLO_REQUIRED")
};
ensure!(
*connection_generation > 0 && *max_message_bytes as usize == MAX_BODY,
"UNSUPPORTED_SERVER_LIMITS"
);
channel.generation = *connection_generation;
channel.accept(reply)?;
Ok(channel)
}
pub fn generation(&self) -> u64 {
self.generation
}
pub fn receive(&mut self) -> Option<Envelope> {
self.incoming.pop_front()
}
pub fn fence(&mut self) {
self.fenced = true;
}
fn validate_outgoing(&self, envelope: &Envelope) -> Result<()> {
ensure!(
rmm_protocol::valid_envelope(envelope, &self.agent, self.generation)
&& envelope.enrollment_generation == self.enrollment,
"INVALID_OUTGOING_ENVELOPE"
);
ensure!(envelope.encoded_len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
Ok(())
}
/// Caller retries durable messages unchanged after uncertainty. Fresh permit
/// requests must instead enter reconciliation on a lost response, not retry.
pub async fn send(&mut self, envelope: &Envelope) -> Result<()> {
self.send_batch(std::slice::from_ref(envelope)).await
}
pub async fn send_batch(&mut self, messages: &[Envelope]) -> Result<()> {
ensure!(
!messages.is_empty() && messages.len() <= MAX_BATCH,
"HTTPS_BATCH_LIMIT"
);
ensure!(
self.incoming.len() + messages.len() < MAX_INCOMING,
"HTTPS_BACKPRESSURE"
);
for e in messages {
self.validate_outgoing(e)?;
ensure!(
!matches!(e.payload, Some(Payload::ClientHello(_))),
"HELLO_REQUIRES_NEW_CHANNEL"
);
if self.fenced || messages.len() > 1 {
ensure!(
matches!(
e.payload,
Some(
Payload::InventorySnapshot(_)
| Payload::JobAccepted(_)
| Payload::JobStateChanged(_)
| Payload::JobResult(_)
)
),
"FENCED_OR_NON_DURABLE_BATCH"
);
}
}
// Check before allocating/cloning the full encoded batch.
let encoded_size = messages
.iter()
.try_fold(2_usize, |n, e| n.checked_add(e.encoded_len() + 5))
.context("HTTPS_BODY_LIMIT")?;
ensure!(encoded_size <= MAX_BODY, "HTTPS_BODY_LIMIT");
let reply = self
.request(
"messages",
MessageBatch {
protocol_major: 1,
messages: messages.to_vec(),
}
.encode_to_vec(),
Duration::from_secs(15),
)
.await?;
self.accept(reply)
}
/// At most one poll per channel (`&mut self`); cancelling a dropped poll is
/// safe, but its heartbeat ID must not be reused for a different envelope.
pub async fn poll(&mut self, heartbeat: Envelope, wait_seconds: u32) -> Result<()> {
ensure!(!self.fenced, "HTTPS_CHANNEL_FENCED");
ensure!(
self.incoming.len() + 2 <= MAX_INCOMING,
"HTTPS_BACKPRESSURE"
);
self.validate_outgoing(&heartbeat)?;
ensure!(
wait_seconds <= 25 && matches!(heartbeat.payload, Some(Payload::Heartbeat(_))),
"INVALID_POLL"
);
let request = PollRequest {
envelope: Some(heartbeat),
wait_seconds,
};
let reply = self
.request(
"poll",
request.encode_to_vec(),
Duration::from_secs(u64::from(wait_seconds) + 10),
)
.await?;
self.accept(reply)
}
async fn request(
&mut self,
route: &str,
body: Vec<u8>,
timeout: Duration,
) -> Result<MessageBatch> {
ensure!(body.len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
tokio::time::sleep_until(self.next_request).await;
// <=10 HTTP requests/sec, with jitter, even with zero-wait polls.
self.next_request =
tokio::time::Instant::now() + Duration::from_millis(rand::random_range(100..=150));
let response = self
.client
.post(format!("{}/agent/v1/{route}", self.origin))
.header(reqwest::header::CONTENT_TYPE, "application/x-protobuf")
.header(reqwest::header::ACCEPT, "application/x-protobuf")
.timeout(timeout)
.body(body)
.send()
.await?;
if response.status() == StatusCode::CONFLICT
|| response.status() == StatusCode::UNAUTHORIZED
|| response.status() == StatusCode::FORBIDDEN
{
self.fence();
}
let bytes = bounded_body(response, MAX_BODY, "application/x-protobuf").await?;
let batch = MessageBatch::decode(bytes.as_slice())?;
ensure!(
batch.protocol_major == 1 && batch.messages.len() <= MAX_BATCH,
"INVALID_SERVER_BATCH"
);
Ok(batch)
}
fn accept(&mut self, reply: MessageBatch) -> Result<()> {
ensure!(
self.incoming.len() + reply.messages.len() <= MAX_INCOMING,
"HTTPS_BACKPRESSURE"
);
// Validate the entire batch before exposing any control message.
for e in &reply.messages {
ensure!(
e.protocol_major == 1
&& e.protocol_minor == 0
&& e.agent_id == self.agent
&& e.enrollment_generation == self.enrollment
&& e.connection_generation == self.generation
&& matches!(
e.payload,
Some(
Payload::ServerHello(_)
| Payload::Ack(_)
| Payload::JobOffer(_)
| Payload::ExecutionPermit(_)
| Payload::CancelRequest(_)
| Payload::SyncResponse(_)
)
),
"INVALID_SERVER_ENVELOPE"
);
if self.fenced {
ensure!(
matches!(e.payload, Some(Payload::Ack(_))),
"HTTPS_CHANNEL_FENCED"
);
}
}
self.incoming.extend(reply.messages);
Ok(())
}
}
+563
View File
@@ -0,0 +1,563 @@
use anyhow::{Result, ensure};
use prost::Message;
use rmm_domain::jobs::{Action, JobPayload, hash};
use rmm_privileged_helper::{Binding, Outcome, Request};
use rmm_protocol::*;
use rusqlite::{Connection, OptionalExtension, params};
use uuid::Uuid;
pub fn supported(job: &JobPayload) -> bool {
if rmm_privileged_helper::apt::is_apt_action(job) {
// A configured environment path alone is never a capability proof.
// Caller binding is independently provisioned, not taken from the job.
return false;
}
// The network process must never execute a script with accidental root.
supported_as(
job,
unsafe { libc::geteuid() },
std::env::var_os("RMM_HELPER_SOCKET").is_some(),
)
}
pub fn supported_with_binding(job: &JobPayload, binding: Option<&Binding>) -> bool {
if rmm_privileged_helper::apt::is_apt_action(job) {
// SAFETY: geteuid has no preconditions.
return unsafe { libc::geteuid() } != 0
&& binding.is_some_and(|b| crate::apt::supported(b, job));
}
if job.execution_identity == "root" {
return unsafe { libc::geteuid() } != 0
&& binding.is_some_and(|b| crate::helper_sync::general_supported(b, job));
}
supported(job)
}
fn supported_as(job: &JobPayload, uid: u32, helper_configured: bool) -> bool {
let unprivileged = uid != 0;
if job.execution_identity == "root" && unprivileged {
return helper_configured
&& matches!(
job.action,
Action::RunScript { .. } | Action::SetServiceState { .. } | Action::Reboot { .. }
);
}
job.execution_identity == "agent_service"
&& unprivileged
&& match &job.action {
Action::RunScript { interpreter, .. } => match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
},
Action::RefreshInventory { scope } => scope == "linux.base",
Action::QueryLogs { .. } => log_request(job).is_ok(),
_ => false,
}
}
pub fn accept(db: &Connection, binding: &Binding, offer: &JobOffer) -> Result<bool> {
let job = binding.job(&offer.payload_bytes)?;
ensure!(
offer.job_id == job.job_id.to_string()
&& offer.attempt_id == job.attempt_id.to_string()
&& offer.job_payload_hash == hash(&offer.payload_bytes)
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
&& offer.plan_hash.len() == 64
&& hex_hash(&offer.plan_hash),
"OFFER_BINDING"
);
ensure!(
supported_with_binding(&job, Some(binding)),
"UNSUPPORTED_CAPABILITY"
);
let bytes = offer.encode_to_vec();
let existing: Option<Vec<u8>> = db
.query_row(
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)
.optional()?;
if let Some(old) = existing {
ensure!(old == bytes, "OFFER_CONFLICT");
return Ok(false);
}
let count: i64 = db.query_row("SELECT count(*) FROM job_attempts", [], |r| r.get(0))?;
ensure!(count < 10000, "JOURNAL_CAPACITY_NEW_EFFECTS_BLOCKED");
db.execute(
"INSERT INTO job_attempts(job,attempt,offer,state) VALUES(?1,?2,?3,'accepted')",
params![offer.job_id, offer.attempt_id, bytes],
)?;
Ok(true)
}
fn hex_hash(s: &str) -> bool {
s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
db.execute(
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
params![envelope.message_id, envelope.encode_to_vec()],
)?;
Ok(())
}
pub fn events(db: &Connection) -> Result<Vec<Envelope>> {
let mut statement = db.prepare("SELECT payload FROM job_events ORDER BY rowid LIMIT 100")?;
let bytes = statement
.query_map([], |r| r.get::<_, Vec<u8>>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
bytes
.into_iter()
.map(|b| Ok(Envelope::decode(b.as_slice())?))
.collect()
}
pub fn ack(db: &mut Connection, id: &str) -> Result<()> {
let tx = db.transaction()?;
crate::reconcile::archive_events(&tx)?;
tx.execute("DELETE FROM job_events WHERE id=?1", [id])?;
tx.commit()?;
Ok(())
}
pub fn next(db: &Connection) -> Result<Option<JobOffer>> {
// One device-wide effect at a time; unknown/reserved attempts retain locks.
let busy: i64 = db.query_row("SELECT count(*) FROM job_attempts WHERE state IN ('requesting','running','unknown','awaiting_reboot')", [], |r| r.get(0))?;
if busy != 0 {
return Ok(None);
}
let mut statement = db.prepare("SELECT offer FROM job_attempts WHERE state='accepted' AND cancel_requested=0 ORDER BY rowid")?;
let rows = statement.query_map([], |r| r.get::<_, Vec<u8>>(0))?;
let now = chrono::Utc::now();
for bytes in rows {
let offer = JobOffer::decode(bytes?.as_slice())?;
let job: JobPayload = serde_json::from_slice(&offer.payload_bytes)?;
if now >= job.not_before || now >= job.start_before {
return Ok(Some(offer));
}
}
Ok(None)
}
pub fn requesting(db: &Connection) -> Result<Option<JobOffer>> {
let bytes: Option<Vec<u8>> = db
.query_row(
"SELECT offer FROM job_attempts WHERE state='requesting' LIMIT 1",
[],
|r| r.get(0),
)
.optional()?;
bytes
.map(|b| Ok(JobOffer::decode(b.as_slice())?))
.transpose()
}
pub fn reserve_request(db: &Connection, offer: &JobOffer) -> Result<()> {
ensure!(db.execute("UPDATE job_attempts SET state='requesting' WHERE job=?1 AND attempt=?2 AND state='accepted' AND cancel_requested=0",
params![offer.job_id,offer.attempt_id])? == 1, "ATTEMPT_NOT_STARTABLE");
Ok(())
}
pub fn transition(db: &Connection, offer: &JobOffer, state: &str, grant: &str) -> Result<u64> {
let previous: String = db.query_row(
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
let allowed = match previous.as_str() {
"accepted" | "requesting" => matches!(
state,
"running" | "failed" | "cancelled" | "expired" | "unknown"
),
"running" | "verifying" => matches!(
state,
"verifying" | "succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
),
// Only a definitive completion of the already-running executor calls
// this reconciliation path. No new effect is started from unknown.
"unknown" | "awaiting_reboot" => matches!(
state,
"succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
),
_ => false,
};
ensure!(allowed, "INVALID_LOCAL_STATE_TRANSITION");
let rev: i64 = db.query_row("UPDATE job_attempts SET state=?3,grant_id=?4,revision=revision+1 WHERE job=?1 AND attempt=?2 RETURNING revision",
params![offer.job_id,offer.attempt_id,state,grant], |r| r.get(0))?;
Ok(u64::try_from(rev)?)
}
pub fn result(offer: &JobOffer, revision: u64, grant: &str, outcome: Outcome) -> JobResult {
JobResult {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
revision,
state: outcome.state,
grant_id: grant.into(),
stdout: outcome.stdout,
stderr: outcome.stderr,
output_truncated: outcome.output_truncated,
reboot_required: outcome.reboot_required,
exit_code: outcome.exit_code,
evidence_json: outcome.evidence.to_string(),
error_code: outcome.error_code,
}
}
pub fn no_start(state: &str, code: &str) -> Outcome {
Outcome {
state: state.into(),
evidence: serde_json::json!({"not_started":true,"postconditions_passed":false}),
..Outcome::unknown(code)
}
}
pub fn execute(job: &JobPayload) -> Outcome {
let result = match &job.action {
Action::RunScript { .. } => rmm_privileged_helper::script(job),
Action::RefreshInventory { .. } => rmm_agent_linux::collect().and_then(|i| {
let bytes = serde_json::to_vec(&i)?;
ensure!(bytes.len() <= job.max_output_bytes as usize, "INVENTORY_OUTPUT_LIMIT");
Ok(Outcome { state: "succeeded".into(), stdout: bytes, stderr: vec![], exit_code: None,
output_truncated: false, reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":true,"source_id":"linux.base","observed_at":chrono::Utc::now(),"completeness":"partial"}), error_code: String::new() })
}),
Action::QueryLogs { .. } => query_logs(job),
_ => Err(anyhow::anyhow!("unsupported action")),
};
result.unwrap_or_else(|_| Outcome::unknown("EXECUTOR_EFFECT_UNKNOWN"))
}
fn allowed_log_units() -> Result<Vec<rmm_agent_linux::actions::ServiceUnit>> {
let path = std::env::var_os("RMM_LOG_UNITS_FILE")
.ok_or_else(|| anyhow::anyhow!("LOG_ALLOWLIST_NOT_CONFIGURED"))?;
let path = std::path::Path::new(&path);
rmm_privileged_helper::private_path(path, unsafe { libc::geteuid() }, false)?;
ensure!(
std::fs::metadata(path)?.len() <= 32768,
"LOG_ALLOWLIST_SIZE"
);
let units: Vec<String> = serde_json::from_slice(&std::fs::read(path)?)?;
ensure!(
!units.is_empty() && units.len() <= 128,
"LOG_ALLOWLIST_SIZE"
);
units
.iter()
.map(|u| rmm_agent_linux::actions::ServiceUnit::new(u))
.collect()
}
pub fn logs_available() -> bool {
allowed_log_units().is_ok()
&& rmm_agent_linux::inventory::capabilities()
.get("logs.journald")
.is_some_and(|o| o.data == Some(true))
}
fn log_request(job: &JobPayload) -> Result<rmm_agent_linux::inventory::JournalRequest> {
log_request_for(job, &allowed_log_units()?)
}
fn log_request_for(
job: &JobPayload,
allowed: &[rmm_agent_linux::actions::ServiceUnit],
) -> Result<rmm_agent_linux::inventory::JournalRequest> {
let Action::QueryLogs {
unit,
since,
until,
max_lines,
} = &job.action
else {
anyhow::bail!("NOT_LOG_QUERY")
};
ensure!(job.max_runtime_seconds >= 20, "LOG_RUNTIME_TOO_SHORT");
rmm_agent_linux::inventory::JournalRequest::new(
rmm_agent_linux::actions::ServiceUnit::new(unit)?,
allowed,
u64::try_from(since.timestamp())?,
u64::try_from(until.timestamp())?,
*max_lines as usize,
(job.max_output_bytes as usize).min(256 * 1024),
)?
.precise_window(*since, *until)
}
fn query_logs(job: &JobPayload) -> Result<Outcome> {
let observation = rmm_agent_linux::inventory::query_journal(&log_request(job)?);
let valid = observation.quality == rmm_agent_linux::inventory::Quality::Valid;
let error = observation.error_code.clone().unwrap_or_default();
let mut value = serde_json::to_value(&observation)?;
while serde_json::to_vec(&value)?.len() > job.max_output_bytes as usize {
let Some(lines) = value["data"]["json_lines"]
.as_array_mut()
.filter(|v| !v.is_empty())
else {
return Ok(Outcome {
state: "failed".into(),
..Outcome::unknown("LOG_RESULT_METADATA_EXCEEDS_OUTPUT_LIMIT")
});
};
lines.remove(0);
value["truncated"] = serde_json::json!(true);
value["completeness"] = serde_json::json!("partial");
value["error_code"] = serde_json::json!("LOG_SERIALIZED_OUTPUT_LIMIT");
}
Ok(Outcome {
state: if valid { "succeeded" } else { "failed" }.into(),
stdout: serde_json::to_vec(&value)?,
stderr: vec![],
exit_code: None,
output_truncated: value["truncated"] == true,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":valid,"source_id":observation.source_id,
"observed_at":value["observed_at"],"quality":value["quality"],"completeness":value["completeness"],
"cursor_status":"not_advanced_no_ack_contract","provider_error_code":error}),
error_code: if valid {
String::new()
} else if error.is_empty() {
"LOG_PROVIDER_UNAVAILABLE".into()
} else {
error
},
})
}
pub fn permit_request(offer: &JobOffer, permit: &ExecutionPermit) -> Result<Request> {
Ok(Request {
job_bytes: offer.payload_bytes.clone(),
permit_bytes: permit.payload_bytes.clone(),
signature: permit.signature.clone(),
plan_hash: offer.plan_hash.clone(),
authorization_id: Uuid::parse_str(&offer.authorization_id)?,
})
}
/// With no backend SyncResponse contract, a reconnect/restart cannot prove an
/// earlier reservation never started. Telemetry and evidence replay continue;
/// all new effects stop. Do not paper over this with a new start request.
#[cfg(test)]
pub fn begin_session(db: &Connection) -> Result<bool> {
let inserted = db.execute(
"INSERT OR IGNORE INTO safety(key,value) VALUES('connected','1')",
[],
)?;
Ok(inserted == 1)
}
#[cfg(test)]
pub fn uncertain(db: &Connection) -> Result<Vec<(JobOffer, String)>> {
let mut statement = db.prepare(
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running')",
)?;
let bytes = statement
.query_map([], |r| {
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
bytes
.into_iter()
.map(|(bytes, grant)| Ok((JobOffer::decode(bytes.as_slice())?, grant)))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage;
fn fixture() -> (std::path::PathBuf, Binding, JobOffer) {
let path = std::env::temp_dir().join(format!("rmm-agent-jobs-{}", Uuid::new_v4()));
storage::prepare_dir(&path).unwrap();
let binding = Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 2,
public_key_hex: "00".repeat(32),
};
let now = chrono::Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: binding.tenant_id,
asset_id: binding.asset_id,
agent_id: binding.agent_id,
enrollment_generation: 1,
assignment_version: 2,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RefreshInventory {
scope: "linux.base".into(),
},
execution_identity: "agent_service".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(10),
max_runtime_seconds: 10,
max_output_bytes: 4096,
};
let bytes = rmm_domain::jobs::encode(&job).unwrap();
let offer = JobOffer {
job_id: job.job_id.to_string(),
attempt_id: job.attempt_id.to_string(),
authorization_id: Uuid::new_v4().to_string(),
plan_hash: hash(b"plan"),
job_payload_hash: hash(&bytes),
payload_bytes: bytes,
};
(path, binding, offer)
}
#[test]
fn sqlite_reopen_offer_dedup_and_no_new_permit_after_ambiguity() {
let (path, binding, offer) = fixture();
let mut db = storage::open(&path).unwrap();
let job = binding.job(&offer.payload_bytes).unwrap();
if unsafe { libc::geteuid() } == 0 {
assert!(!supported(&job));
assert!(accept(&db, &binding, &offer).is_err());
} else {
assert!(accept(&db, &binding, &offer).unwrap());
assert!(!accept(&db, &binding, &offer).unwrap());
let mut changed = offer.clone();
changed.plan_hash = hash(b"different plan");
assert!(accept(&db, &binding, &changed).is_err());
assert!(begin_session(&db).unwrap());
let envelope = Envelope {
message_id: Uuid::new_v4().to_string(),
sequence: storage::next_sequence(&mut db).unwrap(),
payload: Some(Payload::JobAccepted(JobAccepted {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
job_payload_hash: offer.job_payload_hash.clone(),
journal_proof: "durable".into(),
})),
..Default::default()
};
event(&db, &envelope).unwrap();
reserve_request(&db, &offer).unwrap();
assert!(reserve_request(&db, &offer).is_err());
assert!(next(&db).unwrap().is_none());
drop(db);
db = storage::open(&path).unwrap();
assert!(!begin_session(&db).unwrap());
assert_eq!(uncertain(&db).unwrap().len(), 1);
assert_eq!(events(&db).unwrap(), vec![envelope.clone()]);
assert_eq!(transition(&db, &offer, "unknown", "").unwrap(), 1);
assert!(next(&db).unwrap().is_none());
assert!(!accept(&db, &binding, &offer).unwrap());
ack(&mut db, &envelope.message_id).unwrap();
assert!(events(&db).unwrap().is_empty());
assert!(next(&db).unwrap().is_none());
}
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn cancellation_expiry_and_atomic_event_rollback() {
let (path, binding, offer) = fixture();
let mut db = storage::open(&path).unwrap();
if unsafe { libc::geteuid() } != 0 {
accept(&db, &binding, &offer).unwrap();
db.execute("UPDATE job_attempts SET cancel_requested=1", [])
.unwrap();
assert!(reserve_request(&db, &offer).is_err());
let cancelled = no_start("cancelled", "CANCELLED");
assert_eq!(cancelled.evidence["not_started"], true);
let tx = db.transaction().unwrap();
assert_eq!(transition(&tx, &offer, "cancelled", "").unwrap(), 1);
let e = Envelope {
message_id: Uuid::new_v4().to_string(),
payload: Some(Payload::JobResult(result(&offer, 1, "", cancelled))),
..Default::default()
};
event(&tx, &e).unwrap();
tx.rollback().unwrap();
assert!(events(&db).unwrap().is_empty());
assert!(next(&db).unwrap().is_none());
let expired = no_start("expired", "DEADLINE_EXPIRED");
assert_eq!(expired.evidence["not_started"], true);
assert_eq!(transition(&db, &offer, "expired", "").unwrap(), 1);
assert!(next(&db).unwrap().is_none());
} else {
assert!(accept(&db, &binding, &offer).is_err());
}
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn network_process_never_executes_as_accidental_root() {
let (path, binding, offer) = fixture();
let mut job = binding.job(&offer.payload_bytes).unwrap();
assert!(!supported_as(&job, 0, false));
assert!(!supported_as(&job, 0, true));
job.execution_identity = "root".into();
assert!(!supported_as(&job, 1000, false));
assert!(!supported_as(&job, 0, true));
job.action = Action::Reboot {
reason: "explicit root only".into(),
reconnect_timeout_seconds: 60,
dependency_group: "test".into(),
};
assert!(supported_as(&job, 1000, true));
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn scheduled_future_job_does_not_starve_ready_job() {
let (path, binding, offer) = fixture();
let db = storage::open(&path).unwrap();
if unsafe { libc::geteuid() } != 0 {
let mut future = offer.clone();
let mut job: JobPayload = serde_json::from_slice(&future.payload_bytes).unwrap();
job.job_id = Uuid::new_v4();
job.attempt_id = Uuid::new_v4();
job.not_before = chrono::Utc::now() + chrono::Duration::minutes(1);
future.job_id = job.job_id.to_string();
future.attempt_id = job.attempt_id.to_string();
future.payload_bytes = rmm_domain::jobs::encode(&job).unwrap();
future.job_payload_hash = hash(&future.payload_bytes);
accept(&db, &binding, &future).unwrap();
accept(&db, &binding, &offer).unwrap();
assert_eq!(next(&db).unwrap().unwrap().job_id, offer.job_id);
}
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn live_logs_enforce_actual_action_window_allowlist_limits_and_runtime() {
let (path, binding, offer) = fixture();
let mut job = binding.job(&offer.payload_bytes).unwrap();
let unit = rmm_agent_linux::actions::ServiceUnit::new("rmm-test.service").unwrap();
let now = chrono::Utc::now();
job.max_runtime_seconds = 20;
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::minutes(1),
until: now,
max_lines: 10,
};
assert!(log_request_for(&job, &[]).is_err());
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_ok());
job.max_runtime_seconds = 10;
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
job.max_runtime_seconds = 20;
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::hours(25),
until: now,
max_lines: 10,
};
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::minutes(1),
until: now,
max_lines: 1001,
};
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn refresh_is_a_new_measurement_with_real_observation_time() {
let (path, binding, offer) = fixture();
let job = binding.job(&offer.payload_bytes).unwrap();
let before = chrono::Utc::now();
let outcome = execute(&job);
assert_eq!(outcome.state, "succeeded");
let observed =
chrono::DateTime::parse_from_rfc3339(outcome.evidence["observed_at"].as_str().unwrap())
.unwrap();
assert!(observed >= before);
assert!(
serde_json::from_slice::<rmm_domain::Inventory>(&outcome.stdout)
.unwrap()
.validate()
);
std::fs::remove_dir_all(path).unwrap();
}
}
+805
View File
@@ -0,0 +1,805 @@
mod apt;
mod bootstrap;
mod certificate_rotation;
mod helper_sync;
mod https_channel;
mod jobs;
mod reconcile;
#[cfg(test)]
mod runtime_tests;
mod storage;
mod transport;
mod wire_channel;
use anyhow::{Context, Result, ensure};
use chrono::Utc;
use openssl::{
ec::{EcGroup, EcKey},
hash::MessageDigest,
nid::Nid,
pkey::PKey,
x509::{X509NameBuilder, X509Req},
};
use prost::Message;
use rmm_protocol::*;
use sha2::{Digest, Sha256};
use std::{
env, fs,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use uuid::Uuid;
struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
address: std::net::SocketAddr,
ca: Vec<u8>,
id: String,
boot: String,
instance: String,
db: rusqlite::Connection,
started: Instant,
clock: rmm_privileged_helper::TrustedClock,
binding: Option<rmm_privileged_helper::Binding>,
ledger: rusqlite::Connection,
completed_tx: tokio::sync::mpsc::Sender<Completion>,
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
inventory_tx: tokio::sync::mpsc::Sender<Result<(Vec<u8>, i64)>>,
inventory_rx: tokio::sync::mpsc::Receiver<Result<(Vec<u8>, i64)>>,
inventory_inflight: bool,
sync: Option<reconcile::ClientSync>,
sync_clock: C,
helpers: Option<helper_sync::Helpers>,
prepared: Option<reconcile::PreparedSync>,
fence: Arc<EffectFence>,
mode: wire_channel::Mode,
certificates: certificate_rotation::RotationStore,
rotation_policy: certificate_rotation::RotationPolicy,
}
#[derive(Default)]
struct EffectFence {
generation: AtomicU64,
deadline: AtomicI64,
root: AtomicBool,
}
fn report_closed_gate(error: &anyhow::Error) {
let text = error.to_string();
let code = if text.len() <= 96
&& text
.bytes()
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
{
text.as_str()
} else {
"SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE"
};
eprintln!("agent effect gate closed: {code}");
}
impl EffectFence {
fn close(&self) {
self.generation.store(0, Ordering::SeqCst);
self.root.store(false, Ordering::SeqCst);
}
fn allows(&self, generation: u64, root: bool) -> bool {
self.generation.load(Ordering::SeqCst) == generation
&& generation > 0
&& Utc::now().timestamp_millis() < self.deadline.load(Ordering::SeqCst)
&& (!root || self.root.load(Ordering::SeqCst))
}
}
type Completion = (
JobOffer,
rmm_domain::jobs::ExecutionPermitPayload,
rmm_privileged_helper::Outcome,
);
impl<C: reconcile::SyncClock> Agent<C> {
fn close_gate(&mut self) {
self.fence.close();
if let Some(sync) = &mut self.sync {
sync.close();
}
self.prepared = None;
}
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
Ok(reconcile::Channel {
connection_generation: generation,
boot_id: Uuid::parse_str(&self.boot)?,
})
}
fn ready(&self, generation: u64, root: bool) -> bool {
self.channel(generation).is_ok_and(|c| {
self.sync
.as_ref()
.is_some_and(|s| s.can_start(&c, root, &self.sync_clock))
})
}
async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> {
self.fence.close();
let helpers = self.helpers.clone();
let snapshot = tokio::task::spawn_blocking(move || match helpers {
Some(h) => h.snapshot(generation),
None => Ok(None),
})
.await??;
let channel = self.channel(generation)?;
let Some(sync) = &mut self.sync else {
return Ok(());
};
let prepared = sync.prepare(
&mut self.db,
&self.ledger,
channel,
snapshot.as_ref(),
&self.sync_clock,
)?;
wire.send(&prepared.envelope).await?;
self.prepared = Some(prepared);
Ok(())
}
async fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
let prepared = self.prepared.take().context("UNSOLICITED_SYNC_RESPONSE")?;
let sync = self.sync.as_mut().context("SYNC_BINDING_REQUIRED")?;
let applied = sync.apply(&mut self.db, &response, &self.sync_clock)?;
sync.apply_consumption(&mut self.ledger, &self.sync_clock)?;
let reason = if applied
.reason
.bytes()
.all(|b| b.is_ascii_uppercase() || b == b'_')
{
applied.reason.as_str()
} else {
"RECONCILIATION_REQUIRED"
};
eprintln!(
"agent reconciliation committed: decision={}, receipts={}, reason={}",
applied.decision_id,
applied.acknowledged_message_ids.len(),
reason
);
if let Some(instruction) = applied.helper_instruction {
let helpers = self
.helpers
.clone()
.context("HELPER_SYNC_CLIENT_REQUIRED")?;
let request = prepared.request.clone();
let snapshot =
tokio::task::spawn_blocking(move || helpers.apply(&instruction.response, &request))
.await??
.context("HELPER_SYNC_ACK_REQUIRED")?;
if self.ready(generation, false) {
self.sync
.as_mut()
.context("SYNC_REQUIRED")?
.confirm_helper(&snapshot)?;
}
}
if self.ready(generation, false) {
let deadline = self
.sync
.as_ref()
.and_then(|s| s.validity_deadline())
.context("SIGNED_LEASE_REQUIRED")?;
self.fence
.deadline
.store(deadline.timestamp_millis(), Ordering::SeqCst);
self.fence
.root
.store(self.ready(generation, true), Ordering::SeqCst);
self.fence.generation.store(generation, Ordering::SeqCst);
}
Ok(())
}
fn record(
&mut self,
generation: u64,
offer: &JobOffer,
grant: &str,
outcome: rmm_privileged_helper::Outcome,
) -> Result<()> {
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
let tx = self.db.transaction()?;
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
envelope.payload = Some(Payload::JobResult(jobs::result(
offer, revision, grant, outcome,
)));
jobs::event(&tx, &envelope)?;
tx.commit()?;
Ok(())
}
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
// A large replay backlog must not monopolize heartbeat/cancel handling.
// On backpressure timeout, discard this channel rather than continuing
// to use a websocket whose partially sent frame was cancelled.
let events = jobs::events(&self.db)?;
tokio::time::timeout(Duration::from_secs(2), async {
for mut event in events.into_iter().take(8) {
event.connection_generation = generation;
ws.send(&event).await?;
tokio::time::sleep(Duration::from_millis(100)).await;
}
Ok::<_, anyhow::Error>(())
})
.await??;
Ok(())
}
fn permit(
&mut self,
generation: u64,
permit: ExecutionPermit,
safe_session: bool,
) -> Result<()> {
let Some(offer) = jobs::requesting(&self.db)? else {
anyhow::bail!("UNSOLICITED_PERMIT")
};
if !permit.denial_code.is_empty() {
// A denial is not proof that an earlier reserved grant never existed.
self.record(
generation,
&offer,
"",
rmm_privileged_helper::Outcome::unknown("PERMIT_DENIED_RECONCILIATION_REQUIRED"),
)?;
return Ok(());
}
ensure!(
safe_session && self.clock.trusted(),
"CLOCK_OR_SYNC_UNTRUSTED"
);
let binding = self
.binding
.as_ref()
.context("job trust not provisioned")?
.clone();
let request = jobs::permit_request(&offer, &permit)?;
let (job, grant) = binding.verify(&request, Utc::now())?;
ensure!(
self.ready(generation, job.execution_identity == "root"),
"FENCED_SYNC_REQUIRED"
);
ensure!(
jobs::supported_with_binding(&job, Some(&binding)),
"UNSUPPORTED_CAPABILITY"
);
rmm_privileged_helper::consume(&mut self.ledger, &grant)?;
rmm_privileged_helper::save_start(&self.ledger, &grant, &request)?;
let cancelled: bool = self.db.query_row(
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
rusqlite::params![offer.job_id, offer.attempt_id],
|r| r.get(0),
)?;
if cancelled {
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
self.record(
generation,
&offer,
&grant.grant_id.to_string(),
outcome.clone(),
)?;
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
return Ok(());
}
let mut envelope = self.envelope(
generation,
Payload::JobStateChanged(JobStateChanged::default()),
)?;
let tx = self.db.transaction()?;
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
job_id: offer.job_id.clone(),
attempt_id: offer.attempt_id.clone(),
revision,
state: "running".into(),
grant_id: grant.grant_id.to_string(),
}));
jobs::event(&tx, &envelope)?;
tx.commit()?;
let completed = self.completed_tx.clone();
let clock = self.clock.clone();
let fence = self.fence.clone();
tokio::task::spawn_blocking(move || {
// Thread-pool scheduling and durable fsync may have consumed the
// remaining start window. Recheck at the actual effect boundary.
let outcome = if !fence.allows(generation, job.execution_identity == "root")
|| !clock.trusted()
|| binding.verify(&request, Utc::now()).is_err()
{
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
} else if job.execution_identity == "root" {
apt::invoke(&binding, &job, &request).unwrap_or_else(|_| {
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
})
} else {
jobs::execute(&job)
};
let _ = completed.blocking_send((offer, grant, outcome));
});
Ok(())
}
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
Ok(Envelope {
protocol_major: 1,
protocol_minor: 0,
message_id: Uuid::new_v4().to_string(),
agent_id: self.id.clone(),
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
connection_generation: generation,
boot_id: self.boot.clone(),
sequence: storage::next_sequence(&mut self.db)?,
observed_at: Utc::now().timestamp_millis(),
payload: Some(payload),
})
}
fn capabilities(&self) -> Vec<String> {
let mut capabilities = vec!["linux.base/1".into(), "heartbeat/1".into()];
if self.binding.is_some() {
capabilities.push("rmm.sync.v1".into());
}
if let Some(binding) = &self.binding
&& unsafe { libc::geteuid() } != 0
{
if rmm_agent_linux::actions::Interpreter::PosixSh.available() {
capabilities.push("run_script".into());
}
capabilities.push("refresh_inventory".into());
if jobs::logs_available() {
capabilities.push("query_logs".into());
}
capabilities.extend(apt::capabilities(binding));
capabilities.extend(helper_sync::general_capabilities(binding));
}
capabilities.sort();
capabilities.dedup();
capabilities
}
fn queue_inventory(&mut self) {
if self.inventory_inflight || !storage::passive_collection_ready(&self.db).unwrap_or(false)
{
return;
}
self.inventory_inflight = true;
let sender = self.inventory_tx.clone();
let extended = env::var("RMM_EXTENDED_INVENTORY").unwrap_or_else(|_| "1".into()) == "1";
let binding = self.binding.clone();
tokio::task::spawn_blocking(move || {
let result = rmm_agent_linux::wire::snapshot(extended).and_then(|mut inventory| {
if extended {
inventory["observations"]
.as_array_mut()
.context("INVENTORY_OBSERVATIONS")?
.push(apt::planning_observation(binding.as_ref()));
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
}
let bytes = serde_json::to_vec(&inventory)?;
ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT");
Ok((bytes, Utc::now().timestamp_millis()))
});
let _ = sender.blocking_send(result);
});
}
fn inventory_envelope(
&mut self,
generation: u64,
data: Vec<u8>,
observed_at: i64,
) -> Result<Envelope> {
let mut envelope = self.envelope(
generation,
Payload::InventorySnapshot(InventorySnapshot {
snapshot_id: Uuid::new_v4().to_string(),
scope: "linux.base".into(),
schema_version: 1,
chunk_index: 0,
chunk_count: 1,
sha256: Sha256::digest(&data).to_vec(),
data,
completeness: "partial".into(),
}),
)?;
envelope.observed_at = observed_at;
Ok(envelope)
}
async fn session(&mut self) -> Result<()> {
let result = self.session_inner().await;
self.close_gate();
result
}
async fn session_inner(&mut self) -> Result<()> {
self.close_gate();
self.boot = rmm_agent_linux::boot_id()?;
if self.helpers.is_some() {
self.helpers = self
.binding
.as_ref()
.map(helper_sync::Helpers::configured)
.transpose()?;
}
let advertised = self.capabilities();
let hello = self.envelope(
0,
Payload::ClientHello(ClientHello {
agent_version: env!("CARGO_PKG_VERSION").into(),
capabilities: advertised.clone(),
}),
)?;
let identity = self.certificates.current(Utc::now().timestamp())?;
let mut ws =
wire_channel::connect(self.mode, self.address, &self.ca, &identity, hello).await?;
let reply = tokio::time::timeout(Duration::from_secs(20), ws.receive()).await??;
ensure!(
reply.protocol_major == 1
&& reply.protocol_minor == 0
&& reply.agent_id == self.id
&& reply.enrollment_generation
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
"invalid server hello"
);
let Some(Payload::ServerHello(server)) = reply.payload else {
anyhow::bail!("server hello required")
};
let generation = server.connection_generation;
ensure!(
generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32,
"unsupported limits"
);
if let Some(mut pending) = storage::pending(&self.db)? {
// Generation is transport fencing, not part of durable message identity.
pending.connection_generation = generation;
jobs::event(&self.db, &pending)?;
storage::clear_pending(&self.db)?;
}
self.queue_inventory();
self.replay(&ws, generation).await?;
let mut needs_sync = false;
if self.sync.is_some()
&& let Err(error) = self.start_sync(&ws, generation).await
{
report_closed_gate(&error);
self.close_gate();
needs_sync = true;
}
eprintln!("authenticated channel established; inventory queued for durable acknowledgment");
let mut inventory_at = Instant::now();
let mut heartbeat_at = Instant::now();
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
let mut tick = tokio::time::interval(Duration::from_secs(5));
let mut sync_at = Instant::now();
let mut rotation_at = Instant::now();
let mut capability_at = Instant::now();
let mut first_heartbeat = true;
loop {
tokio::select! {
Some(result) = self.inventory_rx.recv() => {
self.inventory_inflight = false;
if let Ok((data,observed_at)) = result {
let inventory = self.inventory_envelope(generation,data,observed_at)?;
jobs::event(&self.db,&inventory)?;
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
}
message = ws.receive() => {
let message = message?;
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
match message.payload {
Some(Payload::Ack(ack)) => {
use rusqlite::OptionalExtension;
let pending:Option<Vec<u8>>=self.db.query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
jobs::ack(&mut self.db, &ack.message_id)?;
},
Some(Payload::SyncResponse(response))=>{if let Err(error)=self.apply_sync(generation,response).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}sync_at=Instant::now();},
Some(Payload::JobOffer(offer)) => {
let binding = self.binding.as_ref().context("job trust not provisioned")?.clone();
let accepted = self.envelope(generation, Payload::JobAccepted(JobAccepted {
job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(),
job_payload_hash: offer.job_payload_hash.clone(), journal_proof: rmm_domain::jobs::hash(&offer.encode_to_vec()),
}))?;
// Clone before borrowing the transaction; acceptance and the
// durable ACK event commit together, before any wire ACK.
let tx = self.db.transaction()?;
let fresh=jobs::accept(&tx, &binding, &offer)?;
jobs::event(&tx, &accepted)?;
tx.commit()?;
needs_sync|=fresh;self.replay(&ws, generation).await?;
}
Some(Payload::ExecutionPermit(permit)) => self.permit(generation, permit, self.ready(generation,false))?,
Some(Payload::CancelRequest(cancel)) => {
let row: Option<(Vec<u8>, String, String)> = {
use rusqlite::OptionalExtension;
self.db.query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?
};
if let Some((bytes,state,grant)) = row {
self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?;
if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; }
// Running script cancellation is not confirmed by an
// inbound request. Its bounded executor reports reality.
}
let ack = self.envelope(generation, Payload::Ack(Ack { message_id: message.message_id }))?;
ws.send(&ack).await?;
}
_ => anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE"),
}
}
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
// Persist the wire result before releasing a consumption lock.
// A crash in between is fail-closed, not a second execution.
let state = outcome.state.clone();
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
let serialized = serde_json::to_vec(&outcome)?;
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?;
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); }
needs_sync=true;self.replay(&ws, generation).await?;
if refresh_capabilities {return Ok(());}
}
_ = tick.tick() => {
self.replay(&ws, generation).await?;
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
if self.prepared.is_some() && !busy && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
if self.prepared.is_none() && self.sync.is_some() && !busy && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
}
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
&& self.clock.trusted() && let Some(offer) = jobs::next(&self.db)? {
let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?;
if Utc::now() >= job.start_before {
self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?;
} else if !self.ready(generation,job.execution_identity=="root") {
needs_sync=true;
} else if Utc::now() >= job.not_before {
// Reserve before send. A lost response must never cause
// a second permit request for this attempt.
jobs::reserve_request(&self.db, &offer)?;
let request = self.envelope(generation, Payload::ExecutionPermitRequest(ExecutionPermitRequest {
job_id: offer.job_id, attempt_id: offer.attempt_id,
job_payload_hash: offer.job_payload_hash, assignment_version: job.assignment_version,
}))?;
ws.send(&request).await?;
}
}
}
}
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
first_heartbeat = false;
let heartbeat = self.envelope(
generation,
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
}),
)?;
ws.send(&heartbeat).await?;
if ws.uses_https() {
let poll = self.envelope(
generation,
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
}),
)?;
ws.poll(poll).await?;
}
heartbeat_at = Instant::now();
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
}
// Planning pins must be refreshed before the 90-second stored-data
// planning limit. One inflight blocking collector bounds pressure;
// heartbeats never perform this metadata work.
if inventory_at.elapsed() >= Duration::from_secs(60) {
self.queue_inventory();
inventory_at = Instant::now();
}
if rotation_at.elapsed() >= Duration::from_secs(30) {
rotation_at = Instant::now();
if self
.certificates
.due(self.rotation_policy, Utc::now().timestamp())?
{
self.close_gate();
drop(ws);
let rotated = self
.certificates
.renew(self.address, Utc::now().timestamp())
.await?;
ensure!(
rotated.needs_reconnect
&& rotated.enrollment_generation
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
"ROTATION_IDENTITY_CHANGED"
);
return Ok(());
}
}
}
}
}
async fn enrollment(dir: &Path, address: std::net::SocketAddr, ca: &[u8]) -> Result<String> {
let identity = dir.join("identity.json");
if identity.exists() {
let response: EnrollmentResponse = serde_json::from_slice(&fs::read(identity)?)?;
Uuid::parse_str(&response.agent_id)?;
return Ok(response.agent_id);
}
let key_path = dir.join("device.key");
let key = if key_path.exists() {
PKey::private_key_from_pem(&fs::read(&key_path)?)?
} else {
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
storage::create_secret(&key_path, &key.private_key_to_pem_pkcs8()?)?;
key
};
let mut csr = X509Req::builder()?;
let mut name = X509NameBuilder::new()?;
name.append_entry_by_text("CN", "new-linux-device")?;
csr.set_subject_name(&name.build())?;
csr.set_pubkey(&key)?;
csr.sign(&key, MessageDigest::sha256())?;
let token_path = env::var("RMM_ENROLLMENT_TOKEN_FILE")
.context("enrollment token file required for first registration")?;
let token = fs::read_to_string(token_path)?.trim().to_owned();
ensure!(token.len() == 64, "invalid enrollment token format");
let client = reqwest::Client::builder()
.https_only(true)
.tls_built_in_root_certs(false)
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
.resolve("rmm-dev.internal", address)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.timeout(Duration::from_secs(15))
.build()?;
let result = client
.post(format!(
"https://rmm-dev.internal:{}/agent/v1/enroll",
address.port()
))
.json(&EnrollmentRequest {
token,
csr_pem: String::from_utf8(csr.build().to_pem()?)?,
})
.send()
.await?;
ensure!(
result.status().is_success(),
"enrollment rejected (response details suppressed)"
);
let response: EnrollmentResponse = result.json().await?;
let certificate = openssl::x509::X509::from_pem(response.certificate_pem.as_bytes())?;
ensure!(
certificate.public_key()?.public_eq(&key),
"enrollment certificate key mismatch"
);
Uuid::parse_str(&response.agent_id)?;
storage::create_secret(&dir.join("device.pem"), response.certificate_pem.as_bytes())?;
storage::create_secret(&identity, &serde_json::to_vec(&response)?)?;
Ok(response.agent_id)
}
#[tokio::main]
async fn main() -> Result<()> {
let args: Vec<_> = env::args_os().skip(1).collect();
if args == [std::ffi::OsString::from("--help")] {
println!(
"rmm-agent [--help]\nEnvironment: RMM_AGENT_STATE, RMM_CA_FILE, RMM_CONNECT_ADDR (TCP only; TLS name rmm-dev.internal), RMM_TRANSPORT=auto|wss|https, RMM_ENROLLMENT_TOKEN_FILE (first enrollment), RMM_JOB_BINDING_FILE, RMM_JOB_SIGNING_KEY_BOOTSTRAP=1, RMM_HELPER_SOCKET, RMM_HELPER_POLICY_FILE, RMM_APT_HELPER_SOCKET, RMM_APT_HELPER_PROOF_FILE, RMM_LOG_UNITS_FILE, RMM_EXTENDED_INVENTORY=0|1 (default 1).\nDevelopment-only rotation test: RMM_AGENT_PROFILE=development and RMM_ROTATION_TEST_AFTER_SECONDS.\nNo argument or environment variable approves jobs or disables TLS/grant verification."
);
return Ok(());
}
ensure!(args.is_empty(), "UNKNOWN_AGENT_ARGUMENT");
let dir =
PathBuf::from(env::var("RMM_AGENT_STATE").unwrap_or_else(|_| "devstate/agent".into()));
storage::prepare_dir(&dir)?;
let _lock = storage::lock(&dir)?;
let address = env::var("RMM_CONNECT_ADDR")
.unwrap_or_else(|_| "127.0.0.1:18443".into())
.parse()?;
let ca = fs::read(env::var("RMM_CA_FILE").unwrap_or_else(|_| "devstate/pki/ca.pem".into()))?;
let id=enrollment(&dir,address,&ca).await.map_err(|_| anyhow::anyhow!("enrollment/identity initialization failed; check trusted CA, token validity and local state"))?;
let mut binding = env::var("RMM_JOB_BINDING_FILE")
.ok()
.map(|path| {
let binding =
rmm_privileged_helper::read_binding(Path::new(&path), unsafe { libc::geteuid() })?;
let identity: EnrollmentResponse =
serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?;
ensure!(
binding.agent_id.to_string() == id
&& binding.asset_id.to_string() == identity.asset_id,
"BOOTSTRAP_IDENTITY_MISMATCH"
);
Ok::<_, anyhow::Error>(binding)
})
.transpose()?;
let certificate_dir = dir.join("certificates");
let certificates = certificate_rotation::RotationStore::open(
&certificate_dir,
&ca,
Uuid::parse_str(&id)?,
binding.as_ref().map_or(1, |b| b.enrollment_generation),
)?;
if !certificate_dir.join("current.json").try_exists()? {
certificates.initialize(
&fs::read(dir.join("device.pem"))?,
&fs::read(dir.join("device.key"))?,
Utc::now().timestamp(),
)?;
}
if env::var("RMM_JOB_SIGNING_KEY_BOOTSTRAP").as_deref() == Ok("1") {
bootstrap::provision_current(
&dir,
address,
&ca,
binding
.as_mut()
.context("bootstrap requires provisioned device binding")?,
&certificates.current(Utc::now().timestamp())?,
)
.await?;
}
let (completed_tx, completed_rx) = tokio::sync::mpsc::channel(1);
let (inventory_tx, inventory_rx) = tokio::sync::mpsc::channel(1);
let sync = binding
.as_ref()
.map(|b| reconcile::ClientSync::new(b.clone()));
let helpers = binding
.as_ref()
.map(helper_sync::Helpers::configured)
.transpose()?;
let mut rotation_policy = certificate_rotation::RotationPolicy::default();
if let Ok(after) = env::var("RMM_ROTATION_TEST_AFTER_SECONDS") {
ensure!(
env::var("RMM_AGENT_PROFILE").as_deref() == Ok("development"),
"ROTATION_TEST_REQUIRES_DEVELOPMENT_PROFILE"
);
rotation_policy.begin_after_seconds = after.parse()?;
ensure!(
rotation_policy.begin_after_seconds > 0,
"ROTATION_TEST_INTERVAL"
);
}
let mode =
wire_channel::Mode::parse(&env::var("RMM_TRANSPORT").unwrap_or_else(|_| "auto".into()))?;
let mut agent = Agent {
address,
ca,
id,
boot: rmm_agent_linux::boot_id()?,
instance: Uuid::new_v4().to_string(),
db: storage::open(&dir)?,
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding,
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() })?,
completed_tx,
completed_rx,
inventory_tx,
inventory_rx,
inventory_inflight: false,
sync,
sync_clock: reconcile::NativeClock::default(),
helpers,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode,
certificates,
rotation_policy,
};
let mut backoff = 1_u64;
loop {
let started = Instant::now();
if let Err(error) = agent.session().await {
report_closed_gate(&error);
}
agent.close_gate();
eprintln!("channel unavailable; retrying with jitter (no jobs can execute)");
if started.elapsed() > Duration::from_secs(60) {
backoff = 1;
}
tokio::time::sleep(https_channel::retry_delay(backoff as u32)).await;
backoff = (backoff + 1).min(6);
}
}
File diff suppressed because it is too large Load Diff
+400
View File
@@ -0,0 +1,400 @@
//! Actual dispatcher/transport tests with a local mTLS fixture, no real server
//! approval, root helper, package/service/reboot or target VM effects.
use super::*;
use openssl::{
asn1::Asn1Time,
bn::BigNum,
ssl::{SslAcceptor, SslMethod, SslVerifyMode},
x509::{
X509, X509NameBuilder,
extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName},
},
};
use rmm_domain::reconciliation::ReconcileDecisionPayload;
use std::{
io::{Read, Write},
net::TcpListener,
};
struct Clock(bool);
#[test]
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
let gate = EffectFence::default();
assert!(!gate.allows(1, false));
gate.deadline
.store(Utc::now().timestamp_millis() + 10000, Ordering::SeqCst);
gate.generation.store(1, Ordering::SeqCst);
assert!(gate.allows(1, false));
assert!(!gate.allows(2, false));
assert!(!gate.allows(1, true));
gate.root.store(true, Ordering::SeqCst);
assert!(gate.allows(1, true));
gate.deadline.store(0, Ordering::SeqCst);
assert!(!gate.allows(1, false));
gate.close();
assert!(!gate.allows(1, true));
}
impl reconcile::SyncClock for Clock {
fn now(&self) -> Result<chrono::DateTime<Utc>> {
ensure!(self.0, "CLOCK_UNTRUSTED");
Ok(Utc::now())
}
}
fn certificate(
name: &str,
issuer: Option<(&X509, &PKey<openssl::pkey::Private>)>,
server: bool,
) -> (X509, PKey<openssl::pkey::Private>) {
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap();
let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap();
let mut subject = X509NameBuilder::new().unwrap();
subject.append_entry_by_text("CN", name).unwrap();
let subject = subject.build();
let mut b = X509::builder().unwrap();
b.set_version(2).unwrap();
b.set_serial_number(
&BigNum::from_u32(rand::random_range(1..u32::MAX))
.unwrap()
.to_asn1_integer()
.unwrap(),
)
.unwrap();
b.set_subject_name(&subject).unwrap();
b.set_issuer_name(issuer.map_or(subject.as_ref(), |(ca, _)| ca.subject_name()))
.unwrap();
b.set_pubkey(&key).unwrap();
b.set_not_before(&Asn1Time::days_from_now(0).unwrap())
.unwrap();
b.set_not_after(&Asn1Time::days_from_now(1).unwrap())
.unwrap();
if issuer.is_none() {
b.append_extension(BasicConstraints::new().critical().ca().build().unwrap())
.unwrap();
b.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap())
.unwrap();
} else {
b.append_extension(BasicConstraints::new().build().unwrap())
.unwrap();
let mut u = ExtendedKeyUsage::new();
if server {
u.server_auth();
} else {
u.client_auth();
}
b.append_extension(u.build().unwrap()).unwrap();
if server {
let context = b.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None);
let san = SubjectAlternativeName::new()
.dns(name)
.build(&context)
.unwrap();
b.append_extension(san).unwrap();
}
}
b.sign(issuer.map_or(&key, |(_, k)| k), MessageDigest::sha256())
.unwrap();
(b.build(), key)
}
fn reply(input: &Envelope, generation: u64, payload: Payload) -> Envelope {
Envelope {
message_id: Uuid::new_v4().to_string(),
connection_generation: generation,
payload: Some(payload),
..input.clone()
}
}
struct Server {
binding: rmm_privileged_helper::Binding,
key: PKey<openssl::pkey::Private>,
generation: u64,
previous: Option<Uuid>,
requests: usize,
heartbeats: usize,
forged: bool,
}
impl Server {
fn response(&mut self, e: &Envelope) -> Envelope {
match e.payload.as_ref().unwrap() {
Payload::ClientHello(h) => {
assert!(h.capabilities.iter().any(|c| c == "rmm.sync.v1"));
reply(
e,
self.generation,
Payload::ServerHello(ServerHello {
connection_generation: self.generation,
max_message_bytes: rmm_domain::MAX_MESSAGE as u32,
}),
)
}
Payload::SyncRequest(r) => {
assert!(r.journal_complete && r.attempts.is_empty());
if let Some(previous) = self.previous {
assert_eq!(r.previous_decision_id, previous.to_string());
}
let now = Utc::now();
let d = ReconcileDecisionPayload {
domain: "rmm.reconciliation-decision".into(),
schema_version: 1,
decision_id: Uuid::new_v4(),
request_id: Uuid::parse_str(&r.request_id).unwrap(),
request_hash: rmm_domain::jobs::hash(&r.encode_to_vec()),
tenant_id: self.binding.tenant_id,
asset_id: self.binding.asset_id,
agent_id: self.binding.agent_id,
enrollment_generation: 1,
assignment_version: 1,
connection_generation: self.generation,
boot_id: Uuid::parse_str(&e.boot_id).unwrap(),
journal_id: Uuid::parse_str(&r.journal_id).unwrap(),
checkpoint: r.checkpoint,
helper_journal_id: None,
helper_checkpoint: 0,
issued_at: now,
valid_until: now + chrono::Duration::seconds(60),
ready: true,
helper_ready: false,
reason: "CONSISTENT".into(),
attempts: vec![],
acknowledged_message_ids: vec![],
key_id: rmm_domain::jobs::hash(&self.key.raw_public_key().unwrap()),
};
let bytes = rmm_domain::jobs::encode(&d).unwrap();
let mut signature = openssl::sign::Signer::new_without_digest(&self.key)
.unwrap()
.sign_oneshot_to_vec(&bytes)
.unwrap();
if self.forged {
signature[0] ^= 1;
} else {
self.previous = Some(d.decision_id);
}
self.requests += 1;
reply(
e,
self.generation,
Payload::SyncResponse(SyncResponse {
decision_bytes: bytes,
signature,
}),
)
}
Payload::Heartbeat(_) => {
self.heartbeats += 1;
reply(
e,
self.generation,
Payload::Ack(Ack {
message_id: e.message_id.clone(),
}),
)
}
Payload::InventorySnapshot(_) => reply(
e,
self.generation,
Payload::Ack(Ack {
message_id: e.message_id.clone(),
}),
),
_ => panic!("fixture received unexpected execution authority"),
}
}
}
fn http_body(stream: &mut impl Read) -> Vec<u8> {
let mut head = vec![];
while !head.ends_with(b"\r\n\r\n") {
let mut b = [0; 1];
stream.read_exact(&mut b).unwrap();
head.push(b[0]);
assert!(head.len() < 8192);
}
let text = String::from_utf8(head).unwrap();
let length: usize = text
.lines()
.filter_map(|line| line.split_once(':'))
.find(|(k, _)| k.eq_ignore_ascii_case("content-length"))
.unwrap()
.1
.trim()
.parse()
.unwrap();
assert!(length <= rmm_domain::MAX_MESSAGE);
let mut bytes = vec![0; length];
stream.read_exact(&mut bytes).unwrap();
bytes
}
async fn run(mode: wire_channel::Mode, trusted: bool, forged: bool, rounds: u64) {
let dir = PathBuf::from("/tmp/opencode").join(format!("runtime-{}", Uuid::new_v4()));
storage::prepare_dir(&dir).unwrap();
let id = Uuid::new_v4();
let (ca, ca_key) = certificate("runtime CA", None, false);
let (server, server_key) = certificate("rmm-dev.internal", Some((&ca, &ca_key)), true);
let (device, device_key) = certificate(&id.to_string(), Some((&ca, &ca_key)), false);
let key = PKey::generate_ed25519().unwrap();
let public = key.raw_public_key().unwrap();
let binding = rmm_privileged_helper::Binding {
tenant_id: Uuid::new_v4(),
asset_id: Uuid::new_v4(),
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
};
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap();
acceptor.set_certificate(&server).unwrap();
acceptor.set_private_key(&server_key).unwrap();
acceptor.cert_store_mut().add_cert(ca.clone()).unwrap();
acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT);
acceptor.set_session_id_context(b"rmm-runtime").unwrap();
let acceptor = acceptor.build();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server_binding = binding.clone();
let worker = std::thread::spawn(move || {
let mut state = Server {
binding: server_binding,
key,
generation: 1,
previous: None,
requests: 0,
heartbeats: 0,
forged,
};
for generation in 1..=rounds {
state.generation = generation;
loop {
let (socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(8)))
.unwrap();
let mut tls = acceptor.accept(socket).unwrap();
assert!(tls.ssl().peer_certificate().is_some());
if mode == wire_channel::Mode::Wss {
let mut ws = tokio_tungstenite::tungstenite::accept(tls).unwrap();
loop {
let message = ws.read().unwrap();
let tokio_tungstenite::tungstenite::Message::Binary(bytes) = message else {
continue;
};
let e = Envelope::decode(bytes).unwrap();
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
state.heartbeats += 1;
let _ = ws.close(None);
break;
}
if matches!(e.payload, Some(Payload::ClientHello(_))) {
ws.send(tokio_tungstenite::tungstenite::Message::Ping(
vec![1].into(),
))
.unwrap();
}
ws.send(tokio_tungstenite::tungstenite::Message::Binary(
state.response(&e).encode_to_vec().into(),
))
.unwrap();
}
break;
} else {
let batch = MessageBatch::decode(http_body(&mut tls).as_slice()).unwrap();
let e = &batch.messages[0];
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
state.heartbeats += 1;
tls.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
break;
}
let response = MessageBatch {
protocol_major: 1,
messages: vec![state.response(e)],
}
.encode_to_vec();
let header = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
response.len()
);
tls.write_all(header.as_bytes()).unwrap();
tls.write_all(&response).unwrap();
}
}
}
(state.requests, state.heartbeats)
});
let certificates = certificate_rotation::RotationStore::open(
&dir.join("certificates"),
&ca.to_pem().unwrap(),
id,
1,
)
.unwrap();
certificates
.initialize(
&device.to_pem().unwrap(),
&device_key.private_key_to_pem_pkcs8().unwrap(),
Utc::now().timestamp(),
)
.unwrap();
let (tx, rx) = tokio::sync::mpsc::channel(1);
let (itx, irx) = tokio::sync::mpsc::channel(1);
let mut agent = Agent {
address,
ca: ca.to_pem().unwrap(),
id: id.to_string(),
boot: rmm_agent_linux::boot_id().unwrap(),
instance: Uuid::new_v4().to_string(),
db: storage::open(&dir).unwrap(),
started: Instant::now(),
clock: rmm_privileged_helper::TrustedClock::new(),
binding: Some(binding.clone()),
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(),
completed_tx: tx,
completed_rx: rx,
inventory_tx: itx,
inventory_rx: irx,
inventory_inflight: false,
sync: Some(reconcile::ClientSync::new(binding)),
sync_clock: Clock(trusted),
helpers: None,
prepared: None,
fence: Arc::new(EffectFence::default()),
mode,
certificates,
rotation_policy: certificate_rotation::RotationPolicy::default(),
};
for _ in 0..rounds {
assert!(
tokio::time::timeout(Duration::from_secs(10), agent.session())
.await
.unwrap()
.is_err()
);
assert!(!agent.fence.allows(1, false));
}
let (requests, heartbeats) = worker.join().unwrap();
assert_eq!(heartbeats, rounds as usize);
if trusted {
assert_eq!(requests, rounds as usize);
let decided: Option<String> = agent
.db
.query_row("SELECT last_decision FROM reconcile_meta", [], |r| r.get(0))
.unwrap();
assert_eq!(decided.is_some(), !forged);
} else {
assert_eq!(requests, 0);
}
drop(agent);
fs::remove_dir_all(dir).unwrap();
}
#[tokio::test]
async fn actual_https_dispatcher_commits_signed_sync_and_clean_reconnect() {
run(wire_channel::Mode::Https, true, false, 2).await;
}
#[tokio::test]
async fn actual_wss_dispatcher_handles_ping_and_clean_reconnect() {
run(wire_channel::Mode::Wss, true, false, 2).await;
}
#[tokio::test]
async fn forged_sync_never_opens_dispatcher_gate() {
run(wire_channel::Mode::Https, true, true, 1).await;
}
#[tokio::test]
async fn untrusted_clock_keeps_telemetry_transport_but_never_requests_start() {
run(wire_channel::Mode::Https, false, false, 1).await;
}
+167
View File
@@ -0,0 +1,167 @@
use anyhow::{Result, ensure};
use fs2::FileExt;
use prost::Message;
use rmm_protocol::Envelope;
use std::{
fs::{self, File, OpenOptions},
io::Write,
os::unix::fs::{OpenOptionsExt, PermissionsExt},
path::Path,
};
pub fn prepare_dir(dir: &Path) -> Result<()> {
if !dir.exists() {
fs::create_dir_all(dir)?;
fs::set_permissions(dir, fs::Permissions::from_mode(0o700))?;
}
let meta = fs::symlink_metadata(dir)?;
ensure!(
meta.is_dir() && !meta.file_type().is_symlink() && meta.permissions().mode() & 0o077 == 0,
"agent state must be a private directory (0700)"
);
// SAFETY: geteuid has no preconditions.
rmm_privileged_helper::private_path(dir, unsafe { libc::geteuid() }, true)?;
Ok(())
}
pub fn create_secret(path: &Path, bytes: &[u8]) -> Result<()> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)?;
file.write_all(bytes)?;
file.sync_all()?;
File::open(
path.parent()
.ok_or_else(|| anyhow::anyhow!("missing parent"))?,
)?
.sync_all()?;
Ok(())
}
pub fn lock(dir: &Path) -> Result<File> {
let f = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(dir.join("agent.lock"))?;
// SAFETY: geteuid has no preconditions.
rmm_privileged_helper::private_path(
&dir.join("agent.lock"),
unsafe { libc::geteuid() },
false,
)?;
f.try_lock_exclusive()?;
Ok(f)
}
pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
let path = dir.join("journal.sqlite");
if !path.exists() {
create_secret(&path, &[])?;
}
// Refuse unsafe preexisting journals rather than opening a symlink.
let uid = unsafe { libc::geteuid() };
rmm_privileged_helper::private_path(&path, uid, false)?;
let db = rusqlite::Connection::open(path)?;
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
db.execute_batch("PRAGMA max_page_count=16384;
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
CREATE TABLE IF NOT EXISTS job_events(id TEXT PRIMARY KEY, payload BLOB NOT NULL CHECK(length(payload)<=1048576));
CREATE TABLE IF NOT EXISTS safety(key TEXT PRIMARY KEY,value TEXT NOT NULL);")?;
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
ensure!(check == "ok", "JOURNAL_INTEGRITY");
crate::reconcile::initialize(&db)?;
File::open(dir)?.sync_all()?;
Ok(db)
}
pub fn next_sequence(db: &mut rusqlite::Connection) -> Result<u64> {
let tx = db.transaction()?;
let seq: i64 = tx.query_row(
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
[],
|r| r.get(0),
)?;
tx.commit()?;
Ok(u64::try_from(seq)?)
}
/// Do not let periodic metadata queue behind unacknowledged durable traffic.
/// Results/control receipts retain priority; no old inventory is deleted.
pub fn passive_collection_ready(db: &rusqlite::Connection) -> Result<bool> {
Ok(
!db.query_row("SELECT EXISTS(SELECT 1 FROM job_events)", [], |row| {
row.get::<_, bool>(0)
})?,
)
}
#[cfg(test)]
pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
db.execute("INSERT INTO pending VALUES(1,?1)", [e.encode_to_vec()])?;
Ok(())
}
#[cfg(test)]
mod passive_pressure_tests {
#[test]
fn pending_durable_traffic_backpressures_collection_without_deletion() {
let db = rusqlite::Connection::open_in_memory().unwrap();
db.execute_batch("CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB NOT NULL)")
.unwrap();
assert!(super::passive_collection_ready(&db).unwrap());
db.execute("INSERT INTO job_events VALUES('unacked',x'01')", [])
.unwrap();
assert!(!super::passive_collection_ready(&db).unwrap());
assert_eq!(
db.query_row("SELECT count(*) FROM job_events", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
db.execute("DELETE FROM job_events WHERE id='unacked'", [])
.unwrap();
assert!(super::passive_collection_ready(&db).unwrap());
}
}
pub fn pending(db: &rusqlite::Connection) -> Result<Option<Envelope>> {
use rusqlite::OptionalExtension;
let data: Option<Vec<u8>> = db
.query_row("SELECT payload FROM pending WHERE id=1", [], |r| r.get(0))
.optional()?;
data.map(|d| Ok(Envelope::decode(d.as_slice())?))
.transpose()
}
pub fn clear_pending(db: &rusqlite::Connection) -> Result<()> {
db.execute("DELETE FROM pending WHERE id=1", [])?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn durable_sequence_pending_and_exclusive_lock() {
let path = std::env::temp_dir().join(format!("rmm-{}", uuid::Uuid::new_v4()));
prepare_dir(&path).unwrap();
let lockfile = lock(&path).unwrap();
assert!(lock(&path).is_err());
let mut db = open(&path).unwrap();
assert_eq!(next_sequence(&mut db).unwrap(), 1);
let e = Envelope {
sequence: 2,
..Default::default()
};
store_pending(&db, &e).unwrap();
drop(db);
let mut db = open(&path).unwrap();
assert_eq!(next_sequence(&mut db).unwrap(), 2);
assert_eq!(pending(&db).unwrap().unwrap(), e);
clear_pending(&db).unwrap();
assert!(pending(&db).unwrap().is_none());
drop(db);
drop(lockfile);
fs::remove_dir_all(path).unwrap();
}
}
+73
View File
@@ -0,0 +1,73 @@
use anyhow::{Context, Result, ensure};
use futures_util::{SinkExt, StreamExt};
use prost::Message;
use rmm_protocol::Envelope;
use rustls::{ClientConfig, RootCertStore};
use std::{io::BufReader, net::SocketAddr, sync::Arc, time::Duration};
use tokio::{net::TcpStream, time::timeout};
use tokio_tungstenite::{
Connector, MaybeTlsStream, WebSocketStream,
tungstenite::{Message as WsMessage, protocol::WebSocketConfig},
};
pub type Channel = WebSocketStream<MaybeTlsStream<TcpStream>>;
pub async fn connect(
address: SocketAddr,
ca: &[u8],
certificate: &[u8],
private_key: &[u8],
) -> Result<Channel> {
ensure!(
address.port() != 0 && !address.ip().is_unspecified() && !address.ip().is_multicast(),
"INVALID_CONNECT_ADDRESS"
);
let mut roots = RootCertStore::empty();
for cert in rustls_pemfile::certs(&mut BufReader::new(ca)) {
roots.add(cert?)?;
}
let certs = rustls_pemfile::certs(&mut BufReader::new(certificate))
.collect::<std::io::Result<Vec<_>>>()?;
let key = rustls_pemfile::private_key(&mut BufReader::new(private_key))?
.context("device key missing")?;
let config = ClientConfig::builder()
.with_root_certificates(roots)
.with_client_auth_cert(certs, key)?;
let tcp = timeout(Duration::from_secs(10), TcpStream::connect(address)).await??;
let ws = WebSocketConfig::default()
.max_message_size(Some(rmm_domain::MAX_MESSAGE))
.max_frame_size(Some(rmm_domain::MAX_MESSAGE));
// URL controls TLS SNI and hostname validation; address overrides TCP only.
let (channel, _) = timeout(
Duration::from_secs(10),
tokio_tungstenite::client_async_tls_with_config(
format!("wss://rmm-dev.internal:{}/agent/v1/connect", address.port()),
tcp,
Some(ws),
Some(Connector::Rustls(Arc::new(config))),
),
)
.await??;
Ok(channel)
}
pub async fn send(ws: &mut Channel, e: &Envelope) -> Result<()> {
timeout(
Duration::from_secs(10),
ws.send(WsMessage::Binary(e.encode_to_vec().into())),
)
.await??;
Ok(())
}
pub async fn receive_live(ws: &mut Channel) -> Result<Envelope> {
loop {
match ws.next().await.context("channel closed")?? {
WsMessage::Binary(bytes) => {
ensure!(bytes.len() <= rmm_domain::MAX_MESSAGE, "message limit");
return Ok(Envelope::decode(bytes)?);
}
WsMessage::Ping(_) => ws.flush().await?,
WsMessage::Pong(_) => {}
WsMessage::Close(_) => anyhow::bail!("channel closed"),
_ => anyhow::bail!("binary envelope required"),
}
}
}
+239
View File
@@ -0,0 +1,239 @@
//! One bounded dispatcher-facing actor for WSS and HTTPS. Never reconnects or
//! retries a permit internally; the owning journal/sync state machine does that.
use crate::{
certificate_rotation::DeviceIdentity,
https_channel::{self, HttpsChannel, WssFailure},
transport,
};
use anyhow::{Context, Result, ensure};
use rmm_protocol::Envelope;
use std::{net::SocketAddr, time::Duration};
use tokio::sync::mpsc;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Mode {
Auto,
Wss,
Https,
}
impl Mode {
pub fn parse(s: &str) -> Result<Self> {
match s {
"auto" => Ok(Self::Auto),
"wss" => Ok(Self::Wss),
"https" => Ok(Self::Https),
_ => anyhow::bail!("RMM_TRANSPORT_INVALID"),
}
}
}
enum Channel {
Wss(Box<transport::Channel>),
Https(HttpsChannel),
}
enum Command {
Send(Envelope),
Poll(Envelope),
}
pub struct Handle {
outgoing: mpsc::Sender<Command>,
incoming: mpsc::Receiver<Result<Envelope>>,
task: tokio::task::JoinHandle<()>,
https: bool,
}
impl Drop for Handle {
fn drop(&mut self) {
self.task.abort();
}
}
impl Handle {
pub async fn send(&self, envelope: &Envelope) -> Result<()> {
tokio::time::timeout(
Duration::from_secs(2),
self.outgoing.send(Command::Send(envelope.clone())),
)
.await??;
Ok(())
}
pub fn uses_https(&self) -> bool {
self.https
}
pub async fn poll(&self, heartbeat: Envelope) -> Result<()> {
ensure!(self.https, "POLL_REQUIRES_HTTPS");
tokio::time::timeout(
Duration::from_secs(2),
self.outgoing.send(Command::Poll(heartbeat)),
)
.await??;
Ok(())
}
pub async fn receive(&mut self) -> Result<Envelope> {
self.incoming.recv().await.context("CHANNEL_CLOSED")?
}
}
fn classify(error: &anyhow::Error) -> WssFailure {
use tokio_tungstenite::tungstenite::Error;
for cause in error.chain() {
if let Some(error) = cause.downcast_ref::<Error>() {
return match error {
Error::Tls(_) => WssFailure::TlsOrIdentity,
Error::Http(response) => match response.status().as_u16() {
401 | 403 => WssFailure::Authentication,
400 | 404 | 405 | 426 | 501 | 502 | 503 => WssFailure::UpgradeBlocked,
_ => WssFailure::Other,
},
_ => WssFailure::Other,
};
}
}
WssFailure::Other
}
pub async fn connect(
mode: Mode,
address: SocketAddr,
ca: &[u8],
identity: &DeviceIdentity,
hello: Envelope,
) -> Result<Handle> {
let channel = if mode == Mode::Https {
None
} else {
match transport::connect(
address,
ca,
&identity.certificate_pem,
&identity.private_key_pem,
)
.await
{
Ok(mut ws) => {
transport::send(&mut ws, &hello).await?;
Some(Channel::Wss(Box::new(ws)))
}
Err(error) if mode == Mode::Auto && classify(&error) == WssFailure::UpgradeBlocked => {
match transport::connect(
address,
ca,
&identity.certificate_pem,
&identity.private_key_pem,
)
.await
{
Ok(mut ws) => {
transport::send(&mut ws, &hello).await?;
Some(Channel::Wss(Box::new(ws)))
}
Err(second) => {
ensure!(
https_channel::allow_https_fallback(classify(&second), 2),
"AUTO_FALLBACK_NOT_AUTHORIZED"
);
None
}
}
}
Err(error) => return Err(error),
}
};
let channel = if let Some(channel) = channel {
channel
} else {
let client = https_channel::mtls_client(
address,
ca,
&identity.certificate_pem,
&identity.private_key_pem,
)?;
let channel = HttpsChannel::connect(client, address, hello).await?;
ensure!(channel.generation() > 0, "HTTPS_GENERATION");
Channel::Https(channel)
};
let https = matches!(channel, Channel::Https(_));
let (outgoing, commands) = mpsc::channel(16);
let (events, incoming) = mpsc::channel(32);
let task = tokio::spawn(async move {
let result = run(channel, commands, &events).await;
if let Err(error) = result {
let _ = events.send(Err(error)).await;
}
});
Ok(Handle {
outgoing,
incoming,
task,
https,
})
}
async fn run(
mut channel: Channel,
mut commands: mpsc::Receiver<Command>,
events: &mpsc::Sender<Result<Envelope>>,
) -> Result<()> {
match &mut channel {
Channel::Wss(ws) => loop {
tokio::select! {
incoming=transport::receive_live(ws)=>{events.send(Ok(incoming?)).await?;},
command=commands.recv()=>{let Some(Command::Send(command))=command else {return Ok(())};transport::send(ws,&command).await?;},
}
},
Channel::Https(http) => {
let mut pending = None;
loop {
while let Some(envelope) = http.receive() {
events.send(Ok(envelope)).await?;
}
let command = match pending.take() {
Some(command) => command,
None => match commands.recv().await {
Some(command) => command,
None => {
http.fence();
return Ok(());
}
},
};
if let Command::Poll(command) = command {
// Dropping a long poll loses no effect/start authority; the
// next durable send or fresh heartbeat drives redelivery.
tokio::select! {
result=http.poll(command,25)=>result?,
next=commands.recv()=>match next{Some(next)=>pending=Some(next),None=>{http.fence();return Ok(())}},
}
} else if let Command::Send(command) = command {
http.send(&command).await?;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn auto_fallback_classifies_upgrade_refusal_but_never_authentication() {
for (status, expected) in [
(426, WssFailure::UpgradeBlocked),
(401, WssFailure::Authentication),
(403, WssFailure::Authentication),
(500, WssFailure::Other),
] {
let response = tokio_tungstenite::tungstenite::http::Response::builder()
.status(status)
.body(None::<Vec<u8>>)
.unwrap();
let error = anyhow::Error::new(tokio_tungstenite::tungstenite::Error::Http(Box::new(
response,
)));
assert_eq!(classify(&error), expected);
}
assert_eq!(
classify(&anyhow::anyhow!("ambiguous reset")),
WssFailure::Other
);
}
#[test]
fn explicit_transport_names_cannot_enable_verification_bypasses() {
assert_eq!(Mode::parse("https").unwrap(), Mode::Https);
assert_eq!(Mode::parse("auto").unwrap(), Mode::Auto);
assert!(Mode::parse("https-insecure").is_err());
}
}