feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "rmm-agent"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
futures-util.workspace = true
|
||||
openssl.workspace = true
|
||||
prost.workspace = true
|
||||
rand.workspace = true
|
||||
reqwest.workspace = true
|
||||
rustls.workspace = true
|
||||
rustls-pemfile.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde.workspace = true
|
||||
sha2.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
uuid.workspace = true
|
||||
rusqlite = { version = "0.32.1", features = ["bundled"] }
|
||||
fs2 = "0.4.3"
|
||||
rmm-domain = { path = "../domain" }
|
||||
rmm-protocol = { path = "../protocol" }
|
||||
rmm-agent-linux = { path = "../agent-linux" }
|
||||
rmm-privileged-helper = { path = "../privileged-helper" }
|
||||
libc = "0.2.177"
|
||||
@@ -0,0 +1,317 @@
|
||||
//! Dedicated APT route: no fallback to generic helper, shell or SSH. Readiness
|
||||
//! is nonce/root-peer/identity/profile/stack pinned before advertising or use.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use rmm_domain::jobs::JobPayload;
|
||||
use rmm_privileged_helper::{Binding, Outcome, Request, apt as provider, readiness};
|
||||
use std::{
|
||||
ffi::OsString,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
pub fn socket_for(
|
||||
job: &JobPayload,
|
||||
general: Option<OsString>,
|
||||
apt: Option<OsString>,
|
||||
) -> Result<PathBuf> {
|
||||
select_socket(provider::is_apt_action(job), general, apt)
|
||||
}
|
||||
fn select_socket(
|
||||
is_apt: bool,
|
||||
general: Option<OsString>,
|
||||
apt: Option<OsString>,
|
||||
) -> Result<PathBuf> {
|
||||
if is_apt {
|
||||
let socket = PathBuf::from(apt.context("APT_HELPER_SOCKET_NOT_CONFIGURED")?);
|
||||
ensure!(
|
||||
socket == Path::new(readiness::SOCKET),
|
||||
"APT_HELPER_SOCKET_PROFILE"
|
||||
);
|
||||
Ok(socket)
|
||||
} else {
|
||||
Ok(PathBuf::from(
|
||||
general.context("GENERAL_HELPER_SOCKET_NOT_CONFIGURED")?,
|
||||
))
|
||||
}
|
||||
}
|
||||
fn status(binding: &Binding) -> Result<readiness::Response> {
|
||||
let path =
|
||||
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_HELPER_PIN_NOT_CONFIGURED")?;
|
||||
let pin = readiness::Pin::read(Path::new(&path), binding)?;
|
||||
let socket =
|
||||
std::env::var_os("RMM_APT_HELPER_SOCKET").context("APT_HELPER_SOCKET_NOT_CONFIGURED")?;
|
||||
readiness::probe(Path::new(&socket), binding, &pin)
|
||||
}
|
||||
pub fn capabilities_from(response: &readiness::Response) -> Vec<String> {
|
||||
if response.profile != "apt_only" {
|
||||
return vec![];
|
||||
}
|
||||
let mut result = vec![];
|
||||
if response.fixture_acceptance_ready && !response.bootstrap_recipes.is_empty() {
|
||||
result.push("apt_native_acceptance".into());
|
||||
}
|
||||
if !response.accepted_recipes.is_empty() {
|
||||
result.push("ensure_package".into());
|
||||
}
|
||||
if response
|
||||
.accepted_recipes
|
||||
.iter()
|
||||
.any(|r| r == provider::UPDATE_RECIPE)
|
||||
{
|
||||
result.push("apply_updates".into());
|
||||
}
|
||||
result
|
||||
}
|
||||
pub fn capabilities(binding: &Binding) -> Vec<String> {
|
||||
status(binding)
|
||||
.map(|s| {
|
||||
let mut caps = capabilities_from(&s);
|
||||
if !caps.is_empty() {
|
||||
let roots: Vec<String> = caps.iter().map(|c| format!("root:{c}")).collect();
|
||||
caps.extend(roots);
|
||||
caps.extend([
|
||||
"helper.root.v1".into(),
|
||||
format!("helper.policy.sha256:{}", s.provider_stack_sha256),
|
||||
"apt.manifest.v2".into(),
|
||||
]);
|
||||
if s.fixture_acceptance_ready && !s.bootstrap_recipes.is_empty() {
|
||||
caps.push("apt.native-acceptance.v1".into());
|
||||
}
|
||||
if !s.accepted_recipes.is_empty() {
|
||||
caps.push("apt.native-accepted.v1".into());
|
||||
}
|
||||
}
|
||||
caps
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Called only by the periodic blocking inventory worker, never heartbeat or a
|
||||
/// caller-selected live query. Failure retains honest unknown/not_configured.
|
||||
pub fn planning_observation(binding: Option<&Binding>) -> serde_json::Value {
|
||||
let result = (|| -> Result<readiness::PlanningResult> {
|
||||
let binding = binding.context("APT_PLANNING_NOT_CONFIGURED")?;
|
||||
let path =
|
||||
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_PLANNING_NOT_CONFIGURED")?;
|
||||
let pin = readiness::Pin::read(Path::new(&path), binding)?;
|
||||
let socket =
|
||||
std::env::var_os("RMM_APT_HELPER_SOCKET").context("APT_PLANNING_NOT_CONFIGURED")?;
|
||||
readiness::planning(Path::new(&socket), binding, &pin)?
|
||||
.planning
|
||||
.context("APT_PLANNING_UNAVAILABLE")
|
||||
})();
|
||||
match result {
|
||||
Ok(planning) => {
|
||||
serde_json::json!({"scope":"linux.apt.planning", "source_id":"root-helper:apt_planning_v1",
|
||||
"observed_at":planning.observed_at, "quality":"valid", "completeness":"complete", "truncated":false, "data":planning})
|
||||
}
|
||||
Err(_) => {
|
||||
serde_json::json!({"scope":"linux.apt.planning", "source_id":"root-helper:apt_planning_v1",
|
||||
"observed_at":chrono::Utc::now(), "quality":"unknown", "completeness":"unknown", "truncated":false,
|
||||
"error_code":if binding.is_none() || std::env::var_os("RMM_APT_HELPER_SOCKET").is_none(){"APT_PLANNING_NOT_CONFIGURED"}else{"APT_PLANNING_UNAVAILABLE"}, "data":null})
|
||||
}
|
||||
}
|
||||
}
|
||||
fn recipe_ready(job: &JobPayload, response: &readiness::Response) -> Result<()> {
|
||||
ensure!(
|
||||
job.execution_identity == "root" && provider::is_apt_action(job),
|
||||
"APT_TYPED_ROOT_ACTION_REQUIRED"
|
||||
);
|
||||
let action = serde_json::to_value(&job.action)?;
|
||||
let manifest: provider::AptExecutionManifest = serde_json::from_value(
|
||||
action
|
||||
.get("apt_manifest")
|
||||
.context("APT_MANIFEST_NOT_CONFIGURED")?
|
||||
.clone(),
|
||||
)?;
|
||||
manifest.resolved_apt_transaction.validate_fixture()?;
|
||||
let recipe = &manifest.resolved_apt_transaction.recipe_id;
|
||||
if job.action.name() == "apt_native_acceptance" {
|
||||
ensure!(
|
||||
manifest.action == "apt-native-acceptance"
|
||||
&& response.fixture_acceptance_ready
|
||||
&& response.bootstrap_recipes.contains(recipe),
|
||||
"APT_FIXTURE_READINESS_REQUIRED"
|
||||
);
|
||||
} else {
|
||||
ensure!(
|
||||
manifest.action == "apt-execute" && response.accepted_recipes.contains(recipe),
|
||||
"APT_NATIVE_ACCEPTANCE_REQUIRED"
|
||||
);
|
||||
ensure!(
|
||||
job.action.name() != "apply_updates" || recipe == provider::UPDATE_RECIPE,
|
||||
"APT_UPDATE_RECIPE_SCOPE"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn supported(binding: &Binding, job: &JobPayload) -> bool {
|
||||
status(binding).and_then(|s| recipe_ready(job, &s)).is_ok()
|
||||
}
|
||||
pub fn invoke(binding: &Binding, job: &JobPayload, request: &Request) -> Result<Outcome> {
|
||||
let socket = socket_for(
|
||||
job,
|
||||
std::env::var_os("RMM_HELPER_SOCKET"),
|
||||
std::env::var_os("RMM_APT_HELPER_SOCKET"),
|
||||
)?;
|
||||
if provider::is_apt_action(job) {
|
||||
recipe_ready(job, &status(binding)?)?;
|
||||
}
|
||||
rmm_privileged_helper::invoke(&socket, request, job.max_runtime_seconds)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn passive_planning_without_binding_is_honest_unknown_without_data() {
|
||||
let observation = planning_observation(None);
|
||||
assert_eq!(observation["scope"], "linux.apt.planning");
|
||||
assert_eq!(observation["quality"], "unknown");
|
||||
assert_eq!(observation["error_code"], "APT_PLANNING_NOT_CONFIGURED");
|
||||
assert!(observation["data"].is_null());
|
||||
assert!(observation.get("received_at").is_none());
|
||||
}
|
||||
#[test]
|
||||
fn typed_actions_require_their_own_fixture_readiness_or_accepted_recipe() {
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let now = chrono::Utc::now();
|
||||
let tx = serde_json::json!({"schema_version":1,"recipe_id":provider::INSTALL_RECIPE,"recipe_version":"1","platform":"ubuntu-24.04-amd64",
|
||||
"resolver_config_version":"rmm-apt-local-v1","no_reboot":true,"status_fingerprint":"a".repeat(64),"baseline_sha256":"b".repeat(64),
|
||||
"packages":[{"name":"rmm-mvp-test","architecture":"all","old_version":null,"new_version":"1.0","origin":"rmm-test",
|
||||
"sha256":rmm_domain::apt_transaction::INSTALL_SHA,"size":20672,
|
||||
"archive_path":format!("/var/lib/rmm-apt-transaction/archives/{}.deb",rmm_domain::apt_transaction::INSTALL_SHA)}]});
|
||||
let bytes = serde_json::to_vec(&tx).unwrap();
|
||||
let manifest = serde_json::json!({"type":"rmm.apt.helper-action","schema_version":2,"action":"apt-native-acceptance","execution_profile":"dev-test",
|
||||
"transaction_sha256":rmm_domain::jobs::hash(&bytes),"resolved_apt_transaction":tx,"transaction_bytes":bytes,
|
||||
"provider_sha256":"a".repeat(64),"hook_sha256":"a".repeat(64),"apt_config_sha256":"a".repeat(64),"runtime_sha256":"a".repeat(64),"test_result_id":"fixture-test"});
|
||||
let mut job:JobPayload=serde_json::from_value(serde_json::json!({"schema_version":1,"tenant_id":id,"asset_id":id,"agent_id":id,
|
||||
"enrollment_generation":1,"assignment_version":1,"job_id":id,"attempt_id":id,"execution_identity":"root",
|
||||
"not_before":now,"start_before":now+chrono::Duration::minutes(1),"max_runtime_seconds":300,"max_output_bytes":4096,
|
||||
"action":{"action":"apt_native_acceptance","recipe_id":provider::INSTALL_RECIPE,"recipe_version":"1","apt_manifest":manifest}})).unwrap();
|
||||
assert!(job.action.validate());
|
||||
let identity = readiness::Identity {
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
signing_key_id: "a".repeat(64),
|
||||
};
|
||||
let mut response = readiness::Response {
|
||||
schema_version: 1,
|
||||
challenge: id,
|
||||
identity,
|
||||
profile: "apt_only".into(),
|
||||
provider_stack_sha256: "a".repeat(64),
|
||||
fixture_acceptance_ready: true,
|
||||
bootstrap_recipes: vec![provider::INSTALL_RECIPE.into()],
|
||||
accepted_recipes: vec![],
|
||||
ledger_device: 1,
|
||||
ledger_inode: 2,
|
||||
lock_device: 1,
|
||||
lock_inode: 3,
|
||||
planning: None,
|
||||
};
|
||||
recipe_ready(&job, &response).unwrap();
|
||||
let mut normal = manifest;
|
||||
normal["action"] = serde_json::json!("apt-execute");
|
||||
job.action=serde_json::from_value(serde_json::json!({"action":"ensure_package","provider":"apt","package":"rmm-mvp-test","version":"1.0",
|
||||
"architecture":"all","recipe_version":"1","installation_context":"system","recipe_id":provider::INSTALL_RECIPE,"apt_manifest":normal})).unwrap();
|
||||
assert!(job.action.validate());
|
||||
assert!(recipe_ready(&job, &response).is_err());
|
||||
response.accepted_recipes = vec![provider::UPDATE_RECIPE.into()];
|
||||
assert!(recipe_ready(&job, &response).is_err());
|
||||
response.accepted_recipes = vec![provider::INSTALL_RECIPE.into()];
|
||||
recipe_ready(&job, &response).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn apt_route_cannot_fall_back_to_general_or_a_caller_path() {
|
||||
assert!(select_socket(true, Some("/run/general.sock".into()), None).is_err());
|
||||
assert!(
|
||||
select_socket(
|
||||
true,
|
||||
Some("/run/general.sock".into()),
|
||||
Some("/tmp/fake.sock".into())
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
select_socket(
|
||||
true,
|
||||
Some("/run/general.sock".into()),
|
||||
Some(readiness::SOCKET.into())
|
||||
)
|
||||
.unwrap(),
|
||||
Path::new(readiness::SOCKET)
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn bootstrap_readiness_never_claims_package_acceptance() {
|
||||
let id = readiness::Identity {
|
||||
tenant_id: uuid::Uuid::new_v4(),
|
||||
asset_id: uuid::Uuid::new_v4(),
|
||||
agent_id: uuid::Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
signing_key_id: "a".repeat(64),
|
||||
};
|
||||
let mut response = readiness::Response {
|
||||
schema_version: 1,
|
||||
challenge: uuid::Uuid::new_v4(),
|
||||
identity: id,
|
||||
profile: "apt_only".into(),
|
||||
provider_stack_sha256: "a".repeat(64),
|
||||
fixture_acceptance_ready: true,
|
||||
bootstrap_recipes: vec![provider::INSTALL_RECIPE.into()],
|
||||
accepted_recipes: vec![],
|
||||
ledger_device: 1,
|
||||
ledger_inode: 2,
|
||||
lock_device: 1,
|
||||
lock_inode: 3,
|
||||
planning: None,
|
||||
};
|
||||
assert_eq!(capabilities_from(&response), vec!["apt_native_acceptance"]);
|
||||
response.accepted_recipes = vec![provider::INSTALL_RECIPE.into()];
|
||||
assert!(!capabilities_from(&response).contains(&"apply_updates".into()));
|
||||
response
|
||||
.accepted_recipes
|
||||
.push(provider::UPDATE_RECIPE.into());
|
||||
assert!(capabilities_from(&response).contains(&"apply_updates".into()));
|
||||
}
|
||||
#[test]
|
||||
fn generic_script_never_uses_apt_socket() {
|
||||
let now = chrono::Utc::now();
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: id,
|
||||
attempt_id: id,
|
||||
action: rmm_domain::jobs::Action::RunScript {
|
||||
interpreter: "posix_sh".into(),
|
||||
script_utf8: "true".into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: None,
|
||||
},
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::minutes(1),
|
||||
max_runtime_seconds: 300,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
assert!(socket_for(&job, None, Some(readiness::SOCKET.into())).is_err());
|
||||
assert_eq!(
|
||||
socket_for(
|
||||
&job,
|
||||
Some("/run/general.sock".into()),
|
||||
Some(readiness::SOCKET.into())
|
||||
)
|
||||
.unwrap(),
|
||||
Path::new("/run/general.sock")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
//! mTLS-only, write-once trust bootstrap. Never replaces a pin from the helper
|
||||
//! or converts a job-supplied key into authority.
|
||||
use anyhow::{Result, ensure};
|
||||
use rmm_privileged_helper::{Binding, SigningKey};
|
||||
#[cfg(test)]
|
||||
use std::fs;
|
||||
use std::{net::SocketAddr, path::Path, time::Duration};
|
||||
|
||||
#[cfg(test)]
|
||||
fn client(dir: &Path, address: SocketAddr, ca: &[u8]) -> Result<reqwest::Client> {
|
||||
let mut identity = fs::read(dir.join("device.pem"))?;
|
||||
identity.extend_from_slice(&fs::read(dir.join("device.key"))?);
|
||||
Ok(reqwest::Client::builder()
|
||||
.https_only(true)
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
||||
.identity(reqwest::Identity::from_pem(&identity)?)
|
||||
.resolve("rmm-dev.internal", address)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(15))
|
||||
.build()?)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn provision(
|
||||
dir: &Path,
|
||||
address: SocketAddr,
|
||||
ca: &[u8],
|
||||
binding: &mut Binding,
|
||||
) -> Result<()> {
|
||||
download(dir, address, binding, client(dir, address, ca)?).await
|
||||
}
|
||||
pub async fn provision_current(
|
||||
dir: &Path,
|
||||
address: SocketAddr,
|
||||
ca: &[u8],
|
||||
binding: &mut Binding,
|
||||
identity: &crate::certificate_rotation::DeviceIdentity,
|
||||
) -> Result<()> {
|
||||
let client = crate::https_channel::mtls_client(
|
||||
address,
|
||||
ca,
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)?;
|
||||
download(dir, address, binding, client).await
|
||||
}
|
||||
async fn download(
|
||||
dir: &Path,
|
||||
address: SocketAddr,
|
||||
binding: &mut Binding,
|
||||
client: reqwest::Client,
|
||||
) -> Result<()> {
|
||||
let mut response = client
|
||||
.get(format!(
|
||||
"https://rmm-dev.internal:{}/api/v1/job-signing-key",
|
||||
address.port()
|
||||
))
|
||||
.timeout(Duration::from_secs(15))
|
||||
.send()
|
||||
.await?;
|
||||
ensure!(
|
||||
response.status() == reqwest::StatusCode::OK,
|
||||
"SIGNING_KEY_AUTHENTICATED_ENDPOINT_REJECTED"
|
||||
);
|
||||
ensure!(
|
||||
response.content_length().is_none_or(|n| n <= 4096),
|
||||
"SIGNING_KEY_SIZE"
|
||||
);
|
||||
let mut bytes = vec![];
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
ensure!(bytes.len() + chunk.len() <= 4096, "SIGNING_KEY_SIZE");
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
let downloaded: SigningKey = serde_json::from_slice(&bytes)?;
|
||||
pin(dir, binding, &downloaded)
|
||||
}
|
||||
fn pin(dir: &Path, binding: &mut Binding, downloaded: &SigningKey) -> Result<()> {
|
||||
downloaded.validate()?;
|
||||
ensure!(
|
||||
binding.public_key_hex.is_empty() || binding.public_key_hex == downloaded.public_key_hex,
|
||||
"PROVISIONED_KEY_MISMATCH"
|
||||
);
|
||||
let path = dir.join("job-signing-key.json");
|
||||
if path.try_exists()? {
|
||||
let cached = SigningKey::read(&path, unsafe { libc::geteuid() })?;
|
||||
ensure!(
|
||||
cached == *downloaded,
|
||||
"SIGNING_KEY_ROTATION_REQUIRES_CONTROLLED_PROVISIONING"
|
||||
);
|
||||
} else {
|
||||
crate::storage::create_secret(&path, &serde_json::to_vec(downloaded)?)?;
|
||||
}
|
||||
binding.public_key_hex = downloaded.public_key_hex.clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
fn key() -> SigningKey {
|
||||
let public = openssl::pkey::PKey::generate_ed25519()
|
||||
.unwrap()
|
||||
.raw_public_key()
|
||||
.unwrap();
|
||||
SigningKey {
|
||||
key_id: rmm_domain::jobs::hash(&public),
|
||||
algorithm: "Ed25519".into(),
|
||||
schema_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn bootstrap_never_rotates_existing_or_provisioned_trust() {
|
||||
let path = std::env::temp_dir().join(format!("rmm-key-bootstrap-{}", uuid::Uuid::new_v4()));
|
||||
crate::storage::prepare_dir(&path).unwrap();
|
||||
let mut binding = Binding {
|
||||
tenant_id: uuid::Uuid::new_v4(),
|
||||
asset_id: uuid::Uuid::new_v4(),
|
||||
agent_id: uuid::Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: String::new(),
|
||||
};
|
||||
let first = key();
|
||||
pin(&path, &mut binding, &first).unwrap();
|
||||
pin(&path, &mut binding, &first).unwrap();
|
||||
assert!(pin(&path, &mut binding, &key()).is_err());
|
||||
binding.public_key_hex.clear();
|
||||
assert!(pin(&path, &mut binding, &key()).is_err());
|
||||
assert_eq!(
|
||||
SigningKey::read(&path.join("job-signing-key.json"), unsafe {
|
||||
libc::geteuid()
|
||||
})
|
||||
.unwrap(),
|
||||
first
|
||||
);
|
||||
let mut bad = first.clone();
|
||||
bad.algorithm = "RSA".into();
|
||||
assert!(bad.validate().is_err());
|
||||
bad = first.clone();
|
||||
bad.schema_version = 2;
|
||||
assert!(bad.validate().is_err());
|
||||
bad = first;
|
||||
bad.key_id = "wrong".into();
|
||||
assert!(bad.validate().is_err());
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
fn certificate(
|
||||
cn: &str,
|
||||
issuer: Option<(
|
||||
&openssl::x509::X509,
|
||||
&openssl::pkey::PKey<openssl::pkey::Private>,
|
||||
)>,
|
||||
server: bool,
|
||||
) -> (
|
||||
openssl::x509::X509,
|
||||
openssl::pkey::PKey<openssl::pkey::Private>,
|
||||
) {
|
||||
use openssl::{
|
||||
asn1::Asn1Time,
|
||||
bn::BigNum,
|
||||
ec::{EcGroup, EcKey},
|
||||
nid::Nid,
|
||||
pkey::PKey,
|
||||
x509::{
|
||||
X509, X509NameBuilder,
|
||||
extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName},
|
||||
},
|
||||
};
|
||||
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap();
|
||||
let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap();
|
||||
let mut name = X509NameBuilder::new().unwrap();
|
||||
name.append_entry_by_text("CN", cn).unwrap();
|
||||
let name = name.build();
|
||||
let mut builder = X509::builder().unwrap();
|
||||
builder.set_version(2).unwrap();
|
||||
builder
|
||||
.set_serial_number(
|
||||
&BigNum::from_u32(rand::random_range(1..u32::MAX))
|
||||
.unwrap()
|
||||
.to_asn1_integer()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
builder.set_subject_name(&name).unwrap();
|
||||
builder
|
||||
.set_issuer_name(issuer.map_or(name.as_ref(), |(ca, _)| ca.subject_name()))
|
||||
.unwrap();
|
||||
builder.set_pubkey(&key).unwrap();
|
||||
builder
|
||||
.set_not_before(&Asn1Time::days_from_now(0).unwrap())
|
||||
.unwrap();
|
||||
builder
|
||||
.set_not_after(&Asn1Time::days_from_now(1).unwrap())
|
||||
.unwrap();
|
||||
if issuer.is_none() {
|
||||
builder
|
||||
.append_extension(BasicConstraints::new().critical().ca().build().unwrap())
|
||||
.unwrap();
|
||||
builder
|
||||
.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap())
|
||||
.unwrap();
|
||||
} else {
|
||||
builder
|
||||
.append_extension(BasicConstraints::new().build().unwrap())
|
||||
.unwrap();
|
||||
let mut usage = ExtendedKeyUsage::new();
|
||||
if server {
|
||||
usage.server_auth();
|
||||
} else {
|
||||
usage.client_auth();
|
||||
}
|
||||
builder.append_extension(usage.build().unwrap()).unwrap();
|
||||
if server {
|
||||
let context = builder.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None);
|
||||
let san = SubjectAlternativeName::new()
|
||||
.dns(cn)
|
||||
.build(&context)
|
||||
.unwrap();
|
||||
builder.append_extension(san).unwrap();
|
||||
}
|
||||
}
|
||||
builder
|
||||
.sign(
|
||||
issuer.map_or(&key, |(_, k)| k),
|
||||
openssl::hash::MessageDigest::sha256(),
|
||||
)
|
||||
.unwrap();
|
||||
(builder.build(), key)
|
||||
}
|
||||
async fn tls_bootstrap(server_name: &str) -> bool {
|
||||
use openssl::ssl::{SslAcceptor, SslMethod, SslVerifyMode};
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
net::TcpListener,
|
||||
};
|
||||
let dir = std::env::temp_dir().join(format!("rmm-mtls-key-{}", uuid::Uuid::new_v4()));
|
||||
crate::storage::prepare_dir(&dir).unwrap();
|
||||
let (ca, ca_key) = certificate("test CA", None, false);
|
||||
let (server, server_key) = certificate(server_name, Some((&ca, &ca_key)), true);
|
||||
let (device, device_key) = certificate("enrolled device", Some((&ca, &ca_key)), false);
|
||||
crate::storage::create_secret(&dir.join("device.pem"), &device.to_pem().unwrap()).unwrap();
|
||||
crate::storage::create_secret(
|
||||
&dir.join("device.key"),
|
||||
&device_key.private_key_to_pem_pkcs8().unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let downloaded = key();
|
||||
let response = serde_json::to_vec(&downloaded).unwrap();
|
||||
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap();
|
||||
acceptor.set_certificate(&server).unwrap();
|
||||
acceptor.set_private_key(&server_key).unwrap();
|
||||
acceptor.cert_store_mut().add_cert(ca.clone()).unwrap();
|
||||
acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT);
|
||||
let acceptor = acceptor.build();
|
||||
let worker = std::thread::spawn(move || {
|
||||
let (socket, _) = listener.accept().unwrap();
|
||||
socket
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
if let Ok(mut tls) = acceptor.accept(socket) {
|
||||
assert!(tls.ssl().peer_certificate().is_some());
|
||||
let mut bytes = [0; 8192];
|
||||
if let Ok(n) = tls.read(&mut bytes) {
|
||||
assert!(
|
||||
String::from_utf8_lossy(&bytes[..n])
|
||||
.starts_with("GET /api/v1/job-signing-key HTTP/1.1")
|
||||
);
|
||||
let header = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
response.len()
|
||||
);
|
||||
let _ = tls.write_all(header.as_bytes());
|
||||
let _ = tls.write_all(&response);
|
||||
}
|
||||
}
|
||||
});
|
||||
let mut binding = Binding {
|
||||
tenant_id: uuid::Uuid::new_v4(),
|
||||
asset_id: uuid::Uuid::new_v4(),
|
||||
agent_id: uuid::Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: String::new(),
|
||||
};
|
||||
let result = provision(&dir, address, &ca.to_pem().unwrap(), &mut binding).await;
|
||||
worker.join().unwrap();
|
||||
if result.is_ok() {
|
||||
assert_eq!(binding.public_key_hex, downloaded.public_key_hex);
|
||||
} else {
|
||||
assert!(!dir.join("job-signing-key.json").exists());
|
||||
}
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
result.is_ok()
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn bootstrap_uses_real_mtls_and_dns_connect_override() {
|
||||
assert!(tls_bootstrap("rmm-dev.internal").await);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn bootstrap_rejects_wrong_server_name_without_creating_trust() {
|
||||
assert!(!tls_bootstrap("wrong.internal").await);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
//! Crash-safe device TLS rotation only; never rotates the helper's signing pin.
|
||||
//! The process must hold the existing exclusive agent-state lock while using this
|
||||
//! store. Install returns a mandatory reconnect/Sync signal, not effect authority.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use openssl::{
|
||||
asn1::Asn1Time,
|
||||
ec::{EcGroup, EcKey},
|
||||
hash::MessageDigest,
|
||||
nid::Nid,
|
||||
pkey::PKey,
|
||||
stack::Stack,
|
||||
x509::{
|
||||
X509, X509NameBuilder, X509PurposeId, X509Req, X509StoreContext, store::X509StoreBuilder,
|
||||
verify::X509VerifyParam,
|
||||
},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::Write,
|
||||
net::SocketAddr,
|
||||
os::unix::fs::{OpenOptionsExt, PermissionsExt},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DAY: i64 = 86400;
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct RotationPolicy {
|
||||
pub begin_after_seconds: i64,
|
||||
pub renew_before_seconds: i64,
|
||||
}
|
||||
impl Default for RotationPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
begin_after_seconds: 20 * DAY,
|
||||
renew_before_seconds: 10 * DAY,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Selector {
|
||||
pub schema_version: u32,
|
||||
pub agent_id: Uuid,
|
||||
pub enrollment_generation: u64,
|
||||
pub version: Uuid,
|
||||
pub previous_version: Option<Uuid>,
|
||||
pub previous_until: Option<i64>,
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Pending {
|
||||
schema_version: u32,
|
||||
version: Uuid,
|
||||
base_version: Uuid,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct RenewalResponse {
|
||||
agent_id: Uuid,
|
||||
certificate_pem: String,
|
||||
overlap_seconds: u32,
|
||||
}
|
||||
pub struct DeviceIdentity {
|
||||
pub selector: Selector,
|
||||
pub certificate_pem: Vec<u8>,
|
||||
pub private_key_pem: Vec<u8>,
|
||||
}
|
||||
pub struct RotationOutcome {
|
||||
pub needs_reconnect: bool,
|
||||
pub enrollment_generation: u64,
|
||||
}
|
||||
pub struct RotationStore {
|
||||
root: PathBuf,
|
||||
ca: Vec<u8>,
|
||||
agent: Uuid,
|
||||
enrollment: u64,
|
||||
}
|
||||
|
||||
fn private_dir(path: &Path) -> Result<()> {
|
||||
if !path.exists() {
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700).create(path)?;
|
||||
}
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
ensure!(
|
||||
metadata.is_dir()
|
||||
&& !metadata.file_type().is_symlink()
|
||||
&& metadata.permissions().mode() & 0o777 == 0o700,
|
||||
"ROTATION_DIRECTORY_NOT_PRIVATE"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn secret(path: &Path) -> Result<Vec<u8>> {
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
ensure!(
|
||||
metadata.is_file()
|
||||
&& !metadata.file_type().is_symlink()
|
||||
&& metadata.permissions().mode() & 0o777 == 0o600
|
||||
&& metadata.len() <= 65536,
|
||||
"ROTATION_FILE_NOT_PRIVATE"
|
||||
);
|
||||
Ok(fs::read(path)?)
|
||||
}
|
||||
fn create(path: &Path, data: &[u8]) -> Result<()> {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(path)?;
|
||||
file.write_all(data)?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
fn atomic_json<T: Serialize>(root: &Path, name: &str, value: &T) -> Result<()> {
|
||||
let temp = root.join(format!(".{}.tmp", Uuid::new_v4()));
|
||||
create(&temp, &serde_json::to_vec(value)?)?;
|
||||
fs::rename(temp, root.join(name))?;
|
||||
File::open(root)?.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RotationStore {
|
||||
/// Bootstrap from the existing enrolled key/certificate, not a new identity.
|
||||
/// Existing current.json is never overwritten by bootstrap input.
|
||||
pub fn open(root: &Path, ca: &[u8], agent: Uuid, enrollment: u64) -> Result<Self> {
|
||||
ensure!(enrollment > 0, "ROTATION_IDENTITY_INVALID");
|
||||
private_dir(root)?;
|
||||
private_dir(&root.join("versions"))?;
|
||||
Ok(Self {
|
||||
root: root.into(),
|
||||
ca: ca.into(),
|
||||
agent,
|
||||
enrollment,
|
||||
})
|
||||
}
|
||||
fn version(&self, id: Uuid) -> PathBuf {
|
||||
self.root.join("versions").join(id.to_string())
|
||||
}
|
||||
pub fn initialize(&self, cert: &[u8], key: &[u8], now: i64) -> Result<()> {
|
||||
ensure!(
|
||||
!self.root.join("current.json").exists(),
|
||||
"ROTATION_ALREADY_INITIALIZED"
|
||||
);
|
||||
self.verify(cert, key, now)?;
|
||||
let version = Uuid::new_v4();
|
||||
let dir = self.version(version);
|
||||
private_dir(&dir)?;
|
||||
create(&dir.join("device.key"), key)?;
|
||||
create(&dir.join("device.pem"), cert)?;
|
||||
File::open(&dir)?.sync_all()?;
|
||||
File::open(self.root.join("versions"))?.sync_all()?;
|
||||
atomic_json(
|
||||
&self.root,
|
||||
"current.json",
|
||||
&Selector {
|
||||
schema_version: 1,
|
||||
agent_id: self.agent,
|
||||
enrollment_generation: self.enrollment,
|
||||
version,
|
||||
previous_version: None,
|
||||
previous_until: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
pub fn current(&self, now: i64) -> Result<DeviceIdentity> {
|
||||
let selector: Selector = serde_json::from_slice(&secret(&self.root.join("current.json"))?)?;
|
||||
ensure!(
|
||||
selector.schema_version == 1
|
||||
&& selector.agent_id == self.agent
|
||||
&& selector.enrollment_generation == self.enrollment,
|
||||
"ROTATION_IDENTITY_CHANGED"
|
||||
);
|
||||
let dir = self.version(selector.version);
|
||||
private_dir(&dir)?;
|
||||
let certificate_pem = secret(&dir.join("device.pem"))?;
|
||||
let private_key_pem = secret(&dir.join("device.key"))?;
|
||||
self.verify(&certificate_pem, &private_key_pem, now)?;
|
||||
Ok(DeviceIdentity {
|
||||
selector,
|
||||
certificate_pem,
|
||||
private_key_pem,
|
||||
})
|
||||
}
|
||||
/// Checks full chain, current validity, clientAuth purpose, key and exact CN.
|
||||
/// X509 SSL_CLIENT purpose rejects server-only certificates and CA leaves.
|
||||
fn verify(&self, pem: &[u8], key: &[u8], now: i64) -> Result<()> {
|
||||
let chain = X509::stack_from_pem(pem)?;
|
||||
let leaf = chain.first().context("ROTATION_CERTIFICATE_MISSING")?;
|
||||
let key = PKey::private_key_from_pem(key)?;
|
||||
ensure!(leaf.public_key()?.public_eq(&key), "ROTATION_KEY_MISMATCH");
|
||||
let names = leaf
|
||||
.subject_name()
|
||||
.entries_by_nid(Nid::COMMONNAME)
|
||||
.collect::<Vec<_>>();
|
||||
ensure!(
|
||||
names.len() == 1 && names[0].data().as_slice() == self.agent.to_string().as_bytes(),
|
||||
"ROTATION_AGENT_MISMATCH"
|
||||
);
|
||||
let time = Asn1Time::from_unix(now)?;
|
||||
ensure!(
|
||||
leaf.not_before() <= time && leaf.not_after() > time,
|
||||
"ROTATION_EXPIRED_OR_NOT_YET_VALID_RECOVERY_REQUIRED"
|
||||
);
|
||||
let mut builder = X509StoreBuilder::new()?;
|
||||
for ca in X509::stack_from_pem(&self.ca)? {
|
||||
builder.add_cert(ca)?;
|
||||
}
|
||||
let mut param = X509VerifyParam::new()?;
|
||||
param.set_time(now);
|
||||
param.set_purpose(X509PurposeId::SSL_CLIENT)?;
|
||||
builder.set_param(¶m)?;
|
||||
let mut intermediates = Stack::new()?;
|
||||
for cert in chain.iter().skip(1) {
|
||||
intermediates.push(cert.clone())?;
|
||||
}
|
||||
let mut context = X509StoreContext::new()?;
|
||||
ensure!(
|
||||
context.init(&builder.build(), leaf, &intermediates, |c| c.verify_cert())?,
|
||||
"ROTATION_CERTIFICATE_UNTRUSTED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn due(&self, policy: RotationPolicy, now: i64) -> Result<bool> {
|
||||
ensure!(
|
||||
policy.begin_after_seconds > 0 && policy.renew_before_seconds > 0,
|
||||
"ROTATION_POLICY_INVALID"
|
||||
);
|
||||
let identity = self.current(now)?;
|
||||
let cert = X509::from_pem(&identity.certificate_pem)?;
|
||||
let time = Asn1Time::from_unix(now)?;
|
||||
let age = cert.not_before().diff(&time)?;
|
||||
let left = time.diff(cert.not_after())?;
|
||||
Ok(
|
||||
i64::from(age.days) * DAY + i64::from(age.secs) >= policy.begin_after_seconds
|
||||
|| i64::from(left.days) * DAY + i64::from(left.secs) <= policy.renew_before_seconds
|
||||
|| self.root.join("pending.json").exists(),
|
||||
)
|
||||
}
|
||||
/// Persist both key and exact CSR before publishing the pending selector.
|
||||
/// Orphaned versions after a crash are harmless and never selected.
|
||||
fn pending(&self, current: &Selector) -> Result<Pending> {
|
||||
let path = self.root.join("pending.json");
|
||||
if path.exists() {
|
||||
let pending: Pending = serde_json::from_slice(&secret(&path)?)?;
|
||||
ensure!(
|
||||
pending.schema_version == 1
|
||||
&& (pending.base_version == current.version
|
||||
|| pending.version == current.version),
|
||||
"ROTATION_PENDING_CONFLICT"
|
||||
);
|
||||
return Ok(pending);
|
||||
}
|
||||
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
|
||||
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
|
||||
let mut name = X509NameBuilder::new()?;
|
||||
name.append_entry_by_text("CN", &self.agent.to_string())?;
|
||||
let mut csr = X509Req::builder()?;
|
||||
csr.set_subject_name(&name.build())?;
|
||||
csr.set_pubkey(&key)?;
|
||||
csr.sign(&key, MessageDigest::sha256())?;
|
||||
let version = Uuid::new_v4();
|
||||
let dir = self.version(version);
|
||||
private_dir(&dir)?;
|
||||
create(&dir.join("device.key"), &key.private_key_to_pem_pkcs8()?)?;
|
||||
create(&dir.join("request.pem"), &csr.build().to_pem()?)?;
|
||||
File::open(dir)?.sync_all()?;
|
||||
File::open(self.root.join("versions"))?.sync_all()?;
|
||||
let pending = Pending {
|
||||
schema_version: 1,
|
||||
version,
|
||||
base_version: current.version,
|
||||
};
|
||||
atomic_json(&self.root, "pending.json", &pending)?;
|
||||
Ok(pending)
|
||||
}
|
||||
/// Retry after response loss uses the original key/CSR and old valid mTLS
|
||||
/// identity. Expiry/revocation remains blocked; no automatic re-enrollment.
|
||||
pub async fn renew(&self, address: SocketAddr, now: i64) -> Result<RotationOutcome> {
|
||||
let old = self.current(now)?;
|
||||
let pending = self.pending(&old.selector)?;
|
||||
if pending.version == old.selector.version {
|
||||
self.clear_pending()?;
|
||||
return Ok(RotationOutcome {
|
||||
needs_reconnect: true,
|
||||
enrollment_generation: self.enrollment,
|
||||
});
|
||||
}
|
||||
let dir = self.version(pending.version);
|
||||
private_dir(&dir)?;
|
||||
let key = secret(&dir.join("device.key"))?;
|
||||
let csr = secret(&dir.join("request.pem"))?;
|
||||
let parsed = X509Req::from_pem(&csr)?;
|
||||
let public = parsed.public_key()?;
|
||||
let private = PKey::private_key_from_pem(&key)?;
|
||||
ensure!(
|
||||
parsed.verify(&public)? && public.public_eq(&private),
|
||||
"ROTATION_PENDING_KEY_MISMATCH"
|
||||
);
|
||||
let client = crate::https_channel::mtls_client(
|
||||
address,
|
||||
&self.ca,
|
||||
&old.certificate_pem,
|
||||
&old.private_key_pem,
|
||||
)?;
|
||||
let response = client
|
||||
.post(format!(
|
||||
"https://rmm-dev.internal:{}/agent/v1/certificates/renew",
|
||||
address.port()
|
||||
))
|
||||
.json(&serde_json::json!({"csr_pem": String::from_utf8(csr)?}))
|
||||
.send()
|
||||
.await?;
|
||||
let bytes = crate::https_channel::bounded_body(response, 65536, "application/json").await?;
|
||||
let response: RenewalResponse = serde_json::from_slice(&bytes)?;
|
||||
ensure!(
|
||||
response.agent_id == self.agent && response.overlap_seconds == 300,
|
||||
"ROTATION_RESPONSE_BINDING_INVALID"
|
||||
);
|
||||
// Never install a certificate which expired during the network request.
|
||||
let verified_at = now.max(chrono::Utc::now().timestamp());
|
||||
self.verify(response.certificate_pem.as_bytes(), &key, verified_at)?;
|
||||
// A crash after writing a response but before selector commit retries the
|
||||
// same CSR. A mismatching response must not overwrite staged evidence.
|
||||
let cert_path = dir.join("device.pem");
|
||||
if cert_path.exists() {
|
||||
ensure!(
|
||||
secret(&cert_path)? == response.certificate_pem.as_bytes(),
|
||||
"ROTATION_RESPONSE_CHANGED"
|
||||
);
|
||||
} else {
|
||||
create(&cert_path, response.certificate_pem.as_bytes())?;
|
||||
}
|
||||
File::open(&dir)?.sync_all()?;
|
||||
atomic_json(
|
||||
&self.root,
|
||||
"current.json",
|
||||
&Selector {
|
||||
schema_version: 1,
|
||||
agent_id: self.agent,
|
||||
enrollment_generation: self.enrollment,
|
||||
version: pending.version,
|
||||
previous_version: Some(old.selector.version),
|
||||
previous_until: Some(verified_at + 300),
|
||||
},
|
||||
)?;
|
||||
// Old version is retained, but is never automatically selected on failure.
|
||||
self.clear_pending()?;
|
||||
Ok(RotationOutcome {
|
||||
needs_reconnect: true,
|
||||
enrollment_generation: self.enrollment,
|
||||
})
|
||||
}
|
||||
fn clear_pending(&self) -> Result<()> {
|
||||
fs::remove_file(self.root.join("pending.json"))?;
|
||||
File::open(&self.root)?.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
use crate::reconcile;
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use rmm_privileged_helper::{Binding, control, readiness, reconcile as root};
|
||||
use rmm_protocol::{SyncAttempt, SyncRequest, SyncResponse};
|
||||
use std::path::{Path, PathBuf};
|
||||
use uuid::Uuid;
|
||||
#[derive(Clone)]
|
||||
struct Endpoint {
|
||||
socket: PathBuf,
|
||||
profile: String,
|
||||
hash: String,
|
||||
}
|
||||
#[derive(Clone)]
|
||||
pub struct Helpers {
|
||||
binding: Binding,
|
||||
endpoints: Vec<Endpoint>,
|
||||
}
|
||||
fn map(s: &root::Snapshot) -> reconcile::HelperSnapshot {
|
||||
reconcile::HelperSnapshot {
|
||||
journal_id: s.journal_id,
|
||||
checkpoint: s.checkpoint,
|
||||
previous_decision_id: s.previous_decision_id,
|
||||
attempts: s
|
||||
.attempts
|
||||
.iter()
|
||||
.map(|a| SyncAttempt {
|
||||
job_id: a.job_id.to_string(),
|
||||
attempt_id: a.attempt_id.to_string(),
|
||||
job_payload_hash: a.job_payload_hash.clone(),
|
||||
grant_id: a.grant_id.to_string(),
|
||||
consumption: a.consumption.clone(),
|
||||
state: a.state.clone(),
|
||||
revision: a.revision,
|
||||
original_boot_id: a.original_boot_id.to_string(),
|
||||
outcome_sha256: a.outcome_sha256.clone(),
|
||||
outcome_bytes: a.outcome_bytes.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
fn general(binding: &Binding) -> Result<control::Response> {
|
||||
let socket = std::env::var_os("RMM_HELPER_SOCKET").context("NO_GENERAL_HELPER")?;
|
||||
let path = std::env::var_os("RMM_HELPER_POLICY_FILE").context("GENERAL_POLICY_PIN_REQUIRED")?;
|
||||
let pin = control::GeneralPin::read(Path::new(&path), binding)?;
|
||||
control::general_probe(Path::new(&socket), binding, &pin)
|
||||
}
|
||||
impl Helpers {
|
||||
pub fn configured(binding: &Binding) -> Result<Self> {
|
||||
let mut endpoints = vec![];
|
||||
if let Some(socket) = std::env::var_os("RMM_HELPER_SOCKET") {
|
||||
let path = std::env::var_os("RMM_HELPER_POLICY_FILE")
|
||||
.context("GENERAL_POLICY_PIN_REQUIRED")?;
|
||||
let pin = control::GeneralPin::read(Path::new(&path), binding)?;
|
||||
endpoints.push(Endpoint {
|
||||
socket: socket.into(),
|
||||
profile: "general".into(),
|
||||
hash: pin.policy_sha256,
|
||||
});
|
||||
}
|
||||
if let Some(socket) = std::env::var_os("RMM_APT_HELPER_SOCKET") {
|
||||
ensure!(
|
||||
Path::new(&socket) == Path::new(readiness::SOCKET),
|
||||
"APT_HELPER_ROUTE"
|
||||
);
|
||||
let path =
|
||||
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_PROOF_REQUIRED")?;
|
||||
let pin = readiness::Pin::read(Path::new(&path), binding)?;
|
||||
endpoints.push(Endpoint {
|
||||
socket: socket.into(),
|
||||
profile: "apt_only".into(),
|
||||
hash: pin.provider_stack_sha256,
|
||||
});
|
||||
}
|
||||
Ok(Self {
|
||||
binding: binding.clone(),
|
||||
endpoints,
|
||||
})
|
||||
}
|
||||
fn validate(&self, e: &Endpoint, r: &control::Response) -> Result<()> {
|
||||
ensure!(
|
||||
r.profile == e.profile && r.policy_sha256 == e.hash,
|
||||
"HELPER_PIN_PROFILE_MISMATCH"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn snapshot(&self, generation: u64) -> Result<Option<reconcile::HelperSnapshot>> {
|
||||
if self.endpoints.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
for _ in 0..3 {
|
||||
let mut snapshots = vec![];
|
||||
let mut inodes = None;
|
||||
for e in &self.endpoints {
|
||||
let q = control::Request::Snapshot {
|
||||
challenge: Uuid::new_v4(),
|
||||
identity: readiness::Identity::from_binding(&self.binding)?,
|
||||
generation,
|
||||
};
|
||||
let r = control::call(&e.socket, &self.binding, &q)?;
|
||||
self.validate(e, &r)?;
|
||||
let ids = (r.ledger_device, r.ledger_inode, r.lock_device, r.lock_inode);
|
||||
if let Some(old) = inodes {
|
||||
ensure!(old == ids, "HELPERS_MUST_SHARE_LEDGER_AND_LOCK");
|
||||
}
|
||||
inodes = Some(ids);
|
||||
snapshots.push(r.snapshot.context("HELPER_SNAPSHOT_REQUIRED")?);
|
||||
}
|
||||
let first = &snapshots[0];
|
||||
if snapshots.iter().all(|s| {
|
||||
s.journal_id == first.journal_id
|
||||
&& s.checkpoint == first.checkpoint
|
||||
&& s.previous_decision_id == first.previous_decision_id
|
||||
&& serde_json::to_vec(&s.attempts).ok()
|
||||
== serde_json::to_vec(&first.attempts).ok()
|
||||
}) {
|
||||
return Ok(Some(map(first)));
|
||||
}
|
||||
}
|
||||
anyhow::bail!("HELPER_CHECKPOINTS_DID_NOT_CONVERGE")
|
||||
}
|
||||
pub fn apply(
|
||||
&self,
|
||||
response: &SyncResponse,
|
||||
request: &SyncRequest,
|
||||
) -> Result<Option<reconcile::HelperSnapshot>> {
|
||||
use prost::Message;
|
||||
let mut applied: Option<root::Snapshot> = None;
|
||||
for e in &self.endpoints {
|
||||
let q = control::Request::Apply {
|
||||
challenge: Uuid::new_v4(),
|
||||
identity: readiness::Identity::from_binding(&self.binding)?,
|
||||
response: root::SignedResponse {
|
||||
decision_bytes: response.decision_bytes.clone(),
|
||||
signature: response.signature.clone(),
|
||||
},
|
||||
request_bytes: request.encode_to_vec(),
|
||||
};
|
||||
let r = control::call(&e.socket, &self.binding, &q)?;
|
||||
self.validate(e, &r)?;
|
||||
let s = r.snapshot.context("APPLIED_HELPER_SNAPSHOT_REQUIRED")?;
|
||||
if let Some(old) = &applied {
|
||||
ensure!(
|
||||
old.journal_id == s.journal_id
|
||||
&& old.checkpoint == s.checkpoint
|
||||
&& old.previous_decision_id == s.previous_decision_id,
|
||||
"HELPER_APPLY_CHECKPOINT_MISMATCH"
|
||||
);
|
||||
}
|
||||
applied = Some(s);
|
||||
}
|
||||
Ok(applied.as_ref().map(map))
|
||||
}
|
||||
}
|
||||
pub fn general_capabilities(binding: &Binding) -> Vec<String> {
|
||||
(|| -> Result<Vec<String>> {
|
||||
let response = general(binding)?;
|
||||
let policy = response.policy.context("NO_GENERAL_POLICY")?;
|
||||
let mut caps = vec![
|
||||
"helper.root.v1".into(),
|
||||
format!("helper.policy.sha256:{}", response.policy_sha256),
|
||||
];
|
||||
let mut actions = vec![];
|
||||
if policy.allow_root_scripts && rmm_agent_linux::actions::Interpreter::PosixSh.available() {
|
||||
actions.push("run_script");
|
||||
}
|
||||
if !policy.allowed_services.is_empty() {
|
||||
actions.extend(["set_service_state", "query_logs"]);
|
||||
}
|
||||
if policy.allow_reboot {
|
||||
actions.push("reboot");
|
||||
}
|
||||
for action in actions {
|
||||
caps.push(action.into());
|
||||
caps.push(format!("root:{action}"));
|
||||
}
|
||||
Ok(caps)
|
||||
})()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
pub fn general_supported(binding: &Binding, job: &rmm_domain::jobs::JobPayload) -> bool {
|
||||
(|| -> Result<bool> {
|
||||
let p = general(binding)?.policy.context("NO_GENERAL_POLICY")?;
|
||||
Ok(match &job.action {
|
||||
rmm_domain::jobs::Action::RunScript { interpreter, .. } => {
|
||||
p.allow_root_scripts
|
||||
&& match interpreter.as_str() {
|
||||
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
|
||||
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
rmm_domain::jobs::Action::SetServiceState { unit, state } => {
|
||||
p.allowed_services.contains(unit)
|
||||
&& matches!(state.as_str(), "running" | "stopped")
|
||||
&& job.max_runtime_seconds >= 180
|
||||
}
|
||||
rmm_domain::jobs::Action::QueryLogs { unit, .. } => {
|
||||
p.allowed_services.contains(unit) && job.max_runtime_seconds >= 20
|
||||
}
|
||||
rmm_domain::jobs::Action::Reboot { .. } => {
|
||||
p.allow_reboot && job.max_runtime_seconds >= 60
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
})()
|
||||
.unwrap_or(false)
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
//! HTTPS alternative to WSS, using the *same* durable envelopes.
|
||||
//!
|
||||
//! The caller owns the journal, sequence allocation and retry policy. In particular,
|
||||
//! HTTP success never removes a journal entry: only a returned concrete `Ack` may
|
||||
//! be passed to the journal consumer. There is deliberately no offline send queue.
|
||||
//! PollRequest has no cursor field in wire profile 1: its fresh heartbeat carries
|
||||
//! the boot/sequence/connection context. Do not invent a JSON cursor or ACK range.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use prost::Message;
|
||||
use reqwest::{Client, StatusCode};
|
||||
use rmm_protocol::{Envelope, MessageBatch, Payload, PollRequest, ServerHello};
|
||||
use std::{collections::VecDeque, net::SocketAddr, time::Duration};
|
||||
|
||||
pub const MAX_BODY: usize = 1024 * 1024;
|
||||
pub const MAX_BATCH: usize = 16;
|
||||
const MAX_INCOMING: usize = 32;
|
||||
|
||||
/// No environment proxy trust, redirects, alternate names or system CA fallback.
|
||||
/// `address` is a parsed RMM_CONNECT_ADDR; it overrides TCP only, never SNI.
|
||||
pub fn mtls_client(address: SocketAddr, ca: &[u8], cert: &[u8], key: &[u8]) -> Result<Client> {
|
||||
ensure!(
|
||||
address.port() != 0 && !address.ip().is_unspecified() && !address.ip().is_multicast(),
|
||||
"INVALID_CONNECT_ADDRESS"
|
||||
);
|
||||
let mut identity = Vec::with_capacity(cert.len() + key.len() + 1);
|
||||
identity.extend_from_slice(cert);
|
||||
identity.push(b'\n');
|
||||
identity.extend_from_slice(key);
|
||||
let client = Client::builder()
|
||||
.https_only(true)
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
||||
.identity(reqwest::Identity::from_pem(&identity)?)
|
||||
.resolve("rmm-dev.internal", address)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(35))
|
||||
.pool_max_idle_per_host(2)
|
||||
.build()?;
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Bounded response reading also applies to chunked bodies, before prost parsing.
|
||||
pub async fn bounded_body(
|
||||
mut response: reqwest::Response,
|
||||
limit: usize,
|
||||
media: &str,
|
||||
) -> Result<Vec<u8>> {
|
||||
ensure!(
|
||||
response.status() == StatusCode::OK,
|
||||
"HTTPS_STATUS_{}",
|
||||
response.status().as_u16()
|
||||
);
|
||||
ensure!(
|
||||
response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
== Some(media),
|
||||
"HTTPS_CONTENT_TYPE"
|
||||
);
|
||||
ensure!(
|
||||
!response
|
||||
.headers()
|
||||
.contains_key(reqwest::header::CONTENT_ENCODING),
|
||||
"HTTPS_COMPRESSION_UNSUPPORTED"
|
||||
);
|
||||
ensure!(
|
||||
response.content_length().is_none_or(|n| n <= limit as u64),
|
||||
"HTTPS_BODY_LIMIT"
|
||||
);
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
ensure!(
|
||||
chunk.len() <= limit.saturating_sub(bytes.len()),
|
||||
"HTTPS_BODY_LIMIT"
|
||||
);
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Only an explicitly classified WebSocket upgrade refusal may select fallback.
|
||||
/// TLS, authentication and ambiguous network failures must remain blocked/retried
|
||||
/// with normal verification. Never infer this classification from an error string.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum WssFailure {
|
||||
UpgradeBlocked,
|
||||
TlsOrIdentity,
|
||||
Authentication,
|
||||
Other,
|
||||
}
|
||||
pub fn allow_https_fallback(failure: WssFailure, attempts: u32) -> bool {
|
||||
failure == WssFailure::UpgradeBlocked && (2..=3).contains(&attempts)
|
||||
}
|
||||
pub fn retry_delay(attempt: u32) -> Duration {
|
||||
let ceiling = (1_u64 << attempt.min(6)).min(60) * 1000;
|
||||
Duration::from_millis(rand::random_range(0..=ceiling))
|
||||
}
|
||||
|
||||
pub struct HttpsChannel {
|
||||
client: Client,
|
||||
origin: String,
|
||||
agent: String,
|
||||
enrollment: u64,
|
||||
generation: u64,
|
||||
incoming: VecDeque<Envelope>,
|
||||
next_request: tokio::time::Instant,
|
||||
fenced: bool,
|
||||
}
|
||||
impl HttpsChannel {
|
||||
/// Performs ClientHello first. ServerHello remains in the receive queue so
|
||||
/// the scheduler can close its effect gate and apply reconciliation normally.
|
||||
pub async fn connect(client: Client, address: SocketAddr, hello: Envelope) -> Result<Self> {
|
||||
ensure!(
|
||||
hello.connection_generation == 0
|
||||
&& matches!(hello.payload, Some(Payload::ClientHello(_))),
|
||||
"CLIENT_HELLO_REQUIRED"
|
||||
);
|
||||
ensure!(
|
||||
rmm_protocol::valid_envelope(&hello, &hello.agent_id, 0),
|
||||
"INVALID_CLIENT_HELLO"
|
||||
);
|
||||
let mut channel = Self {
|
||||
client,
|
||||
origin: format!("https://rmm-dev.internal:{}", address.port()),
|
||||
agent: hello.agent_id.clone(),
|
||||
enrollment: hello.enrollment_generation,
|
||||
generation: 0,
|
||||
incoming: VecDeque::new(),
|
||||
next_request: tokio::time::Instant::now(),
|
||||
fenced: false,
|
||||
};
|
||||
let batch = MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: vec![hello],
|
||||
};
|
||||
let reply = channel
|
||||
.request("messages", batch.encode_to_vec(), Duration::from_secs(15))
|
||||
.await?;
|
||||
let first = reply.messages.first().context("SERVER_HELLO_REQUIRED")?;
|
||||
let Some(Payload::ServerHello(ServerHello {
|
||||
connection_generation,
|
||||
max_message_bytes,
|
||||
})) = &first.payload
|
||||
else {
|
||||
anyhow::bail!("SERVER_HELLO_REQUIRED")
|
||||
};
|
||||
ensure!(
|
||||
*connection_generation > 0 && *max_message_bytes as usize == MAX_BODY,
|
||||
"UNSUPPORTED_SERVER_LIMITS"
|
||||
);
|
||||
channel.generation = *connection_generation;
|
||||
channel.accept(reply)?;
|
||||
Ok(channel)
|
||||
}
|
||||
pub fn generation(&self) -> u64 {
|
||||
self.generation
|
||||
}
|
||||
pub fn receive(&mut self) -> Option<Envelope> {
|
||||
self.incoming.pop_front()
|
||||
}
|
||||
pub fn fence(&mut self) {
|
||||
self.fenced = true;
|
||||
}
|
||||
fn validate_outgoing(&self, envelope: &Envelope) -> Result<()> {
|
||||
ensure!(
|
||||
rmm_protocol::valid_envelope(envelope, &self.agent, self.generation)
|
||||
&& envelope.enrollment_generation == self.enrollment,
|
||||
"INVALID_OUTGOING_ENVELOPE"
|
||||
);
|
||||
ensure!(envelope.encoded_len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
|
||||
Ok(())
|
||||
}
|
||||
/// Caller retries durable messages unchanged after uncertainty. Fresh permit
|
||||
/// requests must instead enter reconciliation on a lost response, not retry.
|
||||
pub async fn send(&mut self, envelope: &Envelope) -> Result<()> {
|
||||
self.send_batch(std::slice::from_ref(envelope)).await
|
||||
}
|
||||
pub async fn send_batch(&mut self, messages: &[Envelope]) -> Result<()> {
|
||||
ensure!(
|
||||
!messages.is_empty() && messages.len() <= MAX_BATCH,
|
||||
"HTTPS_BATCH_LIMIT"
|
||||
);
|
||||
ensure!(
|
||||
self.incoming.len() + messages.len() < MAX_INCOMING,
|
||||
"HTTPS_BACKPRESSURE"
|
||||
);
|
||||
for e in messages {
|
||||
self.validate_outgoing(e)?;
|
||||
ensure!(
|
||||
!matches!(e.payload, Some(Payload::ClientHello(_))),
|
||||
"HELLO_REQUIRES_NEW_CHANNEL"
|
||||
);
|
||||
if self.fenced || messages.len() > 1 {
|
||||
ensure!(
|
||||
matches!(
|
||||
e.payload,
|
||||
Some(
|
||||
Payload::InventorySnapshot(_)
|
||||
| Payload::JobAccepted(_)
|
||||
| Payload::JobStateChanged(_)
|
||||
| Payload::JobResult(_)
|
||||
)
|
||||
),
|
||||
"FENCED_OR_NON_DURABLE_BATCH"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Check before allocating/cloning the full encoded batch.
|
||||
let encoded_size = messages
|
||||
.iter()
|
||||
.try_fold(2_usize, |n, e| n.checked_add(e.encoded_len() + 5))
|
||||
.context("HTTPS_BODY_LIMIT")?;
|
||||
ensure!(encoded_size <= MAX_BODY, "HTTPS_BODY_LIMIT");
|
||||
let reply = self
|
||||
.request(
|
||||
"messages",
|
||||
MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: messages.to_vec(),
|
||||
}
|
||||
.encode_to_vec(),
|
||||
Duration::from_secs(15),
|
||||
)
|
||||
.await?;
|
||||
self.accept(reply)
|
||||
}
|
||||
/// At most one poll per channel (`&mut self`); cancelling a dropped poll is
|
||||
/// safe, but its heartbeat ID must not be reused for a different envelope.
|
||||
pub async fn poll(&mut self, heartbeat: Envelope, wait_seconds: u32) -> Result<()> {
|
||||
ensure!(!self.fenced, "HTTPS_CHANNEL_FENCED");
|
||||
ensure!(
|
||||
self.incoming.len() + 2 <= MAX_INCOMING,
|
||||
"HTTPS_BACKPRESSURE"
|
||||
);
|
||||
self.validate_outgoing(&heartbeat)?;
|
||||
ensure!(
|
||||
wait_seconds <= 25 && matches!(heartbeat.payload, Some(Payload::Heartbeat(_))),
|
||||
"INVALID_POLL"
|
||||
);
|
||||
let request = PollRequest {
|
||||
envelope: Some(heartbeat),
|
||||
wait_seconds,
|
||||
};
|
||||
let reply = self
|
||||
.request(
|
||||
"poll",
|
||||
request.encode_to_vec(),
|
||||
Duration::from_secs(u64::from(wait_seconds) + 10),
|
||||
)
|
||||
.await?;
|
||||
self.accept(reply)
|
||||
}
|
||||
async fn request(
|
||||
&mut self,
|
||||
route: &str,
|
||||
body: Vec<u8>,
|
||||
timeout: Duration,
|
||||
) -> Result<MessageBatch> {
|
||||
ensure!(body.len() <= MAX_BODY, "HTTPS_BODY_LIMIT");
|
||||
tokio::time::sleep_until(self.next_request).await;
|
||||
// <=10 HTTP requests/sec, with jitter, even with zero-wait polls.
|
||||
self.next_request =
|
||||
tokio::time::Instant::now() + Duration::from_millis(rand::random_range(100..=150));
|
||||
let response = self
|
||||
.client
|
||||
.post(format!("{}/agent/v1/{route}", self.origin))
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/x-protobuf")
|
||||
.header(reqwest::header::ACCEPT, "application/x-protobuf")
|
||||
.timeout(timeout)
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
if response.status() == StatusCode::CONFLICT
|
||||
|| response.status() == StatusCode::UNAUTHORIZED
|
||||
|| response.status() == StatusCode::FORBIDDEN
|
||||
{
|
||||
self.fence();
|
||||
}
|
||||
let bytes = bounded_body(response, MAX_BODY, "application/x-protobuf").await?;
|
||||
let batch = MessageBatch::decode(bytes.as_slice())?;
|
||||
ensure!(
|
||||
batch.protocol_major == 1 && batch.messages.len() <= MAX_BATCH,
|
||||
"INVALID_SERVER_BATCH"
|
||||
);
|
||||
Ok(batch)
|
||||
}
|
||||
fn accept(&mut self, reply: MessageBatch) -> Result<()> {
|
||||
ensure!(
|
||||
self.incoming.len() + reply.messages.len() <= MAX_INCOMING,
|
||||
"HTTPS_BACKPRESSURE"
|
||||
);
|
||||
// Validate the entire batch before exposing any control message.
|
||||
for e in &reply.messages {
|
||||
ensure!(
|
||||
e.protocol_major == 1
|
||||
&& e.protocol_minor == 0
|
||||
&& e.agent_id == self.agent
|
||||
&& e.enrollment_generation == self.enrollment
|
||||
&& e.connection_generation == self.generation
|
||||
&& matches!(
|
||||
e.payload,
|
||||
Some(
|
||||
Payload::ServerHello(_)
|
||||
| Payload::Ack(_)
|
||||
| Payload::JobOffer(_)
|
||||
| Payload::ExecutionPermit(_)
|
||||
| Payload::CancelRequest(_)
|
||||
| Payload::SyncResponse(_)
|
||||
)
|
||||
),
|
||||
"INVALID_SERVER_ENVELOPE"
|
||||
);
|
||||
if self.fenced {
|
||||
ensure!(
|
||||
matches!(e.payload, Some(Payload::Ack(_))),
|
||||
"HTTPS_CHANNEL_FENCED"
|
||||
);
|
||||
}
|
||||
}
|
||||
self.incoming.extend(reply.messages);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
use anyhow::{Result, ensure};
|
||||
use prost::Message;
|
||||
use rmm_domain::jobs::{Action, JobPayload, hash};
|
||||
use rmm_privileged_helper::{Binding, Outcome, Request};
|
||||
use rmm_protocol::*;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn supported(job: &JobPayload) -> bool {
|
||||
if rmm_privileged_helper::apt::is_apt_action(job) {
|
||||
// A configured environment path alone is never a capability proof.
|
||||
// Caller binding is independently provisioned, not taken from the job.
|
||||
return false;
|
||||
}
|
||||
// The network process must never execute a script with accidental root.
|
||||
supported_as(
|
||||
job,
|
||||
unsafe { libc::geteuid() },
|
||||
std::env::var_os("RMM_HELPER_SOCKET").is_some(),
|
||||
)
|
||||
}
|
||||
pub fn supported_with_binding(job: &JobPayload, binding: Option<&Binding>) -> bool {
|
||||
if rmm_privileged_helper::apt::is_apt_action(job) {
|
||||
// SAFETY: geteuid has no preconditions.
|
||||
return unsafe { libc::geteuid() } != 0
|
||||
&& binding.is_some_and(|b| crate::apt::supported(b, job));
|
||||
}
|
||||
if job.execution_identity == "root" {
|
||||
return unsafe { libc::geteuid() } != 0
|
||||
&& binding.is_some_and(|b| crate::helper_sync::general_supported(b, job));
|
||||
}
|
||||
supported(job)
|
||||
}
|
||||
fn supported_as(job: &JobPayload, uid: u32, helper_configured: bool) -> bool {
|
||||
let unprivileged = uid != 0;
|
||||
if job.execution_identity == "root" && unprivileged {
|
||||
return helper_configured
|
||||
&& matches!(
|
||||
job.action,
|
||||
Action::RunScript { .. } | Action::SetServiceState { .. } | Action::Reboot { .. }
|
||||
);
|
||||
}
|
||||
job.execution_identity == "agent_service"
|
||||
&& unprivileged
|
||||
&& match &job.action {
|
||||
Action::RunScript { interpreter, .. } => match interpreter.as_str() {
|
||||
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
|
||||
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
|
||||
_ => false,
|
||||
},
|
||||
Action::RefreshInventory { scope } => scope == "linux.base",
|
||||
Action::QueryLogs { .. } => log_request(job).is_ok(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
pub fn accept(db: &Connection, binding: &Binding, offer: &JobOffer) -> Result<bool> {
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"OFFER_BINDING"
|
||||
);
|
||||
ensure!(
|
||||
supported_with_binding(&job, Some(binding)),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
);
|
||||
let bytes = offer.encode_to_vec();
|
||||
let existing: Option<Vec<u8>> = db
|
||||
.query_row(
|
||||
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some(old) = existing {
|
||||
ensure!(old == bytes, "OFFER_CONFLICT");
|
||||
return Ok(false);
|
||||
}
|
||||
let count: i64 = db.query_row("SELECT count(*) FROM job_attempts", [], |r| r.get(0))?;
|
||||
ensure!(count < 10000, "JOURNAL_CAPACITY_NEW_EFFECTS_BLOCKED");
|
||||
db.execute(
|
||||
"INSERT INTO job_attempts(job,attempt,offer,state) VALUES(?1,?2,?3,'accepted')",
|
||||
params![offer.job_id, offer.attempt_id, bytes],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
fn hex_hash(s: &str) -> bool {
|
||||
s.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}
|
||||
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
|
||||
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
||||
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
|
||||
db.execute(
|
||||
"INSERT INTO job_events(id,payload) VALUES(?1,?2)",
|
||||
params![envelope.message_id, envelope.encode_to_vec()],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn events(db: &Connection) -> Result<Vec<Envelope>> {
|
||||
let mut statement = db.prepare("SELECT payload FROM job_events ORDER BY rowid LIMIT 100")?;
|
||||
let bytes = statement
|
||||
.query_map([], |r| r.get::<_, Vec<u8>>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
bytes
|
||||
.into_iter()
|
||||
.map(|b| Ok(Envelope::decode(b.as_slice())?))
|
||||
.collect()
|
||||
}
|
||||
pub fn ack(db: &mut Connection, id: &str) -> Result<()> {
|
||||
let tx = db.transaction()?;
|
||||
crate::reconcile::archive_events(&tx)?;
|
||||
tx.execute("DELETE FROM job_events WHERE id=?1", [id])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn next(db: &Connection) -> Result<Option<JobOffer>> {
|
||||
// One device-wide effect at a time; unknown/reserved attempts retain locks.
|
||||
let busy: i64 = db.query_row("SELECT count(*) FROM job_attempts WHERE state IN ('requesting','running','unknown','awaiting_reboot')", [], |r| r.get(0))?;
|
||||
if busy != 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut statement = db.prepare("SELECT offer FROM job_attempts WHERE state='accepted' AND cancel_requested=0 ORDER BY rowid")?;
|
||||
let rows = statement.query_map([], |r| r.get::<_, Vec<u8>>(0))?;
|
||||
let now = chrono::Utc::now();
|
||||
for bytes in rows {
|
||||
let offer = JobOffer::decode(bytes?.as_slice())?;
|
||||
let job: JobPayload = serde_json::from_slice(&offer.payload_bytes)?;
|
||||
if now >= job.not_before || now >= job.start_before {
|
||||
return Ok(Some(offer));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
pub fn requesting(db: &Connection) -> Result<Option<JobOffer>> {
|
||||
let bytes: Option<Vec<u8>> = db
|
||||
.query_row(
|
||||
"SELECT offer FROM job_attempts WHERE state='requesting' LIMIT 1",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
bytes
|
||||
.map(|b| Ok(JobOffer::decode(b.as_slice())?))
|
||||
.transpose()
|
||||
}
|
||||
pub fn reserve_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
||||
ensure!(db.execute("UPDATE job_attempts SET state='requesting' WHERE job=?1 AND attempt=?2 AND state='accepted' AND cancel_requested=0",
|
||||
params![offer.job_id,offer.attempt_id])? == 1, "ATTEMPT_NOT_STARTABLE");
|
||||
Ok(())
|
||||
}
|
||||
pub fn transition(db: &Connection, offer: &JobOffer, state: &str, grant: &str) -> Result<u64> {
|
||||
let previous: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let allowed = match previous.as_str() {
|
||||
"accepted" | "requesting" => matches!(
|
||||
state,
|
||||
"running" | "failed" | "cancelled" | "expired" | "unknown"
|
||||
),
|
||||
"running" | "verifying" => matches!(
|
||||
state,
|
||||
"verifying" | "succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
|
||||
),
|
||||
// Only a definitive completion of the already-running executor calls
|
||||
// this reconciliation path. No new effect is started from unknown.
|
||||
"unknown" | "awaiting_reboot" => matches!(
|
||||
state,
|
||||
"succeeded" | "failed" | "cancelled" | "unknown" | "awaiting_reboot"
|
||||
),
|
||||
_ => false,
|
||||
};
|
||||
ensure!(allowed, "INVALID_LOCAL_STATE_TRANSITION");
|
||||
let rev: i64 = db.query_row("UPDATE job_attempts SET state=?3,grant_id=?4,revision=revision+1 WHERE job=?1 AND attempt=?2 RETURNING revision",
|
||||
params![offer.job_id,offer.attempt_id,state,grant], |r| r.get(0))?;
|
||||
Ok(u64::try_from(rev)?)
|
||||
}
|
||||
pub fn result(offer: &JobOffer, revision: u64, grant: &str, outcome: Outcome) -> JobResult {
|
||||
JobResult {
|
||||
job_id: offer.job_id.clone(),
|
||||
attempt_id: offer.attempt_id.clone(),
|
||||
revision,
|
||||
state: outcome.state,
|
||||
grant_id: grant.into(),
|
||||
stdout: outcome.stdout,
|
||||
stderr: outcome.stderr,
|
||||
output_truncated: outcome.output_truncated,
|
||||
reboot_required: outcome.reboot_required,
|
||||
exit_code: outcome.exit_code,
|
||||
evidence_json: outcome.evidence.to_string(),
|
||||
error_code: outcome.error_code,
|
||||
}
|
||||
}
|
||||
pub fn no_start(state: &str, code: &str) -> Outcome {
|
||||
Outcome {
|
||||
state: state.into(),
|
||||
evidence: serde_json::json!({"not_started":true,"postconditions_passed":false}),
|
||||
..Outcome::unknown(code)
|
||||
}
|
||||
}
|
||||
pub fn execute(job: &JobPayload) -> Outcome {
|
||||
let result = match &job.action {
|
||||
Action::RunScript { .. } => rmm_privileged_helper::script(job),
|
||||
Action::RefreshInventory { .. } => rmm_agent_linux::collect().and_then(|i| {
|
||||
let bytes = serde_json::to_vec(&i)?;
|
||||
ensure!(bytes.len() <= job.max_output_bytes as usize, "INVENTORY_OUTPUT_LIMIT");
|
||||
Ok(Outcome { state: "succeeded".into(), stdout: bytes, stderr: vec![], exit_code: None,
|
||||
output_truncated: false, reboot_required: false,
|
||||
evidence: serde_json::json!({"postconditions_passed":true,"source_id":"linux.base","observed_at":chrono::Utc::now(),"completeness":"partial"}), error_code: String::new() })
|
||||
}),
|
||||
Action::QueryLogs { .. } => query_logs(job),
|
||||
_ => Err(anyhow::anyhow!("unsupported action")),
|
||||
};
|
||||
result.unwrap_or_else(|_| Outcome::unknown("EXECUTOR_EFFECT_UNKNOWN"))
|
||||
}
|
||||
|
||||
fn allowed_log_units() -> Result<Vec<rmm_agent_linux::actions::ServiceUnit>> {
|
||||
let path = std::env::var_os("RMM_LOG_UNITS_FILE")
|
||||
.ok_or_else(|| anyhow::anyhow!("LOG_ALLOWLIST_NOT_CONFIGURED"))?;
|
||||
let path = std::path::Path::new(&path);
|
||||
rmm_privileged_helper::private_path(path, unsafe { libc::geteuid() }, false)?;
|
||||
ensure!(
|
||||
std::fs::metadata(path)?.len() <= 32768,
|
||||
"LOG_ALLOWLIST_SIZE"
|
||||
);
|
||||
let units: Vec<String> = serde_json::from_slice(&std::fs::read(path)?)?;
|
||||
ensure!(
|
||||
!units.is_empty() && units.len() <= 128,
|
||||
"LOG_ALLOWLIST_SIZE"
|
||||
);
|
||||
units
|
||||
.iter()
|
||||
.map(|u| rmm_agent_linux::actions::ServiceUnit::new(u))
|
||||
.collect()
|
||||
}
|
||||
pub fn logs_available() -> bool {
|
||||
allowed_log_units().is_ok()
|
||||
&& rmm_agent_linux::inventory::capabilities()
|
||||
.get("logs.journald")
|
||||
.is_some_and(|o| o.data == Some(true))
|
||||
}
|
||||
fn log_request(job: &JobPayload) -> Result<rmm_agent_linux::inventory::JournalRequest> {
|
||||
log_request_for(job, &allowed_log_units()?)
|
||||
}
|
||||
fn log_request_for(
|
||||
job: &JobPayload,
|
||||
allowed: &[rmm_agent_linux::actions::ServiceUnit],
|
||||
) -> Result<rmm_agent_linux::inventory::JournalRequest> {
|
||||
let Action::QueryLogs {
|
||||
unit,
|
||||
since,
|
||||
until,
|
||||
max_lines,
|
||||
} = &job.action
|
||||
else {
|
||||
anyhow::bail!("NOT_LOG_QUERY")
|
||||
};
|
||||
ensure!(job.max_runtime_seconds >= 20, "LOG_RUNTIME_TOO_SHORT");
|
||||
rmm_agent_linux::inventory::JournalRequest::new(
|
||||
rmm_agent_linux::actions::ServiceUnit::new(unit)?,
|
||||
allowed,
|
||||
u64::try_from(since.timestamp())?,
|
||||
u64::try_from(until.timestamp())?,
|
||||
*max_lines as usize,
|
||||
(job.max_output_bytes as usize).min(256 * 1024),
|
||||
)?
|
||||
.precise_window(*since, *until)
|
||||
}
|
||||
fn query_logs(job: &JobPayload) -> Result<Outcome> {
|
||||
let observation = rmm_agent_linux::inventory::query_journal(&log_request(job)?);
|
||||
let valid = observation.quality == rmm_agent_linux::inventory::Quality::Valid;
|
||||
let error = observation.error_code.clone().unwrap_or_default();
|
||||
let mut value = serde_json::to_value(&observation)?;
|
||||
while serde_json::to_vec(&value)?.len() > job.max_output_bytes as usize {
|
||||
let Some(lines) = value["data"]["json_lines"]
|
||||
.as_array_mut()
|
||||
.filter(|v| !v.is_empty())
|
||||
else {
|
||||
return Ok(Outcome {
|
||||
state: "failed".into(),
|
||||
..Outcome::unknown("LOG_RESULT_METADATA_EXCEEDS_OUTPUT_LIMIT")
|
||||
});
|
||||
};
|
||||
lines.remove(0);
|
||||
value["truncated"] = serde_json::json!(true);
|
||||
value["completeness"] = serde_json::json!("partial");
|
||||
value["error_code"] = serde_json::json!("LOG_SERIALIZED_OUTPUT_LIMIT");
|
||||
}
|
||||
Ok(Outcome {
|
||||
state: if valid { "succeeded" } else { "failed" }.into(),
|
||||
stdout: serde_json::to_vec(&value)?,
|
||||
stderr: vec![],
|
||||
exit_code: None,
|
||||
output_truncated: value["truncated"] == true,
|
||||
reboot_required: false,
|
||||
evidence: serde_json::json!({"postconditions_passed":valid,"source_id":observation.source_id,
|
||||
"observed_at":value["observed_at"],"quality":value["quality"],"completeness":value["completeness"],
|
||||
"cursor_status":"not_advanced_no_ack_contract","provider_error_code":error}),
|
||||
error_code: if valid {
|
||||
String::new()
|
||||
} else if error.is_empty() {
|
||||
"LOG_PROVIDER_UNAVAILABLE".into()
|
||||
} else {
|
||||
error
|
||||
},
|
||||
})
|
||||
}
|
||||
pub fn permit_request(offer: &JobOffer, permit: &ExecutionPermit) -> Result<Request> {
|
||||
Ok(Request {
|
||||
job_bytes: offer.payload_bytes.clone(),
|
||||
permit_bytes: permit.payload_bytes.clone(),
|
||||
signature: permit.signature.clone(),
|
||||
plan_hash: offer.plan_hash.clone(),
|
||||
authorization_id: Uuid::parse_str(&offer.authorization_id)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// With no backend SyncResponse contract, a reconnect/restart cannot prove an
|
||||
/// earlier reservation never started. Telemetry and evidence replay continue;
|
||||
/// all new effects stop. Do not paper over this with a new start request.
|
||||
#[cfg(test)]
|
||||
pub fn begin_session(db: &Connection) -> Result<bool> {
|
||||
let inserted = db.execute(
|
||||
"INSERT OR IGNORE INTO safety(key,value) VALUES('connected','1')",
|
||||
[],
|
||||
)?;
|
||||
Ok(inserted == 1)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub fn uncertain(db: &Connection) -> Result<Vec<(JobOffer, String)>> {
|
||||
let mut statement = db.prepare(
|
||||
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running')",
|
||||
)?;
|
||||
let bytes = statement
|
||||
.query_map([], |r| {
|
||||
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
bytes
|
||||
.into_iter()
|
||||
.map(|(bytes, grant)| Ok((JobOffer::decode(bytes.as_slice())?, grant)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage;
|
||||
fn fixture() -> (std::path::PathBuf, Binding, JobOffer) {
|
||||
let path = std::env::temp_dir().join(format!("rmm-agent-jobs-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&path).unwrap();
|
||||
let binding = Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 2,
|
||||
public_key_hex: "00".repeat(32),
|
||||
};
|
||||
let now = chrono::Utc::now();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 2,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::RefreshInventory {
|
||||
scope: "linux.base".into(),
|
||||
},
|
||||
execution_identity: "agent_service".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::minutes(10),
|
||||
max_runtime_seconds: 10,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let bytes = rmm_domain::jobs::encode(&job).unwrap();
|
||||
let offer = JobOffer {
|
||||
job_id: job.job_id.to_string(),
|
||||
attempt_id: job.attempt_id.to_string(),
|
||||
authorization_id: Uuid::new_v4().to_string(),
|
||||
plan_hash: hash(b"plan"),
|
||||
job_payload_hash: hash(&bytes),
|
||||
payload_bytes: bytes,
|
||||
};
|
||||
(path, binding, offer)
|
||||
}
|
||||
#[test]
|
||||
fn sqlite_reopen_offer_dedup_and_no_new_permit_after_ambiguity() {
|
||||
let (path, binding, offer) = fixture();
|
||||
let mut db = storage::open(&path).unwrap();
|
||||
let job = binding.job(&offer.payload_bytes).unwrap();
|
||||
if unsafe { libc::geteuid() } == 0 {
|
||||
assert!(!supported(&job));
|
||||
assert!(accept(&db, &binding, &offer).is_err());
|
||||
} else {
|
||||
assert!(accept(&db, &binding, &offer).unwrap());
|
||||
assert!(!accept(&db, &binding, &offer).unwrap());
|
||||
let mut changed = offer.clone();
|
||||
changed.plan_hash = hash(b"different plan");
|
||||
assert!(accept(&db, &binding, &changed).is_err());
|
||||
assert!(begin_session(&db).unwrap());
|
||||
let envelope = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
sequence: storage::next_sequence(&mut db).unwrap(),
|
||||
payload: Some(Payload::JobAccepted(JobAccepted {
|
||||
job_id: offer.job_id.clone(),
|
||||
attempt_id: offer.attempt_id.clone(),
|
||||
job_payload_hash: offer.job_payload_hash.clone(),
|
||||
journal_proof: "durable".into(),
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
event(&db, &envelope).unwrap();
|
||||
reserve_request(&db, &offer).unwrap();
|
||||
assert!(reserve_request(&db, &offer).is_err());
|
||||
assert!(next(&db).unwrap().is_none());
|
||||
drop(db);
|
||||
db = storage::open(&path).unwrap();
|
||||
assert!(!begin_session(&db).unwrap());
|
||||
assert_eq!(uncertain(&db).unwrap().len(), 1);
|
||||
assert_eq!(events(&db).unwrap(), vec![envelope.clone()]);
|
||||
assert_eq!(transition(&db, &offer, "unknown", "").unwrap(), 1);
|
||||
assert!(next(&db).unwrap().is_none());
|
||||
assert!(!accept(&db, &binding, &offer).unwrap());
|
||||
ack(&mut db, &envelope.message_id).unwrap();
|
||||
assert!(events(&db).unwrap().is_empty());
|
||||
assert!(next(&db).unwrap().is_none());
|
||||
}
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn cancellation_expiry_and_atomic_event_rollback() {
|
||||
let (path, binding, offer) = fixture();
|
||||
let mut db = storage::open(&path).unwrap();
|
||||
if unsafe { libc::geteuid() } != 0 {
|
||||
accept(&db, &binding, &offer).unwrap();
|
||||
db.execute("UPDATE job_attempts SET cancel_requested=1", [])
|
||||
.unwrap();
|
||||
assert!(reserve_request(&db, &offer).is_err());
|
||||
let cancelled = no_start("cancelled", "CANCELLED");
|
||||
assert_eq!(cancelled.evidence["not_started"], true);
|
||||
let tx = db.transaction().unwrap();
|
||||
assert_eq!(transition(&tx, &offer, "cancelled", "").unwrap(), 1);
|
||||
let e = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(result(&offer, 1, "", cancelled))),
|
||||
..Default::default()
|
||||
};
|
||||
event(&tx, &e).unwrap();
|
||||
tx.rollback().unwrap();
|
||||
assert!(events(&db).unwrap().is_empty());
|
||||
assert!(next(&db).unwrap().is_none());
|
||||
let expired = no_start("expired", "DEADLINE_EXPIRED");
|
||||
assert_eq!(expired.evidence["not_started"], true);
|
||||
assert_eq!(transition(&db, &offer, "expired", "").unwrap(), 1);
|
||||
assert!(next(&db).unwrap().is_none());
|
||||
} else {
|
||||
assert!(accept(&db, &binding, &offer).is_err());
|
||||
}
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn network_process_never_executes_as_accidental_root() {
|
||||
let (path, binding, offer) = fixture();
|
||||
let mut job = binding.job(&offer.payload_bytes).unwrap();
|
||||
assert!(!supported_as(&job, 0, false));
|
||||
assert!(!supported_as(&job, 0, true));
|
||||
job.execution_identity = "root".into();
|
||||
assert!(!supported_as(&job, 1000, false));
|
||||
assert!(!supported_as(&job, 0, true));
|
||||
job.action = Action::Reboot {
|
||||
reason: "explicit root only".into(),
|
||||
reconnect_timeout_seconds: 60,
|
||||
dependency_group: "test".into(),
|
||||
};
|
||||
assert!(supported_as(&job, 1000, true));
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn scheduled_future_job_does_not_starve_ready_job() {
|
||||
let (path, binding, offer) = fixture();
|
||||
let db = storage::open(&path).unwrap();
|
||||
if unsafe { libc::geteuid() } != 0 {
|
||||
let mut future = offer.clone();
|
||||
let mut job: JobPayload = serde_json::from_slice(&future.payload_bytes).unwrap();
|
||||
job.job_id = Uuid::new_v4();
|
||||
job.attempt_id = Uuid::new_v4();
|
||||
job.not_before = chrono::Utc::now() + chrono::Duration::minutes(1);
|
||||
future.job_id = job.job_id.to_string();
|
||||
future.attempt_id = job.attempt_id.to_string();
|
||||
future.payload_bytes = rmm_domain::jobs::encode(&job).unwrap();
|
||||
future.job_payload_hash = hash(&future.payload_bytes);
|
||||
accept(&db, &binding, &future).unwrap();
|
||||
accept(&db, &binding, &offer).unwrap();
|
||||
assert_eq!(next(&db).unwrap().unwrap().job_id, offer.job_id);
|
||||
}
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn live_logs_enforce_actual_action_window_allowlist_limits_and_runtime() {
|
||||
let (path, binding, offer) = fixture();
|
||||
let mut job = binding.job(&offer.payload_bytes).unwrap();
|
||||
let unit = rmm_agent_linux::actions::ServiceUnit::new("rmm-test.service").unwrap();
|
||||
let now = chrono::Utc::now();
|
||||
job.max_runtime_seconds = 20;
|
||||
job.action = Action::QueryLogs {
|
||||
unit: unit.as_str().into(),
|
||||
since: now - chrono::Duration::minutes(1),
|
||||
until: now,
|
||||
max_lines: 10,
|
||||
};
|
||||
assert!(log_request_for(&job, &[]).is_err());
|
||||
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_ok());
|
||||
job.max_runtime_seconds = 10;
|
||||
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
|
||||
job.max_runtime_seconds = 20;
|
||||
job.action = Action::QueryLogs {
|
||||
unit: unit.as_str().into(),
|
||||
since: now - chrono::Duration::hours(25),
|
||||
until: now,
|
||||
max_lines: 10,
|
||||
};
|
||||
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
|
||||
job.action = Action::QueryLogs {
|
||||
unit: unit.as_str().into(),
|
||||
since: now - chrono::Duration::minutes(1),
|
||||
until: now,
|
||||
max_lines: 1001,
|
||||
};
|
||||
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn refresh_is_a_new_measurement_with_real_observation_time() {
|
||||
let (path, binding, offer) = fixture();
|
||||
let job = binding.job(&offer.payload_bytes).unwrap();
|
||||
let before = chrono::Utc::now();
|
||||
let outcome = execute(&job);
|
||||
assert_eq!(outcome.state, "succeeded");
|
||||
let observed =
|
||||
chrono::DateTime::parse_from_rfc3339(outcome.evidence["observed_at"].as_str().unwrap())
|
||||
.unwrap();
|
||||
assert!(observed >= before);
|
||||
assert!(
|
||||
serde_json::from_slice::<rmm_domain::Inventory>(&outcome.stdout)
|
||||
.unwrap()
|
||||
.validate()
|
||||
);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,805 @@
|
||||
mod apt;
|
||||
mod bootstrap;
|
||||
mod certificate_rotation;
|
||||
mod helper_sync;
|
||||
mod https_channel;
|
||||
mod jobs;
|
||||
mod reconcile;
|
||||
#[cfg(test)]
|
||||
mod runtime_tests;
|
||||
mod storage;
|
||||
mod transport;
|
||||
mod wire_channel;
|
||||
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::Utc;
|
||||
use openssl::{
|
||||
ec::{EcGroup, EcKey},
|
||||
hash::MessageDigest,
|
||||
nid::Nid,
|
||||
pkey::PKey,
|
||||
x509::{X509NameBuilder, X509Req},
|
||||
};
|
||||
use prost::Message;
|
||||
use rmm_protocol::*;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
||||
address: std::net::SocketAddr,
|
||||
ca: Vec<u8>,
|
||||
id: String,
|
||||
boot: String,
|
||||
instance: String,
|
||||
db: rusqlite::Connection,
|
||||
started: Instant,
|
||||
clock: rmm_privileged_helper::TrustedClock,
|
||||
binding: Option<rmm_privileged_helper::Binding>,
|
||||
ledger: rusqlite::Connection,
|
||||
completed_tx: tokio::sync::mpsc::Sender<Completion>,
|
||||
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
||||
inventory_tx: tokio::sync::mpsc::Sender<Result<(Vec<u8>, i64)>>,
|
||||
inventory_rx: tokio::sync::mpsc::Receiver<Result<(Vec<u8>, i64)>>,
|
||||
inventory_inflight: bool,
|
||||
sync: Option<reconcile::ClientSync>,
|
||||
sync_clock: C,
|
||||
helpers: Option<helper_sync::Helpers>,
|
||||
prepared: Option<reconcile::PreparedSync>,
|
||||
fence: Arc<EffectFence>,
|
||||
mode: wire_channel::Mode,
|
||||
certificates: certificate_rotation::RotationStore,
|
||||
rotation_policy: certificate_rotation::RotationPolicy,
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
generation: AtomicU64,
|
||||
deadline: AtomicI64,
|
||||
root: AtomicBool,
|
||||
}
|
||||
fn report_closed_gate(error: &anyhow::Error) {
|
||||
let text = error.to_string();
|
||||
let code = if text.len() <= 96
|
||||
&& text
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
{
|
||||
text.as_str()
|
||||
} else {
|
||||
"SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE"
|
||||
};
|
||||
eprintln!("agent effect gate closed: {code}");
|
||||
}
|
||||
impl EffectFence {
|
||||
fn close(&self) {
|
||||
self.generation.store(0, Ordering::SeqCst);
|
||||
self.root.store(false, Ordering::SeqCst);
|
||||
}
|
||||
fn allows(&self, generation: u64, root: bool) -> bool {
|
||||
self.generation.load(Ordering::SeqCst) == generation
|
||||
&& generation > 0
|
||||
&& Utc::now().timestamp_millis() < self.deadline.load(Ordering::SeqCst)
|
||||
&& (!root || self.root.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
type Completion = (
|
||||
JobOffer,
|
||||
rmm_domain::jobs::ExecutionPermitPayload,
|
||||
rmm_privileged_helper::Outcome,
|
||||
);
|
||||
impl<C: reconcile::SyncClock> Agent<C> {
|
||||
fn close_gate(&mut self) {
|
||||
self.fence.close();
|
||||
if let Some(sync) = &mut self.sync {
|
||||
sync.close();
|
||||
}
|
||||
self.prepared = None;
|
||||
}
|
||||
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
|
||||
Ok(reconcile::Channel {
|
||||
connection_generation: generation,
|
||||
boot_id: Uuid::parse_str(&self.boot)?,
|
||||
})
|
||||
}
|
||||
fn ready(&self, generation: u64, root: bool) -> bool {
|
||||
self.channel(generation).is_ok_and(|c| {
|
||||
self.sync
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.can_start(&c, root, &self.sync_clock))
|
||||
})
|
||||
}
|
||||
async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
self.fence.close();
|
||||
let helpers = self.helpers.clone();
|
||||
let snapshot = tokio::task::spawn_blocking(move || match helpers {
|
||||
Some(h) => h.snapshot(generation),
|
||||
None => Ok(None),
|
||||
})
|
||||
.await??;
|
||||
let channel = self.channel(generation)?;
|
||||
let Some(sync) = &mut self.sync else {
|
||||
return Ok(());
|
||||
};
|
||||
let prepared = sync.prepare(
|
||||
&mut self.db,
|
||||
&self.ledger,
|
||||
channel,
|
||||
snapshot.as_ref(),
|
||||
&self.sync_clock,
|
||||
)?;
|
||||
wire.send(&prepared.envelope).await?;
|
||||
self.prepared = Some(prepared);
|
||||
Ok(())
|
||||
}
|
||||
async fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
||||
let prepared = self.prepared.take().context("UNSOLICITED_SYNC_RESPONSE")?;
|
||||
let sync = self.sync.as_mut().context("SYNC_BINDING_REQUIRED")?;
|
||||
let applied = sync.apply(&mut self.db, &response, &self.sync_clock)?;
|
||||
sync.apply_consumption(&mut self.ledger, &self.sync_clock)?;
|
||||
let reason = if applied
|
||||
.reason
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b == b'_')
|
||||
{
|
||||
applied.reason.as_str()
|
||||
} else {
|
||||
"RECONCILIATION_REQUIRED"
|
||||
};
|
||||
eprintln!(
|
||||
"agent reconciliation committed: decision={}, receipts={}, reason={}",
|
||||
applied.decision_id,
|
||||
applied.acknowledged_message_ids.len(),
|
||||
reason
|
||||
);
|
||||
if let Some(instruction) = applied.helper_instruction {
|
||||
let helpers = self
|
||||
.helpers
|
||||
.clone()
|
||||
.context("HELPER_SYNC_CLIENT_REQUIRED")?;
|
||||
let request = prepared.request.clone();
|
||||
let snapshot =
|
||||
tokio::task::spawn_blocking(move || helpers.apply(&instruction.response, &request))
|
||||
.await??
|
||||
.context("HELPER_SYNC_ACK_REQUIRED")?;
|
||||
if self.ready(generation, false) {
|
||||
self.sync
|
||||
.as_mut()
|
||||
.context("SYNC_REQUIRED")?
|
||||
.confirm_helper(&snapshot)?;
|
||||
}
|
||||
}
|
||||
if self.ready(generation, false) {
|
||||
let deadline = self
|
||||
.sync
|
||||
.as_ref()
|
||||
.and_then(|s| s.validity_deadline())
|
||||
.context("SIGNED_LEASE_REQUIRED")?;
|
||||
self.fence
|
||||
.deadline
|
||||
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
||||
self.fence
|
||||
.root
|
||||
.store(self.ready(generation, true), Ordering::SeqCst);
|
||||
self.fence.generation.store(generation, Ordering::SeqCst);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn record(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
offer: &JobOffer,
|
||||
grant: &str,
|
||||
outcome: rmm_privileged_helper::Outcome,
|
||||
) -> Result<()> {
|
||||
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
|
||||
let tx = self.db.transaction()?;
|
||||
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
||||
envelope.payload = Some(Payload::JobResult(jobs::result(
|
||||
offer, revision, grant, outcome,
|
||||
)));
|
||||
jobs::event(&tx, &envelope)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
// A large replay backlog must not monopolize heartbeat/cancel handling.
|
||||
// On backpressure timeout, discard this channel rather than continuing
|
||||
// to use a websocket whose partially sent frame was cancelled.
|
||||
let events = jobs::events(&self.db)?;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
for mut event in events.into_iter().take(8) {
|
||||
event.connection_generation = generation;
|
||||
ws.send(&event).await?;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
fn permit(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
permit: ExecutionPermit,
|
||||
safe_session: bool,
|
||||
) -> Result<()> {
|
||||
let Some(offer) = jobs::requesting(&self.db)? else {
|
||||
anyhow::bail!("UNSOLICITED_PERMIT")
|
||||
};
|
||||
if !permit.denial_code.is_empty() {
|
||||
// A denial is not proof that an earlier reserved grant never existed.
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
"",
|
||||
rmm_privileged_helper::Outcome::unknown("PERMIT_DENIED_RECONCILIATION_REQUIRED"),
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
safe_session && self.clock.trusted(),
|
||||
"CLOCK_OR_SYNC_UNTRUSTED"
|
||||
);
|
||||
let binding = self
|
||||
.binding
|
||||
.as_ref()
|
||||
.context("job trust not provisioned")?
|
||||
.clone();
|
||||
let request = jobs::permit_request(&offer, &permit)?;
|
||||
let (job, grant) = binding.verify(&request, Utc::now())?;
|
||||
ensure!(
|
||||
self.ready(generation, job.execution_identity == "root"),
|
||||
"FENCED_SYNC_REQUIRED"
|
||||
);
|
||||
ensure!(
|
||||
jobs::supported_with_binding(&job, Some(&binding)),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
);
|
||||
rmm_privileged_helper::consume(&mut self.ledger, &grant)?;
|
||||
rmm_privileged_helper::save_start(&self.ledger, &grant, &request)?;
|
||||
let cancelled: bool = self.db.query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if cancelled {
|
||||
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
&grant.grant_id.to_string(),
|
||||
outcome.clone(),
|
||||
)?;
|
||||
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut envelope = self.envelope(
|
||||
generation,
|
||||
Payload::JobStateChanged(JobStateChanged::default()),
|
||||
)?;
|
||||
let tx = self.db.transaction()?;
|
||||
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
|
||||
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
||||
job_id: offer.job_id.clone(),
|
||||
attempt_id: offer.attempt_id.clone(),
|
||||
revision,
|
||||
state: "running".into(),
|
||||
grant_id: grant.grant_id.to_string(),
|
||||
}));
|
||||
jobs::event(&tx, &envelope)?;
|
||||
tx.commit()?;
|
||||
let completed = self.completed_tx.clone();
|
||||
let clock = self.clock.clone();
|
||||
let fence = self.fence.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Thread-pool scheduling and durable fsync may have consumed the
|
||||
// remaining start window. Recheck at the actual effect boundary.
|
||||
let outcome = if !fence.allows(generation, job.execution_identity == "root")
|
||||
|| !clock.trusted()
|
||||
|| binding.verify(&request, Utc::now()).is_err()
|
||||
{
|
||||
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
|
||||
} else if job.execution_identity == "root" {
|
||||
apt::invoke(&binding, &job, &request).unwrap_or_else(|_| {
|
||||
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
|
||||
})
|
||||
} else {
|
||||
jobs::execute(&job)
|
||||
};
|
||||
let _ = completed.blocking_send((offer, grant, outcome));
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
|
||||
Ok(Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: self.id.clone(),
|
||||
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
connection_generation: generation,
|
||||
boot_id: self.boot.clone(),
|
||||
sequence: storage::next_sequence(&mut self.db)?,
|
||||
observed_at: Utc::now().timestamp_millis(),
|
||||
payload: Some(payload),
|
||||
})
|
||||
}
|
||||
fn capabilities(&self) -> Vec<String> {
|
||||
let mut capabilities = vec!["linux.base/1".into(), "heartbeat/1".into()];
|
||||
if self.binding.is_some() {
|
||||
capabilities.push("rmm.sync.v1".into());
|
||||
}
|
||||
if let Some(binding) = &self.binding
|
||||
&& unsafe { libc::geteuid() } != 0
|
||||
{
|
||||
if rmm_agent_linux::actions::Interpreter::PosixSh.available() {
|
||||
capabilities.push("run_script".into());
|
||||
}
|
||||
capabilities.push("refresh_inventory".into());
|
||||
if jobs::logs_available() {
|
||||
capabilities.push("query_logs".into());
|
||||
}
|
||||
capabilities.extend(apt::capabilities(binding));
|
||||
capabilities.extend(helper_sync::general_capabilities(binding));
|
||||
}
|
||||
capabilities.sort();
|
||||
capabilities.dedup();
|
||||
capabilities
|
||||
}
|
||||
fn queue_inventory(&mut self) {
|
||||
if self.inventory_inflight || !storage::passive_collection_ready(&self.db).unwrap_or(false)
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.inventory_inflight = true;
|
||||
let sender = self.inventory_tx.clone();
|
||||
let extended = env::var("RMM_EXTENDED_INVENTORY").unwrap_or_else(|_| "1".into()) == "1";
|
||||
let binding = self.binding.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = rmm_agent_linux::wire::snapshot(extended).and_then(|mut inventory| {
|
||||
if extended {
|
||||
inventory["observations"]
|
||||
.as_array_mut()
|
||||
.context("INVENTORY_OBSERVATIONS")?
|
||||
.push(apt::planning_observation(binding.as_ref()));
|
||||
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
|
||||
}
|
||||
let bytes = serde_json::to_vec(&inventory)?;
|
||||
ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT");
|
||||
Ok((bytes, Utc::now().timestamp_millis()))
|
||||
});
|
||||
let _ = sender.blocking_send(result);
|
||||
});
|
||||
}
|
||||
fn inventory_envelope(
|
||||
&mut self,
|
||||
generation: u64,
|
||||
data: Vec<u8>,
|
||||
observed_at: i64,
|
||||
) -> Result<Envelope> {
|
||||
let mut envelope = self.envelope(
|
||||
generation,
|
||||
Payload::InventorySnapshot(InventorySnapshot {
|
||||
snapshot_id: Uuid::new_v4().to_string(),
|
||||
scope: "linux.base".into(),
|
||||
schema_version: 1,
|
||||
chunk_index: 0,
|
||||
chunk_count: 1,
|
||||
sha256: Sha256::digest(&data).to_vec(),
|
||||
data,
|
||||
completeness: "partial".into(),
|
||||
}),
|
||||
)?;
|
||||
envelope.observed_at = observed_at;
|
||||
Ok(envelope)
|
||||
}
|
||||
async fn session(&mut self) -> Result<()> {
|
||||
let result = self.session_inner().await;
|
||||
self.close_gate();
|
||||
result
|
||||
}
|
||||
async fn session_inner(&mut self) -> Result<()> {
|
||||
self.close_gate();
|
||||
self.boot = rmm_agent_linux::boot_id()?;
|
||||
if self.helpers.is_some() {
|
||||
self.helpers = self
|
||||
.binding
|
||||
.as_ref()
|
||||
.map(helper_sync::Helpers::configured)
|
||||
.transpose()?;
|
||||
}
|
||||
let advertised = self.capabilities();
|
||||
let hello = self.envelope(
|
||||
0,
|
||||
Payload::ClientHello(ClientHello {
|
||||
agent_version: env!("CARGO_PKG_VERSION").into(),
|
||||
capabilities: advertised.clone(),
|
||||
}),
|
||||
)?;
|
||||
let identity = self.certificates.current(Utc::now().timestamp())?;
|
||||
let mut ws =
|
||||
wire_channel::connect(self.mode, self.address, &self.ca, &identity, hello).await?;
|
||||
let reply = tokio::time::timeout(Duration::from_secs(20), ws.receive()).await??;
|
||||
ensure!(
|
||||
reply.protocol_major == 1
|
||||
&& reply.protocol_minor == 0
|
||||
&& reply.agent_id == self.id
|
||||
&& reply.enrollment_generation
|
||||
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
"invalid server hello"
|
||||
);
|
||||
let Some(Payload::ServerHello(server)) = reply.payload else {
|
||||
anyhow::bail!("server hello required")
|
||||
};
|
||||
let generation = server.connection_generation;
|
||||
ensure!(
|
||||
generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32,
|
||||
"unsupported limits"
|
||||
);
|
||||
if let Some(mut pending) = storage::pending(&self.db)? {
|
||||
// Generation is transport fencing, not part of durable message identity.
|
||||
pending.connection_generation = generation;
|
||||
jobs::event(&self.db, &pending)?;
|
||||
storage::clear_pending(&self.db)?;
|
||||
}
|
||||
self.queue_inventory();
|
||||
self.replay(&ws, generation).await?;
|
||||
let mut needs_sync = false;
|
||||
if self.sync.is_some()
|
||||
&& let Err(error) = self.start_sync(&ws, generation).await
|
||||
{
|
||||
report_closed_gate(&error);
|
||||
self.close_gate();
|
||||
needs_sync = true;
|
||||
}
|
||||
eprintln!("authenticated channel established; inventory queued for durable acknowledgment");
|
||||
let mut inventory_at = Instant::now();
|
||||
let mut heartbeat_at = Instant::now();
|
||||
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
||||
let mut sync_at = Instant::now();
|
||||
let mut rotation_at = Instant::now();
|
||||
let mut capability_at = Instant::now();
|
||||
let mut first_heartbeat = true;
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(result) = self.inventory_rx.recv() => {
|
||||
self.inventory_inflight = false;
|
||||
if let Ok((data,observed_at)) = result {
|
||||
let inventory = self.inventory_envelope(generation,data,observed_at)?;
|
||||
jobs::event(&self.db,&inventory)?;
|
||||
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
|
||||
}
|
||||
message = ws.receive() => {
|
||||
let message = message?;
|
||||
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
|
||||
match message.payload {
|
||||
Some(Payload::Ack(ack)) => {
|
||||
use rusqlite::OptionalExtension;
|
||||
let pending:Option<Vec<u8>>=self.db.query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
|
||||
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
|
||||
jobs::ack(&mut self.db, &ack.message_id)?;
|
||||
},
|
||||
Some(Payload::SyncResponse(response))=>{if let Err(error)=self.apply_sync(generation,response).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}sync_at=Instant::now();},
|
||||
Some(Payload::JobOffer(offer)) => {
|
||||
let binding = self.binding.as_ref().context("job trust not provisioned")?.clone();
|
||||
let accepted = self.envelope(generation, Payload::JobAccepted(JobAccepted {
|
||||
job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(),
|
||||
job_payload_hash: offer.job_payload_hash.clone(), journal_proof: rmm_domain::jobs::hash(&offer.encode_to_vec()),
|
||||
}))?;
|
||||
// Clone before borrowing the transaction; acceptance and the
|
||||
// durable ACK event commit together, before any wire ACK.
|
||||
let tx = self.db.transaction()?;
|
||||
let fresh=jobs::accept(&tx, &binding, &offer)?;
|
||||
jobs::event(&tx, &accepted)?;
|
||||
tx.commit()?;
|
||||
needs_sync|=fresh;self.replay(&ws, generation).await?;
|
||||
}
|
||||
Some(Payload::ExecutionPermit(permit)) => self.permit(generation, permit, self.ready(generation,false))?,
|
||||
Some(Payload::CancelRequest(cancel)) => {
|
||||
let row: Option<(Vec<u8>, String, String)> = {
|
||||
use rusqlite::OptionalExtension;
|
||||
self.db.query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?
|
||||
};
|
||||
if let Some((bytes,state,grant)) = row {
|
||||
self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
||||
if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; }
|
||||
// Running script cancellation is not confirmed by an
|
||||
// inbound request. Its bounded executor reports reality.
|
||||
}
|
||||
let ack = self.envelope(generation, Payload::Ack(Ack { message_id: message.message_id }))?;
|
||||
ws.send(&ack).await?;
|
||||
}
|
||||
_ => anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE"),
|
||||
}
|
||||
}
|
||||
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
|
||||
// Persist the wire result before releasing a consumption lock.
|
||||
// A crash in between is fail-closed, not a second execution.
|
||||
let state = outcome.state.clone();
|
||||
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
|
||||
let serialized = serde_json::to_vec(&outcome)?;
|
||||
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
|
||||
let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?;
|
||||
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
|
||||
if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); }
|
||||
needs_sync=true;self.replay(&ws, generation).await?;
|
||||
if refresh_capabilities {return Ok(());}
|
||||
}
|
||||
_ = tick.tick() => {
|
||||
self.replay(&ws, generation).await?;
|
||||
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
||||
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
||||
if self.prepared.is_some() && !busy && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
||||
if self.prepared.is_none() && self.sync.is_some() && !busy && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
||||
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
||||
}
|
||||
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
|
||||
&& self.clock.trusted() && let Some(offer) = jobs::next(&self.db)? {
|
||||
let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?;
|
||||
if Utc::now() >= job.start_before {
|
||||
self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?;
|
||||
} else if !self.ready(generation,job.execution_identity=="root") {
|
||||
needs_sync=true;
|
||||
} else if Utc::now() >= job.not_before {
|
||||
// Reserve before send. A lost response must never cause
|
||||
// a second permit request for this attempt.
|
||||
jobs::reserve_request(&self.db, &offer)?;
|
||||
let request = self.envelope(generation, Payload::ExecutionPermitRequest(ExecutionPermitRequest {
|
||||
job_id: offer.job_id, attempt_id: offer.attempt_id,
|
||||
job_payload_hash: offer.job_payload_hash, assignment_version: job.assignment_version,
|
||||
}))?;
|
||||
ws.send(&request).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
|
||||
first_heartbeat = false;
|
||||
let heartbeat = self.envelope(
|
||||
generation,
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
}),
|
||||
)?;
|
||||
ws.send(&heartbeat).await?;
|
||||
if ws.uses_https() {
|
||||
let poll = self.envelope(
|
||||
generation,
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
}),
|
||||
)?;
|
||||
ws.poll(poll).await?;
|
||||
}
|
||||
heartbeat_at = Instant::now();
|
||||
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
}
|
||||
// Planning pins must be refreshed before the 90-second stored-data
|
||||
// planning limit. One inflight blocking collector bounds pressure;
|
||||
// heartbeats never perform this metadata work.
|
||||
if inventory_at.elapsed() >= Duration::from_secs(60) {
|
||||
self.queue_inventory();
|
||||
inventory_at = Instant::now();
|
||||
}
|
||||
if rotation_at.elapsed() >= Duration::from_secs(30) {
|
||||
rotation_at = Instant::now();
|
||||
if self
|
||||
.certificates
|
||||
.due(self.rotation_policy, Utc::now().timestamp())?
|
||||
{
|
||||
self.close_gate();
|
||||
drop(ws);
|
||||
let rotated = self
|
||||
.certificates
|
||||
.renew(self.address, Utc::now().timestamp())
|
||||
.await?;
|
||||
ensure!(
|
||||
rotated.needs_reconnect
|
||||
&& rotated.enrollment_generation
|
||||
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
"ROTATION_IDENTITY_CHANGED"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enrollment(dir: &Path, address: std::net::SocketAddr, ca: &[u8]) -> Result<String> {
|
||||
let identity = dir.join("identity.json");
|
||||
if identity.exists() {
|
||||
let response: EnrollmentResponse = serde_json::from_slice(&fs::read(identity)?)?;
|
||||
Uuid::parse_str(&response.agent_id)?;
|
||||
return Ok(response.agent_id);
|
||||
}
|
||||
let key_path = dir.join("device.key");
|
||||
let key = if key_path.exists() {
|
||||
PKey::private_key_from_pem(&fs::read(&key_path)?)?
|
||||
} else {
|
||||
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
|
||||
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
|
||||
storage::create_secret(&key_path, &key.private_key_to_pem_pkcs8()?)?;
|
||||
key
|
||||
};
|
||||
let mut csr = X509Req::builder()?;
|
||||
let mut name = X509NameBuilder::new()?;
|
||||
name.append_entry_by_text("CN", "new-linux-device")?;
|
||||
csr.set_subject_name(&name.build())?;
|
||||
csr.set_pubkey(&key)?;
|
||||
csr.sign(&key, MessageDigest::sha256())?;
|
||||
let token_path = env::var("RMM_ENROLLMENT_TOKEN_FILE")
|
||||
.context("enrollment token file required for first registration")?;
|
||||
let token = fs::read_to_string(token_path)?.trim().to_owned();
|
||||
ensure!(token.len() == 64, "invalid enrollment token format");
|
||||
let client = reqwest::Client::builder()
|
||||
.https_only(true)
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
||||
.resolve("rmm-dev.internal", address)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(Duration::from_secs(15))
|
||||
.build()?;
|
||||
let result = client
|
||||
.post(format!(
|
||||
"https://rmm-dev.internal:{}/agent/v1/enroll",
|
||||
address.port()
|
||||
))
|
||||
.json(&EnrollmentRequest {
|
||||
token,
|
||||
csr_pem: String::from_utf8(csr.build().to_pem()?)?,
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
ensure!(
|
||||
result.status().is_success(),
|
||||
"enrollment rejected (response details suppressed)"
|
||||
);
|
||||
let response: EnrollmentResponse = result.json().await?;
|
||||
let certificate = openssl::x509::X509::from_pem(response.certificate_pem.as_bytes())?;
|
||||
ensure!(
|
||||
certificate.public_key()?.public_eq(&key),
|
||||
"enrollment certificate key mismatch"
|
||||
);
|
||||
Uuid::parse_str(&response.agent_id)?;
|
||||
storage::create_secret(&dir.join("device.pem"), response.certificate_pem.as_bytes())?;
|
||||
storage::create_secret(&identity, &serde_json::to_vec(&response)?)?;
|
||||
Ok(response.agent_id)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args: Vec<_> = env::args_os().skip(1).collect();
|
||||
if args == [std::ffi::OsString::from("--help")] {
|
||||
println!(
|
||||
"rmm-agent [--help]\nEnvironment: RMM_AGENT_STATE, RMM_CA_FILE, RMM_CONNECT_ADDR (TCP only; TLS name rmm-dev.internal), RMM_TRANSPORT=auto|wss|https, RMM_ENROLLMENT_TOKEN_FILE (first enrollment), RMM_JOB_BINDING_FILE, RMM_JOB_SIGNING_KEY_BOOTSTRAP=1, RMM_HELPER_SOCKET, RMM_HELPER_POLICY_FILE, RMM_APT_HELPER_SOCKET, RMM_APT_HELPER_PROOF_FILE, RMM_LOG_UNITS_FILE, RMM_EXTENDED_INVENTORY=0|1 (default 1).\nDevelopment-only rotation test: RMM_AGENT_PROFILE=development and RMM_ROTATION_TEST_AFTER_SECONDS.\nNo argument or environment variable approves jobs or disables TLS/grant verification."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(args.is_empty(), "UNKNOWN_AGENT_ARGUMENT");
|
||||
let dir =
|
||||
PathBuf::from(env::var("RMM_AGENT_STATE").unwrap_or_else(|_| "devstate/agent".into()));
|
||||
storage::prepare_dir(&dir)?;
|
||||
let _lock = storage::lock(&dir)?;
|
||||
let address = env::var("RMM_CONNECT_ADDR")
|
||||
.unwrap_or_else(|_| "127.0.0.1:18443".into())
|
||||
.parse()?;
|
||||
let ca = fs::read(env::var("RMM_CA_FILE").unwrap_or_else(|_| "devstate/pki/ca.pem".into()))?;
|
||||
let id=enrollment(&dir,address,&ca).await.map_err(|_| anyhow::anyhow!("enrollment/identity initialization failed; check trusted CA, token validity and local state"))?;
|
||||
let mut binding = env::var("RMM_JOB_BINDING_FILE")
|
||||
.ok()
|
||||
.map(|path| {
|
||||
let binding =
|
||||
rmm_privileged_helper::read_binding(Path::new(&path), unsafe { libc::geteuid() })?;
|
||||
let identity: EnrollmentResponse =
|
||||
serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?;
|
||||
ensure!(
|
||||
binding.agent_id.to_string() == id
|
||||
&& binding.asset_id.to_string() == identity.asset_id,
|
||||
"BOOTSTRAP_IDENTITY_MISMATCH"
|
||||
);
|
||||
Ok::<_, anyhow::Error>(binding)
|
||||
})
|
||||
.transpose()?;
|
||||
let certificate_dir = dir.join("certificates");
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&certificate_dir,
|
||||
&ca,
|
||||
Uuid::parse_str(&id)?,
|
||||
binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
||||
)?;
|
||||
if !certificate_dir.join("current.json").try_exists()? {
|
||||
certificates.initialize(
|
||||
&fs::read(dir.join("device.pem"))?,
|
||||
&fs::read(dir.join("device.key"))?,
|
||||
Utc::now().timestamp(),
|
||||
)?;
|
||||
}
|
||||
if env::var("RMM_JOB_SIGNING_KEY_BOOTSTRAP").as_deref() == Ok("1") {
|
||||
bootstrap::provision_current(
|
||||
&dir,
|
||||
address,
|
||||
&ca,
|
||||
binding
|
||||
.as_mut()
|
||||
.context("bootstrap requires provisioned device binding")?,
|
||||
&certificates.current(Utc::now().timestamp())?,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let (completed_tx, completed_rx) = tokio::sync::mpsc::channel(1);
|
||||
let (inventory_tx, inventory_rx) = tokio::sync::mpsc::channel(1);
|
||||
let sync = binding
|
||||
.as_ref()
|
||||
.map(|b| reconcile::ClientSync::new(b.clone()));
|
||||
let helpers = binding
|
||||
.as_ref()
|
||||
.map(helper_sync::Helpers::configured)
|
||||
.transpose()?;
|
||||
let mut rotation_policy = certificate_rotation::RotationPolicy::default();
|
||||
if let Ok(after) = env::var("RMM_ROTATION_TEST_AFTER_SECONDS") {
|
||||
ensure!(
|
||||
env::var("RMM_AGENT_PROFILE").as_deref() == Ok("development"),
|
||||
"ROTATION_TEST_REQUIRES_DEVELOPMENT_PROFILE"
|
||||
);
|
||||
rotation_policy.begin_after_seconds = after.parse()?;
|
||||
ensure!(
|
||||
rotation_policy.begin_after_seconds > 0,
|
||||
"ROTATION_TEST_INTERVAL"
|
||||
);
|
||||
}
|
||||
let mode =
|
||||
wire_channel::Mode::parse(&env::var("RMM_TRANSPORT").unwrap_or_else(|_| "auto".into()))?;
|
||||
let mut agent = Agent {
|
||||
address,
|
||||
ca,
|
||||
id,
|
||||
boot: rmm_agent_linux::boot_id()?,
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db: storage::open(&dir)?,
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding,
|
||||
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() })?,
|
||||
completed_tx,
|
||||
completed_rx,
|
||||
inventory_tx,
|
||||
inventory_rx,
|
||||
inventory_inflight: false,
|
||||
sync,
|
||||
sync_clock: reconcile::NativeClock::default(),
|
||||
helpers,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode,
|
||||
certificates,
|
||||
rotation_policy,
|
||||
};
|
||||
let mut backoff = 1_u64;
|
||||
loop {
|
||||
let started = Instant::now();
|
||||
if let Err(error) = agent.session().await {
|
||||
report_closed_gate(&error);
|
||||
}
|
||||
agent.close_gate();
|
||||
eprintln!("channel unavailable; retrying with jitter (no jobs can execute)");
|
||||
if started.elapsed() > Duration::from_secs(60) {
|
||||
backoff = 1;
|
||||
}
|
||||
tokio::time::sleep(https_channel::retry_delay(backoff as u32)).await;
|
||||
backoff = (backoff + 1).min(6);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,400 @@
|
||||
//! Actual dispatcher/transport tests with a local mTLS fixture, no real server
|
||||
//! approval, root helper, package/service/reboot or target VM effects.
|
||||
use super::*;
|
||||
use openssl::{
|
||||
asn1::Asn1Time,
|
||||
bn::BigNum,
|
||||
ssl::{SslAcceptor, SslMethod, SslVerifyMode},
|
||||
x509::{
|
||||
X509, X509NameBuilder,
|
||||
extension::{BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectAlternativeName},
|
||||
},
|
||||
};
|
||||
use rmm_domain::reconciliation::ReconcileDecisionPayload;
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
net::TcpListener,
|
||||
};
|
||||
|
||||
struct Clock(bool);
|
||||
#[test]
|
||||
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
|
||||
let gate = EffectFence::default();
|
||||
assert!(!gate.allows(1, false));
|
||||
gate.deadline
|
||||
.store(Utc::now().timestamp_millis() + 10000, Ordering::SeqCst);
|
||||
gate.generation.store(1, Ordering::SeqCst);
|
||||
assert!(gate.allows(1, false));
|
||||
assert!(!gate.allows(2, false));
|
||||
assert!(!gate.allows(1, true));
|
||||
gate.root.store(true, Ordering::SeqCst);
|
||||
assert!(gate.allows(1, true));
|
||||
gate.deadline.store(0, Ordering::SeqCst);
|
||||
assert!(!gate.allows(1, false));
|
||||
gate.close();
|
||||
assert!(!gate.allows(1, true));
|
||||
}
|
||||
impl reconcile::SyncClock for Clock {
|
||||
fn now(&self) -> Result<chrono::DateTime<Utc>> {
|
||||
ensure!(self.0, "CLOCK_UNTRUSTED");
|
||||
Ok(Utc::now())
|
||||
}
|
||||
}
|
||||
fn certificate(
|
||||
name: &str,
|
||||
issuer: Option<(&X509, &PKey<openssl::pkey::Private>)>,
|
||||
server: bool,
|
||||
) -> (X509, PKey<openssl::pkey::Private>) {
|
||||
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap();
|
||||
let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap();
|
||||
let mut subject = X509NameBuilder::new().unwrap();
|
||||
subject.append_entry_by_text("CN", name).unwrap();
|
||||
let subject = subject.build();
|
||||
let mut b = X509::builder().unwrap();
|
||||
b.set_version(2).unwrap();
|
||||
b.set_serial_number(
|
||||
&BigNum::from_u32(rand::random_range(1..u32::MAX))
|
||||
.unwrap()
|
||||
.to_asn1_integer()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
b.set_subject_name(&subject).unwrap();
|
||||
b.set_issuer_name(issuer.map_or(subject.as_ref(), |(ca, _)| ca.subject_name()))
|
||||
.unwrap();
|
||||
b.set_pubkey(&key).unwrap();
|
||||
b.set_not_before(&Asn1Time::days_from_now(0).unwrap())
|
||||
.unwrap();
|
||||
b.set_not_after(&Asn1Time::days_from_now(1).unwrap())
|
||||
.unwrap();
|
||||
if issuer.is_none() {
|
||||
b.append_extension(BasicConstraints::new().critical().ca().build().unwrap())
|
||||
.unwrap();
|
||||
b.append_extension(KeyUsage::new().key_cert_sign().crl_sign().build().unwrap())
|
||||
.unwrap();
|
||||
} else {
|
||||
b.append_extension(BasicConstraints::new().build().unwrap())
|
||||
.unwrap();
|
||||
let mut u = ExtendedKeyUsage::new();
|
||||
if server {
|
||||
u.server_auth();
|
||||
} else {
|
||||
u.client_auth();
|
||||
}
|
||||
b.append_extension(u.build().unwrap()).unwrap();
|
||||
if server {
|
||||
let context = b.x509v3_context(issuer.map(|(ca, _)| ca.as_ref()), None);
|
||||
let san = SubjectAlternativeName::new()
|
||||
.dns(name)
|
||||
.build(&context)
|
||||
.unwrap();
|
||||
b.append_extension(san).unwrap();
|
||||
}
|
||||
}
|
||||
b.sign(issuer.map_or(&key, |(_, k)| k), MessageDigest::sha256())
|
||||
.unwrap();
|
||||
(b.build(), key)
|
||||
}
|
||||
fn reply(input: &Envelope, generation: u64, payload: Payload) -> Envelope {
|
||||
Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
connection_generation: generation,
|
||||
payload: Some(payload),
|
||||
..input.clone()
|
||||
}
|
||||
}
|
||||
struct Server {
|
||||
binding: rmm_privileged_helper::Binding,
|
||||
key: PKey<openssl::pkey::Private>,
|
||||
generation: u64,
|
||||
previous: Option<Uuid>,
|
||||
requests: usize,
|
||||
heartbeats: usize,
|
||||
forged: bool,
|
||||
}
|
||||
impl Server {
|
||||
fn response(&mut self, e: &Envelope) -> Envelope {
|
||||
match e.payload.as_ref().unwrap() {
|
||||
Payload::ClientHello(h) => {
|
||||
assert!(h.capabilities.iter().any(|c| c == "rmm.sync.v1"));
|
||||
reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::ServerHello(ServerHello {
|
||||
connection_generation: self.generation,
|
||||
max_message_bytes: rmm_domain::MAX_MESSAGE as u32,
|
||||
}),
|
||||
)
|
||||
}
|
||||
Payload::SyncRequest(r) => {
|
||||
assert!(r.journal_complete && r.attempts.is_empty());
|
||||
if let Some(previous) = self.previous {
|
||||
assert_eq!(r.previous_decision_id, previous.to_string());
|
||||
}
|
||||
let now = Utc::now();
|
||||
let d = ReconcileDecisionPayload {
|
||||
domain: "rmm.reconciliation-decision".into(),
|
||||
schema_version: 1,
|
||||
decision_id: Uuid::new_v4(),
|
||||
request_id: Uuid::parse_str(&r.request_id).unwrap(),
|
||||
request_hash: rmm_domain::jobs::hash(&r.encode_to_vec()),
|
||||
tenant_id: self.binding.tenant_id,
|
||||
asset_id: self.binding.asset_id,
|
||||
agent_id: self.binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
connection_generation: self.generation,
|
||||
boot_id: Uuid::parse_str(&e.boot_id).unwrap(),
|
||||
journal_id: Uuid::parse_str(&r.journal_id).unwrap(),
|
||||
checkpoint: r.checkpoint,
|
||||
helper_journal_id: None,
|
||||
helper_checkpoint: 0,
|
||||
issued_at: now,
|
||||
valid_until: now + chrono::Duration::seconds(60),
|
||||
ready: true,
|
||||
helper_ready: false,
|
||||
reason: "CONSISTENT".into(),
|
||||
attempts: vec![],
|
||||
acknowledged_message_ids: vec![],
|
||||
key_id: rmm_domain::jobs::hash(&self.key.raw_public_key().unwrap()),
|
||||
};
|
||||
let bytes = rmm_domain::jobs::encode(&d).unwrap();
|
||||
let mut signature = openssl::sign::Signer::new_without_digest(&self.key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&bytes)
|
||||
.unwrap();
|
||||
if self.forged {
|
||||
signature[0] ^= 1;
|
||||
} else {
|
||||
self.previous = Some(d.decision_id);
|
||||
}
|
||||
self.requests += 1;
|
||||
reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::SyncResponse(SyncResponse {
|
||||
decision_bytes: bytes,
|
||||
signature,
|
||||
}),
|
||||
)
|
||||
}
|
||||
Payload::Heartbeat(_) => {
|
||||
self.heartbeats += 1;
|
||||
reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::Ack(Ack {
|
||||
message_id: e.message_id.clone(),
|
||||
}),
|
||||
)
|
||||
}
|
||||
Payload::InventorySnapshot(_) => reply(
|
||||
e,
|
||||
self.generation,
|
||||
Payload::Ack(Ack {
|
||||
message_id: e.message_id.clone(),
|
||||
}),
|
||||
),
|
||||
_ => panic!("fixture received unexpected execution authority"),
|
||||
}
|
||||
}
|
||||
}
|
||||
fn http_body(stream: &mut impl Read) -> Vec<u8> {
|
||||
let mut head = vec![];
|
||||
while !head.ends_with(b"\r\n\r\n") {
|
||||
let mut b = [0; 1];
|
||||
stream.read_exact(&mut b).unwrap();
|
||||
head.push(b[0]);
|
||||
assert!(head.len() < 8192);
|
||||
}
|
||||
let text = String::from_utf8(head).unwrap();
|
||||
let length: usize = text
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case("content-length"))
|
||||
.unwrap()
|
||||
.1
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert!(length <= rmm_domain::MAX_MESSAGE);
|
||||
let mut bytes = vec![0; length];
|
||||
stream.read_exact(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
async fn run(mode: wire_channel::Mode, trusted: bool, forged: bool, rounds: u64) {
|
||||
let dir = PathBuf::from("/tmp/opencode").join(format!("runtime-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&dir).unwrap();
|
||||
let id = Uuid::new_v4();
|
||||
let (ca, ca_key) = certificate("runtime CA", None, false);
|
||||
let (server, server_key) = certificate("rmm-dev.internal", Some((&ca, &ca_key)), true);
|
||||
let (device, device_key) = certificate(&id.to_string(), Some((&ca, &ca_key)), false);
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let public = key.raw_public_key().unwrap();
|
||||
let binding = rmm_privileged_helper::Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
};
|
||||
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls_server()).unwrap();
|
||||
acceptor.set_certificate(&server).unwrap();
|
||||
acceptor.set_private_key(&server_key).unwrap();
|
||||
acceptor.cert_store_mut().add_cert(ca.clone()).unwrap();
|
||||
acceptor.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT);
|
||||
acceptor.set_session_id_context(b"rmm-runtime").unwrap();
|
||||
let acceptor = acceptor.build();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server_binding = binding.clone();
|
||||
let worker = std::thread::spawn(move || {
|
||||
let mut state = Server {
|
||||
binding: server_binding,
|
||||
key,
|
||||
generation: 1,
|
||||
previous: None,
|
||||
requests: 0,
|
||||
heartbeats: 0,
|
||||
forged,
|
||||
};
|
||||
for generation in 1..=rounds {
|
||||
state.generation = generation;
|
||||
loop {
|
||||
let (socket, _) = listener.accept().unwrap();
|
||||
socket
|
||||
.set_read_timeout(Some(Duration::from_secs(8)))
|
||||
.unwrap();
|
||||
let mut tls = acceptor.accept(socket).unwrap();
|
||||
assert!(tls.ssl().peer_certificate().is_some());
|
||||
if mode == wire_channel::Mode::Wss {
|
||||
let mut ws = tokio_tungstenite::tungstenite::accept(tls).unwrap();
|
||||
loop {
|
||||
let message = ws.read().unwrap();
|
||||
let tokio_tungstenite::tungstenite::Message::Binary(bytes) = message else {
|
||||
continue;
|
||||
};
|
||||
let e = Envelope::decode(bytes).unwrap();
|
||||
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
|
||||
state.heartbeats += 1;
|
||||
let _ = ws.close(None);
|
||||
break;
|
||||
}
|
||||
if matches!(e.payload, Some(Payload::ClientHello(_))) {
|
||||
ws.send(tokio_tungstenite::tungstenite::Message::Ping(
|
||||
vec![1].into(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
ws.send(tokio_tungstenite::tungstenite::Message::Binary(
|
||||
state.response(&e).encode_to_vec().into(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
let batch = MessageBatch::decode(http_body(&mut tls).as_slice()).unwrap();
|
||||
let e = &batch.messages[0];
|
||||
if matches!(e.payload, Some(Payload::Heartbeat(_))) {
|
||||
state.heartbeats += 1;
|
||||
tls.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
|
||||
break;
|
||||
}
|
||||
let response = MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: vec![state.response(e)],
|
||||
}
|
||||
.encode_to_vec();
|
||||
let header = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
response.len()
|
||||
);
|
||||
tls.write_all(header.as_bytes()).unwrap();
|
||||
tls.write_all(&response).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
(state.requests, state.heartbeats)
|
||||
});
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&dir.join("certificates"),
|
||||
&ca.to_pem().unwrap(),
|
||||
id,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
certificates
|
||||
.initialize(
|
||||
&device.to_pem().unwrap(),
|
||||
&device_key.private_key_to_pem_pkcs8().unwrap(),
|
||||
Utc::now().timestamp(),
|
||||
)
|
||||
.unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
let (itx, irx) = tokio::sync::mpsc::channel(1);
|
||||
let mut agent = Agent {
|
||||
address,
|
||||
ca: ca.to_pem().unwrap(),
|
||||
id: id.to_string(),
|
||||
boot: rmm_agent_linux::boot_id().unwrap(),
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db: storage::open(&dir).unwrap(),
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding: Some(binding.clone()),
|
||||
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap(),
|
||||
completed_tx: tx,
|
||||
completed_rx: rx,
|
||||
inventory_tx: itx,
|
||||
inventory_rx: irx,
|
||||
inventory_inflight: false,
|
||||
sync: Some(reconcile::ClientSync::new(binding)),
|
||||
sync_clock: Clock(trusted),
|
||||
helpers: None,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode,
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
};
|
||||
for _ in 0..rounds {
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(10), agent.session())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_err()
|
||||
);
|
||||
assert!(!agent.fence.allows(1, false));
|
||||
}
|
||||
let (requests, heartbeats) = worker.join().unwrap();
|
||||
assert_eq!(heartbeats, rounds as usize);
|
||||
if trusted {
|
||||
assert_eq!(requests, rounds as usize);
|
||||
let decided: Option<String> = agent
|
||||
.db
|
||||
.query_row("SELECT last_decision FROM reconcile_meta", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(decided.is_some(), !forged);
|
||||
} else {
|
||||
assert_eq!(requests, 0);
|
||||
}
|
||||
drop(agent);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn actual_https_dispatcher_commits_signed_sync_and_clean_reconnect() {
|
||||
run(wire_channel::Mode::Https, true, false, 2).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn actual_wss_dispatcher_handles_ping_and_clean_reconnect() {
|
||||
run(wire_channel::Mode::Wss, true, false, 2).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn forged_sync_never_opens_dispatcher_gate() {
|
||||
run(wire_channel::Mode::Https, true, true, 1).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn untrusted_clock_keeps_telemetry_transport_but_never_requests_start() {
|
||||
run(wire_channel::Mode::Https, false, false, 1).await;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
use anyhow::{Result, ensure};
|
||||
use fs2::FileExt;
|
||||
use prost::Message;
|
||||
use rmm_protocol::Envelope;
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::Write,
|
||||
os::unix::fs::{OpenOptionsExt, PermissionsExt},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
pub fn prepare_dir(dir: &Path) -> Result<()> {
|
||||
if !dir.exists() {
|
||||
fs::create_dir_all(dir)?;
|
||||
fs::set_permissions(dir, fs::Permissions::from_mode(0o700))?;
|
||||
}
|
||||
let meta = fs::symlink_metadata(dir)?;
|
||||
ensure!(
|
||||
meta.is_dir() && !meta.file_type().is_symlink() && meta.permissions().mode() & 0o077 == 0,
|
||||
"agent state must be a private directory (0700)"
|
||||
);
|
||||
// SAFETY: geteuid has no preconditions.
|
||||
rmm_privileged_helper::private_path(dir, unsafe { libc::geteuid() }, true)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn create_secret(path: &Path, bytes: &[u8]) -> Result<()> {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(path)?;
|
||||
file.write_all(bytes)?;
|
||||
file.sync_all()?;
|
||||
File::open(
|
||||
path.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("missing parent"))?,
|
||||
)?
|
||||
.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn lock(dir: &Path) -> Result<File> {
|
||||
let f = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(dir.join("agent.lock"))?;
|
||||
// SAFETY: geteuid has no preconditions.
|
||||
rmm_privileged_helper::private_path(
|
||||
&dir.join("agent.lock"),
|
||||
unsafe { libc::geteuid() },
|
||||
false,
|
||||
)?;
|
||||
f.try_lock_exclusive()?;
|
||||
Ok(f)
|
||||
}
|
||||
pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
|
||||
let path = dir.join("journal.sqlite");
|
||||
if !path.exists() {
|
||||
create_secret(&path, &[])?;
|
||||
}
|
||||
// Refuse unsafe preexisting journals rather than opening a symlink.
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
rmm_privileged_helper::private_path(&path, uid, false)?;
|
||||
let db = rusqlite::Connection::open(path)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("PRAGMA max_page_count=16384;
|
||||
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
|
||||
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
|
||||
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
|
||||
CREATE TABLE IF NOT EXISTS job_events(id TEXT PRIMARY KEY, payload BLOB NOT NULL CHECK(length(payload)<=1048576));
|
||||
CREATE TABLE IF NOT EXISTS safety(key TEXT PRIMARY KEY,value TEXT NOT NULL);")?;
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "JOURNAL_INTEGRITY");
|
||||
crate::reconcile::initialize(&db)?;
|
||||
File::open(dir)?.sync_all()?;
|
||||
Ok(db)
|
||||
}
|
||||
pub fn next_sequence(db: &mut rusqlite::Connection) -> Result<u64> {
|
||||
let tx = db.transaction()?;
|
||||
let seq: i64 = tx.query_row(
|
||||
"UPDATE counter SET seq=seq+1 WHERE id=1 RETURNING seq",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(u64::try_from(seq)?)
|
||||
}
|
||||
/// Do not let periodic metadata queue behind unacknowledged durable traffic.
|
||||
/// Results/control receipts retain priority; no old inventory is deleted.
|
||||
pub fn passive_collection_ready(db: &rusqlite::Connection) -> Result<bool> {
|
||||
Ok(
|
||||
!db.query_row("SELECT EXISTS(SELECT 1 FROM job_events)", [], |row| {
|
||||
row.get::<_, bool>(0)
|
||||
})?,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
|
||||
db.execute("INSERT INTO pending VALUES(1,?1)", [e.encode_to_vec()])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod passive_pressure_tests {
|
||||
#[test]
|
||||
fn pending_durable_traffic_backpressures_collection_without_deletion() {
|
||||
let db = rusqlite::Connection::open_in_memory().unwrap();
|
||||
db.execute_batch("CREATE TABLE job_events(id TEXT PRIMARY KEY,payload BLOB NOT NULL)")
|
||||
.unwrap();
|
||||
assert!(super::passive_collection_ready(&db).unwrap());
|
||||
db.execute("INSERT INTO job_events VALUES('unacked',x'01')", [])
|
||||
.unwrap();
|
||||
assert!(!super::passive_collection_ready(&db).unwrap());
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM job_events", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
db.execute("DELETE FROM job_events WHERE id='unacked'", [])
|
||||
.unwrap();
|
||||
assert!(super::passive_collection_ready(&db).unwrap());
|
||||
}
|
||||
}
|
||||
pub fn pending(db: &rusqlite::Connection) -> Result<Option<Envelope>> {
|
||||
use rusqlite::OptionalExtension;
|
||||
let data: Option<Vec<u8>> = db
|
||||
.query_row("SELECT payload FROM pending WHERE id=1", [], |r| r.get(0))
|
||||
.optional()?;
|
||||
data.map(|d| Ok(Envelope::decode(d.as_slice())?))
|
||||
.transpose()
|
||||
}
|
||||
pub fn clear_pending(db: &rusqlite::Connection) -> Result<()> {
|
||||
db.execute("DELETE FROM pending WHERE id=1", [])?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn durable_sequence_pending_and_exclusive_lock() {
|
||||
let path = std::env::temp_dir().join(format!("rmm-{}", uuid::Uuid::new_v4()));
|
||||
prepare_dir(&path).unwrap();
|
||||
let lockfile = lock(&path).unwrap();
|
||||
assert!(lock(&path).is_err());
|
||||
let mut db = open(&path).unwrap();
|
||||
assert_eq!(next_sequence(&mut db).unwrap(), 1);
|
||||
let e = Envelope {
|
||||
sequence: 2,
|
||||
..Default::default()
|
||||
};
|
||||
store_pending(&db, &e).unwrap();
|
||||
drop(db);
|
||||
let mut db = open(&path).unwrap();
|
||||
assert_eq!(next_sequence(&mut db).unwrap(), 2);
|
||||
assert_eq!(pending(&db).unwrap().unwrap(), e);
|
||||
clear_pending(&db).unwrap();
|
||||
assert!(pending(&db).unwrap().is_none());
|
||||
drop(db);
|
||||
drop(lockfile);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use prost::Message;
|
||||
use rmm_protocol::Envelope;
|
||||
use rustls::{ClientConfig, RootCertStore};
|
||||
use std::{io::BufReader, net::SocketAddr, sync::Arc, time::Duration};
|
||||
use tokio::{net::TcpStream, time::timeout};
|
||||
use tokio_tungstenite::{
|
||||
Connector, MaybeTlsStream, WebSocketStream,
|
||||
tungstenite::{Message as WsMessage, protocol::WebSocketConfig},
|
||||
};
|
||||
|
||||
pub type Channel = WebSocketStream<MaybeTlsStream<TcpStream>>;
|
||||
pub async fn connect(
|
||||
address: SocketAddr,
|
||||
ca: &[u8],
|
||||
certificate: &[u8],
|
||||
private_key: &[u8],
|
||||
) -> Result<Channel> {
|
||||
ensure!(
|
||||
address.port() != 0 && !address.ip().is_unspecified() && !address.ip().is_multicast(),
|
||||
"INVALID_CONNECT_ADDRESS"
|
||||
);
|
||||
let mut roots = RootCertStore::empty();
|
||||
for cert in rustls_pemfile::certs(&mut BufReader::new(ca)) {
|
||||
roots.add(cert?)?;
|
||||
}
|
||||
let certs = rustls_pemfile::certs(&mut BufReader::new(certificate))
|
||||
.collect::<std::io::Result<Vec<_>>>()?;
|
||||
let key = rustls_pemfile::private_key(&mut BufReader::new(private_key))?
|
||||
.context("device key missing")?;
|
||||
let config = ClientConfig::builder()
|
||||
.with_root_certificates(roots)
|
||||
.with_client_auth_cert(certs, key)?;
|
||||
let tcp = timeout(Duration::from_secs(10), TcpStream::connect(address)).await??;
|
||||
let ws = WebSocketConfig::default()
|
||||
.max_message_size(Some(rmm_domain::MAX_MESSAGE))
|
||||
.max_frame_size(Some(rmm_domain::MAX_MESSAGE));
|
||||
// URL controls TLS SNI and hostname validation; address overrides TCP only.
|
||||
let (channel, _) = timeout(
|
||||
Duration::from_secs(10),
|
||||
tokio_tungstenite::client_async_tls_with_config(
|
||||
format!("wss://rmm-dev.internal:{}/agent/v1/connect", address.port()),
|
||||
tcp,
|
||||
Some(ws),
|
||||
Some(Connector::Rustls(Arc::new(config))),
|
||||
),
|
||||
)
|
||||
.await??;
|
||||
Ok(channel)
|
||||
}
|
||||
pub async fn send(ws: &mut Channel, e: &Envelope) -> Result<()> {
|
||||
timeout(
|
||||
Duration::from_secs(10),
|
||||
ws.send(WsMessage::Binary(e.encode_to_vec().into())),
|
||||
)
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
pub async fn receive_live(ws: &mut Channel) -> Result<Envelope> {
|
||||
loop {
|
||||
match ws.next().await.context("channel closed")?? {
|
||||
WsMessage::Binary(bytes) => {
|
||||
ensure!(bytes.len() <= rmm_domain::MAX_MESSAGE, "message limit");
|
||||
return Ok(Envelope::decode(bytes)?);
|
||||
}
|
||||
WsMessage::Ping(_) => ws.flush().await?,
|
||||
WsMessage::Pong(_) => {}
|
||||
WsMessage::Close(_) => anyhow::bail!("channel closed"),
|
||||
_ => anyhow::bail!("binary envelope required"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! One bounded dispatcher-facing actor for WSS and HTTPS. Never reconnects or
|
||||
//! retries a permit internally; the owning journal/sync state machine does that.
|
||||
use crate::{
|
||||
certificate_rotation::DeviceIdentity,
|
||||
https_channel::{self, HttpsChannel, WssFailure},
|
||||
transport,
|
||||
};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use rmm_protocol::Envelope;
|
||||
use std::{net::SocketAddr, time::Duration};
|
||||
use tokio::sync::mpsc;
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
Auto,
|
||||
Wss,
|
||||
Https,
|
||||
}
|
||||
impl Mode {
|
||||
pub fn parse(s: &str) -> Result<Self> {
|
||||
match s {
|
||||
"auto" => Ok(Self::Auto),
|
||||
"wss" => Ok(Self::Wss),
|
||||
"https" => Ok(Self::Https),
|
||||
_ => anyhow::bail!("RMM_TRANSPORT_INVALID"),
|
||||
}
|
||||
}
|
||||
}
|
||||
enum Channel {
|
||||
Wss(Box<transport::Channel>),
|
||||
Https(HttpsChannel),
|
||||
}
|
||||
enum Command {
|
||||
Send(Envelope),
|
||||
Poll(Envelope),
|
||||
}
|
||||
pub struct Handle {
|
||||
outgoing: mpsc::Sender<Command>,
|
||||
incoming: mpsc::Receiver<Result<Envelope>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
https: bool,
|
||||
}
|
||||
impl Drop for Handle {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
impl Handle {
|
||||
pub async fn send(&self, envelope: &Envelope) -> Result<()> {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
self.outgoing.send(Command::Send(envelope.clone())),
|
||||
)
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
pub fn uses_https(&self) -> bool {
|
||||
self.https
|
||||
}
|
||||
pub async fn poll(&self, heartbeat: Envelope) -> Result<()> {
|
||||
ensure!(self.https, "POLL_REQUIRES_HTTPS");
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
self.outgoing.send(Command::Poll(heartbeat)),
|
||||
)
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
pub async fn receive(&mut self) -> Result<Envelope> {
|
||||
self.incoming.recv().await.context("CHANNEL_CLOSED")?
|
||||
}
|
||||
}
|
||||
fn classify(error: &anyhow::Error) -> WssFailure {
|
||||
use tokio_tungstenite::tungstenite::Error;
|
||||
for cause in error.chain() {
|
||||
if let Some(error) = cause.downcast_ref::<Error>() {
|
||||
return match error {
|
||||
Error::Tls(_) => WssFailure::TlsOrIdentity,
|
||||
Error::Http(response) => match response.status().as_u16() {
|
||||
401 | 403 => WssFailure::Authentication,
|
||||
400 | 404 | 405 | 426 | 501 | 502 | 503 => WssFailure::UpgradeBlocked,
|
||||
_ => WssFailure::Other,
|
||||
},
|
||||
_ => WssFailure::Other,
|
||||
};
|
||||
}
|
||||
}
|
||||
WssFailure::Other
|
||||
}
|
||||
pub async fn connect(
|
||||
mode: Mode,
|
||||
address: SocketAddr,
|
||||
ca: &[u8],
|
||||
identity: &DeviceIdentity,
|
||||
hello: Envelope,
|
||||
) -> Result<Handle> {
|
||||
let channel = if mode == Mode::Https {
|
||||
None
|
||||
} else {
|
||||
match transport::connect(
|
||||
address,
|
||||
ca,
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(mut ws) => {
|
||||
transport::send(&mut ws, &hello).await?;
|
||||
Some(Channel::Wss(Box::new(ws)))
|
||||
}
|
||||
Err(error) if mode == Mode::Auto && classify(&error) == WssFailure::UpgradeBlocked => {
|
||||
match transport::connect(
|
||||
address,
|
||||
ca,
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(mut ws) => {
|
||||
transport::send(&mut ws, &hello).await?;
|
||||
Some(Channel::Wss(Box::new(ws)))
|
||||
}
|
||||
Err(second) => {
|
||||
ensure!(
|
||||
https_channel::allow_https_fallback(classify(&second), 2),
|
||||
"AUTO_FALLBACK_NOT_AUTHORIZED"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
};
|
||||
let channel = if let Some(channel) = channel {
|
||||
channel
|
||||
} else {
|
||||
let client = https_channel::mtls_client(
|
||||
address,
|
||||
ca,
|
||||
&identity.certificate_pem,
|
||||
&identity.private_key_pem,
|
||||
)?;
|
||||
let channel = HttpsChannel::connect(client, address, hello).await?;
|
||||
ensure!(channel.generation() > 0, "HTTPS_GENERATION");
|
||||
Channel::Https(channel)
|
||||
};
|
||||
let https = matches!(channel, Channel::Https(_));
|
||||
let (outgoing, commands) = mpsc::channel(16);
|
||||
let (events, incoming) = mpsc::channel(32);
|
||||
let task = tokio::spawn(async move {
|
||||
let result = run(channel, commands, &events).await;
|
||||
if let Err(error) = result {
|
||||
let _ = events.send(Err(error)).await;
|
||||
}
|
||||
});
|
||||
Ok(Handle {
|
||||
outgoing,
|
||||
incoming,
|
||||
task,
|
||||
https,
|
||||
})
|
||||
}
|
||||
async fn run(
|
||||
mut channel: Channel,
|
||||
mut commands: mpsc::Receiver<Command>,
|
||||
events: &mpsc::Sender<Result<Envelope>>,
|
||||
) -> Result<()> {
|
||||
match &mut channel {
|
||||
Channel::Wss(ws) => loop {
|
||||
tokio::select! {
|
||||
incoming=transport::receive_live(ws)=>{events.send(Ok(incoming?)).await?;},
|
||||
command=commands.recv()=>{let Some(Command::Send(command))=command else {return Ok(())};transport::send(ws,&command).await?;},
|
||||
}
|
||||
},
|
||||
Channel::Https(http) => {
|
||||
let mut pending = None;
|
||||
loop {
|
||||
while let Some(envelope) = http.receive() {
|
||||
events.send(Ok(envelope)).await?;
|
||||
}
|
||||
let command = match pending.take() {
|
||||
Some(command) => command,
|
||||
None => match commands.recv().await {
|
||||
Some(command) => command,
|
||||
None => {
|
||||
http.fence();
|
||||
return Ok(());
|
||||
}
|
||||
},
|
||||
};
|
||||
if let Command::Poll(command) = command {
|
||||
// Dropping a long poll loses no effect/start authority; the
|
||||
// next durable send or fresh heartbeat drives redelivery.
|
||||
tokio::select! {
|
||||
result=http.poll(command,25)=>result?,
|
||||
next=commands.recv()=>match next{Some(next)=>pending=Some(next),None=>{http.fence();return Ok(())}},
|
||||
}
|
||||
} else if let Command::Send(command) = command {
|
||||
http.send(&command).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn auto_fallback_classifies_upgrade_refusal_but_never_authentication() {
|
||||
for (status, expected) in [
|
||||
(426, WssFailure::UpgradeBlocked),
|
||||
(401, WssFailure::Authentication),
|
||||
(403, WssFailure::Authentication),
|
||||
(500, WssFailure::Other),
|
||||
] {
|
||||
let response = tokio_tungstenite::tungstenite::http::Response::builder()
|
||||
.status(status)
|
||||
.body(None::<Vec<u8>>)
|
||||
.unwrap();
|
||||
let error = anyhow::Error::new(tokio_tungstenite::tungstenite::Error::Http(Box::new(
|
||||
response,
|
||||
)));
|
||||
assert_eq!(classify(&error), expected);
|
||||
}
|
||||
assert_eq!(
|
||||
classify(&anyhow::anyhow!("ambiguous reset")),
|
||||
WssFailure::Other
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn explicit_transport_names_cannot_enable_verification_bypasses() {
|
||||
assert_eq!(Mode::parse("https").unwrap(), Mode::Https);
|
||||
assert_eq!(Mode::parse("auto").unwrap(), Mode::Auto);
|
||||
assert!(Mode::parse("https-insecure").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
#[path = "../src/certificate_rotation.rs"]
|
||||
mod certificate_rotation;
|
||||
// Rotation uses the HTTPS implementation too. Run its transport contract in this
|
||||
// harness rather than compiling an untested/unused copy of the channel module.
|
||||
#[path = "https_channel.rs"]
|
||||
mod transport_tests;
|
||||
use certificate_rotation::*;
|
||||
use common::*;
|
||||
use std::{
|
||||
fs,
|
||||
os::unix::fs::DirBuilderExt,
|
||||
os::unix::fs::PermissionsExt,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use transport_tests::{common, https_channel};
|
||||
use uuid::Uuid;
|
||||
|
||||
struct Temp(PathBuf);
|
||||
impl Temp {
|
||||
fn new() -> Self {
|
||||
let path =
|
||||
PathBuf::from("/tmp/opencode").join(format!("agent-transport-{}", Uuid::new_v4()));
|
||||
fs::DirBuilder::new().mode(0o700).create(&path).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
}
|
||||
impl Drop for Temp {
|
||||
fn drop(&mut self) {
|
||||
fs::remove_dir_all(&self.0).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn csr_cert(pki: &Pki, csr: &str, cn: &str) -> Vec<u8> {
|
||||
let csr = openssl::x509::X509Req::from_pem(csr.as_bytes()).unwrap();
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
pki.issue(
|
||||
&csr.public_key().unwrap(),
|
||||
cn,
|
||||
false,
|
||||
now - 60,
|
||||
now + 30 * 86400,
|
||||
)
|
||||
.to_pem()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn initialized(pki: &Pki, temp: &Temp, agent: Uuid, now: i64) -> RotationStore {
|
||||
let store = RotationStore::open(&temp.0, &pki.ca.to_pem().unwrap(), agent, 1).unwrap();
|
||||
let key = key();
|
||||
let cert = pki.issue(
|
||||
&key,
|
||||
&agent.to_string(),
|
||||
false,
|
||||
now - 21 * 86400,
|
||||
now + 9 * 86400,
|
||||
);
|
||||
store
|
||||
.initialize(
|
||||
&cert.to_pem().unwrap(),
|
||||
&key.private_key_to_pem_pkcs8().unwrap(),
|
||||
now,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_response_reuses_csr_and_atomically_installs_matching_key() {
|
||||
let pki = Pki::new();
|
||||
let temp = Temp::new();
|
||||
let agent = Uuid::new_v4();
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let store = initialized(&pki, &temp, agent, now);
|
||||
assert!(store.due(RotationPolicy::default(), now).unwrap());
|
||||
let original = store.current(now).unwrap();
|
||||
let cached = Arc::new(Mutex::new(None::<(String, Vec<u8>)>));
|
||||
let cache = cached.clone();
|
||||
let issuer = Pki {
|
||||
ca: pki.ca.clone(),
|
||||
ca_key: pki.ca_key.clone(),
|
||||
};
|
||||
let server = Server::start(&pki, "rmm-dev.internal", move |r| {
|
||||
assert_eq!(r.path, "/agent/v1/certificates/renew");
|
||||
assert_eq!(r.content_type, "application/json");
|
||||
let json: serde_json::Value = serde_json::from_slice(&r.body).unwrap();
|
||||
let csr = json["csr_pem"].as_str().unwrap();
|
||||
let mut cache = cache.lock().unwrap();
|
||||
if let Some((old, cert)) = cache.as_ref() {
|
||||
assert_eq!(old, csr);
|
||||
return Some(Response { status: 200, media: "application/json", body: serde_json::to_vec(&serde_json::json!({"agent_id":agent,"certificate_pem":String::from_utf8(cert.clone()).unwrap(),"overlap_seconds":300})).unwrap() });
|
||||
}
|
||||
*cache = Some((csr.into(), csr_cert(&issuer, csr, &agent.to_string())));
|
||||
None
|
||||
});
|
||||
assert!(store.renew(server.address, now).await.is_err());
|
||||
assert_eq!(
|
||||
store.current(now).unwrap().selector.version,
|
||||
original.selector.version
|
||||
);
|
||||
// Reopen simulates process loss after pending key/CSR and lost response.
|
||||
let reopened = RotationStore::open(&temp.0, &pki.ca.to_pem().unwrap(), agent, 1).unwrap();
|
||||
let result = reopened.renew(server.address, now).await.unwrap();
|
||||
assert!(result.needs_reconnect);
|
||||
assert_eq!(result.enrollment_generation, 1);
|
||||
let installed = reopened.current(now).unwrap();
|
||||
assert_ne!(installed.selector.version, original.selector.version);
|
||||
assert_eq!(
|
||||
installed.selector.previous_version,
|
||||
Some(original.selector.version)
|
||||
);
|
||||
assert!(installed.selector.previous_until.unwrap() >= now + 300);
|
||||
assert_ne!(installed.private_key_pem, original.private_key_pem);
|
||||
assert!(
|
||||
temp.0
|
||||
.join("versions")
|
||||
.join(original.selector.version.to_string())
|
||||
.join("device.key")
|
||||
.exists()
|
||||
);
|
||||
assert!(!reopened.due(RotationPolicy::default(), now).unwrap());
|
||||
for entry in fs::read_dir(temp.0.join("versions")).unwrap() {
|
||||
let dir = entry.unwrap().path();
|
||||
assert_eq!(
|
||||
fs::metadata(&dir).unwrap().permissions().mode() & 0o777,
|
||||
0o700
|
||||
);
|
||||
for file in fs::read_dir(dir).unwrap() {
|
||||
assert_eq!(
|
||||
file.unwrap().metadata().unwrap().permissions().mode() & 0o777,
|
||||
0o600
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_unknown_issuer_changed_identity_key_mismatch_and_server_eku_block() {
|
||||
let pki = Pki::new();
|
||||
let agent = Uuid::new_v4();
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
for mode in 0..5 {
|
||||
let temp = Temp::new();
|
||||
let store = RotationStore::open(&temp.0, &pki.ca.to_pem().unwrap(), agent, 1).unwrap();
|
||||
let key = key();
|
||||
let wrong_key = common::key();
|
||||
let other = Pki::new();
|
||||
let cert = match mode {
|
||||
0 => pki.issue(&key, &agent.to_string(), false, now - 86400, now - 1),
|
||||
1 => other.issue(&key, &agent.to_string(), false, now - 60, now + 86400),
|
||||
2 => pki.issue(
|
||||
&key,
|
||||
&Uuid::new_v4().to_string(),
|
||||
false,
|
||||
now - 60,
|
||||
now + 86400,
|
||||
),
|
||||
3 => pki.issue(&wrong_key, &agent.to_string(), false, now - 60, now + 86400),
|
||||
_ => pki.issue(&key, &agent.to_string(), true, now - 60, now + 86400),
|
||||
};
|
||||
assert!(
|
||||
store
|
||||
.initialize(
|
||||
&cert.to_pem().unwrap(),
|
||||
&key.private_key_to_pem_pkcs8().unwrap(),
|
||||
now
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!temp.0.join("current.json").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_stages_do_not_change_selector_and_expiry_never_reenrolls() {
|
||||
let pki = Pki::new();
|
||||
let temp = Temp::new();
|
||||
let agent = Uuid::new_v4();
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let store = initialized(&pki, &temp, agent, now);
|
||||
let original = store.current(now).unwrap().selector.version;
|
||||
let orphan = temp.0.join("versions").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir(&orphan).unwrap();
|
||||
fs::set_permissions(&orphan, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
fs::write(orphan.join("device.key"), b"incomplete crash stage").unwrap();
|
||||
assert_eq!(store.current(now).unwrap().selector.version, original);
|
||||
assert!(
|
||||
store
|
||||
.renew("127.0.0.1:1".parse().unwrap(), now + 10 * 86400)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!temp.0.join("pending.json").exists());
|
||||
let changed =
|
||||
RotationStore::open(&temp.0, &pki.ca.to_pem().unwrap(), Uuid::new_v4(), 1).unwrap();
|
||||
assert!(changed.current(now).is_err());
|
||||
let generation = RotationStore::open(&temp.0, &pki.ca.to_pem().unwrap(), agent, 2).unwrap();
|
||||
assert!(generation.current(now).is_err());
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
#[path = "../../../tests/agent_transport/common.rs"]
|
||||
pub(crate) mod common;
|
||||
#[path = "../src/https_channel.rs"]
|
||||
pub(crate) mod https_channel;
|
||||
|
||||
use common::*;
|
||||
use https_channel::*;
|
||||
use prost::Message;
|
||||
use rmm_protocol::*;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
fn envelope(payload: Payload, generation: u64) -> Envelope {
|
||||
Envelope {
|
||||
protocol_major: 1,
|
||||
protocol_minor: 0,
|
||||
agent_id: "agent".into(),
|
||||
enrollment_generation: 1,
|
||||
connection_generation: generation,
|
||||
boot_id: uuid::Uuid::new_v4().to_string(),
|
||||
message_id: uuid::Uuid::new_v4().to_string(),
|
||||
sequence: 1,
|
||||
observed_at: chrono::Utc::now().timestamp_millis(),
|
||||
payload: Some(payload),
|
||||
}
|
||||
}
|
||||
fn hello() -> Envelope {
|
||||
envelope(Payload::ClientHello(ClientHello::default()), 0)
|
||||
}
|
||||
fn heartbeat(generation: u64) -> Envelope {
|
||||
envelope(Payload::Heartbeat(Heartbeat::default()), generation)
|
||||
}
|
||||
fn reply(e: &Envelope, payload: Payload) -> Envelope {
|
||||
Envelope {
|
||||
connection_generation: 7,
|
||||
payload: Some(payload),
|
||||
..e.clone()
|
||||
}
|
||||
}
|
||||
fn batch(messages: Vec<Envelope>) -> Response {
|
||||
Response::protobuf(
|
||||
MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages,
|
||||
}
|
||||
.encode_to_vec(),
|
||||
)
|
||||
}
|
||||
fn client(pki: &Pki, address: std::net::SocketAddr) -> reqwest::Client {
|
||||
let key = key();
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let cert = pki.issue(&key, "agent", false, now - 60, now + 86400);
|
||||
mtls_client(
|
||||
address,
|
||||
&pki.ca.to_pem().unwrap(),
|
||||
&cert.to_pem().unwrap(),
|
||||
&key.private_key_to_pem_pkcs8().unwrap(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_mtls_hello_poll_durable_replay_and_fencing() {
|
||||
let pki = Pki::new();
|
||||
let results = Arc::new(Mutex::new(Vec::new()));
|
||||
let seen = results.clone();
|
||||
let stored = Arc::new(Mutex::new(std::collections::HashMap::new()));
|
||||
let durable = stored.clone();
|
||||
let server = Server::start(&pki, "rmm-dev.internal", move |r| {
|
||||
assert_eq!(r.content_type, "application/x-protobuf");
|
||||
if r.path.ends_with("/poll") {
|
||||
let poll = PollRequest::decode(r.body.as_slice()).unwrap();
|
||||
assert_eq!(poll.wait_seconds, 25);
|
||||
let e = poll.envelope.unwrap();
|
||||
return Some(batch(vec![reply(
|
||||
&e,
|
||||
Payload::Ack(Ack {
|
||||
message_id: e.message_id.clone(),
|
||||
}),
|
||||
)]));
|
||||
}
|
||||
let b = MessageBatch::decode(r.body.as_slice()).unwrap();
|
||||
let e = &b.messages[0];
|
||||
if matches!(e.payload, Some(Payload::ClientHello(_))) {
|
||||
return Some(batch(vec![reply(
|
||||
e,
|
||||
Payload::ServerHello(ServerHello {
|
||||
connection_generation: 7,
|
||||
max_message_bytes: MAX_BODY as u32,
|
||||
}),
|
||||
)]));
|
||||
}
|
||||
if matches!(e.payload, Some(Payload::JobResult(_))) {
|
||||
let mut durable = durable.lock().unwrap();
|
||||
let bytes = e.encode_to_vec();
|
||||
if let Some(previous) = durable.get(&e.message_id) {
|
||||
assert_eq!(previous, &bytes);
|
||||
} else {
|
||||
durable.insert(e.message_id.clone(), bytes);
|
||||
}
|
||||
seen.lock().unwrap().push(r.body.clone());
|
||||
if seen.lock().unwrap().len() == 1 {
|
||||
return None;
|
||||
} // committed response lost
|
||||
}
|
||||
Some(batch(vec![reply(
|
||||
e,
|
||||
Payload::Ack(Ack {
|
||||
message_id: e.message_id.clone(),
|
||||
}),
|
||||
)]))
|
||||
});
|
||||
let mut channel = HttpsChannel::connect(client(&pki, server.address), server.address, hello())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(channel.generation(), 7);
|
||||
assert!(matches!(
|
||||
channel.receive().unwrap().payload,
|
||||
Some(Payload::ServerHello(_))
|
||||
));
|
||||
let result = envelope(Payload::JobResult(JobResult::default()), 7);
|
||||
assert!(channel.send(&result).await.is_err());
|
||||
assert!(channel.receive().is_none()); // no fabricated ACK on HTTP/IO failure
|
||||
channel.send(&result).await.unwrap();
|
||||
assert_eq!(
|
||||
channel.receive().unwrap().payload,
|
||||
Some(Payload::Ack(Ack {
|
||||
message_id: result.message_id.clone()
|
||||
}))
|
||||
);
|
||||
assert_eq!(
|
||||
results.lock().unwrap().as_slice(),
|
||||
&[
|
||||
MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: vec![result.clone()]
|
||||
}
|
||||
.encode_to_vec(),
|
||||
MessageBatch {
|
||||
protocol_major: 1,
|
||||
messages: vec![result.clone()]
|
||||
}
|
||||
.encode_to_vec()
|
||||
]
|
||||
);
|
||||
channel.poll(heartbeat(7), 25).await.unwrap();
|
||||
channel.receive().unwrap();
|
||||
assert!(channel.poll(heartbeat(7), 26).await.is_err());
|
||||
assert!(channel.send(&heartbeat(6)).await.is_err());
|
||||
channel.fence();
|
||||
assert!(channel.poll(heartbeat(7), 0).await.is_err());
|
||||
channel.send(&result).await.unwrap(); // late durable evidence still reliable
|
||||
assert_eq!(stored.lock().unwrap().len(), 1); // one durable application record
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bad_dns_bad_ca_and_missing_client_identity_are_rejected() {
|
||||
let pki = Pki::new();
|
||||
let wrong_dns = Server::start(&pki, "wrong.internal", |_| {
|
||||
panic!("bad TLS reached application")
|
||||
});
|
||||
assert!(
|
||||
HttpsChannel::connect(client(&pki, wrong_dns.address), wrong_dns.address, hello())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let good = Server::start(&pki, "rmm-dev.internal", |_| {
|
||||
panic!("bad TLS reached application")
|
||||
});
|
||||
assert!(
|
||||
HttpsChannel::connect(client(&Pki::new(), good.address), good.address, hello())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let without_identity = reqwest::Client::builder()
|
||||
.https_only(true)
|
||||
.no_proxy()
|
||||
.tls_built_in_root_certs(false)
|
||||
.add_root_certificate(reqwest::Certificate::from_pem(&pki.ca.to_pem().unwrap()).unwrap())
|
||||
.resolve("rmm-dev.internal", good.address)
|
||||
.build()
|
||||
.unwrap();
|
||||
assert!(
|
||||
HttpsChannel::connect(without_identity, good.address, hello())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_content_limits_generation_and_redirect_fail_closed() {
|
||||
let pki = Pki::new();
|
||||
for mode in 0..5 {
|
||||
let server = Server::start(&pki, "rmm-dev.internal", move |r| {
|
||||
let b = MessageBatch::decode(r.body.as_slice()).unwrap();
|
||||
let e = &b.messages[0];
|
||||
let mut out = batch(vec![reply(
|
||||
e,
|
||||
Payload::ServerHello(ServerHello {
|
||||
connection_generation: 7,
|
||||
max_message_bytes: MAX_BODY as u32,
|
||||
}),
|
||||
)]);
|
||||
match mode {
|
||||
0 => out.media = "text/html",
|
||||
1 => out.body = vec![0; MAX_BODY + 1],
|
||||
2 => {
|
||||
let mut e = reply(e, Payload::Ack(Ack::default()));
|
||||
e.connection_generation = 6;
|
||||
out = batch(vec![e]);
|
||||
}
|
||||
3 => out.status = 302,
|
||||
_ => out.status = 409,
|
||||
}
|
||||
Some(out)
|
||||
});
|
||||
assert!(
|
||||
HttpsChannel::connect(client(&pki, server.address), server.address, hello())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_is_explicit_bounded_and_never_tls_bypass() {
|
||||
assert!(allow_https_fallback(WssFailure::UpgradeBlocked, 2));
|
||||
assert!(!allow_https_fallback(WssFailure::UpgradeBlocked, 4));
|
||||
for reason in [
|
||||
WssFailure::TlsOrIdentity,
|
||||
WssFailure::Authentication,
|
||||
WssFailure::Other,
|
||||
] {
|
||||
assert!(!allow_https_fallback(reason, 2));
|
||||
}
|
||||
for attempt in 0..100 {
|
||||
assert!(retry_delay(attempt).as_secs() <= 60);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user