fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -157,6 +157,7 @@ fn load() -> Result<Option<Loaded>> {
|
||||
}))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
super::dev_transport_faults::validate_environment()?;
|
||||
load().map(|_| ())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Development-only, exact-device WSS handshake rejection after normal mTLS
|
||||
//! binding. HTTPS and every execution/approval path remain untouched.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use rmm_domain::jobs::hash;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::PgPool;
|
||||
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Config {
|
||||
schema_version: u32,
|
||||
fault_id: Uuid,
|
||||
tenant_id: Uuid,
|
||||
agent_id: Uuid,
|
||||
issued_at: DateTime<Utc>,
|
||||
expires_at: DateTime<Utc>,
|
||||
}
|
||||
impl Config {
|
||||
fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == 1
|
||||
&& !self.fault_id.is_nil()
|
||||
&& !self.tenant_id.is_nil()
|
||||
&& !self.agent_id.is_nil()
|
||||
&& self.issued_at.timestamp() > 0
|
||||
&& self.expires_at > self.issued_at
|
||||
&& self.expires_at - self.issued_at <= Duration::minutes(15),
|
||||
"DEV_TRANSPORT_FAULT_CONFIG"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn active(&self, tenant: Uuid, agent: Uuid, now: DateTime<Utc>) -> bool {
|
||||
self.tenant_id == tenant
|
||||
&& self.agent_id == agent
|
||||
&& now >= self.issued_at
|
||||
&& now < self.expires_at
|
||||
}
|
||||
}
|
||||
fn configured_path(
|
||||
flag: Option<&str>,
|
||||
path: Option<&str>,
|
||||
profile: &str,
|
||||
dev: &str,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
match (flag, path) {
|
||||
(None | Some("false"), None) => Ok(None),
|
||||
(Some("true"), Some("/root-policy/transport-fault.json")) => {
|
||||
ensure!(
|
||||
profile == "development" && dev == "true",
|
||||
"DEV_TRANSPORT_FAULT_PROFILE"
|
||||
);
|
||||
Ok(Some(PathBuf::from("/root-policy/transport-fault.json")))
|
||||
}
|
||||
_ => anyhow::bail!("DEV_TRANSPORT_FAULT_EXPLICIT_OPT_IN"),
|
||||
}
|
||||
}
|
||||
fn load() -> Result<Option<(Config, String)>> {
|
||||
let flag = std::env::var("RMM_DEV_TRANSPORT_FAULTS").ok();
|
||||
let path = std::env::var("RMM_DEV_TRANSPORT_FAULT_FILE").ok();
|
||||
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
|
||||
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
|
||||
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = fs::symlink_metadata(path.parent().context("DEV_TRANSPORT_FAULT_PARENT")?)?;
|
||||
let metadata = fs::symlink_metadata(&path)?;
|
||||
ensure!(
|
||||
directory.is_dir()
|
||||
&& directory.uid() == 10001
|
||||
&& directory.mode() & 0o777 == 0o700
|
||||
&& metadata.is_file()
|
||||
&& metadata.uid() == 10001
|
||||
&& metadata.mode() & 0o777 == 0o600
|
||||
&& metadata.nlink() == 1
|
||||
&& metadata.len() <= 4096,
|
||||
"DEV_TRANSPORT_FAULT_PRIVATE_CONFIG"
|
||||
);
|
||||
let raw = fs::read(path)?;
|
||||
ensure!(raw.len() <= 4096, "DEV_TRANSPORT_FAULT_CONFIG_SIZE");
|
||||
let config: Config = serde_json::from_slice(&raw)?;
|
||||
config.validate()?;
|
||||
Ok(Some((config, hash(&raw))))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
load().map(|_| ())
|
||||
}
|
||||
fn slot_id(fault: Uuid, slot: u8) -> Uuid {
|
||||
use sha2::{Digest, Sha256};
|
||||
let digest = Sha256::digest(format!("rmm.dev-transport-fault.v1:{fault}:{slot}"));
|
||||
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
|
||||
}
|
||||
fn same_configuration(value: &Value, sha256: &str) -> Result<()> {
|
||||
ensure!(
|
||||
value.get("config_sha256") == Some(&json!(sha256)),
|
||||
"DEV_TRANSPORT_FAULT_ID_REUSED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Caller has already checked certificate expiry and authoritative device_binding.
|
||||
/// No upgrade/ClientHello/JobOffer occurs on this path. At most two narrow durable
|
||||
/// observations are inserted; every matching WSS request stays denied until expiry.
|
||||
pub(crate) async fn reject_wss(
|
||||
pool: &PgPool,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
certificate_sha256: &str,
|
||||
) -> Result<bool> {
|
||||
let Some((config, sha256)) = load()? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !config.active(tenant, agent, Utc::now()) {
|
||||
return Ok(false);
|
||||
}
|
||||
ensure!(
|
||||
certificate_sha256.len() == 64
|
||||
&& certificate_sha256
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
|
||||
"DEV_TRANSPORT_FAULT_PEER_FINGERPRINT"
|
||||
);
|
||||
let mut tx = rmm_persistence::tenant_tx(pool, tenant).await?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
||||
.bind(format!("dev-transport-fault:{}", config.fault_id))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
for slot in 1..=2 {
|
||||
let id = slot_id(config.fault_id, slot);
|
||||
let existing: Option<Value> = sqlx::query_scalar(
|
||||
"SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_transport_fault_adapter'",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some(value) = existing {
|
||||
same_configuration(&value, &sha256)?;
|
||||
} else {
|
||||
let value = json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
|
||||
"fault_id":config.fault_id,"agent_id":agent,"tenant_id":tenant,"slot":slot,
|
||||
"config_sha256":sha256,"certificate_sha256":certificate_sha256,
|
||||
"handshake_id":Uuid::new_v4(),"status":503,
|
||||
"detail":"authenticated_mtls_device_wss_rejected_before_upgrade"});
|
||||
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_transport_fault_adapter','dev_transport_fault.wss_rejected',$3) ON CONFLICT(tenant_id,event_id) DO NOTHING")
|
||||
.bind(tenant).bind(id).bind(value).execute(&mut *tx).await?;
|
||||
break;
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn opt_in_is_explicit_and_production_activation_is_rejected() {
|
||||
assert!(
|
||||
configured_path(None, None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
configured_path(Some("false"), None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for (flag, path, profile, dev) in [
|
||||
(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"production",
|
||||
"true",
|
||||
),
|
||||
(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"false",
|
||||
),
|
||||
(
|
||||
Some("false"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"true",
|
||||
),
|
||||
(Some("true"), None, "development", "true"),
|
||||
(Some("true"), Some("/etc/passwd"), "development", "true"),
|
||||
] {
|
||||
assert!(configured_path(flag, path, profile, dev).is_err());
|
||||
}
|
||||
assert!(
|
||||
configured_path(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"true"
|
||||
)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn exact_scope_and_bounded_window_never_affect_another_device() {
|
||||
let now = Utc::now();
|
||||
let mut config = Config {
|
||||
schema_version: 1,
|
||||
fault_id: Uuid::new_v4(),
|
||||
tenant_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
issued_at: now,
|
||||
expires_at: now + Duration::minutes(15),
|
||||
};
|
||||
config.validate().unwrap();
|
||||
assert!(config.active(config.tenant_id, config.agent_id, now));
|
||||
assert!(!config.active(Uuid::new_v4(), config.agent_id, now));
|
||||
assert!(!config.active(config.tenant_id, Uuid::new_v4(), now));
|
||||
assert!(!config.active(
|
||||
config.tenant_id,
|
||||
config.agent_id,
|
||||
now - Duration::seconds(1)
|
||||
));
|
||||
assert!(!config.active(config.tenant_id, config.agent_id, config.expires_at));
|
||||
config.expires_at += Duration::milliseconds(1);
|
||||
assert!(config.validate().is_err());
|
||||
config.expires_at = now;
|
||||
assert!(config.validate().is_err());
|
||||
config.expires_at = now + Duration::minutes(1);
|
||||
config.agent_id = Uuid::nil();
|
||||
assert!(config.validate().is_err());
|
||||
}
|
||||
#[test]
|
||||
fn slots_are_bounded_stable_and_cannot_change_configuration() {
|
||||
let fault = Uuid::new_v4();
|
||||
assert_eq!(slot_id(fault, 1), slot_id(fault, 1));
|
||||
assert_ne!(slot_id(fault, 1), slot_id(fault, 2));
|
||||
assert_ne!(slot_id(fault, 1), slot_id(Uuid::new_v4(), 1));
|
||||
let value = json!({"config_sha256":"a".repeat(64)});
|
||||
same_configuration(&value, &"a".repeat(64)).unwrap();
|
||||
assert!(same_configuration(&value, &"b".repeat(64)).is_err());
|
||||
assert!(same_configuration(&json!({}), &"a".repeat(64)).is_err());
|
||||
}
|
||||
}
|
||||
@@ -92,6 +92,12 @@ pub async fn connect(
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
|
||||
let (tenant, agent) = binding.ok_or(StatusCode::UNAUTHORIZED)?;
|
||||
if dev_transport_faults::reject_wss(&state.pool, tenant, agent, &fingerprint)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
{
|
||||
return Err(StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
Ok(ws
|
||||
.max_message_size(MAX_MESSAGE)
|
||||
.max_frame_size(MAX_MESSAGE)
|
||||
@@ -226,3 +232,6 @@ async fn channel(
|
||||
}
|
||||
#[path = "dev_delivery_faults.rs"]
|
||||
pub(crate) mod dev_delivery_faults;
|
||||
|
||||
#[path = "dev_transport_faults.rs"]
|
||||
pub(crate) mod dev_transport_faults;
|
||||
|
||||
Reference in New Issue
Block a user