From 1f60183b7e8b72dfdca7019e71a6411748f2927e Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Wed, 18 Nov 2026 20:20:10 +0100 Subject: [PATCH] Fit larger Linux package inventories within bounded snapshots --- crates/agent-linux/src/wire.rs | 59 +++++++++++++++++++++---- crates/agent/src/main.rs | 3 +- crates/domain/src/inventory_document.rs | 22 ++++++++- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/crates/agent-linux/src/wire.rs b/crates/agent-linux/src/wire.rs index 9a6e70a..b8aacc7 100644 --- a/crates/agent-linux/src/wire.rs +++ b/crates/agent-linux/src/wire.rs @@ -1,8 +1,20 @@ //! Additive JSON observation mapping inside the existing snapshot payload. This //! does not introduce a protobuf tag or a fabricated receive time. -use anyhow::Result; +use anyhow::{Result, ensure}; +use rmm_domain::inventory_document::LINUX_SOFTWARE_OBSERVATION_MAX_BYTES; use serde_json::{Value, json}; +fn truncate_rows(observation: &mut Value) -> bool { + let Some(rows) = observation["data"].as_array_mut().filter(|v| !v.is_empty()) else { + return false; + }; + rows.truncate(rows.len() / 2); + observation["truncated"] = json!(true); + observation["completeness"] = json!("partial"); + observation["error_code"] = json!("WIRE_SCOPE_LIMIT"); + true +} + pub fn observations(inventory: &crate::inventory::LinuxInventory) -> Result> { let value = serde_json::to_value(inventory)?; let mut result = Vec::new(); @@ -15,17 +27,15 @@ pub fn observations(inventory: &crate::inventory::LinuxInventory) -> Result max_bytes { - observation["truncated"] = json!(true); - observation["completeness"] = json!("partial"); - observation["error_code"] = json!("WIRE_SCOPE_LIMIT"); - if let Some(rows) = observation["data"].as_array_mut().filter(|v| !v.is_empty()) { - rows.truncate(rows.len() / 2); - } else { + if !truncate_rows(&mut observation) { + observation["truncated"] = json!(true); + observation["completeness"] = json!("partial"); + observation["error_code"] = json!("WIRE_SCOPE_LIMIT"); observation["data"] = Value::Null; observation["quality"] = json!("unknown"); break; @@ -47,6 +57,20 @@ pub fn snapshot(extended: bool) -> Result { Ok(base) } +pub fn encode_snapshot(mut inventory: Value) -> Result> { + let mut bytes = serde_json::to_vec(&inventory)?; + // Preserve the other scopes when a large package list fills the snapshot. + while bytes.len() > 512 * 1024 { + let software = inventory + .get_mut("observations") + .and_then(Value::as_array_mut) + .and_then(|scopes| scopes.iter_mut().find(|scope| scope["scope"] == "linux.software")); + ensure!(software.is_some_and(truncate_rows), "INVENTORY_WIRE_LIMIT"); + bytes = serde_json::to_vec(&inventory)?; + } + Ok(bytes) +} + #[cfg(test)] mod tests { use super::*; @@ -67,7 +91,7 @@ mod tests { assert!(!observation["source_id"].as_str().unwrap().is_empty()); assert!(observation.get("received_at").is_none()); let limit = if observation["scope"] == "linux.software" { - rmm_domain::inventory_document::SOFTWARE_OBSERVATION_MAX_BYTES + LINUX_SOFTWARE_OBSERVATION_MAX_BYTES } else { 32768 }; @@ -97,4 +121,21 @@ mod tests { assert_eq!(software["truncated"], true); assert_eq!(software["completeness"], "partial"); } + #[test] + fn snapshot_limit_shrinks_software_and_preserves_other_scopes() { + let network = json!({"scope":"linux.interfaces","data":"x".repeat(280 * 1024)}); + let data = vec!["x".repeat(1024); 240]; + let snapshot = json!({"observations":[ + network, + {"scope":"linux.software","completeness":"complete","data":data}, + ]}); + let bytes = encode_snapshot(snapshot).unwrap(); + let bounded: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(bounded["observations"][0], network); + assert_eq!(bounded["observations"][1]["completeness"], "partial"); + assert_eq!( + bounded["observations"][1]["data"].as_array().unwrap().len(), + 120 + ); + } } diff --git a/crates/agent/src/main.rs b/crates/agent/src/main.rs index 44cbb0f..eb16aac 100644 --- a/crates/agent/src/main.rs +++ b/crates/agent/src/main.rs @@ -513,8 +513,7 @@ impl Agent { .push(agent_update_catalog::observation(report, Utc::now())); inventory["collection_completed_at"] = serde_json::json!(Utc::now()); } - let bytes = serde_json::to_vec(&inventory)?; - ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT"); + let bytes = rmm_agent_linux::wire::encode_snapshot(inventory)?; Ok((bytes, Utc::now().timestamp_millis())) }); let _ = sender.blocking_send(result); diff --git a/crates/domain/src/inventory_document.rs b/crates/domain/src/inventory_document.rs index fe4adc8..2fd5778 100644 --- a/crates/domain/src/inventory_document.rs +++ b/crates/domain/src/inventory_document.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; pub const SOFTWARE_OBSERVATION_MAX_BYTES: usize = 128 * 1024; +pub const LINUX_SOFTWARE_OBSERVATION_MAX_BYTES: usize = 256 * 1024; pub fn firmware_identifier(value: &str) -> Option { let value = value.trim_matches(|c: char| c == '\0' || c.is_whitespace()); @@ -167,8 +168,8 @@ pub fn validate_for_scope( let mut scopes = std::collections::BTreeSet::new(); for item in observations { let max_bytes = match (scope, item["scope"].as_str()) { - ("linux.base", Some("linux.software")) | ("windows.base", Some("windows.software")) => - SOFTWARE_OBSERVATION_MAX_BYTES, + ("linux.base", Some("linux.software")) => LINUX_SOFTWARE_OBSERVATION_MAX_BYTES, + ("windows.base", Some("windows.software")) => SOFTWARE_OBSERVATION_MAX_BYTES, _ => 32768, }; if serde_json::to_vec(item) @@ -269,6 +270,23 @@ mod tests { assert!(validate(&serde_json::to_vec(&unknown).unwrap(), now).is_err()); } #[test] + fn larger_linux_package_lists_fit_the_software_scope() { + let now = Utc::now(); + let mut v = base(); + let package = json!({"name":"some-package","version":"1.0","architecture":"amd64"}); + v["collection_started_at"] = json!(now); + v["collection_completed_at"] = json!(now); + v["observations"] = json!([{"scope":"linux.software","source_id":"dpkg-query","observed_at":now,"quality":"valid","completeness":"complete","truncated":false,"error_code":null,"data":vec![package; 2200]}]); + let bytes = serde_json::to_vec(&v).unwrap(); + assert!(bytes.len() > SOFTWARE_OBSERVATION_MAX_BYTES); + assert!(validate(&bytes, now).is_ok()); + v["observations"][0]["scope"] = json!("linux.hardware"); + assert_eq!( + validate(&serde_json::to_vec(&v).unwrap(), now).unwrap_err(), + "OBSERVATION_SIZE" + ); + } + #[test] fn windows_scope_does_not_weaken_linux_or_accept_platform_spoofing() { let now = Utc::now(); let mut v = base();