Fit larger Linux package inventories within bounded snapshots
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -1,8 +1,20 @@
|
||||
//! Additive JSON observation mapping inside the existing snapshot payload. This
|
||||
//! does not introduce a protobuf tag or a fabricated receive time.
|
||||
use anyhow::Result;
|
||||
use anyhow::{Result, ensure};
|
||||
use rmm_domain::inventory_document::LINUX_SOFTWARE_OBSERVATION_MAX_BYTES;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn truncate_rows(observation: &mut Value) -> bool {
|
||||
let Some(rows) = observation["data"].as_array_mut().filter(|v| !v.is_empty()) else {
|
||||
return false;
|
||||
};
|
||||
rows.truncate(rows.len() / 2);
|
||||
observation["truncated"] = json!(true);
|
||||
observation["completeness"] = json!("partial");
|
||||
observation["error_code"] = json!("WIRE_SCOPE_LIMIT");
|
||||
true
|
||||
}
|
||||
|
||||
pub fn observations(inventory: &crate::inventory::LinuxInventory) -> Result<Vec<Value>> {
|
||||
let value = serde_json::to_value(inventory)?;
|
||||
let mut result = Vec::new();
|
||||
@@ -15,17 +27,15 @@ pub fn observations(inventory: &crate::inventory::LinuxInventory) -> Result<Vec<
|
||||
// Bound each scope independently. Never drop unsupported/error metadata
|
||||
// or imply that absent/truncated rows were deleted from a device.
|
||||
let max_bytes = if name == "software" {
|
||||
rmm_domain::inventory_document::SOFTWARE_OBSERVATION_MAX_BYTES
|
||||
LINUX_SOFTWARE_OBSERVATION_MAX_BYTES
|
||||
} else {
|
||||
32768
|
||||
};
|
||||
while serde_json::to_vec(&observation)?.len() > max_bytes {
|
||||
observation["truncated"] = json!(true);
|
||||
observation["completeness"] = json!("partial");
|
||||
observation["error_code"] = json!("WIRE_SCOPE_LIMIT");
|
||||
if let Some(rows) = observation["data"].as_array_mut().filter(|v| !v.is_empty()) {
|
||||
rows.truncate(rows.len() / 2);
|
||||
} else {
|
||||
if !truncate_rows(&mut observation) {
|
||||
observation["truncated"] = json!(true);
|
||||
observation["completeness"] = json!("partial");
|
||||
observation["error_code"] = json!("WIRE_SCOPE_LIMIT");
|
||||
observation["data"] = Value::Null;
|
||||
observation["quality"] = json!("unknown");
|
||||
break;
|
||||
@@ -47,6 +57,20 @@ pub fn snapshot(extended: bool) -> Result<Value> {
|
||||
Ok(base)
|
||||
}
|
||||
|
||||
pub fn encode_snapshot(mut inventory: Value) -> Result<Vec<u8>> {
|
||||
let mut bytes = serde_json::to_vec(&inventory)?;
|
||||
// Preserve the other scopes when a large package list fills the snapshot.
|
||||
while bytes.len() > 512 * 1024 {
|
||||
let software = inventory
|
||||
.get_mut("observations")
|
||||
.and_then(Value::as_array_mut)
|
||||
.and_then(|scopes| scopes.iter_mut().find(|scope| scope["scope"] == "linux.software"));
|
||||
ensure!(software.is_some_and(truncate_rows), "INVENTORY_WIRE_LIMIT");
|
||||
bytes = serde_json::to_vec(&inventory)?;
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -67,7 +91,7 @@ mod tests {
|
||||
assert!(!observation["source_id"].as_str().unwrap().is_empty());
|
||||
assert!(observation.get("received_at").is_none());
|
||||
let limit = if observation["scope"] == "linux.software" {
|
||||
rmm_domain::inventory_document::SOFTWARE_OBSERVATION_MAX_BYTES
|
||||
LINUX_SOFTWARE_OBSERVATION_MAX_BYTES
|
||||
} else {
|
||||
32768
|
||||
};
|
||||
@@ -97,4 +121,21 @@ mod tests {
|
||||
assert_eq!(software["truncated"], true);
|
||||
assert_eq!(software["completeness"], "partial");
|
||||
}
|
||||
#[test]
|
||||
fn snapshot_limit_shrinks_software_and_preserves_other_scopes() {
|
||||
let network = json!({"scope":"linux.interfaces","data":"x".repeat(280 * 1024)});
|
||||
let data = vec!["x".repeat(1024); 240];
|
||||
let snapshot = json!({"observations":[
|
||||
network,
|
||||
{"scope":"linux.software","completeness":"complete","data":data},
|
||||
]});
|
||||
let bytes = encode_snapshot(snapshot).unwrap();
|
||||
let bounded: Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(bounded["observations"][0], network);
|
||||
assert_eq!(bounded["observations"][1]["completeness"], "partial");
|
||||
assert_eq!(
|
||||
bounded["observations"][1]["data"].as_array().unwrap().len(),
|
||||
120
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -513,8 +513,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
.push(agent_update_catalog::observation(report, Utc::now()));
|
||||
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
|
||||
}
|
||||
let bytes = serde_json::to_vec(&inventory)?;
|
||||
ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT");
|
||||
let bytes = rmm_agent_linux::wire::encode_snapshot(inventory)?;
|
||||
Ok((bytes, Utc::now().timestamp_millis()))
|
||||
});
|
||||
let _ = sender.blocking_send(result);
|
||||
|
||||
@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
pub const SOFTWARE_OBSERVATION_MAX_BYTES: usize = 128 * 1024;
|
||||
pub const LINUX_SOFTWARE_OBSERVATION_MAX_BYTES: usize = 256 * 1024;
|
||||
|
||||
pub fn firmware_identifier(value: &str) -> Option<String> {
|
||||
let value = value.trim_matches(|c: char| c == '\0' || c.is_whitespace());
|
||||
@@ -167,8 +168,8 @@ pub fn validate_for_scope(
|
||||
let mut scopes = std::collections::BTreeSet::new();
|
||||
for item in observations {
|
||||
let max_bytes = match (scope, item["scope"].as_str()) {
|
||||
("linux.base", Some("linux.software")) | ("windows.base", Some("windows.software")) =>
|
||||
SOFTWARE_OBSERVATION_MAX_BYTES,
|
||||
("linux.base", Some("linux.software")) => LINUX_SOFTWARE_OBSERVATION_MAX_BYTES,
|
||||
("windows.base", Some("windows.software")) => SOFTWARE_OBSERVATION_MAX_BYTES,
|
||||
_ => 32768,
|
||||
};
|
||||
if serde_json::to_vec(item)
|
||||
@@ -269,6 +270,23 @@ mod tests {
|
||||
assert!(validate(&serde_json::to_vec(&unknown).unwrap(), now).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn larger_linux_package_lists_fit_the_software_scope() {
|
||||
let now = Utc::now();
|
||||
let mut v = base();
|
||||
let package = json!({"name":"some-package","version":"1.0","architecture":"amd64"});
|
||||
v["collection_started_at"] = json!(now);
|
||||
v["collection_completed_at"] = json!(now);
|
||||
v["observations"] = json!([{"scope":"linux.software","source_id":"dpkg-query","observed_at":now,"quality":"valid","completeness":"complete","truncated":false,"error_code":null,"data":vec![package; 2200]}]);
|
||||
let bytes = serde_json::to_vec(&v).unwrap();
|
||||
assert!(bytes.len() > SOFTWARE_OBSERVATION_MAX_BYTES);
|
||||
assert!(validate(&bytes, now).is_ok());
|
||||
v["observations"][0]["scope"] = json!("linux.hardware");
|
||||
assert_eq!(
|
||||
validate(&serde_json::to_vec(&v).unwrap(), now).unwrap_err(),
|
||||
"OBSERVATION_SIZE"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn windows_scope_does_not_weaken_linux_or_accept_platform_spoofing() {
|
||||
let now = Utc::now();
|
||||
let mut v = base();
|
||||
|
||||
Reference in New Issue
Block a user