Show available RAM and measured CPU usage for devices
Rust development slice / linux (push) Failing after 8s

This commit is contained in:
2026-11-18 18:37:18 +01:00
parent fb2376443c
commit b3fe35b86b
17 changed files with 254 additions and 30 deletions
+48 -11
View File
@@ -4,12 +4,12 @@ use crate::{
actions::{ServiceUnit, reboot_required},
process,
};
use rmm_domain::inventory_document::DeviceType;
use rmm_domain::inventory_document::{CpuTimes, DeviceType};
use std::{
collections::BTreeMap,
ffi::{CStr, CString},
fs,
io::Read,
io::{BufRead, BufReader, Read},
mem::MaybeUninit,
net::{Ipv4Addr, Ipv6Addr},
path::Path,
@@ -432,6 +432,41 @@ pub fn operating_system() -> Observation<OperatingSystem> {
Observation::unavailable("linux.os", Quality::Error, "OS_SOURCE_UNAVAILABLE")
}
}
fn aggregate_jiffies(stat: &str) -> Option<Vec<u64>> {
stat.lines()
.find(|line| line.starts_with("cpu "))
.and_then(|line| {
line.split_whitespace()
.skip(1)
.map(str::parse)
.collect::<Result<_, _>>()
.ok()
})
}
fn linux_cpu_times(jiffies: &[u64]) -> Option<CpuTimes> {
// Linux already includes guest and guest_nice in user and nice.
let total = jiffies
.iter()
.take(8)
.try_fold(0_u64, |total, ticks| total.checked_add(*ticks))?;
let idle = jiffies
.get(3)?
.checked_add(jiffies.get(4).copied().unwrap_or_default())?;
Some(CpuTimes { total, idle })
}
pub fn cpu_times() -> Option<CpuTimes> {
let mut line = String::new();
BufReader::new(fs::File::open("/proc/stat").ok()?.take(4096))
.read_line(&mut line)
.ok()?;
if !line.ends_with('\n') {
return None;
}
linux_cpu_times(&aggregate_jiffies(&line)?)
}
pub fn cpu() -> Observation<Cpu> {
let info = read("/proc/cpuinfo", 1024 * 1024);
let stat = read("/proc/stat", 256 * 1024);
@@ -460,15 +495,7 @@ pub fn cpu() -> Observation<Cpu> {
.filter(|(k, _)| k.trim() == "model name")
.map(|(_, v)| v.trim().to_owned())
});
let aggregate_jiffies: Option<Vec<u64>> =
stat.lines().find(|l| l.starts_with("cpu ")).and_then(|l| {
l.split_whitespace()
.skip(1)
.map(str::parse)
.collect::<Result<Vec<_>, _>>()
.ok()
});
match aggregate_jiffies {
match aggregate_jiffies(&stat) {
Some(aggregate_jiffies) if logical_processors > 0 => Observation::valid(
"linux.cpu:/proc/cpuinfo,/proc/stat",
Cpu {
@@ -1400,6 +1427,16 @@ mod tests {
assert_eq!(request.since_filter, "@100.123457");
assert_eq!(request.until_filter, "@101.987654");
}
#[test]
fn cpu_time_excludes_double_counted_guest_ticks_and_io_wait() {
let before = linux_cpu_times(&[10, 0, 5, 60, 10, 0, 0, 0, 4, 0]).unwrap();
let after = linux_cpu_times(&[30, 0, 5, 120, 30, 0, 0, 0, 20, 0]).unwrap();
assert_eq!(after.usage_since(before), Some(2_000));
assert_eq!(after.usage_since(after), None);
assert_eq!(before.usage_since(after), None);
assert!(linux_cpu_times(&[10, 0, 0]).is_none());
}
#[test]
fn machine_types_distinguish_proxmox_guests_and_probe_failures() {
assert_eq!(machine_type(true), Some(DeviceType::Hypervisor));
@@ -185,6 +185,7 @@ mod native {
Payload::Heartbeat(Heartbeat {
instance_id: Uuid::new_v4().to_string(),
uptime_seconds: 1,
system_usage: None,
}),
)?;
let acknowledgement = send(client, &heartbeat).await?;
+45 -9
View File
@@ -2,7 +2,7 @@
//! uninstall commands; registry entries are evidence, not verified MSI state.
use anyhow::{Result, ensure};
use chrono::Utc;
use rmm_domain::inventory_document::DeviceType;
use rmm_domain::inventory_document::{CpuTimes, DeviceType};
use serde_json::{Value, json};
use std::{
mem::{size_of, zeroed},
@@ -10,8 +10,12 @@ use std::{
time::{Duration, Instant},
};
use windows_sys::Win32::{
Foundation::{ERROR_FILE_NOT_FOUND, ERROR_NO_MORE_ITEMS, ERROR_SUCCESS},
System::{Registry::*, SystemInformation::*},
Foundation::{ERROR_FILE_NOT_FOUND, ERROR_NO_MORE_ITEMS, ERROR_SUCCESS, FILETIME},
System::{
Registry::*,
SystemInformation::*,
Threading::{GetActiveProcessorGroupCount, GetSystemTimes},
},
};
struct Registry(HKEY);
@@ -141,21 +145,53 @@ fn machine_type() -> Option<DeviceType> {
)
}
fn hardware() -> Result<Value> {
let mut system: SYSTEM_INFO = unsafe { zeroed() };
unsafe {
GetNativeSystemInfo(&mut system);
}
fn memory_status() -> Result<MEMORYSTATUSEX> {
let mut memory: MEMORYSTATUSEX = unsafe { zeroed() };
memory.dwLength = size_of::<MEMORYSTATUSEX>() as u32;
ensure!(
unsafe { GlobalMemoryStatusEx(&mut memory) } != 0
&& system.dwNumberOfProcessors > 0
&& memory.ullTotalPhys > 0
&& memory.ullAvailPhys <= memory.ullTotalPhys
&& memory.dwMemoryLoad <= 100,
"NATIVE_HARDWARE_QUERY"
);
Ok(memory)
}
pub fn physical_memory() -> Option<(u64, u64)> {
let memory = memory_status().ok()?;
Some((memory.ullAvailPhys, memory.ullTotalPhys))
}
pub fn cpu_times() -> Option<CpuTimes> {
// GetSystemTimes on a multi-group machine only reports the caller's group.
if unsafe { GetActiveProcessorGroupCount() } != 1 {
return None;
}
let (mut idle, mut kernel, mut user) = (
FILETIME::default(),
FILETIME::default(),
FILETIME::default(),
);
if unsafe { GetSystemTimes(&mut idle, &mut kernel, &mut user) } == 0 {
return None;
}
let ticks =
|time: FILETIME| (u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime);
// Windows includes idle time in the kernel counter.
Some(CpuTimes {
total: ticks(kernel).checked_add(ticks(user))?,
idle: ticks(idle),
})
}
fn hardware() -> Result<Value> {
let mut system: SYSTEM_INFO = unsafe { zeroed() };
unsafe {
GetNativeSystemInfo(&mut system);
}
ensure!(system.dwNumberOfProcessors > 0, "NATIVE_HARDWARE_QUERY");
let memory = memory_status()?;
let cpu_model = open(
HKEY_LOCAL_MACHINE,
r"HARDWARE\DESCRIPTION\System\CentralProcessor\0",
+13 -1
View File
@@ -8,7 +8,7 @@ use rmm_agent_core::{
reconcile::{self, SyncClock},
};
use rmm_agent_windows::certificate_rotation::{ProtectedKeyFiles, RotationStore};
use rmm_agent_windows::{CAPABILITIES, job_binding::readonly_binding, journal::Journal, native};
use rmm_agent_windows::{CAPABILITIES, inventory, job_binding::readonly_binding, journal::Journal, native};
use rmm_domain::jobs::{Action, ExecutionPermitPayload};
use rmm_protocol::*;
use rusqlite::{Connection, OptionalExtension};
@@ -2602,6 +2602,7 @@ impl Agent {
let mut rotation_at = Instant::now();
let mut rotation_after = Duration::from_secs(30);
let mut heartbeat = Instant::now() - Duration::from_secs(60);
let mut previous_cpu = None;
let mut heartbeat_after = Duration::from_secs(30);
let mut tick = tokio::time::interval(Duration::from_millis(20));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
@@ -2776,9 +2777,19 @@ impl Agent {
// inventory offline. Retry setup at the next heartbeat.
eprintln!("PASSIVE_SIGNED_TIME_UNAVAILABLE");
}
let cpu = inventory::cpu_times();
let cpu_usage_basis_points = cpu.zip(previous_cpu)
.and_then(|(current, previous)| current.usage_since(previous));
previous_cpu = cpu;
let memory = inventory::physical_memory();
let payload = Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
system_usage: Some(SystemUsage {
cpu_usage_basis_points,
ram_available_bytes: memory.map(|(available, _)| available),
ram_total_bytes: memory.map(|(_, total)| total),
}),
});
let h = self.envelope(generation, payload)?;
channel.send(&h).await?;
@@ -2788,6 +2799,7 @@ impl Agent {
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
system_usage: None,
}),
)?;
channel.poll(poll).await?;
+12
View File
@@ -649,6 +649,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
let mut inventory_at = Instant::now();
let mut update_at = Instant::now();
let mut heartbeat_at = Instant::now();
let mut previous_cpu = None;
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
let mut tick = tokio::time::interval(Duration::from_secs(5));
let mut sync_at = Instant::now();
@@ -809,11 +810,21 @@ impl<C: reconcile::SyncClock> Agent<C> {
}
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
first_heartbeat = false;
let cpu = rmm_agent_linux::inventory::cpu_times();
let cpu_usage_basis_points = cpu.zip(previous_cpu)
.and_then(|(current, previous)| current.usage_since(previous));
previous_cpu = cpu;
let memory = rmm_agent_linux::inventory::memory().data;
let heartbeat = self.envelope(
generation,
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
system_usage: Some(SystemUsage {
cpu_usage_basis_points,
ram_available_bytes: memory.as_ref().and_then(|m| m.available_bytes),
ram_total_bytes: memory.map(|m| m.total_bytes),
}),
}),
)?;
ws.send(&heartbeat).await?;
@@ -823,6 +834,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
Payload::Heartbeat(Heartbeat {
instance_id: self.instance.clone(),
uptime_seconds: self.started.elapsed().as_secs(),
system_usage: None,
}),
)?;
ws.poll(poll).await?;
+16
View File
@@ -11,6 +11,22 @@ pub enum DeviceType {
Hypervisor,
}
#[derive(Clone, Copy)]
pub struct CpuTimes {
pub total: u64,
pub idle: u64,
}
impl CpuTimes {
pub fn usage_since(self, previous: Self) -> Option<u32> {
let total = self.total.checked_sub(previous.total)?;
let idle = self.idle.checked_sub(previous.idle)?;
let busy = total.checked_sub(idle)?;
(total != 0)
.then(|| ((u128::from(busy) * 10_000 + u128::from(total / 2)) / u128::from(total)) as u32)
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Observation {
+12
View File
@@ -81,6 +81,17 @@ pub struct Heartbeat {
pub instance_id: String,
#[prost(uint64, tag = "2")]
pub uptime_seconds: u64,
#[prost(message, optional, tag = "3")]
pub system_usage: Option<SystemUsage>,
}
#[derive(Clone, PartialEq, Message)]
pub struct SystemUsage {
#[prost(uint32, optional, tag = "1")]
pub cpu_usage_basis_points: Option<u32>,
#[prost(uint64, optional, tag = "2")]
pub ram_available_bytes: Option<u64>,
#[prost(uint64, optional, tag = "3")]
pub ram_total_bytes: Option<u64>,
}
#[derive(Clone, PartialEq, Message)]
pub struct InventorySnapshot {
@@ -156,6 +167,7 @@ mod tests {
payload: Some(Payload::Heartbeat(Heartbeat {
instance_id: "x".into(),
uptime_seconds: 1,
system_usage: None,
})),
..Default::default()
};
+47 -1
View File
@@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
use prost::Message;
use rmm_domain::MAX_MESSAGE;
use rmm_protocol::*;
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use sqlx::Row;
use uuid::Uuid;
@@ -146,7 +147,12 @@ pub async fn ingest(
match &e.payload {
Some(Payload::Heartbeat(h)) => {
Uuid::parse_str(&h.instance_id)?;
sqlx::query("UPDATE agent_installations SET last_contact_at=now() WHERE agent_id=$1 AND revoked_at IS NULL").bind(agent).execute(&mut *tx).await?;
let usage = h.system_usage.as_ref().and_then(system_usage);
sqlx::query("UPDATE agent_installations SET last_contact_at=now(),
system_usage=COALESCE($2,system_usage),
system_usage_observed_at=CASE WHEN $2::jsonb IS NULL THEN system_usage_observed_at ELSE $3 END
WHERE agent_id=$1 AND revoked_at IS NULL")
.bind(agent).bind(usage).bind(observed).execute(&mut *tx).await?;
}
Some(Payload::InventorySnapshot(s)) => {
ensure!(
@@ -197,3 +203,43 @@ pub async fn ingest(
e.connection_generation,
))
}
fn system_usage(sample: &SystemUsage) -> Option<Value> {
let cpu = sample
.cpu_usage_basis_points
.filter(|&value| value <= 10_000);
let memory = sample
.ram_available_bytes
.zip(sample.ram_total_bytes)
.filter(|&(available, total)| total > 0 && available <= total);
if cpu.is_none() && memory.is_none() {
return None;
}
Some(json!({
"cpu_usage_percent": cpu.map(|value| f64::from(value) / 100.0),
"ram_available_bytes": memory.map(|(available, _)| available),
"ram_total_bytes": memory.map(|(_, total)| total),
}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accepts_zero_usage_and_ignores_invalid_metrics_without_losing_the_heartbeat() {
let mut sample = SystemUsage {
cpu_usage_basis_points: Some(0),
ram_available_bytes: Some(0),
ram_total_bytes: Some(1),
};
assert_eq!(
system_usage(&sample),
Some(json!({"cpu_usage_percent": 0.0, "ram_available_bytes": 0, "ram_total_bytes": 1}))
);
sample.cpu_usage_basis_points = Some(10_001);
assert!(system_usage(&sample).unwrap()["cpu_usage_percent"].is_null());
sample.ram_available_bytes = Some(2);
assert_eq!(system_usage(&sample), None);
}
}
+1
View File
@@ -2,6 +2,7 @@ WITH classified AS (
SELECT a.tenant_id,a.asset_id,a.agent_id,a.created_at,a.revoked_at,a.last_contact_at,
a.last_authenticated_at,a.last_authenticated_certificate_sha256,a.certificate_sha256,
a.enrollment_generation,a.assignment_version,a.capabilities,a.reported_agent_version,
a.system_usage,a.system_usage_observed_at,
asset.archived_at,t.name AS customer_name,
CASE
WHEN a.revoked_at IS NOT NULL THEN 'revoked'
+18 -2
View File
@@ -352,6 +352,17 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool) -> Value {
let data = row.get::<Option<Value>, _>("data").unwrap_or(Value::Null);
let cpu = observed(&data, "linux.cpu");
let windows = observed(&data, "windows.hardware");
let linux_memory = observed(&data, "linux.memory");
let usage = row
.get::<Option<Value>, _>("system_usage")
.unwrap_or(Value::Null);
let usage_at: Option<DateTime<Utc>> = row.get("system_usage_observed_at");
let inventory_at: Option<DateTime<Utc>> = row.get("observed_at");
let ram_available = usage["ram_available_bytes"].as_u64();
let ram_total = usage["ram_total_bytes"]
.as_u64()
.or_else(|| linux_memory["total_bytes"].as_u64())
.or_else(|| windows["memory_total_bytes"].as_u64());
let device_type =
device_type(observed(&data, "linux.hardware")).or_else(|| device_type(windows));
let last = row.get::<Option<DateTime<Utc>>, _>("last_contact_at");
@@ -372,9 +383,14 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool) -> Value {
"disk_free_bytes":data["root_available_bytes"].as_u64(),"disk_total_bytes":data["root_total_bytes"].as_u64(),
"cpu_model":cpu["model"].as_str().or_else(||windows["cpu_model"].as_str()),
"cpu_logical_count":cpu["logical_processors"].as_u64().or_else(||windows["logical_processors_visible"].as_u64()),
"ram_total_bytes":observed(&data,"linux.memory")["total_bytes"].as_u64().or_else(||windows["memory_total_bytes"].as_u64()),
"cpu_usage_percent":usage["cpu_usage_percent"],
"cpu_usage_observed_at":usage["cpu_usage_percent"].is_number().then_some(usage_at).flatten(),
"ram_total_bytes":ram_total,
"ram_available_bytes":ram_available.or_else(|| linux_memory["available_bytes"].as_u64().or_else(||windows["memory_available_bytes"].as_u64())
.filter(|available| ram_total.is_some_and(|total| *available <= total))),
"ram_observed_at":if ram_available.is_some() {usage_at} else {inventory_at},
"last_contact_at":last,"agent_version":row.get::<Option<String>,_>("reported_agent_version"),
"presence":presence,"observed_at":row.get::<Option<DateTime<Utc>>,_>("observed_at"),
"presence":presence,"observed_at":inventory_at,
"received_at":row.get::<Option<DateTime<Utc>>,_>("received_at"),
"source_id":row.get::<Option<String>,_>("source_id"),"quality":if data.is_null(){"unknown"}else{"valid"},
"completeness":"partial","health":"unknown"