Show available RAM and measured CPU usage for devices
Rust development slice / linux (push) Failing after 8s
Rust development slice / linux (push) Failing after 8s
This commit is contained in:
@@ -4,12 +4,12 @@ use crate::{
|
||||
actions::{ServiceUnit, reboot_required},
|
||||
process,
|
||||
};
|
||||
use rmm_domain::inventory_document::DeviceType;
|
||||
use rmm_domain::inventory_document::{CpuTimes, DeviceType};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
ffi::{CStr, CString},
|
||||
fs,
|
||||
io::Read,
|
||||
io::{BufRead, BufReader, Read},
|
||||
mem::MaybeUninit,
|
||||
net::{Ipv4Addr, Ipv6Addr},
|
||||
path::Path,
|
||||
@@ -432,6 +432,41 @@ pub fn operating_system() -> Observation<OperatingSystem> {
|
||||
Observation::unavailable("linux.os", Quality::Error, "OS_SOURCE_UNAVAILABLE")
|
||||
}
|
||||
}
|
||||
fn aggregate_jiffies(stat: &str) -> Option<Vec<u64>> {
|
||||
stat.lines()
|
||||
.find(|line| line.starts_with("cpu "))
|
||||
.and_then(|line| {
|
||||
line.split_whitespace()
|
||||
.skip(1)
|
||||
.map(str::parse)
|
||||
.collect::<Result<_, _>>()
|
||||
.ok()
|
||||
})
|
||||
}
|
||||
|
||||
fn linux_cpu_times(jiffies: &[u64]) -> Option<CpuTimes> {
|
||||
// Linux already includes guest and guest_nice in user and nice.
|
||||
let total = jiffies
|
||||
.iter()
|
||||
.take(8)
|
||||
.try_fold(0_u64, |total, ticks| total.checked_add(*ticks))?;
|
||||
let idle = jiffies
|
||||
.get(3)?
|
||||
.checked_add(jiffies.get(4).copied().unwrap_or_default())?;
|
||||
Some(CpuTimes { total, idle })
|
||||
}
|
||||
|
||||
pub fn cpu_times() -> Option<CpuTimes> {
|
||||
let mut line = String::new();
|
||||
BufReader::new(fs::File::open("/proc/stat").ok()?.take(4096))
|
||||
.read_line(&mut line)
|
||||
.ok()?;
|
||||
if !line.ends_with('\n') {
|
||||
return None;
|
||||
}
|
||||
linux_cpu_times(&aggregate_jiffies(&line)?)
|
||||
}
|
||||
|
||||
pub fn cpu() -> Observation<Cpu> {
|
||||
let info = read("/proc/cpuinfo", 1024 * 1024);
|
||||
let stat = read("/proc/stat", 256 * 1024);
|
||||
@@ -460,15 +495,7 @@ pub fn cpu() -> Observation<Cpu> {
|
||||
.filter(|(k, _)| k.trim() == "model name")
|
||||
.map(|(_, v)| v.trim().to_owned())
|
||||
});
|
||||
let aggregate_jiffies: Option<Vec<u64>> =
|
||||
stat.lines().find(|l| l.starts_with("cpu ")).and_then(|l| {
|
||||
l.split_whitespace()
|
||||
.skip(1)
|
||||
.map(str::parse)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.ok()
|
||||
});
|
||||
match aggregate_jiffies {
|
||||
match aggregate_jiffies(&stat) {
|
||||
Some(aggregate_jiffies) if logical_processors > 0 => Observation::valid(
|
||||
"linux.cpu:/proc/cpuinfo,/proc/stat",
|
||||
Cpu {
|
||||
@@ -1400,6 +1427,16 @@ mod tests {
|
||||
assert_eq!(request.since_filter, "@100.123457");
|
||||
assert_eq!(request.until_filter, "@101.987654");
|
||||
}
|
||||
#[test]
|
||||
fn cpu_time_excludes_double_counted_guest_ticks_and_io_wait() {
|
||||
let before = linux_cpu_times(&[10, 0, 5, 60, 10, 0, 0, 0, 4, 0]).unwrap();
|
||||
let after = linux_cpu_times(&[30, 0, 5, 120, 30, 0, 0, 0, 20, 0]).unwrap();
|
||||
assert_eq!(after.usage_since(before), Some(2_000));
|
||||
assert_eq!(after.usage_since(after), None);
|
||||
assert_eq!(before.usage_since(after), None);
|
||||
assert!(linux_cpu_times(&[10, 0, 0]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn machine_types_distinguish_proxmox_guests_and_probe_failures() {
|
||||
assert_eq!(machine_type(true), Some(DeviceType::Hypervisor));
|
||||
|
||||
@@ -185,6 +185,7 @@ mod native {
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: Uuid::new_v4().to_string(),
|
||||
uptime_seconds: 1,
|
||||
system_usage: None,
|
||||
}),
|
||||
)?;
|
||||
let acknowledgement = send(client, &heartbeat).await?;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//! uninstall commands; registry entries are evidence, not verified MSI state.
|
||||
use anyhow::{Result, ensure};
|
||||
use chrono::Utc;
|
||||
use rmm_domain::inventory_document::DeviceType;
|
||||
use rmm_domain::inventory_document::{CpuTimes, DeviceType};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
mem::{size_of, zeroed},
|
||||
@@ -10,8 +10,12 @@ use std::{
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use windows_sys::Win32::{
|
||||
Foundation::{ERROR_FILE_NOT_FOUND, ERROR_NO_MORE_ITEMS, ERROR_SUCCESS},
|
||||
System::{Registry::*, SystemInformation::*},
|
||||
Foundation::{ERROR_FILE_NOT_FOUND, ERROR_NO_MORE_ITEMS, ERROR_SUCCESS, FILETIME},
|
||||
System::{
|
||||
Registry::*,
|
||||
SystemInformation::*,
|
||||
Threading::{GetActiveProcessorGroupCount, GetSystemTimes},
|
||||
},
|
||||
};
|
||||
|
||||
struct Registry(HKEY);
|
||||
@@ -141,21 +145,53 @@ fn machine_type() -> Option<DeviceType> {
|
||||
)
|
||||
}
|
||||
|
||||
fn hardware() -> Result<Value> {
|
||||
let mut system: SYSTEM_INFO = unsafe { zeroed() };
|
||||
unsafe {
|
||||
GetNativeSystemInfo(&mut system);
|
||||
}
|
||||
fn memory_status() -> Result<MEMORYSTATUSEX> {
|
||||
let mut memory: MEMORYSTATUSEX = unsafe { zeroed() };
|
||||
memory.dwLength = size_of::<MEMORYSTATUSEX>() as u32;
|
||||
ensure!(
|
||||
unsafe { GlobalMemoryStatusEx(&mut memory) } != 0
|
||||
&& system.dwNumberOfProcessors > 0
|
||||
&& memory.ullTotalPhys > 0
|
||||
&& memory.ullAvailPhys <= memory.ullTotalPhys
|
||||
&& memory.dwMemoryLoad <= 100,
|
||||
"NATIVE_HARDWARE_QUERY"
|
||||
);
|
||||
Ok(memory)
|
||||
}
|
||||
|
||||
pub fn physical_memory() -> Option<(u64, u64)> {
|
||||
let memory = memory_status().ok()?;
|
||||
Some((memory.ullAvailPhys, memory.ullTotalPhys))
|
||||
}
|
||||
|
||||
pub fn cpu_times() -> Option<CpuTimes> {
|
||||
// GetSystemTimes on a multi-group machine only reports the caller's group.
|
||||
if unsafe { GetActiveProcessorGroupCount() } != 1 {
|
||||
return None;
|
||||
}
|
||||
let (mut idle, mut kernel, mut user) = (
|
||||
FILETIME::default(),
|
||||
FILETIME::default(),
|
||||
FILETIME::default(),
|
||||
);
|
||||
if unsafe { GetSystemTimes(&mut idle, &mut kernel, &mut user) } == 0 {
|
||||
return None;
|
||||
}
|
||||
let ticks =
|
||||
|time: FILETIME| (u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime);
|
||||
// Windows includes idle time in the kernel counter.
|
||||
Some(CpuTimes {
|
||||
total: ticks(kernel).checked_add(ticks(user))?,
|
||||
idle: ticks(idle),
|
||||
})
|
||||
}
|
||||
|
||||
fn hardware() -> Result<Value> {
|
||||
let mut system: SYSTEM_INFO = unsafe { zeroed() };
|
||||
unsafe {
|
||||
GetNativeSystemInfo(&mut system);
|
||||
}
|
||||
ensure!(system.dwNumberOfProcessors > 0, "NATIVE_HARDWARE_QUERY");
|
||||
let memory = memory_status()?;
|
||||
let cpu_model = open(
|
||||
HKEY_LOCAL_MACHINE,
|
||||
r"HARDWARE\DESCRIPTION\System\CentralProcessor\0",
|
||||
|
||||
@@ -8,7 +8,7 @@ use rmm_agent_core::{
|
||||
reconcile::{self, SyncClock},
|
||||
};
|
||||
use rmm_agent_windows::certificate_rotation::{ProtectedKeyFiles, RotationStore};
|
||||
use rmm_agent_windows::{CAPABILITIES, job_binding::readonly_binding, journal::Journal, native};
|
||||
use rmm_agent_windows::{CAPABILITIES, inventory, job_binding::readonly_binding, journal::Journal, native};
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload};
|
||||
use rmm_protocol::*;
|
||||
use rusqlite::{Connection, OptionalExtension};
|
||||
@@ -2602,6 +2602,7 @@ impl Agent {
|
||||
let mut rotation_at = Instant::now();
|
||||
let mut rotation_after = Duration::from_secs(30);
|
||||
let mut heartbeat = Instant::now() - Duration::from_secs(60);
|
||||
let mut previous_cpu = None;
|
||||
let mut heartbeat_after = Duration::from_secs(30);
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
@@ -2776,9 +2777,19 @@ impl Agent {
|
||||
// inventory offline. Retry setup at the next heartbeat.
|
||||
eprintln!("PASSIVE_SIGNED_TIME_UNAVAILABLE");
|
||||
}
|
||||
let cpu = inventory::cpu_times();
|
||||
let cpu_usage_basis_points = cpu.zip(previous_cpu)
|
||||
.and_then(|(current, previous)| current.usage_since(previous));
|
||||
previous_cpu = cpu;
|
||||
let memory = inventory::physical_memory();
|
||||
let payload = Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
system_usage: Some(SystemUsage {
|
||||
cpu_usage_basis_points,
|
||||
ram_available_bytes: memory.map(|(available, _)| available),
|
||||
ram_total_bytes: memory.map(|(_, total)| total),
|
||||
}),
|
||||
});
|
||||
let h = self.envelope(generation, payload)?;
|
||||
channel.send(&h).await?;
|
||||
@@ -2788,6 +2799,7 @@ impl Agent {
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
system_usage: None,
|
||||
}),
|
||||
)?;
|
||||
channel.poll(poll).await?;
|
||||
|
||||
@@ -649,6 +649,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
let mut inventory_at = Instant::now();
|
||||
let mut update_at = Instant::now();
|
||||
let mut heartbeat_at = Instant::now();
|
||||
let mut previous_cpu = None;
|
||||
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
||||
let mut sync_at = Instant::now();
|
||||
@@ -809,11 +810,21 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
}
|
||||
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
|
||||
first_heartbeat = false;
|
||||
let cpu = rmm_agent_linux::inventory::cpu_times();
|
||||
let cpu_usage_basis_points = cpu.zip(previous_cpu)
|
||||
.and_then(|(current, previous)| current.usage_since(previous));
|
||||
previous_cpu = cpu;
|
||||
let memory = rmm_agent_linux::inventory::memory().data;
|
||||
let heartbeat = self.envelope(
|
||||
generation,
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
system_usage: Some(SystemUsage {
|
||||
cpu_usage_basis_points,
|
||||
ram_available_bytes: memory.as_ref().and_then(|m| m.available_bytes),
|
||||
ram_total_bytes: memory.map(|m| m.total_bytes),
|
||||
}),
|
||||
}),
|
||||
)?;
|
||||
ws.send(&heartbeat).await?;
|
||||
@@ -823,6 +834,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
Payload::Heartbeat(Heartbeat {
|
||||
instance_id: self.instance.clone(),
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
system_usage: None,
|
||||
}),
|
||||
)?;
|
||||
ws.poll(poll).await?;
|
||||
|
||||
@@ -11,6 +11,22 @@ pub enum DeviceType {
|
||||
Hypervisor,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct CpuTimes {
|
||||
pub total: u64,
|
||||
pub idle: u64,
|
||||
}
|
||||
|
||||
impl CpuTimes {
|
||||
pub fn usage_since(self, previous: Self) -> Option<u32> {
|
||||
let total = self.total.checked_sub(previous.total)?;
|
||||
let idle = self.idle.checked_sub(previous.idle)?;
|
||||
let busy = total.checked_sub(idle)?;
|
||||
(total != 0)
|
||||
.then(|| ((u128::from(busy) * 10_000 + u128::from(total / 2)) / u128::from(total)) as u32)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Observation {
|
||||
|
||||
@@ -81,6 +81,17 @@ pub struct Heartbeat {
|
||||
pub instance_id: String,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub uptime_seconds: u64,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub system_usage: Option<SystemUsage>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Message)]
|
||||
pub struct SystemUsage {
|
||||
#[prost(uint32, optional, tag = "1")]
|
||||
pub cpu_usage_basis_points: Option<u32>,
|
||||
#[prost(uint64, optional, tag = "2")]
|
||||
pub ram_available_bytes: Option<u64>,
|
||||
#[prost(uint64, optional, tag = "3")]
|
||||
pub ram_total_bytes: Option<u64>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Message)]
|
||||
pub struct InventorySnapshot {
|
||||
@@ -156,6 +167,7 @@ mod tests {
|
||||
payload: Some(Payload::Heartbeat(Heartbeat {
|
||||
instance_id: "x".into(),
|
||||
uptime_seconds: 1,
|
||||
system_usage: None,
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
|
||||
use prost::Message;
|
||||
use rmm_domain::MAX_MESSAGE;
|
||||
use rmm_protocol::*;
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
@@ -146,7 +147,12 @@ pub async fn ingest(
|
||||
match &e.payload {
|
||||
Some(Payload::Heartbeat(h)) => {
|
||||
Uuid::parse_str(&h.instance_id)?;
|
||||
sqlx::query("UPDATE agent_installations SET last_contact_at=now() WHERE agent_id=$1 AND revoked_at IS NULL").bind(agent).execute(&mut *tx).await?;
|
||||
let usage = h.system_usage.as_ref().and_then(system_usage);
|
||||
sqlx::query("UPDATE agent_installations SET last_contact_at=now(),
|
||||
system_usage=COALESCE($2,system_usage),
|
||||
system_usage_observed_at=CASE WHEN $2::jsonb IS NULL THEN system_usage_observed_at ELSE $3 END
|
||||
WHERE agent_id=$1 AND revoked_at IS NULL")
|
||||
.bind(agent).bind(usage).bind(observed).execute(&mut *tx).await?;
|
||||
}
|
||||
Some(Payload::InventorySnapshot(s)) => {
|
||||
ensure!(
|
||||
@@ -197,3 +203,43 @@ pub async fn ingest(
|
||||
e.connection_generation,
|
||||
))
|
||||
}
|
||||
|
||||
fn system_usage(sample: &SystemUsage) -> Option<Value> {
|
||||
let cpu = sample
|
||||
.cpu_usage_basis_points
|
||||
.filter(|&value| value <= 10_000);
|
||||
let memory = sample
|
||||
.ram_available_bytes
|
||||
.zip(sample.ram_total_bytes)
|
||||
.filter(|&(available, total)| total > 0 && available <= total);
|
||||
if cpu.is_none() && memory.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(json!({
|
||||
"cpu_usage_percent": cpu.map(|value| f64::from(value) / 100.0),
|
||||
"ram_available_bytes": memory.map(|(available, _)| available),
|
||||
"ram_total_bytes": memory.map(|(_, total)| total),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accepts_zero_usage_and_ignores_invalid_metrics_without_losing_the_heartbeat() {
|
||||
let mut sample = SystemUsage {
|
||||
cpu_usage_basis_points: Some(0),
|
||||
ram_available_bytes: Some(0),
|
||||
ram_total_bytes: Some(1),
|
||||
};
|
||||
assert_eq!(
|
||||
system_usage(&sample),
|
||||
Some(json!({"cpu_usage_percent": 0.0, "ram_available_bytes": 0, "ram_total_bytes": 1}))
|
||||
);
|
||||
sample.cpu_usage_basis_points = Some(10_001);
|
||||
assert!(system_usage(&sample).unwrap()["cpu_usage_percent"].is_null());
|
||||
sample.ram_available_bytes = Some(2);
|
||||
assert_eq!(system_usage(&sample), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ WITH classified AS (
|
||||
SELECT a.tenant_id,a.asset_id,a.agent_id,a.created_at,a.revoked_at,a.last_contact_at,
|
||||
a.last_authenticated_at,a.last_authenticated_certificate_sha256,a.certificate_sha256,
|
||||
a.enrollment_generation,a.assignment_version,a.capabilities,a.reported_agent_version,
|
||||
a.system_usage,a.system_usage_observed_at,
|
||||
asset.archived_at,t.name AS customer_name,
|
||||
CASE
|
||||
WHEN a.revoked_at IS NOT NULL THEN 'revoked'
|
||||
|
||||
@@ -352,6 +352,17 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool) -> Value {
|
||||
let data = row.get::<Option<Value>, _>("data").unwrap_or(Value::Null);
|
||||
let cpu = observed(&data, "linux.cpu");
|
||||
let windows = observed(&data, "windows.hardware");
|
||||
let linux_memory = observed(&data, "linux.memory");
|
||||
let usage = row
|
||||
.get::<Option<Value>, _>("system_usage")
|
||||
.unwrap_or(Value::Null);
|
||||
let usage_at: Option<DateTime<Utc>> = row.get("system_usage_observed_at");
|
||||
let inventory_at: Option<DateTime<Utc>> = row.get("observed_at");
|
||||
let ram_available = usage["ram_available_bytes"].as_u64();
|
||||
let ram_total = usage["ram_total_bytes"]
|
||||
.as_u64()
|
||||
.or_else(|| linux_memory["total_bytes"].as_u64())
|
||||
.or_else(|| windows["memory_total_bytes"].as_u64());
|
||||
let device_type =
|
||||
device_type(observed(&data, "linux.hardware")).or_else(|| device_type(windows));
|
||||
let last = row.get::<Option<DateTime<Utc>>, _>("last_contact_at");
|
||||
@@ -372,9 +383,14 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool) -> Value {
|
||||
"disk_free_bytes":data["root_available_bytes"].as_u64(),"disk_total_bytes":data["root_total_bytes"].as_u64(),
|
||||
"cpu_model":cpu["model"].as_str().or_else(||windows["cpu_model"].as_str()),
|
||||
"cpu_logical_count":cpu["logical_processors"].as_u64().or_else(||windows["logical_processors_visible"].as_u64()),
|
||||
"ram_total_bytes":observed(&data,"linux.memory")["total_bytes"].as_u64().or_else(||windows["memory_total_bytes"].as_u64()),
|
||||
"cpu_usage_percent":usage["cpu_usage_percent"],
|
||||
"cpu_usage_observed_at":usage["cpu_usage_percent"].is_number().then_some(usage_at).flatten(),
|
||||
"ram_total_bytes":ram_total,
|
||||
"ram_available_bytes":ram_available.or_else(|| linux_memory["available_bytes"].as_u64().or_else(||windows["memory_available_bytes"].as_u64())
|
||||
.filter(|available| ram_total.is_some_and(|total| *available <= total))),
|
||||
"ram_observed_at":if ram_available.is_some() {usage_at} else {inventory_at},
|
||||
"last_contact_at":last,"agent_version":row.get::<Option<String>,_>("reported_agent_version"),
|
||||
"presence":presence,"observed_at":row.get::<Option<DateTime<Utc>>,_>("observed_at"),
|
||||
"presence":presence,"observed_at":inventory_at,
|
||||
"received_at":row.get::<Option<DateTime<Utc>>,_>("received_at"),
|
||||
"source_id":row.get::<Option<String>,_>("source_id"),"quality":if data.is_null(){"unknown"}else{"valid"},
|
||||
"completeness":"partial","health":"unknown"
|
||||
|
||||
Reference in New Issue
Block a user