Compare commits

...
Author SHA1 Message Date
kevin_janssen 62cb52bc22 Inventory Linux whole disks and account mounted filesystems by block topology
Rust development slice / linux (push) Failing after 11s
2026-11-20 09:34:17 +01:00
3 changed files with 439 additions and 4 deletions
+11 -3
View File
@@ -4,6 +4,7 @@ use crate::{
actions::{ServiceUnit, reboot_required},
process,
};
mod disks;
use rmm_domain::inventory_document::{CpuTimes, DeviceType, firmware_identifier};
use std::{
collections::BTreeMap,
@@ -200,6 +201,8 @@ pub struct Dns {
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct Volume {
#[serde(skip)]
pub(crate) devno: String,
pub mount_point: String,
pub filesystem: String,
pub device: String,
@@ -275,6 +278,7 @@ pub struct LinuxInventory {
pub routes: Observation<Vec<Route>>,
pub dns: Observation<Dns>,
pub volumes: Observation<Vec<Observation<Volume>>>,
pub disks: Observation<Vec<disks::Disk>>,
pub software: Observation<Vec<SoftwarePackage>>,
pub users: Observation<Vec<String>>,
pub updates: Observation<UpdateStatus>,
@@ -282,6 +286,8 @@ pub struct LinuxInventory {
pub smart: Observation<()>,
}
pub fn collect_native() -> LinuxInventory {
let volumes = volumes();
let disks = disks::collect(&volumes);
LinuxInventory {
os: operating_system(),
cpu: cpu(),
@@ -289,7 +295,8 @@ pub fn collect_native() -> LinuxInventory {
interfaces: interfaces(),
routes: routes(),
dns: dns(),
volumes: volumes(),
volumes,
disks,
software: software(),
users: logged_in_users(),
updates: updates(),
@@ -937,7 +944,7 @@ pub fn volumes() -> Observation<Vec<Observation<Volume>>> {
));
continue;
};
observations.push(volume(&mount, right[0], right[1]));
observations.push(volume(&mount, left[2], right[0], right[1]));
}
Some(observations)
})
@@ -955,7 +962,7 @@ impl Observation<Vec<Observation<Volume>>> {
self
}
}
fn volume(mount: &str, filesystem: &str, device: &str) -> Observation<Volume> {
fn volume(mount: &str, devno: &str, filesystem: &str, device: &str) -> Observation<Volume> {
// Remote filesystems can hang statvfs indefinitely; explicitly do not probe.
const LOCAL: &[&str] = &[
"ext2", "ext3", "ext4", "xfs", "btrfs", "tmpfs", "overlay", "vfat", "exfat", "f2fs", "zfs",
@@ -999,6 +1006,7 @@ fn volume(mount: &str, filesystem: &str, device: &str) -> Observation<Volume> {
Observation::valid(
&source,
Volume {
devno: devno.to_owned(),
mount_point: mount.into(),
filesystem: filesystem.into(),
device: mount_unescape(device).unwrap_or_else(|| device.into()),
+427
View File
@@ -0,0 +1,427 @@
//! Disk capacity is a hardware measurement. Filesystem usage can only be
//! attributed where the block topology and mount identity agree.
use super::{Observation, Quality, Volume, command};
use std::{
collections::{BTreeMap, BTreeSet},
fs,
os::unix::fs::{FileTypeExt, MetadataExt},
path::Path,
};
const SOURCE: &str = "linux.sysfs:block;mountinfo;statvfs";
#[derive(Debug, Clone, serde::Serialize)]
pub struct Disk {
name: String,
kind: &'static str,
model: Option<String>,
members: BTreeSet<String>,
total_bytes: u64,
used_bytes: Option<u64>,
available_bytes: Option<u64>,
}
impl Disk {
fn new(name: String, kind: &'static str, total_bytes: u64) -> Self {
Self {
name,
kind,
model: None,
members: BTreeSet::new(),
total_bytes,
used_bytes: None,
available_bytes: None,
}
}
}
struct Node {
parents: Vec<String>,
disk: bool,
stacked: bool,
label: String,
}
type Nodes = BTreeMap<String, Node>;
type Rows = BTreeMap<String, Disk>;
fn attr(path: impl AsRef<Path>) -> Option<String> {
fs::read_to_string(path)
.ok()
.map(|s| s.trim().to_owned())
.filter(|s| !s.is_empty() && s.len() <= 256)
}
fn devno(dev: u64) -> String {
format!("{}:{}", libc::major(dev), libc::minor(dev))
}
fn block_id(path: &str) -> Option<String> {
if !path.starts_with("/dev/") {
return None;
}
let m = fs::metadata(path).ok()?;
m.file_type().is_block_device().then(|| devno(m.rdev()))
}
fn topology() -> std::io::Result<(Nodes, BTreeMap<String, String>, Rows)> {
let mut nodes = Nodes::new();
let mut ids = BTreeMap::new();
let mut rows = Rows::new();
for entry in fs::read_dir("/sys/class/block")? {
let entry = entry?;
let path = entry.path();
let name = entry.file_name().to_string_lossy().into_owned();
let Some(id) = attr(path.join("dev")) else {
continue;
};
let Ok(real) = path.canonicalize() else {
continue;
};
let part = path.join("partition").exists();
let parents = if part {
real.parent()
.and_then(Path::file_name)
.map(|n| n.to_string_lossy().into_owned())
.into_iter()
.collect()
} else {
fs::read_dir(path.join("slaves"))
.into_iter()
.flatten()
.flatten()
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect::<Vec<_>>()
};
// virtio/SCSI/Xen disks in guests have real bus paths too. Exclude
// synthetic block devices, optical drives and eMMC boot areas.
let disk = !part
&& parents.is_empty()
&& !real.starts_with("/sys/devices/virtual/block")
&& attr(path.join("device/type")).is_none_or(|t| t == "0" || t == "14")
&& !name.starts_with("sr")
&& !(name.starts_with("mmcblk") && name.contains("boot"));
if disk {
if let Some(size) = attr(path.join("size"))
.and_then(|s| s.parse::<u64>().ok())
.and_then(|s| s.checked_mul(512))
.filter(|n| *n > 0)
{
let mut row = Disk::new(name.clone(), "disk", size);
row.model =
attr(path.join("device/model")).or_else(|| attr(path.join("device/name")));
rows.insert(name.clone(), row);
}
}
ids.insert(id, name.clone());
nodes.insert(
name.clone(),
Node {
stacked: !part && !parents.is_empty(),
disk,
parents,
label: attr(path.join("dm/name")).unwrap_or(name),
},
);
}
Ok((nodes, ids, rows))
}
fn owners(name: &str, nodes: &Nodes) -> (BTreeSet<String>, bool) {
let mut pending = vec![name];
let mut visited = BTreeSet::new();
let mut disks = BTreeSet::new();
let mut stacked = false;
while let Some(n) = pending.pop() {
if !visited.insert(n) {
continue;
}
if let Some(node) = nodes.get(n) {
if node.disk {
disks.insert(n.to_owned());
}
stacked |= node.stacked;
pending.extend(node.parents.iter().map(String::as_str));
}
}
(disks, stacked)
}
fn project(
nodes: &Nodes,
ids: &BTreeMap<String, String>,
rows: &mut Rows,
volumes: &[&Volume],
btrfs: &BTreeMap<String, (String, BTreeSet<String>, bool)>,
) {
let mut seen = BTreeSet::new();
let mut invalid = BTreeSet::new();
for v in volumes {
if v.total_bytes == 0 || v.available_bytes > v.free_bytes || v.free_bytes > v.total_bytes {
continue;
}
let (members, key, group) = if v.filesystem == "btrfs" {
// Subvolumes can have different anonymous st_dev numbers. Group by
// the filesystem UUID and never allocate replicated chunks per disk.
let Some((uuid, members, single)) = block_id(&v.device).and_then(|id| btrfs.get(&id))
else {
continue;
};
(
members.clone(),
format!("btrfs:{uuid}"),
(!single).then(|| format!("Btrfs {uuid}")),
)
} else {
let Some(name) = ids.get(&v.devno) else {
continue;
};
let (members, stacked) = owners(name, nodes);
if members.is_empty() {
continue;
}
(
members,
v.devno.clone(),
stacked.then(|| nodes[name].label.clone()),
)
};
if !seen.insert(key.clone()) {
continue;
}
let used = v.total_bytes - v.free_bytes;
if let Some(name) = group {
let mut row = Disk::new(name, "group", v.total_bytes);
row.members = members;
row.used_bytes = Some(used);
row.available_bytes = Some(v.available_bytes);
rows.insert(format!("group:{key}"), row);
} else if members.len() == 1 {
let disk = members.first().unwrap();
if invalid.contains(disk) {
continue;
}
let Some(row) = rows.get_mut(disk) else {
continue;
};
let u = row.used_bytes.unwrap_or(0).checked_add(used);
let a = row
.available_bytes
.unwrap_or(0)
.checked_add(v.available_bytes);
if u.zip(a)
.and_then(|(u, a)| u.checked_add(a))
.is_some_and(|n| n <= row.total_bytes)
{
row.used_bytes = u;
row.available_bytes = a;
} else {
invalid.insert(disk.clone());
row.used_bytes = None;
row.available_bytes = None;
}
}
}
}
fn btrfs_members(
nodes: &Nodes,
ids: &BTreeMap<String, String>,
) -> BTreeMap<String, (String, BTreeSet<String>, bool)> {
let mut result = BTreeMap::new();
for fs in fs::read_dir("/sys/fs/btrfs")
.into_iter()
.flatten()
.flatten()
{
let mut members = BTreeSet::new();
let mut devices = Vec::new();
let mut direct = true;
for dev in fs::read_dir(fs.path().join("devices"))
.into_iter()
.flatten()
.flatten()
{
if let Some(id) = attr(dev.path().join("dev")) {
if let Some(name) = ids.get(&id) {
let (disks, stacked) = owners(name, nodes);
direct &= !stacked && disks.len() == 1;
members.extend(disks);
} else {
direct = false
}
devices.push(id);
} else {
direct = false
}
}
let single = direct && devices.len() == 1;
for id in devices {
result.insert(
id,
(
fs.file_name().to_string_lossy().into_owned(),
members.clone(),
single,
),
);
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
fn volume(id: &str, total: u64, free: u64, available: u64) -> Volume {
Volume {
devno: id.into(),
mount_point: "/".into(),
filesystem: "ext4".into(),
device: "".into(),
total_bytes: total,
free_bytes: free,
available_bytes: available,
available_percent: None,
total_inodes: 0,
free_inodes: 0,
available_inodes: 0,
available_inode_percent: None,
}
}
#[test]
fn partitions_count_once_pools_stay_separate_and_unmounted_is_unknown() {
let mut nodes = Nodes::new();
let mut rows = Rows::new();
for name in ["sda", "sdb", "vdc"] {
nodes.insert(
name.into(),
Node {
disk: true,
stacked: false,
parents: vec![],
label: name.into(),
},
);
rows.insert(name.into(), Disk::new(name.into(), "disk", 1000));
}
for (name, parents, stacked) in [
("sda1", vec!["sda"], false),
("sda2", vec!["sda"], false),
("md0", vec!["sda", "sdb"], true),
] {
nodes.insert(
name.into(),
Node {
disk: false,
stacked,
parents: parents.into_iter().map(String::from).collect(),
label: name.into(),
},
);
}
let ids = [("8:1", "sda1"), ("8:2", "sda2"), ("9:0", "md0")]
.map(|(a, b)| (a.into(), b.into()))
.into();
let a = volume("8:1", 100, 40, 30);
let b = volume("8:2", 200, 100, 80);
let raid = volume("9:0", 500, 250, 240);
project(
&nodes,
&ids,
&mut rows,
&[&a, &a, &b, &raid, &raid],
&BTreeMap::new(),
);
assert_eq!(
(rows["sda"].used_bytes, rows["sda"].available_bytes),
(Some(160), Some(110))
);
assert_eq!(rows["sdb"].used_bytes, None);
assert_eq!(rows["vdc"].available_bytes, None);
assert_eq!(
rows["group:9:0"].members,
["sda".into(), "sdb".into()].into()
);
assert_eq!(rows["group:9:0"].used_bytes, Some(250));
assert_eq!(rows.len(), 4);
}
}
fn zfs(rows: &mut Rows, nodes: &Nodes, ids: &BTreeMap<String, String>, volumes: &[&Volume]) {
let pools: BTreeSet<_> = volumes
.iter()
.filter(|v| v.filesystem == "zfs")
.filter_map(|v| v.device.split('/').next())
.collect();
if pools.is_empty() {
return;
}
// A pool's root dataset includes its children; don't sum dataset free space.
let usage = command(
"/usr/sbin/zfs",
&[
"list",
"-Hp",
"-t",
"filesystem",
"-o",
"name,used,available",
],
32768,
);
let topology = command("/usr/sbin/zpool", &["status", "-LP"], 32768);
let mut pool = "";
let mut members: BTreeMap<&str, BTreeSet<String>> = BTreeMap::new();
for line in topology
.data
.as_deref()
.unwrap_or("")
.lines()
.map(str::trim)
{
if let Some(p) = line.strip_prefix("pool: ") {
pool = p;
continue;
}
if let Some(name) = line
.split_whitespace()
.next()
.and_then(block_id)
.and_then(|id| ids.get(&id))
{
members
.entry(pool)
.or_default()
.extend(owners(name, nodes).0);
}
}
for p in pools {
let measure = usage
.data
.as_deref()
.unwrap_or("")
.lines()
.find_map(|line| {
let mut cols = line.split('\t');
if cols.next() != Some(p) {
return None;
}
let used = cols.next()?.parse::<u64>().ok()?;
let free = cols.next()?.parse::<u64>().ok()?;
Some((used, free, used.checked_add(free)?))
});
let mut row = Disk::new(format!("ZFS {p}"), "group", measure.map_or(0, |m| m.2));
row.used_bytes = measure.map(|m| m.0);
row.available_bytes = measure.map(|m| m.1);
row.members = members.remove(p).unwrap_or_default();
rows.insert(format!("group:zfs:{p}"), row);
}
}
pub(super) fn collect(volumes: &Observation<Vec<Observation<Volume>>>) -> Observation<Vec<Disk>> {
let Ok((nodes, ids, mut rows)) = topology() else {
return Observation::unavailable(SOURCE, Quality::Unknown, "BLOCK_TOPOLOGY_UNAVAILABLE");
};
let mounted: Vec<_> = volumes
.data
.iter()
.flatten()
.filter_map(|o| o.data.as_ref())
// An over-mounted or replaced mountpoint may now refer to another device.
.filter(|v| fs::metadata(&v.mount_point).is_ok_and(|m| devno(m.dev()) == v.devno))
.collect();
let btrfs = btrfs_members(&nodes, &ids);
project(&nodes, &ids, &mut rows, &mounted, &btrfs);
zfs(&mut rows, &nodes, &ids, &mounted);
let mut result: Vec<_> = rows.into_values().collect();
result.sort_by(|a, b| (a.kind, &a.name).cmp(&(b.kind, &b.name)));
Observation::valid(SOURCE, result).partial("FILESYSTEM_COVERAGE_PARTIAL")
}
+1 -1
View File
@@ -78,7 +78,7 @@ mod tests {
fn native_wire_scopes_preserve_quality_time_and_unsupported() {
let native = crate::inventory::collect_native();
let mapped = observations(&native).unwrap();
assert_eq!(mapped.len(), 12);
assert_eq!(mapped.len(), 13);
let smart = mapped.iter().find(|v| v["scope"] == "linux.smart").unwrap();
assert_eq!(smart["quality"], "unsupported");
assert!(smart["data"].is_null());