Compare commits

...
16 changed files with 724 additions and 6 deletions
+4
View File
@@ -1,4 +1,5 @@
//! Additive bounded passive metadata; the strict original Inventory stays intact.
mod opnsense;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
@@ -64,6 +65,9 @@ pub fn validate_for_scope(
now: DateTime<Utc>,
scope: &str,
) -> Result<Value, &'static str> {
if scope == "opnsense.base" {
return opnsense::validate(bytes);
}
if scope == "agent.update" {
if bytes.len() > 65536 {
return Err("AGENT_UPDATE_INVENTORY_SIZE");
@@ -0,0 +1,107 @@
//! Allowlisted wire document. No appliance configuration or VPN secrets on the wire.
use serde::Deserialize;
use serde_json::Value;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Snapshot {
os: String,
hostname: String,
architecture: String,
root_total_bytes: u64,
root_available_bytes: u64,
cpu_model: String,
logical_processors: u32,
memory_total_bytes: u64,
firewall: Rows,
vpn_configured: Rows,
wireguard: Rows,
openvpn: Rows,
ipsec: Rows,
hardware: Option<Hardware>,
software: Option<Records<Software>>,
interfaces: Option<Records<Interface>>,
users: Option<Rows>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Hardware {
device_type: String,
booted_at: chrono::DateTime<chrono::Utc>,
manufacturer: Option<String>,
bios_serial_number: Option<String>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Records<T> { status: String, entries: Vec<T> }
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Software { name: String, version: String }
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Interface { name: String, mac: Option<String>, addresses: Vec<String> }
impl<T> Records<T> {
fn valid(&self, limit: usize, check: impl Fn(&T) -> bool) -> bool {
matches!(self.status.as_str(),"ok"|"partial"|"unavailable")
&& (self.status != "unavailable" || self.entries.is_empty())
&& self.entries.len() <= limit && self.entries.iter().all(check)
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Rows {
status: String,
entries: Vec<String>,
}
impl Rows {
fn valid(&self) -> bool {
matches!(self.status.as_str(), "ok" | "partial" | "unavailable")
&& (self.status != "unavailable" || self.entries.is_empty())
&& self.entries.len() <= 128
&& self.entries.iter().all(|s| clean(s, 512))
}
}
fn clean(s: &str, limit: usize) -> bool {
!s.trim().is_empty() && s.len() <= limit && !s.chars().any(char::is_control)
}
pub(super) fn validate(bytes: &[u8]) -> Result<Value, &'static str> {
let err = "INVALID_OPNSENSE_INVENTORY";
if bytes.len() > 384 * 1024 { return Err(err); }
let s: Snapshot = serde_json::from_slice(bytes).map_err(|_| err)?;
if !s.os.starts_with("OPNsense ") || !clean(&s.os, 128)
|| !clean(&s.hostname, 255) || s.architecture != "amd64"
|| s.root_total_bytes == 0 || s.root_available_bytes > s.root_total_bytes
|| !clean(&s.cpu_model, 256) || s.logical_processors == 0 || s.memory_total_bytes == 0
|| ![&s.firewall,&s.vpn_configured,&s.wireguard,&s.openvpn,&s.ipsec].iter().all(|r| r.valid())
|| s.users.is_some_and(|r| !r.valid())
|| s.hardware.is_some_and(|h| !matches!(h.device_type.as_str(),"physical"|"virtual")
|| h.booted_at.timestamp() < 0
|| [h.manufacturer,h.bios_serial_number].into_iter().flatten().any(|v| !clean(&v,256)))
|| s.software.is_some_and(|r| !r.valid(1024, |p| clean(&p.name,256) && clean(&p.version,256)))
|| s.interfaces.is_some_and(|r| !r.valid(128, |i| clean(&i.name,128)
&& i.mac.as_ref().is_none_or(|m| clean(m,64)) && i.addresses.len()<=256
&& i.addresses.iter().all(|s| clean(s,128))))
{ return Err(err); }
serde_json::from_slice(bytes).map_err(|_| err)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn no_config_blob_and_no_false_empty_on_error() {
let rows = json!({"status":"ok","entries":[]});
let mut doc = json!({"os":"OPNsense 26.7","hostname":"test","architecture":"amd64",
"root_total_bytes":20,"root_available_bytes":10,"cpu_model":"Test","logical_processors":2,
"memory_total_bytes":10,"firewall":rows,"vpn_configured":rows,"wireguard":rows,"ipsec":rows,"openvpn":rows});
assert!(validate(doc.to_string().as_bytes()).is_ok());
doc["wireguard"]["private_key"] = json!("not-permitted");
assert!(validate(doc.to_string().as_bytes()).is_err());
doc["wireguard"] = json!({"status":"unavailable","entries":["peer"]});
assert!(validate(doc.to_string().as_bytes()).is_err());
}
}
+1 -1
View File
@@ -160,7 +160,7 @@ pub async fn ingest(
ensure!(
matches!(
s.scope.as_str(),
"linux.base" | "windows.base" | "agent.update"
"linux.base" | "windows.base" | "opnsense.base" | "agent.update"
) && s.schema_version == 1
&& s.chunk_count == 1
&& s.chunk_index == 0
+1 -1
View File
@@ -34,7 +34,7 @@ SELECT c.*,o.observed_at,o.received_at,o.data,o.source_id FROM classified c
LEFT JOIN LATERAL (
SELECT observed_at,received_at,data,source_id FROM observations
WHERE tenant_id=c.tenant_id AND agent_id=c.agent_id AND received_at<=$2
AND source_id IN ('linux.base','windows.base')
AND source_id IN ('linux.base','windows.base','opnsense.base')
ORDER BY observed_at DESC,received_at DESC,sequence DESC LIMIT 1
) o ON true
WHERE ($6::text IS NULL OR (c.lifecycle='current')=($6='current'))
+5 -3
View File
@@ -321,6 +321,7 @@ fn device_type(hardware: &Value) -> Option<DeviceType> {
fn inventory_device_type(data: &Value) -> Option<DeviceType> {
device_type(observed(data, "linux.hardware"))
.or_else(|| device_type(observed(data, "windows.hardware")))
.or_else(|| device_type(&data["hardware"]))
}
fn hostname_filter(query: Option<&str>) -> Result<&str, Error> {
@@ -517,7 +518,8 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool, hosts: &HashMap<Uuid, Ho
let ram_total = usage["ram_total_bytes"]
.as_u64()
.or_else(|| linux_memory["total_bytes"].as_u64())
.or_else(|| windows["memory_total_bytes"].as_u64());
.or_else(|| windows["memory_total_bytes"].as_u64())
.or_else(|| data["memory_total_bytes"].as_u64());
let device_type = inventory_device_type(&data);
let last = row.get::<Option<DateTime<Utc>>, _>("last_contact_at");
let revoked = row.get::<Option<DateTime<Utc>>, _>("revoked_at");
@@ -536,8 +538,8 @@ fn project(row: &PgRow, now: DateTime<Utc>, full: bool, hosts: &HashMap<Uuid, Ho
"hostname":data["hostname"].as_str(),"purpose":row.get::<Option<String>,_>("purpose"),
"device_type":device_type,"os":os_label(&data),
"disk_free_bytes":data["root_available_bytes"].as_u64(),"disk_total_bytes":data["root_total_bytes"].as_u64(),
"cpu_model":cpu["model"].as_str().or_else(||windows["cpu_model"].as_str()),
"cpu_logical_count":cpu["logical_processors"].as_u64().or_else(||windows["logical_processors_visible"].as_u64()),
"cpu_model":cpu["model"].as_str().or_else(||windows["cpu_model"].as_str()).or_else(||data["cpu_model"].as_str()),
"cpu_logical_count":cpu["logical_processors"].as_u64().or_else(||windows["logical_processors_visible"].as_u64()).or_else(||data["logical_processors"].as_u64()),
"cpu_usage_percent":cpu_usage,
"cpu_usage_observed_at":cpu_observed_at,
"ram_total_bytes":ram_total,
+18
View File
@@ -0,0 +1,18 @@
# OPNsense-Testclient
Test-VM `discovery-opnsense` auf `qemu:///system`, 2 vCPU, 1536 MiB RAM, 12 GiB virtuelle Platte. Basis: offizielles Nano-Image OPNsense 26.7 amd64; SHA-256 des komprimierten Images `28d5e2f37e40d87468a924e3006ef10e2ddc6de485b85333d9e3958c84d0cb9d`, gegen die Herstellerankündigung verglichen und die Signatur des entpackten Images geprüft. Start bei Bedarf: `virsh -c qemu:///system start discovery-opnsense`. Konsole: `virsh -c qemu:///system console discovery-opnsense`. Das LAN `192.168.247.0/24` ist eine isolierte libvirt-Bridge, das WAN `192.168.246.0/24` verwendet NAT. Management vom Notebook: `https://192.168.247.1`. Der Hostpin stammt aus der seriellen Konsole; Zugangswerte liegen ausschließlich im privaten Zugangsspeicher des ursprünglichen Checkouts.
## Quelle und Entwicklungsinstallation
Der OPNsense-eigene Python-Interpreter führt zwei Zipapps aus. Der lokale Collector läuft mit den für PF, VPN-Sockets und Konfiguration nötigen Rechten. Er veröffentlicht ausschließlich den ausgewählten Bestand atomar in `/var/db/discovery-observations/inventory.json`. Das Dienstkonto `discovery` kann die Datei lesen, aber weder Collector noch Cache verändern. Nur dieses Konto hält die Geräteidentität und nutzt den ausgehenden HTTPS-Kanal. Für die Erfassung wird keine eingehende Netzwerkverbindung benötigt.
1. In einer Build-venv `grpcio-tools==1.76.0` und `protobuf==6.33.5` installieren. `python deploy/opnsense/build.py ARTIFAKTORDNER` generiert die Protobuf-Bindung aus `proto/agent.proto` und packt die portable Python-Laufzeit mit ein. `python deploy/opnsense/test_agent.py ARTIFAKTORDNER/agent.pyz` prüft die Wiederholung einer nicht bestätigten Inventarmeldung mit unveränderter Nachrichtenidentität.
2. Über den gepinnten Entwicklungszugang `agent.pyz`, `collect.pyz`, `discovery_agent` und den **bestätigten öffentlichen CA-Anker des Gerätegateways** als `ca.pem` in einen privaten Stagingordner im Gast übertragen. `sh install-dev.sh STAGINGORDNER` auf dem Gast ausführen. Diese Strecke ist eine direkte Entwicklungsbereitstellung, kein signierter Releasekanal.
3. Bei einer frischen Installation in der Discovery-WebUI den Testkunden wählen und einen Einmalcode erzeugen. Den Code über geschützte Standardeingabe an `su -m discovery -c '/usr/local/bin/python3 -I /usr/local/libexec/discovery-agent/agent.pyz enroll'` übergeben. Danach `service discovery_agent start`. Bei vorhandener Identität nicht neu enrollen. Der Dienst verwaltet beide Prozesse; `rc.conf.local` aktiviert sie beim Start.
4. In **Geräte** den Hostnamen öffnen. Das Inventar wird ungefähr einmal pro Minute erhoben, der Heartbeat etwa alle 20 Sekunden gesendet. Das Betriebssystem liefert CPU-Zeitdifferenzen und eine RAM-Schätzung aus freien und inaktiven Seiten. Fehlgeschlagene Quellen erscheinen als „Nicht ermittelbar“, ein erfolgreicher leerer Befund gesondert. Pro Liste werden höchstens 128 Zeilen gemeldet; Auszüge sind gekennzeichnet.
**VPN:** Beschreibungen und Aktivierung der OPNsense-Modelle für WireGuard-Instanzen/-Peers, OpenVPN-Instanzen und IPsec-Verbindungen. Separat: WireGuard-Handshakes (kein dauerhafter Online-Status), OpenVPN-Managementstatus und IKE-Sicherheitsassoziationen. Ohne erreichbaren IPsec-Dienst ist der Laufzeitstatus unbekannt. Keine privaten Schlüssel, PSKs, Zertifikate oder vollständige Gerätekonfiguration im Inventar.
**Firewall:** aktiver PF-Hauptregelsatz in Kernelreihenfolge, einschließlich automatisch erzeugter Regeln. Kein Editor und kein rekursives Auslesen der Anchor-Regelsätze.
**Grenzen:** Das passive Profil meldet keine Skript-, Root- oder Release-Capability. Der bestehende Linux-Jobpfad verwendet Linux-spezifische Uhren, Helper und Interpreterprüfung. Für Parität benötigt OPNsense eigene native Adapter für den signierten Abgleich, den Startgrant und das Ausführungsjournal sowie einen eigenen signierten Installer-/Updatekanal. Der OPNsense-Gast darf nicht als Linuxziel für APT- oder Bash-Vorlagen angeboten werden.
+215
View File
@@ -0,0 +1,215 @@
"""Unprivileged OPNsense telemetry channel. Input never selects a local command."""
import hashlib
import http.client
import json
import os
from pathlib import Path
import re
import sqlite3
import ssl
import subprocess
import sys
import time
import uuid
import agent_pb2 as pb
STATE = Path('/var/db/discovery-agent')
CA = '/usr/local/etc/discovery-agent/ca.pem'
HOST = 'discovery.ease-technologies.de'
MEDIA = 'application/x-protobuf'
VERSION = '0.1.0-opnsense.4'
def post(ctx, path, data, media):
conn = http.client.HTTPSConnection(HOST, context=ctx, timeout=15)
try:
conn.request('POST', '/agent/v1/'+path, data, {'Content-Type':media, 'Accept':media})
res = conn.getresponse()
if res.status != 200 or res.getheader('Content-Type', '').split(';')[0] != media:
raise ValueError(f'{path}: HTTP {res.status}')
body = res.read(1024*1024 + 1)
if len(body) > 1024*1024:
raise ValueError('response size')
return body
finally:
conn.close()
def context():
return ssl.create_default_context(cafile=CA)
def enroll():
# Existing keys are never overwritten, including after an ambiguous POST.
key = STATE/'device.key'
if key.exists() or (STATE/'identity.json').exists():
raise RuntimeError('installation already initialized')
token = sys.stdin.readline(128).strip()
if not re.fullmatch('[0-9a-f]{64}', token):
raise ValueError('invalid enrollment code')
csr = subprocess.check_output(['/usr/bin/openssl','req','-new','-newkey','ec','-pkeyopt',
'ec_paramgen_curve:P-256','-nodes','-subj','/CN=discovery-agent','-keyout',str(key)],
stderr=subprocess.DEVNULL)
result = json.loads(post(context(), 'enroll', json.dumps(dict(token=token, csr_pem=csr.decode())),
'application/json'))
for name in ('agent_id','asset_id'):
uuid.UUID(result[name])
(STATE/'device.pem').write_text(result.pop('certificate_pem'))
context().load_cert_chain(STATE/'device.pem', key)
(STATE/'identity.json').write_text(json.dumps(result))
def durable_write(path, data):
tmp = path.with_suffix('.new')
with tmp.open('wb') as f:
f.write(data)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
fd = os.open(STATE, os.O_DIRECTORY)
try:
os.fsync(fd)
finally:
os.close(fd)
def renew(ctx, identity):
# Reuse a durable request after a lost reply; gateway renewal is idempotent
# for the same CSR. Keep the currently working certificate until key match.
cert, key, csr = (STATE/n for n in ('device.pem','device.key','renewal.csr'))
if not csr.exists():
check = subprocess.run(['/usr/bin/openssl','x509','-in',str(cert),'-noout','-checkend','604800'],
stdout=subprocess.DEVNULL, check=False)
if check.returncode == 0:
return False
durable_write(csr,subprocess.check_output(['/usr/bin/openssl','req','-new','-key',str(key),
'-subj','/CN=discovery-agent']))
result = json.loads(post(ctx,'certificates/renew',json.dumps({'csr_pem':csr.read_text()}),'application/json'))
if result['agent_id'] != identity['agent_id']:
raise ValueError('renewal identity mismatch')
candidate = STATE/'candidate.pem'
durable_write(candidate,result['certificate_pem'].encode())
context().load_cert_chain(candidate,key)
durable_write(cert,candidate.read_bytes())
candidate.unlink()
csr.unlink()
return True
def usage(previous):
def counters(*keys):
return [int(x) for x in subprocess.check_output(['/sbin/sysctl', '-n', *keys],timeout=3).split()]
ticks = counters('kern.cp_time')
free, inactive, page, total = counters('vm.stats.vm.v_free_count','vm.stats.vm.v_inactive_count',
'hw.pagesize','hw.physmem')
sample = pb.SystemUsage(ram_available_bytes=min((free+inactive)*page,total),ram_total_bytes=total)
if previous and len(previous) == len(ticks) and all(x >= y for x,y in zip(ticks,previous)):
delta = sum(ticks)-sum(previous)
if delta > 0:
sample.cpu_usage_basis_points = 10000*(delta-ticks[-1]+previous[-1])//delta
return sample,ticks
def session(db, identity, boot, instance):
ctx = context()
ctx.load_cert_chain(STATE/'device.pem', STATE/'device.key')
generation = 0
def send(kind, payload, captured=None):
# A lost inventory ACK replays the exact durable envelope, including its
# original time, boot and sequence; only the connection fence changes.
with db:
pending = db.execute('SELECT envelope FROM pending WHERE id=1').fetchone() if kind=='inventory_snapshot' else None
if pending:
e = pb.Envelope.FromString(pending[0])
if e.agent_id != identity['agent_id']:
raise ValueError('pending snapshot identity')
e.connection_generation = generation
else:
seq = db.execute('UPDATE sequence SET n=n+1 RETURNING n').fetchone()[0]
e = pb.Envelope(protocol_major=1, message_id=str(uuid.uuid4()), agent_id=identity['agent_id'],
enrollment_generation=1, connection_generation=generation, boot_id=boot,
sequence=seq, observed_at=captured or time.time_ns()//1_000_000)
getattr(e, kind).CopyFrom(payload)
if kind == 'inventory_snapshot':
db.execute('INSERT INTO pending VALUES(1,?,?)',(e.SerializeToString(),e.observed_at))
batch = pb.MessageBatch(protocol_major=1, messages=[e])
reply = pb.MessageBatch.FromString(post(ctx, 'messages', batch.SerializeToString(), MEDIA))
if reply.protocol_major != 1 or len(reply.messages) != 1:
raise ValueError('unexpected reply')
r = reply.messages[0]
if (r.protocol_major,r.agent_id,r.enrollment_generation,r.boot_id,r.sequence) != (
1,e.agent_id,1,e.boot_id,e.sequence):
raise ValueError('reply identity')
if kind == 'client_hello':
if (r.WhichOneof('payload') != 'server_hello' or r.connection_generation == 0 or
r.connection_generation != r.server_hello.connection_generation or
r.server_hello.max_message_bytes != 1024*1024):
raise ValueError('invalid server hello')
return r.connection_generation
if (r.connection_generation != generation or r.WhichOneof('payload') != 'ack'
or r.ack.message_id != e.message_id):
raise ValueError('unconfirmed message')
if kind == 'inventory_snapshot':
with db:
db.execute('DELETE FROM pending WHERE id=1')
if renew(ctx, identity):
ctx = context()
ctx.load_cert_chain(STATE/'device.pem', STATE/'device.key')
generation = send('client_hello',pb.ClientHello(agent_version=VERSION, capabilities=['heartbeat/1','opnsense.base/1']))
last = None
previous = None
check_cert = time.monotonic()+12*3600
while True:
if time.monotonic() >= check_cert:
if renew(ctx, identity):
return
check_cert = time.monotonic()+12*3600
sample,previous = usage(previous)
send('heartbeat',pb.Heartbeat(instance_id=instance,uptime_seconds=int(time.clock_gettime(time.CLOCK_UPTIME)),
system_usage=sample))
with db:
# The ingestion contract accepts observations for 72 hours.
db.execute('DELETE FROM pending WHERE observed < ?', (time.time_ns()//1_000_000 - 72*3600*1000,))
if db.execute('SELECT 1 FROM pending').fetchone():
send('inventory_snapshot',None)
file = Path('/var/db/discovery-observations/inventory.json')
if file.exists():
with file.open('rb') as f:
stamp = os.fstat(f.fileno()).st_mtime_ns
if stamp != last and 0 <= time.time()-stamp/1e9 <= 120:
raw = f.read(384*1024+1)
if len(raw) > 384*1024:
raise ValueError('snapshot size')
send('inventory_snapshot',pb.InventorySnapshot(snapshot_id=str(uuid.uuid4()),
scope='opnsense.base',schema_version=1,chunk_count=1,data=raw,
sha256=hashlib.sha256(raw).digest(),completeness='partial'),stamp//1_000_000)
last = stamp
time.sleep(20)
def main():
os.umask(0o077)
if sys.argv[1:] == ['enroll']:
return enroll()
identity = json.loads((STATE/'identity.json').read_bytes())
db = sqlite3.connect(STATE/'channel.sqlite')
with db:
db.execute('CREATE TABLE IF NOT EXISTS sequence(id INTEGER PRIMARY KEY CHECK(id=1),n INTEGER NOT NULL)')
db.execute('CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),envelope BLOB NOT NULL,observed INTEGER NOT NULL)')
db.execute('INSERT OR IGNORE INTO sequence VALUES(1,0)')
started = subprocess.check_output(['/sbin/sysctl','-n','kern.boottime']).decode().split('}')[0]
boot = str(uuid.uuid5(uuid.UUID(identity['agent_id']),started))
instance = str(uuid.uuid4())
while True:
try:
session(db, identity, boot, instance)
except Exception as exc:
print('OPNsense channel:', type(exc).__name__, str(exc)[:160], flush=True)
time.sleep(30)
if __name__ == '__main__':
main()
+33
View File
@@ -0,0 +1,33 @@
"""Build two FreeBSD Python zipapps; use grpcio-tools 1.76.0 + protobuf 6.33.5."""
import importlib.metadata as metadata
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import zipapp
here = Path(__file__).resolve().parent
out = Path(sys.argv[1])
out.mkdir(exist_ok=True, parents=True)
if metadata.version('protobuf') != '6.33.5' or metadata.version('grpcio-tools') != '1.76.0':
sys.exit('Unexpected protocol builder dependencies')
with tempfile.TemporaryDirectory() as tmp:
work = Path(tmp)
proto = here.parents[1]/'proto'
subprocess.run([sys.executable,'-m','grpc_tools.protoc','-I'+str(proto),'--python_out='+tmp,
str(proto/'agent.proto')],check=True)
dist = metadata.distribution('protobuf')
for f in dist.files:
if str(f).startswith('google/protobuf/') and f.suffix == '.py':
dst = work/f
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(dist.locate_file(f), dst)
(work/'google/__init__.py').touch()
license_file = next(f for f in dist.files if f.name=='LICENSE' and '.dist-info/' in str(f))
shutil.copyfile(dist.locate_file(license_file),work/'LICENSE-protobuf')
shutil.copyfile(here/'agent.py',work/'__main__.py')
zipapp.create_archive(work,out/'agent.pyz',compressed=True)
with tempfile.TemporaryDirectory() as tmp:
shutil.copyfile(here/'collect.py',Path(tmp)/'__main__.py')
zipapp.create_archive(tmp,out/'collect.pyz',compressed=True)
+161
View File
@@ -0,0 +1,161 @@
"""Bounded local inventory projection. Runs independently of the network agent."""
import datetime
import json
import os
from pathlib import Path
import re
import resource
import socket
import subprocess
import tempfile
import time
import xml.etree.ElementTree as ET
CACHE = Path('/var/db/discovery-observations')
def run(*args):
# Appliance commands may produce much more than one snapshot. Cap the
# producer before reading its output, including on a misconfigured host.
def limits():
resource.setrlimit(resource.RLIMIT_FSIZE, (2**21, 2**21))
with tempfile.TemporaryFile() as out:
subprocess.run(args, stdout=out, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
check=True, timeout=10, preexec_fn=limits)
out.seek(0)
return out.read(2**21).decode('utf8', errors='replace').strip()
def sysctl(name):
return run('/sbin/sysctl', '-n', name)
def rows(provider):
entries = []
partial = False
try:
for text in provider():
if len(entries) == 128:
partial = True
break
text = ' '.join(text.split())
if len(text.encode()) > 512:
partial = True
text = text.encode()[:509].decode(errors='ignore') + '…'
if text:
entries.append(text)
except (OSError, ValueError, KeyError, TypeError, ET.ParseError, subprocess.SubprocessError):
return dict(status='partial' if entries else 'unavailable', entries=entries)
return dict(status='partial' if partial else 'ok', entries=entries)
def configured():
# Project only these model paths and display fields. Never serialize XML.
with open('/conf/config.xml', 'rb') as f:
data = f.read(8 * 1024 * 1024 + 1)
if len(data) > 8 * 1024 * 1024:
raise ValueError('config size')
root = ET.fromstring(data)
for title, path, field in [
('WireGuard-Instanz', 'wireguard/server/servers/server', 'name'),
('WireGuard-Peer', 'wireguard/client/clients/client', 'name'),
('IPsec', 'Swanctl/Connections/Connection', 'description'),
('OpenVPN', 'OpenVPN/Instances/Instance', 'description'),
]:
for item in root.findall('OPNsense/' + path):
enabled = {'0': 'deaktiviert', '1': 'aktiviert'}.get(item.findtext('enabled'), 'Status unbekannt')
yield f"{title} · {item.findtext(field) or item.get('uuid') or '–'} · {enabled}"
def wireguard():
for line in run('/usr/bin/wg', 'show', 'all', 'latest-handshakes').splitlines():
interface, public_key, epoch = line.split()
stamp = datetime.datetime.fromtimestamp(int(epoch), datetime.timezone.utc).isoformat() if int(epoch) else 'nie'
yield f'{interface} · {public_key} · {stamp}'
def openvpn():
data = json.loads(run('/usr/local/opnsense/scripts/openvpn/ovpn_status.py', '--options', 'client,server'))
for role in ('server', 'client'):
for instance, status in data.get(role, {}).items():
yield f"{role} {instance} · {status.get('status', 'unbekannt')}"
for peer in status.get('client_list', []):
yield f"{instance} · {peer.get('common_name', '–')} · {peer.get('real_address', '–')}"
def ipsec():
# Only IKE summary lines, not CHILD_SA details or authentication settings.
for line in run('/usr/local/sbin/swanctl', '--list-sas').splitlines():
if re.match(r'^\S.*: #\d+, ', line):
yield line
def interfaces():
item = None
for line in run('/sbin/ifconfig','-f','inet:cidr,inet6:cidr','-a').splitlines():
if line and not line[0].isspace():
if item:
yield item
item = dict(name=line.split(':',1)[0],mac=None,addresses=[])
elif item:
fields = line.split()
if len(fields)>1 and fields[0]=='ether':
item['mac'] = fields[1]
elif len(fields)>1 and fields[0] in ('inet','inet6'):
item['addresses'].append(fields[1])
if item:
yield item
def packages():
for line in run('/usr/sbin/pkg','query','%n\t%v').splitlines():
name,version = line.split('\t',1)
yield dict(name=name,version=version)
def inventory_rows(fn, limit):
entries = []
try:
for item in fn():
if len(entries) >= limit:
return dict(status='partial',entries=entries)
entries.append(item)
return dict(status='ok',entries=entries)
except (OSError,ValueError,subprocess.SubprocessError):
return dict(status='partial' if entries else 'unavailable',entries=entries)
def snapshot():
disk = os.statvfs('/')
boot = int(re.search(r'sec = (\d+)',sysctl('kern.boottime'))[1])
hardware = {'device_type':'physical' if sysctl('kern.vm_guest')=='none' else 'virtual',
'booted_at':datetime.datetime.fromtimestamp(boot,datetime.timezone.utc).isoformat()}
for key,name in [('smbios.system.maker','manufacturer'),('smbios.system.serial','bios_serial_number')]:
try:
value = run('/bin/kenv',key)
if value and value.lower() not in ('none','unknown','not specified','default string'):
hardware[name]=value[:128]
except (OSError,subprocess.SubprocessError):
pass
return dict(os=run('/usr/local/sbin/opnsense-version'), hostname=socket.gethostname(),
architecture=os.uname().machine, root_total_bytes=disk.f_blocks*disk.f_frsize,
root_available_bytes=max(0,disk.f_bavail)*disk.f_frsize, cpu_model=sysctl('hw.model'),
logical_processors=int(sysctl('hw.ncpu')), memory_total_bytes=int(sysctl('hw.physmem')),
firewall=rows(lambda: run('/sbin/pfctl', '-sr').splitlines()), vpn_configured=rows(configured),
wireguard=rows(wireguard), openvpn=rows(openvpn), ipsec=rows(ipsec),
hardware=hardware, interfaces=inventory_rows(interfaces,128), software=inventory_rows(packages,1024),
users=rows(lambda:sorted(set(run('/usr/bin/users').split()))))
if __name__ == '__main__':
while True:
try:
with tempfile.NamedTemporaryFile(dir=CACHE, delete=False) as f:
f.write(json.dumps(snapshot(), ensure_ascii=False).encode())
os.fchmod(f.fileno(), 0o640)
os.replace(f.name, CACHE/'inventory.json')
except Exception as exc:
print('OPNsense collection:', type(exc).__name__, flush=True)
if 'f' in locals():
Path(f.name).unlink(missing_ok=True)
time.sleep(60)
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
# PROVIDE: discovery_agent
# REQUIRE: NETWORKING
# KEYWORD: shutdown
. /etc/rc.subr
name=discovery_agent
rcvar=discovery_agent_enable
load_rc_config "$name"
: "${discovery_agent_enable:=NO}"
start_cmd=discovery_start
stop_cmd=discovery_stop
discovery_start() {
for part in collect agent; do
pid=/var/run/discovery-$part.pid
[ -z "$(check_pidfile "$pid" /usr/sbin/daemon)" ] || continue
user=root
[ "$part" != agent ] || user=discovery
/usr/sbin/daemon -f -r -R 10 -P "$pid" -u "$user" -S -T "discovery-$part" \
/usr/local/bin/python3 -I /usr/local/libexec/discovery-agent/"$part".pyz || return 1
done
}
discovery_stop() {
for part in agent collect; do
pid=$(check_pidfile "/var/run/discovery-$part.pid" /usr/sbin/daemon)
[ -z "$pid" ] || kill -TERM "$pid"
done
}
run_rc_command "$1"
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# Direct artifact deployment in the test guest. Enrollment is a separate step.
set -eu
[ "$(id -u)" = 0 ] && [ "$(uname -s)" = FreeBSD ] && [ -x /usr/local/sbin/opnsense-version ]
[ "$#" = 1 ] || { echo 'Usage: install-dev.sh /path/to/staged-artifacts' >&2; exit 2; }
src=$1
for f in agent.pyz collect.pyz discovery_agent ca.pem; do [ -f "$src/$f" ]; done
/usr/bin/openssl x509 -in "$src/ca.pem" -noout -checkend 0 >/dev/null
pw groupshow discovery >/dev/null 2>&1 || pw groupadd discovery
pw usershow discovery >/dev/null 2>&1 || pw useradd discovery -g discovery -d /var/db/discovery-agent -s /usr/sbin/nologin
[ "$(id -gn discovery)" = discovery ] && [ "$(id -u discovery)" != 0 ]
app=/usr/local/libexec/discovery-agent
conf=/usr/local/etc/discovery-agent
rc=/usr/local/etc/rc.d/discovery_agent
install -d -o root -g wheel -m 0755 "$app" "$conf"
install -d -o discovery -g discovery -m 0700 /var/db/discovery-agent
install -d -o root -g discovery -m 2750 /var/db/discovery-observations
if [ -e "$conf/ca.pem" ]; then
cmp -s "$conf/ca.pem" "$src/ca.pem" || { echo 'Existing trust anchor differs' >&2; exit 1; }
fi
for part in collect agent; do
install -o root -g wheel -m 0444 "$src/$part.pyz" "$app/$part.next"
done
if [ -x "$rc" ]; then
"$rc" onestop
for _ in 1 2 3 4 5; do
[ -f /var/run/discovery-agent.pid ] || [ -f /var/run/discovery-collect.pid ] || break
sleep 1
done
if [ -f /var/run/discovery-agent.pid ] || [ -f /var/run/discovery-collect.pid ]; then
echo 'Existing service still running' >&2; exit 1
fi
fi
for part in collect agent; do mv -f "$app/$part.next" "$app/$part.pyz"; done
install -o root -g wheel -m 0444 "$src/ca.pem" "$conf/ca.pem"
install -o root -g wheel -m 0555 "$src/discovery_agent" "$rc"
sysrc -f /etc/rc.conf.local discovery_agent_enable=YES >/dev/null
if [ -s /var/db/discovery-agent/identity.json ]; then
"$rc" start
else
echo 'Ready for enrollment under the discovery service account.'
fi
+67
View File
@@ -0,0 +1,67 @@
"""Run from the protocol-builder venv: python test_agent.py <built agent.pyz>."""
import importlib.util
import os
from pathlib import Path
import sqlite3
import sys
import tempfile
import unittest
from unittest.mock import patch
import uuid
sys.path.insert(0,sys.argv.pop(1))
spec = importlib.util.spec_from_file_location('opn_channel', Path(__file__).with_name('agent.py'))
a = importlib.util.module_from_spec(spec)
spec.loader.exec_module(a)
class Reconnect(unittest.TestCase):
def test_lost_ack_keeps_original_identity_after_restart(self):
with tempfile.TemporaryDirectory() as temp:
db = sqlite3.connect(':memory:')
db.executescript('CREATE TABLE sequence(id INTEGER PRIMARY KEY,n INTEGER);INSERT INTO sequence VALUES(1,0);'
'CREATE TABLE pending(id INTEGER PRIMARY KEY,envelope BLOB,observed INTEGER);')
cache = Path(temp)/'inventory.json'
cache.write_text('{}')
identity = {'agent_id':str(uuid.uuid4())}
instance = str(uuid.uuid4())
captures = []
generation = 0
def gateway(_ctx,path,data,media):
nonlocal generation
req = a.pb.MessageBatch.FromString(data).messages[0]
reply = a.pb.Envelope()
reply.CopyFrom(req)
kind = req.WhichOneof('payload')
if kind == 'client_hello':
generation += 1
reply.connection_generation = generation
reply.server_hello.CopyFrom(a.pb.ServerHello(connection_generation=generation,max_message_bytes=1024*1024))
else:
if kind == 'inventory_snapshot':
captures.append(req)
if len(captures)==1: raise OSError('lost ACK')
reply.ack.CopyFrom(a.pb.Ack(message_id=req.message_id))
return a.pb.MessageBatch(protocol_major=1,messages=[reply]).SerializeToString()
with patch.object(a,'context'), patch.object(a,'renew',return_value=False), \
patch.object(a,'post',gateway), patch.object(a,'Path',return_value=cache), \
patch.object(a,'usage',return_value=(a.pb.SystemUsage(),[])), \
patch.object(a.time,'clock_gettime',return_value=100), \
patch.object(a.time,'CLOCK_UPTIME',1,create=True), patch.object(a.time,'sleep',side_effect=StopIteration):
with self.assertRaises(OSError):
a.session(db,identity,str(uuid.uuid4()),instance)
self.assertIsNotNone(db.execute('SELECT 1 FROM pending').fetchone())
os.unlink(cache)
with self.assertRaises(StopIteration):
a.session(db,identity,str(uuid.uuid4()),instance)
self.assertIsNone(db.execute('SELECT 1 FROM pending').fetchone())
self.assertEqual(len(captures),2)
self.assertNotEqual(captures[0].connection_generation,captures[1].connection_generation)
captures[0].connection_generation=captures[1].connection_generation
self.assertEqual(captures[0],captures[1])
if __name__=='__main__':
unittest.main()
+6
View File
@@ -0,0 +1,6 @@
SET LOCAL lock_timeout = '3s';
SET LOCAL statement_timeout = '30s';
ALTER TABLE observations DROP CONSTRAINT observations_source_id_check;
ALTER TABLE observations ADD CONSTRAINT observations_source_id_check
CHECK (source_id IN ('linux.base','windows.base','agent.update','opnsense.base')) NOT VALID;
ALTER TABLE observations VALIDATE CONSTRAINT observations_source_id_check;
+9 -1
View File
@@ -14,7 +14,15 @@ function values(value: unknown) {
return Array.isArray(value) ? value.flatMap(value => text(value) ?? []) : [];
}
function observation(device: RecordValue, name: string) {
function observation(device: RecordValue, name: string): RecordValue | undefined {
if (device.source_id === 'opnsense.base') {
const inv = object(device.inventory);
if (name === 'hardware' || name === 'os') return { data: inv.hardware };
const report = object(inv[name]);
if (report.status === 'ok' || report.status === 'partial')
return { data: report.entries, truncated: report.status === 'partial' };
return undefined;
}
const observations = object(device.inventory).observations;
return Array.isArray(observations) ? observations.map(object).find(item =>
item.quality === 'valid' && (item.scope === 'linux.' + name || item.scope === 'windows.' + name)
+2
View File
@@ -17,6 +17,7 @@ import { allowed } from './safety';
import { CatalogCheckNotice } from './catalog-check';
import { devicePlatform } from './script-catalog';
import { DeviceInventoryPanels, systemDetails } from './device-inventory';
import { OPNsenseDetails } from './opnsense';
import { DeviceCustomerTransfer } from './device-customer-transfer';
import { HypervisorAssignment, HypervisorLabel } from './device-hypervisor';
import { ErrorBox, errorText, useResource } from './resource';
@@ -300,6 +301,7 @@ function DeviceDetail({
success={purposeSaved}
onSaved={() => { setPurposeSaved(true); detail.reload(); }}
/>
<OPNsenseDetails device={d} />
<DeviceInventoryPanels device={d} />
{volumes.length > 0 && (
<section>
+24
View File
@@ -0,0 +1,24 @@
import type { RecordValue } from './api';
export function OPNsenseDetails({ device }: { device: RecordValue }) {
if (device.source_id !== 'opnsense.base') return null;
const inventory = device.inventory as RecordValue | null;
function list(key: string, title: string) {
const report = inventory?.[key] as RecordValue | undefined;
const available = report?.status === 'ok' || report?.status === 'partial';
const rows = Array.isArray(report?.entries) ? report.entries.filter((x): x is string => typeof x === 'string') : [];
return <div key={key}><h3>{title}</h3>{!available ? <p>Nicht ermittelbar</p> : <>
{report?.status === 'partial' && <p>Auszug – unvollständige Liste</p>}
{rows.length ? <ol>{rows.map((line, i) => <li key={i}><code>{line}</code></li>)}</ol> : <p>Keine Einträge gemeldet.</p>}
</>}</div>;
}
return <>
<section><h2>VPN-Verbindungen</h2>
{list('vpn_configured', 'Konfigurierte VPNs')}
{list('wireguard', 'WireGuard · letzter Handshake je Peer')}
{list('openvpn', 'OpenVPN · Laufzeitstatus')}
{list('ipsec', 'IPsec · aktive Sicherheitsassoziationen')}
</section>
<section><h2>Firewallregeln</h2>{list('firewall', 'Aktive PF-Filterregeln · Hauptregelsatz in Auswertungsreihenfolge')}</section>
</>;
}