"""Unprivileged OPNsense telemetry channel. Input never selects a local command.""" import hashlib import http.client import json import os from pathlib import Path import re import sqlite3 import ssl import subprocess import sys import time import uuid import agent_pb2 as pb STATE = Path('/var/db/discovery-agent') CA = '/usr/local/etc/discovery-agent/ca.pem' HOST = 'discovery.ease-technologies.de' MEDIA = 'application/x-protobuf' VERSION = '0.1.0-opnsense.4' def post(ctx, path, data, media): conn = http.client.HTTPSConnection(HOST, context=ctx, timeout=15) try: conn.request('POST', '/agent/v1/'+path, data, {'Content-Type':media, 'Accept':media}) res = conn.getresponse() if res.status != 200 or res.getheader('Content-Type', '').split(';')[0] != media: raise ValueError(f'{path}: HTTP {res.status}') body = res.read(1024*1024 + 1) if len(body) > 1024*1024: raise ValueError('response size') return body finally: conn.close() def context(): return ssl.create_default_context(cafile=CA) def enroll(): # Existing keys are never overwritten, including after an ambiguous POST. key = STATE/'device.key' if key.exists() or (STATE/'identity.json').exists(): raise RuntimeError('installation already initialized') token = sys.stdin.readline(128).strip() if not re.fullmatch('[0-9a-f]{64}', token): raise ValueError('invalid enrollment code') csr = subprocess.check_output(['/usr/bin/openssl','req','-new','-newkey','ec','-pkeyopt', 'ec_paramgen_curve:P-256','-nodes','-subj','/CN=discovery-agent','-keyout',str(key)], stderr=subprocess.DEVNULL) result = json.loads(post(context(), 'enroll', json.dumps(dict(token=token, csr_pem=csr.decode())), 'application/json')) for name in ('agent_id','asset_id'): uuid.UUID(result[name]) (STATE/'device.pem').write_text(result.pop('certificate_pem')) context().load_cert_chain(STATE/'device.pem', key) (STATE/'identity.json').write_text(json.dumps(result)) def durable_write(path, data): tmp = path.with_suffix('.new') with tmp.open('wb') as f: f.write(data) f.flush() os.fsync(f.fileno()) os.replace(tmp, path) fd = os.open(STATE, os.O_DIRECTORY) try: os.fsync(fd) finally: os.close(fd) def renew(ctx, identity): # Reuse a durable request after a lost reply; gateway renewal is idempotent # for the same CSR. Keep the currently working certificate until key match. cert, key, csr = (STATE/n for n in ('device.pem','device.key','renewal.csr')) if not csr.exists(): check = subprocess.run(['/usr/bin/openssl','x509','-in',str(cert),'-noout','-checkend','604800'], stdout=subprocess.DEVNULL, check=False) if check.returncode == 0: return False durable_write(csr,subprocess.check_output(['/usr/bin/openssl','req','-new','-key',str(key), '-subj','/CN=discovery-agent'])) result = json.loads(post(ctx,'certificates/renew',json.dumps({'csr_pem':csr.read_text()}),'application/json')) if result['agent_id'] != identity['agent_id']: raise ValueError('renewal identity mismatch') candidate = STATE/'candidate.pem' durable_write(candidate,result['certificate_pem'].encode()) context().load_cert_chain(candidate,key) durable_write(cert,candidate.read_bytes()) candidate.unlink() csr.unlink() return True def usage(previous): def counters(*keys): return [int(x) for x in subprocess.check_output(['/sbin/sysctl', '-n', *keys],timeout=3).split()] ticks = counters('kern.cp_time') free, inactive, page, total = counters('vm.stats.vm.v_free_count','vm.stats.vm.v_inactive_count', 'hw.pagesize','hw.physmem') sample = pb.SystemUsage(ram_available_bytes=min((free+inactive)*page,total),ram_total_bytes=total) if previous and len(previous) == len(ticks) and all(x >= y for x,y in zip(ticks,previous)): delta = sum(ticks)-sum(previous) if delta > 0: sample.cpu_usage_basis_points = 10000*(delta-ticks[-1]+previous[-1])//delta return sample,ticks def session(db, identity, boot, instance): ctx = context() ctx.load_cert_chain(STATE/'device.pem', STATE/'device.key') generation = 0 def send(kind, payload, captured=None): # A lost inventory ACK replays the exact durable envelope, including its # original time, boot and sequence; only the connection fence changes. with db: pending = db.execute('SELECT envelope FROM pending WHERE id=1').fetchone() if kind=='inventory_snapshot' else None if pending: e = pb.Envelope.FromString(pending[0]) if e.agent_id != identity['agent_id']: raise ValueError('pending snapshot identity') e.connection_generation = generation else: seq = db.execute('UPDATE sequence SET n=n+1 RETURNING n').fetchone()[0] e = pb.Envelope(protocol_major=1, message_id=str(uuid.uuid4()), agent_id=identity['agent_id'], enrollment_generation=1, connection_generation=generation, boot_id=boot, sequence=seq, observed_at=captured or time.time_ns()//1_000_000) getattr(e, kind).CopyFrom(payload) if kind == 'inventory_snapshot': db.execute('INSERT INTO pending VALUES(1,?,?)',(e.SerializeToString(),e.observed_at)) batch = pb.MessageBatch(protocol_major=1, messages=[e]) reply = pb.MessageBatch.FromString(post(ctx, 'messages', batch.SerializeToString(), MEDIA)) if reply.protocol_major != 1 or len(reply.messages) != 1: raise ValueError('unexpected reply') r = reply.messages[0] if (r.protocol_major,r.agent_id,r.enrollment_generation,r.boot_id,r.sequence) != ( 1,e.agent_id,1,e.boot_id,e.sequence): raise ValueError('reply identity') if kind == 'client_hello': if (r.WhichOneof('payload') != 'server_hello' or r.connection_generation == 0 or r.connection_generation != r.server_hello.connection_generation or r.server_hello.max_message_bytes != 1024*1024): raise ValueError('invalid server hello') return r.connection_generation if (r.connection_generation != generation or r.WhichOneof('payload') != 'ack' or r.ack.message_id != e.message_id): raise ValueError('unconfirmed message') if kind == 'inventory_snapshot': with db: db.execute('DELETE FROM pending WHERE id=1') if renew(ctx, identity): ctx = context() ctx.load_cert_chain(STATE/'device.pem', STATE/'device.key') generation = send('client_hello',pb.ClientHello(agent_version=VERSION, capabilities=['heartbeat/1','opnsense.base/1'])) last = None previous = None check_cert = time.monotonic()+12*3600 while True: if time.monotonic() >= check_cert: if renew(ctx, identity): return check_cert = time.monotonic()+12*3600 sample,previous = usage(previous) send('heartbeat',pb.Heartbeat(instance_id=instance,uptime_seconds=int(time.clock_gettime(time.CLOCK_UPTIME)), system_usage=sample)) with db: # The ingestion contract accepts observations for 72 hours. db.execute('DELETE FROM pending WHERE observed < ?', (time.time_ns()//1_000_000 - 72*3600*1000,)) if db.execute('SELECT 1 FROM pending').fetchone(): send('inventory_snapshot',None) file = Path('/var/db/discovery-observations/inventory.json') if file.exists(): with file.open('rb') as f: stamp = os.fstat(f.fileno()).st_mtime_ns if stamp != last and 0 <= time.time()-stamp/1e9 <= 120: raw = f.read(384*1024+1) if len(raw) > 384*1024: raise ValueError('snapshot size') send('inventory_snapshot',pb.InventorySnapshot(snapshot_id=str(uuid.uuid4()), scope='opnsense.base',schema_version=1,chunk_count=1,data=raw, sha256=hashlib.sha256(raw).digest(),completeness='partial'),stamp//1_000_000) last = stamp time.sleep(20) def main(): os.umask(0o077) if sys.argv[1:] == ['enroll']: return enroll() identity = json.loads((STATE/'identity.json').read_bytes()) db = sqlite3.connect(STATE/'channel.sqlite') with db: db.execute('CREATE TABLE IF NOT EXISTS sequence(id INTEGER PRIMARY KEY CHECK(id=1),n INTEGER NOT NULL)') db.execute('CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),envelope BLOB NOT NULL,observed INTEGER NOT NULL)') db.execute('INSERT OR IGNORE INTO sequence VALUES(1,0)') started = subprocess.check_output(['/sbin/sysctl','-n','kern.boottime']).decode().split('}')[0] boot = str(uuid.uuid5(uuid.UUID(identity['agent_id']),started)) instance = str(uuid.uuid4()) while True: try: session(db, identity, boot, instance) except Exception as exc: print('OPNsense channel:', type(exc).__name__, str(exc)[:160], flush=True) time.sleep(30) if __name__ == '__main__': main()