216 lines
9.5 KiB
Python
216 lines
9.5 KiB
Python
"""Unprivileged OPNsense telemetry channel. Input never selects a local command."""
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import sqlite3
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
import agent_pb2 as pb
|
|
|
|
STATE = Path('/var/db/discovery-agent')
|
|
CA = '/usr/local/etc/discovery-agent/ca.pem'
|
|
HOST = 'discovery.ease-technologies.de'
|
|
MEDIA = 'application/x-protobuf'
|
|
VERSION = '0.1.0-opnsense.4'
|
|
|
|
|
|
def post(ctx, path, data, media):
|
|
conn = http.client.HTTPSConnection(HOST, context=ctx, timeout=15)
|
|
try:
|
|
conn.request('POST', '/agent/v1/'+path, data, {'Content-Type':media, 'Accept':media})
|
|
res = conn.getresponse()
|
|
if res.status != 200 or res.getheader('Content-Type', '').split(';')[0] != media:
|
|
raise ValueError(f'{path}: HTTP {res.status}')
|
|
body = res.read(1024*1024 + 1)
|
|
if len(body) > 1024*1024:
|
|
raise ValueError('response size')
|
|
return body
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def context():
|
|
return ssl.create_default_context(cafile=CA)
|
|
|
|
|
|
def enroll():
|
|
# Existing keys are never overwritten, including after an ambiguous POST.
|
|
key = STATE/'device.key'
|
|
if key.exists() or (STATE/'identity.json').exists():
|
|
raise RuntimeError('installation already initialized')
|
|
token = sys.stdin.readline(128).strip()
|
|
if not re.fullmatch('[0-9a-f]{64}', token):
|
|
raise ValueError('invalid enrollment code')
|
|
csr = subprocess.check_output(['/usr/bin/openssl','req','-new','-newkey','ec','-pkeyopt',
|
|
'ec_paramgen_curve:P-256','-nodes','-subj','/CN=discovery-agent','-keyout',str(key)],
|
|
stderr=subprocess.DEVNULL)
|
|
result = json.loads(post(context(), 'enroll', json.dumps(dict(token=token, csr_pem=csr.decode())),
|
|
'application/json'))
|
|
for name in ('agent_id','asset_id'):
|
|
uuid.UUID(result[name])
|
|
(STATE/'device.pem').write_text(result.pop('certificate_pem'))
|
|
context().load_cert_chain(STATE/'device.pem', key)
|
|
(STATE/'identity.json').write_text(json.dumps(result))
|
|
|
|
|
|
def durable_write(path, data):
|
|
tmp = path.with_suffix('.new')
|
|
with tmp.open('wb') as f:
|
|
f.write(data)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.replace(tmp, path)
|
|
fd = os.open(STATE, os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def renew(ctx, identity):
|
|
# Reuse a durable request after a lost reply; gateway renewal is idempotent
|
|
# for the same CSR. Keep the currently working certificate until key match.
|
|
cert, key, csr = (STATE/n for n in ('device.pem','device.key','renewal.csr'))
|
|
if not csr.exists():
|
|
check = subprocess.run(['/usr/bin/openssl','x509','-in',str(cert),'-noout','-checkend','604800'],
|
|
stdout=subprocess.DEVNULL, check=False)
|
|
if check.returncode == 0:
|
|
return False
|
|
durable_write(csr,subprocess.check_output(['/usr/bin/openssl','req','-new','-key',str(key),
|
|
'-subj','/CN=discovery-agent']))
|
|
result = json.loads(post(ctx,'certificates/renew',json.dumps({'csr_pem':csr.read_text()}),'application/json'))
|
|
if result['agent_id'] != identity['agent_id']:
|
|
raise ValueError('renewal identity mismatch')
|
|
candidate = STATE/'candidate.pem'
|
|
durable_write(candidate,result['certificate_pem'].encode())
|
|
context().load_cert_chain(candidate,key)
|
|
durable_write(cert,candidate.read_bytes())
|
|
candidate.unlink()
|
|
csr.unlink()
|
|
return True
|
|
|
|
|
|
def usage(previous):
|
|
def counters(*keys):
|
|
return [int(x) for x in subprocess.check_output(['/sbin/sysctl', '-n', *keys],timeout=3).split()]
|
|
ticks = counters('kern.cp_time')
|
|
free, inactive, page, total = counters('vm.stats.vm.v_free_count','vm.stats.vm.v_inactive_count',
|
|
'hw.pagesize','hw.physmem')
|
|
sample = pb.SystemUsage(ram_available_bytes=min((free+inactive)*page,total),ram_total_bytes=total)
|
|
if previous and len(previous) == len(ticks) and all(x >= y for x,y in zip(ticks,previous)):
|
|
delta = sum(ticks)-sum(previous)
|
|
if delta > 0:
|
|
sample.cpu_usage_basis_points = 10000*(delta-ticks[-1]+previous[-1])//delta
|
|
return sample,ticks
|
|
|
|
|
|
def session(db, identity, boot, instance):
|
|
ctx = context()
|
|
ctx.load_cert_chain(STATE/'device.pem', STATE/'device.key')
|
|
generation = 0
|
|
|
|
def send(kind, payload, captured=None):
|
|
# A lost inventory ACK replays the exact durable envelope, including its
|
|
# original time, boot and sequence; only the connection fence changes.
|
|
with db:
|
|
pending = db.execute('SELECT envelope FROM pending WHERE id=1').fetchone() if kind=='inventory_snapshot' else None
|
|
if pending:
|
|
e = pb.Envelope.FromString(pending[0])
|
|
if e.agent_id != identity['agent_id']:
|
|
raise ValueError('pending snapshot identity')
|
|
e.connection_generation = generation
|
|
else:
|
|
seq = db.execute('UPDATE sequence SET n=n+1 RETURNING n').fetchone()[0]
|
|
e = pb.Envelope(protocol_major=1, message_id=str(uuid.uuid4()), agent_id=identity['agent_id'],
|
|
enrollment_generation=1, connection_generation=generation, boot_id=boot,
|
|
sequence=seq, observed_at=captured or time.time_ns()//1_000_000)
|
|
getattr(e, kind).CopyFrom(payload)
|
|
if kind == 'inventory_snapshot':
|
|
db.execute('INSERT INTO pending VALUES(1,?,?)',(e.SerializeToString(),e.observed_at))
|
|
batch = pb.MessageBatch(protocol_major=1, messages=[e])
|
|
reply = pb.MessageBatch.FromString(post(ctx, 'messages', batch.SerializeToString(), MEDIA))
|
|
if reply.protocol_major != 1 or len(reply.messages) != 1:
|
|
raise ValueError('unexpected reply')
|
|
r = reply.messages[0]
|
|
if (r.protocol_major,r.agent_id,r.enrollment_generation,r.boot_id,r.sequence) != (
|
|
1,e.agent_id,1,e.boot_id,e.sequence):
|
|
raise ValueError('reply identity')
|
|
if kind == 'client_hello':
|
|
if (r.WhichOneof('payload') != 'server_hello' or r.connection_generation == 0 or
|
|
r.connection_generation != r.server_hello.connection_generation or
|
|
r.server_hello.max_message_bytes != 1024*1024):
|
|
raise ValueError('invalid server hello')
|
|
return r.connection_generation
|
|
if (r.connection_generation != generation or r.WhichOneof('payload') != 'ack'
|
|
or r.ack.message_id != e.message_id):
|
|
raise ValueError('unconfirmed message')
|
|
if kind == 'inventory_snapshot':
|
|
with db:
|
|
db.execute('DELETE FROM pending WHERE id=1')
|
|
|
|
if renew(ctx, identity):
|
|
ctx = context()
|
|
ctx.load_cert_chain(STATE/'device.pem', STATE/'device.key')
|
|
generation = send('client_hello',pb.ClientHello(agent_version=VERSION, capabilities=['heartbeat/1','opnsense.base/1']))
|
|
last = None
|
|
previous = None
|
|
check_cert = time.monotonic()+12*3600
|
|
while True:
|
|
if time.monotonic() >= check_cert:
|
|
if renew(ctx, identity):
|
|
return
|
|
check_cert = time.monotonic()+12*3600
|
|
sample,previous = usage(previous)
|
|
send('heartbeat',pb.Heartbeat(instance_id=instance,uptime_seconds=int(time.clock_gettime(time.CLOCK_UPTIME)),
|
|
system_usage=sample))
|
|
with db:
|
|
# The ingestion contract accepts observations for 72 hours.
|
|
db.execute('DELETE FROM pending WHERE observed < ?', (time.time_ns()//1_000_000 - 72*3600*1000,))
|
|
if db.execute('SELECT 1 FROM pending').fetchone():
|
|
send('inventory_snapshot',None)
|
|
file = Path('/var/db/discovery-observations/inventory.json')
|
|
if file.exists():
|
|
with file.open('rb') as f:
|
|
stamp = os.fstat(f.fileno()).st_mtime_ns
|
|
if stamp != last and 0 <= time.time()-stamp/1e9 <= 120:
|
|
raw = f.read(384*1024+1)
|
|
if len(raw) > 384*1024:
|
|
raise ValueError('snapshot size')
|
|
send('inventory_snapshot',pb.InventorySnapshot(snapshot_id=str(uuid.uuid4()),
|
|
scope='opnsense.base',schema_version=1,chunk_count=1,data=raw,
|
|
sha256=hashlib.sha256(raw).digest(),completeness='partial'),stamp//1_000_000)
|
|
last = stamp
|
|
time.sleep(20)
|
|
|
|
|
|
def main():
|
|
os.umask(0o077)
|
|
if sys.argv[1:] == ['enroll']:
|
|
return enroll()
|
|
identity = json.loads((STATE/'identity.json').read_bytes())
|
|
db = sqlite3.connect(STATE/'channel.sqlite')
|
|
with db:
|
|
db.execute('CREATE TABLE IF NOT EXISTS sequence(id INTEGER PRIMARY KEY CHECK(id=1),n INTEGER NOT NULL)')
|
|
db.execute('CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),envelope BLOB NOT NULL,observed INTEGER NOT NULL)')
|
|
db.execute('INSERT OR IGNORE INTO sequence VALUES(1,0)')
|
|
started = subprocess.check_output(['/sbin/sysctl','-n','kern.boottime']).decode().split('}')[0]
|
|
boot = str(uuid.uuid5(uuid.UUID(identity['agent_id']),started))
|
|
instance = str(uuid.uuid4())
|
|
while True:
|
|
try:
|
|
session(db, identity, boot, instance)
|
|
except Exception as exc:
|
|
print('OPNsense channel:', type(exc).__name__, str(exc)[:160], flush=True)
|
|
time.sleep(30)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|