Files
rust-discovery/tools/dev_opnsense_worker.py
T
kevin_janssenandClaude Opus 5.5 93429e4394 Keep OPNsense telemetry when the script worker cannot start and build it for FreeBSD 14.1
A worker linked on the FreeBSD 15 test guest needs FBSD_1.9 and does not load on
OPNsense 24.7; the agent then retried the job session every 30 s and never sent a
heartbeat or inventory. The agent now pauses script capabilities for an hour and
reconnects without them. tools/dev_opnsense_worker.py links the worker against the
official FreeBSD 14.1 pkgbase libraries. The installer checks that the worker loads
before binding it and reports success only after a heartbeat.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 12:17:34 +01:00

123 lines
5.8 KiB
Python

#!/usr/bin/env python3
"""Build the native OPNsense script worker `rmm-opnsense-jobs` for the oldest supported release.
The build runs in the OPNsense test guest (toolchain `/opt/discovery-rust`), but links
against the FreeBSD 14.1 base libraries (OPNsense 24.7) instead of the guest's own
FreeBSD. A worker linked natively on FreeBSD 15 needs `FBSD_1.9` from libc and does
not load on OPNsense 24.x. The sysroot consists of the official pkgbase packages of
14.1 (`pkg.freebsd.org/FreeBSD:14:amd64/base_release_1`), each checked against the
SHA-256 of the repository catalogue, and is kept in `/var/tmp/fbsd141` on the guest.
Usage: python3 tools/dev_opnsense_worker.py OUT_FILE
Then: python3 tools/dev_opnsense_bundle.py --worker OUT_FILE
"""
import argparse
import hashlib
import io
import json
from pathlib import Path
import shlex
import subprocess
import tarfile
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
CREDENTIALS = ROOT / 'credentials'
SSH = ['ssh', '-F', '/dev/null', '-T', '-i', str(CREDENTIALS / 'opnsense-dev-key'),
'-o', 'UserKnownHostsFile=' + str(CREDENTIALS / 'opnsense-dev.known_hosts'),
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
'-o', 'IdentitiesOnly=yes', '-o', 'ConnectTimeout=10', '-o', 'ServerAliveInterval=30', 'root@192.168.247.1']
REPO = 'https://pkg.freebsd.org/FreeBSD:14:amd64/base_release_1'
PACKAGES = {'FreeBSD-clibs', 'FreeBSD-clibs-dev', 'FreeBSD-runtime', 'FreeBSD-runtime-dev',
'FreeBSD-libexecinfo', 'FreeBSD-libexecinfo-dev', 'FreeBSD-libcompiler_rt-dev',
# libprocstat, libmemstat, libdevstat (linked by Rust std on FreeBSD)
'FreeBSD-utilities', 'FreeBSD-utilities-dev'}
SYSROOT = '/var/tmp/fbsd141/root'
SOURCE = '/root/discovery-stage/worker-src'
def packages():
"""(file, sha256) of the 14.1 library packages from the HTTPS repository catalogue."""
with urllib.request.urlopen(REPO + '/packagesite.pkg', timeout=60) as res:
raw = res.read(16 * 1024 * 1024)
# zstd-compressed tar; the system tar knows the format.
site = subprocess.run(['tar', '--zstd', '-xOf', '-', 'packagesite.yaml'], input=raw, check=True,
capture_output=True).stdout.decode()
found = [json.loads(line) for line in site.splitlines() if line.strip()]
found = [(d['repopath'], d['sum']) for d in found if d['name'] in PACKAGES and d['version'].startswith('14.1')]
if len(found) != len(PACKAGES):
raise SystemExit('pkgbase 14.1: nicht alle Bibliothekspakete im Katalog')
return found
def stamp_of(found):
return ' '.join(sorted(name for name, _ in found))
def sysroot(found):
"""Download on the notebook (the guest's own network is unreliable), verify, unpack in the guest."""
stamp = stamp_of(found)
current = subprocess.run([*SSH, 'cat /var/tmp/fbsd141/stamp'], capture_output=True).stdout.decode().strip()
if current == stamp:
return
# Packages carry schg flags (libc, init); the sysroot is plain data without them.
subprocess.run([*SSH, 'sh -s'], check=True, input=(
'set -eu\nif [ -d /var/tmp/fbsd141 ]; then chflags -R noschg /var/tmp/fbsd141; rm -rf /var/tmp/fbsd141; fi\n'
f'mkdir -p {SYSROOT}\n').encode())
for name, digest in found:
with urllib.request.urlopen(REPO + '/' + name, timeout=120) as res:
data = res.read(128 * 1024 * 1024)
if hashlib.sha256(data).hexdigest() != digest:
raise SystemExit(f'{name}: Prüfsumme stimmt nicht mit dem Katalog überein')
target = shlex.quote('/var/tmp/fbsd141/' + name)
subprocess.run([*SSH, f'cat >{target}'], input=data, check=True)
subprocess.run([*SSH, 'sh -s'], check=True, input=(
f'set -eu\n[ "$(sha256 -q {target})" = {digest} ]\n'
f'tar -xf {target} -C {SYSROOT} --no-fflags --exclude "+*" 2>/dev/null || true\nrm {target}\n').encode())
subprocess.run([*SSH, 'sh -s'], check=True, input=(
f'set -eu\n[ -f {SYSROOT}/usr/lib/crt1.o ] && [ -f {SYSROOT}/lib/libc.so.7 ]\n'
f'echo {shlex.quote(stamp)} >/var/tmp/fbsd141/stamp\n').encode())
BUILD = f"""set -eu
cd {SOURCE}
export PATH=/opt/discovery-rust/bin:$PATH CARGO_TARGET_DIR=/var/tmp/opnsense-worker-target
export RUST_LIBC_UNSTABLE_FREEBSD_VERSION=14 CC=clang CFLAGS="--sysroot={SYSROOT}"
export RUSTFLAGS="-C linker=clang -C link-arg=--sysroot={SYSROOT} -C link-arg=-fuse-ld=lld"
cargo build --locked --release -p rmm-opnsense-jobs >&2
bin=$CARGO_TARGET_DIR/release/rmm-opnsense-jobs
if readelf -V "$bin" | grep -q 'FBSD_1\\.9'; then echo 'worker still needs FBSD_1.9' >&2; exit 1; fi
cat "$bin"
"""
def source_tar():
files = subprocess.run(['git', 'ls-files', '-z', '--', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml',
'proto', 'crates'], cwd=ROOT, check=True, capture_output=True).stdout.split(b'\0')
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode='w') as tar:
for name in filter(None, files):
path = ROOT / name.decode()
if path.is_file():
tar.add(path, arcname=name.decode())
return buf.getvalue()
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('out', type=Path)
args = parser.parse_args()
sysroot(packages())
subprocess.run([*SSH, f'rm -rf {SOURCE} && mkdir -p {SOURCE}'], check=True)
subprocess.run([*SSH, f'tar -xf - -C {SOURCE}'], input=source_tar(), check=True)
binary = subprocess.run([*SSH, 'sh -s'], input=BUILD.encode(), capture_output=True)
if not binary.stdout.startswith(b'\x7fELF'):
raise SystemExit(binary.stderr.decode()[-2000:])
args.out.write_bytes(binary.stdout)
args.out.chmod(0o755)
print(f'{args.out}: {len(binary.stdout)} bytes, linked against FreeBSD 14.1')
if __name__ == '__main__':
main()