#!/usr/bin/env python3 """Build the native OPNsense script worker `rmm-opnsense-jobs` for the oldest supported release. The build runs in the OPNsense test guest (toolchain `/opt/discovery-rust`), but links against the FreeBSD 14.1 base libraries (OPNsense 24.7) instead of the guest's own FreeBSD. A worker linked natively on FreeBSD 15 needs `FBSD_1.9` from libc and does not load on OPNsense 24.x. The sysroot consists of the official pkgbase packages of 14.1 (`pkg.freebsd.org/FreeBSD:14:amd64/base_release_1`), each checked against the SHA-256 of the repository catalogue, and is kept in `/var/tmp/fbsd141` on the guest. Usage: python3 tools/dev_opnsense_worker.py OUT_FILE Then: python3 tools/dev_opnsense_bundle.py --worker OUT_FILE """ import argparse import hashlib import io import json from pathlib import Path import shlex import subprocess import tarfile import urllib.request ROOT = Path(__file__).resolve().parents[1] CREDENTIALS = ROOT / 'credentials' SSH = ['ssh', '-F', '/dev/null', '-T', '-i', str(CREDENTIALS / 'opnsense-dev-key'), '-o', 'UserKnownHostsFile=' + str(CREDENTIALS / 'opnsense-dev.known_hosts'), '-o', 'GlobalKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', '-o', 'ConnectTimeout=10', '-o', 'ServerAliveInterval=30', 'root@192.168.247.1'] REPO = 'https://pkg.freebsd.org/FreeBSD:14:amd64/base_release_1' PACKAGES = {'FreeBSD-clibs', 'FreeBSD-clibs-dev', 'FreeBSD-runtime', 'FreeBSD-runtime-dev', 'FreeBSD-libexecinfo', 'FreeBSD-libexecinfo-dev', 'FreeBSD-libcompiler_rt-dev', # libprocstat, libmemstat, libdevstat (linked by Rust std on FreeBSD) 'FreeBSD-utilities', 'FreeBSD-utilities-dev'} SYSROOT = '/var/tmp/fbsd141/root' SOURCE = '/root/discovery-stage/worker-src' def packages(): """(file, sha256) of the 14.1 library packages from the HTTPS repository catalogue.""" with urllib.request.urlopen(REPO + '/packagesite.pkg', timeout=60) as res: raw = res.read(16 * 1024 * 1024) # zstd-compressed tar; the system tar knows the format. site = subprocess.run(['tar', '--zstd', '-xOf', '-', 'packagesite.yaml'], input=raw, check=True, capture_output=True).stdout.decode() found = [json.loads(line) for line in site.splitlines() if line.strip()] found = [(d['repopath'], d['sum']) for d in found if d['name'] in PACKAGES and d['version'].startswith('14.1')] if len(found) != len(PACKAGES): raise SystemExit('pkgbase 14.1: nicht alle Bibliothekspakete im Katalog') return found def stamp_of(found): return ' '.join(sorted(name for name, _ in found)) def sysroot(found): """Download on the notebook (the guest's own network is unreliable), verify, unpack in the guest.""" stamp = stamp_of(found) current = subprocess.run([*SSH, 'cat /var/tmp/fbsd141/stamp'], capture_output=True).stdout.decode().strip() if current == stamp: return # Packages carry schg flags (libc, init); the sysroot is plain data without them. subprocess.run([*SSH, 'sh -s'], check=True, input=( 'set -eu\nif [ -d /var/tmp/fbsd141 ]; then chflags -R noschg /var/tmp/fbsd141; rm -rf /var/tmp/fbsd141; fi\n' f'mkdir -p {SYSROOT}\n').encode()) for name, digest in found: with urllib.request.urlopen(REPO + '/' + name, timeout=120) as res: data = res.read(128 * 1024 * 1024) if hashlib.sha256(data).hexdigest() != digest: raise SystemExit(f'{name}: Prüfsumme stimmt nicht mit dem Katalog überein') target = shlex.quote('/var/tmp/fbsd141/' + name) subprocess.run([*SSH, f'cat >{target}'], input=data, check=True) subprocess.run([*SSH, 'sh -s'], check=True, input=( f'set -eu\n[ "$(sha256 -q {target})" = {digest} ]\n' f'tar -xf {target} -C {SYSROOT} --no-fflags --exclude "+*" 2>/dev/null || true\nrm {target}\n').encode()) subprocess.run([*SSH, 'sh -s'], check=True, input=( f'set -eu\n[ -f {SYSROOT}/usr/lib/crt1.o ] && [ -f {SYSROOT}/lib/libc.so.7 ]\n' f'echo {shlex.quote(stamp)} >/var/tmp/fbsd141/stamp\n').encode()) BUILD = f"""set -eu cd {SOURCE} export PATH=/opt/discovery-rust/bin:$PATH CARGO_TARGET_DIR=/var/tmp/opnsense-worker-target export RUST_LIBC_UNSTABLE_FREEBSD_VERSION=14 CC=clang CFLAGS="--sysroot={SYSROOT}" export RUSTFLAGS="-C linker=clang -C link-arg=--sysroot={SYSROOT} -C link-arg=-fuse-ld=lld" cargo build --locked --release -p rmm-opnsense-jobs >&2 bin=$CARGO_TARGET_DIR/release/rmm-opnsense-jobs if readelf -V "$bin" | grep -q 'FBSD_1\\.9'; then echo 'worker still needs FBSD_1.9' >&2; exit 1; fi cat "$bin" """ def source_tar(): files = subprocess.run(['git', 'ls-files', '-z', '--', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'proto', 'crates'], cwd=ROOT, check=True, capture_output=True).stdout.split(b'\0') buf = io.BytesIO() with tarfile.open(fileobj=buf, mode='w') as tar: for name in filter(None, files): path = ROOT / name.decode() if path.is_file(): tar.add(path, arcname=name.decode()) return buf.getvalue() def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('out', type=Path) args = parser.parse_args() sysroot(packages()) subprocess.run([*SSH, f'rm -rf {SOURCE} && mkdir -p {SOURCE}'], check=True) subprocess.run([*SSH, f'tar -xf - -C {SOURCE}'], input=source_tar(), check=True) binary = subprocess.run([*SSH, 'sh -s'], input=BUILD.encode(), capture_output=True) if not binary.stdout.startswith(b'\x7fELF'): raise SystemExit(binary.stderr.decode()[-2000:]) args.out.write_bytes(binary.stdout) args.out.chmod(0o755) print(f'{args.out}: {len(binary.stdout)} bytes, linked against FreeBSD 14.1') if __name__ == '__main__': main()