#!/usr/bin/env python3 """Build the project's Rust binaries from a Git revision in the existing builders.""" import argparse import fcntl import hashlib import json import os from pathlib import Path import shlex import shutil import subprocess import sys import tarfile import tempfile ROOT = Path(__file__).resolve().parents[1] TARGETS = { "server-arm64": ( "linux/arm64", "aarch64-unknown-linux-gnu", "sha256:30ae3d90753bc39b0a98a897784c662aff9598ffab7a98a6298e8aa79145c2ec", [("rmm-server", "rmm-server"), ("rmm-time-broker-linux", "rmm-time-broker-linux")], ), "linux-x64": ( "linux/amd64", "x86_64-unknown-linux-gnu", "sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6", [("rmm-agent", "rmm-agent"), ("rmm-privileged-helper", "rmm-privileged-helper"), ("rmm-updater-worker", "rmm-agent-updater")], ), "windows-gnu-x64": ( "linux/amd64", "x86_64-pc-windows-gnu", "sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9", [("rmm-agent-windows", "rmm-agent-windows"), ("rmm-agent-windows", "rmm-helper-windows")], ), } SOURCE_PATHS = ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "crates", "proto", "migrations", "tools/apt-transaction", "deploy/linux", "tests"] def git(*args): return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip() def prepare(target, jobs, windows_updater=False): platform, triple, image, packages = TARGETS[target] packages = list(packages) cargo = ["cargo", "build", "--release", "--locked", "--offline", "--jobs", str(jobs), "--target", triple] suffix = "" if target == "windows-gnu-x64": suffix = ".exe" cargo += ["--features", "rmm-agent-windows/system-helper"] if windows_updater: packages.append(("rmm-agent-windows", "rmm-updater-windows")) for package in dict.fromkeys(package for package, _ in packages): cargo += ["-p", package] binaries = [binary + suffix for _, binary in packages] for _, binary in packages: cargo += ["--bin", binary] return platform, triple, image, binaries, cargo def build_archive(work, archive, revision, plan): platform, triple, image, binaries, cargo = plan source, cache = work / "src", work / "cargo" command = ["docker", "run", "--rm", "--pull=never", "--platform", platform, "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{source}:/src:ro", "-v", f"{cache}:/target", "-w", "/src", "-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]] print(shlex.join(command), flush=True) os.umask(0o077) subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL) work.mkdir(parents=True, exist_ok=True) with (work / "build.lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX) source.mkdir(exist_ok=True) cache.mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=work) as temporary: staging = Path(temporary) unpacked = staging / "src" unpacked.mkdir() subprocess.run(["tar", "-xf", archive, "-C", unpacked], check=True) # Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime # even when switching back to an older commit with an older Git timestamp. subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True) subprocess.run(command, check=True) artifacts = staging / "artifacts" artifacts.mkdir() hashes = {} for name in binaries: path = artifacts / name built = cache / triple / "release" / name if built.resolve() != built or not built.is_file(): raise SystemExit(f"Missing regular build artifact: {name}") shutil.copy2(built, path) hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest() manifest = {"revision": revision, "target": triple, "builder": image, "sha256": hashes} manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode() (artifacts / "manifest.json").write_bytes(manifest_bytes) destination = work / "artifacts" / revision / hashlib.sha256(manifest_bytes).hexdigest() destination.parent.mkdir(parents=True, exist_ok=True) if destination.exists(): if ((destination / "manifest.json").read_bytes() != manifest_bytes or any(hashlib.sha256((destination / n).read_bytes()).hexdigest() != h for n, h in hashes.items())): raise SystemExit(f"Previous output was modified: {destination}") else: artifacts.rename(destination) print(f"Artifacts: {destination}\n{json.dumps(hashes, indent=2)}") def remote_build(archive, revision, jobs): from dev_stage import ssh_prefix # The only SSH payload is this build driver and the selected Git sources. # The worker has no dependency on a checkout or on the notebook's profiles. worker = Path(__file__).resolve() checksums = [hashlib.sha256(p.read_bytes()).hexdigest() for p in (worker, archive)] shell = """set -eu umask 077 stage=$(mktemp -d) trap 'rm -rf -- "$stage"' EXIT cd "$stage" tar -xf - build.py source.tar.gz printf '%s build.py\\n%s source.tar.gz\\n' "$1" "$2" | sha256sum -c - >/dev/null python3 -I build.py _server_worker "$3" "$4" """ with tempfile.TemporaryFile() as bundle: with tarfile.open(fileobj=bundle, mode="w") as package: package.add(worker, arcname="build.py") package.add(archive, arcname="source.tar.gz") bundle.seek(0) command = shlex.join(["sh", "-c", shell, "build", *checksums, revision, str(jobs)]) result = subprocess.run([*ssh_prefix("server"), command], stdin=bundle) if result.returncode: raise SystemExit(f"ARM64 build via SSH failed (exit {result.returncode}).") def main(): parser = argparse.ArgumentParser( description="Build Rust binaries from a committed revision in the project's pinned Docker builders.", epilog="Build hosts need tar, rsync and Docker. WebUI: npm ci and npm run build in web/. " "Native Windows MSVC: use deploy/windows/Build-NativeMsvc.ps1 (-Mode Preflight, then -Mode Build).", ) parser.add_argument("target", choices=TARGETS) parser.add_argument("--ref", default="HEAD", help="Git commit or ref (default: HEAD). Uncommitted edits are excluded.") parser.add_argument("--jobs", type=int, default=2, help="Maximum parallel Cargo jobs (default: 2).") parser.add_argument("--ssh", action="store_true", help="Build server-arm64 on the configured server, in ~/rmm-dev-build/.") parser.add_argument("--plan", action="store_true", help="Show the build plan without connecting, writing or building.") args = parser.parse_args() if args.jobs < 1: parser.error("--jobs must be positive") if args.ssh and args.target != "server-arm64": parser.error("--ssh applies to server-arm64") revision = git("rev-parse", "--verify", "--end-of-options", args.ref + "^{commit}") windows_updater = args.target == "windows-gnu-x64" and git( "ls-tree", "--name-only", revision, "crates/agent-windows/src/bin/rmm-updater-windows.rs") plan = prepare(args.target, args.jobs, windows_updater) platform, _, image, binaries, cargo = plan host = "server via SSH" if args.ssh else "local" print(f"{host}: {args.target} {revision}\n{platform} {image}\n{', '.join(binaries)}\n{shlex.join(cargo)}", flush=True) if args.plan: return with tempfile.TemporaryDirectory() as temporary: archive = Path(temporary) / "source.tar.gz" subprocess.run(["git", "archive", "-o", archive, revision, "--", *SOURCE_PATHS], cwd=ROOT, check=True) if args.ssh: remote_build(archive, revision, args.jobs) else: build_archive(ROOT / "devstate" / "build" / args.target, archive, revision, plan) if __name__ == "__main__": try: if sys.argv[1:2] == ["_server_worker"]: revision, jobs_arg = sys.argv[2:] if (len(revision) not in (40, 64) or any(c not in "0123456789abcdef" for c in revision) or not jobs_arg.isascii() or not jobs_arg.isdecimal() or int(jobs_arg) < 1): raise SystemExit("Invalid packed build request.") work = Path.home() / "rmm-dev-build" / "server-arm64" build_archive(work, Path("source.tar.gz").resolve(), revision, prepare("server-arm64", int(jobs_arg))) else: main() except (OSError, subprocess.CalledProcessError) as error: raise SystemExit(str(error)) from None