Files
rust-discovery/tools/build.py
T

217 lines
10 KiB
Python

#!/usr/bin/env python3
"""Build the project's Rust binaries from a Git revision in the existing builders."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
ROOT = Path(__file__).resolve().parents[1]
TARGETS = {
"server-arm64": (
"linux/arm64", "aarch64-unknown-linux-gnu",
"sha256:30ae3d90753bc39b0a98a897784c662aff9598ffab7a98a6298e8aa79145c2ec",
[("rmm-server", "rmm-server"), ("rmm-time-broker-linux", "rmm-time-broker-linux")],
),
"linux-x64": (
"linux/amd64", "x86_64-unknown-linux-gnu",
"sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6",
[("rmm-agent", "rmm-agent"), ("rmm-privileged-helper", "rmm-privileged-helper"),
("rmm-updater-worker", "rmm-agent-updater")],
),
"windows-gnu-x64": (
"linux/amd64", "x86_64-pc-windows-gnu",
"sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9",
[("rmm-agent-windows", "rmm-agent-windows"), ("rmm-agent-windows", "rmm-helper-windows")],
),
}
SOURCE_PATHS = ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "crates", "proto",
"migrations", "tools/apt-transaction", "deploy/linux", "tests"]
def git(*args):
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
def prepare(target, jobs, windows_updater=False):
platform, triple, image, packages = TARGETS[target]
packages = list(packages)
cargo = ["cargo", "build", "--release", "--locked", "--offline", "--jobs", str(jobs), "--target", triple]
suffix = ""
if target == "windows-gnu-x64":
suffix = ".exe"
cargo += ["--features", "rmm-agent-windows/system-helper"]
if windows_updater:
packages.append(("rmm-agent-windows", "rmm-updater-windows"))
for package in dict.fromkeys(package for package, _ in packages):
cargo += ["-p", package]
binaries = [binary + suffix for _, binary in packages]
for _, binary in packages:
cargo += ["--bin", binary]
return platform, triple, image, binaries, cargo
def crate_registry(work, docker, image, triple):
# Old revisions can build entirely from the registry embedded in the image.
# New lockfiles get a persistent package cache. Only Cargo fetch needs the
# network; build scripts and the compiler always run without it.
registry = work / "cargo-registry"
mount = ["-v", f"{registry}:/packages", "-e", "CARGO_HOME=/packages"]
fetch = ["--entrypoint", "cargo", image, "fetch", "--locked", "--target", triple]
offline = [*docker, "--network=none"]
cache = mount if registry.exists() else []
probe = subprocess.run([*offline, *cache, *fetch, "--offline"], capture_output=True)
if probe.returncode == 0:
return cache
if not registry.exists():
with tempfile.TemporaryDirectory(dir=work) as temporary:
staging = Path(temporary) / "packages"
staging.mkdir()
subprocess.run([
*offline, "-v", f"{staging}:/packages", "--entrypoint", "sh", image, "-c",
"mkdir /packages/registry && cp -R /usr/local/cargo/registry/cache "
"/usr/local/cargo/registry/index /packages/registry/",
], check=True)
staging.rename(registry)
print("Fetching locked Cargo dependencies for " + triple, flush=True)
subprocess.run([*docker, *mount, *fetch], check=True)
return mount
def build_archive(work, archive, revision, plan):
platform, triple, image, binaries, cargo = plan
source, cache = work / "src", work / "cargo"
docker = ["docker", "run", "--rm", "--pull=never", "--platform", platform,
"--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges",
"--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}",
"-v", f"{source}:/src:ro", "-w", "/src"]
os.umask(0o077)
subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL)
work.mkdir(parents=True, exist_ok=True)
with (work / "build.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
source.mkdir(exist_ok=True)
cache.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(dir=work) as temporary:
staging = Path(temporary)
unpacked = staging / "src"
unpacked.mkdir()
subprocess.run(["tar", "-xf", archive, "-C", unpacked], check=True)
# Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime
# even when switching back to an older commit with an older Git timestamp.
subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True)
registry = crate_registry(work, docker, image, triple)
command = [*docker, "--network=none", *registry, "-v", f"{cache}:/target",
"-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]]
print(shlex.join(command), flush=True)
subprocess.run(command, check=True)
artifacts = staging / "artifacts"
artifacts.mkdir()
hashes = {}
for name in binaries:
path = artifacts / name
built = cache / triple / "release" / name
if built.resolve() != built or not built.is_file():
raise SystemExit(f"Missing regular build artifact: {name}")
shutil.copy2(built, path)
hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
manifest = {"revision": revision, "target": triple, "builder": image, "sha256": hashes}
manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode()
(artifacts / "manifest.json").write_bytes(manifest_bytes)
destination = work / "artifacts" / revision / hashlib.sha256(manifest_bytes).hexdigest()
destination.parent.mkdir(parents=True, exist_ok=True)
if destination.exists():
if ((destination / "manifest.json").read_bytes() != manifest_bytes
or any(hashlib.sha256((destination / n).read_bytes()).hexdigest() != h for n, h in hashes.items())):
raise SystemExit(f"Previous output was modified: {destination}")
else:
artifacts.rename(destination)
print(f"Artifacts: {destination}\n{json.dumps(hashes, indent=2)}")
def remote_build(archive, revision, jobs):
from dev_stage import ssh_prefix
# The only SSH payload is this build driver and the selected Git sources.
# The worker has no dependency on a checkout or on the notebook's profiles.
worker = Path(__file__).resolve()
checksums = [hashlib.sha256(p.read_bytes()).hexdigest() for p in (worker, archive)]
shell = """set -eu
umask 077
stage=$(mktemp -d)
trap 'rm -rf -- "$stage"' EXIT
cd "$stage"
tar -xf - build.py source.tar.gz
printf '%s build.py\\n%s source.tar.gz\\n' "$1" "$2" | sha256sum -c - >/dev/null
python3 -I build.py _server_worker "$3" "$4"
"""
with tempfile.TemporaryFile() as bundle:
with tarfile.open(fileobj=bundle, mode="w") as package:
package.add(worker, arcname="build.py")
package.add(archive, arcname="source.tar.gz")
bundle.seek(0)
command = shlex.join(["sh", "-c", shell, "build", *checksums, revision, str(jobs)])
result = subprocess.run([*ssh_prefix("server"), command], stdin=bundle)
if result.returncode:
raise SystemExit(f"ARM64 build via SSH failed (exit {result.returncode}).")
def main():
parser = argparse.ArgumentParser(
description="Build Rust binaries from a committed revision in the project's pinned Docker builders.",
epilog="Build hosts need tar, rsync and Docker. WebUI: npm ci and npm run build in web/. "
"Native Windows MSVC: use deploy/windows/Build-NativeMsvc.ps1 (-Mode Preflight, then -Mode Build).",
)
parser.add_argument("target", choices=TARGETS)
parser.add_argument("--ref", default="HEAD", help="Git commit or ref (default: HEAD). Uncommitted edits are excluded.")
parser.add_argument("--jobs", type=int, default=2, help="Maximum parallel Cargo jobs (default: 2).")
parser.add_argument("--ssh", action="store_true", help="Build server-arm64 on the configured server, in ~/rmm-dev-build/.")
parser.add_argument("--plan", action="store_true", help="Show the build plan without connecting, writing or building.")
args = parser.parse_args()
if args.jobs < 1:
parser.error("--jobs must be positive")
if args.ssh and args.target != "server-arm64":
parser.error("--ssh applies to server-arm64")
revision = git("rev-parse", "--verify", "--end-of-options", args.ref + "^{commit}")
windows_updater = args.target == "windows-gnu-x64" and git(
"ls-tree", "--name-only", revision, "crates/agent-windows/src/bin/rmm-updater-windows.rs")
plan = prepare(args.target, args.jobs, windows_updater)
platform, _, image, binaries, cargo = plan
host = "server via SSH" if args.ssh else "local"
print(f"{host}: {args.target} {revision}\n{platform} {image}\n{', '.join(binaries)}\n{shlex.join(cargo)}", flush=True)
if args.plan:
return
with tempfile.TemporaryDirectory() as temporary:
archive = Path(temporary) / "source.tar.gz"
subprocess.run(["git", "archive", "-o", archive, revision, "--", *SOURCE_PATHS], cwd=ROOT, check=True)
if args.ssh:
remote_build(archive, revision, args.jobs)
else:
build_archive(ROOT / "devstate" / "build" / args.target, archive, revision, plan)
if __name__ == "__main__":
try:
if sys.argv[1:2] == ["_server_worker"]:
revision, jobs_arg = sys.argv[2:]
if (len(revision) not in (40, 64) or any(c not in "0123456789abcdef" for c in revision)
or not jobs_arg.isascii() or not jobs_arg.isdecimal() or int(jobs_arg) < 1):
raise SystemExit("Invalid packed build request.")
work = Path.home() / "rmm-dev-build" / "server-arm64"
build_archive(work, Path("source.tar.gz").resolve(), revision, prepare("server-arm64", int(jobs_arg)))
else:
main()
except (OSError, subprocess.CalledProcessError) as error:
raise SystemExit(str(error)) from None