217 lines
10 KiB
Python
217 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Build the project's Rust binaries from a Git revision in the existing builders."""
|
|
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
TARGETS = {
|
|
"server-arm64": (
|
|
"linux/arm64", "aarch64-unknown-linux-gnu",
|
|
"sha256:30ae3d90753bc39b0a98a897784c662aff9598ffab7a98a6298e8aa79145c2ec",
|
|
[("rmm-server", "rmm-server"), ("rmm-time-broker-linux", "rmm-time-broker-linux")],
|
|
),
|
|
"linux-x64": (
|
|
"linux/amd64", "x86_64-unknown-linux-gnu",
|
|
"sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6",
|
|
[("rmm-agent", "rmm-agent"), ("rmm-privileged-helper", "rmm-privileged-helper"),
|
|
("rmm-updater-worker", "rmm-agent-updater")],
|
|
),
|
|
"windows-gnu-x64": (
|
|
"linux/amd64", "x86_64-pc-windows-gnu",
|
|
"sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9",
|
|
[("rmm-agent-windows", "rmm-agent-windows"), ("rmm-agent-windows", "rmm-helper-windows")],
|
|
),
|
|
}
|
|
SOURCE_PATHS = ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "crates", "proto",
|
|
"migrations", "tools/apt-transaction", "deploy/linux", "tests"]
|
|
|
|
|
|
def git(*args):
|
|
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
|
|
|
|
|
|
def prepare(target, jobs, windows_updater=False):
|
|
platform, triple, image, packages = TARGETS[target]
|
|
packages = list(packages)
|
|
cargo = ["cargo", "build", "--release", "--locked", "--offline", "--jobs", str(jobs), "--target", triple]
|
|
suffix = ""
|
|
if target == "windows-gnu-x64":
|
|
suffix = ".exe"
|
|
cargo += ["--features", "rmm-agent-windows/system-helper"]
|
|
if windows_updater:
|
|
packages.append(("rmm-agent-windows", "rmm-updater-windows"))
|
|
for package in dict.fromkeys(package for package, _ in packages):
|
|
cargo += ["-p", package]
|
|
binaries = [binary + suffix for _, binary in packages]
|
|
for _, binary in packages:
|
|
cargo += ["--bin", binary]
|
|
|
|
return platform, triple, image, binaries, cargo
|
|
|
|
|
|
def crate_registry(work, docker, image, triple):
|
|
# Old revisions can build entirely from the registry embedded in the image.
|
|
# New lockfiles get a persistent package cache. Only Cargo fetch needs the
|
|
# network; build scripts and the compiler always run without it.
|
|
registry = work / "cargo-registry"
|
|
mount = ["-v", f"{registry}:/packages", "-e", "CARGO_HOME=/packages"]
|
|
fetch = ["--entrypoint", "cargo", image, "fetch", "--locked", "--target", triple]
|
|
offline = [*docker, "--network=none"]
|
|
cache = mount if registry.exists() else []
|
|
probe = subprocess.run([*offline, *cache, *fetch, "--offline"], capture_output=True)
|
|
if probe.returncode == 0:
|
|
return cache
|
|
|
|
if not registry.exists():
|
|
with tempfile.TemporaryDirectory(dir=work) as temporary:
|
|
staging = Path(temporary) / "packages"
|
|
staging.mkdir()
|
|
subprocess.run([
|
|
*offline, "-v", f"{staging}:/packages", "--entrypoint", "sh", image, "-c",
|
|
"mkdir /packages/registry && cp -R /usr/local/cargo/registry/cache "
|
|
"/usr/local/cargo/registry/index /packages/registry/",
|
|
], check=True)
|
|
staging.rename(registry)
|
|
print("Fetching locked Cargo dependencies for " + triple, flush=True)
|
|
subprocess.run([*docker, *mount, *fetch], check=True)
|
|
return mount
|
|
|
|
|
|
def build_archive(work, archive, revision, plan):
|
|
platform, triple, image, binaries, cargo = plan
|
|
source, cache = work / "src", work / "cargo"
|
|
docker = ["docker", "run", "--rm", "--pull=never", "--platform", platform,
|
|
"--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges",
|
|
"--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}",
|
|
"-v", f"{source}:/src:ro", "-w", "/src"]
|
|
os.umask(0o077)
|
|
subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL)
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
with (work / "build.lock").open("a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
source.mkdir(exist_ok=True)
|
|
cache.mkdir(exist_ok=True)
|
|
with tempfile.TemporaryDirectory(dir=work) as temporary:
|
|
staging = Path(temporary)
|
|
unpacked = staging / "src"
|
|
unpacked.mkdir()
|
|
subprocess.run(["tar", "-xf", archive, "-C", unpacked], check=True)
|
|
# Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime
|
|
# even when switching back to an older commit with an older Git timestamp.
|
|
subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True)
|
|
registry = crate_registry(work, docker, image, triple)
|
|
command = [*docker, "--network=none", *registry, "-v", f"{cache}:/target",
|
|
"-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]]
|
|
print(shlex.join(command), flush=True)
|
|
subprocess.run(command, check=True)
|
|
artifacts = staging / "artifacts"
|
|
artifacts.mkdir()
|
|
hashes = {}
|
|
for name in binaries:
|
|
path = artifacts / name
|
|
built = cache / triple / "release" / name
|
|
if built.resolve() != built or not built.is_file():
|
|
raise SystemExit(f"Missing regular build artifact: {name}")
|
|
shutil.copy2(built, path)
|
|
hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
manifest = {"revision": revision, "target": triple, "builder": image, "sha256": hashes}
|
|
manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode()
|
|
(artifacts / "manifest.json").write_bytes(manifest_bytes)
|
|
destination = work / "artifacts" / revision / hashlib.sha256(manifest_bytes).hexdigest()
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
if destination.exists():
|
|
if ((destination / "manifest.json").read_bytes() != manifest_bytes
|
|
or any(hashlib.sha256((destination / n).read_bytes()).hexdigest() != h for n, h in hashes.items())):
|
|
raise SystemExit(f"Previous output was modified: {destination}")
|
|
else:
|
|
artifacts.rename(destination)
|
|
print(f"Artifacts: {destination}\n{json.dumps(hashes, indent=2)}")
|
|
|
|
|
|
def remote_build(archive, revision, jobs):
|
|
from dev_stage import ssh_prefix
|
|
|
|
# The only SSH payload is this build driver and the selected Git sources.
|
|
# The worker has no dependency on a checkout or on the notebook's profiles.
|
|
worker = Path(__file__).resolve()
|
|
checksums = [hashlib.sha256(p.read_bytes()).hexdigest() for p in (worker, archive)]
|
|
shell = """set -eu
|
|
umask 077
|
|
stage=$(mktemp -d)
|
|
trap 'rm -rf -- "$stage"' EXIT
|
|
cd "$stage"
|
|
tar -xf - build.py source.tar.gz
|
|
printf '%s build.py\\n%s source.tar.gz\\n' "$1" "$2" | sha256sum -c - >/dev/null
|
|
python3 -I build.py _server_worker "$3" "$4"
|
|
"""
|
|
with tempfile.TemporaryFile() as bundle:
|
|
with tarfile.open(fileobj=bundle, mode="w") as package:
|
|
package.add(worker, arcname="build.py")
|
|
package.add(archive, arcname="source.tar.gz")
|
|
bundle.seek(0)
|
|
command = shlex.join(["sh", "-c", shell, "build", *checksums, revision, str(jobs)])
|
|
result = subprocess.run([*ssh_prefix("server"), command], stdin=bundle)
|
|
if result.returncode:
|
|
raise SystemExit(f"ARM64 build via SSH failed (exit {result.returncode}).")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Build Rust binaries from a committed revision in the project's pinned Docker builders.",
|
|
epilog="Build hosts need tar, rsync and Docker. WebUI: npm ci and npm run build in web/. "
|
|
"Native Windows MSVC: use deploy/windows/Build-NativeMsvc.ps1 (-Mode Preflight, then -Mode Build).",
|
|
)
|
|
parser.add_argument("target", choices=TARGETS)
|
|
parser.add_argument("--ref", default="HEAD", help="Git commit or ref (default: HEAD). Uncommitted edits are excluded.")
|
|
parser.add_argument("--jobs", type=int, default=2, help="Maximum parallel Cargo jobs (default: 2).")
|
|
parser.add_argument("--ssh", action="store_true", help="Build server-arm64 on the configured server, in ~/rmm-dev-build/.")
|
|
parser.add_argument("--plan", action="store_true", help="Show the build plan without connecting, writing or building.")
|
|
args = parser.parse_args()
|
|
if args.jobs < 1:
|
|
parser.error("--jobs must be positive")
|
|
if args.ssh and args.target != "server-arm64":
|
|
parser.error("--ssh applies to server-arm64")
|
|
revision = git("rev-parse", "--verify", "--end-of-options", args.ref + "^{commit}")
|
|
windows_updater = args.target == "windows-gnu-x64" and git(
|
|
"ls-tree", "--name-only", revision, "crates/agent-windows/src/bin/rmm-updater-windows.rs")
|
|
plan = prepare(args.target, args.jobs, windows_updater)
|
|
platform, _, image, binaries, cargo = plan
|
|
host = "server via SSH" if args.ssh else "local"
|
|
print(f"{host}: {args.target} {revision}\n{platform} {image}\n{', '.join(binaries)}\n{shlex.join(cargo)}", flush=True)
|
|
if args.plan:
|
|
return
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
archive = Path(temporary) / "source.tar.gz"
|
|
subprocess.run(["git", "archive", "-o", archive, revision, "--", *SOURCE_PATHS], cwd=ROOT, check=True)
|
|
if args.ssh:
|
|
remote_build(archive, revision, args.jobs)
|
|
else:
|
|
build_archive(ROOT / "devstate" / "build" / args.target, archive, revision, plan)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
if sys.argv[1:2] == ["_server_worker"]:
|
|
revision, jobs_arg = sys.argv[2:]
|
|
if (len(revision) not in (40, 64) or any(c not in "0123456789abcdef" for c in revision)
|
|
or not jobs_arg.isascii() or not jobs_arg.isdecimal() or int(jobs_arg) < 1):
|
|
raise SystemExit("Invalid packed build request.")
|
|
work = Path.home() / "rmm-dev-build" / "server-arm64"
|
|
build_archive(work, Path("source.tar.gz").resolve(), revision, prepare("server-arm64", int(jobs_arg)))
|
|
else:
|
|
main()
|
|
except (OSError, subprocess.CalledProcessError) as error:
|
|
raise SystemExit(str(error)) from None
|