Add versioned script library, reconcile permit refusals and compact helper captures
This commit is contained in:
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the project's Rust binaries from a Git revision in the existing builders."""
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TARGETS = {
|
||||
"server-arm64": (
|
||||
"linux/arm64", "aarch64-unknown-linux-gnu",
|
||||
"sha256:30ae3d90753bc39b0a98a897784c662aff9598ffab7a98a6298e8aa79145c2ec",
|
||||
[("rmm-server", "rmm-server"), ("rmm-time-broker-linux", "rmm-time-broker-linux")],
|
||||
),
|
||||
"linux-x64": (
|
||||
"linux/amd64", "x86_64-unknown-linux-gnu",
|
||||
"sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6",
|
||||
[("rmm-agent", "rmm-agent"), ("rmm-privileged-helper", "rmm-privileged-helper"),
|
||||
("rmm-updater-worker", "rmm-agent-updater")],
|
||||
),
|
||||
"windows-gnu-x64": (
|
||||
"linux/amd64", "x86_64-pc-windows-gnu",
|
||||
"sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9",
|
||||
[("rmm-agent-windows", "rmm-agent-windows"), ("rmm-agent-windows", "rmm-helper-windows")],
|
||||
),
|
||||
}
|
||||
SOURCE_PATHS = ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "crates", "proto",
|
||||
"migrations", "tools/apt-transaction", "deploy/linux", "tests"]
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def prepare(target, jobs, windows_updater=False):
|
||||
platform, triple, image, packages = TARGETS[target]
|
||||
packages = list(packages)
|
||||
cargo = ["cargo", "build", "--release", "--locked", "--offline", "--jobs", str(jobs), "--target", triple]
|
||||
suffix = ""
|
||||
if target == "windows-gnu-x64":
|
||||
suffix = ".exe"
|
||||
cargo += ["--features", "rmm-agent-windows/system-helper"]
|
||||
if windows_updater:
|
||||
packages.append(("rmm-agent-windows", "rmm-updater-windows"))
|
||||
for package in dict.fromkeys(package for package, _ in packages):
|
||||
cargo += ["-p", package]
|
||||
binaries = [binary + suffix for _, binary in packages]
|
||||
for _, binary in packages:
|
||||
cargo += ["--bin", binary]
|
||||
|
||||
return platform, triple, image, binaries, cargo
|
||||
|
||||
|
||||
def build_archive(work, archive, revision, plan):
|
||||
platform, triple, image, binaries, cargo = plan
|
||||
source, cache = work / "src", work / "cargo"
|
||||
command = ["docker", "run", "--rm", "--pull=never", "--platform", platform, "--network=none",
|
||||
"--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges",
|
||||
"--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}",
|
||||
"-v", f"{source}:/src:ro", "-v", f"{cache}:/target", "-w", "/src",
|
||||
"-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]]
|
||||
print(shlex.join(command), flush=True)
|
||||
os.umask(0o077)
|
||||
subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL)
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
with (work / "build.lock").open("a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
source.mkdir(exist_ok=True)
|
||||
cache.mkdir(exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=work) as temporary:
|
||||
staging = Path(temporary)
|
||||
unpacked = staging / "src"
|
||||
unpacked.mkdir()
|
||||
subprocess.run(["tar", "-xf", archive, "-C", unpacked], check=True)
|
||||
# Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime
|
||||
# even when switching back to an older commit with an older Git timestamp.
|
||||
subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True)
|
||||
subprocess.run(command, check=True)
|
||||
artifacts = staging / "artifacts"
|
||||
artifacts.mkdir()
|
||||
hashes = {}
|
||||
for name in binaries:
|
||||
path = artifacts / name
|
||||
built = cache / triple / "release" / name
|
||||
if built.resolve() != built or not built.is_file():
|
||||
raise SystemExit(f"Missing regular build artifact: {name}")
|
||||
shutil.copy2(built, path)
|
||||
hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
manifest = {"revision": revision, "target": triple, "builder": image, "sha256": hashes}
|
||||
manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode()
|
||||
(artifacts / "manifest.json").write_bytes(manifest_bytes)
|
||||
destination = work / "artifacts" / revision / hashlib.sha256(manifest_bytes).hexdigest()
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.exists():
|
||||
if ((destination / "manifest.json").read_bytes() != manifest_bytes
|
||||
or any(hashlib.sha256((destination / n).read_bytes()).hexdigest() != h for n, h in hashes.items())):
|
||||
raise SystemExit(f"Previous output was modified: {destination}")
|
||||
else:
|
||||
artifacts.rename(destination)
|
||||
print(f"Artifacts: {destination}\n{json.dumps(hashes, indent=2)}")
|
||||
|
||||
|
||||
def remote_build(archive, revision, jobs):
|
||||
from dev_stage import ssh_prefix
|
||||
|
||||
# The only SSH payload is this build driver and the selected Git sources.
|
||||
# The worker has no dependency on a checkout or on the notebook's profiles.
|
||||
worker = Path(__file__).resolve()
|
||||
checksums = [hashlib.sha256(p.read_bytes()).hexdigest() for p in (worker, archive)]
|
||||
shell = """set -eu
|
||||
umask 077
|
||||
stage=$(mktemp -d)
|
||||
trap 'rm -rf -- "$stage"' EXIT
|
||||
cd "$stage"
|
||||
tar -xf - build.py source.tar.gz
|
||||
printf '%s build.py\\n%s source.tar.gz\\n' "$1" "$2" | sha256sum -c - >/dev/null
|
||||
python3 -I build.py _server_worker "$3" "$4"
|
||||
"""
|
||||
with tempfile.TemporaryFile() as bundle:
|
||||
with tarfile.open(fileobj=bundle, mode="w") as package:
|
||||
package.add(worker, arcname="build.py")
|
||||
package.add(archive, arcname="source.tar.gz")
|
||||
bundle.seek(0)
|
||||
command = shlex.join(["sh", "-c", shell, "build", *checksums, revision, str(jobs)])
|
||||
result = subprocess.run([*ssh_prefix("server"), command], stdin=bundle)
|
||||
if result.returncode:
|
||||
raise SystemExit(f"ARM64 build via SSH failed (exit {result.returncode}).")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build Rust binaries from a committed revision in the project's pinned Docker builders.",
|
||||
epilog="Build hosts need tar, rsync and Docker. WebUI: npm ci and npm run build in web/. "
|
||||
"Native Windows MSVC: use deploy/windows/Build-NativeMsvc.ps1 (-Mode Preflight, then -Mode Build).",
|
||||
)
|
||||
parser.add_argument("target", choices=TARGETS)
|
||||
parser.add_argument("--ref", default="HEAD", help="Git commit or ref (default: HEAD). Uncommitted edits are excluded.")
|
||||
parser.add_argument("--jobs", type=int, default=2, help="Maximum parallel Cargo jobs (default: 2).")
|
||||
parser.add_argument("--ssh", action="store_true", help="Build server-arm64 on the configured server, in ~/rmm-dev-build/.")
|
||||
parser.add_argument("--plan", action="store_true", help="Show the build plan without connecting, writing or building.")
|
||||
args = parser.parse_args()
|
||||
if args.jobs < 1:
|
||||
parser.error("--jobs must be positive")
|
||||
if args.ssh and args.target != "server-arm64":
|
||||
parser.error("--ssh applies to server-arm64")
|
||||
revision = git("rev-parse", "--verify", "--end-of-options", args.ref + "^{commit}")
|
||||
windows_updater = args.target == "windows-gnu-x64" and git(
|
||||
"ls-tree", "--name-only", revision, "crates/agent-windows/src/bin/rmm-updater-windows.rs")
|
||||
plan = prepare(args.target, args.jobs, windows_updater)
|
||||
platform, _, image, binaries, cargo = plan
|
||||
host = "server via SSH" if args.ssh else "local"
|
||||
print(f"{host}: {args.target} {revision}\n{platform} {image}\n{', '.join(binaries)}\n{shlex.join(cargo)}", flush=True)
|
||||
if args.plan:
|
||||
return
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
archive = Path(temporary) / "source.tar.gz"
|
||||
subprocess.run(["git", "archive", "-o", archive, revision, "--", *SOURCE_PATHS], cwd=ROOT, check=True)
|
||||
if args.ssh:
|
||||
remote_build(archive, revision, args.jobs)
|
||||
else:
|
||||
build_archive(ROOT / "devstate" / "build" / args.target, archive, revision, plan)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
if sys.argv[1:2] == ["_server_worker"]:
|
||||
revision, jobs_arg = sys.argv[2:]
|
||||
if (len(revision) not in (40, 64) or any(c not in "0123456789abcdef" for c in revision)
|
||||
or not jobs_arg.isascii() or not jobs_arg.isdecimal() or int(jobs_arg) < 1):
|
||||
raise SystemExit("Invalid packed build request.")
|
||||
work = Path.home() / "rmm-dev-build" / "server-arm64"
|
||||
build_archive(work, Path("source.tar.gz").resolve(), revision, prepare("server-arm64", int(jobs_arg)))
|
||||
else:
|
||||
main()
|
||||
except (OSError, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(str(error)) from None
|
||||
Reference in New Issue
Block a user