Add versioned script library, reconcile permit refusals and compact helper captures

This commit is contained in:
2026-11-17 15:45:56 +01:00
parent a576c10e8a
commit 8e2fef7007
29 changed files with 1225 additions and 50 deletions
+186
View File
@@ -0,0 +1,186 @@
#!/usr/bin/env python3
"""Build the project's Rust binaries from a Git revision in the existing builders."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
ROOT = Path(__file__).resolve().parents[1]
TARGETS = {
"server-arm64": (
"linux/arm64", "aarch64-unknown-linux-gnu",
"sha256:30ae3d90753bc39b0a98a897784c662aff9598ffab7a98a6298e8aa79145c2ec",
[("rmm-server", "rmm-server"), ("rmm-time-broker-linux", "rmm-time-broker-linux")],
),
"linux-x64": (
"linux/amd64", "x86_64-unknown-linux-gnu",
"sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6",
[("rmm-agent", "rmm-agent"), ("rmm-privileged-helper", "rmm-privileged-helper"),
("rmm-updater-worker", "rmm-agent-updater")],
),
"windows-gnu-x64": (
"linux/amd64", "x86_64-pc-windows-gnu",
"sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9",
[("rmm-agent-windows", "rmm-agent-windows"), ("rmm-agent-windows", "rmm-helper-windows")],
),
}
SOURCE_PATHS = ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "crates", "proto",
"migrations", "tools/apt-transaction", "deploy/linux", "tests"]
def git(*args):
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
def prepare(target, jobs, windows_updater=False):
platform, triple, image, packages = TARGETS[target]
packages = list(packages)
cargo = ["cargo", "build", "--release", "--locked", "--offline", "--jobs", str(jobs), "--target", triple]
suffix = ""
if target == "windows-gnu-x64":
suffix = ".exe"
cargo += ["--features", "rmm-agent-windows/system-helper"]
if windows_updater:
packages.append(("rmm-agent-windows", "rmm-updater-windows"))
for package in dict.fromkeys(package for package, _ in packages):
cargo += ["-p", package]
binaries = [binary + suffix for _, binary in packages]
for _, binary in packages:
cargo += ["--bin", binary]
return platform, triple, image, binaries, cargo
def build_archive(work, archive, revision, plan):
platform, triple, image, binaries, cargo = plan
source, cache = work / "src", work / "cargo"
command = ["docker", "run", "--rm", "--pull=never", "--platform", platform, "--network=none",
"--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges",
"--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}",
"-v", f"{source}:/src:ro", "-v", f"{cache}:/target", "-w", "/src",
"-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]]
print(shlex.join(command), flush=True)
os.umask(0o077)
subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL)
work.mkdir(parents=True, exist_ok=True)
with (work / "build.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
source.mkdir(exist_ok=True)
cache.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(dir=work) as temporary:
staging = Path(temporary)
unpacked = staging / "src"
unpacked.mkdir()
subprocess.run(["tar", "-xf", archive, "-C", unpacked], check=True)
# Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime
# even when switching back to an older commit with an older Git timestamp.
subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True)
subprocess.run(command, check=True)
artifacts = staging / "artifacts"
artifacts.mkdir()
hashes = {}
for name in binaries:
path = artifacts / name
built = cache / triple / "release" / name
if built.resolve() != built or not built.is_file():
raise SystemExit(f"Missing regular build artifact: {name}")
shutil.copy2(built, path)
hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
manifest = {"revision": revision, "target": triple, "builder": image, "sha256": hashes}
manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode()
(artifacts / "manifest.json").write_bytes(manifest_bytes)
destination = work / "artifacts" / revision / hashlib.sha256(manifest_bytes).hexdigest()
destination.parent.mkdir(parents=True, exist_ok=True)
if destination.exists():
if ((destination / "manifest.json").read_bytes() != manifest_bytes
or any(hashlib.sha256((destination / n).read_bytes()).hexdigest() != h for n, h in hashes.items())):
raise SystemExit(f"Previous output was modified: {destination}")
else:
artifacts.rename(destination)
print(f"Artifacts: {destination}\n{json.dumps(hashes, indent=2)}")
def remote_build(archive, revision, jobs):
from dev_stage import ssh_prefix
# The only SSH payload is this build driver and the selected Git sources.
# The worker has no dependency on a checkout or on the notebook's profiles.
worker = Path(__file__).resolve()
checksums = [hashlib.sha256(p.read_bytes()).hexdigest() for p in (worker, archive)]
shell = """set -eu
umask 077
stage=$(mktemp -d)
trap 'rm -rf -- "$stage"' EXIT
cd "$stage"
tar -xf - build.py source.tar.gz
printf '%s build.py\\n%s source.tar.gz\\n' "$1" "$2" | sha256sum -c - >/dev/null
python3 -I build.py _server_worker "$3" "$4"
"""
with tempfile.TemporaryFile() as bundle:
with tarfile.open(fileobj=bundle, mode="w") as package:
package.add(worker, arcname="build.py")
package.add(archive, arcname="source.tar.gz")
bundle.seek(0)
command = shlex.join(["sh", "-c", shell, "build", *checksums, revision, str(jobs)])
result = subprocess.run([*ssh_prefix("server"), command], stdin=bundle)
if result.returncode:
raise SystemExit(f"ARM64 build via SSH failed (exit {result.returncode}).")
def main():
parser = argparse.ArgumentParser(
description="Build Rust binaries from a committed revision in the project's pinned Docker builders.",
epilog="Build hosts need tar, rsync and Docker. WebUI: npm ci and npm run build in web/. "
"Native Windows MSVC: use deploy/windows/Build-NativeMsvc.ps1 (-Mode Preflight, then -Mode Build).",
)
parser.add_argument("target", choices=TARGETS)
parser.add_argument("--ref", default="HEAD", help="Git commit or ref (default: HEAD). Uncommitted edits are excluded.")
parser.add_argument("--jobs", type=int, default=2, help="Maximum parallel Cargo jobs (default: 2).")
parser.add_argument("--ssh", action="store_true", help="Build server-arm64 on the configured server, in ~/rmm-dev-build/.")
parser.add_argument("--plan", action="store_true", help="Show the build plan without connecting, writing or building.")
args = parser.parse_args()
if args.jobs < 1:
parser.error("--jobs must be positive")
if args.ssh and args.target != "server-arm64":
parser.error("--ssh applies to server-arm64")
revision = git("rev-parse", "--verify", "--end-of-options", args.ref + "^{commit}")
windows_updater = args.target == "windows-gnu-x64" and git(
"ls-tree", "--name-only", revision, "crates/agent-windows/src/bin/rmm-updater-windows.rs")
plan = prepare(args.target, args.jobs, windows_updater)
platform, _, image, binaries, cargo = plan
host = "server via SSH" if args.ssh else "local"
print(f"{host}: {args.target} {revision}\n{platform} {image}\n{', '.join(binaries)}\n{shlex.join(cargo)}", flush=True)
if args.plan:
return
with tempfile.TemporaryDirectory() as temporary:
archive = Path(temporary) / "source.tar.gz"
subprocess.run(["git", "archive", "-o", archive, revision, "--", *SOURCE_PATHS], cwd=ROOT, check=True)
if args.ssh:
remote_build(archive, revision, args.jobs)
else:
build_archive(ROOT / "devstate" / "build" / args.target, archive, revision, plan)
if __name__ == "__main__":
try:
if sys.argv[1:2] == ["_server_worker"]:
revision, jobs_arg = sys.argv[2:]
if (len(revision) not in (40, 64) or any(c not in "0123456789abcdef" for c in revision)
or not jobs_arg.isascii() or not jobs_arg.isdecimal() or int(jobs_arg) < 1):
raise SystemExit("Invalid packed build request.")
work = Path.home() / "rmm-dev-build" / "server-arm64"
build_archive(work, Path("source.tar.gz").resolve(), revision, prepare("server-arm64", int(jobs_arg)))
else:
main()
except (OSError, subprocess.CalledProcessError) as error:
raise SystemExit(str(error)) from None
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Backfill lossless storage for old helper snapshots on a development guest."""
import argparse
from contextlib import closing
from pathlib import Path
import sqlite3
def compact(db, schema, batch=32):
db.execute('PRAGMA foreign_keys=ON')
db.execute('BEGIN IMMEDIATE')
try:
if db.execute("SELECT type FROM sqlite_schema WHERE name='helper_sync_captures'").fetchone() == ('table',):
statement = ''
for line in schema.splitlines(keepends=True):
statement += line
if sqlite3.complete_statement(statement):
db.execute(statement)
statement = ''
db.commit()
except Exception:
db.rollback()
raise
cursor = -1
total = 0
while True:
db.execute('BEGIN IMMEDIATE')
try:
rows = db.execute(
"SELECT checkpoint,snapshot,fingerprint,captured_at FROM helper_capture_headers "
"WHERE checkpoint>? AND tail_id IS NULL ORDER BY checkpoint LIMIT ?", (cursor, batch)
).fetchall()
for checkpoint, raw, fingerprint, captured_at in rows:
cursor = checkpoint
if b',"attempts":' not in raw:
continue
db.execute("DELETE FROM helper_capture_headers WHERE checkpoint=?", (checkpoint,))
db.execute("INSERT INTO helper_sync_captures VALUES(?,?,?,?)", (checkpoint, raw, fingerprint, captured_at))
restored = db.execute("SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?", (checkpoint,)).fetchone()[0]
if restored != raw:
raise ValueError('snapshot roundtrip failed')
total += 1
db.commit()
except Exception:
db.rollback()
raise
if len(rows) < batch:
return total
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('ledger', type=Path)
parser.add_argument('--schema', type=Path, default=Path(__file__).resolve().parents[1] / 'crates/agent-core/src/helper_capture_storage.sql')
args = parser.parse_args()
with closing(sqlite3.connect(args.ledger.resolve(strict=True).as_uri()+'?mode=rw', uri=True, timeout=30)) as db:
db.execute('PRAGMA max_page_count=262144')
rows = compact(db, args.schema.read_text())
print(f"{rows} snapshots backfilled; {db.execute('PRAGMA freelist_count').fetchone()[0]} free pages")
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Vorhandene Entwicklungsartefakte per SSH in das Benutzerprofil des Ziels kopieren."""
import argparse
import base64
import hashlib
from pathlib import Path, PureWindowsPath
import re
import shlex
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "deploy"))
from vm_access_check import load_config
from vm_mvp_acceptance import protected_json
def ssh_prefix(target):
credentials = ROOT / "credentials"
config = load_config(credentials / ("linux-vm.conf" if target == "linux" else "server.conf"))
host = config["username"] + "@" + config["address"]
key = str(Path(config["ssh_key_path"]).expanduser())
base = ["ssh", "-F", "/dev/null", "-T"]
direct = [*base, "-p", config.get("port", "22"), "-i", key, "-o", "StrictHostKeyChecking=yes",
"-o", "BatchMode=yes", "-o", "ForwardAgent=no", "-o", "IdentitiesOnly=yes"]
if target != "windows":
return [*direct, "-o", "ConnectTimeout=10", host]
# Match the pinned access registered by deploy/windows/prepare_dev_access.py.
# Reconstruct the proxy from validated fields before using a profile containing a command.
jump = shlex.join([*direct, "-W", "127.0.0.1:22240", host])
prefix = [*base, "-i", key, "-o", "ProxyCommand=" + jump, "-o", "HostKeyAlias=rmm-windows-dev",
"-o", "UserKnownHostsFile=" + str(credentials / "windows-dev.known_hosts"),
"-o", "GlobalKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=yes",
"-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", "ForwardAgent=no",
"-o", "ConnectTimeout=10", "rmm-dev@127.0.0.1"]
if protected_json(credentials / "windows-dev-access.json").get("ssh_prefix") != prefix:
raise ValueError("Windows-Profil stimmt nicht mit dem registrierten SSH-Weg überein")
return prefix
UNIX = r'''
set -eu
umask 077
hash=$1; name=$2; mode=$3
root="$HOME/rmm-dev-stage"; dir="$root/$hash"; dst="$dir/$name"
for folder in "$root" "$dir"; do
if [ ! -e "$folder" ] && [ ! -L "$folder" ]; then
if [ "$mode" = probe ]; then echo missing; exit 0; fi
mkdir -m 700 "$folder" || [ -d "$folder" ]
fi
[ ! -L "$folder" ] && [ -d "$folder" ] &&
[ "$(stat -c '%u:%a' "$folder")" = "$(id -u):700" ] || exit 1
done
verify() {
[ ! -L "$1" ] && [ -f "$1" ] && [ "$(sha256sum < "$1" | cut -d' ' -f1)" = "$hash" ]
}
if [ -e "$dst" ] || [ -L "$dst" ]; then verify "$dst"; echo verified; exit 0; fi
if [ "$mode" = probe ]; then echo missing; exit 0; fi
part=$(mktemp "$dir/.upload.XXXXXXXX")
trap 'rm -f -- "$part"' EXIT
cat > "$part"
verify "$part"
ln -T -- "$part" "$dst" || verify "$dst"
verify "$dst"
echo verified
'''
WINDOWS = r'''
$ErrorActionPreference = 'Stop'
$root = Join-Path $env:USERPROFILE 'rmm-dev-stage'
$dir = Join-Path $root $hash
$dst = Join-Path $dir $name
foreach ($folder in @($root,$dir)) {
if (!(Test-Path -LiteralPath $folder)) {
if ($mode -eq 'probe') { 'missing'; exit 0 }
[void][IO.Directory]::CreateDirectory($folder)
}
$item = Get-Item -LiteralPath $folder -Force
if (!$item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
throw 'Invalid staging directory'
}
}
function Verify-File($file) {
$item = Get-Item -LiteralPath $file -Force
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
(Get-FileHash -LiteralPath $file -Algorithm SHA256).Hash -ne $hash) { throw 'Invalid artifact' }
}
if (Test-Path -LiteralPath $dst) { Verify-File $dst; 'verified'; exit 0 }
if ($mode -eq 'probe') { 'missing'; exit 0 }
$part = Join-Path $dir ('.upload.' + [Guid]::NewGuid())
$out = [IO.File]::Open($part, 'CreateNew', 'Write', 'None')
try {
try { [Console]::OpenStandardInput().CopyTo($out) } finally { $out.Dispose() }
Verify-File $part
try { [IO.File]::Move($part,$dst) } catch [IO.IOException] { Verify-File $dst }
Verify-File $dst
'verified'
} finally { [IO.File]::Delete($part) }
'''
def remote_command(target, digest, name, mode):
if target == "windows":
# name and digest have a restricted alphabet before PowerShell sees them.
script = f"$hash='{digest}'\n$name='{name}'\n$mode='{mode}'\n" + WINDOWS
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
return "powershell.exe -NoLogo -NoProfile -NonInteractive -EncodedCommand " + encoded
return shlex.join(["sh", "-c", UNIX, "dev-stage", digest, name, mode])
def transfer(prefix, command, file=None):
result = subprocess.run([*prefix, command], stdin=file or subprocess.DEVNULL,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=600)
status = result.stdout.strip()
if result.returncode or status not in (b"missing", b"verified"):
raise RuntimeError("SSH-Verbindung, Stagingpfad oder SHA-256-Prüfung fehlgeschlagen")
return status
def main():
parser = argparse.ArgumentParser(description=__doc__, epilog=(
"Ziel: rmm-dev-stage/<SHA-256>/<Dateiname> im SSH-Benutzerprofil. "
"Identische Dateien werden wiederverwendet. Dienste und Releases werden separat aktiviert."))
parser.add_argument("--plan", action="store_true", help="Nur lokale Hashes und Zielpfade anzeigen")
parser.add_argument("target", choices=("server", "linux", "windows"), help="Entwicklungsziel")
parser.add_argument("files", nargs="+", type=Path, metavar="DATEI", help="Vorhandene lokale Artefakte")
args = parser.parse_args()
for path in args.files:
name = path.name
if (not path.is_file() or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,126}", name)
or name.endswith(".") or PureWindowsPath(name).is_reserved()):
parser.error("Reguläre Datei mit portablem Artefaktnamen erwartet")
if path.resolve().is_relative_to(ROOT / "credentials"):
parser.error("Zugangsprofile sind keine Entwicklungsartefakte")
prefix = None if args.plan else ssh_prefix(args.target)
for path in args.files:
with path.open("rb") as file:
hashed = hashlib.sha256()
for block in iter(lambda: file.read(1024 * 1024), b""):
hashed.update(block)
digest = hashed.hexdigest()
destination = f"{args.target}:rmm-dev-stage/{digest}/{path.name}"
if args.plan:
print("geplant", destination)
continue
if transfer(prefix, remote_command(args.target, digest, path.name, "probe")) == b"verified":
print("vorhanden und geprüft", destination)
continue
file.seek(0)
if transfer(prefix, remote_command(args.target, digest, path.name, "stage"), file) != b"verified":
raise RuntimeError("Zielhash nicht bestätigt")
print("bereitgestellt und geprüft", destination)
if __name__ == "__main__":
try:
main()
except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
# Do not include the subprocess command (Windows SSH contains the private access path).
message = str(error) if isinstance(error, (ValueError, RuntimeError)) else type(error).__name__
sys.exit(f"Staging fehlgeschlagen: {message}")