Add versioned script library, reconcile permit refusals and compact helper captures
This commit is contained in:
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the project's Rust binaries from a Git revision in the existing builders."""
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TARGETS = {
|
||||
"server-arm64": (
|
||||
"linux/arm64", "aarch64-unknown-linux-gnu",
|
||||
"sha256:30ae3d90753bc39b0a98a897784c662aff9598ffab7a98a6298e8aa79145c2ec",
|
||||
[("rmm-server", "rmm-server"), ("rmm-time-broker-linux", "rmm-time-broker-linux")],
|
||||
),
|
||||
"linux-x64": (
|
||||
"linux/amd64", "x86_64-unknown-linux-gnu",
|
||||
"sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6",
|
||||
[("rmm-agent", "rmm-agent"), ("rmm-privileged-helper", "rmm-privileged-helper"),
|
||||
("rmm-updater-worker", "rmm-agent-updater")],
|
||||
),
|
||||
"windows-gnu-x64": (
|
||||
"linux/amd64", "x86_64-pc-windows-gnu",
|
||||
"sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9",
|
||||
[("rmm-agent-windows", "rmm-agent-windows"), ("rmm-agent-windows", "rmm-helper-windows")],
|
||||
),
|
||||
}
|
||||
SOURCE_PATHS = ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "crates", "proto",
|
||||
"migrations", "tools/apt-transaction", "deploy/linux", "tests"]
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def prepare(target, jobs, windows_updater=False):
|
||||
platform, triple, image, packages = TARGETS[target]
|
||||
packages = list(packages)
|
||||
cargo = ["cargo", "build", "--release", "--locked", "--offline", "--jobs", str(jobs), "--target", triple]
|
||||
suffix = ""
|
||||
if target == "windows-gnu-x64":
|
||||
suffix = ".exe"
|
||||
cargo += ["--features", "rmm-agent-windows/system-helper"]
|
||||
if windows_updater:
|
||||
packages.append(("rmm-agent-windows", "rmm-updater-windows"))
|
||||
for package in dict.fromkeys(package for package, _ in packages):
|
||||
cargo += ["-p", package]
|
||||
binaries = [binary + suffix for _, binary in packages]
|
||||
for _, binary in packages:
|
||||
cargo += ["--bin", binary]
|
||||
|
||||
return platform, triple, image, binaries, cargo
|
||||
|
||||
|
||||
def build_archive(work, archive, revision, plan):
|
||||
platform, triple, image, binaries, cargo = plan
|
||||
source, cache = work / "src", work / "cargo"
|
||||
command = ["docker", "run", "--rm", "--pull=never", "--platform", platform, "--network=none",
|
||||
"--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges",
|
||||
"--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}",
|
||||
"-v", f"{source}:/src:ro", "-v", f"{cache}:/target", "-w", "/src",
|
||||
"-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]]
|
||||
print(shlex.join(command), flush=True)
|
||||
os.umask(0o077)
|
||||
subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL)
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
with (work / "build.lock").open("a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
source.mkdir(exist_ok=True)
|
||||
cache.mkdir(exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=work) as temporary:
|
||||
staging = Path(temporary)
|
||||
unpacked = staging / "src"
|
||||
unpacked.mkdir()
|
||||
subprocess.run(["tar", "-xf", archive, "-C", unpacked], check=True)
|
||||
# Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime
|
||||
# even when switching back to an older commit with an older Git timestamp.
|
||||
subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True)
|
||||
subprocess.run(command, check=True)
|
||||
artifacts = staging / "artifacts"
|
||||
artifacts.mkdir()
|
||||
hashes = {}
|
||||
for name in binaries:
|
||||
path = artifacts / name
|
||||
built = cache / triple / "release" / name
|
||||
if built.resolve() != built or not built.is_file():
|
||||
raise SystemExit(f"Missing regular build artifact: {name}")
|
||||
shutil.copy2(built, path)
|
||||
hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
manifest = {"revision": revision, "target": triple, "builder": image, "sha256": hashes}
|
||||
manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode()
|
||||
(artifacts / "manifest.json").write_bytes(manifest_bytes)
|
||||
destination = work / "artifacts" / revision / hashlib.sha256(manifest_bytes).hexdigest()
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.exists():
|
||||
if ((destination / "manifest.json").read_bytes() != manifest_bytes
|
||||
or any(hashlib.sha256((destination / n).read_bytes()).hexdigest() != h for n, h in hashes.items())):
|
||||
raise SystemExit(f"Previous output was modified: {destination}")
|
||||
else:
|
||||
artifacts.rename(destination)
|
||||
print(f"Artifacts: {destination}\n{json.dumps(hashes, indent=2)}")
|
||||
|
||||
|
||||
def remote_build(archive, revision, jobs):
|
||||
from dev_stage import ssh_prefix
|
||||
|
||||
# The only SSH payload is this build driver and the selected Git sources.
|
||||
# The worker has no dependency on a checkout or on the notebook's profiles.
|
||||
worker = Path(__file__).resolve()
|
||||
checksums = [hashlib.sha256(p.read_bytes()).hexdigest() for p in (worker, archive)]
|
||||
shell = """set -eu
|
||||
umask 077
|
||||
stage=$(mktemp -d)
|
||||
trap 'rm -rf -- "$stage"' EXIT
|
||||
cd "$stage"
|
||||
tar -xf - build.py source.tar.gz
|
||||
printf '%s build.py\\n%s source.tar.gz\\n' "$1" "$2" | sha256sum -c - >/dev/null
|
||||
python3 -I build.py _server_worker "$3" "$4"
|
||||
"""
|
||||
with tempfile.TemporaryFile() as bundle:
|
||||
with tarfile.open(fileobj=bundle, mode="w") as package:
|
||||
package.add(worker, arcname="build.py")
|
||||
package.add(archive, arcname="source.tar.gz")
|
||||
bundle.seek(0)
|
||||
command = shlex.join(["sh", "-c", shell, "build", *checksums, revision, str(jobs)])
|
||||
result = subprocess.run([*ssh_prefix("server"), command], stdin=bundle)
|
||||
if result.returncode:
|
||||
raise SystemExit(f"ARM64 build via SSH failed (exit {result.returncode}).")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build Rust binaries from a committed revision in the project's pinned Docker builders.",
|
||||
epilog="Build hosts need tar, rsync and Docker. WebUI: npm ci and npm run build in web/. "
|
||||
"Native Windows MSVC: use deploy/windows/Build-NativeMsvc.ps1 (-Mode Preflight, then -Mode Build).",
|
||||
)
|
||||
parser.add_argument("target", choices=TARGETS)
|
||||
parser.add_argument("--ref", default="HEAD", help="Git commit or ref (default: HEAD). Uncommitted edits are excluded.")
|
||||
parser.add_argument("--jobs", type=int, default=2, help="Maximum parallel Cargo jobs (default: 2).")
|
||||
parser.add_argument("--ssh", action="store_true", help="Build server-arm64 on the configured server, in ~/rmm-dev-build/.")
|
||||
parser.add_argument("--plan", action="store_true", help="Show the build plan without connecting, writing or building.")
|
||||
args = parser.parse_args()
|
||||
if args.jobs < 1:
|
||||
parser.error("--jobs must be positive")
|
||||
if args.ssh and args.target != "server-arm64":
|
||||
parser.error("--ssh applies to server-arm64")
|
||||
revision = git("rev-parse", "--verify", "--end-of-options", args.ref + "^{commit}")
|
||||
windows_updater = args.target == "windows-gnu-x64" and git(
|
||||
"ls-tree", "--name-only", revision, "crates/agent-windows/src/bin/rmm-updater-windows.rs")
|
||||
plan = prepare(args.target, args.jobs, windows_updater)
|
||||
platform, _, image, binaries, cargo = plan
|
||||
host = "server via SSH" if args.ssh else "local"
|
||||
print(f"{host}: {args.target} {revision}\n{platform} {image}\n{', '.join(binaries)}\n{shlex.join(cargo)}", flush=True)
|
||||
if args.plan:
|
||||
return
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
archive = Path(temporary) / "source.tar.gz"
|
||||
subprocess.run(["git", "archive", "-o", archive, revision, "--", *SOURCE_PATHS], cwd=ROOT, check=True)
|
||||
if args.ssh:
|
||||
remote_build(archive, revision, args.jobs)
|
||||
else:
|
||||
build_archive(ROOT / "devstate" / "build" / args.target, archive, revision, plan)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
if sys.argv[1:2] == ["_server_worker"]:
|
||||
revision, jobs_arg = sys.argv[2:]
|
||||
if (len(revision) not in (40, 64) or any(c not in "0123456789abcdef" for c in revision)
|
||||
or not jobs_arg.isascii() or not jobs_arg.isdecimal() or int(jobs_arg) < 1):
|
||||
raise SystemExit("Invalid packed build request.")
|
||||
work = Path.home() / "rmm-dev-build" / "server-arm64"
|
||||
build_archive(work, Path("source.tar.gz").resolve(), revision, prepare("server-arm64", int(jobs_arg)))
|
||||
else:
|
||||
main()
|
||||
except (OSError, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(str(error)) from None
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill lossless storage for old helper snapshots on a development guest."""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
|
||||
def compact(db, schema, batch=32):
|
||||
db.execute('PRAGMA foreign_keys=ON')
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
if db.execute("SELECT type FROM sqlite_schema WHERE name='helper_sync_captures'").fetchone() == ('table',):
|
||||
statement = ''
|
||||
for line in schema.splitlines(keepends=True):
|
||||
statement += line
|
||||
if sqlite3.complete_statement(statement):
|
||||
db.execute(statement)
|
||||
statement = ''
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
cursor = -1
|
||||
total = 0
|
||||
while True:
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
rows = db.execute(
|
||||
"SELECT checkpoint,snapshot,fingerprint,captured_at FROM helper_capture_headers "
|
||||
"WHERE checkpoint>? AND tail_id IS NULL ORDER BY checkpoint LIMIT ?", (cursor, batch)
|
||||
).fetchall()
|
||||
for checkpoint, raw, fingerprint, captured_at in rows:
|
||||
cursor = checkpoint
|
||||
if b',"attempts":' not in raw:
|
||||
continue
|
||||
db.execute("DELETE FROM helper_capture_headers WHERE checkpoint=?", (checkpoint,))
|
||||
db.execute("INSERT INTO helper_sync_captures VALUES(?,?,?,?)", (checkpoint, raw, fingerprint, captured_at))
|
||||
restored = db.execute("SELECT snapshot FROM helper_sync_captures WHERE checkpoint=?", (checkpoint,)).fetchone()[0]
|
||||
if restored != raw:
|
||||
raise ValueError('snapshot roundtrip failed')
|
||||
total += 1
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
if len(rows) < batch:
|
||||
return total
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('ledger', type=Path)
|
||||
parser.add_argument('--schema', type=Path, default=Path(__file__).resolve().parents[1] / 'crates/agent-core/src/helper_capture_storage.sql')
|
||||
args = parser.parse_args()
|
||||
with closing(sqlite3.connect(args.ledger.resolve(strict=True).as_uri()+'?mode=rw', uri=True, timeout=30)) as db:
|
||||
db.execute('PRAGMA max_page_count=262144')
|
||||
rows = compact(db, args.schema.read_text())
|
||||
print(f"{rows} snapshots backfilled; {db.execute('PRAGMA freelist_count').fetchone()[0]} free pages")
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Vorhandene Entwicklungsartefakte per SSH in das Benutzerprofil des Ziels kopieren."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
from pathlib import Path, PureWindowsPath
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "deploy"))
|
||||
from vm_access_check import load_config
|
||||
from vm_mvp_acceptance import protected_json
|
||||
|
||||
|
||||
def ssh_prefix(target):
|
||||
credentials = ROOT / "credentials"
|
||||
config = load_config(credentials / ("linux-vm.conf" if target == "linux" else "server.conf"))
|
||||
host = config["username"] + "@" + config["address"]
|
||||
key = str(Path(config["ssh_key_path"]).expanduser())
|
||||
base = ["ssh", "-F", "/dev/null", "-T"]
|
||||
direct = [*base, "-p", config.get("port", "22"), "-i", key, "-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "BatchMode=yes", "-o", "ForwardAgent=no", "-o", "IdentitiesOnly=yes"]
|
||||
if target != "windows":
|
||||
return [*direct, "-o", "ConnectTimeout=10", host]
|
||||
# Match the pinned access registered by deploy/windows/prepare_dev_access.py.
|
||||
# Reconstruct the proxy from validated fields before using a profile containing a command.
|
||||
jump = shlex.join([*direct, "-W", "127.0.0.1:22240", host])
|
||||
prefix = [*base, "-i", key, "-o", "ProxyCommand=" + jump, "-o", "HostKeyAlias=rmm-windows-dev",
|
||||
"-o", "UserKnownHostsFile=" + str(credentials / "windows-dev.known_hosts"),
|
||||
"-o", "GlobalKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", "ForwardAgent=no",
|
||||
"-o", "ConnectTimeout=10", "rmm-dev@127.0.0.1"]
|
||||
if protected_json(credentials / "windows-dev-access.json").get("ssh_prefix") != prefix:
|
||||
raise ValueError("Windows-Profil stimmt nicht mit dem registrierten SSH-Weg überein")
|
||||
return prefix
|
||||
|
||||
|
||||
UNIX = r'''
|
||||
set -eu
|
||||
umask 077
|
||||
hash=$1; name=$2; mode=$3
|
||||
root="$HOME/rmm-dev-stage"; dir="$root/$hash"; dst="$dir/$name"
|
||||
for folder in "$root" "$dir"; do
|
||||
if [ ! -e "$folder" ] && [ ! -L "$folder" ]; then
|
||||
if [ "$mode" = probe ]; then echo missing; exit 0; fi
|
||||
mkdir -m 700 "$folder" || [ -d "$folder" ]
|
||||
fi
|
||||
[ ! -L "$folder" ] && [ -d "$folder" ] &&
|
||||
[ "$(stat -c '%u:%a' "$folder")" = "$(id -u):700" ] || exit 1
|
||||
done
|
||||
verify() {
|
||||
[ ! -L "$1" ] && [ -f "$1" ] && [ "$(sha256sum < "$1" | cut -d' ' -f1)" = "$hash" ]
|
||||
}
|
||||
if [ -e "$dst" ] || [ -L "$dst" ]; then verify "$dst"; echo verified; exit 0; fi
|
||||
if [ "$mode" = probe ]; then echo missing; exit 0; fi
|
||||
part=$(mktemp "$dir/.upload.XXXXXXXX")
|
||||
trap 'rm -f -- "$part"' EXIT
|
||||
cat > "$part"
|
||||
verify "$part"
|
||||
ln -T -- "$part" "$dst" || verify "$dst"
|
||||
verify "$dst"
|
||||
echo verified
|
||||
'''
|
||||
|
||||
WINDOWS = r'''
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = Join-Path $env:USERPROFILE 'rmm-dev-stage'
|
||||
$dir = Join-Path $root $hash
|
||||
$dst = Join-Path $dir $name
|
||||
foreach ($folder in @($root,$dir)) {
|
||||
if (!(Test-Path -LiteralPath $folder)) {
|
||||
if ($mode -eq 'probe') { 'missing'; exit 0 }
|
||||
[void][IO.Directory]::CreateDirectory($folder)
|
||||
}
|
||||
$item = Get-Item -LiteralPath $folder -Force
|
||||
if (!$item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw 'Invalid staging directory'
|
||||
}
|
||||
}
|
||||
function Verify-File($file) {
|
||||
$item = Get-Item -LiteralPath $file -Force
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
(Get-FileHash -LiteralPath $file -Algorithm SHA256).Hash -ne $hash) { throw 'Invalid artifact' }
|
||||
}
|
||||
if (Test-Path -LiteralPath $dst) { Verify-File $dst; 'verified'; exit 0 }
|
||||
if ($mode -eq 'probe') { 'missing'; exit 0 }
|
||||
$part = Join-Path $dir ('.upload.' + [Guid]::NewGuid())
|
||||
$out = [IO.File]::Open($part, 'CreateNew', 'Write', 'None')
|
||||
try {
|
||||
try { [Console]::OpenStandardInput().CopyTo($out) } finally { $out.Dispose() }
|
||||
Verify-File $part
|
||||
try { [IO.File]::Move($part,$dst) } catch [IO.IOException] { Verify-File $dst }
|
||||
Verify-File $dst
|
||||
'verified'
|
||||
} finally { [IO.File]::Delete($part) }
|
||||
'''
|
||||
|
||||
|
||||
def remote_command(target, digest, name, mode):
|
||||
if target == "windows":
|
||||
# name and digest have a restricted alphabet before PowerShell sees them.
|
||||
script = f"$hash='{digest}'\n$name='{name}'\n$mode='{mode}'\n" + WINDOWS
|
||||
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
|
||||
return "powershell.exe -NoLogo -NoProfile -NonInteractive -EncodedCommand " + encoded
|
||||
return shlex.join(["sh", "-c", UNIX, "dev-stage", digest, name, mode])
|
||||
|
||||
|
||||
def transfer(prefix, command, file=None):
|
||||
result = subprocess.run([*prefix, command], stdin=file or subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=600)
|
||||
status = result.stdout.strip()
|
||||
if result.returncode or status not in (b"missing", b"verified"):
|
||||
raise RuntimeError("SSH-Verbindung, Stagingpfad oder SHA-256-Prüfung fehlgeschlagen")
|
||||
return status
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, epilog=(
|
||||
"Ziel: rmm-dev-stage/<SHA-256>/<Dateiname> im SSH-Benutzerprofil. "
|
||||
"Identische Dateien werden wiederverwendet. Dienste und Releases werden separat aktiviert."))
|
||||
parser.add_argument("--plan", action="store_true", help="Nur lokale Hashes und Zielpfade anzeigen")
|
||||
parser.add_argument("target", choices=("server", "linux", "windows"), help="Entwicklungsziel")
|
||||
parser.add_argument("files", nargs="+", type=Path, metavar="DATEI", help="Vorhandene lokale Artefakte")
|
||||
args = parser.parse_args()
|
||||
for path in args.files:
|
||||
name = path.name
|
||||
if (not path.is_file() or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,126}", name)
|
||||
or name.endswith(".") or PureWindowsPath(name).is_reserved()):
|
||||
parser.error("Reguläre Datei mit portablem Artefaktnamen erwartet")
|
||||
if path.resolve().is_relative_to(ROOT / "credentials"):
|
||||
parser.error("Zugangsprofile sind keine Entwicklungsartefakte")
|
||||
prefix = None if args.plan else ssh_prefix(args.target)
|
||||
for path in args.files:
|
||||
with path.open("rb") as file:
|
||||
hashed = hashlib.sha256()
|
||||
for block in iter(lambda: file.read(1024 * 1024), b""):
|
||||
hashed.update(block)
|
||||
digest = hashed.hexdigest()
|
||||
destination = f"{args.target}:rmm-dev-stage/{digest}/{path.name}"
|
||||
if args.plan:
|
||||
print("geplant", destination)
|
||||
continue
|
||||
if transfer(prefix, remote_command(args.target, digest, path.name, "probe")) == b"verified":
|
||||
print("vorhanden und geprüft", destination)
|
||||
continue
|
||||
file.seek(0)
|
||||
if transfer(prefix, remote_command(args.target, digest, path.name, "stage"), file) != b"verified":
|
||||
raise RuntimeError("Zielhash nicht bestätigt")
|
||||
print("bereitgestellt und geprüft", destination)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
|
||||
# Do not include the subprocess command (Windows SSH contains the private access path).
|
||||
message = str(error) if isinstance(error, (ValueError, RuntimeError)) else type(error).__name__
|
||||
sys.exit(f"Staging fehlgeschlagen: {message}")
|
||||
Reference in New Issue
Block a user