Files
rust-discovery/deploy/vm_job_bootstrap.py
BuTzZ 6c3203d725
Rust development slice / linux (push) Failing after 1m10s
feat: snapshot Linux-first RMM MVP and pause handover
2026-11-12 22:06:25 +01:00

322 lines
17 KiB
Python

#!/usr/bin/env python3
"""Explicitly authorized non-root systemd bootstrap, preserving the old identity.
No enrollment, job approval, package operation, reboot or journal repair exists
here. Installation is create-only and refuses partially installed resources.
Rollback is manual: stop an idle service, verify manifests, rename state back
on the same filesystem and restore original root ownership. Never run both paths.
"""
import base64
from contextlib import closing, contextmanager
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import tempfile
import uuid
import vm_access_check as access
from vm_passive_test import execute, remote_python, require
ROOT = access.ROOT
@contextmanager
def connection():
config = access.load_config(ROOT / 'credentials/linux-vm.conf')
known = Path.home() / '.ssh/known_hosts'
port, address = config.get('port', '22'), config['address']
execute(['ssh-keygen', '-F', address if port == '22' else f'[{address}]:{port}', '-f', str(known)])
with tempfile.TemporaryDirectory(prefix='rmm-bootstrap-', dir=ROOT / 'credentials') as folder:
folder = Path(folder)
secret, askpass = folder / 'password', folder / 'askpass'
secret.write_text(config.get('password', '') + '\n'); secret.chmod(0o600)
askpass.write_text('#!/bin/sh\nexec cat "$RMM_SSH_PASSWORD_FILE"\n'); askpass.chmod(0o700)
env = dict(os.environ, SSH_ASKPASS=str(askpass), SSH_ASKPASS_REQUIRE='force',
DISPLAY='rmm-bootstrap:0', RMM_SSH_PASSWORD_FILE=str(secret))
ssh = ['ssh', '-F', '/dev/null', '-T', '-p', port,
'-o', 'StrictHostKeyChecking=yes', '-o', f'UserKnownHostsFile={known}',
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', 'ForwardAgent=no',
'-o', 'ConnectTimeout=10', '-o', 'ConnectionAttempts=1',
'-o', 'NumberOfPasswordPrompts=1', '-o', 'LogLevel=ERROR']
if config.get('ssh_key_path'):
ssh += ['-i', str(Path(config['ssh_key_path']).expanduser()), '-o', 'IdentitiesOnly=yes']
yield ssh, config['username'] + '@' + address, env
def manifest(directory):
"""Byte identity plus read-only SQLite schema/row evidence; never returns rows."""
import sqlite3
result = {}
for path in sorted(directory.rglob('*')):
meta = path.lstat()
require(not path.is_symlink() and (path.is_dir() or path.is_file()), 'unsafe state entry')
if path.is_file():
require(meta.st_nlink == 1, 'state file has external hard links')
result[str(path.relative_to(directory))] = hashlib.sha256(path.read_bytes()).hexdigest()
databases = {}
for path in sorted(directory.rglob('*.sqlite')):
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity failed')
schema = db.execute('SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY type,name').fetchall()
tables = {}
for kind, name, _, _ in schema:
if kind != 'table':
continue
quoted = '"' + name.replace('"', '""') + '"'
rows = db.execute('SELECT * FROM ' + quoted).fetchall()
digest = hashlib.sha256(repr(sorted(rows, key=repr)).encode()).hexdigest()
tables[name] = dict(rows=len(rows), digest=digest)
databases[path.name] = dict(schema_hash=hashlib.sha256(repr(schema).encode()).hexdigest(), tables=tables)
return dict(files=result, databases=databases)
def remote_install(payload, root=Path('/')):
"""Runs only on the VM as root through verified SSH stdin."""
import fcntl
import grp
import pwd
os.umask(0o077)
owner = os.geteuid()
require(root != Path('/') or owner == 0, 'bootstrap requires root SSH')
source = Path(payload['source'])
state = source / 'state'
destination = root / 'var/lib/rmm-agent'
config = root / 'etc/rmm'
binary = root / 'usr/local/libexec/rmm-agent'
unit = root / 'etc/systemd/system/rmm-agent.service'
require(state.is_dir() and not state.is_symlink() and state.stat().st_uid == owner,
'preserved root-owned state missing')
require(not destination.exists() and not destination.is_symlink(), 'destination must be absent')
require(not config.is_symlink() and not binary.is_symlink() and not unit.is_symlink(), 'unsafe destination symlink')
for parent in (destination.parent, config.parent, binary.parent if binary.parent.exists() else binary.parent.parent, unit.parent):
require(parent.is_dir() and not parent.is_symlink() and parent.stat().st_uid == owner
and parent.stat().st_mode & 0o022 == 0, 'unsafe install parent')
require(state.stat().st_dev == destination.parent.stat().st_dev, 'atomic migration requires same filesystem')
lock = (state / 'agent.lock').open('rb')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
for process in Path('/proc').glob('[0-9]*'):
try:
require(Path(os.readlink(process / 'exe')).name != 'rmm-agent', 'existing agent process')
except (FileNotFoundError, PermissionError):
pass
for descriptor in (process / 'fd').glob('*'):
try:
opened = os.readlink(descriptor)
require(not opened.startswith(str(state) + '/') or
(process.name == str(os.getpid()) and opened == str(state / 'agent.lock')),
'state has active file handles')
except (FileNotFoundError, PermissionError):
pass
identity = json.loads((state / 'identity.json').read_text())
require(identity['agent_id'] == payload['binding']['agent_id'] and
identity['asset_id'] == payload['binding']['asset_id'], 'server/device identity mismatch')
before = manifest(state)
report_path = config / 'vm-bootstrap-report.json'
report = dict(created=[], migration_complete=False, before=before,
original_state=str(state), service_started=False, attempts=[])
if config.exists():
require(config.stat().st_uid == owner and config.stat().st_mode & 0o022 == 0,
'unsafe existing config directory')
require(report_path.is_file() and not report_path.is_symlink() and
report_path.stat().st_uid == owner and report_path.stat().st_mode & 0o777 == 0o600,
'existing configuration has no protected bootstrap provenance')
report = json.loads(report_path.read_text())
require(not report['migration_complete'] and report['original_state'] == str(state)
and report['before'] == before and str(config) in report['created'],
'partial bootstrap provenance or original state changed')
require({p.name for p in config.iterdir()} <= {'vm-bootstrap-report.json', 'ca.pem', 'agent-binding.json'},
'unknown configuration file')
report.setdefault('attempts', [])
encoded = base64.b64decode(payload['binary'], validate=True)
require(hashlib.sha256(encoded).hexdigest() == payload['binary_sha256'], 'artifact hash mismatch')
require(encoded[:6] == b'\x7fELF\x02\x01' and encoded[18:20] == b'\x3e\x00', 'wrong ELF platform')
ca = payload['ca'].encode()
require(b'BEGIN CERTIFICATE' in ca and b'PRIVATE KEY' not in ca, 'public CA required')
try:
account = pwd.getpwnam('rmm-agent')
except KeyError:
account = None
if account:
require(0 < account.pw_uid < 1000 and account.pw_gid > 0 and
account.pw_shell in ('/usr/sbin/nologin', '/sbin/nologin') and
grp.getgrnam('rmm-agent').gr_gid == account.pw_gid and
not any('rmm-agent' in g.gr_mem for g in grp.getgrall()), 'existing account unsafe')
else:
try:
grp.getgrnam('rmm-agent')
raise RuntimeError('existing group without dedicated account; review required')
except KeyError:
pass
if binary.exists():
require(str(binary) in report['created'] and binary.stat().st_uid == owner and
binary.stat().st_mode & 0o777 == 0o755 and
hashlib.sha256(binary.read_bytes()).hexdigest() == payload['previous_binary_sha256'],
'existing binary is not the proven prior artifact')
require(not unit.exists() or str(unit) in report['created'], 'foreign existing unit')
def create(path, data, mode, uid=owner, gid=None):
if gid is None:
gid = os.getegid()
if path.exists():
require(str(path) in report['created'] and path.read_bytes() == data and
path.stat().st_uid == uid and path.stat().st_mode & 0o777 == mode,
'existing resource differs from proven bootstrap')
return
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(data); stream.flush(); os.fsync(stream.fileno())
os.fchown(stream.fileno(), uid, gid); os.fchmod(stream.fileno(), mode)
report['created'].append(str(path))
def save_report():
temporary = config / ('.bootstrap-report-' + uuid.uuid4().hex)
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'w') as stream:
json.dump(report, stream, indent=2); stream.flush(); os.fsync(stream.fileno())
os.replace(temporary, report_path)
attempt = {'completed': False}
report['attempts'].append(attempt)
try:
if not config.exists():
config.mkdir(mode=0o755)
report['created'].append(str(config))
os.chmod(config, 0o755)
save_report()
if not binary.parent.exists():
binary.parent.mkdir(mode=0o755)
report['created'].append(str(binary.parent))
elif binary.parent.stat().st_mode & 0o777 != 0o755:
require({p.name for p in binary.parent.iterdir()} <= {'rmm-agent'},
'nonstandard binary directory requires review')
os.chmod(binary.parent, 0o755)
save_report()
if account is None:
subprocess.run(['groupadd', '--system', 'rmm-agent'], check=True, capture_output=True)
report['created'].append('system-group:rmm-agent')
save_report()
subprocess.run(['useradd', '--system', '--gid', 'rmm-agent', '--home-dir', str(destination),
'--no-create-home', '--shell', '/usr/sbin/nologin', 'rmm-agent'],
check=True, capture_output=True)
report['created'].append('system-user:rmm-agent')
save_report()
account = pwd.getpwnam('rmm-agent')
require(account.pw_uid != 0 and account.pw_gid != 0, 'account must be non-root')
staged = binary.with_name('.rmm-agent-' + uuid.uuid4().hex)
try:
create(staged, encoded, 0o755)
subprocess.run(['/lib64/ld-linux-x86-64.so.2', '--list', str(staged)],
check=True, capture_output=True)
os.replace(staged, binary)
report['created'].remove(str(staged))
if str(binary) not in report['created']:
report['created'].append(str(binary))
report['binary_sha256'] = payload['binary_sha256']
finally:
if staged.exists():
staged.unlink()
report['created'].remove(str(staged))
save_report()
create(config / 'ca.pem', ca, 0o644)
# Binding reader requires service ownership/0600. Root owns the directory;
# the service cannot replace the file or change its pinned authority.
create(config / 'agent-binding.json', json.dumps(payload['binding']).encode(),
0o600, account.pw_uid, account.pw_gid)
create(unit, payload['unit'].encode(), 0o644)
os.rename(state, destination)
report['migration_complete'] = True
for path in [destination] + list(destination.rglob('*')):
require(not path.is_symlink(), 'unexpected migration symlink')
os.chown(path, account.pw_uid, account.pw_gid)
os.chmod(path, 0o700 if path.is_dir() else 0o600)
after = manifest(destination)
require(before == after, 'migration changed state bytes or SQLite data')
report.update(after=after, byte_and_sqlite_preservation=True,
agent_uid=account.pw_uid, agent_gid=account.pw_gid)
subprocess.run(['systemctl', 'daemon-reload'], check=True, capture_output=True)
attempt['completed'] = True
except Exception as error:
attempt['error'] = sanitized_error(error)
raise
finally:
if config.is_dir():
save_report()
lock.close()
# No automatic start: caller first establishes and verifies reverse tunnel.
return dict(migration_complete=True, byte_and_sqlite_preservation=True,
agent_uid=account.pw_uid, service_started=False,
created=report['created'], databases=after['databases'])
def sanitized_error(error):
import traceback
return {'exception_class': type(error).__name__,
'traceback_lines': [frame.lineno for frame in traceback.extract_tb(error.__traceback__)]}
def remote_script(payload, staging_root=None):
source = Path(__file__).read_text()
source = source[:source.index('\ndef install():')]
source = source.replace('import vm_access_check as access\nfrom vm_passive_test import execute, remote_python, require\n\nROOT = access.ROOT',
'def require(value, message):\n if not value: raise RuntimeError(message)\nROOT = Path("/")')
argument = '' if staging_root is None else ', root=Path(' + repr(str(staging_root)) + ')'
return source + '\ntry:\n print(json.dumps({"result": remote_install(' + repr(payload) + argument + ')}))\nexcept Exception as error:\n print(json.dumps({"error": sanitized_error(error)}))\n'
def install():
sys.path.insert(0, str(ROOT / 'tests'))
from backend_contract import request
from local_integration import A, CA, STATE
prior = json.loads((STATE / 'vm-passive/result.json').read_text())
require(prior.get('agent_stopped') and prior.get('tunnel_stopped'), 'old lifecycle not stopped')
agent = str(uuid.UUID(prior['agent_id']))
operator = (STATE / 'operator-a.token').read_text().strip()
status, _, devices = request(f'/api/v1/tenants/{A}/devices', token=operator)
require(status == 200 and not devices['truncated'], 'device lookup incomplete')
device = next((x for x in devices['items'] if x['agent_id'] == agent), None)
require(device is not None, 'preserved enrolled device absent from server')
require('enrollment_generation' in device and 'assignment_version' in device,
'backend must expose authenticated identity generations before bootstrap')
status, _, key = request('/api/v1/job-signing-key', token=operator)
require(status == 200 and key['algorithm'] == 'Ed25519' and key['schema_version'] == 1,
'authenticated signing key GET rejected')
raw = bytes.fromhex(key['public_key_hex'])
require(len(raw) == 32 and hashlib.sha256(raw).hexdigest() == key['key_id'], 'invalid signing key')
binding = {name: device[name] for name in ('asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')}
binding.update(tenant_id=A, public_key_hex=raw.hex())
artifact = STATE / 'vm-jobs/docker/rmm-agent'
versions = execute(['readelf', '--version-info', str(artifact)]).decode()
required = max(tuple(map(int, x.split('.'))) for x in re.findall(r'GLIBC_(\d+\.\d+)', versions))
require(required <= (2, 39), 'artifact exceeds Ubuntu 24.04 glibc')
data = artifact.read_bytes()
payload = dict(source=prior['remote_directory'], binding=binding,
binary=base64.b64encode(data).decode(), binary_sha256=hashlib.sha256(data).hexdigest(),
ca=CA.read_text(), unit=(ROOT / 'deploy/linux/rmm-agent.service').read_text())
previous = STATE / 'vm-jobs/build/release/rmm-agent'
payload['previous_binary_sha256'] = hashlib.sha256(previous.read_bytes()).hexdigest()
# No source/module import on the VM; all trusted code and data travel on stdin.
code = remote_script(payload)
with connection() as (ssh, target, env):
response = json.loads(remote_python(ssh, target, env, code))
if 'error' in response:
print('REMOTE_BOOTSTRAP_ERROR=' + json.dumps(response['error']))
raise RuntimeError('remote bootstrap failed; inspect protected report before retry')
result = response['result']
result.update(artifact_sha256=payload['binary_sha256'], required_glibc='.'.join(map(str, required)),
agent_id=device['agent_id'], asset_id=device['asset_id'], signing_key_authenticated=True)
report = STATE / 'vm-jobs/bootstrap.json'
with report.open('x') as stream:
json.dump(result, stream, indent=2)
report.chmod(0o600)
print('PASS create-only non-root systemd bootstrap; identity/SQLite bytes preserved; service not started')
if __name__ == '__main__':
os.umask(0o077)
try:
install()
except Exception:
print('BLOCKED: bootstrap incomplete; private diagnostics withheld; inspect managed state before retry')
raise SystemExit(2) from None