feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded real-VM passive acceptance check. No root bootstrap or job execution."""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import vm_access_check as access
|
||||
|
||||
ROOT = access.ROOT
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
import local_integration as api
|
||||
|
||||
|
||||
def require(value, message):
|
||||
if not value:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def execute(args, **kwargs):
|
||||
result = subprocess.run(args, capture_output=True, timeout=40, **kwargs)
|
||||
require(result.returncode == 0, "command failed; private diagnostics withheld")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def remote_python(ssh, target, env, code):
|
||||
return execute(ssh + [target, "python3 -"], env=env, input=code.encode())
|
||||
|
||||
|
||||
PREFLIGHT = r'''
|
||||
import glob,json,os,platform,socket,subprocess
|
||||
agents=0
|
||||
for path in glob.glob('/proc/[0-9]*/exe'):
|
||||
try:
|
||||
if os.path.basename(os.readlink(path))=='rmm-agent': agents+=1
|
||||
except OSError: pass
|
||||
ports=subprocess.run(['ss','-H','-ltn'],capture_output=True,text=True,check=True).stdout
|
||||
occupied=any(line.split()[3].rsplit(':',1)[-1]=='18443' for line in ports.splitlines())
|
||||
libs=subprocess.run(['/sbin/ldconfig','-p'],capture_output=True,text=True,check=True).stdout
|
||||
print(json.dumps(dict(architecture=platform.machine(),glibc=platform.libc_ver()[1],
|
||||
hostname=socket.gethostname(),agents=agents,occupied=occupied,libraries=libs,
|
||||
test_directories=len(glob.glob(os.path.expanduser('~/rmm-passive-test.*'))))))
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
os.umask(0o077)
|
||||
config = access.load_config(ROOT / "credentials/linux-vm.conf")
|
||||
known = Path.home() / ".ssh/known_hosts"
|
||||
address, port = config["address"], config.get("port", "22")
|
||||
lookup = address if port == "22" else f"[{address}]:{port}"
|
||||
require(known.is_file(), "trusted host key file missing")
|
||||
execute(["ssh-keygen", "-F", lookup, "-f", str(known)])
|
||||
for path in Path('/proc').glob('[0-9]*/cmdline'):
|
||||
try:
|
||||
args = path.read_bytes().split(b'\0')
|
||||
except OSError:
|
||||
continue
|
||||
require(not (args[0].split(b'/')[-1] == b'ssh' and
|
||||
any(b'18443:127.0.0.1:18443' in x for x in args)),
|
||||
"existing reverse tunnel found; refusing competing listener")
|
||||
require(api.request('/healthz')[0] == 200, "local trusted TLS health failed")
|
||||
with tempfile.TemporaryDirectory(prefix="rmm-vm-", dir=ROOT / "credentials") as private:
|
||||
private = Path(private)
|
||||
password = private / "password"
|
||||
password.write_text(config.get("password", "") + "\n")
|
||||
password.chmod(0o600)
|
||||
askpass = private / "askpass"
|
||||
askpass.write_text('#!/bin/sh\nexec cat "$RMM_SSH_PASSWORD_FILE"\n')
|
||||
askpass.chmod(0o700)
|
||||
env = dict(os.environ, SSH_ASKPASS=str(askpass), SSH_ASKPASS_REQUIRE="force",
|
||||
DISPLAY="rmm-test:0", RMM_SSH_PASSWORD_FILE=str(password))
|
||||
ssh = ["ssh", "-F", "/dev/null", "-T", "-p", port,
|
||||
"-o", "StrictHostKeyChecking=yes", "-o", f"UserKnownHostsFile={known}",
|
||||
"-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10",
|
||||
"-o", "ConnectionAttempts=1", "-o", "ForwardAgent=no",
|
||||
"-o", "NumberOfPasswordPrompts=1", "-o", "LogLevel=ERROR",
|
||||
"-o", "PreferredAuthentications=publickey,password"]
|
||||
if config.get("ssh_key_path"):
|
||||
ssh += ["-i", str(Path(config['ssh_key_path']).expanduser()), "-o", "IdentitiesOnly=yes"]
|
||||
target = f"{config['username']}@{address}"
|
||||
facts = json.loads(remote_python(ssh, target, env, PREFLIGHT))
|
||||
require(facts['architecture'] == 'x86_64', "unsupported VM architecture")
|
||||
require(facts['agents'] == 0 and not facts['occupied'], "VM agent or port already occupied; nothing stopped")
|
||||
print("PASS existing-process/port preflight; prior private test directory count:", facts['test_directories'])
|
||||
artifact = private / "rmm-agent"
|
||||
execute(["docker", "compose", "--env-file", str(ROOT / "devstate/.env"),
|
||||
"-f", str(ROOT / "deploy/compose.yaml"), "-p", "rust-discovery-dev",
|
||||
"cp", "server:/usr/local/bin/rmm-agent", str(artifact)])
|
||||
artifact.chmod(0o700)
|
||||
elf = artifact.read_bytes()
|
||||
require(elf[:6] == b'\x7fELF\x02\x01' and int.from_bytes(elf[18:20], 'little') == 62,
|
||||
"agent artifact is not Linux x86-64 ELF")
|
||||
versions = execute(["readelf", "--version-info", str(artifact)]).decode()
|
||||
required_glibc = max(tuple(map(int, x.split('.'))) for x in re.findall(r'GLIBC_(\d+\.\d+)', versions))
|
||||
require(tuple(map(int, facts['glibc'].split('.'))) >= required_glibc, "agent requires newer glibc")
|
||||
dynamic = execute(["readelf", "-d", str(artifact)]).decode()
|
||||
libraries = re.findall(r'\(NEEDED\).*\[(.*?)\]', dynamic)
|
||||
require(libraries and all(re.search(r'^\s*' + re.escape(lib) + r'\s', facts['libraries'], re.M) for lib in libraries),
|
||||
"VM runtime library missing")
|
||||
print("PASS artifact ELF/glibc/dependency compatibility; required glibc:", '.'.join(map(str, required_glibc)))
|
||||
control = str(private / 'control')
|
||||
tunnel_log = private / 'ssh.log'
|
||||
remote_dir = None
|
||||
agent_pid = None
|
||||
report = {'artifact_sha256': hashlib.sha256(elf).hexdigest(),
|
||||
'architecture': facts['architecture'], 'glibc': facts['glibc']}
|
||||
report_dir = ROOT / 'devstate/vm-passive'
|
||||
report_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
require(not (report_dir / 'active.json').exists(), "previous managed test record exists; inspect before retry")
|
||||
with tunnel_log.open('wb') as log:
|
||||
tunnel = subprocess.Popen(ssh + ['-M', '-S', control, '-N', '-T',
|
||||
'-o', 'ExitOnForwardFailure=yes', '-o', 'ServerAliveInterval=30',
|
||||
'-o', 'ServerAliveCountMax=3', '-R', '127.0.0.1:18443:127.0.0.1:18443', target],
|
||||
env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
||||
try:
|
||||
deadline = time.monotonic() + 15
|
||||
while not Path(control).exists():
|
||||
require(tunnel.poll() is None, "strict SSH reverse tunnel failed")
|
||||
require(time.monotonic() < deadline, "SSH control socket timeout")
|
||||
time.sleep(.1)
|
||||
connected = ssh + ['-S', control, '-o', 'BatchMode=yes']
|
||||
execute(connected + ['-O', 'check', target], env=env)
|
||||
listeners = execute(connected + [target, "ss -H -ltn 'sport = :18443'"], env=env).decode()
|
||||
endpoints = [line.split()[3] for line in listeners.splitlines()]
|
||||
require(endpoints == ['127.0.0.1:18443'], "reverse listener not exclusively IPv4 loopback")
|
||||
print("PASS PC-initiated SSH reverse tunnel; VM listener exclusively 127.0.0.1:18443")
|
||||
raw_token = api.token(api.A)
|
||||
bundle = io.BytesIO()
|
||||
with tarfile.open(fileobj=bundle, mode='w') as archive:
|
||||
for name, data, mode in [('rmm-agent', elf, 0o700),
|
||||
('ca.pem', api.CA.read_bytes(), 0o600), ('enroll.token', raw_token.encode(), 0o600)]:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size, info.mode = len(data), mode
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
remote_dir = execute(connected + [target,
|
||||
'umask 077; d=$(mktemp -d "$HOME/rmm-passive-test.XXXXXXXX") || exit; '
|
||||
'tar -xf - -C "$d" || exit; printf "%s" "$d"'], env=env, input=bundle.getvalue()).decode()
|
||||
require(remote_dir.startswith('/') and '\n' not in remote_dir, "invalid private deployment path")
|
||||
quoted = shlex.quote(remote_dir)
|
||||
# The loader verifies symbol versions before any agent main function runs.
|
||||
execute(connected + [target, f'/lib64/ld-linux-x86-64.so.2 --list {quoted}/rmm-agent'], env=env)
|
||||
print("PASS VM dynamic loader resolved concrete artifact before native agent launch")
|
||||
launch = f'''import os,subprocess
|
||||
directory={remote_dir!r}
|
||||
os.umask(0o077)
|
||||
env=dict(os.environ,RMM_AGENT_STATE=directory+'/state',RMM_CA_FILE=directory+'/ca.pem',
|
||||
RMM_ENROLLMENT_TOKEN_FILE=directory+'/enroll.token',RMM_CONNECT_ADDR='127.0.0.1:18443')
|
||||
with open(directory+'/agent.log','ab') as log:
|
||||
process=subprocess.Popen([directory+'/rmm-agent'],cwd=directory,env=env,
|
||||
stdin=subprocess.DEVNULL,stdout=log,stderr=log,start_new_session=True)
|
||||
print(process.pid)
|
||||
'''
|
||||
agent_pid = int(remote_python(connected, target, env, launch))
|
||||
# Identify only the exact binary in the newly created private directory.
|
||||
code = f'''import glob,json,os
|
||||
path={remote_dir!r}+'/rmm-agent'
|
||||
pids=[]
|
||||
for p in glob.glob('/proc/[0-9]*/exe'):
|
||||
try:
|
||||
if os.readlink(p)==path:pids.append(int(p.split('/')[2]))
|
||||
except OSError:pass
|
||||
print(json.dumps(pids))
|
||||
'''
|
||||
pids = json.loads(remote_python(connected, target, env, code))
|
||||
require(pids == [agent_pid], "new VM agent process not uniquely identified")
|
||||
agent_pid = pids[0]
|
||||
report.update(remote_directory=remote_dir, agent_pid=agent_pid, tunnel_pid=tunnel.pid)
|
||||
(report_dir / 'active.json').write_text(json.dumps(report))
|
||||
identity = None
|
||||
observed = None
|
||||
contacts = set()
|
||||
deadline = time.monotonic() + 100
|
||||
viewer = (ROOT / 'devstate/viewer-a.token').read_text().strip()
|
||||
while time.monotonic() < deadline:
|
||||
if identity is None:
|
||||
result = subprocess.run(connected + [target, f'cat {quoted}/state/identity.json'],
|
||||
capture_output=True, env=env, timeout=15)
|
||||
if result.returncode == 0:
|
||||
identity = json.loads(result.stdout)
|
||||
if identity:
|
||||
status, data = api.request(f'/api/v1/tenants/{api.A}/devices', viewer)
|
||||
require(status == 200, "tenant reader failed")
|
||||
observed = next((x for x in json.loads(data)['items'] if x['agent_id'] == identity['agent_id']), None)
|
||||
if observed and observed['last_contact_at']:
|
||||
contacts.add(observed['last_contact_at'])
|
||||
if observed and observed['inventory'] and len(contacts) >= 2:
|
||||
break
|
||||
require(tunnel.poll() is None, "tunnel ended before acceptance")
|
||||
time.sleep(2)
|
||||
require(observed and observed['inventory'] and len(contacts) >= 2, "real VM inventory/two fresh heartbeats not observed")
|
||||
inventory = observed['inventory']
|
||||
require(inventory['hostname'] == facts['hostname'] and inventory['architecture'] == 'x86_64'
|
||||
and '24.04' in inventory['os'] and inventory['root_available_bytes'] > 0
|
||||
and inventory['root_available_inodes'] > 0 and observed['observed_at'] and observed['received_at']
|
||||
and observed['presence'] == 'online', "VM inventory evidence invalid")
|
||||
viewer_b = (ROOT / 'devstate/viewer-b.token').read_text().strip()
|
||||
status, data = api.request(f'/api/v1/tenants/{api.B}/devices', viewer_b)
|
||||
require(status == 200 and identity['agent_id'] not in data.decode(), "VM tenant isolation failed")
|
||||
execute(connected + [target, f'rm -- {quoted}/enroll.token'], env=env)
|
||||
report.update(agent_id=identity['agent_id'], observed_at=observed['observed_at'],
|
||||
received_at=observed['received_at'], heartbeat_times=sorted(contacts),
|
||||
verified=True, tenant_isolation=True)
|
||||
print("PASS actual VM enrollment/mTLS-WSS/inventory, two distinct fresh heartbeats, tenant isolation")
|
||||
finally:
|
||||
try:
|
||||
if remote_dir and agent_pid:
|
||||
cleanup = f'''import os,signal,time
|
||||
pid={agent_pid}
|
||||
expected={remote_dir!r}+'/rmm-agent'
|
||||
try:
|
||||
if os.readlink('/proc/'+str(pid)+'/exe')==expected: os.kill(pid,signal.SIGTERM)
|
||||
except FileNotFoundError:pass
|
||||
for _ in range(30):
|
||||
try:
|
||||
if os.readlink('/proc/'+str(pid)+'/exe')!=expected:break
|
||||
except FileNotFoundError:break
|
||||
time.sleep(.1)
|
||||
else:raise RuntimeError('agent did not stop')
|
||||
token={remote_dir!r}+'/enroll.token'
|
||||
try:os.unlink(token)
|
||||
except FileNotFoundError:pass
|
||||
print('stopped')
|
||||
'''
|
||||
remote_python(ssh + ['-S', control, '-o', 'BatchMode=yes'], target, env, cleanup)
|
||||
report['agent_stopped'] = True
|
||||
finally:
|
||||
tunnel.terminate()
|
||||
tunnel.wait(timeout=15)
|
||||
report['tunnel_stopped'] = True
|
||||
(report_dir / 'result.json').write_text(json.dumps(report, indent=2))
|
||||
if report.get('agent_stopped'):
|
||||
(report_dir / 'active.json').unlink(missing_ok=True)
|
||||
print("PASS bounded test stopped its own agent/tunnel; private PID/path evidence: devstate/vm-passive/result.json")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except RuntimeError as error:
|
||||
print("FAILED:", error)
|
||||
raise SystemExit(1) from None
|
||||
except Exception:
|
||||
print("FAILED: VM passive test incomplete; private details withheld; inspect managed state before retry")
|
||||
raise SystemExit(1) from None
|
||||
Reference in New Issue
Block a user