Files
BuTzZ 6c3203d725
Rust development slice / linux (push) Failing after 1m10s
feat: snapshot Linux-first RMM MVP and pause handover
2026-11-12 22:06:25 +01:00

202 lines
13 KiB
Python

#!/usr/bin/env python3
"""Bounded real systemd-agent/MCP/browser Hello-World check; no root jobs."""
import datetime
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
import vm_job_bootstrap as bootstrap
from vm_passive_test import execute, remote_python, require
ROOT = bootstrap.ROOT
sys.path.insert(0, str(ROOT / 'tests'))
from backend_contract import request
from local_integration import A, B, STATE, sql
def store(path, value):
with path.open('w') as stream:
json.dump(value, stream, indent=2)
path.chmod(0o600)
def can_stop(states):
return isinstance(states, list) and all(state in ('succeeded', 'failed', 'cancelled', 'expired') for state in states)
def run():
os.umask(0o077)
directory = STATE / 'vm-jobs'
installed = json.loads((directory / 'bootstrap.json').read_text())
require(installed['migration_complete'] and installed['byte_and_sqlite_preservation'],
'controlled migration not proven')
require(not (directory / 'active.json').exists(), 'previous active lifecycle requires review')
agent = str(uuid.UUID(installed['agent_id']))
asset = str(uuid.UUID(installed['asset_id']))
operator = (STATE / 'operator-a.token').read_text().strip()
viewer_b = (STATE / 'viewer-b.token').read_text().strip()
base = f'/api/v1/tenants/{A}'
report = dict(agent_id=agent, asset_id=asset, testsimulation=True, service_started=False,
tunnel_started=False, jobs_verified=False)
active = directory / 'active.json'
def counts():
return sql(f"SELECT json_build_array((SELECT count(*) FROM jobs WHERE agent_id='{agent}'),(SELECT count(*) FROM outbox o JOIN jobs j USING(tenant_id,job_id) WHERE j.agent_id='{agent}'),(SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id) WHERE j.agent_id='{agent}'));")
def idle():
states = json.loads(sql(f"SELECT coalesce(json_agg(state),'[]'::json) FROM jobs WHERE agent_id='{agent}';"))
return can_stop(states)
baseline = json.loads(counts())
require(baseline == [0, 0, 0], 'existing VM job history requires separate continuation; no new run')
with bootstrap.connection() as (ssh, target, env):
# Refuse to compete with any existing reverse tunnel or native agent.
for path in Path('/proc').glob('[0-9]*/cmdline'):
try:
args = path.read_bytes().split(b'\0')
except OSError:
continue
require(not (args[0].split(b'/')[-1] == b'ssh' and
any(b'18443:127.0.0.1:18443' in x for x in args)), 'existing reverse tunnel')
preflight = '''import json,os,subprocess,pwd
from pathlib import Path
ports=subprocess.run(['ss','-H','-ltn'],capture_output=True,text=True,check=True).stdout
occupied=any(x.split()[3].rsplit(':',1)[-1]=='18443' for x in ports.splitlines())
status=subprocess.run(['systemctl','is-active','rmm-agent.service'],capture_output=True,text=True).stdout.strip()
account=pwd.getpwnam('rmm-agent')
print(json.dumps(dict(port_occupied=occupied,already_active=status=='active',uid=account.pw_uid,
state_uid=Path('/var/lib/rmm-agent').stat().st_uid,old_source_exists=Path(%r).exists())))
''' % (json.loads((STATE / 'vm-passive/result.json').read_text())['remote_directory'] + '/state')
facts = json.loads(remote_python(ssh, target, env, preflight))
require(not facts['port_occupied'] and not facts['already_active'] and
facts['uid'] == installed['agent_uid'] and facts['state_uid'] == facts['uid']
and facts['uid'] > 0 and not facts['old_source_exists'], 'VM service readiness failed')
control = str(directory / 'ssh-control')
require(not Path(control).exists(), 'prior SSH control socket requires review')
with (directory / 'tunnel.log').open('wb') as log:
tunnel = subprocess.Popen(ssh + ['-M', '-S', control, '-N', '-T',
'-o', 'ExitOnForwardFailure=yes', '-o', 'ServerAliveInterval=30',
'-o', 'ServerAliveCountMax=3', '-R', '127.0.0.1:18443:127.0.0.1:18443', target],
env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
report.update(tunnel_started=True, tunnel_pid=tunnel.pid)
store(active, report)
connected = ssh + ['-S', control, '-o', 'BatchMode=yes']
try:
deadline = time.monotonic() + 15
while not Path(control).exists():
require(tunnel.poll() is None and time.monotonic() < deadline, 'strict reverse tunnel setup failed')
time.sleep(.1)
listeners = execute(connected + [target, "ss -H -ltn 'sport = :18443'"], env=env).decode()
require([line.split()[3] for line in listeners.splitlines()] == ['127.0.0.1:18443'],
'reverse listener not exclusively IPv4 loopback')
execute(connected + [target, 'systemctl start rmm-agent.service'], env=env)
report['service_started'] = True
store(active, report)
contacts = set()
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
status, _, device = request(base + '/devices/' + asset, token=operator)
require(status == 200 and device['agent_id'] == agent, 'preserved server identity mismatch')
if device['last_contact_at']:
moment = datetime.datetime.fromisoformat(device['last_contact_at'].replace('Z', '+00:00'))
if (datetime.datetime.now(datetime.timezone.utc) - moment).total_seconds() < 90:
contacts.add(device['last_contact_at'])
if len(contacts) >= 2 and 'run_script' in device['capabilities']:
break
time.sleep(2)
require(len(contacts) >= 2 and 'run_script' in device['capabilities'], 'fresh native agent heartbeats/capability missing')
report['heartbeat_times'] = sorted(contacts)
code = '''import json,subprocess
from pathlib import Path
pid=int(subprocess.run(['systemctl','show','--property=MainPID','--value','rmm-agent.service'],capture_output=True,text=True,check=True).stdout)
uid=Path('/proc/'+str(pid)).stat().st_uid
logs=subprocess.run(['journalctl','-u','rmm-agent.service','--since','5 minutes ago','-o','cat','--no-pager'],capture_output=True,text=True,check=True).stdout
print(json.dumps(dict(agent_pid=pid,agent_uid=uid,sync_blocked='restore/reconnect sync unavailable' in logs)))
'''
runtime = json.loads(remote_python(connected, target, env, code))
report.update(runtime)
require(runtime['agent_uid'] == installed['agent_uid'] and not runtime['sync_blocked'],
'non-root runtime or reconnect synchronization gate blocked')
require(request(f'/api/v1/tenants/{B}/devices/{asset}', token=viewer_b)[0] == 404,
'cross-tenant device exposure')
headers = {'Accept': 'application/json, text/event-stream', 'MCP-Protocol-Version': '2025-11-25'}
init = {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {
'protocolVersion': '2025-11-25', 'capabilities': {}, 'clientInfo': {'name': 'real-vm-hello', 'version': '1'}}}
status, response_headers, response = request('/mcp', 'POST', init, operator, headers)
require(status == 200 and response['result']['protocolVersion'] == '2025-11-25', 'MCP initialize failed')
session = response_headers.get('mcp-session-id', response_headers.get('Mcp-Session-Id'))
if session:
headers['Mcp-Session-Id'] = session
require(request('/mcp', 'POST', {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, operator, headers)[0] == 202,
'MCP session initialization failed')
def call(name, arguments, identifier):
status, _, response = request('/mcp', 'POST', {'jsonrpc': '2.0', 'id': identifier,
'method': 'tools/call', 'params': {'name': name, 'arguments': arguments}}, operator, headers)
require(status == 200 and 'result' in response and not response['result'].get('isError'), 'MCP tool failed')
return response['result']['structuredContent']
now = datetime.datetime.now(datetime.timezone.utc)
plan_input = dict(tenant_id=A, action='run_script', targets={'asset_ids': [asset]},
parameters={'variants': [{'os_family': 'linux', 'interpreter': 'posix_sh', 'encoding': 'utf-8',
'script_utf8': "printf '%s\\n' 'Hello World'\n"}], 'arguments': [], 'execution_identity': 'agent_service',
'timeout_seconds': 60, 'max_output_bytes': 65536, 'expected_stdout': 'Hello World\n'},
schedule={'not_before': now.isoformat(), 'start_before': (now + datetime.timedelta(minutes=15)).isoformat(),
'offline_policy': 'wait_until_deadline'}, rollout={'max_parallel': 1, 'halt_after_failures': 1},
reason='Real Linux VM · non-root systemd agent · simulated dev/test UI approval')
plan = call('operations.plan', {'tenant_id': A, 'input': plan_input}, 2)
require(hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash']
and len(plan['jobs']) == 1 and plan['jobs'][0]['agent_id'] == agent, 'immutable plan binding mismatch')
report.update(plan_id=plan['plan_id'], plan_hash=plan['plan_hash'], job_id=plan['jobs'][0]['job_id'])
store(active, report)
pending = call('operations.submit', {'tenant_id': A, 'plan_id': plan['plan_id'],
'idempotency_key': 'vm-hello-' + uuid.uuid4().hex}, 3)
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0
and pending['execution_state'] is None and json.loads(counts()) == baseline,
'preapproval executable-artifact invariant failed')
operation = str(uuid.UUID(pending['operation_id']))
report.update(operation_id=operation, preapproval_counts=baseline)
store(active, report)
browser = subprocess.run([sys.executable, str(ROOT / 'deploy/browser/run_vm_jobs.py'),
'--operation-id', operation, '--asset-id', asset, '--dev-test-approve'], timeout=150)
report['browser_exit_code'] = browser.returncode
current = request(base + '/operations/' + operation, token=operator)[2]
report['final_operation'] = current
require(browser.returncode == 0 and current['approval_state'] == 'approved' and
current['job_count'] == 1 and current['execution_state'] == 'succeeded',
'browser or VM result not confirmed; never retry effect')
job = request(base + '/jobs/' + report['job_id'], token=operator)[2]
result = job['result']
require(job['state'] == 'succeeded' and bytes(result['stdout']) == b'Hello World\n' and
result['exit_code'] == 0 and result['evidence']['postconditions_passed'] is True,
'exact output/postconditions not confirmed')
audit = sql(f"SELECT json_build_object('approval_origin',approval_origin,'testsimulation',testsimulation,'principal',principal,'plan_hash',plan_hash) FROM authorizations WHERE tenant_id='{A}' AND operation_id='{operation}';")
report['approval_evidence'] = json.loads(audit)
require(report['approval_evidence']['testsimulation'] is True and
report['approval_evidence']['approval_origin'] == 'dev_test_browser_simulation' and
report['approval_evidence']['plan_hash'] == plan['plan_hash'], 'approval simulation audit mismatch')
report.update(jobs_verified=True, postapproval_counts=json.loads(counts()), output_verified=True)
print('PASS actual non-root systemd VM Hello World via MCP pending request and explicit simulated UI click')
finally:
# Unknown/queued/in-flight jobs retain the service and owned tunnel.
# No effect is retried and no process is killed on an ambiguous result.
if idle():
if report['service_started']:
execute(connected + [target, 'systemctl stop rmm-agent.service'], env=env)
report['service_stopped'] = True
tunnel.terminate(); tunnel.wait(timeout=15)
report['tunnel_stopped'] = True
active.unlink(missing_ok=True)
else:
report['lifecycle_retained_for_active_or_unknown_jobs'] = True
store(active, report)
store(directory / 'result.json', report)
if __name__ == '__main__':
try:
run()
except Exception as error:
print('VM_JOB_CHECK_INCOMPLETE=' + json.dumps(bootstrap.sanitized_error(error)))
raise SystemExit(2) from None