202 lines
13 KiB
Python
202 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded real systemd-agent/MCP/browser Hello-World check; no root jobs."""
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
import vm_job_bootstrap as bootstrap
|
|
from vm_passive_test import execute, remote_python, require
|
|
|
|
ROOT = bootstrap.ROOT
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from backend_contract import request
|
|
from local_integration import A, B, STATE, sql
|
|
|
|
|
|
def store(path, value):
|
|
with path.open('w') as stream:
|
|
json.dump(value, stream, indent=2)
|
|
path.chmod(0o600)
|
|
|
|
|
|
def can_stop(states):
|
|
return isinstance(states, list) and all(state in ('succeeded', 'failed', 'cancelled', 'expired') for state in states)
|
|
|
|
|
|
def run():
|
|
os.umask(0o077)
|
|
directory = STATE / 'vm-jobs'
|
|
installed = json.loads((directory / 'bootstrap.json').read_text())
|
|
require(installed['migration_complete'] and installed['byte_and_sqlite_preservation'],
|
|
'controlled migration not proven')
|
|
require(not (directory / 'active.json').exists(), 'previous active lifecycle requires review')
|
|
agent = str(uuid.UUID(installed['agent_id']))
|
|
asset = str(uuid.UUID(installed['asset_id']))
|
|
operator = (STATE / 'operator-a.token').read_text().strip()
|
|
viewer_b = (STATE / 'viewer-b.token').read_text().strip()
|
|
base = f'/api/v1/tenants/{A}'
|
|
report = dict(agent_id=agent, asset_id=asset, testsimulation=True, service_started=False,
|
|
tunnel_started=False, jobs_verified=False)
|
|
active = directory / 'active.json'
|
|
def counts():
|
|
return sql(f"SELECT json_build_array((SELECT count(*) FROM jobs WHERE agent_id='{agent}'),(SELECT count(*) FROM outbox o JOIN jobs j USING(tenant_id,job_id) WHERE j.agent_id='{agent}'),(SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id) WHERE j.agent_id='{agent}'));")
|
|
def idle():
|
|
states = json.loads(sql(f"SELECT coalesce(json_agg(state),'[]'::json) FROM jobs WHERE agent_id='{agent}';"))
|
|
return can_stop(states)
|
|
baseline = json.loads(counts())
|
|
require(baseline == [0, 0, 0], 'existing VM job history requires separate continuation; no new run')
|
|
with bootstrap.connection() as (ssh, target, env):
|
|
# Refuse to compete with any existing reverse tunnel or native agent.
|
|
for path in Path('/proc').glob('[0-9]*/cmdline'):
|
|
try:
|
|
args = path.read_bytes().split(b'\0')
|
|
except OSError:
|
|
continue
|
|
require(not (args[0].split(b'/')[-1] == b'ssh' and
|
|
any(b'18443:127.0.0.1:18443' in x for x in args)), 'existing reverse tunnel')
|
|
preflight = '''import json,os,subprocess,pwd
|
|
from pathlib import Path
|
|
ports=subprocess.run(['ss','-H','-ltn'],capture_output=True,text=True,check=True).stdout
|
|
occupied=any(x.split()[3].rsplit(':',1)[-1]=='18443' for x in ports.splitlines())
|
|
status=subprocess.run(['systemctl','is-active','rmm-agent.service'],capture_output=True,text=True).stdout.strip()
|
|
account=pwd.getpwnam('rmm-agent')
|
|
print(json.dumps(dict(port_occupied=occupied,already_active=status=='active',uid=account.pw_uid,
|
|
state_uid=Path('/var/lib/rmm-agent').stat().st_uid,old_source_exists=Path(%r).exists())))
|
|
''' % (json.loads((STATE / 'vm-passive/result.json').read_text())['remote_directory'] + '/state')
|
|
facts = json.loads(remote_python(ssh, target, env, preflight))
|
|
require(not facts['port_occupied'] and not facts['already_active'] and
|
|
facts['uid'] == installed['agent_uid'] and facts['state_uid'] == facts['uid']
|
|
and facts['uid'] > 0 and not facts['old_source_exists'], 'VM service readiness failed')
|
|
control = str(directory / 'ssh-control')
|
|
require(not Path(control).exists(), 'prior SSH control socket requires review')
|
|
with (directory / 'tunnel.log').open('wb') as log:
|
|
tunnel = subprocess.Popen(ssh + ['-M', '-S', control, '-N', '-T',
|
|
'-o', 'ExitOnForwardFailure=yes', '-o', 'ServerAliveInterval=30',
|
|
'-o', 'ServerAliveCountMax=3', '-R', '127.0.0.1:18443:127.0.0.1:18443', target],
|
|
env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
|
report.update(tunnel_started=True, tunnel_pid=tunnel.pid)
|
|
store(active, report)
|
|
connected = ssh + ['-S', control, '-o', 'BatchMode=yes']
|
|
try:
|
|
deadline = time.monotonic() + 15
|
|
while not Path(control).exists():
|
|
require(tunnel.poll() is None and time.monotonic() < deadline, 'strict reverse tunnel setup failed')
|
|
time.sleep(.1)
|
|
listeners = execute(connected + [target, "ss -H -ltn 'sport = :18443'"], env=env).decode()
|
|
require([line.split()[3] for line in listeners.splitlines()] == ['127.0.0.1:18443'],
|
|
'reverse listener not exclusively IPv4 loopback')
|
|
execute(connected + [target, 'systemctl start rmm-agent.service'], env=env)
|
|
report['service_started'] = True
|
|
store(active, report)
|
|
contacts = set()
|
|
deadline = time.monotonic() + 100
|
|
while time.monotonic() < deadline:
|
|
status, _, device = request(base + '/devices/' + asset, token=operator)
|
|
require(status == 200 and device['agent_id'] == agent, 'preserved server identity mismatch')
|
|
if device['last_contact_at']:
|
|
moment = datetime.datetime.fromisoformat(device['last_contact_at'].replace('Z', '+00:00'))
|
|
if (datetime.datetime.now(datetime.timezone.utc) - moment).total_seconds() < 90:
|
|
contacts.add(device['last_contact_at'])
|
|
if len(contacts) >= 2 and 'run_script' in device['capabilities']:
|
|
break
|
|
time.sleep(2)
|
|
require(len(contacts) >= 2 and 'run_script' in device['capabilities'], 'fresh native agent heartbeats/capability missing')
|
|
report['heartbeat_times'] = sorted(contacts)
|
|
code = '''import json,subprocess
|
|
from pathlib import Path
|
|
pid=int(subprocess.run(['systemctl','show','--property=MainPID','--value','rmm-agent.service'],capture_output=True,text=True,check=True).stdout)
|
|
uid=Path('/proc/'+str(pid)).stat().st_uid
|
|
logs=subprocess.run(['journalctl','-u','rmm-agent.service','--since','5 minutes ago','-o','cat','--no-pager'],capture_output=True,text=True,check=True).stdout
|
|
print(json.dumps(dict(agent_pid=pid,agent_uid=uid,sync_blocked='restore/reconnect sync unavailable' in logs)))
|
|
'''
|
|
runtime = json.loads(remote_python(connected, target, env, code))
|
|
report.update(runtime)
|
|
require(runtime['agent_uid'] == installed['agent_uid'] and not runtime['sync_blocked'],
|
|
'non-root runtime or reconnect synchronization gate blocked')
|
|
require(request(f'/api/v1/tenants/{B}/devices/{asset}', token=viewer_b)[0] == 404,
|
|
'cross-tenant device exposure')
|
|
headers = {'Accept': 'application/json, text/event-stream', 'MCP-Protocol-Version': '2025-11-25'}
|
|
init = {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {
|
|
'protocolVersion': '2025-11-25', 'capabilities': {}, 'clientInfo': {'name': 'real-vm-hello', 'version': '1'}}}
|
|
status, response_headers, response = request('/mcp', 'POST', init, operator, headers)
|
|
require(status == 200 and response['result']['protocolVersion'] == '2025-11-25', 'MCP initialize failed')
|
|
session = response_headers.get('mcp-session-id', response_headers.get('Mcp-Session-Id'))
|
|
if session:
|
|
headers['Mcp-Session-Id'] = session
|
|
require(request('/mcp', 'POST', {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, operator, headers)[0] == 202,
|
|
'MCP session initialization failed')
|
|
def call(name, arguments, identifier):
|
|
status, _, response = request('/mcp', 'POST', {'jsonrpc': '2.0', 'id': identifier,
|
|
'method': 'tools/call', 'params': {'name': name, 'arguments': arguments}}, operator, headers)
|
|
require(status == 200 and 'result' in response and not response['result'].get('isError'), 'MCP tool failed')
|
|
return response['result']['structuredContent']
|
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
plan_input = dict(tenant_id=A, action='run_script', targets={'asset_ids': [asset]},
|
|
parameters={'variants': [{'os_family': 'linux', 'interpreter': 'posix_sh', 'encoding': 'utf-8',
|
|
'script_utf8': "printf '%s\\n' 'Hello World'\n"}], 'arguments': [], 'execution_identity': 'agent_service',
|
|
'timeout_seconds': 60, 'max_output_bytes': 65536, 'expected_stdout': 'Hello World\n'},
|
|
schedule={'not_before': now.isoformat(), 'start_before': (now + datetime.timedelta(minutes=15)).isoformat(),
|
|
'offline_policy': 'wait_until_deadline'}, rollout={'max_parallel': 1, 'halt_after_failures': 1},
|
|
reason='Real Linux VM · non-root systemd agent · simulated dev/test UI approval')
|
|
plan = call('operations.plan', {'tenant_id': A, 'input': plan_input}, 2)
|
|
require(hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash']
|
|
and len(plan['jobs']) == 1 and plan['jobs'][0]['agent_id'] == agent, 'immutable plan binding mismatch')
|
|
report.update(plan_id=plan['plan_id'], plan_hash=plan['plan_hash'], job_id=plan['jobs'][0]['job_id'])
|
|
store(active, report)
|
|
pending = call('operations.submit', {'tenant_id': A, 'plan_id': plan['plan_id'],
|
|
'idempotency_key': 'vm-hello-' + uuid.uuid4().hex}, 3)
|
|
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0
|
|
and pending['execution_state'] is None and json.loads(counts()) == baseline,
|
|
'preapproval executable-artifact invariant failed')
|
|
operation = str(uuid.UUID(pending['operation_id']))
|
|
report.update(operation_id=operation, preapproval_counts=baseline)
|
|
store(active, report)
|
|
browser = subprocess.run([sys.executable, str(ROOT / 'deploy/browser/run_vm_jobs.py'),
|
|
'--operation-id', operation, '--asset-id', asset, '--dev-test-approve'], timeout=150)
|
|
report['browser_exit_code'] = browser.returncode
|
|
current = request(base + '/operations/' + operation, token=operator)[2]
|
|
report['final_operation'] = current
|
|
require(browser.returncode == 0 and current['approval_state'] == 'approved' and
|
|
current['job_count'] == 1 and current['execution_state'] == 'succeeded',
|
|
'browser or VM result not confirmed; never retry effect')
|
|
job = request(base + '/jobs/' + report['job_id'], token=operator)[2]
|
|
result = job['result']
|
|
require(job['state'] == 'succeeded' and bytes(result['stdout']) == b'Hello World\n' and
|
|
result['exit_code'] == 0 and result['evidence']['postconditions_passed'] is True,
|
|
'exact output/postconditions not confirmed')
|
|
audit = sql(f"SELECT json_build_object('approval_origin',approval_origin,'testsimulation',testsimulation,'principal',principal,'plan_hash',plan_hash) FROM authorizations WHERE tenant_id='{A}' AND operation_id='{operation}';")
|
|
report['approval_evidence'] = json.loads(audit)
|
|
require(report['approval_evidence']['testsimulation'] is True and
|
|
report['approval_evidence']['approval_origin'] == 'dev_test_browser_simulation' and
|
|
report['approval_evidence']['plan_hash'] == plan['plan_hash'], 'approval simulation audit mismatch')
|
|
report.update(jobs_verified=True, postapproval_counts=json.loads(counts()), output_verified=True)
|
|
print('PASS actual non-root systemd VM Hello World via MCP pending request and explicit simulated UI click')
|
|
finally:
|
|
# Unknown/queued/in-flight jobs retain the service and owned tunnel.
|
|
# No effect is retried and no process is killed on an ambiguous result.
|
|
if idle():
|
|
if report['service_started']:
|
|
execute(connected + [target, 'systemctl stop rmm-agent.service'], env=env)
|
|
report['service_stopped'] = True
|
|
tunnel.terminate(); tunnel.wait(timeout=15)
|
|
report['tunnel_stopped'] = True
|
|
active.unlink(missing_ok=True)
|
|
else:
|
|
report['lifecycle_retained_for_active_or_unknown_jobs'] = True
|
|
store(active, report)
|
|
store(directory / 'result.json', report)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
run()
|
|
except Exception as error:
|
|
print('VM_JOB_CHECK_INCOMPLETE=' + json.dumps(bootstrap.sanitized_error(error)))
|
|
raise SystemExit(2) from None
|