feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded real systemd-agent/MCP/browser Hello-World check; no root jobs."""
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import vm_job_bootstrap as bootstrap
|
||||
from vm_passive_test import execute, remote_python, require
|
||||
|
||||
ROOT = bootstrap.ROOT
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from backend_contract import request
|
||||
from local_integration import A, B, STATE, sql
|
||||
|
||||
|
||||
def store(path, value):
|
||||
with path.open('w') as stream:
|
||||
json.dump(value, stream, indent=2)
|
||||
path.chmod(0o600)
|
||||
|
||||
|
||||
def can_stop(states):
|
||||
return isinstance(states, list) and all(state in ('succeeded', 'failed', 'cancelled', 'expired') for state in states)
|
||||
|
||||
|
||||
def run():
|
||||
os.umask(0o077)
|
||||
directory = STATE / 'vm-jobs'
|
||||
installed = json.loads((directory / 'bootstrap.json').read_text())
|
||||
require(installed['migration_complete'] and installed['byte_and_sqlite_preservation'],
|
||||
'controlled migration not proven')
|
||||
require(not (directory / 'active.json').exists(), 'previous active lifecycle requires review')
|
||||
agent = str(uuid.UUID(installed['agent_id']))
|
||||
asset = str(uuid.UUID(installed['asset_id']))
|
||||
operator = (STATE / 'operator-a.token').read_text().strip()
|
||||
viewer_b = (STATE / 'viewer-b.token').read_text().strip()
|
||||
base = f'/api/v1/tenants/{A}'
|
||||
report = dict(agent_id=agent, asset_id=asset, testsimulation=True, service_started=False,
|
||||
tunnel_started=False, jobs_verified=False)
|
||||
active = directory / 'active.json'
|
||||
def counts():
|
||||
return sql(f"SELECT json_build_array((SELECT count(*) FROM jobs WHERE agent_id='{agent}'),(SELECT count(*) FROM outbox o JOIN jobs j USING(tenant_id,job_id) WHERE j.agent_id='{agent}'),(SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id) WHERE j.agent_id='{agent}'));")
|
||||
def idle():
|
||||
states = json.loads(sql(f"SELECT coalesce(json_agg(state),'[]'::json) FROM jobs WHERE agent_id='{agent}';"))
|
||||
return can_stop(states)
|
||||
baseline = json.loads(counts())
|
||||
require(baseline == [0, 0, 0], 'existing VM job history requires separate continuation; no new run')
|
||||
with bootstrap.connection() as (ssh, target, env):
|
||||
# Refuse to compete with any existing reverse tunnel or native agent.
|
||||
for path in Path('/proc').glob('[0-9]*/cmdline'):
|
||||
try:
|
||||
args = path.read_bytes().split(b'\0')
|
||||
except OSError:
|
||||
continue
|
||||
require(not (args[0].split(b'/')[-1] == b'ssh' and
|
||||
any(b'18443:127.0.0.1:18443' in x for x in args)), 'existing reverse tunnel')
|
||||
preflight = '''import json,os,subprocess,pwd
|
||||
from pathlib import Path
|
||||
ports=subprocess.run(['ss','-H','-ltn'],capture_output=True,text=True,check=True).stdout
|
||||
occupied=any(x.split()[3].rsplit(':',1)[-1]=='18443' for x in ports.splitlines())
|
||||
status=subprocess.run(['systemctl','is-active','rmm-agent.service'],capture_output=True,text=True).stdout.strip()
|
||||
account=pwd.getpwnam('rmm-agent')
|
||||
print(json.dumps(dict(port_occupied=occupied,already_active=status=='active',uid=account.pw_uid,
|
||||
state_uid=Path('/var/lib/rmm-agent').stat().st_uid,old_source_exists=Path(%r).exists())))
|
||||
''' % (json.loads((STATE / 'vm-passive/result.json').read_text())['remote_directory'] + '/state')
|
||||
facts = json.loads(remote_python(ssh, target, env, preflight))
|
||||
require(not facts['port_occupied'] and not facts['already_active'] and
|
||||
facts['uid'] == installed['agent_uid'] and facts['state_uid'] == facts['uid']
|
||||
and facts['uid'] > 0 and not facts['old_source_exists'], 'VM service readiness failed')
|
||||
control = str(directory / 'ssh-control')
|
||||
require(not Path(control).exists(), 'prior SSH control socket requires review')
|
||||
with (directory / 'tunnel.log').open('wb') as log:
|
||||
tunnel = subprocess.Popen(ssh + ['-M', '-S', control, '-N', '-T',
|
||||
'-o', 'ExitOnForwardFailure=yes', '-o', 'ServerAliveInterval=30',
|
||||
'-o', 'ServerAliveCountMax=3', '-R', '127.0.0.1:18443:127.0.0.1:18443', target],
|
||||
env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
||||
report.update(tunnel_started=True, tunnel_pid=tunnel.pid)
|
||||
store(active, report)
|
||||
connected = ssh + ['-S', control, '-o', 'BatchMode=yes']
|
||||
try:
|
||||
deadline = time.monotonic() + 15
|
||||
while not Path(control).exists():
|
||||
require(tunnel.poll() is None and time.monotonic() < deadline, 'strict reverse tunnel setup failed')
|
||||
time.sleep(.1)
|
||||
listeners = execute(connected + [target, "ss -H -ltn 'sport = :18443'"], env=env).decode()
|
||||
require([line.split()[3] for line in listeners.splitlines()] == ['127.0.0.1:18443'],
|
||||
'reverse listener not exclusively IPv4 loopback')
|
||||
execute(connected + [target, 'systemctl start rmm-agent.service'], env=env)
|
||||
report['service_started'] = True
|
||||
store(active, report)
|
||||
contacts = set()
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
status, _, device = request(base + '/devices/' + asset, token=operator)
|
||||
require(status == 200 and device['agent_id'] == agent, 'preserved server identity mismatch')
|
||||
if device['last_contact_at']:
|
||||
moment = datetime.datetime.fromisoformat(device['last_contact_at'].replace('Z', '+00:00'))
|
||||
if (datetime.datetime.now(datetime.timezone.utc) - moment).total_seconds() < 90:
|
||||
contacts.add(device['last_contact_at'])
|
||||
if len(contacts) >= 2 and 'run_script' in device['capabilities']:
|
||||
break
|
||||
time.sleep(2)
|
||||
require(len(contacts) >= 2 and 'run_script' in device['capabilities'], 'fresh native agent heartbeats/capability missing')
|
||||
report['heartbeat_times'] = sorted(contacts)
|
||||
code = '''import json,subprocess
|
||||
from pathlib import Path
|
||||
pid=int(subprocess.run(['systemctl','show','--property=MainPID','--value','rmm-agent.service'],capture_output=True,text=True,check=True).stdout)
|
||||
uid=Path('/proc/'+str(pid)).stat().st_uid
|
||||
logs=subprocess.run(['journalctl','-u','rmm-agent.service','--since','5 minutes ago','-o','cat','--no-pager'],capture_output=True,text=True,check=True).stdout
|
||||
print(json.dumps(dict(agent_pid=pid,agent_uid=uid,sync_blocked='restore/reconnect sync unavailable' in logs)))
|
||||
'''
|
||||
runtime = json.loads(remote_python(connected, target, env, code))
|
||||
report.update(runtime)
|
||||
require(runtime['agent_uid'] == installed['agent_uid'] and not runtime['sync_blocked'],
|
||||
'non-root runtime or reconnect synchronization gate blocked')
|
||||
require(request(f'/api/v1/tenants/{B}/devices/{asset}', token=viewer_b)[0] == 404,
|
||||
'cross-tenant device exposure')
|
||||
headers = {'Accept': 'application/json, text/event-stream', 'MCP-Protocol-Version': '2025-11-25'}
|
||||
init = {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {
|
||||
'protocolVersion': '2025-11-25', 'capabilities': {}, 'clientInfo': {'name': 'real-vm-hello', 'version': '1'}}}
|
||||
status, response_headers, response = request('/mcp', 'POST', init, operator, headers)
|
||||
require(status == 200 and response['result']['protocolVersion'] == '2025-11-25', 'MCP initialize failed')
|
||||
session = response_headers.get('mcp-session-id', response_headers.get('Mcp-Session-Id'))
|
||||
if session:
|
||||
headers['Mcp-Session-Id'] = session
|
||||
require(request('/mcp', 'POST', {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, operator, headers)[0] == 202,
|
||||
'MCP session initialization failed')
|
||||
def call(name, arguments, identifier):
|
||||
status, _, response = request('/mcp', 'POST', {'jsonrpc': '2.0', 'id': identifier,
|
||||
'method': 'tools/call', 'params': {'name': name, 'arguments': arguments}}, operator, headers)
|
||||
require(status == 200 and 'result' in response and not response['result'].get('isError'), 'MCP tool failed')
|
||||
return response['result']['structuredContent']
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
plan_input = dict(tenant_id=A, action='run_script', targets={'asset_ids': [asset]},
|
||||
parameters={'variants': [{'os_family': 'linux', 'interpreter': 'posix_sh', 'encoding': 'utf-8',
|
||||
'script_utf8': "printf '%s\\n' 'Hello World'\n"}], 'arguments': [], 'execution_identity': 'agent_service',
|
||||
'timeout_seconds': 60, 'max_output_bytes': 65536, 'expected_stdout': 'Hello World\n'},
|
||||
schedule={'not_before': now.isoformat(), 'start_before': (now + datetime.timedelta(minutes=15)).isoformat(),
|
||||
'offline_policy': 'wait_until_deadline'}, rollout={'max_parallel': 1, 'halt_after_failures': 1},
|
||||
reason='Real Linux VM · non-root systemd agent · simulated dev/test UI approval')
|
||||
plan = call('operations.plan', {'tenant_id': A, 'input': plan_input}, 2)
|
||||
require(hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash']
|
||||
and len(plan['jobs']) == 1 and plan['jobs'][0]['agent_id'] == agent, 'immutable plan binding mismatch')
|
||||
report.update(plan_id=plan['plan_id'], plan_hash=plan['plan_hash'], job_id=plan['jobs'][0]['job_id'])
|
||||
store(active, report)
|
||||
pending = call('operations.submit', {'tenant_id': A, 'plan_id': plan['plan_id'],
|
||||
'idempotency_key': 'vm-hello-' + uuid.uuid4().hex}, 3)
|
||||
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0
|
||||
and pending['execution_state'] is None and json.loads(counts()) == baseline,
|
||||
'preapproval executable-artifact invariant failed')
|
||||
operation = str(uuid.UUID(pending['operation_id']))
|
||||
report.update(operation_id=operation, preapproval_counts=baseline)
|
||||
store(active, report)
|
||||
browser = subprocess.run([sys.executable, str(ROOT / 'deploy/browser/run_vm_jobs.py'),
|
||||
'--operation-id', operation, '--asset-id', asset, '--dev-test-approve'], timeout=150)
|
||||
report['browser_exit_code'] = browser.returncode
|
||||
current = request(base + '/operations/' + operation, token=operator)[2]
|
||||
report['final_operation'] = current
|
||||
require(browser.returncode == 0 and current['approval_state'] == 'approved' and
|
||||
current['job_count'] == 1 and current['execution_state'] == 'succeeded',
|
||||
'browser or VM result not confirmed; never retry effect')
|
||||
job = request(base + '/jobs/' + report['job_id'], token=operator)[2]
|
||||
result = job['result']
|
||||
require(job['state'] == 'succeeded' and bytes(result['stdout']) == b'Hello World\n' and
|
||||
result['exit_code'] == 0 and result['evidence']['postconditions_passed'] is True,
|
||||
'exact output/postconditions not confirmed')
|
||||
audit = sql(f"SELECT json_build_object('approval_origin',approval_origin,'testsimulation',testsimulation,'principal',principal,'plan_hash',plan_hash) FROM authorizations WHERE tenant_id='{A}' AND operation_id='{operation}';")
|
||||
report['approval_evidence'] = json.loads(audit)
|
||||
require(report['approval_evidence']['testsimulation'] is True and
|
||||
report['approval_evidence']['approval_origin'] == 'dev_test_browser_simulation' and
|
||||
report['approval_evidence']['plan_hash'] == plan['plan_hash'], 'approval simulation audit mismatch')
|
||||
report.update(jobs_verified=True, postapproval_counts=json.loads(counts()), output_verified=True)
|
||||
print('PASS actual non-root systemd VM Hello World via MCP pending request and explicit simulated UI click')
|
||||
finally:
|
||||
# Unknown/queued/in-flight jobs retain the service and owned tunnel.
|
||||
# No effect is retried and no process is killed on an ambiguous result.
|
||||
if idle():
|
||||
if report['service_started']:
|
||||
execute(connected + [target, 'systemctl stop rmm-agent.service'], env=env)
|
||||
report['service_stopped'] = True
|
||||
tunnel.terminate(); tunnel.wait(timeout=15)
|
||||
report['tunnel_stopped'] = True
|
||||
active.unlink(missing_ok=True)
|
||||
else:
|
||||
report['lifecycle_retained_for_active_or_unknown_jobs'] = True
|
||||
store(active, report)
|
||||
store(directory / 'result.json', report)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
run()
|
||||
except Exception as error:
|
||||
print('VM_JOB_CHECK_INCOMPLETE=' + json.dumps(bootstrap.sanitized_error(error)))
|
||||
raise SystemExit(2) from None
|
||||
Reference in New Issue
Block a user