Files
rust-discovery/deploy/cloud_mvp_acceptance.py
2026-11-13 17:56:24 +01:00

223 lines
15 KiB
Python

#!/usr/bin/env python3
"""Cloud-only MCP → pending → actual dev/test WebUI → VM result acceptance.
No local executor/server, enrollment, direct approval API/SQL or retry exists.
Credentials stay on the cloud server; the browser reads its protected login
file. All SQL is fixed read-only evidence, scoped to the one cloud VM.
"""
import argparse
import hashlib
import http.client
import json
import os
from pathlib import Path
import shutil
import subprocess
import socket
import ssl
import time
import uuid
import remote_client_bootstrap as bootstrap
import vm_mvp_acceptance as contract
ROOT = bootstrap.ROOT
TENANT = bootstrap.TENANT
require = bootstrap.require
def prepare_browser():
cache=ROOT/'web/node_modules/.cache/cloud-acceptance'
nss=cache/'home/.pki/nssdb';nss.mkdir(mode=0o700,parents=True,exist_ok=True)
for directory in (cache,cache/'home',cache/'home/.pki',nss):directory.chmod(0o700)
ca=ROOT/'credentials/server-ca.pem';ssl.create_default_context(cafile=str(ca))
tool=shutil.which('certutil') or str(ROOT/'web/node_modules/.cache/nss-tools/root/usr/bin/certutil')
commands=[]
if not (nss/'cert9.db').exists():commands.append([tool,'-N','-d','sql:'+str(nss),'--empty-password'])
commands.extend(([tool,'-A','-d','sql:'+str(nss),'-n','RMM cloud acceptance CA','-t','C,,','-i',str(ca)],
[tool,'-V','-d','sql:'+str(nss),'-n','RMM cloud acceptance CA','-u','L']))
for cmd in commands:
contract.require(subprocess.run(cmd,capture_output=True,timeout=15).returncode==0,'isolated_browser_trust_unavailable')
def vm_transport(payload):
import sqlite3
mode=payload['mode'];require(mode in ('wss','https'),'transport_allowlist')
state=Path('/var/lib/rmm-agent-remote')
identity=json.loads((state/'identity.json').read_text())
require(identity['agent_id']==payload['agent_id'],'fixed_cloud_vm_identity')
binary=Path('/usr/local/libexec/rmm-agent-remote')
require(hashlib.sha256(binary.read_bytes()).hexdigest()==payload['binary_sha256'],'fixed_artifact')
with sqlite3.connect('file:'+str(state/'journal.sqlite')+'?mode=ro',uri=True) as db:
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'unresolved_native_job')
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'environment_override_conflict')
directory=Path('/etc/systemd/system/rmm-agent-remote.service.d')
require(not directory.is_symlink(),'unsafe_dropin_directory')
directory.mkdir(mode=0o755,exist_ok=True);directory.chmod(0o755)
require(directory.stat().st_uid==0 and directory.stat().st_mode&0o022==0,'unsafe_dropin_owner')
path=directory/'95-cloud-acceptance-transport.conf'
allowed=['[Service]\nEnvironment=RMM_TRANSPORT='+m+'\n' for m in ('wss','https')]
require(not path.is_symlink() and (not path.exists() or path.stat().st_uid==0 and path.read_text() in allowed),'unknown_transport_dropin')
if mode=='wss':
if path.exists():path.unlink()
else:
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_TRUNC|os.O_NOFOLLOW,0o644)
with os.fdopen(fd,'w') as stream:stream.write(allowed[1]);stream.flush();os.fsync(stream.fileno())
path.chmod(0o644)
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True)
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True)
return dict(requested=mode,only_cloud_vm=True)
def server(payload):
root = Path('/opt/rust-discovery')
if payload['kind'] == 'http':
require(payload['path'] in ('/mcp', '/auth/config'), 'no administrative mutation endpoint')
class HTTPS(http.client.HTTPSConnection):
def connect(self):
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1',18443),timeout=10), server_hostname=self.host)
client = HTTPS('localhost',18443,context=ssl.create_default_context(cafile=str(root/'devstate/pki/ca.pem')))
headers = {'Content-Type':'application/json', **payload.get('headers',{})}
require(not any(k.lower() in ('authorization','cookie','x-csrf-token') for k in headers), 'caller credential injection')
if payload['path'] == '/mcp':
principal=payload.get('principal','a'); require(principal in ('a','b'),'fixed_test_principal')
headers['Authorization'] = 'Bearer '+(root/('devstate/operator-'+principal+'.token')).read_text().strip()
client.request(payload['method'],payload['path'],body=json.dumps(payload['body']) if payload.get('body') is not None else None,headers=headers)
response = client.getresponse(); status = response.status; returned = dict(response.getheaders())
if 'text/event-stream' in response.getheader('Content-Type',''):
data = None
for _ in range(128):
line = response.readline(1024*1024)
if not line: break
if line.startswith(b'data: ') and line[6:].strip():
item = json.loads(line[6:])
if 'result' in item or 'error' in item: data=item; break
else:
raw=response.read(1024*1024); data=json.loads(raw) if raw else None
client.close()
return dict(status=status,headers=returned,data=data)
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed test tenant required')
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"'"
if payload['kind'] == 'gate':
job = str(uuid.UUID(payload['job_id'])) if payload.get('job_id') else None
selected = scope + (" AND job_id='"+job+"'" if job else '')
query = ("SELECT json_build_object('jobs',(SELECT count(*) FROM jobs WHERE "+selected+"),"
"'outbox',(SELECT count(*) FROM outbox o JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE "+
selected.replace('tenant_id=', 'j.tenant_id=').replace('agent_id=', 'j.agent_id=').replace('job_id=', 'j.job_id=')+"),"
"'permits',(SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE "+
selected.replace('tenant_id=', 'j.tenant_id=').replace('agent_id=', 'j.agent_id=').replace('job_id=', 'j.job_id=')+"),"
"'unresolved',(SELECT count(*) FROM jobs WHERE "+scope+" AND state NOT IN ('succeeded','failed','cancelled','expired')));")
else:
require(payload['kind'] == 'audit', 'unknown evidence query')
operation = str(uuid.UUID(payload['operation_id']))
query = ("SELECT json_build_object('origin',approval_origin,'simulation',testsimulation,'principal',principal,'plan_hash',plan_hash) "
"FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' "
"AND EXISTS(SELECT 1 FROM jobs WHERE "+scope+" AND operation_id='"+operation+"');")
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
result=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
return json.loads(result) if result else {}
class Backend:
def __init__(self,principal='a'):
contract.require(principal in ('a','b'),'fixed_test_principal')
self.principal=principal
self.sequence=1
self.headers={'Accept':'application/json, text/event-stream','MCP-Protocol-Version':'2025-11-25'}
response=self.request(dict(jsonrpc='2.0',id=1,method='initialize',params=dict(protocolVersion='2025-11-25',capabilities={},clientInfo=dict(name='cloud-vm-mvp',version='1'))))
contract.require(response['status']==200 and response['data']['result']['protocolVersion']=='2025-11-25','mcp_initialize')
session=next(v for k,v in response['headers'].items() if k.lower()=='mcp-session-id')
self.headers['Mcp-Session-Id']=session
contract.require(self.request(dict(jsonrpc='2.0',method='notifications/initialized'))['status']==202,'mcp_initialized')
def request(self,body):
return bootstrap.rpc('server.conf',server,dict(kind='http',path='/mcp',method='POST',headers=self.headers,body=body,principal=self.principal))
def call(self,name,args):
contract.require(name in ('tenants.list','inventory.get_device','logs.query','operations.plan','operations.submit','jobs.get','jobs.cancel'),'tool_allowlist')
self.sequence+=1
response=self.request(dict(jsonrpc='2.0',id=self.sequence,method='tools/call',params=dict(name=name,arguments=args)))
data=response.get('data',{}) or {}
contract.require(response['status']==200 and 'result' in data and not data['result'].get('isError'),'mcp_tool_failed')
return data['result']['structuredContent']
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
parser.add_argument('--case',choices=('hello','refresh_inventory','cancel_queued'),default='hello')
parser.add_argument('--transport',choices=('wss','https'))
args=parser.parse_args();os.umask(0o077)
contract.require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook_must_not_have_client')
prepare_browser()
installed=contract.protected_json(ROOT/'devstate/remote-vm.json')
identity=installed['identity']
settings=dict(tenant_id=TENANT,**identity)
def evidence(kind,**extra):
return bootstrap.rpc('server.conf',server,dict(kind=kind,tenant_id=TENANT,agent_id=identity['agent_id'],**extra))
contract.require(evidence('gate')['unresolved']==0,'unresolved_job_no_new_attempt')
if args.transport:
bootstrap.rpc('linux-vm.conf',vm_transport,dict(mode=args.transport,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256']))
end=time.monotonic()+90
while True:
observed=bootstrap.rpc('server.conf',bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
if observed.get('transport')==args.transport and observed.get('sync_ready') and not observed.get('restore_blocked'):break
contract.require(time.monotonic()<end,'native_transport_sync_not_ready');time.sleep(2)
backend=Backend()
device=backend.call('inventory.get_device',dict(tenant_id=TENANT,asset_id=identity['asset_id']))
contract.require(device['agent_id']==identity['agent_id'] and device['presence']=='online'
and 0 <= (contract.now()-contract.timestamp(device['last_contact_at'])).total_seconds()<90,'fresh_cloud_vm_required')
observed=bootstrap.rpc('server.conf',bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
contract.require(observed['sync_ready'] and not observed['restore_blocked'],'signed_sync_gate_required')
needed={'refresh_inventory' if args.case=='refresh_inventory' else 'run_script','rmm.sync.v1'}
contract.require(needed<=set(device['capabilities']),'actual_vm_capability_required')
request=contract.case_input(args.case,settings)
plan=backend.call('operations.plan',dict(tenant_id=TENANT,input=request))
contract.require(hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest()==plan['plan_hash'] and len(plan['jobs'])==1,'immutable_plan')
job=plan['jobs'][0]
folder=ROOT/'devstate/cloud-acceptance'/uuid.uuid4().hex;folder.mkdir(mode=0o700,parents=True)
checkpoint=dict(profile='dev/test',case=args.case,expected_principal='simulated-testprincipal-a',**settings,
plan=plan,job=job,stage='planned',no_retry=True,transport=observed['transport'],server_connection=observed)
path=folder/'checkpoint.json'
def save():
path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
save()
pending=backend.call('operations.submit',dict(tenant_id=TENANT,plan_id=plan['plan_id'],idempotency_key='cloud-'+folder.name))
gate=evidence('gate',job_id=job['job_id'])
contract.require(pending['approval_state']=='pending_approval' and pending['job_count']==0 and pending['execution_state'] is None
and all(gate[k]==0 for k in ('jobs','outbox','permits')),'pending_has_zero_execution_artifacts')
checkpoint.update(operation_id=pending['operation_id'],preapproval=gate,stage='pending_approval');save()
process=subprocess.run(['node',str(ROOT/'deploy/browser/cloud_approve.cjs'),str(path)],capture_output=True,timeout=120)
contract.require(process.returncode==0,'browser_incomplete_preserve_checkpoint_never_retry')
if args.case=='cancel_queued':
backend.call('jobs.cancel',dict(tenant_id=TENANT,job_id=job['job_id'],reason='Explicit cloud dev/test cancellation before fixed future start'))
deadline=time.monotonic()+105
while True:
result=backend.call('jobs.get',dict(tenant_id=TENANT,job_id=job['job_id']))
if result['state'] in contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'):break
contract.require(time.monotonic()<deadline,'result_timeout_no_retry');time.sleep(2)
checkpoint.update(result=result,stage=result['state']);save()
if args.case=='cancel_queued':
contract.require(result['state']=='cancelled','confirmed_cancellation_required')
else:
contract.require(result['state']=='succeeded' and result['result']['evidence']['postconditions_passed'],'native_postconditions_required')
if args.case=='hello':
contract.require(result['result']['exit_code']==0 and bytes(result['result']['stdout'])==b'Hello World\n','exact_stdout_required')
elif args.case=='refresh_inventory':
contract.require(contract.timestamp(result['result']['evidence']['observed_at'])>=contract.timestamp(request['schedule']['not_before']),'fresh_inventory_required')
gate=evidence('gate',job_id=job['job_id']);audit=evidence('audit',operation_id=pending['operation_id'])
expected_counts=[1,1,0] if args.case=='cancel_queued' else [1,1,1]
contract.require([gate[k] for k in ('jobs','outbox','permits')]==expected_counts,'exact_job_outbox_grant_counts')
contract.require(audit==dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']),'interactive_simulation_audit')
checkpoint.update(verified=True,postapproval=gate,audit=audit);save()
print('PASS cloud VM '+args.case+': MCP pending 0/0/0, actual simulated UI approval, verified counts '+str(expected_counts))
if __name__=='__main__':
try:main()
except Exception as error:
print('CLOUD_ACCEPTANCE_INCOMPLETE:'+ (str(error) if isinstance(error,contract.GuardError) else type(error).__name__))
raise SystemExit(2) from None