feat: deploy cloud-only RMM and verify audited APT diagnostics
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cloud-only MCP → pending → actual dev/test WebUI → VM result acceptance.
|
||||
|
||||
No local executor/server, enrollment, direct approval API/SQL or retry exists.
|
||||
Credentials stay on the cloud server; the browser reads its protected login
|
||||
file. All SQL is fixed read-only evidence, scoped to the one cloud VM.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import remote_client_bootstrap as bootstrap
|
||||
import vm_mvp_acceptance as contract
|
||||
|
||||
ROOT = bootstrap.ROOT
|
||||
TENANT = bootstrap.TENANT
|
||||
require = bootstrap.require
|
||||
|
||||
|
||||
def prepare_browser():
|
||||
cache=ROOT/'web/node_modules/.cache/cloud-acceptance'
|
||||
nss=cache/'home/.pki/nssdb';nss.mkdir(mode=0o700,parents=True,exist_ok=True)
|
||||
for directory in (cache,cache/'home',cache/'home/.pki',nss):directory.chmod(0o700)
|
||||
ca=ROOT/'credentials/server-ca.pem';ssl.create_default_context(cafile=str(ca))
|
||||
tool=shutil.which('certutil') or str(ROOT/'web/node_modules/.cache/nss-tools/root/usr/bin/certutil')
|
||||
commands=[]
|
||||
if not (nss/'cert9.db').exists():commands.append([tool,'-N','-d','sql:'+str(nss),'--empty-password'])
|
||||
commands.extend(([tool,'-A','-d','sql:'+str(nss),'-n','RMM cloud acceptance CA','-t','C,,','-i',str(ca)],
|
||||
[tool,'-V','-d','sql:'+str(nss),'-n','RMM cloud acceptance CA','-u','L']))
|
||||
for cmd in commands:
|
||||
contract.require(subprocess.run(cmd,capture_output=True,timeout=15).returncode==0,'isolated_browser_trust_unavailable')
|
||||
|
||||
|
||||
def vm_transport(payload):
|
||||
import sqlite3
|
||||
mode=payload['mode'];require(mode in ('wss','https'),'transport_allowlist')
|
||||
state=Path('/var/lib/rmm-agent-remote')
|
||||
identity=json.loads((state/'identity.json').read_text())
|
||||
require(identity['agent_id']==payload['agent_id'],'fixed_cloud_vm_identity')
|
||||
binary=Path('/usr/local/libexec/rmm-agent-remote')
|
||||
require(hashlib.sha256(binary.read_bytes()).hexdigest()==payload['binary_sha256'],'fixed_artifact')
|
||||
with sqlite3.connect('file:'+str(state/'journal.sqlite')+'?mode=ro',uri=True) as db:
|
||||
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'unresolved_native_job')
|
||||
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'environment_override_conflict')
|
||||
directory=Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
||||
require(not directory.is_symlink(),'unsafe_dropin_directory')
|
||||
directory.mkdir(mode=0o755,exist_ok=True);directory.chmod(0o755)
|
||||
require(directory.stat().st_uid==0 and directory.stat().st_mode&0o022==0,'unsafe_dropin_owner')
|
||||
path=directory/'95-cloud-acceptance-transport.conf'
|
||||
allowed=['[Service]\nEnvironment=RMM_TRANSPORT='+m+'\n' for m in ('wss','https')]
|
||||
require(not path.is_symlink() and (not path.exists() or path.stat().st_uid==0 and path.read_text() in allowed),'unknown_transport_dropin')
|
||||
if mode=='wss':
|
||||
if path.exists():path.unlink()
|
||||
else:
|
||||
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_TRUNC|os.O_NOFOLLOW,0o644)
|
||||
with os.fdopen(fd,'w') as stream:stream.write(allowed[1]);stream.flush();os.fsync(stream.fileno())
|
||||
path.chmod(0o644)
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True)
|
||||
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True)
|
||||
return dict(requested=mode,only_cloud_vm=True)
|
||||
|
||||
|
||||
def server(payload):
|
||||
root = Path('/opt/rust-discovery')
|
||||
if payload['kind'] == 'http':
|
||||
require(payload['path'] in ('/mcp', '/auth/config'), 'no administrative mutation endpoint')
|
||||
class HTTPS(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1',18443),timeout=10), server_hostname=self.host)
|
||||
client = HTTPS('localhost',18443,context=ssl.create_default_context(cafile=str(root/'devstate/pki/ca.pem')))
|
||||
headers = {'Content-Type':'application/json', **payload.get('headers',{})}
|
||||
require(not any(k.lower() in ('authorization','cookie','x-csrf-token') for k in headers), 'caller credential injection')
|
||||
if payload['path'] == '/mcp':
|
||||
principal=payload.get('principal','a'); require(principal in ('a','b'),'fixed_test_principal')
|
||||
headers['Authorization'] = 'Bearer '+(root/('devstate/operator-'+principal+'.token')).read_text().strip()
|
||||
client.request(payload['method'],payload['path'],body=json.dumps(payload['body']) if payload.get('body') is not None else None,headers=headers)
|
||||
response = client.getresponse(); status = response.status; returned = dict(response.getheaders())
|
||||
if 'text/event-stream' in response.getheader('Content-Type',''):
|
||||
data = None
|
||||
for _ in range(128):
|
||||
line = response.readline(1024*1024)
|
||||
if not line: break
|
||||
if line.startswith(b'data: ') and line[6:].strip():
|
||||
item = json.loads(line[6:])
|
||||
if 'result' in item or 'error' in item: data=item; break
|
||||
else:
|
||||
raw=response.read(1024*1024); data=json.loads(raw) if raw else None
|
||||
client.close()
|
||||
return dict(status=status,headers=returned,data=data)
|
||||
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed test tenant required')
|
||||
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"'"
|
||||
if payload['kind'] == 'gate':
|
||||
job = str(uuid.UUID(payload['job_id'])) if payload.get('job_id') else None
|
||||
selected = scope + (" AND job_id='"+job+"'" if job else '')
|
||||
query = ("SELECT json_build_object('jobs',(SELECT count(*) FROM jobs WHERE "+selected+"),"
|
||||
"'outbox',(SELECT count(*) FROM outbox o JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE "+
|
||||
selected.replace('tenant_id=', 'j.tenant_id=').replace('agent_id=', 'j.agent_id=').replace('job_id=', 'j.job_id=')+"),"
|
||||
"'permits',(SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE "+
|
||||
selected.replace('tenant_id=', 'j.tenant_id=').replace('agent_id=', 'j.agent_id=').replace('job_id=', 'j.job_id=')+"),"
|
||||
"'unresolved',(SELECT count(*) FROM jobs WHERE "+scope+" AND state NOT IN ('succeeded','failed','cancelled','expired')));")
|
||||
else:
|
||||
require(payload['kind'] == 'audit', 'unknown evidence query')
|
||||
operation = str(uuid.UUID(payload['operation_id']))
|
||||
query = ("SELECT json_build_object('origin',approval_origin,'simulation',testsimulation,'principal',principal,'plan_hash',plan_hash) "
|
||||
"FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' "
|
||||
"AND EXISTS(SELECT 1 FROM jobs WHERE "+scope+" AND operation_id='"+operation+"');")
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
result=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
|
||||
return json.loads(result) if result else {}
|
||||
|
||||
|
||||
class Backend:
|
||||
def __init__(self,principal='a'):
|
||||
contract.require(principal in ('a','b'),'fixed_test_principal')
|
||||
self.principal=principal
|
||||
self.sequence=1
|
||||
self.headers={'Accept':'application/json, text/event-stream','MCP-Protocol-Version':'2025-11-25'}
|
||||
response=self.request(dict(jsonrpc='2.0',id=1,method='initialize',params=dict(protocolVersion='2025-11-25',capabilities={},clientInfo=dict(name='cloud-vm-mvp',version='1'))))
|
||||
contract.require(response['status']==200 and response['data']['result']['protocolVersion']=='2025-11-25','mcp_initialize')
|
||||
session=next(v for k,v in response['headers'].items() if k.lower()=='mcp-session-id')
|
||||
self.headers['Mcp-Session-Id']=session
|
||||
contract.require(self.request(dict(jsonrpc='2.0',method='notifications/initialized'))['status']==202,'mcp_initialized')
|
||||
|
||||
def request(self,body):
|
||||
return bootstrap.rpc('server.conf',server,dict(kind='http',path='/mcp',method='POST',headers=self.headers,body=body,principal=self.principal))
|
||||
|
||||
def call(self,name,args):
|
||||
contract.require(name in ('tenants.list','inventory.get_device','operations.plan','operations.submit','jobs.get','jobs.cancel'),'tool_allowlist')
|
||||
self.sequence+=1
|
||||
response=self.request(dict(jsonrpc='2.0',id=self.sequence,method='tools/call',params=dict(name=name,arguments=args)))
|
||||
data=response.get('data',{}) or {}
|
||||
contract.require(response['status']==200 and 'result' in data and not data['result'].get('isError'),'mcp_tool_failed')
|
||||
return data['result']['structuredContent']
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
||||
parser.add_argument('--case',choices=('hello','refresh_inventory','cancel_queued'),default='hello')
|
||||
parser.add_argument('--transport',choices=('wss','https'))
|
||||
args=parser.parse_args();os.umask(0o077)
|
||||
contract.require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook_must_not_have_client')
|
||||
prepare_browser()
|
||||
installed=contract.protected_json(ROOT/'devstate/remote-vm.json')
|
||||
identity=installed['identity']
|
||||
settings=dict(tenant_id=TENANT,**identity)
|
||||
def evidence(kind,**extra):
|
||||
return bootstrap.rpc('server.conf',server,dict(kind=kind,tenant_id=TENANT,agent_id=identity['agent_id'],**extra))
|
||||
contract.require(evidence('gate')['unresolved']==0,'unresolved_job_no_new_attempt')
|
||||
if args.transport:
|
||||
bootstrap.rpc('linux-vm.conf',vm_transport,dict(mode=args.transport,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256']))
|
||||
end=time.monotonic()+90
|
||||
while True:
|
||||
observed=bootstrap.rpc('server.conf',bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
if observed.get('transport')==args.transport and observed.get('sync_ready') and not observed.get('restore_blocked'):break
|
||||
contract.require(time.monotonic()<end,'native_transport_sync_not_ready');time.sleep(2)
|
||||
backend=Backend()
|
||||
device=backend.call('inventory.get_device',dict(tenant_id=TENANT,asset_id=identity['asset_id']))
|
||||
contract.require(device['agent_id']==identity['agent_id'] and device['presence']=='online'
|
||||
and 0 <= (contract.now()-contract.timestamp(device['last_contact_at'])).total_seconds()<90,'fresh_cloud_vm_required')
|
||||
observed=bootstrap.rpc('server.conf',bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
contract.require(observed['sync_ready'] and not observed['restore_blocked'],'signed_sync_gate_required')
|
||||
needed={'refresh_inventory' if args.case=='refresh_inventory' else 'run_script','rmm.sync.v1'}
|
||||
contract.require(needed<=set(device['capabilities']),'actual_vm_capability_required')
|
||||
request=contract.case_input(args.case,settings)
|
||||
plan=backend.call('operations.plan',dict(tenant_id=TENANT,input=request))
|
||||
contract.require(hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest()==plan['plan_hash'] and len(plan['jobs'])==1,'immutable_plan')
|
||||
job=plan['jobs'][0]
|
||||
folder=ROOT/'devstate/cloud-acceptance'/uuid.uuid4().hex;folder.mkdir(mode=0o700,parents=True)
|
||||
checkpoint=dict(profile='dev/test',case=args.case,expected_principal='simulated-testprincipal-a',**settings,
|
||||
plan=plan,job=job,stage='planned',no_retry=True,transport=observed['transport'],server_connection=observed)
|
||||
path=folder/'checkpoint.json'
|
||||
def save():
|
||||
path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
||||
save()
|
||||
pending=backend.call('operations.submit',dict(tenant_id=TENANT,plan_id=plan['plan_id'],idempotency_key='cloud-'+folder.name))
|
||||
gate=evidence('gate',job_id=job['job_id'])
|
||||
contract.require(pending['approval_state']=='pending_approval' and pending['job_count']==0 and pending['execution_state'] is None
|
||||
and all(gate[k]==0 for k in ('jobs','outbox','permits')),'pending_has_zero_execution_artifacts')
|
||||
checkpoint.update(operation_id=pending['operation_id'],preapproval=gate,stage='pending_approval');save()
|
||||
process=subprocess.run(['node',str(ROOT/'deploy/browser/cloud_approve.cjs'),str(path)],capture_output=True,timeout=120)
|
||||
contract.require(process.returncode==0,'browser_incomplete_preserve_checkpoint_never_retry')
|
||||
if args.case=='cancel_queued':
|
||||
backend.call('jobs.cancel',dict(tenant_id=TENANT,job_id=job['job_id'],reason='Explicit cloud dev/test cancellation before fixed future start'))
|
||||
deadline=time.monotonic()+105
|
||||
while True:
|
||||
result=backend.call('jobs.get',dict(tenant_id=TENANT,job_id=job['job_id']))
|
||||
if result['state'] in contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'):break
|
||||
contract.require(time.monotonic()<deadline,'result_timeout_no_retry');time.sleep(2)
|
||||
checkpoint.update(result=result,stage=result['state']);save()
|
||||
if args.case=='cancel_queued':
|
||||
contract.require(result['state']=='cancelled','confirmed_cancellation_required')
|
||||
else:
|
||||
contract.require(result['state']=='succeeded' and result['result']['evidence']['postconditions_passed'],'native_postconditions_required')
|
||||
if args.case=='hello':
|
||||
contract.require(result['result']['exit_code']==0 and bytes(result['result']['stdout'])==b'Hello World\n','exact_stdout_required')
|
||||
elif args.case=='refresh_inventory':
|
||||
contract.require(contract.timestamp(result['result']['evidence']['observed_at'])>=contract.timestamp(request['schedule']['not_before']),'fresh_inventory_required')
|
||||
gate=evidence('gate',job_id=job['job_id']);audit=evidence('audit',operation_id=pending['operation_id'])
|
||||
expected_counts=[1,1,0] if args.case=='cancel_queued' else [1,1,1]
|
||||
contract.require([gate[k] for k in ('jobs','outbox','permits')]==expected_counts,'exact_job_outbox_grant_counts')
|
||||
contract.require(audit==dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']),'interactive_simulation_audit')
|
||||
checkpoint.update(verified=True,postapproval=gate,audit=audit);save()
|
||||
print('PASS cloud VM '+args.case+': MCP pending 0/0/0, actual simulated UI approval, verified counts '+str(expected_counts))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_ACCEPTANCE_INCOMPLETE:'+ (str(error) if isinstance(error,contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
Reference in New Issue
Block a user