Files
rust-discovery/deploy/cloud_transport_acceptance.py
2026-11-13 19:46:40 +01:00

197 lines
14 KiB
Python

#!/usr/bin/env python3
"""Passive Cloud auto-WSS→HTTPS fallback with two actual mTLS WSS rejections.
No RMM job, plan, approval or new identity. Only explicit, bounded dev/test
transport fault configuration and preserving agent/service reconnects exist.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import time
import uuid
import cloud_mvp_acceptance as cloud
require = cloud.contract.require
STATE = cloud.ROOT/'devstate/cloud-transport'
def auto_mode(payload):
import sqlite3
state = Path('/var/lib/rmm-agent-remote')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed permanent identity')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['agent_sha256'],'known qualified agent')
for path,query in ((state/'journal.sqlite',"SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
(Path('/var/lib/rmm-helper/consumption.sqlite'),'SELECT count(*) FROM consumed WHERE result IS NULL')):
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0,'all native effects must be settled')
unit = Path('/etc/systemd/system/rmm-agent-remote.service')
require(unit.read_text().splitlines().count('Environment=RMM_TRANSPORT=auto') == 1,'known automatic base service')
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'no environment-file transport override')
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
selected = directory/'95-cloud-acceptance-transport.conf'
for path in directory.glob('*.conf'):
require(not path.is_symlink(),'transport drop-in symlink')
require(path == selected or 'RMM_TRANSPORT=' not in path.read_text(),'unknown transport override')
changed = selected.exists()
if changed:
require(selected.stat().st_uid == 0 and selected.read_text() in
('[Service]\nEnvironment=RMM_TRANSPORT=wss\n','[Service]\nEnvironment=RMM_TRANSPORT=https\n'),'known existing test override')
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-auto-transport-'+str(uuid.UUID(payload['id'])))
backup.mkdir(mode=0o700)
os.rename(selected,backup/'previous-transport.conf')
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
restarted = changed or payload.get('restart') is True
if restarted:
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
return dict(mode='auto',preserved_identity=True,removed_test_override=changed,restarted=restarted)
def configure_server(payload):
root = Path('/opt/rust-discovery');compose = root/'deploy/server.compose.yaml'
directory = root/'devstate/root-policy';path = directory/'transport-fault.json'
require(not compose.is_symlink() and hashlib.sha256(compose.read_bytes()).hexdigest() == payload['compose_sha256'],'independently known compose required')
require(not directory.is_symlink() and directory.is_dir() and directory.stat().st_uid == 10001
and directory.stat().st_mode & 0o777 == 0o700,'existing private read-only mount')
require(not path.is_symlink(),'transport config symlink')
old = path.read_bytes() if path.exists() else None
actual_sha = hashlib.sha256(old).hexdigest() if old is not None else None
expected = payload['expected_config_sha256']
# Cleanup may accept either of two PREVIOUSLY known hashes when an SSH
# response was lost; it never adopts an arbitrary hash read from the host.
require(actual_sha in expected if isinstance(expected,list) else actual_sha == expected,'known prior transport config')
tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']))
require(tenant == '11111111-1111-4111-8111-111111111111','fixed development tenant')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(compose)]
query = "SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND state NOT IN ('succeeded','failed','cancelled','expired');"
require(subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'],
input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip() == b'0','no open Cloud jobs')
enabled = payload['enabled'];require(type(enabled) is bool,'explicit opt-in')
backup = root/'devstate'/('transport-fault-'+str(uuid.UUID(payload['id'])))
backup.mkdir(mode=0o700)
def create(target,raw,uid=0):
fd = os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as stream:
stream.write(raw);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,uid)
if old is not None:create(backup/'transport-fault-before.json',old)
if enabled:
value = payload['config'];require(value['tenant_id'] == tenant and value['agent_id'] == agent,'exact scope')
raw = json.dumps(value,separators=(',',':')).encode()
create(directory/'transport-fault.next',raw,10001);os.replace(directory/'transport-fault.next',path)
else:raw = old
overlay = backup/'transport.compose.yaml'
# Literal overlay; no existing .env secret is read, returned or replaced.
body = 'services:\n'
for role in ('api','gateway'):
body += ' '+role+':\n environment:\n RMM_DEV_TRANSPORT_FAULTS: "'+('true' if enabled else 'false')+'"\n'
if enabled:body += ' RMM_DEV_TRANSPORT_FAULT_FILE: /root-policy/transport-fault.json\n'
create(overlay,body.encode())
cmd += ['-f',str(overlay)]
subprocess.run(cmd+['config','-q'],capture_output=True,check=True,timeout=15)
completed = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],capture_output=True,timeout=25).returncode == 0
return dict(deployed=completed,enabled=enabled,backup=str(backup),config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None)
def server_evidence(payload):
root = Path('/opt/rust-discovery');tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']));fault = str(uuid.UUID(payload['fault_id']))
require(tenant == '11111111-1111-4111-8111-111111111111','fixed test tenant')
query = ("SELECT json_build_object('receipts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data) ORDER BY created_at),'[]'::json) "
"FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_transport_fault_adapter' AND data->>'fault_id'='"+fault+"'),"
"'device',(SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,"
"'transport',transport,'connection_generation',connection_generation,'last_contact_at',last_contact_at,'last_authenticated_at',last_authenticated_at,"
"'certificate_sha256',last_authenticated_certificate_sha256,'sync_ready',sync_ready,'restore_blocked',restore_blocked) "
"FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'));")
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout)
def runtime_flags(payload):
root = Path('/opt/rust-discovery')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
result = {}
for role in ('api','gateway'):
ident = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip()
require(ident and '\n' not in ident,'one running role')
values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',ident],capture_output=True,check=True,timeout=10).stdout)
keys = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_TRANSPORT_FAULTS','RMM_DEV_TRANSPORT_FAULT_FILE')
result[role] = {v.split('=',1)[0]:v.split('=',1)[1] for v in values if v.split('=',1)[0] in keys}
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--execute-dev-test-transport',action='store_true',required=True)
parser.add_argument('--expected-agent-sha256',required=True)
parser.add_argument('--expected-compose-sha256',required=True)
parser.add_argument('--expected-config-sha256')
args = parser.parse_args();os.umask(0o077)
for value in (args.expected_agent_sha256,args.expected_compose_sha256):require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified pin')
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
scope = dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'])
def gate():return cloud.bootstrap.rpc('server.conf',cloud.server,dict(scope,kind='gate'))
before = gate();require(before['unresolved'] == 0,'earlier effects must be closed')
ident = str(uuid.uuid4());folder = STATE/ident;folder.mkdir(mode=0o700,parents=True)
evidence_scope = dict(scope,fault_id=ident)
auto = cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=ident,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,restart=True))
checkpoint = dict(profile='dev/test',testsimulation=True,case='actual_auto_fallback',id=ident,identity=identity,preapproval_counts=before,auto_mode=auto,no_jobs_created=True)
path = folder/'checkpoint.json'
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
def wait_transport(transport,after=None):
end=time.monotonic()+150
while True:
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);device=observed['device']
if (device['transport'] == transport and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at']
and (after is None or cloud.contract.timestamp(device['last_contact_at']) > after)):return observed
require(time.monotonic() < end,'fresh actual transport heartbeat and signed Sync missing');time.sleep(2)
save();checkpoint['initial_wss']=wait_transport('wss');save()
now=cloud.contract.now();config=dict(schema_version=1,fault_id=ident,**scope,issued_at=now.isoformat(),expires_at=(now+dt.timedelta(minutes=10)).isoformat())
raw=json.dumps(config,separators=(',',':')).encode();checkpoint['config']=config;save()
enabled=None;complete=False;attempted=False
try:
attempted=True
enabled=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=ident,enabled=True,**scope,compose_sha256=args.expected_compose_sha256,
expected_config_sha256=args.expected_config_sha256,config=config))
checkpoint['enabled']=enabled;save();require(enabled['deployed'],'transport opt-in incomplete')
end=time.monotonic()+150
while True:
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);receipts=observed['receipts'];device=observed['device']
if (len(receipts) == 2 and device['transport'] == 'https' and device['sync_ready'] and not device['restore_blocked']
and device['last_contact_at'] and cloud.contract.timestamp(device['last_contact_at']) > cloud.contract.timestamp(receipts[1]['created_at'])):break
require(time.monotonic() < end,'two authenticated WSS503 and actual HTTPS heartbeat missing');time.sleep(2)
require({r['data']['slot'] for r in receipts} == {1,2} and len({r['data']['handshake_id'] for r in receipts}) == 2
and all(r['data']['status'] == 503 and r['data']['agent_id'] == identity['agent_id'] and r['data']['tenant_id'] == cloud.TENANT
and r['data']['config_sha256'] == hashlib.sha256(raw).hexdigest() and r['data']['certificate_sha256'] == device['certificate_sha256']
and r['data']['testsimulation'] is True for r in receipts),'actual exact peer fault receipts')
require(all(device[k] == checkpoint['initial_wss']['device'][k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version'))
and all(device[k] == identity[k] for k in ('agent_id','asset_id')),'permanent identity and actual initial versions unchanged')
checkpoint['https']=observed;save();complete=True
finally:
if attempted:
closed=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,**scope,compose_sha256=args.expected_compose_sha256,
expected_config_sha256=[hashlib.sha256(raw).hexdigest(),args.expected_config_sha256]))
checkpoint['disabled']=closed;save();require(closed['deployed'],'transport fault cleanup incomplete')
flags=cloud.bootstrap.rpc('server.conf',runtime_flags,{})
require(len(flags) == 2 and all(v.get('RMM_DEV_TRANSPORT_FAULTS') in (None,'false') and 'RMM_DEV_TRANSPORT_FAULT_FILE' not in v for v in flags.values()),'both real roles disabled')
checkpoint['disabled_runtime']=flags;save()
# Auto chooses transport at connect time. An existing HTTPS actor
# need not switch itself back after the WSS denial is removed.
checkpoint['restoring_auto']=cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=str(uuid.uuid4()),agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,restart=True));save()
checkpoint['restored_wss']=wait_transport('wss',cloud.contract.now());save()
require(complete,'fallback not proven')
after=gate();require(after == before,'passive fallback must leave job/outbox/grant counts unchanged')
checkpoint.update(post_counts=after,verified=True,stage='native_dev_test_accepted');save()
print('PASS passive Cloud auto WSS→HTTPS: two authenticated503, fresh HTTPS heartbeat+Sync, normal WSS restored, identity and job counts preserved')
if __name__ == '__main__':
try:main()
except Exception as error:
print('CLOUD_TRANSPORT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
raise SystemExit(2) from None