#!/usr/bin/env python3 """Passive Cloud auto-WSS→HTTPS fallback with two actual mTLS WSS rejections. No RMM job, plan, approval or new identity. Only explicit, bounded dev/test transport fault configuration and preserving agent/service reconnects exist. """ import argparse import datetime as dt import hashlib import json import os from pathlib import Path import re import time import uuid import cloud_mvp_acceptance as cloud require = cloud.contract.require STATE = cloud.ROOT/'devstate/cloud-transport' def auto_mode(payload): import sqlite3 state = Path('/var/lib/rmm-agent-remote') require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed permanent identity') require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['agent_sha256'],'known qualified agent') for path,query in ((state/'journal.sqlite',"SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"), (Path('/var/lib/rmm-helper/consumption.sqlite'),'SELECT count(*) FROM consumed WHERE result IS NULL')): with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0,'all native effects must be settled') unit = Path('/etc/systemd/system/rmm-agent-remote.service') require(unit.read_text().splitlines().count('Environment=RMM_TRANSPORT=auto') == 1,'known automatic base service') require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'no environment-file transport override') directory = Path('/etc/systemd/system/rmm-agent-remote.service.d') selected = directory/'95-cloud-acceptance-transport.conf' for path in directory.glob('*.conf'): require(not path.is_symlink(),'transport drop-in symlink') require(path == selected or 'RMM_TRANSPORT=' not in path.read_text(),'unknown transport override') changed = selected.exists() if changed: require(selected.stat().st_uid == 0 and selected.read_text() in ('[Service]\nEnvironment=RMM_TRANSPORT=wss\n','[Service]\nEnvironment=RMM_TRANSPORT=https\n'),'known existing test override') backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-auto-transport-'+str(uuid.UUID(payload['id']))) backup.mkdir(mode=0o700) os.rename(selected,backup/'previous-transport.conf') subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10) restarted = changed or payload.get('restart') is True if restarted: subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True,timeout=20) return dict(mode='auto',preserved_identity=True,removed_test_override=changed,restarted=restarted) def configure_server(payload): root = Path('/opt/rust-discovery');compose = root/'deploy/server.compose.yaml' directory = root/'devstate/root-policy';path = directory/'transport-fault.json' require(not compose.is_symlink() and hashlib.sha256(compose.read_bytes()).hexdigest() == payload['compose_sha256'],'independently known compose required') require(not directory.is_symlink() and directory.is_dir() and directory.stat().st_uid == 10001 and directory.stat().st_mode & 0o777 == 0o700,'existing private read-only mount') require(not path.is_symlink(),'transport config symlink') old = path.read_bytes() if path.exists() else None actual_sha = hashlib.sha256(old).hexdigest() if old is not None else None expected = payload['expected_config_sha256'] # Cleanup may accept either of two PREVIOUSLY known hashes when an SSH # response was lost; it never adopts an arbitrary hash read from the host. require(actual_sha in expected if isinstance(expected,list) else actual_sha == expected,'known prior transport config') tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id'])) require(tenant == '11111111-1111-4111-8111-111111111111','fixed development tenant') cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(compose)] query = "SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND state NOT IN ('succeeded','failed','cancelled','expired');" require(subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'], input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip() == b'0','no open Cloud jobs') enabled = payload['enabled'];require(type(enabled) is bool,'explicit opt-in') backup = root/'devstate'/('transport-fault-'+str(uuid.UUID(payload['id']))) backup.mkdir(mode=0o700) def create(target,raw,uid=0): fd = os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) with os.fdopen(fd,'wb') as stream: stream.write(raw);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,uid) if old is not None:create(backup/'transport-fault-before.json',old) if enabled: value = payload['config'];require(value['tenant_id'] == tenant and value['agent_id'] == agent,'exact scope') raw = json.dumps(value,separators=(',',':')).encode() create(directory/'transport-fault.next',raw,10001);os.replace(directory/'transport-fault.next',path) else:raw = old overlay = backup/'transport.compose.yaml' # Literal overlay; no existing .env secret is read, returned or replaced. body = 'services:\n' for role in ('api','gateway'): body += ' '+role+':\n environment:\n RMM_DEV_TRANSPORT_FAULTS: "'+('true' if enabled else 'false')+'"\n' if enabled:body += ' RMM_DEV_TRANSPORT_FAULT_FILE: /root-policy/transport-fault.json\n' create(overlay,body.encode()) cmd += ['-f',str(overlay)] subprocess.run(cmd+['config','-q'],capture_output=True,check=True,timeout=15) completed = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],capture_output=True,timeout=25).returncode == 0 return dict(deployed=completed,enabled=enabled,backup=str(backup),config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None) def server_evidence(payload): root = Path('/opt/rust-discovery');tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']));fault = str(uuid.UUID(payload['fault_id'])) require(tenant == '11111111-1111-4111-8111-111111111111','fixed test tenant') query = ("SELECT json_build_object('receipts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data) ORDER BY created_at),'[]'::json) " "FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_transport_fault_adapter' AND data->>'fault_id'='"+fault+"')," "'device',(SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,'assignment_version',assignment_version," "'transport',transport,'connection_generation',connection_generation,'last_contact_at',last_contact_at,'last_authenticated_at',last_authenticated_at," "'certificate_sha256',last_authenticated_certificate_sha256,'sync_ready',sync_ready,'restore_blocked',restore_blocked) " "FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'));") cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), 'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'] return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout) def runtime_flags(payload): root = Path('/opt/rust-discovery') cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')] result = {} for role in ('api','gateway'): ident = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip() require(ident and '\n' not in ident,'one running role') values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',ident],capture_output=True,check=True,timeout=10).stdout) keys = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_TRANSPORT_FAULTS','RMM_DEV_TRANSPORT_FAULT_FILE') result[role] = {v.split('=',1)[0]:v.split('=',1)[1] for v in values if v.split('=',1)[0] in keys} return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--execute-dev-test-transport',action='store_true',required=True) parser.add_argument('--expected-agent-sha256',required=True) parser.add_argument('--expected-compose-sha256',required=True) parser.add_argument('--expected-config-sha256') args = parser.parse_args();os.umask(0o077) for value in (args.expected_agent_sha256,args.expected_compose_sha256):require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified pin') require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only') identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity'] scope = dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id']) def gate():return cloud.bootstrap.rpc('server.conf',cloud.server,dict(scope,kind='gate')) before = gate();require(before['unresolved'] == 0,'earlier effects must be closed') ident = str(uuid.uuid4());folder = STATE/ident;folder.mkdir(mode=0o700,parents=True) evidence_scope = dict(scope,fault_id=ident) auto = cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=ident,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,restart=True)) checkpoint = dict(profile='dev/test',testsimulation=True,case='actual_auto_fallback',id=ident,identity=identity,preapproval_counts=before,auto_mode=auto,no_jobs_created=True) path = folder/'checkpoint.json' def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600) def wait_transport(transport,after=None): end=time.monotonic()+150 while True: observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);device=observed['device'] if (device['transport'] == transport and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at'] and (after is None or cloud.contract.timestamp(device['last_contact_at']) > after)):return observed require(time.monotonic() < end,'fresh actual transport heartbeat and signed Sync missing');time.sleep(2) save();checkpoint['initial_wss']=wait_transport('wss');save() now=cloud.contract.now();config=dict(schema_version=1,fault_id=ident,**scope,issued_at=now.isoformat(),expires_at=(now+dt.timedelta(minutes=10)).isoformat()) raw=json.dumps(config,separators=(',',':')).encode();checkpoint['config']=config;save() enabled=None;complete=False;attempted=False try: attempted=True enabled=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=ident,enabled=True,**scope,compose_sha256=args.expected_compose_sha256, expected_config_sha256=args.expected_config_sha256,config=config)) checkpoint['enabled']=enabled;save();require(enabled['deployed'],'transport opt-in incomplete') end=time.monotonic()+150 while True: observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);receipts=observed['receipts'];device=observed['device'] if (len(receipts) == 2 and device['transport'] == 'https' and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at'] and cloud.contract.timestamp(device['last_contact_at']) > cloud.contract.timestamp(receipts[1]['created_at'])):break require(time.monotonic() < end,'two authenticated WSS503 and actual HTTPS heartbeat missing');time.sleep(2) require({r['data']['slot'] for r in receipts} == {1,2} and len({r['data']['handshake_id'] for r in receipts}) == 2 and all(r['data']['status'] == 503 and r['data']['agent_id'] == identity['agent_id'] and r['data']['tenant_id'] == cloud.TENANT and r['data']['config_sha256'] == hashlib.sha256(raw).hexdigest() and r['data']['certificate_sha256'] == device['certificate_sha256'] and r['data']['testsimulation'] is True for r in receipts),'actual exact peer fault receipts') require(all(device[k] == checkpoint['initial_wss']['device'][k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version')) and all(device[k] == identity[k] for k in ('agent_id','asset_id')),'permanent identity and actual initial versions unchanged') checkpoint['https']=observed;save();complete=True finally: if attempted: closed=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,**scope,compose_sha256=args.expected_compose_sha256, expected_config_sha256=[hashlib.sha256(raw).hexdigest(),args.expected_config_sha256])) checkpoint['disabled']=closed;save();require(closed['deployed'],'transport fault cleanup incomplete') flags=cloud.bootstrap.rpc('server.conf',runtime_flags,{}) require(len(flags) == 2 and all(v.get('RMM_DEV_TRANSPORT_FAULTS') in (None,'false') and 'RMM_DEV_TRANSPORT_FAULT_FILE' not in v for v in flags.values()),'both real roles disabled') checkpoint['disabled_runtime']=flags;save() # Auto chooses transport at connect time. An existing HTTPS actor # need not switch itself back after the WSS denial is removed. checkpoint['restoring_auto']=cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=str(uuid.uuid4()),agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,restart=True));save() checkpoint['restored_wss']=wait_transport('wss',cloud.contract.now());save() require(complete,'fallback not proven') after=gate();require(after == before,'passive fallback must leave job/outbox/grant counts unchanged') checkpoint.update(post_counts=after,verified=True,stage='native_dev_test_accepted');save() print('PASS passive Cloud auto WSS→HTTPS: two authenticated503, fresh HTTPS heartbeat+Sync, normal WSS restored, identity and job counts preserved') if __name__ == '__main__': try:main() except Exception as error: print('CLOUD_TRANSPORT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__)) raise SystemExit(2) from None