197 lines
14 KiB
Python
197 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Passive Cloud auto-WSS→HTTPS fallback with two actual mTLS WSS rejections.
|
|
|
|
No RMM job, plan, approval or new identity. Only explicit, bounded dev/test
|
|
transport fault configuration and preserving agent/service reconnects exist.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import uuid
|
|
|
|
import cloud_mvp_acceptance as cloud
|
|
|
|
require = cloud.contract.require
|
|
STATE = cloud.ROOT/'devstate/cloud-transport'
|
|
|
|
|
|
def auto_mode(payload):
|
|
import sqlite3
|
|
state = Path('/var/lib/rmm-agent-remote')
|
|
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed permanent identity')
|
|
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['agent_sha256'],'known qualified agent')
|
|
for path,query in ((state/'journal.sqlite',"SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
|
|
(Path('/var/lib/rmm-helper/consumption.sqlite'),'SELECT count(*) FROM consumed WHERE result IS NULL')):
|
|
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0,'all native effects must be settled')
|
|
unit = Path('/etc/systemd/system/rmm-agent-remote.service')
|
|
require(unit.read_text().splitlines().count('Environment=RMM_TRANSPORT=auto') == 1,'known automatic base service')
|
|
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'no environment-file transport override')
|
|
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
|
selected = directory/'95-cloud-acceptance-transport.conf'
|
|
for path in directory.glob('*.conf'):
|
|
require(not path.is_symlink(),'transport drop-in symlink')
|
|
require(path == selected or 'RMM_TRANSPORT=' not in path.read_text(),'unknown transport override')
|
|
changed = selected.exists()
|
|
if changed:
|
|
require(selected.stat().st_uid == 0 and selected.read_text() in
|
|
('[Service]\nEnvironment=RMM_TRANSPORT=wss\n','[Service]\nEnvironment=RMM_TRANSPORT=https\n'),'known existing test override')
|
|
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-auto-transport-'+str(uuid.UUID(payload['id'])))
|
|
backup.mkdir(mode=0o700)
|
|
os.rename(selected,backup/'previous-transport.conf')
|
|
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
|
|
restarted = changed or payload.get('restart') is True
|
|
if restarted:
|
|
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
|
|
return dict(mode='auto',preserved_identity=True,removed_test_override=changed,restarted=restarted)
|
|
|
|
|
|
def configure_server(payload):
|
|
root = Path('/opt/rust-discovery');compose = root/'deploy/server.compose.yaml'
|
|
directory = root/'devstate/root-policy';path = directory/'transport-fault.json'
|
|
require(not compose.is_symlink() and hashlib.sha256(compose.read_bytes()).hexdigest() == payload['compose_sha256'],'independently known compose required')
|
|
require(not directory.is_symlink() and directory.is_dir() and directory.stat().st_uid == 10001
|
|
and directory.stat().st_mode & 0o777 == 0o700,'existing private read-only mount')
|
|
require(not path.is_symlink(),'transport config symlink')
|
|
old = path.read_bytes() if path.exists() else None
|
|
actual_sha = hashlib.sha256(old).hexdigest() if old is not None else None
|
|
expected = payload['expected_config_sha256']
|
|
# Cleanup may accept either of two PREVIOUSLY known hashes when an SSH
|
|
# response was lost; it never adopts an arbitrary hash read from the host.
|
|
require(actual_sha in expected if isinstance(expected,list) else actual_sha == expected,'known prior transport config')
|
|
tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']))
|
|
require(tenant == '11111111-1111-4111-8111-111111111111','fixed development tenant')
|
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(compose)]
|
|
query = "SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND state NOT IN ('succeeded','failed','cancelled','expired');"
|
|
require(subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'],
|
|
input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip() == b'0','no open Cloud jobs')
|
|
enabled = payload['enabled'];require(type(enabled) is bool,'explicit opt-in')
|
|
backup = root/'devstate'/('transport-fault-'+str(uuid.UUID(payload['id'])))
|
|
backup.mkdir(mode=0o700)
|
|
def create(target,raw,uid=0):
|
|
fd = os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
with os.fdopen(fd,'wb') as stream:
|
|
stream.write(raw);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,uid)
|
|
if old is not None:create(backup/'transport-fault-before.json',old)
|
|
if enabled:
|
|
value = payload['config'];require(value['tenant_id'] == tenant and value['agent_id'] == agent,'exact scope')
|
|
raw = json.dumps(value,separators=(',',':')).encode()
|
|
create(directory/'transport-fault.next',raw,10001);os.replace(directory/'transport-fault.next',path)
|
|
else:raw = old
|
|
overlay = backup/'transport.compose.yaml'
|
|
# Literal overlay; no existing .env secret is read, returned or replaced.
|
|
body = 'services:\n'
|
|
for role in ('api','gateway'):
|
|
body += ' '+role+':\n environment:\n RMM_DEV_TRANSPORT_FAULTS: "'+('true' if enabled else 'false')+'"\n'
|
|
if enabled:body += ' RMM_DEV_TRANSPORT_FAULT_FILE: /root-policy/transport-fault.json\n'
|
|
create(overlay,body.encode())
|
|
cmd += ['-f',str(overlay)]
|
|
subprocess.run(cmd+['config','-q'],capture_output=True,check=True,timeout=15)
|
|
completed = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],capture_output=True,timeout=25).returncode == 0
|
|
return dict(deployed=completed,enabled=enabled,backup=str(backup),config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None)
|
|
|
|
|
|
def server_evidence(payload):
|
|
root = Path('/opt/rust-discovery');tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']));fault = str(uuid.UUID(payload['fault_id']))
|
|
require(tenant == '11111111-1111-4111-8111-111111111111','fixed test tenant')
|
|
query = ("SELECT json_build_object('receipts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data) ORDER BY created_at),'[]'::json) "
|
|
"FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_transport_fault_adapter' AND data->>'fault_id'='"+fault+"'),"
|
|
"'device',(SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,"
|
|
"'transport',transport,'connection_generation',connection_generation,'last_contact_at',last_contact_at,'last_authenticated_at',last_authenticated_at,"
|
|
"'certificate_sha256',last_authenticated_certificate_sha256,'sync_ready',sync_ready,'restore_blocked',restore_blocked) "
|
|
"FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'));")
|
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout)
|
|
|
|
|
|
def runtime_flags(payload):
|
|
root = Path('/opt/rust-discovery')
|
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
|
|
result = {}
|
|
for role in ('api','gateway'):
|
|
ident = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip()
|
|
require(ident and '\n' not in ident,'one running role')
|
|
values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',ident],capture_output=True,check=True,timeout=10).stdout)
|
|
keys = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_TRANSPORT_FAULTS','RMM_DEV_TRANSPORT_FAULT_FILE')
|
|
result[role] = {v.split('=',1)[0]:v.split('=',1)[1] for v in values if v.split('=',1)[0] in keys}
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--execute-dev-test-transport',action='store_true',required=True)
|
|
parser.add_argument('--expected-agent-sha256',required=True)
|
|
parser.add_argument('--expected-compose-sha256',required=True)
|
|
parser.add_argument('--expected-config-sha256')
|
|
args = parser.parse_args();os.umask(0o077)
|
|
for value in (args.expected_agent_sha256,args.expected_compose_sha256):require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified pin')
|
|
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
|
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
|
scope = dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'])
|
|
def gate():return cloud.bootstrap.rpc('server.conf',cloud.server,dict(scope,kind='gate'))
|
|
before = gate();require(before['unresolved'] == 0,'earlier effects must be closed')
|
|
ident = str(uuid.uuid4());folder = STATE/ident;folder.mkdir(mode=0o700,parents=True)
|
|
evidence_scope = dict(scope,fault_id=ident)
|
|
auto = cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=ident,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,restart=True))
|
|
checkpoint = dict(profile='dev/test',testsimulation=True,case='actual_auto_fallback',id=ident,identity=identity,preapproval_counts=before,auto_mode=auto,no_jobs_created=True)
|
|
path = folder/'checkpoint.json'
|
|
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
|
def wait_transport(transport,after=None):
|
|
end=time.monotonic()+150
|
|
while True:
|
|
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);device=observed['device']
|
|
if (device['transport'] == transport and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at']
|
|
and (after is None or cloud.contract.timestamp(device['last_contact_at']) > after)):return observed
|
|
require(time.monotonic() < end,'fresh actual transport heartbeat and signed Sync missing');time.sleep(2)
|
|
save();checkpoint['initial_wss']=wait_transport('wss');save()
|
|
now=cloud.contract.now();config=dict(schema_version=1,fault_id=ident,**scope,issued_at=now.isoformat(),expires_at=(now+dt.timedelta(minutes=10)).isoformat())
|
|
raw=json.dumps(config,separators=(',',':')).encode();checkpoint['config']=config;save()
|
|
enabled=None;complete=False;attempted=False
|
|
try:
|
|
attempted=True
|
|
enabled=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=ident,enabled=True,**scope,compose_sha256=args.expected_compose_sha256,
|
|
expected_config_sha256=args.expected_config_sha256,config=config))
|
|
checkpoint['enabled']=enabled;save();require(enabled['deployed'],'transport opt-in incomplete')
|
|
end=time.monotonic()+150
|
|
while True:
|
|
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);receipts=observed['receipts'];device=observed['device']
|
|
if (len(receipts) == 2 and device['transport'] == 'https' and device['sync_ready'] and not device['restore_blocked']
|
|
and device['last_contact_at'] and cloud.contract.timestamp(device['last_contact_at']) > cloud.contract.timestamp(receipts[1]['created_at'])):break
|
|
require(time.monotonic() < end,'two authenticated WSS503 and actual HTTPS heartbeat missing');time.sleep(2)
|
|
require({r['data']['slot'] for r in receipts} == {1,2} and len({r['data']['handshake_id'] for r in receipts}) == 2
|
|
and all(r['data']['status'] == 503 and r['data']['agent_id'] == identity['agent_id'] and r['data']['tenant_id'] == cloud.TENANT
|
|
and r['data']['config_sha256'] == hashlib.sha256(raw).hexdigest() and r['data']['certificate_sha256'] == device['certificate_sha256']
|
|
and r['data']['testsimulation'] is True for r in receipts),'actual exact peer fault receipts')
|
|
require(all(device[k] == checkpoint['initial_wss']['device'][k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version'))
|
|
and all(device[k] == identity[k] for k in ('agent_id','asset_id')),'permanent identity and actual initial versions unchanged')
|
|
checkpoint['https']=observed;save();complete=True
|
|
finally:
|
|
if attempted:
|
|
closed=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,**scope,compose_sha256=args.expected_compose_sha256,
|
|
expected_config_sha256=[hashlib.sha256(raw).hexdigest(),args.expected_config_sha256]))
|
|
checkpoint['disabled']=closed;save();require(closed['deployed'],'transport fault cleanup incomplete')
|
|
flags=cloud.bootstrap.rpc('server.conf',runtime_flags,{})
|
|
require(len(flags) == 2 and all(v.get('RMM_DEV_TRANSPORT_FAULTS') in (None,'false') and 'RMM_DEV_TRANSPORT_FAULT_FILE' not in v for v in flags.values()),'both real roles disabled')
|
|
checkpoint['disabled_runtime']=flags;save()
|
|
# Auto chooses transport at connect time. An existing HTTPS actor
|
|
# need not switch itself back after the WSS denial is removed.
|
|
checkpoint['restoring_auto']=cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=str(uuid.uuid4()),agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,restart=True));save()
|
|
checkpoint['restored_wss']=wait_transport('wss',cloud.contract.now());save()
|
|
require(complete,'fallback not proven')
|
|
after=gate();require(after == before,'passive fallback must leave job/outbox/grant counts unchanged')
|
|
checkpoint.update(post_counts=after,verified=True,stage='native_dev_test_accepted');save()
|
|
print('PASS passive Cloud auto WSS→HTTPS: two authenticated503, fresh HTTPS heartbeat+Sync, normal WSS restored, identity and job counts preserved')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:main()
|
|
except Exception as error:
|
|
print('CLOUD_TRANSPORT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
|
raise SystemExit(2) from None
|