#!/usr/bin/env python3 """Cloud wire-fault acceptance through real dev/test WebUI, never direct approval. Default invocation does nothing. Explicit execution requires the independently reviewed previous compose hash and zero unresolved jobs. A failed checkpoint is retained; this program has no resume/retry/requeue or identity-reset command. """ import argparse import datetime as dt import hashlib import json import os from pathlib import Path import re import subprocess import time import uuid import cloud_mvp_acceptance as cloud ROOT = cloud.ROOT STATE = ROOT / 'devstate/cloud-delivery' require = cloud.contract.require MODES = {'duplicate_offer': 'duplicate_offer_once', 'lost_result_ack': 'drop_result_ack_once'} def connect_backend(): deadline = time.monotonic()+45 while True: try: return cloud.Backend() except Exception: require(time.monotonic() < deadline,'ordinary MCP session unavailable after server restart') time.sleep(2) def configure_server(payload): """Root-private bootstrap config only; never creates/approves executable work.""" import datetime as dt import re root = Path('/opt/rust-discovery') compose = root/'deploy/server.compose.yaml' env = root/'devstate/.env' directory = root/'devstate/root-policy' config = directory/'delivery-fault.json' for path in (compose, env, directory): require(not path.is_symlink(), 'configuration symlink') require(compose.stat().st_uid == 0 and env.stat().st_uid == 0 and env.stat().st_mode & 0o777 == 0o600, 'root-owned private environment') require(directory.is_dir() and directory.stat().st_uid == 10001 and directory.stat().st_mode & 0o777 == 0o700, 'existing private root-policy mount') require(hashlib.sha256(compose.read_bytes()).hexdigest() == payload['expected_compose_sha256'], 'compose drift') require(not config.is_symlink(), 'fault config symlink') old_config = config.read_bytes() if config.exists() else None require((hashlib.sha256(old_config).hexdigest() if old_config is not None else None) == payload.get('expected_config_sha256'), 'fault config drift') tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id'])) require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed test tenant') cmd = ['docker','compose','--env-file',str(env),'-f',str(compose)] query = ("SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+ "' AND state NOT IN ('succeeded','failed','cancelled','expired');") count = subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'], input=query.encode(), capture_output=True, check=True, timeout=20).stdout.strip() require(count == b'0', 'unknown/unsettled work remains fenced') enabled = payload['enabled']; require(type(enabled) is bool, 'explicit boolean opt-in') if enabled: value = payload['config']; end = dt.datetime.fromisoformat(value['expires_at']) require(value['schema_version'] == 1 and value['tenant_id'] == tenant and value['agent_id'] == agent and value['mode'] in ('duplicate_offer_once','drop_result_ack_once') and value['transport'] in ('wss','https') and dt.datetime.now(dt.timezone.utc) < end <= dt.datetime.now(dt.timezone.utc)+dt.timedelta(minutes=15), 'bounded exact fault') raw = json.dumps(value, separators=(',',':')).encode() else: raw = old_config text = env.read_text() keys = ('RMM_DEV_DELIVERY_FAULTS','RMM_DEV_DELIVERY_FAULT_FILE') lines = text.splitlines() for key in keys: require(sum(bool(re.match(r'^\s*(?:export\s+)?'+key+r'\s*=', line)) for line in lines) <= 1, 'ambiguous fault environment') lines = [line for line in lines if not any(re.match(r'^\s*(?:export\s+)?'+key+r'\s*=', line) for key in keys)] lines.append('RMM_DEV_DELIVERY_FAULTS='+('true' if enabled else 'false')) if enabled: lines.append('RMM_DEV_DELIVERY_FAULT_FILE=/root-policy/delivery-fault.json') new_env = ('\n'.join(lines)+'\n').encode() compose_raw = payload['compose'].encode() require(hashlib.sha256(compose_raw).hexdigest() == payload['compose_sha256'], 'reviewed compose bytes') backup = root/'devstate'/('delivery-fault-config-'+str(uuid.UUID(payload['id']))) backup.mkdir(mode=0o700) def create(path, data, mode=0o600, uid=0): fd = os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode) with os.fdopen(fd,'wb') as stream: stream.write(data);stream.flush();os.fsync(stream.fileno()) os.fchown(stream.fileno(),uid,uid);os.fchmod(stream.fileno(),mode) create(backup/'server.compose.yaml',compose.read_bytes()) create(backup/'server.env',env.read_bytes()) # Stays root-private on server. if old_config is not None: create(backup/'delivery-fault-before.json',old_config) overlay = backup/'delivery.compose.yaml' create(overlay, b'services:\n api:\n environment:\n RMM_DEV_DELIVERY_FAULT_FILE: /root-policy/delivery-fault.json\n gateway:\n environment:\n RMM_DEV_DELIVERY_FAULT_FILE: /root-policy/delivery-fault.json\n') # Validate staged compose without printing expanded environment or secrets. staged_compose = compose.with_name('server.compose.delivery-next.yaml') staged_env = env.with_name('.env.delivery-next') create(staged_compose,compose_raw,0o644);create(staged_env,new_env) clean_env = {k:v for k,v in os.environ.items() if k not in keys} staged = ['docker','compose','--env-file',str(staged_env),'-f',str(staged_compose)] if enabled: staged += ['-f',str(overlay)] subprocess.run(staged+['config','-q'],env=clean_env,capture_output=True,check=True,timeout=30) if enabled: next_config = directory/'delivery-fault.next' create(next_config,raw,uid=10001);os.replace(next_config,config) os.replace(staged_env,env);os.replace(staged_compose,compose) cmd = ['docker','compose','--env-file',str(env),'-f',str(compose)] if enabled: cmd += ['-f',str(overlay)] result = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],env=clean_env,capture_output=True,timeout=60) return dict(deployed=result.returncode == 0, enabled=enabled, backup=str(backup), config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None, compose_sha256=hashlib.sha256(compose_raw).hexdigest(), approvals_created=False) def fault_evidence(payload): root = Path('/opt/rust-discovery') tenant = str(uuid.UUID(payload['tenant_id'])); fault = str(uuid.UUID(payload['fault_id'])) require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed test tenant') query = ("SELECT COALESCE(json_agg(json_build_object('type',event_type,'data',data,'created_at',created_at) ORDER BY created_at),'[]'::json) " "FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_delivery_fault_adapter' " "AND data->>'fault_id'='"+fault+"';") cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), 'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'] return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout) def runtime_fault_state(payload): """Return only the four public fault/profile environment fields.""" root = Path('/opt/rust-discovery') cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')] result = {} for role in ('api','gateway'): identifier = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip() require(identifier and '\n' not in identifier,'single running role required') values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',identifier],capture_output=True,check=True,timeout=10).stdout) allow = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_DELIVERY_FAULTS','RMM_DEV_DELIVERY_FAULT_FILE') result[role] = {value.split('=',1)[0]:value.split('=',1)[1] for value in values if value.split('=',1)[0] in allow} return result def stored_vm_evidence(payload): """Read only retained journal/consumption rows, never execute a collector.""" import sqlite3 state = Path('/var/lib/rmm-agent-remote') require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed native identity') job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id'])) with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','journal integrity') rows = db.execute('SELECT state,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall() unacked = db.execute('SELECT count(*) FROM job_events WHERE id=?',(str(uuid.UUID(payload['message_id'])),)).fetchone()[0] unresolved = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db: consumed = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(job,attempt)).fetchall() return dict(attempts=[dict(state=r[0],grant_id=r[1]) for r in rows],consumed=[r[0] for r in consumed], original_message_unacked=unacked,unresolved=unresolved) def native_gate(payload): import sqlite3 state = Path('/var/lib/rmm-agent-remote') require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed native identity') with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','journal integrity') return dict(unresolved=db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]) def counter_script(fault_id, read_only=False): identifier = str(uuid.UUID(fault_id)) prefix = ('set -eu\numask 077\nd=/var/lib/rmm-agent-remote/delivery-'+identifier+'\n') guards = ('test -d "$d"\ntest ! -L "$d"\ntest "$(stat -c \'%u:%a\' "$d")" = "$(id -u):700"\n') if read_only: return prefix+guards+'test -f "$d/count"\ntest ! -L "$d/count"\ntest "$(cat "$d/count")" = 1\nprintf \'%s\\n\' \'RMM_DELIVERY_COUNT=1\'\n' return (prefix+'if [ ! -e "$d" ]; then mkdir "$d"; fi\n'+guards+ 'mkdir "$d/lock"\ntrap \'rmdir "$d/lock"\' EXIT\nn=0\n'+ 'if [ -e "$d/count" ]; then test -f "$d/count"; test ! -L "$d/count"; n=$(cat "$d/count"); fi\n'+ 'case "$n" in [0-9]) ;; *) exit 1 ;; esac\nn=$((n+1))\n'+ 'printf \'%s\\n\' "$n" > "$d/count.next"\nmv "$d/count.next" "$d/count"\n'+ 'printf \'RMM_DELIVERY_COUNT=%s\\n\' "$n"\n') def verify_fault(records, config, config_sha): expected = {'claim','emitted','accepted_1','accepted_2'} if config['mode'] == 'duplicate_offer_once' else {'claim','replay'} slots = {r['type'].removeprefix('dev_delivery_fault.'):r['data'] for r in records} if set(slots) != expected or len(records) != len(expected): return False for value in slots.values(): require(all(value[k] == config[k] for k in ('fault_id','mode','transport','agent_id','job_id','attempt_id','plan_sha256','job_payload_sha256')) and value['config_sha256'] == config_sha and value['testsimulation'] is True and value['noProductionEvidence'] is True, 'exact durable fault receipt required') claim = slots['claim'] if config['mode'] == 'duplicate_offer_once': require(slots['emitted']['message_id'] == claim['message_id'] and slots['emitted']['normalized_envelope_sha256'] == claim['normalized_envelope_sha256'] and slots['accepted_1']['message_id'] != slots['accepted_2']['message_id'], 'actual duplicate and distinct client acceptances') else: require(slots['replay']['message_id'] == claim['message_id'] and slots['replay']['normalized_envelope_sha256'] == claim['normalized_envelope_sha256'], 'same original result replay required') return True def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--execute-dev-test-ui',action='store_true',required=True) parser.add_argument('--case',choices=tuple(MODES),required=True) parser.add_argument('--expected-compose-sha256',required=True) parser.add_argument('--expected-config-sha256') args = parser.parse_args();os.umask(0o077) require(re.fullmatch('[0-9a-f]{64}',args.expected_compose_sha256) is not None,'reviewed previous compose SHA required') require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook must remain development only') cloud.prepare_browser() identity = cloud.contract.protected_json(ROOT/'devstate/remote-vm.json')['identity'] backend = connect_backend() settings = dict(tenant_id=cloud.TENANT,**identity) def evidence(kind,**extra): return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind=kind,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**extra)) require(evidence('gate')['unresolved'] == 0,'unknown/unsettled work remains fenced') require(cloud.bootstrap.rpc('linux-vm.conf',native_gate,dict(agent_id=identity['agent_id']))['unresolved'] == 0,'native unresolved work remains fenced') observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id'])) require(observed['sync_ready'] and not observed['restore_blocked'] and observed['transport'] in ('wss','https'),'signed native sync gate') device = backend.call('inventory.get_device',dict(tenant_id=cloud.TENANT,asset_id=identity['asset_id'])) require(device['agent_id'] == identity['agent_id'] and device['presence'] == 'online' and 0 <= (cloud.contract.now()-cloud.contract.timestamp(device['last_contact_at'])).total_seconds() < 90 and {'run_script','rmm.sync.v1'} <= set(device['capabilities']),'fresh native cloud executor') fault_id = str(uuid.uuid4()) folder = STATE/fault_id;folder.mkdir(mode=0o700,parents=True) checkpoint = dict(profile='dev/test',case=args.case,expected_principal='simulated-testprincipal-a',**settings, fault_id=fault_id,transport=observed['transport'],no_retry=True,stage='planned') path = folder/'checkpoint.json' def save(): path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600) def execute_script(read_only): nonlocal backend request = cloud.contract.case_input('hello',settings) request['parameters']['variants'][0]['script_utf8'] = counter_script(fault_id,read_only) request['parameters']['expected_stdout'] = 'RMM_DELIVERY_COUNT=1\n' request['reason'] = 'Explicit dev/test cloud delivery '+args.case+(' independent counter read' if read_only else ' single effect') plan = backend.call('operations.plan',dict(tenant_id=cloud.TENANT,input=request)) require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'],'one immutable fixed job') job = plan['jobs'][0] pending = backend.call('operations.submit',dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='delivery-'+fault_id+('-read' if read_only else '-effect'))) gate = evidence('gate',job_id=job['job_id']) require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and all(gate[k] == 0 for k in ('jobs','outbox','permits')),'pending must have zero effects') record = dict(plan=plan,job=job,operation_id=pending['operation_id'],preapproval=gate,request=request,read_only=read_only) checkpoint['postcheck' if read_only else 'effect'] = record;save() if not read_only: now = cloud.contract.now() config = dict(schema_version=1,fault_id=fault_id,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'], attempt_id=job['payload']['attempt_id'],plan_sha256=plan['plan_hash'],job_payload_sha256=job['job_payload_hash'], mode=MODES[args.case],transport=observed['transport'],issued_at=now.isoformat(), expires_at=min(now+dt.timedelta(minutes=10),cloud.contract.timestamp(job['payload']['start_before'])).isoformat()) checkpoint['config'] = config;save() compose = (ROOT/'deploy/server.compose.yaml').read_text() deployed = cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=True,config=config, tenant_id=cloud.TENANT,agent_id=identity['agent_id'],expected_compose_sha256=args.expected_compose_sha256, expected_config_sha256=args.expected_config_sha256,compose=compose,compose_sha256=hashlib.sha256(compose.encode()).hexdigest())) checkpoint['enabled'] = deployed;save() require(deployed['deployed'],'fault configuration deployment incomplete; preserve checkpoint') # API restart invalidates its in-memory MCP sessions. Reinitialize # through the ordinary transport; never reuse an old bearer session. backend = connect_backend() ui = dict(checkpoint,**record) ui_path = folder/('postcheck-ui.json' if read_only else 'effect-ui.json') cloud.contract.write_private(ui_path,ui) result = subprocess.run(['node',str(ROOT/'deploy/browser/cloud_delivery_acceptance.cjs'),str(ui_path)],capture_output=True,timeout=120) require(result.returncode == 0,'interactive browser incomplete; never repeat approval') end = time.monotonic()+120 while True: result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])) record['result'] = result;save() if result['state'] in cloud.contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break require(time.monotonic() < end,'native result deadline; preserve unknown');time.sleep(2) require(result['state'] == 'succeeded' and result['result']['exit_code'] == 0 and result['result']['evidence']['postconditions_passed'] and bytes(result['result']['stdout']) == b'RMM_DELIVERY_COUNT=1\n', 'native counter postcondition') record['postapproval'] = evidence('gate',job_id=job['job_id']) record['audit'] = evidence('audit',operation_id=pending['operation_id']);save() require([record['postapproval'][k] for k in ('jobs','outbox','permits')] == [1,1,1] and record['audit'] == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']), 'one authorized job/outbox/grant') return job save() job = execute_script(False) end = time.monotonic()+60 while True: records = cloud.bootstrap.rpc('server.conf',fault_evidence,dict(tenant_id=cloud.TENANT,fault_id=fault_id)) checkpoint['fault_receipts'] = records;save() if verify_fault(records,checkpoint['config'],checkpoint['enabled']['config_sha256']): message = next(r['data']['message_id'] for r in records if r['type'] == 'dev_delivery_fault.claim') native = cloud.bootstrap.rpc('linux-vm.conf',stored_vm_evidence,dict(agent_id=identity['agent_id'],job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],message_id=message)) checkpoint['native'] = native;save() if native['original_message_unacked'] == 0 and native['unresolved'] == 0: break require(time.monotonic() < end,'actual delivery fault/replay acknowledgement missing; never retry');time.sleep(2) require(len(native['attempts']) == len(native['consumed']) == 1 and native['attempts'][0]['state'] == 'succeeded' and native['attempts'][0]['grant_id'] == native['consumed'][0], 'one durable native consumption') enabled = checkpoint['enabled'];compose = (ROOT/'deploy/server.compose.yaml').read_text() closed = cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,tenant_id=cloud.TENANT, agent_id=identity['agent_id'],expected_compose_sha256=enabled['compose_sha256'],expected_config_sha256=enabled['config_sha256'], compose=compose,compose_sha256=hashlib.sha256(compose.encode()).hexdigest())) checkpoint['disabled'] = closed;save() require(closed['deployed'] and not closed['enabled'],'fault opt-in not confirmed closed') checkpoint['disabled_runtime'] = cloud.bootstrap.rpc('server.conf',runtime_fault_state,{}) save() require(all(value == dict(RMM_PROFILE='development',RMM_DEV_AUTH='true',RMM_DEV_DELIVERY_FAULTS='false') for value in checkpoint['disabled_runtime'].values()),'both actual server roles must have fault opt-in disabled and FILE absent') backend = connect_backend() execute_script(True) checkpoint.update(verified=True,stage='native_dev_test_accepted');save() print('PASS cloud '+args.case+': actual wire fault, one attempt/grant/consumption, independent UI-approved counter=1; dev/test only') if __name__ == '__main__': try: main() except Exception as error: print('CLOUD_DELIVERY_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__)) raise SystemExit(2) from None