#!/usr/bin/env python3 """One bounded Cloud agent SIGKILL; independent root helper closes its outcome. Requires fixed dev/test UI approval twice, a qualified agent and known closed root-policy hash. No API/DB approval, ledger edits, retry or unsigned recovery. Do not execute alongside another VM/runtime/approval operation. """ import argparse import datetime as dt import hashlib import json import os from pathlib import Path import re import subprocess import time import uuid import cloud_delivery_acceptance as delivery import cloud_apt_followup as prior import cloud_root_evidence import cloud_root_policy cloud = delivery.cloud require = cloud.contract.require STATE = cloud.ROOT/'devstate/cloud-crash' def script(identifier, read_only=False): identifier = str(uuid.UUID(identifier)) source = (cloud.ROOT/'deploy/linux/agent-crash-counter.py').read_text() return ("exec /usr/bin/python3 -I - "+('read' if read_only else 'effect')+' '+identifier+ " <<'RMM_AGENT_CRASH_COUNTER'\n"+source+'\nRMM_AGENT_CRASH_COUNTER\n') def vm(payload): """Read retained evidence; optionally signal only the exact bound MainPID.""" import signal import sqlite3 import stat state = Path('/var/lib/rmm-agent-remote') require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'permanent identity') for path,key in ((Path('/usr/local/libexec/rmm-agent-remote'),'agent_sha256'), (Path('/usr/local/libexec/rmm-privileged-helper'),'helper_sha256')): require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == payload[key],'qualified pinned artifact') def process(unit, expected_sha): raw = subprocess.run(['systemctl','show',unit,'-p','MainPID','--value'],capture_output=True,check=True,timeout=10).stdout pid = int(raw);require(pid > 1,'active pinned service') require(hashlib.sha256(Path('/proc/'+str(pid)+'/exe').read_bytes()).hexdigest() == expected_sha,'running service artifact') start = Path('/proc/'+str(pid)+'/stat').read_text().rsplit(')',1)[1].split()[19] return dict(pid=pid,start=start) processes = dict(agent=process('rmm-agent-remote.service',payload['agent_sha256']), helper=process('rmm-helper-cloud.service',payload['helper_sha256'])) result = dict(processes=processes) with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','native journal integrity') result['native_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] if payload.get('job_id'): job = str(uuid.UUID(payload['job_id']));attempt = str(uuid.UUID(payload['attempt_id'])) rows = db.execute('SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall() result['attempts'] = [dict(state=r[0],revision=r[1],grant_id=r[2]) for r in rows] result['agent_last_decision'] = db.execute('SELECT last_decision FROM reconcile_meta').fetchone()[0] helper = Path('/var/lib/rmm-helper/consumption.sqlite') with sqlite3.connect(helper.as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','root ledger integrity') result['ledger'] = dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0], unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0], faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]) if payload.get('job_id'): rows = db.execute('SELECT c.grant_id,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.job=? AND c.attempt=?',(job,attempt)).fetchall() result['root_attempts'] = [] for grant,finished,start,outcome in rows: stored = json.loads(start);request = stored['request'];permit = json.loads(bytes(request['permit_bytes'])) require(permit['job_id'] == job and permit['attempt_id'] == attempt and permit['agent_id'] == payload['agent_id'] and permit['grant_id'] == grant and hashlib.sha256(bytes(request['job_bytes'])).hexdigest() == payload['job_payload_sha256'] and request['plan_hash'] == payload['plan_sha256'],'exact durable root start binding') result['root_attempts'].append(dict(grant_id=grant,finished=json.loads(finished) if finished else None, outcome=json.loads(outcome) if outcome else None,start_sha256=hashlib.sha256(start).hexdigest())) head = db.execute('SELECT last_decision FROM helper_sync_meta').fetchone()[0] if head: row = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?',(head,)).fetchone() if row: signed = json.loads(row[0]);decision = json.loads(bytes(signed['decision_bytes'])) selected = [a for a in decision['attempts'] if a['job_id'] == job and a['attempt_id'] == attempt] result['signed_sync'] = dict(decision_id=head,ready=decision['ready'],helper_ready=decision['helper_ready'], payload_sha256=hashlib.sha256(bytes(signed['decision_bytes'])).hexdigest(),signature_sha256=hashlib.sha256(bytes(signed['signature'])).hexdigest(), selected=selected) if payload.get('job_id'): with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db: result['agent_consumed'] = [dict(grant_id=g,finished=json.loads(o) if o else None) for g,o in db.execute('SELECT grant_id,result FROM consumed WHERE job=? AND attempt=?',(job,attempt))] identifier = str(uuid.UUID(payload['id']));directory = helper.parent/('crash-'+identifier) require(not directory.is_symlink(),'counter symlink') result['counter'] = None if directory.exists(): require(directory.stat().st_uid == 0 and stat.S_IMODE(directory.stat().st_mode) == 0o700,'private root counter') path = directory/'count' if path.exists(): fd = os.open(path,os.O_RDONLY|os.O_NOFOLLOW) with os.fdopen(fd,'rb') as stream: metadata = os.fstat(stream.fileno());require(stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_nlink == 1 and stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16,'fixed root counter file') raw = stream.read();result['counter'] = int(raw) if payload['mode'] == 'kill': require(processes == payload['processes'],'exact original process instance') require(len(result['attempts']) == len(result['root_attempts']) == len(result['agent_consumed']) == 1 and result['attempts'][0]['state'] == 'running' and result['root_attempts'][0]['finished'] is None and result['root_attempts'][0]['outcome'] is None and result['counter'] == 1 and result['ledger'] == dict(consumed=payload['baseline_consumed']+1,unresolved=1,faults=0),'one actual running root effect; deterministic independent closure') require(result['attempts'][0]['grant_id'] == result['root_attempts'][0]['grant_id'] == result['agent_consumed'][0]['grant_id'],'one consumed grant on both sides') backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-agent-crash-'+identifier) backup.mkdir(mode=0o700) fd = os.open(backup/'before-signal.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) with os.fdopen(fd,'w') as stream:json.dump(result,stream);stream.flush();os.fsync(stream.fileno()) # pidfd prevents a PID-reuse race. No unit stop, cgroup/process-tree kill, # helper signal or package-manager intervention exists here. fd = os.pidfd_open(processes['agent']['pid']) try: require(process('rmm-agent-remote.service',payload['agent_sha256']) == processes['agent'],'same agent before signal') signal.pidfd_send_signal(fd,signal.SIGKILL) finally: os.close(fd) require(process('rmm-helper-cloud.service',payload['helper_sha256']) == processes['helper'],'independent helper survived') return dict(before=result,agent_signal='SIGKILL',helper_unchanged=True,backup=str(backup)) require(payload['mode'] == 'inspect','fixed fault operation') return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--execute-dev-test-ui',action='store_true',required=True) parser.add_argument('--expected-agent-sha256',required=True) parser.add_argument('--expected-helper-sha256',required=True) parser.add_argument('--expected-root-policy-sha256',required=True) args = parser.parse_args();os.umask(0o077) for value in (args.expected_agent_sha256,args.expected_helper_sha256,args.expected_root_policy_sha256): require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified artifact/config pin') require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only') cloud.prepare_browser() identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity'] settings = dict(tenant_id=cloud.TENANT,**identity) backend = cloud.Backend() def evidence(kind,**extra):return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind=kind,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**extra)) require(evidence('gate')['unresolved'] == 0,'previous central effects must be closed') identifier = str(uuid.uuid4()) scope = dict(id=identifier,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,helper_sha256=args.expected_helper_sha256) initial = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(scope,mode='inspect')) require(initial['native_unresolved'] == initial['ledger']['unresolved'] == initial['ledger']['faults'] == 0 and initial['counter'] is None,'native/history must be settled; untouched unique counter') observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id'])) require(observed['sync_ready'] and not observed['restore_blocked'],'signed sync before new root plan') peers = cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,dict(settings,general_only=True)) pin = peers['general'];binding = pin['identity'] root_scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')} root_scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['policy_sha256'], acceptance_evidence_id='cloud-agent-crash-independent-helper:'+identifier,allowed_services=[],allow_root_scripts=True,allow_reboot=False) policy = dict(schema_version=1,profile='development',expires_at=(cloud.contract.now()+dt.timedelta(minutes=15)).isoformat(),scopes=[root_scope]) folder = STATE/identifier;folder.mkdir(mode=0o700,parents=True) checkpoint = dict(profile='dev/test',case='agent_crash',expected_principal='simulated-testprincipal-a',**settings, id=identifier,initial=initial,qualified=scope,stage='prepared_no_job',no_retry=True) path = folder/'checkpoint.json' def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600) save() opened = cloud.bootstrap.rpc('server.conf',prior.narrow_policy,dict(id=identifier,expected_sha256=args.expected_root_policy_sha256,policy=policy)) checkpoint['policy_open'] = opened;save() def browser(value, name): file = folder/(name+'.json');cloud.contract.write_private(file,value) process = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_crash_acceptance.cjs'),str(file)],capture_output=True,timeout=120) require(process.returncode == 0,'actual UI incomplete; preserve pending plan; never resubmit/approve again') def authorize(read_only): # A busy helper may have temporarily removed root capabilities from the # new process's hello. Wait for genuine measured readvertisement/Sync; # never plan a new effect against a stale capability or readiness flag. deadline = time.monotonic()+100 while True: device = backend.call('inventory.get_device',dict(tenant_id=cloud.TENANT,asset_id=identity['asset_id'])) current = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id'])) if (device['agent_id'] == identity['agent_id'] and device['presence'] == 'online' and {'run_script','root:run_script','helper.root.v1','helper.policy.sha256:'+pin['policy_sha256']} <= set(device['capabilities']) and current['sync_ready'] and not current['restore_blocked']):break require(time.monotonic() < deadline,'fresh root capability and signed Sync before new plan');time.sleep(2) request = cloud.contract.case_input('service_setup',settings) request['parameters'].update(variants=[dict(os_family='linux',interpreter='posix_sh',encoding='utf-8',script_utf8=script(identifier,read_only))], expected_stdout='RMM_CRASH_COUNT=1\n',timeout_seconds=120,max_output_bytes=65536) request['reason'] = 'Explicit bounded Cloud agent crash '+('independent final counter read' if read_only else 'one root counter effect, helper remains alive for90s') common = dict(profile='dev/test',case='agent_crash',expected_principal=checkpoint['expected_principal'],**settings,id=identifier,read_only=read_only,request=request) name = 'postcheck' if read_only else 'effect' output = folder/(name+'-prepared.json') browser(dict(common,phase='prepare',output_file=str(output)),name+'-prepare') prepared = cloud.contract.protected_json(output);plan=prepared['plan'];pending=prepared['operation'] require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'],'one immutable fixed plan') job = plan['jobs'][0];gate = evidence('gate',job_id=job['job_id']) stored = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,operation_id=pending['operation_id'])) require(stored['approval_state'] == 'pending_approval' and stored['job_count'] == 0 and stored['execution_state'] is None and stored['plan_hash'] == plan['plan_hash'] and all(gate[k] == 0 for k in ('jobs','outbox','permits')),'pending zero artifacts') record = dict(plan=plan,job=job,operation_id=pending['operation_id'],preapproval=gate,request=request) checkpoint[name] = record;save() browser(dict(common,phase='approve',**record),name+'-approve') return record effect = authorize(False);job=effect['job'] fixed = dict(scope,job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],job_payload_sha256=job['job_payload_hash'],plan_sha256=effect['plan']['plan_hash']) deadline = time.monotonic()+90 while True: before = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect')) if before['counter'] == 1 and before['root_attempts'] and before['root_attempts'][0]['outcome'] is None:break require(time.monotonic() < deadline,'counter/running marker missing; never kill outside bound window');time.sleep(1) killed = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='kill',processes=initial['processes'],baseline_consumed=initial['ledger']['consumed'])) checkpoint['signal'] = killed;save() deadline = time.monotonic()+100 while True: result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])) if result['state'] == 'unknown': unknown = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect')) if unknown['root_attempts'][0]['outcome'] is None and unknown['attempts'][0]['state'] == 'unknown':break require(result['state'] not in cloud.contract.TERMINAL and time.monotonic() < deadline,'actual durable and central unknown not observed; preserve same attempt');time.sleep(1) require(result['result']['error_code'] == 'AGENT_PROCESS_INTERRUPTED' and unknown['counter'] == 1 and unknown['processes']['agent'] != initial['processes']['agent'] and unknown['processes']['helper'] == initial['processes']['helper'],'actual new agent process; independent live helper') checkpoint['unknown'] = dict(central=result,native=unknown);save() deadline = time.monotonic()+180 while True: final = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])) native = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect')) sync = native.get('signed_sync',{}) if (final['state'] in cloud.contract.TERMINAL and native['ledger']['unresolved'] == 0 and sync.get('ready') and sync.get('helper_ready') and native['agent_last_decision'] == sync.get('decision_id')):break require(time.monotonic() < deadline,'same-attempt definitive reconciliation missing; never unlock/retry');time.sleep(2) require(final['state'] == 'succeeded' and final['result']['exit_code'] == 0 and bytes(final['result']['stdout']) == b'RMM_CRASH_COUNT=1\n' and final['result']['evidence']['postconditions_passed'] and native['counter'] == 1 and len(native['root_attempts']) == len(native['agent_consumed']) == len(native['attempts']) == 1 and native['ledger'] == dict(consumed=initial['ledger']['consumed']+1,unresolved=0,faults=0),'one actual helper completion and consumption') selected = sync['selected'];grant = killed['before']['root_attempts'][0]['grant_id'] require(len(selected) == 1 and selected[0]['grant_id'] == grant and selected[0]['unlock'] is True and selected[0]['outcome_state'] == 'succeeded' and selected[0]['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED' and selected[0]['revision'] > unknown['attempts'][0]['revision'] and selected[0]['job_payload_hash'] == fixed['job_payload_sha256'] and selected[0]['plan_hash'] == fixed['plan_sha256'] and native['root_attempts'][0]['grant_id'] == native['agent_consumed'][0]['grant_id'] == grant,'exact existing signed Sync closure') checkpoint.update(final=final,native=native,stage='reconciled');save() def audit(record): record['postapproval'] = evidence('gate',job_id=record['job']['job_id']) record['audit'] = evidence('audit',operation_id=record['operation_id']);save() require([record['postapproval'][k] for k in ('jobs','outbox','permits')] == [1,1,1] and record['audit'] == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=record['plan']['plan_hash']),'one authorized immutable attempt') audit(effect) postcheck = authorize(True);deadline=time.monotonic()+90 while True: checked = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=postcheck['job']['job_id'])) if checked['state'] in cloud.contract.TERMINAL or checked['state'] == 'unknown':break require(time.monotonic() < deadline,'independent read outcome missing');time.sleep(2) postcheck['result'] = checked;save() require(checked['state'] == 'succeeded' and checked['result']['exit_code'] == 0 and checked['result']['evidence']['postconditions_passed'] and bytes(checked['result']['stdout']) == b'RMM_CRASH_COUNT=1\n','independent final counter still one') audit(postcheck) require(evidence('gate')['unresolved'] == 0,'all effects must be closed before policy close') closed = cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=opened['policy_sha256'],backup=opened['backup'])) checkpoint.update(policy_closed=closed,verified=True,stage='native_dev_test_accepted');save() print('PASS bounded Cloud agent crash: durable+central unknown, same-attempt root outcome and signed Sync, one effect, separate UI-approved counter1; dev/test only') if __name__ == '__main__': try:main() except Exception as error: print('CLOUD_CRASH_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__)) raise SystemExit(2) from None