263 lines
20 KiB
Python
263 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""One bounded Cloud agent SIGKILL; independent root helper closes its outcome.
|
|
|
|
Requires fixed dev/test UI approval twice, a qualified agent and known closed
|
|
root-policy hash. No API/DB approval, ledger edits, retry or unsigned recovery.
|
|
Do not execute alongside another VM/runtime/approval operation.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
import cloud_delivery_acceptance as delivery
|
|
import cloud_apt_followup as prior
|
|
import cloud_root_evidence
|
|
import cloud_root_policy
|
|
|
|
cloud = delivery.cloud
|
|
require = cloud.contract.require
|
|
STATE = cloud.ROOT/'devstate/cloud-crash'
|
|
|
|
|
|
def script(identifier, read_only=False):
|
|
identifier = str(uuid.UUID(identifier))
|
|
source = (cloud.ROOT/'deploy/linux/agent-crash-counter.py').read_text()
|
|
return ("exec /usr/bin/python3 -I - "+('read' if read_only else 'effect')+' '+identifier+
|
|
" <<'RMM_AGENT_CRASH_COUNTER'\n"+source+'\nRMM_AGENT_CRASH_COUNTER\n')
|
|
|
|
|
|
def vm(payload):
|
|
"""Read retained evidence; optionally signal only the exact bound MainPID."""
|
|
import signal
|
|
import sqlite3
|
|
import stat
|
|
state = Path('/var/lib/rmm-agent-remote')
|
|
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'permanent identity')
|
|
for path,key in ((Path('/usr/local/libexec/rmm-agent-remote'),'agent_sha256'),
|
|
(Path('/usr/local/libexec/rmm-privileged-helper'),'helper_sha256')):
|
|
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == payload[key],'qualified pinned artifact')
|
|
def process(unit, expected_sha):
|
|
raw = subprocess.run(['systemctl','show',unit,'-p','MainPID','--value'],capture_output=True,check=True,timeout=10).stdout
|
|
pid = int(raw);require(pid > 1,'active pinned service')
|
|
require(hashlib.sha256(Path('/proc/'+str(pid)+'/exe').read_bytes()).hexdigest() == expected_sha,'running service artifact')
|
|
start = Path('/proc/'+str(pid)+'/stat').read_text().rsplit(')',1)[1].split()[19]
|
|
return dict(pid=pid,start=start)
|
|
processes = dict(agent=process('rmm-agent-remote.service',payload['agent_sha256']),
|
|
helper=process('rmm-helper-cloud.service',payload['helper_sha256']))
|
|
result = dict(processes=processes)
|
|
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','native journal integrity')
|
|
result['native_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
|
if payload.get('job_id'):
|
|
job = str(uuid.UUID(payload['job_id']));attempt = str(uuid.UUID(payload['attempt_id']))
|
|
rows = db.execute('SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
|
|
result['attempts'] = [dict(state=r[0],revision=r[1],grant_id=r[2]) for r in rows]
|
|
result['agent_last_decision'] = db.execute('SELECT last_decision FROM reconcile_meta').fetchone()[0]
|
|
helper = Path('/var/lib/rmm-helper/consumption.sqlite')
|
|
with sqlite3.connect(helper.as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','root ledger integrity')
|
|
result['ledger'] = dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
|
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
|
|
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0])
|
|
if payload.get('job_id'):
|
|
rows = db.execute('SELECT c.grant_id,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.job=? AND c.attempt=?',(job,attempt)).fetchall()
|
|
result['root_attempts'] = []
|
|
for grant,finished,start,outcome in rows:
|
|
stored = json.loads(start);request = stored['request'];permit = json.loads(bytes(request['permit_bytes']))
|
|
require(permit['job_id'] == job and permit['attempt_id'] == attempt and permit['agent_id'] == payload['agent_id']
|
|
and permit['grant_id'] == grant and hashlib.sha256(bytes(request['job_bytes'])).hexdigest() == payload['job_payload_sha256']
|
|
and request['plan_hash'] == payload['plan_sha256'],'exact durable root start binding')
|
|
result['root_attempts'].append(dict(grant_id=grant,finished=json.loads(finished) if finished else None,
|
|
outcome=json.loads(outcome) if outcome else None,start_sha256=hashlib.sha256(start).hexdigest()))
|
|
head = db.execute('SELECT last_decision FROM helper_sync_meta').fetchone()[0]
|
|
if head:
|
|
row = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?',(head,)).fetchone()
|
|
if row:
|
|
signed = json.loads(row[0]);decision = json.loads(bytes(signed['decision_bytes']))
|
|
selected = [a for a in decision['attempts'] if a['job_id'] == job and a['attempt_id'] == attempt]
|
|
result['signed_sync'] = dict(decision_id=head,ready=decision['ready'],helper_ready=decision['helper_ready'],
|
|
payload_sha256=hashlib.sha256(bytes(signed['decision_bytes'])).hexdigest(),signature_sha256=hashlib.sha256(bytes(signed['signature'])).hexdigest(),
|
|
selected=selected)
|
|
if payload.get('job_id'):
|
|
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
|
result['agent_consumed'] = [dict(grant_id=g,finished=json.loads(o) if o else None) for g,o in db.execute('SELECT grant_id,result FROM consumed WHERE job=? AND attempt=?',(job,attempt))]
|
|
identifier = str(uuid.UUID(payload['id']));directory = helper.parent/('crash-'+identifier)
|
|
require(not directory.is_symlink(),'counter symlink')
|
|
result['counter'] = None
|
|
if directory.exists():
|
|
require(directory.stat().st_uid == 0 and stat.S_IMODE(directory.stat().st_mode) == 0o700,'private root counter')
|
|
path = directory/'count'
|
|
if path.exists():
|
|
fd = os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
|
|
with os.fdopen(fd,'rb') as stream:
|
|
metadata = os.fstat(stream.fileno());require(stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0
|
|
and metadata.st_nlink == 1 and stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16,'fixed root counter file')
|
|
raw = stream.read();result['counter'] = int(raw)
|
|
if payload['mode'] == 'kill':
|
|
require(processes == payload['processes'],'exact original process instance')
|
|
require(len(result['attempts']) == len(result['root_attempts']) == len(result['agent_consumed']) == 1
|
|
and result['attempts'][0]['state'] == 'running' and result['root_attempts'][0]['finished'] is None
|
|
and result['root_attempts'][0]['outcome'] is None and result['counter'] == 1
|
|
and result['ledger'] == dict(consumed=payload['baseline_consumed']+1,unresolved=1,faults=0),'one actual running root effect; deterministic independent closure')
|
|
require(result['attempts'][0]['grant_id'] == result['root_attempts'][0]['grant_id'] == result['agent_consumed'][0]['grant_id'],'one consumed grant on both sides')
|
|
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-agent-crash-'+identifier)
|
|
backup.mkdir(mode=0o700)
|
|
fd = os.open(backup/'before-signal.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
with os.fdopen(fd,'w') as stream:json.dump(result,stream);stream.flush();os.fsync(stream.fileno())
|
|
# pidfd prevents a PID-reuse race. No unit stop, cgroup/process-tree kill,
|
|
# helper signal or package-manager intervention exists here.
|
|
fd = os.pidfd_open(processes['agent']['pid'])
|
|
try:
|
|
require(process('rmm-agent-remote.service',payload['agent_sha256']) == processes['agent'],'same agent before signal')
|
|
signal.pidfd_send_signal(fd,signal.SIGKILL)
|
|
finally: os.close(fd)
|
|
require(process('rmm-helper-cloud.service',payload['helper_sha256']) == processes['helper'],'independent helper survived')
|
|
return dict(before=result,agent_signal='SIGKILL',helper_unchanged=True,backup=str(backup))
|
|
require(payload['mode'] == 'inspect','fixed fault operation')
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
|
parser.add_argument('--expected-agent-sha256',required=True)
|
|
parser.add_argument('--expected-helper-sha256',required=True)
|
|
parser.add_argument('--expected-root-policy-sha256',required=True)
|
|
args = parser.parse_args();os.umask(0o077)
|
|
for value in (args.expected_agent_sha256,args.expected_helper_sha256,args.expected_root_policy_sha256):
|
|
require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified artifact/config pin')
|
|
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
|
cloud.prepare_browser()
|
|
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
|
settings = dict(tenant_id=cloud.TENANT,**identity)
|
|
backend = cloud.Backend()
|
|
def evidence(kind,**extra):return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind=kind,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**extra))
|
|
require(evidence('gate')['unresolved'] == 0,'previous central effects must be closed')
|
|
identifier = str(uuid.uuid4())
|
|
scope = dict(id=identifier,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,helper_sha256=args.expected_helper_sha256)
|
|
initial = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(scope,mode='inspect'))
|
|
require(initial['native_unresolved'] == initial['ledger']['unresolved'] == initial['ledger']['faults'] == 0
|
|
and initial['counter'] is None,'native/history must be settled; untouched unique counter')
|
|
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
|
require(observed['sync_ready'] and not observed['restore_blocked'],'signed sync before new root plan')
|
|
peers = cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,dict(settings,general_only=True))
|
|
pin = peers['general'];binding = pin['identity']
|
|
root_scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
|
|
root_scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['policy_sha256'],
|
|
acceptance_evidence_id='cloud-agent-crash-independent-helper:'+identifier,allowed_services=[],allow_root_scripts=True,allow_reboot=False)
|
|
policy = dict(schema_version=1,profile='development',expires_at=(cloud.contract.now()+dt.timedelta(minutes=15)).isoformat(),scopes=[root_scope])
|
|
folder = STATE/identifier;folder.mkdir(mode=0o700,parents=True)
|
|
checkpoint = dict(profile='dev/test',case='agent_crash',expected_principal='simulated-testprincipal-a',**settings,
|
|
id=identifier,initial=initial,qualified=scope,stage='prepared_no_job',no_retry=True)
|
|
path = folder/'checkpoint.json'
|
|
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
|
save()
|
|
opened = cloud.bootstrap.rpc('server.conf',prior.narrow_policy,dict(id=identifier,expected_sha256=args.expected_root_policy_sha256,policy=policy))
|
|
checkpoint['policy_open'] = opened;save()
|
|
def browser(value, name):
|
|
file = folder/(name+'.json');cloud.contract.write_private(file,value)
|
|
process = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_crash_acceptance.cjs'),str(file)],capture_output=True,timeout=120)
|
|
require(process.returncode == 0,'actual UI incomplete; preserve pending plan; never resubmit/approve again')
|
|
def authorize(read_only):
|
|
# A busy helper may have temporarily removed root capabilities from the
|
|
# new process's hello. Wait for genuine measured readvertisement/Sync;
|
|
# never plan a new effect against a stale capability or readiness flag.
|
|
deadline = time.monotonic()+100
|
|
while True:
|
|
device = backend.call('inventory.get_device',dict(tenant_id=cloud.TENANT,asset_id=identity['asset_id']))
|
|
current = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
|
if (device['agent_id'] == identity['agent_id'] and device['presence'] == 'online'
|
|
and {'run_script','root:run_script','helper.root.v1','helper.policy.sha256:'+pin['policy_sha256']} <= set(device['capabilities'])
|
|
and current['sync_ready'] and not current['restore_blocked']):break
|
|
require(time.monotonic() < deadline,'fresh root capability and signed Sync before new plan');time.sleep(2)
|
|
request = cloud.contract.case_input('service_setup',settings)
|
|
request['parameters'].update(variants=[dict(os_family='linux',interpreter='posix_sh',encoding='utf-8',script_utf8=script(identifier,read_only))],
|
|
expected_stdout='RMM_CRASH_COUNT=1\n',timeout_seconds=120,max_output_bytes=65536)
|
|
request['reason'] = 'Explicit bounded Cloud agent crash '+('independent final counter read' if read_only else 'one root counter effect, helper remains alive for90s')
|
|
common = dict(profile='dev/test',case='agent_crash',expected_principal=checkpoint['expected_principal'],**settings,id=identifier,read_only=read_only,request=request)
|
|
name = 'postcheck' if read_only else 'effect'
|
|
output = folder/(name+'-prepared.json')
|
|
browser(dict(common,phase='prepare',output_file=str(output)),name+'-prepare')
|
|
prepared = cloud.contract.protected_json(output);plan=prepared['plan'];pending=prepared['operation']
|
|
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'],'one immutable fixed plan')
|
|
job = plan['jobs'][0];gate = evidence('gate',job_id=job['job_id'])
|
|
stored = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,operation_id=pending['operation_id']))
|
|
require(stored['approval_state'] == 'pending_approval' and stored['job_count'] == 0 and stored['execution_state'] is None
|
|
and stored['plan_hash'] == plan['plan_hash'] and all(gate[k] == 0 for k in ('jobs','outbox','permits')),'pending zero artifacts')
|
|
record = dict(plan=plan,job=job,operation_id=pending['operation_id'],preapproval=gate,request=request)
|
|
checkpoint[name] = record;save()
|
|
browser(dict(common,phase='approve',**record),name+'-approve')
|
|
return record
|
|
effect = authorize(False);job=effect['job']
|
|
fixed = dict(scope,job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],job_payload_sha256=job['job_payload_hash'],plan_sha256=effect['plan']['plan_hash'])
|
|
deadline = time.monotonic()+90
|
|
while True:
|
|
before = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
|
if before['counter'] == 1 and before['root_attempts'] and before['root_attempts'][0]['outcome'] is None:break
|
|
require(time.monotonic() < deadline,'counter/running marker missing; never kill outside bound window');time.sleep(1)
|
|
killed = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='kill',processes=initial['processes'],baseline_consumed=initial['ledger']['consumed']))
|
|
checkpoint['signal'] = killed;save()
|
|
deadline = time.monotonic()+100
|
|
while True:
|
|
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
|
if result['state'] == 'unknown':
|
|
unknown = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
|
if unknown['root_attempts'][0]['outcome'] is None and unknown['attempts'][0]['state'] == 'unknown':break
|
|
require(result['state'] not in cloud.contract.TERMINAL and time.monotonic() < deadline,'actual durable and central unknown not observed; preserve same attempt');time.sleep(1)
|
|
require(result['result']['error_code'] == 'AGENT_PROCESS_INTERRUPTED' and unknown['counter'] == 1
|
|
and unknown['processes']['agent'] != initial['processes']['agent']
|
|
and unknown['processes']['helper'] == initial['processes']['helper'],'actual new agent process; independent live helper')
|
|
checkpoint['unknown'] = dict(central=result,native=unknown);save()
|
|
deadline = time.monotonic()+180
|
|
while True:
|
|
final = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
|
native = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
|
sync = native.get('signed_sync',{})
|
|
if (final['state'] in cloud.contract.TERMINAL and native['ledger']['unresolved'] == 0 and sync.get('ready') and sync.get('helper_ready')
|
|
and native['agent_last_decision'] == sync.get('decision_id')):break
|
|
require(time.monotonic() < deadline,'same-attempt definitive reconciliation missing; never unlock/retry');time.sleep(2)
|
|
require(final['state'] == 'succeeded' and final['result']['exit_code'] == 0 and bytes(final['result']['stdout']) == b'RMM_CRASH_COUNT=1\n'
|
|
and final['result']['evidence']['postconditions_passed'] and native['counter'] == 1
|
|
and len(native['root_attempts']) == len(native['agent_consumed']) == len(native['attempts']) == 1
|
|
and native['ledger'] == dict(consumed=initial['ledger']['consumed']+1,unresolved=0,faults=0),'one actual helper completion and consumption')
|
|
selected = sync['selected'];grant = killed['before']['root_attempts'][0]['grant_id']
|
|
require(len(selected) == 1 and selected[0]['grant_id'] == grant and selected[0]['unlock'] is True
|
|
and selected[0]['outcome_state'] == 'succeeded' and selected[0]['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
|
|
and selected[0]['revision'] > unknown['attempts'][0]['revision']
|
|
and selected[0]['job_payload_hash'] == fixed['job_payload_sha256'] and selected[0]['plan_hash'] == fixed['plan_sha256']
|
|
and native['root_attempts'][0]['grant_id'] == native['agent_consumed'][0]['grant_id'] == grant,'exact existing signed Sync closure')
|
|
checkpoint.update(final=final,native=native,stage='reconciled');save()
|
|
def audit(record):
|
|
record['postapproval'] = evidence('gate',job_id=record['job']['job_id'])
|
|
record['audit'] = evidence('audit',operation_id=record['operation_id']);save()
|
|
require([record['postapproval'][k] for k in ('jobs','outbox','permits')] == [1,1,1]
|
|
and record['audit'] == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=record['plan']['plan_hash']),'one authorized immutable attempt')
|
|
audit(effect)
|
|
postcheck = authorize(True);deadline=time.monotonic()+90
|
|
while True:
|
|
checked = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=postcheck['job']['job_id']))
|
|
if checked['state'] in cloud.contract.TERMINAL or checked['state'] == 'unknown':break
|
|
require(time.monotonic() < deadline,'independent read outcome missing');time.sleep(2)
|
|
postcheck['result'] = checked;save()
|
|
require(checked['state'] == 'succeeded' and checked['result']['exit_code'] == 0
|
|
and checked['result']['evidence']['postconditions_passed'] and bytes(checked['result']['stdout']) == b'RMM_CRASH_COUNT=1\n','independent final counter still one')
|
|
audit(postcheck)
|
|
require(evidence('gate')['unresolved'] == 0,'all effects must be closed before policy close')
|
|
closed = cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=opened['policy_sha256'],backup=opened['backup']))
|
|
checkpoint.update(policy_closed=closed,verified=True,stage='native_dev_test_accepted');save()
|
|
print('PASS bounded Cloud agent crash: durable+central unknown, same-attempt root outcome and signed Sync, one effect, separate UI-approved counter1; dev/test only')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:main()
|
|
except Exception as error:
|
|
print('CLOUD_CRASH_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
|
raise SystemExit(2) from None
|