Files
rust-discovery/deploy/cloud_offline_acceptance.py
T
2026-11-13 20:24:01 +01:00

343 lines
28 KiB
Python

#!/usr/bin/env python3
"""Cloud-only active-agent offline/start-window/revocation acceptance.
Requires explicit execution. Plans start pending with zero executable artifacts;
approval and plan revocation use actual dev/test WebUI controls. A narrowly
scoped systemd BPF network fault affects only the existing Cloud agent unit.
SSH is used solely for that transport bootstrap and retained-state evidence.
No enrollment, package action, journal reset, approval API or automatic retry.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
import cloud_mvp_acceptance as cloud
CASES = ('offline_wait', 'expired_offline', 'revoked_before_start')
SERVER_IP = '88.99.80.202'
STATE = cloud.ROOT/'devstate/cloud-offline'
require = cloud.contract.require
def counter_script(identifier):
identifier = str(uuid.UUID(identifier))
return ('set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-'+identifier+'\n'
'test ! -e "$d"\nmkdir "$d"\n'
'printf \'%s\\n\' \'1\' > "$d/count"\n'
'test "$(cat "$d/count")" = 1\n'
'printf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n')
def server_evidence(payload):
"""Fixed SELECTs only, never creates/revokes/approves an operation."""
root = Path('/opt/rust-discovery')
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
job = str(uuid.UUID(payload['job_id'])); operation = str(uuid.UUID(payload['operation_id']))
plan = str(uuid.UUID(payload['plan_id']))
generation = int(payload['previous_generation']); require(generation > 0, 'previous fenced channel')
import datetime as dt
after = dt.datetime.fromisoformat(payload['after'].replace('Z','+00:00')).isoformat() if payload.get('after') else '1970-01-01T00:00:00+00:00'
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND job_id='"+job+"'"
query = ("SELECT json_build_object('observed_at',now(),'state',(SELECT state FROM jobs WHERE "+scope+"),"
"'start_reserved',COALESCE((SELECT start_reserved FROM jobs WHERE "+scope+"),false),"
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'approval_state',(SELECT approval_state FROM approval_requests WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"'),"
"'revoked_authorizations',(SELECT count(*) FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' AND revoked_at IS NOT NULL),"
"'starts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data)),'[]'::json) FROM audit_events "
"WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"'),"
"'fresh_signed_sync_before_start',EXISTS(SELECT 1 FROM sync_checkpoints c WHERE c.tenant_id='"+tenant+"' AND c.agent_id='"+agent+"' "
"AND c.connection_generation>"+str(generation)+" AND (convert_from(c.payload,'UTF8')::jsonb->>'ready')::boolean "
"AND c.created_at>='"+after+"'::timestamptz AND c.created_at<=(SELECT min(created_at) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')),"
"'revocation',(SELECT json_build_object('principal',principal,'origin',data->>'approval_origin',"
"'simulation',data->'testsimulation','interactive_session',data->>'session_hash' IS NOT NULL,"
"'plan_id',data->'decision'->>'plan_id','operation_id',data->'decision'->>'operation_id') "
"FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='plan_revoked' AND data->'decision'->>'plan_id'='"+plan+"' ORDER BY created_at DESC LIMIT 1));")
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
def vm_network(payload):
"""One operational transport fault, plus reads of existing durable evidence."""
import contextlib
import datetime as dt
import errno
import fcntl
import grp
import sqlite3
import stat
identifier = str(uuid.UUID(payload['id']))
phase = payload['phase']; require(phase in ('preflight','window'), 'fixed network phase')
action = payload['action']; require(action in ('state','block','restore'), 'fixed network operation')
state = Path('/var/lib/rmm-agent-remote'); helper = Path('/var/lib/rmm-helper')
unit = 'rmm-agent-remote.service'; account = pwd.getpwnam('rmm-agent')
require(os.geteuid() == 0 and account.pw_uid == 993 and account.pw_gid > 0
and account.pw_gid == grp.getgrnam('rmm-agent').gr_gid
and state.stat().st_uid == account.pw_uid, 'fixed unprivileged named Cloud agent')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed Cloud identity')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed current agent binary')
def properties():
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
return dict(line.split('=',1) for line in raw.splitlines())
@contextlib.contextmanager
def agent_read(path):
require(path in (state/'journal.sqlite',state/'consumption.sqlite') and not path.is_symlink(), 'fixed agent database')
owner = path.stat()
require(stat.S_ISREG(owner.st_mode) and owner.st_uid == account.pw_uid
and owner.st_gid == account.pw_gid, 'agent database ownership')
sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')]
existing = set()
for p in sidecars:
require(not p.is_symlink(), 'SQLite sidecar symlink')
if p.exists():
require(p.stat().st_uid == owner.st_uid, 'existing SQLite sidecar owner')
existing.add(p)
db = None
try:
db = sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
yield db
finally:
if db is not None: db.close()
# sqlite3's transaction context alone does NOT close a connection.
# A root readonly opener can create these two fixed WAL files;
# correct only such newly created files, even on a query failure.
for p in sidecars:
if p not in existing and p.exists():
fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
try:
s = os.fstat(fd)
require(stat.S_ISREG(s.st_mode) and s.st_nlink == 1
and s.st_uid in (0,owner.st_uid), 'new fixed SQLite sidecar')
if s.st_uid == 0: os.fchown(fd,owner.st_uid,owner.st_gid)
finally: os.close(fd)
def snapshot():
result = dict(agent_service=properties())
with agent_read(state/'journal.sqlite') as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
if payload.get('job_id'):
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
rows = db.execute('SELECT state,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
result['attempts'] = [dict(state=r[0],grant_id=r[1]) for r in rows]
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
else:
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
with agent_read(state/'consumption.sqlite') as db:
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
with contextlib.closing(sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
serial = [[dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
result['helper_ledger_sha256'] = hashlib.sha256(json.dumps(serial,separators=(',',':')).encode()).hexdigest()
result['helper_consumed'] = len(rows); result['helper_unresolved'] = sum(row[5] is None for row in rows)
effect_directory = state/('offline-'+identifier)
if effect_directory.exists():
m = effect_directory.lstat();require(stat.S_ISDIR(m.st_mode) and m.st_uid == 993 and m.st_mode&0o777 == 0o700, 'fixed stored effect directory')
else: require(not effect_directory.is_symlink(), 'stored effect directory symlink')
marker = effect_directory/'count'
if marker.exists():
m = marker.lstat(); require(stat.S_ISREG(m.st_mode) and m.st_uid == account.pw_uid and m.st_mode&0o777 == 0o600 and m.st_size <= 16, 'fixed stored counter evidence')
result['count'] = marker.read_text()
else:
require(not marker.is_symlink(), 'stored counter symlink'); result['count'] = None
return result
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
require(not directory.is_symlink(), 'network dropin directory')
directory.mkdir(mode=0o755,exist_ok=True)
require(directory.stat().st_uid == 0 and directory.stat().st_mode&0o022 == 0, 'network dropin ownership')
dropin = directory/'96-cloud-offline-acceptance.conf'
expected = ('# Cloud MVP offline '+identifier+' '+phase+'\n[Service]\nIPAddressDeny=88.99.80.202/32\n').encode()
require(not dropin.is_symlink(), 'network dropin symlink')
timer = 'rmm-offline-rollback-'+identifier+'-'+phase
before = snapshot()
if action == 'state': return before
if action == 'restore':
if dropin.exists():
require(dropin.stat().st_uid == 0 and dropin.read_bytes() == expected, 'exact network fault ownership')
dropin.unlink()
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
subprocess.run(['systemctl','stop',timer+'.timer'],capture_output=True,timeout=10)
return dict(restored=True,after=snapshot())
# State reads and cleanup of our exact transport fault remain possible
# after an unknown result. They never clear a journal fence or job state.
require(before['agent_fault'] is None and before['helper_fault'] is None and before['helper_unresolved'] == 0 and before['other_unresolved'] == 0, 'unrelated or unknown work remains fenced')
if payload.get('helper_ledger_sha256'):
require(before['helper_ledger_sha256'] == payload['helper_ledger_sha256'], 'existing Root ledger changed')
require(not dropin.exists() and before['agent_service']['ActiveState'] == 'active' and not before['agent_service']['IPAddressAllow'], 'clean active unit and no allow override')
require(before['count'] is None and not before['consumed'], 'no prior effect/consumption')
if phase == 'window':
start = dt.datetime.fromisoformat(payload['not_before'].replace('Z','+00:00'))
require(dt.datetime.now(dt.timezone.utc) < start-dt.timedelta(seconds=60) and before['attempts'] == [dict(state='accepted',grant_id='')], 'fixed future accepted attempt with zero grant')
else:
require(not payload.get('job_id'), 'preflight has no job')
# Arm bounded automatic transport rollback before creating the deny rule.
# The timer can remove only these exact bytes and never touches job state.
rollback_dir = Path('/var/lib/rmm-offline-bootstrap')
require(not rollback_dir.is_symlink(), 'rollback directory symlink')
rollback_dir.mkdir(mode=0o700,exist_ok=True)
require(rollback_dir.stat().st_uid == 0 and rollback_dir.stat().st_mode&0o777 == 0o700, 'private bootstrap rollback directory')
rollback = rollback_dir/(identifier+'-'+phase+'.py')
code = ('import os,subprocess\nfrom pathlib import Path\np=Path('+repr(str(dropin))+')\n'
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(expected)+':\n'
' p.unlink()\n subprocess.run(["systemctl","daemon-reload"],check=True)\n'
' subprocess.run(["systemctl","restart","rmm-agent-remote.service"],check=True)\n')
fd = os.open(rollback,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream: stream.write(code);stream.flush();os.fsync(stream.fileno())
subprocess.run(['systemd-run','--quiet','--unit='+timer,'--on-active=420s','--property=Type=oneshot','/usr/bin/python3',str(rollback)],capture_output=True,check=True,timeout=10)
# Quiesce this accepted/unstarted process only while applying the tested
# policy. It must be running again BEFORE not_before; a stopped-agent
# interval never counts toward the active offline acceptance window.
subprocess.run(['systemctl','stop',unit],capture_output=True,check=True,timeout=20)
stopped = snapshot()
require(stopped['agent_service']['ActiveState'] == 'inactive' and not stopped['consumed'] and stopped['count'] is None, 'quiescent unstarted network setup')
fd = os.open(dropin,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
with os.fdopen(fd,'wb') as stream: stream.write(expected);stream.flush();os.fsync(stream.fileno())
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
after = snapshot(); require(after['agent_service']['ActiveState'] == 'active' and int(after['agent_service']['MainPID']) > 0 and after['count'] is None and not after['consumed'], 'active offline agent and zero effect')
if phase == 'window': require(dt.datetime.now(dt.timezone.utc) < start, 'agent must be active offline before earliest start')
group = after['agent_service']['ControlGroup']
require((group == '/system.slice/rmm-agent-remote.service' or group.startswith('/system.slice/rmm-agent-remote.service/')) and '..' not in group, 'actual agent cgroup')
cgroup = Path('/sys/fs/cgroup'+group)/'cgroup.procs'
require(cgroup.is_file(), 'unified cgroup transport enforcement required')
# This separate bootstrap child moves ONLY ITS OWN PID, drops privilege and
# attempts TCP only. No agent credential, TLS override or appdata is used.
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
'os.setgroups([]);os.setgid('+str(account.pw_gid)+');os.setuid('+str(account.pw_uid)+')\n'
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
'except OSError as e: result={"connected":False,"errno":e.errno}\n'
'print(json.dumps(result))\n')
result = subprocess.run(['/usr/bin/python3','-I','-c',probe],capture_output=True,check=True,timeout=4)
blocked = json.loads(result.stdout)
require(blocked.get('connected') is False and blocked.get('errno') in (errno.EACCES,errno.EPERM), 'actual same-cgroup BPF denial must be EACCES/EPERM; timeout is not proof')
after = snapshot()
require(after['helper_ledger_sha256'] == before['helper_ledger_sha256'], 'Root ledger preserved during network bootstrap')
return dict(blocked=True,probe=blocked,after=after,rollback_after_seconds=420,unit=unit,server_ip='88.99.80.202',port=443)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
parser.add_argument('--case',choices=CASES,required=True)
args = parser.parse_args(); os.umask(0o077)
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(), 'notebook is development only')
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
identity = installed['identity']; settings = dict(tenant_id=cloud.TENANT,**identity)
backend = cloud.Backend(); identifier = str(uuid.uuid4())
folder = STATE/identifier; folder.mkdir(mode=0o700,parents=True)
checkpoint = dict(profile='dev/test',case=args.case,id=identifier,expected_principal='simulated-testprincipal-a',**settings,no_retry=True,stage='preflight')
serial = 0
def save():
nonlocal serial
cloud.contract.write_private(folder/('%03d-%s.json'%(serial,checkpoint['stage'])),checkpoint);serial += 1
def gate(job_id=None):
return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**({'job_id':job_id} if job_id else {})))
def native(action,phase='window',**extra):
p = dict(action=action,phase=phase,id=identifier,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256'],**extra)
if 'job' in checkpoint and phase == 'window': p.update(job_id=checkpoint['job']['job_id'],attempt_id=checkpoint['job']['payload']['attempt_id'])
return cloud.bootstrap.rpc('linux-vm.conf',vm_network,p)
def healthy(previous_generation=None):
end = time.monotonic()+100
while True:
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
if observed.get('sync_ready') and not observed.get('restore_blocked') and (previous_generation is None or observed['connection_generation'] > previous_generation): return observed
require(time.monotonic() < end, 'ordinary signed reconnect did not become healthy');time.sleep(2)
require(gate()['unresolved'] == 0, 'unresolved central work remains fenced')
checkpoint['initial_channel'] = healthy(); initial = native('state','preflight')
ledger_sha = initial['helper_ledger_sha256']; checkpoint['initial_native'] = initial;save()
checkpoint['network_preflight'] = native('block','preflight',helper_ledger_sha256=ledger_sha);save()
checkpoint['preflight_restored'] = native('restore','preflight',helper_ledger_sha256=ledger_sha);save()
checkpoint['healthy_before_plan'] = healthy(checkpoint['initial_channel']['connection_generation'])
start = cloud.contract.now()+dt.timedelta(seconds=150); end = start+dt.timedelta(seconds=130)
request = cloud.contract.case_input('hello',settings,start=start)
request['parameters']['variants'][0]['script_utf8'] = counter_script(identifier)
request['parameters']['expected_stdout'] = 'RMM_OFFLINE_COUNT=1\n'
request['schedule'] = cloud.contract.schedule(start,end)
request['reason'] = 'Explicit dev/test active Cloud agent '+args.case+'; one fixed attempt, no implicit retry'
plan = backend.call('operations.plan',dict(tenant_id=cloud.TENANT,input=request)); require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'one immutable fixed job')
job = plan['jobs'][0]; pending = backend.call('operations.submit',dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier))
checkpoint.update(plan=plan,job=job,request=request,operation_id=pending['operation_id'],stage='pending_approval',preapproval=gate(job['job_id']))
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and all(checkpoint['preapproval'][k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts');save()
def browser(phase):
cloud.prepare_browser(); path = folder/('ui-'+phase+'.json'); cloud.contract.write_private(path,dict(checkpoint,phase=phase))
result = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_offline_acceptance.cjs'),str(path)],capture_output=True,timeout=90)
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint and never repeat approval')
browser('approve'); checkpoint['stage'] = 'approved';save()
deadline = time.monotonic()+45
while True:
server = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])); retained = native('state',helper_ledger_sha256=ledger_sha)
require(gate(job['job_id'])['permits'] == 0 and not retained['consumed'] and retained['count'] is None, 'future attempt already has a grant/effect')
if server['state'] == 'accepted' and retained['attempts'] == [dict(state='accepted',grant_id='')]: break
require(time.monotonic() < deadline, 'future offer was not durably accepted');time.sleep(2)
checkpoint.update(accepted=server,native_accepted=retained,stage='accepted_unstarted');save()
checkpoint['network_fault'] = native('block',not_before=job['payload']['not_before'],helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'active_offline';save()
channel = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
resume = cloud.contract.timestamp(job['payload']['start_before'] if args.case == 'expired_offline' else job['payload']['not_before'])+dt.timedelta(seconds=10)
if args.case == 'revoked_before_start': browser('revoke');checkpoint['ui_revoked'] = True;save()
sequence = checkpoint['network_fault']['after']['sequence']; windows = []
while cloud.contract.now() < resume:
retained = native('state',helper_ledger_sha256=ledger_sha); counts = gate(job['job_id'])
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
require(retained['agent_service']['ActiveState'] == 'active' and retained['agent_service']['MainPID'] == checkpoint['network_fault']['after']['agent_service']['MainPID']
and not retained['consumed'] and retained['count'] is None and counts['permits'] == 0 and observed['last_contact_at'] == channel['last_contact_at'], 'active offline window had a grant/effect/contact or agent exited')
windows.append(dict(observed_at=cloud.contract.now().isoformat(),sequence=retained['sequence'],grants=counts['permits'],count=retained['count'],active=True))
checkpoint['offline_window'] = windows;save();time.sleep(5)
require(windows and windows[-1]['sequence'] > sequence, 'running offline agent did not persist reconnect attempts')
checkpoint['server_offline_tail'] = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation']))
require(checkpoint['server_offline_tail']['grants'] == 0 and not checkpoint['server_offline_tail']['start_reserved'], 'no start immediately before network restoration')
checkpoint['network_restored'] = native('restore',helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'network_restored';save()
checkpoint['healthy_reconnect'] = healthy(channel['connection_generation'])
deadline = time.monotonic()+100
while True:
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
if result['state'] in cloud.contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
require(time.monotonic() < deadline, 'terminal native result missing; no retry');time.sleep(2)
expected = {'offline_wait':'succeeded','expired_offline':'expired','revoked_before_start':'cancelled'}[args.case]
require(result['state'] == expected, 'actual terminal state differs from bounded case')
native_final = native('state',helper_ledger_sha256=ledger_sha)
wanted = 1 if args.case == 'offline_wait' else 0
final_gate = gate(job['job_id']); require([final_gate[k] for k in ('jobs','outbox','permits')] == [1,1,wanted] and final_gate['unresolved'] == 0, 'exact job/outbox/start-grant counts')
require(len(native_final['consumed']) == wanted and native_final['count'] == ('1\n' if wanted else None), 'exact native consumption and stored effect counter')
if wanted: require(result['result']['exit_code'] == 0 and bytes(result['result']['stdout']) == b'RMM_OFFLINE_COUNT=1\n' and result['result']['evidence']['postconditions_passed'], 'native one-effect postcondition')
if args.case == 'offline_wait':
observed_at = cloud.contract.now()
require(observed_at < min(cloud.contract.timestamp(plan['expires_at']),cloud.contract.timestamp(job['payload']['start_before'])), 'original submit idempotency must be observed while its plan is valid')
submit_request = dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier)
same = backend.call('operations.submit',submit_request)
after_submit = gate(job['job_id'])
counts = lambda observed: {k:observed[k] for k in ('jobs','outbox','permits')}
require(same['operation_id'] == pending['operation_id'] and counts(final_gate) == counts(after_submit) == dict(jobs=1,outbox=1,permits=1), 'same original submit must reuse its operation without new executable artifacts')
receipt = dict(verified=True,case=args.case,operation_id=pending['operation_id'],request=submit_request,
response_operation_id=same['operation_id'],before_counts=counts(final_gate),after_counts=counts(after_submit),observed_at=observed_at.isoformat())
cloud.contract.write_private(folder/'idempotency-receipt.json',receipt)
checkpoint['idempotency_receipt'] = receipt;save()
audit = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='audit',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],operation_id=pending['operation_id']))
require(audit == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']), 'exact interactive approval audit')
details = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation'],after=checkpoint['server_offline_tail']['observed_at']))
require(details['grants'] == wanted and len(details['starts']) == wanted and details['start_reserved'] == bool(wanted), 'retained start reservation evidence')
if wanted: require(details['fresh_signed_sync_before_start'], 'fresh healthy signed reconnect must precede the one start reservation')
if args.case == 'revoked_before_start':
require(details['approval_state'] == 'revoked' and details['revoked_authorizations'] == 1 and details['revocation'] == dict(principal=checkpoint['expected_principal'],origin='dev_test_browser_simulation',simulation=True,interactive_session=True,plan_id=plan['plan_id'],operation_id=pending['operation_id']), 'actual interactive plan revocation audit')
checkpoint.update(result=result,native_final=native_final,postapproval=final_gate,audit=audit,server_evidence=details,stage='native_dev_test_accepted',verified=True);save()
print('PASS Cloud '+args.case+': active offline BPF verified, WebUI gate, exact grants/effect='+str(wanted))
if __name__ == '__main__':
try: main()
except Exception as error:
message = str(error)
print('CLOUD_OFFLINE_INCOMPLETE:'+ (message if isinstance(error,cloud.contract.GuardError)
or message.startswith('remote bootstrap incomplete: ') else type(error).__name__))
raise SystemExit(2) from None