304 lines
26 KiB
Python
304 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Cloud-only active-agent offline/start-window/revocation acceptance.
|
|
|
|
Requires explicit execution. Plans start pending with zero executable artifacts;
|
|
approval and plan revocation use actual dev/test WebUI controls. A narrowly
|
|
scoped systemd BPF network fault affects only the existing Cloud agent unit.
|
|
SSH is used solely for that transport bootstrap and retained-state evidence.
|
|
No enrollment, package action, journal reset, approval API or automatic retry.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
import cloud_mvp_acceptance as cloud
|
|
|
|
CASES = ('offline_wait', 'expired_offline', 'revoked_before_start')
|
|
SERVER_IP = '88.99.80.202'
|
|
STATE = cloud.ROOT/'devstate/cloud-offline'
|
|
require = cloud.contract.require
|
|
|
|
|
|
def counter_script(identifier):
|
|
identifier = str(uuid.UUID(identifier))
|
|
return ('set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-'+identifier+'\n'
|
|
'test ! -e "$d"\nmkdir "$d"\n'
|
|
'printf \'%s\\n\' \'1\' > "$d/count"\n'
|
|
'test "$(cat "$d/count")" = 1\n'
|
|
'printf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n')
|
|
|
|
|
|
def server_evidence(payload):
|
|
"""Fixed SELECTs only, never creates/revokes/approves an operation."""
|
|
root = Path('/opt/rust-discovery')
|
|
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
|
job = str(uuid.UUID(payload['job_id'])); operation = str(uuid.UUID(payload['operation_id']))
|
|
plan = str(uuid.UUID(payload['plan_id']))
|
|
generation = int(payload['previous_generation']); require(generation > 0, 'previous fenced channel')
|
|
import datetime as dt
|
|
after = dt.datetime.fromisoformat(payload['after'].replace('Z','+00:00')).isoformat() if payload.get('after') else '1970-01-01T00:00:00+00:00'
|
|
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
|
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND job_id='"+job+"'"
|
|
query = ("SELECT json_build_object('observed_at',now(),'state',(SELECT state FROM jobs WHERE "+scope+"),"
|
|
"'start_reserved',COALESCE((SELECT start_reserved FROM jobs WHERE "+scope+"),false),"
|
|
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
|
"'approval_state',(SELECT approval_state FROM approval_requests WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"'),"
|
|
"'revoked_authorizations',(SELECT count(*) FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' AND revoked_at IS NOT NULL),"
|
|
"'starts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data)),'[]'::json) FROM audit_events "
|
|
"WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"'),"
|
|
"'fresh_signed_sync_before_start',EXISTS(SELECT 1 FROM sync_checkpoints c WHERE c.tenant_id='"+tenant+"' AND c.agent_id='"+agent+"' "
|
|
"AND c.connection_generation>"+str(generation)+" AND (convert_from(c.payload,'UTF8')::jsonb->>'ready')::boolean "
|
|
"AND c.created_at>='"+after+"'::timestamptz AND c.created_at<=(SELECT min(created_at) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')),"
|
|
"'revocation',(SELECT json_build_object('principal',principal,'origin',data->>'approval_origin',"
|
|
"'simulation',data->'testsimulation','interactive_session',data->>'session_hash' IS NOT NULL,"
|
|
"'plan_id',data->'decision'->>'plan_id','operation_id',data->'decision'->>'operation_id') "
|
|
"FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='plan_revoked' AND data->'decision'->>'plan_id'='"+plan+"' ORDER BY created_at DESC LIMIT 1));")
|
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
|
|
|
|
|
def vm_network(payload):
|
|
"""One operational transport fault, plus reads of existing durable evidence."""
|
|
import datetime as dt
|
|
import errno
|
|
import fcntl
|
|
import sqlite3
|
|
import stat
|
|
identifier = str(uuid.UUID(payload['id']))
|
|
phase = payload['phase']; require(phase in ('preflight','window'), 'fixed network phase')
|
|
action = payload['action']; require(action in ('state','block','restore'), 'fixed network operation')
|
|
state = Path('/var/lib/rmm-agent-remote'); helper = Path('/var/lib/rmm-helper')
|
|
unit = 'rmm-agent-remote.service'; account = pwd.getpwnam('rmm-agent')
|
|
require(os.geteuid() == 0 and account.pw_uid == 993 and account.pw_gid == 993, 'fixed unprivileged Cloud agent')
|
|
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed Cloud identity')
|
|
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed current agent binary')
|
|
def properties():
|
|
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
|
|
return dict(line.split('=',1) for line in raw.splitlines())
|
|
def snapshot():
|
|
result = dict(agent_service=properties())
|
|
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
|
|
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
|
|
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
|
|
if payload.get('job_id'):
|
|
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
|
rows = db.execute('SELECT state,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
|
|
result['attempts'] = [dict(state=r[0],grant_id=r[1]) for r in rows]
|
|
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
|
|
else:
|
|
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
|
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
|
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
|
|
with sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
|
|
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
|
|
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
|
|
serial = [[dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
|
|
result['helper_ledger_sha256'] = hashlib.sha256(json.dumps(serial,separators=(',',':')).encode()).hexdigest()
|
|
result['helper_consumed'] = len(rows); result['helper_unresolved'] = sum(row[5] is None for row in rows)
|
|
effect_directory = state/('offline-'+identifier)
|
|
if effect_directory.exists():
|
|
m = effect_directory.lstat();require(stat.S_ISDIR(m.st_mode) and m.st_uid == 993 and m.st_mode&0o777 == 0o700, 'fixed stored effect directory')
|
|
else: require(not effect_directory.is_symlink(), 'stored effect directory symlink')
|
|
marker = effect_directory/'count'
|
|
if marker.exists():
|
|
m = marker.lstat(); require(stat.S_ISREG(m.st_mode) and m.st_uid == account.pw_uid and m.st_mode&0o777 == 0o600 and m.st_size <= 16, 'fixed stored counter evidence')
|
|
result['count'] = marker.read_text()
|
|
else:
|
|
require(not marker.is_symlink(), 'stored counter symlink'); result['count'] = None
|
|
return result
|
|
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
|
require(not directory.is_symlink(), 'network dropin directory')
|
|
directory.mkdir(mode=0o755,exist_ok=True)
|
|
require(directory.stat().st_uid == 0 and directory.stat().st_mode&0o022 == 0, 'network dropin ownership')
|
|
dropin = directory/'96-cloud-offline-acceptance.conf'
|
|
expected = ('# Cloud MVP offline '+identifier+' '+phase+'\n[Service]\nIPAddressDeny=88.99.80.202/32\n').encode()
|
|
require(not dropin.is_symlink(), 'network dropin symlink')
|
|
timer = 'rmm-offline-rollback-'+identifier+'-'+phase
|
|
before = snapshot()
|
|
require(before['agent_fault'] is None and before['helper_fault'] is None and before['helper_unresolved'] == 0 and before['other_unresolved'] == 0, 'unrelated or unknown work remains fenced')
|
|
if payload.get('helper_ledger_sha256'):
|
|
require(before['helper_ledger_sha256'] == payload['helper_ledger_sha256'], 'existing Root ledger changed')
|
|
if action == 'state': return before
|
|
if action == 'restore':
|
|
if dropin.exists():
|
|
require(dropin.stat().st_uid == 0 and dropin.read_bytes() == expected, 'exact network fault ownership')
|
|
dropin.unlink()
|
|
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
|
|
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
|
|
subprocess.run(['systemctl','stop',timer+'.timer'],capture_output=True,timeout=10)
|
|
return dict(restored=True,after=snapshot())
|
|
require(not dropin.exists() and before['agent_service']['ActiveState'] == 'active' and not before['agent_service']['IPAddressAllow'], 'clean active unit and no allow override')
|
|
require(before['count'] is None and not before['consumed'], 'no prior effect/consumption')
|
|
if phase == 'window':
|
|
start = dt.datetime.fromisoformat(payload['not_before'].replace('Z','+00:00'))
|
|
require(dt.datetime.now(dt.timezone.utc) < start-dt.timedelta(seconds=60) and before['attempts'] == [dict(state='accepted',grant_id='')], 'fixed future accepted attempt with zero grant')
|
|
else:
|
|
require(not payload.get('job_id'), 'preflight has no job')
|
|
# Arm bounded automatic transport rollback before creating the deny rule.
|
|
# The timer can remove only these exact bytes and never touches job state.
|
|
rollback_dir = Path('/var/lib/rmm-offline-bootstrap')
|
|
require(not rollback_dir.is_symlink(), 'rollback directory symlink')
|
|
rollback_dir.mkdir(mode=0o700,exist_ok=True)
|
|
require(rollback_dir.stat().st_uid == 0 and rollback_dir.stat().st_mode&0o777 == 0o700, 'private bootstrap rollback directory')
|
|
rollback = rollback_dir/(identifier+'-'+phase+'.py')
|
|
code = ('import os,subprocess\nfrom pathlib import Path\np=Path('+repr(str(dropin))+')\n'
|
|
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(expected)+':\n'
|
|
' p.unlink()\n subprocess.run(["systemctl","daemon-reload"],check=True)\n'
|
|
' subprocess.run(["systemctl","restart","rmm-agent-remote.service"],check=True)\n')
|
|
fd = os.open(rollback,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
with os.fdopen(fd,'w') as stream: stream.write(code);stream.flush();os.fsync(stream.fileno())
|
|
subprocess.run(['systemd-run','--quiet','--unit='+timer,'--on-active=420s','--property=Type=oneshot','/usr/bin/python3',str(rollback)],capture_output=True,check=True,timeout=10)
|
|
# Quiesce this accepted/unstarted process only while applying the tested
|
|
# policy. It must be running again BEFORE not_before; a stopped-agent
|
|
# interval never counts toward the active offline acceptance window.
|
|
subprocess.run(['systemctl','stop',unit],capture_output=True,check=True,timeout=20)
|
|
stopped = snapshot()
|
|
require(stopped['agent_service']['ActiveState'] == 'inactive' and not stopped['consumed'] and stopped['count'] is None, 'quiescent unstarted network setup')
|
|
fd = os.open(dropin,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
|
|
with os.fdopen(fd,'wb') as stream: stream.write(expected);stream.flush();os.fsync(stream.fileno())
|
|
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
|
|
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
|
|
after = snapshot(); require(after['agent_service']['ActiveState'] == 'active' and int(after['agent_service']['MainPID']) > 0 and after['count'] is None and not after['consumed'], 'active offline agent and zero effect')
|
|
if phase == 'window': require(dt.datetime.now(dt.timezone.utc) < start, 'agent must be active offline before earliest start')
|
|
group = after['agent_service']['ControlGroup']
|
|
require((group == '/system.slice/rmm-agent-remote.service' or group.startswith('/system.slice/rmm-agent-remote.service/')) and '..' not in group, 'actual agent cgroup')
|
|
cgroup = Path('/sys/fs/cgroup'+group)/'cgroup.procs'
|
|
require(cgroup.is_file(), 'unified cgroup transport enforcement required')
|
|
# This separate bootstrap child moves ONLY ITS OWN PID, drops privilege and
|
|
# attempts TCP only. No agent credential, TLS override or appdata is used.
|
|
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
|
|
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
|
|
'os.setgroups([]);os.setgid(993);os.setuid(993)\n'
|
|
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
|
|
'except OSError as e: result={"connected":False,"errno":e.errno}\n'
|
|
'print(json.dumps(result))\n')
|
|
result = subprocess.run(['/usr/bin/python3','-I','-c',probe],capture_output=True,check=True,timeout=4)
|
|
blocked = json.loads(result.stdout)
|
|
require(blocked.get('connected') is False and blocked.get('errno') in (errno.EACCES,errno.EPERM), 'actual same-cgroup BPF denial must be EACCES/EPERM; timeout is not proof')
|
|
after = snapshot()
|
|
require(after['helper_ledger_sha256'] == before['helper_ledger_sha256'], 'Root ledger preserved during network bootstrap')
|
|
return dict(blocked=True,probe=blocked,after=after,rollback_after_seconds=420,unit=unit,server_ip='88.99.80.202',port=443)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
|
parser.add_argument('--case',choices=CASES,required=True)
|
|
args = parser.parse_args(); os.umask(0o077)
|
|
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(), 'notebook is development only')
|
|
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
|
identity = installed['identity']; settings = dict(tenant_id=cloud.TENANT,**identity)
|
|
backend = cloud.Backend(); identifier = str(uuid.uuid4())
|
|
folder = STATE/identifier; folder.mkdir(mode=0o700,parents=True)
|
|
checkpoint = dict(profile='dev/test',case=args.case,id=identifier,expected_principal='simulated-testprincipal-a',**settings,no_retry=True,stage='preflight')
|
|
serial = 0
|
|
def save():
|
|
nonlocal serial
|
|
cloud.contract.write_private(folder/('%03d-%s.json'%(serial,checkpoint['stage'])),checkpoint);serial += 1
|
|
def gate(job_id=None):
|
|
return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**({'job_id':job_id} if job_id else {})))
|
|
def native(action,phase='window',**extra):
|
|
p = dict(action=action,phase=phase,id=identifier,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256'],**extra)
|
|
if 'job' in checkpoint and phase == 'window': p.update(job_id=checkpoint['job']['job_id'],attempt_id=checkpoint['job']['payload']['attempt_id'])
|
|
return cloud.bootstrap.rpc('linux-vm.conf',vm_network,p)
|
|
def healthy(previous_generation=None):
|
|
end = time.monotonic()+100
|
|
while True:
|
|
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
|
if observed.get('sync_ready') and not observed.get('restore_blocked') and (previous_generation is None or observed['connection_generation'] > previous_generation): return observed
|
|
require(time.monotonic() < end, 'ordinary signed reconnect did not become healthy');time.sleep(2)
|
|
require(gate()['unresolved'] == 0, 'unresolved central work remains fenced')
|
|
checkpoint['initial_channel'] = healthy(); initial = native('state','preflight')
|
|
ledger_sha = initial['helper_ledger_sha256']; checkpoint['initial_native'] = initial;save()
|
|
checkpoint['network_preflight'] = native('block','preflight',helper_ledger_sha256=ledger_sha);save()
|
|
checkpoint['preflight_restored'] = native('restore','preflight',helper_ledger_sha256=ledger_sha);save()
|
|
checkpoint['healthy_before_plan'] = healthy(checkpoint['initial_channel']['connection_generation'])
|
|
start = cloud.contract.now()+dt.timedelta(seconds=150); end = start+dt.timedelta(seconds=130)
|
|
request = cloud.contract.case_input('hello',settings,start=start)
|
|
request['parameters']['variants'][0]['script_utf8'] = counter_script(identifier)
|
|
request['parameters']['expected_stdout'] = 'RMM_OFFLINE_COUNT=1\n'
|
|
request['schedule'] = cloud.contract.schedule(start,end)
|
|
request['reason'] = 'Explicit dev/test active Cloud agent '+args.case+'; one fixed attempt, no implicit retry'
|
|
plan = backend.call('operations.plan',dict(tenant_id=cloud.TENANT,input=request)); require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'one immutable fixed job')
|
|
job = plan['jobs'][0]; pending = backend.call('operations.submit',dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier))
|
|
checkpoint.update(plan=plan,job=job,request=request,operation_id=pending['operation_id'],stage='pending_approval',preapproval=gate(job['job_id']))
|
|
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and all(checkpoint['preapproval'][k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts');save()
|
|
def browser(phase):
|
|
cloud.prepare_browser(); path = folder/('ui-'+phase+'.json'); cloud.contract.write_private(path,dict(checkpoint,phase=phase))
|
|
result = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_offline_acceptance.cjs'),str(path)],capture_output=True,timeout=90)
|
|
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint and never repeat approval')
|
|
browser('approve'); checkpoint['stage'] = 'approved';save()
|
|
deadline = time.monotonic()+45
|
|
while True:
|
|
server = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])); retained = native('state',helper_ledger_sha256=ledger_sha)
|
|
require(gate(job['job_id'])['permits'] == 0 and not retained['consumed'] and retained['count'] is None, 'future attempt already has a grant/effect')
|
|
if server['state'] == 'accepted' and retained['attempts'] == [dict(state='accepted',grant_id='')]: break
|
|
require(time.monotonic() < deadline, 'future offer was not durably accepted');time.sleep(2)
|
|
checkpoint.update(accepted=server,native_accepted=retained,stage='accepted_unstarted');save()
|
|
checkpoint['network_fault'] = native('block',not_before=job['payload']['not_before'],helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'active_offline';save()
|
|
channel = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
|
resume = cloud.contract.timestamp(job['payload']['start_before'] if args.case == 'expired_offline' else job['payload']['not_before'])+dt.timedelta(seconds=10)
|
|
if args.case == 'revoked_before_start': browser('revoke');checkpoint['ui_revoked'] = True;save()
|
|
sequence = checkpoint['network_fault']['after']['sequence']; windows = []
|
|
while cloud.contract.now() < resume:
|
|
retained = native('state',helper_ledger_sha256=ledger_sha); counts = gate(job['job_id'])
|
|
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
|
require(retained['agent_service']['ActiveState'] == 'active' and retained['agent_service']['MainPID'] == checkpoint['network_fault']['after']['agent_service']['MainPID']
|
|
and not retained['consumed'] and retained['count'] is None and counts['permits'] == 0 and observed['last_contact_at'] == channel['last_contact_at'], 'active offline window had a grant/effect/contact or agent exited')
|
|
windows.append(dict(observed_at=cloud.contract.now().isoformat(),sequence=retained['sequence'],grants=counts['permits'],count=retained['count'],active=True))
|
|
checkpoint['offline_window'] = windows;save();time.sleep(5)
|
|
require(windows and windows[-1]['sequence'] > sequence, 'running offline agent did not persist reconnect attempts')
|
|
checkpoint['server_offline_tail'] = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation']))
|
|
require(checkpoint['server_offline_tail']['grants'] == 0 and not checkpoint['server_offline_tail']['start_reserved'], 'no start immediately before network restoration')
|
|
checkpoint['network_restored'] = native('restore',helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'network_restored';save()
|
|
checkpoint['healthy_reconnect'] = healthy(channel['connection_generation'])
|
|
deadline = time.monotonic()+100
|
|
while True:
|
|
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
|
if result['state'] in cloud.contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
|
|
require(time.monotonic() < deadline, 'terminal native result missing; no retry');time.sleep(2)
|
|
expected = {'offline_wait':'succeeded','expired_offline':'expired','revoked_before_start':'cancelled'}[args.case]
|
|
require(result['state'] == expected, 'actual terminal state differs from bounded case')
|
|
native_final = native('state',helper_ledger_sha256=ledger_sha)
|
|
wanted = 1 if args.case == 'offline_wait' else 0
|
|
final_gate = gate(job['job_id']); require([final_gate[k] for k in ('jobs','outbox','permits')] == [1,1,wanted] and final_gate['unresolved'] == 0, 'exact job/outbox/start-grant counts')
|
|
require(len(native_final['consumed']) == wanted and native_final['count'] == ('1\n' if wanted else None), 'exact native consumption and stored effect counter')
|
|
if wanted: require(result['result']['exit_code'] == 0 and bytes(result['result']['stdout']) == b'RMM_OFFLINE_COUNT=1\n' and result['result']['evidence']['postconditions_passed'], 'native one-effect postcondition')
|
|
if args.case == 'offline_wait':
|
|
observed_at = cloud.contract.now()
|
|
require(observed_at < min(cloud.contract.timestamp(plan['expires_at']),cloud.contract.timestamp(job['payload']['start_before'])), 'original submit idempotency must be observed while its plan is valid')
|
|
submit_request = dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier)
|
|
same = backend.call('operations.submit',submit_request)
|
|
after_submit = gate(job['job_id'])
|
|
counts = lambda observed: {k:observed[k] for k in ('jobs','outbox','permits')}
|
|
require(same['operation_id'] == pending['operation_id'] and counts(final_gate) == counts(after_submit) == dict(jobs=1,outbox=1,permits=1), 'same original submit must reuse its operation without new executable artifacts')
|
|
receipt = dict(verified=True,case=args.case,operation_id=pending['operation_id'],request=submit_request,
|
|
response_operation_id=same['operation_id'],before_counts=counts(final_gate),after_counts=counts(after_submit),observed_at=observed_at.isoformat())
|
|
cloud.contract.write_private(folder/'idempotency-receipt.json',receipt)
|
|
checkpoint['idempotency_receipt'] = receipt;save()
|
|
audit = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='audit',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],operation_id=pending['operation_id']))
|
|
require(audit == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']), 'exact interactive approval audit')
|
|
details = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation'],after=checkpoint['server_offline_tail']['observed_at']))
|
|
require(details['grants'] == wanted and len(details['starts']) == wanted and details['start_reserved'] == bool(wanted), 'retained start reservation evidence')
|
|
if wanted: require(details['fresh_signed_sync_before_start'], 'fresh healthy signed reconnect must precede the one start reservation')
|
|
if args.case == 'revoked_before_start':
|
|
require(details['approval_state'] == 'revoked' and details['revoked_authorizations'] == 1 and details['revocation'] == dict(principal=checkpoint['expected_principal'],origin='dev_test_browser_simulation',simulation=True,interactive_session=True,plan_id=plan['plan_id'],operation_id=pending['operation_id']), 'actual interactive plan revocation audit')
|
|
checkpoint.update(result=result,native_final=native_final,postapproval=final_gate,audit=audit,server_evidence=details,stage='native_dev_test_accepted',verified=True);save()
|
|
print('PASS Cloud '+args.case+': active offline BPF verified, WebUI gate, exact grants/effect='+str(wanted))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except Exception as error:
|
|
print('CLOUD_OFFLINE_INCOMPLETE:'+ (str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
|
raise SystemExit(2) from None
|