Preserve timely reboot evidence and document blocked original reconciliation
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 22:03:24 +01:00
parent 80906a3eb6
commit 4102b97fb9
9 changed files with 387 additions and 13 deletions
+24
View File
@@ -243,6 +243,30 @@ def legacy_helper_unit_metadata(payload):
clock_query_filters=[line for line in lines if line in ('SystemCallFilter=adjtimex clock_adjtime','SystemCallFilter=adjtimex')])
def start_helper_after_boot_clock(payload):
"""Start the existing helper after its clock guard failed; never a job."""
import contextlib
import sqlite3
unit=Path('/etc/systemd/system/rmm-helper-cloud.service')
require(hashlib.sha256(unit.read_bytes()).hexdigest()==payload['unit_sha256']
and hashlib.sha256(Path('/usr/local/libexec/rmm-privileged-helper').read_bytes()).hexdigest()==payload['helper_sha256'],'same qualified helper/unit')
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
require(boot!=payload['old_boot_id'],'actual guest already rebooted; never request another reboot')
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy ledger')
rows=db.execute('SELECT c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows)==1 and rows[0][:2]==(payload['job_id'],payload['attempt_id'])
and json.loads(rows[0][2])['state']=='awaiting_reboot','only exact existing reboot may await reconciliation')
before=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()
require(before==b'failed','known boot-start failure only')
subprocess.run(['systemctl','start','rmm-helper-cloud.service'],capture_output=True,check=True,timeout=20)
active=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()==b'active'
return dict(started_existing_helper=active,new_boot_id=boot,ledger_reset=False,
jobs_created=False,grants_created=False,reboot_requested=False)
def disable_legacy_helper_autostart(payload):
"""Disable only the stopped historical dev helper; never stop/start a unit."""
import contextlib
+5 -1
View File
@@ -254,7 +254,11 @@ def close(ident):
m=cloud.contract;folder=STATE/str(uuid.UUID(ident));require(m.protected_json(folder/'result.json')['verified'],'normal verified reconnect required before rollback')
session=m.protected_json(folder/'before.json');target=session['target'];opened=m.protected_json(folder/'policy-open.json');enabled=m.protected_json(folder/'helper-enabled.json')
require(evidence(target,'gate')['unresolved']==0,'all effects closed')
closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
earlier=folder/'policy-closed-for-upgrade.json'
if earlier.exists():
closed=m.protected_json(earlier);current=cloud.bootstrap.rpc('server.conf',server_policy,dict(action='state'))
require(current['closed'] and current['policy_sha256']==closed['policy_sha256'],'exact earlier deny-only closure retained')
else:closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
m.write_private(folder/'policy-closed.json',closed)
restored=cloud.bootstrap.rpc('linux-vm.conf',vm_policy,dict(session['pins'],id=ident,enabled=False,config_sha256=enabled['config_sha256'],pin_sha256=enabled['pin_sha256'],original_backup=enabled['backup']))
require(restored['general_pin']==session['before']['general_pin'] and restored['config_sha256']==session['before']['config_sha256'] and restored['pin_sha256']==session['before']['pin_sha256'] and restored['preserved_state'] and not restored['reboot_requested'],'original non-reboot policy/pin bytes restored without another reboot')
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Preserve the one consumed reboot while updating its read-only verifier."""
import argparse
import base64
import hashlib
import io
import json
from pathlib import Path
import tarfile
import cloud_reboot_acceptance as reboot
import cloud_server_upgrade as upgrade
cloud=reboot.cloud
require=cloud.contract.require
def agent_transport(payload):
import contextlib
import sqlite3
require(payload['action'] in ('stop','start'),'own agent transport direction')
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'qualified agent')
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
require(boot==payload['new_boot_id'] and boot!=payload['old_boot_id'],'already completed one guest boot')
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy Root ledger')
rows=db.execute('SELECT c.grant_id,c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows)==1 and rows[0][:3]==(payload['grant_id'],payload['job_id'],payload['attempt_id']) and json.loads(rows[0][3])['state']=='awaiting_reboot','only original consumed reboot may await proof')
subprocess.run(['systemctl',payload['action'],'rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
return dict(action=payload['action'],new_boot_id=boot,jobs_created=False,grants_created=False,reboot_requested=False)
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--id',required=True);p.add_argument('--reboot-id',required=True);args=p.parse_args()
import uuid
candidate=str(uuid.UUID(args.id));sid=str(uuid.UUID(args.reboot_id));folder=reboot.STATE/sid
session=cloud.contract.protected_json(folder/'before.json');checkpoint=cloud.contract.protected_json(folder/'checkpoint.json');job=checkpoint['job'];target=session['target']
require(sid=='d015c3ef-f3aa-4522-a3c2-fcb6099df17b' and job['job_id']=='8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa','one exact already approved reboot')
gate=lambda:reboot.evidence(target,'gate')
require(gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1),'only known consumed reboot unresolved')
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
grant='25a4bf3f-772c-487d-825d-73bcbbc418fd'
require(current['state'] in ('unknown','awaiting_reboot') and current['result']['grant_id']==grant,'same immutable reboot attempt')
scope=dict(session['pins'],job_id=job['job_id'],attempt_id=job['attempt_id'])
before=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
require(before['root_unresolved']==1 and before['other_unresolved']==0 and before['selected_root_grants']==[grant]
and before['boot_id']!=session['old_boot_id'] and len(before['root_rows'])==len(session['before']['root_rows'])+1
and all(before['root_rows'].get(k)==v for k,v in session['before']['root_rows'].items()),'all old Root rows and exact consumed reboot preserved')
audit=reboot.evidence(target,'audit',operation_id=checkpoint['operation_id'])
require(audit==dict(origin='dev_test_browser_simulation',simulation=True,principal='simulated-testprincipal-a',plan_hash=checkpoint['plan']['plan_hash']),'original actual UI approval')
status=cloud.bootstrap.rpc('server.conf',upgrade.status,dict(id=candidate))
require(status['finished'] and status['exit_code']==0 and status['source_unchanged'] and status['runtime_tag_unchanged'],'qualified separate server image')
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(action='state'))
if not policy['closed']:
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(id=sid,action='close',previous_sha256=policy['policy_sha256']))
cloud.contract.write_private(folder/'policy-closed-for-upgrade.json',policy)
transport=dict(agent_id=target['agent_id'],agent_sha256=session['pins']['agent_sha256'],old_boot_id=session['old_boot_id'],new_boot_id=before['boot_id'],grant_id=grant,job_id=job['job_id'],attempt_id=job['attempt_id'])
stopped=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='stop'))
frozen=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
require(frozen['root_rows']==before['root_rows'] and frozen['database_inodes']==before['database_inodes'] and frozen['services']['rmm-agent-remote.service']['ActiveState']=='inactive','quiesced agent and original evidence retained')
cloud.contract.write_private(folder/'proof-upgrade-before.json',dict(candidate=candidate,gate=gate(),current=current,native=frozen,transport=stopped,policy=policy))
stream=io.BytesIO()
with tarfile.open(fileobj=stream,mode='w:') as archive:
for path in sorted((cloud.ROOT/'web/dist').rglob('*')):
if path.is_file():
require(not path.is_symlink(),'built UI symlink');archive.add(path,arcname=str(path.relative_to(cloud.ROOT/'web/dist')),recursive=False)
raw=stream.getvalue()
result=cloud.bootstrap.rpc('server.conf',upgrade.deploy,dict(id=candidate,closed_policy_sha256=policy['policy_sha256'],previous_ui_index_sha256='a3513f928e20ea11d53f1cd27e8735d911555e1c66d5d97ca8bdefaff60e71fa',web_tar=base64.b64encode(raw).decode(),web_sha256=hashlib.sha256(raw).hexdigest()))
after=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
require(result['deployed'] and gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1) and after['root_rows']==frozen['root_rows'] and after['database_inodes']==frozen['database_inodes'],'deployment preserves exact unresolved effect before normal resumption')
cloud.contract.write_private(folder/'proof-upgrade-deployed.json',dict(verified=True,candidate=candidate,result=result,native=after,counts=gate()))
resumed=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='start'))
cloud.contract.write_private(folder/'proof-upgrade-resumed.json',resumed)
print('PASS exact reboot evidence preserved during qualified server upgrade; ordinary signed resumption required; no second reboot')
if __name__=='__main__':
try:main()
except Exception as error:
message=str(error);print(message if message.startswith('remote bootstrap incomplete: ') else 'REBOOT_PROOF_UPGRADE_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+11
View File
@@ -11,6 +11,17 @@ import cloud_mvp_acceptance as cloud
require = cloud.contract.require
def reboot_receipts(payload):
tenant=str(uuid.UUID(payload['tenant_id']));agent=str(uuid.UUID(payload['agent_id']));job=str(uuid.UUID(payload['job_id']))
require(tenant=='11111111-1111-4111-8111-111111111111','fixed development tenant')
query=("BEGIN READ ONLY; SELECT json_build_object('observed_at',now(),'installation',(SELECT json_build_object('boot_id',connection_boot_id,'generation',connection_generation,'last_contact_at',last_contact_at,'sync_ready',sync_ready,'helper_ready',helper_sync_ready) FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'),"
"'job',(SELECT json_build_object('state',state,'revision',revision,'result',result) FROM jobs WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'grant',(SELECT json_build_object('grant_id',grant_id,'created_at',created_at) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'inventories',(SELECT COALESCE(json_agg(x),'[]'::json) FROM (SELECT message_id,boot_id,observed_at,received_at,source_id FROM observations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' ORDER BY received_at DESC LIMIT 20) x)); COMMIT;")
root=Path('/opt/rust-discovery');cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
def server(payload):
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')