Preserve timely reboot evidence and document blocked original reconciliation
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -243,6 +243,30 @@ def legacy_helper_unit_metadata(payload):
|
||||
clock_query_filters=[line for line in lines if line in ('SystemCallFilter=adjtimex clock_adjtime','SystemCallFilter=adjtimex')])
|
||||
|
||||
|
||||
def start_helper_after_boot_clock(payload):
|
||||
"""Start the existing helper after its clock guard failed; never a job."""
|
||||
import contextlib
|
||||
import sqlite3
|
||||
unit=Path('/etc/systemd/system/rmm-helper-cloud.service')
|
||||
require(hashlib.sha256(unit.read_bytes()).hexdigest()==payload['unit_sha256']
|
||||
and hashlib.sha256(Path('/usr/local/libexec/rmm-privileged-helper').read_bytes()).hexdigest()==payload['helper_sha256'],'same qualified helper/unit')
|
||||
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
|
||||
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
require(boot!=payload['old_boot_id'],'actual guest already rebooted; never request another reboot')
|
||||
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and
|
||||
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy ledger')
|
||||
rows=db.execute('SELECT c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
|
||||
require(len(rows)==1 and rows[0][:2]==(payload['job_id'],payload['attempt_id'])
|
||||
and json.loads(rows[0][2])['state']=='awaiting_reboot','only exact existing reboot may await reconciliation')
|
||||
before=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()
|
||||
require(before==b'failed','known boot-start failure only')
|
||||
subprocess.run(['systemctl','start','rmm-helper-cloud.service'],capture_output=True,check=True,timeout=20)
|
||||
active=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()==b'active'
|
||||
return dict(started_existing_helper=active,new_boot_id=boot,ledger_reset=False,
|
||||
jobs_created=False,grants_created=False,reboot_requested=False)
|
||||
|
||||
|
||||
def disable_legacy_helper_autostart(payload):
|
||||
"""Disable only the stopped historical dev helper; never stop/start a unit."""
|
||||
import contextlib
|
||||
|
||||
@@ -254,7 +254,11 @@ def close(ident):
|
||||
m=cloud.contract;folder=STATE/str(uuid.UUID(ident));require(m.protected_json(folder/'result.json')['verified'],'normal verified reconnect required before rollback')
|
||||
session=m.protected_json(folder/'before.json');target=session['target'];opened=m.protected_json(folder/'policy-open.json');enabled=m.protected_json(folder/'helper-enabled.json')
|
||||
require(evidence(target,'gate')['unresolved']==0,'all effects closed')
|
||||
closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
|
||||
earlier=folder/'policy-closed-for-upgrade.json'
|
||||
if earlier.exists():
|
||||
closed=m.protected_json(earlier);current=cloud.bootstrap.rpc('server.conf',server_policy,dict(action='state'))
|
||||
require(current['closed'] and current['policy_sha256']==closed['policy_sha256'],'exact earlier deny-only closure retained')
|
||||
else:closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
|
||||
m.write_private(folder/'policy-closed.json',closed)
|
||||
restored=cloud.bootstrap.rpc('linux-vm.conf',vm_policy,dict(session['pins'],id=ident,enabled=False,config_sha256=enabled['config_sha256'],pin_sha256=enabled['pin_sha256'],original_backup=enabled['backup']))
|
||||
require(restored['general_pin']==session['before']['general_pin'] and restored['config_sha256']==session['before']['config_sha256'] and restored['pin_sha256']==session['before']['pin_sha256'] and restored['preserved_state'] and not restored['reboot_requested'],'original non-reboot policy/pin bytes restored without another reboot')
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserve the one consumed reboot while updating its read-only verifier."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import cloud_reboot_acceptance as reboot
|
||||
import cloud_server_upgrade as upgrade
|
||||
|
||||
cloud=reboot.cloud
|
||||
require=cloud.contract.require
|
||||
|
||||
|
||||
def agent_transport(payload):
|
||||
import contextlib
|
||||
import sqlite3
|
||||
require(payload['action'] in ('stop','start'),'own agent transport direction')
|
||||
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'qualified agent')
|
||||
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
require(boot==payload['new_boot_id'] and boot!=payload['old_boot_id'],'already completed one guest boot')
|
||||
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy Root ledger')
|
||||
rows=db.execute('SELECT c.grant_id,c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
|
||||
require(len(rows)==1 and rows[0][:3]==(payload['grant_id'],payload['job_id'],payload['attempt_id']) and json.loads(rows[0][3])['state']=='awaiting_reboot','only original consumed reboot may await proof')
|
||||
subprocess.run(['systemctl',payload['action'],'rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
|
||||
return dict(action=payload['action'],new_boot_id=boot,jobs_created=False,grants_created=False,reboot_requested=False)
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--id',required=True);p.add_argument('--reboot-id',required=True);args=p.parse_args()
|
||||
import uuid
|
||||
candidate=str(uuid.UUID(args.id));sid=str(uuid.UUID(args.reboot_id));folder=reboot.STATE/sid
|
||||
session=cloud.contract.protected_json(folder/'before.json');checkpoint=cloud.contract.protected_json(folder/'checkpoint.json');job=checkpoint['job'];target=session['target']
|
||||
require(sid=='d015c3ef-f3aa-4522-a3c2-fcb6099df17b' and job['job_id']=='8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa','one exact already approved reboot')
|
||||
gate=lambda:reboot.evidence(target,'gate')
|
||||
require(gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1),'only known consumed reboot unresolved')
|
||||
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
|
||||
grant='25a4bf3f-772c-487d-825d-73bcbbc418fd'
|
||||
require(current['state'] in ('unknown','awaiting_reboot') and current['result']['grant_id']==grant,'same immutable reboot attempt')
|
||||
scope=dict(session['pins'],job_id=job['job_id'],attempt_id=job['attempt_id'])
|
||||
before=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
|
||||
require(before['root_unresolved']==1 and before['other_unresolved']==0 and before['selected_root_grants']==[grant]
|
||||
and before['boot_id']!=session['old_boot_id'] and len(before['root_rows'])==len(session['before']['root_rows'])+1
|
||||
and all(before['root_rows'].get(k)==v for k,v in session['before']['root_rows'].items()),'all old Root rows and exact consumed reboot preserved')
|
||||
audit=reboot.evidence(target,'audit',operation_id=checkpoint['operation_id'])
|
||||
require(audit==dict(origin='dev_test_browser_simulation',simulation=True,principal='simulated-testprincipal-a',plan_hash=checkpoint['plan']['plan_hash']),'original actual UI approval')
|
||||
status=cloud.bootstrap.rpc('server.conf',upgrade.status,dict(id=candidate))
|
||||
require(status['finished'] and status['exit_code']==0 and status['source_unchanged'] and status['runtime_tag_unchanged'],'qualified separate server image')
|
||||
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(action='state'))
|
||||
if not policy['closed']:
|
||||
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(id=sid,action='close',previous_sha256=policy['policy_sha256']))
|
||||
cloud.contract.write_private(folder/'policy-closed-for-upgrade.json',policy)
|
||||
transport=dict(agent_id=target['agent_id'],agent_sha256=session['pins']['agent_sha256'],old_boot_id=session['old_boot_id'],new_boot_id=before['boot_id'],grant_id=grant,job_id=job['job_id'],attempt_id=job['attempt_id'])
|
||||
stopped=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='stop'))
|
||||
frozen=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
|
||||
require(frozen['root_rows']==before['root_rows'] and frozen['database_inodes']==before['database_inodes'] and frozen['services']['rmm-agent-remote.service']['ActiveState']=='inactive','quiesced agent and original evidence retained')
|
||||
cloud.contract.write_private(folder/'proof-upgrade-before.json',dict(candidate=candidate,gate=gate(),current=current,native=frozen,transport=stopped,policy=policy))
|
||||
stream=io.BytesIO()
|
||||
with tarfile.open(fileobj=stream,mode='w:') as archive:
|
||||
for path in sorted((cloud.ROOT/'web/dist').rglob('*')):
|
||||
if path.is_file():
|
||||
require(not path.is_symlink(),'built UI symlink');archive.add(path,arcname=str(path.relative_to(cloud.ROOT/'web/dist')),recursive=False)
|
||||
raw=stream.getvalue()
|
||||
result=cloud.bootstrap.rpc('server.conf',upgrade.deploy,dict(id=candidate,closed_policy_sha256=policy['policy_sha256'],previous_ui_index_sha256='a3513f928e20ea11d53f1cd27e8735d911555e1c66d5d97ca8bdefaff60e71fa',web_tar=base64.b64encode(raw).decode(),web_sha256=hashlib.sha256(raw).hexdigest()))
|
||||
after=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
|
||||
require(result['deployed'] and gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1) and after['root_rows']==frozen['root_rows'] and after['database_inodes']==frozen['database_inodes'],'deployment preserves exact unresolved effect before normal resumption')
|
||||
cloud.contract.write_private(folder/'proof-upgrade-deployed.json',dict(verified=True,candidate=candidate,result=result,native=after,counts=gate()))
|
||||
resumed=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='start'))
|
||||
cloud.contract.write_private(folder/'proof-upgrade-resumed.json',resumed)
|
||||
print('PASS exact reboot evidence preserved during qualified server upgrade; ordinary signed resumption required; no second reboot')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
message=str(error);print(message if message.startswith('remote bootstrap incomplete: ') else 'REBOOT_PROOF_UPGRADE_INCOMPLETE:'+type(error).__name__)
|
||||
raise SystemExit(2) from None
|
||||
@@ -11,6 +11,17 @@ import cloud_mvp_acceptance as cloud
|
||||
require = cloud.contract.require
|
||||
|
||||
|
||||
def reboot_receipts(payload):
|
||||
tenant=str(uuid.UUID(payload['tenant_id']));agent=str(uuid.UUID(payload['agent_id']));job=str(uuid.UUID(payload['job_id']))
|
||||
require(tenant=='11111111-1111-4111-8111-111111111111','fixed development tenant')
|
||||
query=("BEGIN READ ONLY; SELECT json_build_object('observed_at',now(),'installation',(SELECT json_build_object('boot_id',connection_boot_id,'generation',connection_generation,'last_contact_at',last_contact_at,'sync_ready',sync_ready,'helper_ready',helper_sync_ready) FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'),"
|
||||
"'job',(SELECT json_build_object('state',state,'revision',revision,'result',result) FROM jobs WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||
"'grant',(SELECT json_build_object('grant_id',grant_id,'created_at',created_at) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||
"'inventories',(SELECT COALESCE(json_agg(x),'[]'::json) FROM (SELECT message_id,boot_id,observed_at,received_at,source_id FROM observations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' ORDER BY received_at DESC LIMIT 20) x)); COMMIT;")
|
||||
root=Path('/opt/rust-discovery');cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
||||
|
||||
|
||||
def server(payload):
|
||||
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
||||
|
||||
Reference in New Issue
Block a user