Files
rust-discovery/deploy/script_console_finish_start.py
T
kevin_janssen cfd32eac72
Rust development slice / linux (push) Failing after 6s
Record preserved Linux full-maintenance runtime startup
2026-11-14 17:57:04 +01:00

301 lines
23 KiB
Python

#!/usr/bin/env python3
"""Finish the two never-started services; never start/stop/restart General.
Exact original installation + first-start receipts preserved. Each remaining
service has its own durable intent before one start, with a bounded native probe.
"""
import argparse
import hashlib
import inspect
import json
import os
from pathlib import Path
import uuid
ROOT=Path(__file__).resolve().parents[1]
PRIOR_SHA='c0c0ecf1e1bcb2d2f9ce0bcf6973aa430e9e28ba86823ca5ad0954744a589238'
if hashlib.sha256((ROOT/'deploy/script_console_continue.py').read_bytes()).hexdigest()!=PRIOR_SHA:
raise RuntimeError('Original continuation source stays unchanged')
import script_console_continue as prior
old=prior.old
support=prior.support
PREVIOUS=ROOT/'devstate/script-console-continue/43dae370-6862-4cf4-a83c-0b2612af6c79'
PREVIOUS_INTENT_SHA='f9059e74b3abca44f49216cbde45e036ac4701354d76d0cc9cca8acc41aaff0c'
PREVIOUS_READBACK=PREVIOUS/'readback-b49d10b0-fda3-459a-b286-e6911c9c37eb.json'
PREVIOUS_READBACK_SHA='5aac963e78fa5e587d9137ddd7394f6fde19cd1cfedbbb8528517b0d15e18006'
STATE=ROOT/'devstate/script-console-finish-start'
def require(value,code):
if not value:raise RuntimeError(code)
def guards():
prior.guards()
require(support.sha(ROOT/'deploy/script_console_continue.py')==PRIOR_SHA
and support.sha(PREVIOUS/'intent.json')==PREVIOUS_INTENT_SHA
and support.sha(PREVIOUS_READBACK)==PREVIOUS_READBACK_SHA,'exact original continuation/source/readback')
def finish_vm(request):
import base64
import datetime as dt
import fcntl
import time
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import stat
import subprocess
import traceback
import uuid
os.umask(0o077)
p=request['original'];ident=p['id'];recovery=str(uuid.UUID(request['id']))
require(recovery==request['id'] and ident=='2e27c312-58cc-444d-92a6-863a11ac6ffd'
and request['action'] in ('finish','read') and os.geteuid()==0 and os.uname().machine=='x86_64','fixed root continuation')
case=Path('/var/lib/rmm-bootstrap-history')/('script-console-v1-'+ident)
# Exact reviewed original helpers are copied below, no installation branch.
def encode(v):return json.dumps(v,sort_keys=True,separators=(',',':')).encode()
def sha(b):return hashlib.sha256(b).hexdigest()
def fsync_dir(path):
fd=os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try:os.fsync(fd)
finally:os.close(fd)
def create(path,raw,mode=0o600,gid=0):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as out:
out.write(raw);out.flush();os.fchmod(out.fileno(),mode);os.fchown(out.fileno(),0,gid);os.fsync(out.fileno())
fsync_dir(path.parent)
def safe(path,directory=False,owner=0):
m=path.lstat();require(not path.is_symlink() and m.st_uid==owner and not m.st_mode&0o022
and (stat.S_ISDIR(m.st_mode) if directory else stat.S_ISREG(m.st_mode) and m.st_nlink==1),'safe owned path')
return m
def run(args,**kwargs):return subprocess.run(args,capture_output=True,check=True,timeout=25,**kwargs).stdout
def svc(name):
return dict(line.split('=',1) for line in run(['systemctl','show',name,'-p','ActiveState','-p','MainPID','-p','UnitFileState']).decode().splitlines())
def retained_files():
# Never open private device keys. Retain all file identities, sizes,
# modes and change timestamps; SQL content has its own full row witness.
value={}
for root in (Path('/var/lib/rmm-agent-remote'),Path('/var/lib/rmm-helper')):
for path in [root,*sorted(root.rglob('*'))]:
m=path.lstat();require(not path.is_symlink() and (stat.S_ISDIR(m.st_mode) or stat.S_ISREG(m.st_mode)),'no unexpected retained state object')
if str(path).endswith(('-journal','-wal','-shm')):raise RuntimeError('Settled journal required')
value[str(path)]=[m.st_dev,m.st_ino,m.st_uid,m.st_gid,m.st_mode,m.st_size,m.st_mtime_ns,m.st_ctime_ns]
return value
def sql_witness():
result={}
for name,identity in p['preflight']['database_inodes'].items():
path=Path(name);m=path.lstat();require([m.st_dev,m.st_ino]==identity and not path.is_symlink(),'preserved DB identity')
require(all(not os.path.lexists(str(path)+suffix) for suffix in ('-wal','-shm','-journal')),'no live SQLite sidecars')
db=sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
try:
db.execute('PRAGMA query_only=ON');db.execute('BEGIN')
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','retained SQL integrity')
tables=dict(db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"))
rows={}
for table in tables:
require(table.replace('_','').isalnum(),'fixed SQLite identifier')
rows[table]=[sha(encode([dict(blob_sha256=sha(v),bytes=len(v)) if isinstance(v,bytes) else v for v in row]))
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')]
result[name]=dict(schema=tables,rows=rows,identity=identity)
if path.name=='journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'no running job')
if path.parent.name=='rmm-helper':
current=db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
exact={r[0]:sha(json.dumps([dict(sha256=sha(v),size=len(v)) if isinstance(v,bytes) else v for v in r],separators=(',',':')).encode()) for r in current}
require(exact==p['preflight']['root_rows'] and len(exact)==18 and all(r[5] is not None for r in current),'exact retained Root18')
finally:db.close()
return result
def description(path):
if not os.path.lexists(path):return dict(present=False)
m=path.lstat();require(not path.is_symlink() and stat.S_ISREG(m.st_mode),'fixed regular public file')
return dict(present=True,device=m.st_dev,inode=m.st_ino,uid=m.st_uid,gid=m.st_gid,
mode=stat.S_IMODE(m.st_mode),bytes=m.st_size,symlink=False,regular=True,sha256=sha(path.read_bytes()))
def receipt(path):
safe(path);require(path.stat().st_size<=8*1024*1024,'bounded private receipt')
return json.loads(path.read_bytes())
def services():
value={}
for name in ('rmm-helper-cloud.service','rmm-full-maintenance-helper.service','rmm-agent-remote.service','rmm-apt-helper.service'):
reply=subprocess.run(['systemctl','show',name,'-p','LoadState','-p','ActiveState','-p','SubState','-p','MainPID','-p','UnitFileState'],capture_output=True,timeout=25)
value[name]=dict(exit_code=reply.returncode,properties=dict(line.split('=',1) for line in reply.stdout.decode().splitlines() if '=' in line))
return value
safe(case,True);safe(case/'intent.json')
require((case/'intent.json').read_bytes()==encode(p),'complete original remote intent')
marker=case/'finish-start-intent.json';result_path=case/'finish-result.json'
if request['action']=='read':
return dict(id=recovery,readback_only=True,original_case=ident,services=services(),
original_report_historical=receipt(case/'report.json'),
original_restart_intent_present=os.path.lexists(case/'restart-intent.json'),
finish_entered=os.path.lexists(marker),finish_intent=receipt(marker) if os.path.lexists(marker) else None,
first_continuation_report_historical=receipt(case/'continuation-result.json'),
historical_finish_result=receipt(result_path) if os.path.lexists(result_path) else None,
current_files={name:description(Path(path)) for name,path in (('agent','/usr/local/libexec/rmm-agent-remote'),('helper','/usr/local/libexec/rmm-privileged-helper'))},
state_writes=False,rmm_jobs=False,approvals=False,retry_allowed=False)
previous=request['first_continuation_readback']
require((case/'continuation-start-intent.json').read_bytes()==encode(previous['continuation_intent'])
and (case/'continuation-result.json').read_bytes()==encode(previous['historical_continuation_result']),
'original first-start marker/result retained byte-for-byte')
require(previous['historical_continuation_result']['completed'] is False
and previous['historical_continuation_result']['started']==[],'known first General probe stopped the sequence')
for label in ('general','full','agent'):
require(not os.path.lexists(case/('continuation-'+label+'-started.json'))
and not os.path.lexists(case/('finish-'+label+'-start-intent.json'))
and not os.path.lexists(case/('finish-'+label+'-started.json')),'no previous confirmed or remaining-service start')
def probe(name,uid):
state=svc(name);require(state['ActiveState']=='active' and int(state['MainPID'])>0,'service process not ready')
pid=int(state['MainPID']);proc=Path('/proc',str(pid));ticks=(proc/'stat').read_text().rsplit(')',1)[1].split()[19]
status=(proc/'status').read_text().splitlines();uids=next(x.split()[1:] for x in status if x.startswith('Uid:'));gids=next(x.split()[1:] for x in status if x.startswith('Gid:'))
image=sha((proc/'exe').read_bytes());key='agent' if name=='rmm-agent-remote.service' else 'helper'
require(uids==[str(uid)]*4 and gids==['984']*4 and image==p['artifacts'][key]['sha256']
and (proc/'stat').read_text().rsplit(')',1)[1].split()[19]==ticks and svc(name)['MainPID']==str(pid)
and Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same actual process/account/image/boot')
return dict(name=name,pid=pid,start_ticks=ticks,sha256=image,uids=uids,gids=gids)
def exact_general():
value=probe('rmm-helper-cloud.service',0)
require(value['pid']==65562 and value['start_ticks']=='7308498','original already-running General birth')
return value
def poll_started(name,uid):
deadline=time.monotonic()+5;first_birth=None;settled=None
while time.monotonic()<deadline:
state=svc(name);pid=int(state['MainPID'])
if pid>0:
try:birth=(pid,Path('/proc',str(pid),'stat').read_text().rsplit(')',1)[1].split()[19])
except FileNotFoundError:birth=None
if birth is not None:
if first_birth is None:first_birth=birth
require(birth==first_birth,'no service PID/birth replacement during first start')
try:value=probe(name,uid)
except (RuntimeError,FileNotFoundError,ProcessLookupError):value=None
if value is not None and value==settled:return value
settled=value;time.sleep(0.05)
raise RuntimeError('First-start process did not settle in five seconds; readback only')
gate=request['fresh_server_gate']
require(gate['closed'] is True and gate['counts']==dict(jobs=30,outbox=30,permits=26,unresolved=0)
and gate['policy_sha256']==p['central_policy_sha256'],'fresh central scope still closed')
observed_at=dt.datetime.fromisoformat(gate['observed_at'])
require(observed_at.utcoffset() is not None and -5<=(dt.datetime.now(dt.timezone.utc)-observed_at).total_seconds()<=60,'fresh bounded server gate')
require(not os.path.lexists(case/'restart-intent.json') and not os.path.lexists(marker)
and not os.path.lexists(result_path),'no prior original or continuation start')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same boot before continuation')
observed=request['original_readback']
for name,expected in observed['markers'].items():
current=description(case/(name+'.json'))
require(current=={k:v for k,v in expected.items() if k!='historical_content'},'original markers retained exactly')
require(description(case/'report.json')=={k:v for k,v in observed['historical_report'].items() if k!='historical_content'},'original failed report retained exactly')
report=receipt(case/'report.json')
require(report['stage']=='incomplete_manual_review' and report['preserved_before_restart'] is True
and report['completed_preserving_bootstrap'] is False and report['full_job_accepted'] is False,'exact installed-before-first-start phase')
states=services()
for name,item in states.items():
require(item['exit_code']==0 and item['properties']==previous['services'][name]['properties']
and ((name=='rmm-helper-cloud.service' and item['properties']['ActiveState']=='active' and item['properties']['MainPID']=='65562')
or (name!='rmm-helper-cloud.service' and item['properties']['ActiveState']=='inactive' and item['properties']['MainPID']=='0')),'exact General running and remaining services inactive')
general_before=exact_general()
binaries={'agent':Path('/usr/local/libexec/rmm-agent-remote'),'helper':Path('/usr/local/libexec/rmm-privileged-helper')}
for key,path in binaries.items():
require(description(path)==observed['files'][key+'_current'] and sha(path.read_bytes())==p['artifacts'][key]['sha256'],'same qualified installed image')
require(description(case/(key+'-previous'))==observed['files'][key+'_previous'],'same preserved old image')
require(not os.path.lexists(case/(key+'-next')),'no leftover image exchange')
for name,expected in p['baseline']['controls'].items():
if name in [str(x) for x in binaries.values()]:continue
path=Path(name);m=path.lstat()
require(not path.is_symlink() and dict(device=m.st_dev,inode=m.st_ino,mode=stat.S_IMODE(m.st_mode),uid=m.st_uid,gid=m.st_gid,bytes=m.st_size,sha256=sha(path.read_bytes()))==expected,'all General/APT/package controls unchanged')
# GeneralPin/Identity serde declaration order is the original inspector's
# order. The historical report uses sorted object keys and is not the raw
# pin-file encoding. Reconstruct the original exact compact bytes explicitly.
full_pin={k:report['full_pin'][k] for k in ('schema_version','profile','identity','policy_sha256','helper_binary_sha256')}
full_pin['identity']={k:full_pin['identity'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version','signing_key_id')}
new_files={
'/etc/rmm-remote/helper-full-maintenance.json':(base64.b64decode(p['baseline']['full_config_candidate']['bytes_b64'],validate=True),0o600,0),
'/etc/rmm-remote/full-maintenance-helper-proof.json':(json.dumps(full_pin,separators=(',',':')).encode(),0o640,984),
'/etc/systemd/system/rmm-full-maintenance-helper.service':(base64.b64decode(p['unit_bytes_b64'],validate=True),0o644,0),
'/etc/systemd/system/rmm-agent-remote.service.d/60-full-maintenance.conf':(b'[Service]\nEnvironment=RMM_FULL_MAINTENANCE_HELPER_SOCKET=/run/rmm-helper-maintenance/control.sock\nEnvironment=RMM_FULL_MAINTENANCE_POLICY_FILE=/etc/rmm-remote/full-maintenance-helper-proof.json\n',0o644,0)}
for name,(raw,mode,gid) in new_files.items():
path=Path(name);m=safe(path)
require(m.st_gid==gid and stat.S_IMODE(m.st_mode)==mode and path.read_bytes()==raw,'exact new Full control bytes/modes')
env={'PATH':'/usr/sbin:/usr/bin:/sbin:/bin','LANG':'C','LC_ALL':'C','RMM_HELPER_CONFIG':'/etc/rmm-remote/helper-full-maintenance.json'}
pin=json.loads(run([str(binaries['helper']),'--inspect-full-maintenance-profile'],env=env))
require(pin==report['full_pin'] and pin['helper_binary_sha256']==p['artifacts']['helper']['sha256']
and pin['identity']==p['general_pin']['identity'],'actual protected Full image/policy/identity')
env['RMM_HELPER_CONFIG']='/etc/rmm-remote/helper-general.json'
require(json.loads(run([str(binaries['helper']),'--inspect-general-profile'],env=env))==p['general_pin'],'preserved General profile')
names=('User','Group','NoNewPrivileges','PrivateNetwork','PrivateTmp','ProtectSystem','ProtectHome',
'ProtectKernelTunables','ProtectKernelModules','ProtectControlGroups','RestrictSUIDSGID',
'RestrictAddressFamilies','SystemCallFilter','RootDirectory','RootImage','NeedDaemonReload','FragmentPath','DropInPaths')
effective=dict(line.split('=',1) for line in run(['systemctl','show','rmm-full-maintenance-helper.service',*[a for n in names for a in ('-p',n)]]).decode().splitlines())
require(effective['User']=='root' and effective['Group']=='rmm-agent'
and all(effective[n]=='no' for n in names[2:11])
and all(effective[n] in ('','~') for n in names[11:13])
and effective['RootDirectory']==effective['RootImage']==effective['DropInPaths']==''
and effective['NeedDaemonReload']=='no' and effective['FragmentPath']=='/etc/systemd/system/rmm-full-maintenance-helper.service','exact unrestricted loaded Fullunit; empty allow/deny lists only')
locks=[]
for path in (Path('/var/lib/rmm-agent-remote/agent.lock'),Path('/var/lib/rmm-helper/executor.lock')):
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(fd)
try:
before_sql=receipt(case/'sql-before.json');before_files=receipt(case/'files-before.json')
require(sha((case/'sql-before.json').read_bytes())==previous['continuation_intent']['sql_before_sha256']
and sha((case/'files-before.json').read_bytes())==previous['continuation_intent']['files_before_sha256'],'exact original preserved state witnesses')
require(sql_witness()==before_sql and retained_files()==before_files,'all original SQL rows/inodes/PKI metadata preserved before first start')
require(services()==states and exact_general()==general_before
and not os.path.lexists(case/'restart-intent.json'),'fresh General-birth and remaining-stopped CAS')
first_start_not_before=previous['continuation_intent']['first_start_not_before']
remaining_start_not_before=dt.datetime.now(dt.timezone.utc).isoformat()
create(marker,encode(dict(id=recovery,original_case=ident,runner_sha256=request['runner_sha256'],first_start_not_before=first_start_not_before,remaining_start_not_before=remaining_start_not_before,
general_before=general_before,original_report_sha256=description(case/'report.json')['sha256'],sql_before_sha256=sha((case/'sql-before.json').read_bytes()),
files_before_sha256=sha((case/'files-before.json').read_bytes()),first_start_once=True,no_retry=True,
fresh_server_gate=gate,effective_full_unit=effective)))
finally:
for fd in reversed(locks):fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
outcome=dict(id=recovery,original_case=ident,first_start_entered=True,completed=False,first_start_not_before=first_start_not_before,remaining_start_not_before=remaining_start_not_before,
general_before=general_before,
preserved_before_first_start=True,signed_sync_verified=False,full_job_accepted=False,
direct_product_state_writes=False,rmm_jobs=False,approvals=False,package_effects=False,reboot_requested=False,started=[])
try:
for name,key,uid in (('rmm-full-maintenance-helper.service','full',0),('rmm-agent-remote.service','agent',993)):
require(exact_general()==general_before,'General remains the same process; never start it again')
create(case/('finish-'+key+'-start-intent.json'),encode(dict(id=recovery,name=name,start_once=True,no_retry=True,
observed_at=dt.datetime.now(dt.timezone.utc).isoformat())))
run(['systemctl','start',name])
evidence=poll_started(name,uid)
create(case/('finish-'+key+'-started.json'),encode(evidence));outcome['started'].append(evidence)
require(exact_general()==general_before,'same original General after remaining starts')
require(svc('rmm-apt-helper.service')['ActiveState']=='inactive' and svc('rmm-apt-helper.service')['UnitFileState']=='disabled','APT remains closed')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same final boot')
outcome['completed']=True
except Exception as e:outcome.update(error_class=type(e).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(e.__traceback__)])
create(result_path,encode(outcome));return outcome
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('mode',choices=('prepare-local','finish','read'));parser.add_argument('--case');args=parser.parse_args()
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True);guards()
before=support.load(PREVIOUS/'intent.json')
def expected(case):return dict(id=case,runner_sha256=support.sha(__file__),prior_sha256=PRIOR_SHA,
original=before['original'],original_readback=before['original_readback'],first_continuation_readback=support.load(PREVIOUS_READBACK))
if args.mode=='prepare-local':
require(args.case is None,'new local-only remaining-start intent');case=str(uuid.uuid4());folder=STATE/case;folder.mkdir(mode=0o700)
support.save(folder/'intent.json',expected(case))
else:
case=str(uuid.UUID(args.case));require(case==args.case,'original canonical finish case');folder=STATE/case;intent=support.load(folder/'intent.json')
require(intent==expected(case),'complete original same-source JSON for every finish/read')
guards()
if args.mode=='finish':
support.save(folder/'finish-entered-local.json',dict(start_once=True,no_retry=True))
gate=old.rpc(old.server_prefix(),old.server_gate,intent['original'],folder);support.save(folder/'server-gate.json',gate)
guards();result=old.rpc(support.access_settings(old.pre.ACCESS_ROOT,'linux'),finish_vm,dict(intent,action='finish',fresh_server_gate=gate),folder)
support.save(folder/'finish-reply.json',result)
require(result['completed'] is True and result['preserved_before_first_start'] is True
and result['signed_sync_verified'] is False and result['full_job_accepted'] is False
and result['rmm_jobs'] is False and result['approvals'] is False,'finish incomplete; readback only')
else:
result=old.rpc(support.access_settings(old.pre.ACCESS_ROOT,'linux'),finish_vm,dict(intent,action='read'),folder)
support.save(folder/('readback-'+str(uuid.uuid4())+'.json'),result)
print(json.dumps(dict(case_id=case,mode=args.mode,intent=str(folder/'intent.json'),intent_sha256=support.sha(folder/'intent.json'),target_called=args.mode!='prepare-local',rmm_jobs=False,approvals=False)))
if __name__=='__main__':main()