Record preserved Linux full-maintenance runtime startup
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-14 17:57:04 +01:00
parent 7ef068fff2
commit cfd32eac72
5 changed files with 704 additions and 15 deletions
+257
View File
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""One preserving continuation of the installed, never-started Linux Full pair.
No stop, swap, install, rollback, SQL write, job or approval. Parent executes after
review; absent original restart intent and exact fresh pre-start CAS required.
"""
import argparse
import base64
import hashlib
import inspect
import json
import os
from pathlib import Path
import uuid
ROOT=Path(__file__).resolve().parents[1]
INSTALL_SHA='667b56f7e5f085db249f27a7094919c7c0e6055dabc9f0600cb261c34760cd26'
if hashlib.sha256((ROOT/'deploy/script_console_install.py').read_bytes()).hexdigest()!=INSTALL_SHA:
raise RuntimeError('Original install source must stay unchanged')
import script_console_install as old
support=old.support
ORIGINAL=ROOT/'devstate/script-console-install/2e27c312-58cc-444d-92a6-863a11ac6ffd'
ORIGINAL_SHA='8aec7bf7df13bc4968f47ca1554c813406409530613ce95241e74a81204cd02e'
READBACK=ORIGINAL/'readback-36f94d52-caf4-4b63-ac07-827edda4f806.json'
READBACK_SHA='b8964cd12ffd57226081980edfa1d33c99718fcb470979a29aa02a8af8fbd9ba'
STATE=ROOT/'devstate/script-console-continue'
def require(value,code):
if not value:raise RuntimeError(code)
def guards():
require(support.sha(ROOT/'deploy/script_console_install.py')==INSTALL_SHA
and support.sha(ROOT/'deploy/script_console_bootstrap.py')==old.PREFLIGHT_SHA
and support.sha(ORIGINAL/'intent.json')==ORIGINAL_SHA and support.sha(READBACK)==READBACK_SHA,
'exact original install/readback/source')
old.pre.native.verify_support()
manifest=support.load(old.pre.native.PACK/'manifest.json')
require(support.sha(old.pre.native.PACK/'manifest.json')==old.pre.native.MANIFEST_SHA
and support.sha(old.pre.NATIVE)==old.pre.NATIVE_SHA and len(manifest['source_files'])==158
and all(support.sha(ROOT/n)==h for n,h in manifest['source_files'].items()),'unchanged158 inputs and native Linux84')
def continue_vm(request):
import base64
import datetime as dt
import fcntl
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import stat
import subprocess
import traceback
import uuid
os.umask(0o077)
p=request['original'];ident=p['id'];recovery=str(uuid.UUID(request['id']))
require(recovery==request['id'] and ident=='2e27c312-58cc-444d-92a6-863a11ac6ffd'
and request['action'] in ('continue','read') and os.geteuid()==0 and os.uname().machine=='x86_64','fixed root continuation')
case=Path('/var/lib/rmm-bootstrap-history')/('script-console-v1-'+ident)
# Exact reviewed original helpers are copied below, no installation branch.
def encode(v):return json.dumps(v,sort_keys=True,separators=(',',':')).encode()
def sha(b):return hashlib.sha256(b).hexdigest()
def fsync_dir(path):
fd=os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try:os.fsync(fd)
finally:os.close(fd)
def create(path,raw,mode=0o600,gid=0):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as out:
out.write(raw);out.flush();os.fchmod(out.fileno(),mode);os.fchown(out.fileno(),0,gid);os.fsync(out.fileno())
fsync_dir(path.parent)
def safe(path,directory=False,owner=0):
m=path.lstat();require(not path.is_symlink() and m.st_uid==owner and not m.st_mode&0o022
and (stat.S_ISDIR(m.st_mode) if directory else stat.S_ISREG(m.st_mode) and m.st_nlink==1),'safe owned path')
return m
def run(args,**kwargs):return subprocess.run(args,capture_output=True,check=True,timeout=25,**kwargs).stdout
def svc(name):
return dict(line.split('=',1) for line in run(['systemctl','show',name,'-p','ActiveState','-p','MainPID','-p','UnitFileState']).decode().splitlines())
def retained_files():
# Never open private device keys. Retain all file identities, sizes,
# modes and change timestamps; SQL content has its own full row witness.
value={}
for root in (Path('/var/lib/rmm-agent-remote'),Path('/var/lib/rmm-helper')):
for path in [root,*sorted(root.rglob('*'))]:
m=path.lstat();require(not path.is_symlink() and (stat.S_ISDIR(m.st_mode) or stat.S_ISREG(m.st_mode)),'no unexpected retained state object')
if str(path).endswith(('-journal','-wal','-shm')):raise RuntimeError('Settled journal required')
value[str(path)]=[m.st_dev,m.st_ino,m.st_uid,m.st_gid,m.st_mode,m.st_size,m.st_mtime_ns,m.st_ctime_ns]
return value
def sql_witness():
result={}
for name,identity in p['preflight']['database_inodes'].items():
path=Path(name);m=path.lstat();require([m.st_dev,m.st_ino]==identity and not path.is_symlink(),'preserved DB identity')
require(all(not os.path.lexists(str(path)+suffix) for suffix in ('-wal','-shm','-journal')),'no live SQLite sidecars')
db=sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
try:
db.execute('PRAGMA query_only=ON');db.execute('BEGIN')
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','retained SQL integrity')
tables=dict(db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"))
rows={}
for table in tables:
require(table.replace('_','').isalnum(),'fixed SQLite identifier')
rows[table]=[sha(encode([dict(blob_sha256=sha(v),bytes=len(v)) if isinstance(v,bytes) else v for v in row]))
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')]
result[name]=dict(schema=tables,rows=rows,identity=identity)
if path.name=='journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'no running job')
if path.parent.name=='rmm-helper':
current=db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
exact={r[0]:sha(json.dumps([dict(sha256=sha(v),size=len(v)) if isinstance(v,bytes) else v for v in r],separators=(',',':')).encode()) for r in current}
require(exact==p['preflight']['root_rows'] and len(exact)==18 and all(r[5] is not None for r in current),'exact retained Root18')
finally:db.close()
return result
def description(path):
if not os.path.lexists(path):return dict(present=False)
m=path.lstat();require(not path.is_symlink() and stat.S_ISREG(m.st_mode),'fixed regular public file')
return dict(present=True,device=m.st_dev,inode=m.st_ino,uid=m.st_uid,gid=m.st_gid,
mode=stat.S_IMODE(m.st_mode),bytes=m.st_size,symlink=False,regular=True,sha256=sha(path.read_bytes()))
def receipt(path):
safe(path);require(path.stat().st_size<=8*1024*1024,'bounded private receipt')
return json.loads(path.read_bytes())
def services():
value={}
for name in ('rmm-helper-cloud.service','rmm-full-maintenance-helper.service','rmm-agent-remote.service','rmm-apt-helper.service'):
reply=subprocess.run(['systemctl','show',name,'-p','LoadState','-p','ActiveState','-p','SubState','-p','MainPID','-p','UnitFileState'],capture_output=True,timeout=25)
value[name]=dict(exit_code=reply.returncode,properties=dict(line.split('=',1) for line in reply.stdout.decode().splitlines() if '=' in line))
return value
safe(case,True);safe(case/'intent.json')
require((case/'intent.json').read_bytes()==encode(p),'complete original remote intent')
marker=case/'continuation-start-intent.json';result_path=case/'continuation-result.json'
if request['action']=='read':
return dict(id=recovery,readback_only=True,original_case=ident,services=services(),
original_report_historical=receipt(case/'report.json'),
original_restart_intent_present=os.path.lexists(case/'restart-intent.json'),
continuation_entered=os.path.lexists(marker),continuation_intent=receipt(marker) if os.path.lexists(marker) else None,
historical_continuation_result=receipt(result_path) if os.path.lexists(result_path) else None,
current_files={name:description(Path(path)) for name,path in (('agent','/usr/local/libexec/rmm-agent-remote'),('helper','/usr/local/libexec/rmm-privileged-helper'))},
state_writes=False,rmm_jobs=False,approvals=False,retry_allowed=False)
gate=request['fresh_server_gate']
require(gate['closed'] is True and gate['counts']==dict(jobs=30,outbox=30,permits=26,unresolved=0)
and gate['policy_sha256']==p['central_policy_sha256'],'fresh central scope still closed')
observed_at=dt.datetime.fromisoformat(gate['observed_at'])
require(observed_at.utcoffset() is not None and -5<=(dt.datetime.now(dt.timezone.utc)-observed_at).total_seconds()<=60,'fresh bounded server gate')
require(not os.path.lexists(case/'restart-intent.json') and not os.path.lexists(marker)
and not os.path.lexists(result_path),'no prior original or continuation start')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same boot before continuation')
observed=request['original_readback']
for name,expected in observed['markers'].items():
current=description(case/(name+'.json'))
require(current=={k:v for k,v in expected.items() if k!='historical_content'},'original markers retained exactly')
require(description(case/'report.json')=={k:v for k,v in observed['historical_report'].items() if k!='historical_content'},'original failed report retained exactly')
report=receipt(case/'report.json')
require(report['stage']=='incomplete_manual_review' and report['preserved_before_restart'] is True
and report['completed_preserving_bootstrap'] is False and report['full_job_accepted'] is False,'exact installed-before-first-start phase')
states=services()
for name,item in states.items():
require(item['exit_code']==0 and item['properties']==observed['services'][name]['properties']
and item['properties']['ActiveState']=='inactive' and item['properties']['MainPID']=='0','all four services still inactive')
binaries={'agent':Path('/usr/local/libexec/rmm-agent-remote'),'helper':Path('/usr/local/libexec/rmm-privileged-helper')}
for key,path in binaries.items():
require(description(path)==observed['files'][key+'_current'] and sha(path.read_bytes())==p['artifacts'][key]['sha256'],'same qualified installed image')
require(description(case/(key+'-previous'))==observed['files'][key+'_previous'],'same preserved old image')
require(not os.path.lexists(case/(key+'-next')),'no leftover image exchange')
for name,expected in p['baseline']['controls'].items():
if name in [str(x) for x in binaries.values()]:continue
path=Path(name);m=path.lstat()
require(not path.is_symlink() and dict(device=m.st_dev,inode=m.st_ino,mode=stat.S_IMODE(m.st_mode),uid=m.st_uid,gid=m.st_gid,bytes=m.st_size,sha256=sha(path.read_bytes()))==expected,'all General/APT/package controls unchanged')
# GeneralPin/Identity serde declaration order is the original inspector's
# order. The historical report uses sorted object keys and is not the raw
# pin-file encoding. Reconstruct the original exact compact bytes explicitly.
full_pin={k:report['full_pin'][k] for k in ('schema_version','profile','identity','policy_sha256','helper_binary_sha256')}
full_pin['identity']={k:full_pin['identity'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version','signing_key_id')}
new_files={
'/etc/rmm-remote/helper-full-maintenance.json':(base64.b64decode(p['baseline']['full_config_candidate']['bytes_b64'],validate=True),0o600,0),
'/etc/rmm-remote/full-maintenance-helper-proof.json':(json.dumps(full_pin,separators=(',',':')).encode(),0o640,984),
'/etc/systemd/system/rmm-full-maintenance-helper.service':(base64.b64decode(p['unit_bytes_b64'],validate=True),0o644,0),
'/etc/systemd/system/rmm-agent-remote.service.d/60-full-maintenance.conf':(b'[Service]\nEnvironment=RMM_FULL_MAINTENANCE_HELPER_SOCKET=/run/rmm-helper-maintenance/control.sock\nEnvironment=RMM_FULL_MAINTENANCE_POLICY_FILE=/etc/rmm-remote/full-maintenance-helper-proof.json\n',0o644,0)}
for name,(raw,mode,gid) in new_files.items():
path=Path(name);m=safe(path)
require(m.st_gid==gid and stat.S_IMODE(m.st_mode)==mode and path.read_bytes()==raw,'exact new Full control bytes/modes')
env={'PATH':'/usr/sbin:/usr/bin:/sbin:/bin','LANG':'C','LC_ALL':'C','RMM_HELPER_CONFIG':'/etc/rmm-remote/helper-full-maintenance.json'}
pin=json.loads(run([str(binaries['helper']),'--inspect-full-maintenance-profile'],env=env))
require(pin==report['full_pin'] and pin['helper_binary_sha256']==p['artifacts']['helper']['sha256']
and pin['identity']==p['general_pin']['identity'],'actual protected Full image/policy/identity')
env['RMM_HELPER_CONFIG']='/etc/rmm-remote/helper-general.json'
require(json.loads(run([str(binaries['helper']),'--inspect-general-profile'],env=env))==p['general_pin'],'preserved General profile')
names=('User','Group','NoNewPrivileges','PrivateNetwork','PrivateTmp','ProtectSystem','ProtectHome',
'ProtectKernelTunables','ProtectKernelModules','ProtectControlGroups','RestrictSUIDSGID',
'RestrictAddressFamilies','SystemCallFilter','RootDirectory','RootImage','NeedDaemonReload','FragmentPath','DropInPaths')
effective=dict(line.split('=',1) for line in run(['systemctl','show','rmm-full-maintenance-helper.service',*[a for n in names for a in ('-p',n)]]).decode().splitlines())
require(effective['User']=='root' and effective['Group']=='rmm-agent'
and all(effective[n]=='no' for n in names[2:11])
and all(effective[n] in ('','~') for n in names[11:13])
and effective['RootDirectory']==effective['RootImage']==effective['DropInPaths']==''
and effective['NeedDaemonReload']=='no' and effective['FragmentPath']=='/etc/systemd/system/rmm-full-maintenance-helper.service','exact unrestricted loaded Fullunit; empty allow/deny lists only')
locks=[]
for path in (Path('/var/lib/rmm-agent-remote/agent.lock'),Path('/var/lib/rmm-helper/executor.lock')):
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(fd)
try:
before_sql=receipt(case/'sql-before.json');before_files=receipt(case/'files-before.json')
require(sql_witness()==before_sql and retained_files()==before_files,'all original SQL rows/inodes/PKI metadata preserved before first start')
require(services()==states and not os.path.lexists(case/'restart-intent.json'),'fresh final stopped-state CAS')
first_start_not_before=dt.datetime.now(dt.timezone.utc).isoformat()
create(marker,encode(dict(id=recovery,original_case=ident,runner_sha256=request['runner_sha256'],first_start_not_before=first_start_not_before,
original_report_sha256=description(case/'report.json')['sha256'],sql_before_sha256=sha((case/'sql-before.json').read_bytes()),
files_before_sha256=sha((case/'files-before.json').read_bytes()),first_start_once=True,no_retry=True,
fresh_server_gate=gate,effective_full_unit=effective)))
finally:
for fd in reversed(locks):fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
outcome=dict(id=recovery,original_case=ident,first_start_entered=True,completed=False,first_start_not_before=first_start_not_before,
preserved_before_first_start=True,signed_sync_verified=False,full_job_accepted=False,
direct_product_state_writes=False,rmm_jobs=False,approvals=False,package_effects=False,reboot_requested=False,started=[])
try:
for name,key,uid in (('rmm-helper-cloud.service','general',0),('rmm-full-maintenance-helper.service','full',0),('rmm-agent-remote.service','agent',993)):
run(['systemctl','start',name]);state=svc(name);require(state['ActiveState']=='active' and int(state['MainPID'])>0,'exact first service start active')
pid=int(state['MainPID']);proc=Path('/proc',str(pid));ticks=(proc/'stat').read_text().rsplit(')',1)[1].split()[19]
status=(proc/'status').read_text().splitlines();uids=next(x.split()[1:] for x in status if x.startswith('Uid:'));gids=next(x.split()[1:] for x in status if x.startswith('Gid:'))
image=sha((proc/'exe').read_bytes())
require(uids==[str(uid)]*4 and gids==['984']*4 and image==p['artifacts']['agent' if key=='agent' else 'helper']['sha256']
and (proc/'stat').read_text().rsplit(')',1)[1].split()[19]==ticks and svc(name)['MainPID']==str(pid),'actual same new process/code/account')
evidence=dict(name=name,pid=pid,start_ticks=ticks,sha256=image,uids=uids,gids=gids)
create(case/('continuation-'+key+'-started.json'),encode(evidence));outcome['started'].append(evidence)
require(svc('rmm-apt-helper.service')['ActiveState']=='inactive' and svc('rmm-apt-helper.service')['UnitFileState']=='disabled','APT remains closed')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same final boot')
outcome['completed']=True
except Exception as e:outcome.update(error_class=type(e).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(e.__traceback__)])
create(result_path,encode(outcome));return outcome
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('mode',choices=('prepare-local','continue','read'));parser.add_argument('--case');args=parser.parse_args()
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True);guards()
if args.mode=='prepare-local':
require(args.case is None,'new local-only continuation intent');case=str(uuid.uuid4());folder=STATE/case;folder.mkdir(mode=0o700)
intent=dict(id=case,runner_sha256=support.sha(__file__),install_sha256=INSTALL_SHA,original=support.load(ORIGINAL/'intent.json'),original_readback=support.load(READBACK))
support.save(folder/'intent.json',intent)
else:
case=str(uuid.UUID(args.case));require(case==args.case,'original canonical recovery case');folder=STATE/case;intent=support.load(folder/'intent.json')
require(intent==dict(id=case,runner_sha256=support.sha(__file__),install_sha256=INSTALL_SHA,original=support.load(ORIGINAL/'intent.json'),original_readback=support.load(READBACK)),'exact original same-source recovery JSON')
guards()
if args.mode=='continue':
support.save(folder/'continue-entered-local.json',dict(start_once=True,no_retry=True))
gate=old.rpc(old.server_prefix(),old.server_gate,intent['original'],folder);support.save(folder/'server-gate.json',gate)
guards();result=old.rpc(support.access_settings(old.pre.ACCESS_ROOT,'linux'),continue_vm,dict(intent,action='continue',fresh_server_gate=gate),folder)
support.save(folder/'continue-reply.json',result)
require(result['completed'] is True and result['preserved_before_first_start'] is True
and result['signed_sync_verified'] is False and result['full_job_accepted'] is False
and result['rmm_jobs'] is False and result['approvals'] is False,'continuation incomplete; readback only')
else:
result=old.rpc(support.access_settings(old.pre.ACCESS_ROOT,'linux'),continue_vm,dict(intent,action='read'),folder)
support.save(folder/('readback-'+str(uuid.uuid4())+'.json'),result)
print(json.dumps(dict(case_id=case,mode=args.mode,intent=str(folder/'intent.json'),intent_sha256=support.sha(folder/'intent.json'),target_called=args.mode!='prepare-local',rmm_jobs=False,approvals=False)))
if __name__=='__main__':main()