258 lines
20 KiB
Python
258 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""One preserving continuation of the installed, never-started Linux Full pair.
|
|
|
|
No stop, swap, install, rollback, SQL write, job or approval. Parent executes after
|
|
review; absent original restart intent and exact fresh pre-start CAS required.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import uuid
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
INSTALL_SHA='667b56f7e5f085db249f27a7094919c7c0e6055dabc9f0600cb261c34760cd26'
|
|
if hashlib.sha256((ROOT/'deploy/script_console_install.py').read_bytes()).hexdigest()!=INSTALL_SHA:
|
|
raise RuntimeError('Original install source must stay unchanged')
|
|
import script_console_install as old
|
|
support=old.support
|
|
ORIGINAL=ROOT/'devstate/script-console-install/2e27c312-58cc-444d-92a6-863a11ac6ffd'
|
|
ORIGINAL_SHA='8aec7bf7df13bc4968f47ca1554c813406409530613ce95241e74a81204cd02e'
|
|
READBACK=ORIGINAL/'readback-36f94d52-caf4-4b63-ac07-827edda4f806.json'
|
|
READBACK_SHA='b8964cd12ffd57226081980edfa1d33c99718fcb470979a29aa02a8af8fbd9ba'
|
|
STATE=ROOT/'devstate/script-console-continue'
|
|
|
|
|
|
def require(value,code):
|
|
if not value:raise RuntimeError(code)
|
|
|
|
|
|
def guards():
|
|
require(support.sha(ROOT/'deploy/script_console_install.py')==INSTALL_SHA
|
|
and support.sha(ROOT/'deploy/script_console_bootstrap.py')==old.PREFLIGHT_SHA
|
|
and support.sha(ORIGINAL/'intent.json')==ORIGINAL_SHA and support.sha(READBACK)==READBACK_SHA,
|
|
'exact original install/readback/source')
|
|
old.pre.native.verify_support()
|
|
manifest=support.load(old.pre.native.PACK/'manifest.json')
|
|
require(support.sha(old.pre.native.PACK/'manifest.json')==old.pre.native.MANIFEST_SHA
|
|
and support.sha(old.pre.NATIVE)==old.pre.NATIVE_SHA and len(manifest['source_files'])==158
|
|
and all(support.sha(ROOT/n)==h for n,h in manifest['source_files'].items()),'unchanged158 inputs and native Linux84')
|
|
|
|
|
|
def continue_vm(request):
|
|
import base64
|
|
import datetime as dt
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import stat
|
|
import subprocess
|
|
import traceback
|
|
import uuid
|
|
os.umask(0o077)
|
|
p=request['original'];ident=p['id'];recovery=str(uuid.UUID(request['id']))
|
|
require(recovery==request['id'] and ident=='2e27c312-58cc-444d-92a6-863a11ac6ffd'
|
|
and request['action'] in ('continue','read') and os.geteuid()==0 and os.uname().machine=='x86_64','fixed root continuation')
|
|
case=Path('/var/lib/rmm-bootstrap-history')/('script-console-v1-'+ident)
|
|
# Exact reviewed original helpers are copied below, no installation branch.
|
|
def encode(v):return json.dumps(v,sort_keys=True,separators=(',',':')).encode()
|
|
def sha(b):return hashlib.sha256(b).hexdigest()
|
|
def fsync_dir(path):
|
|
fd=os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
|
try:os.fsync(fd)
|
|
finally:os.close(fd)
|
|
def create(path,raw,mode=0o600,gid=0):
|
|
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
|
|
with os.fdopen(fd,'wb') as out:
|
|
out.write(raw);out.flush();os.fchmod(out.fileno(),mode);os.fchown(out.fileno(),0,gid);os.fsync(out.fileno())
|
|
fsync_dir(path.parent)
|
|
def safe(path,directory=False,owner=0):
|
|
m=path.lstat();require(not path.is_symlink() and m.st_uid==owner and not m.st_mode&0o022
|
|
and (stat.S_ISDIR(m.st_mode) if directory else stat.S_ISREG(m.st_mode) and m.st_nlink==1),'safe owned path')
|
|
return m
|
|
def run(args,**kwargs):return subprocess.run(args,capture_output=True,check=True,timeout=25,**kwargs).stdout
|
|
def svc(name):
|
|
return dict(line.split('=',1) for line in run(['systemctl','show',name,'-p','ActiveState','-p','MainPID','-p','UnitFileState']).decode().splitlines())
|
|
def retained_files():
|
|
# Never open private device keys. Retain all file identities, sizes,
|
|
# modes and change timestamps; SQL content has its own full row witness.
|
|
value={}
|
|
for root in (Path('/var/lib/rmm-agent-remote'),Path('/var/lib/rmm-helper')):
|
|
for path in [root,*sorted(root.rglob('*'))]:
|
|
m=path.lstat();require(not path.is_symlink() and (stat.S_ISDIR(m.st_mode) or stat.S_ISREG(m.st_mode)),'no unexpected retained state object')
|
|
if str(path).endswith(('-journal','-wal','-shm')):raise RuntimeError('Settled journal required')
|
|
value[str(path)]=[m.st_dev,m.st_ino,m.st_uid,m.st_gid,m.st_mode,m.st_size,m.st_mtime_ns,m.st_ctime_ns]
|
|
return value
|
|
def sql_witness():
|
|
result={}
|
|
for name,identity in p['preflight']['database_inodes'].items():
|
|
path=Path(name);m=path.lstat();require([m.st_dev,m.st_ino]==identity and not path.is_symlink(),'preserved DB identity')
|
|
require(all(not os.path.lexists(str(path)+suffix) for suffix in ('-wal','-shm','-journal')),'no live SQLite sidecars')
|
|
db=sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
|
|
try:
|
|
db.execute('PRAGMA query_only=ON');db.execute('BEGIN')
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','retained SQL integrity')
|
|
tables=dict(db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"))
|
|
rows={}
|
|
for table in tables:
|
|
require(table.replace('_','').isalnum(),'fixed SQLite identifier')
|
|
rows[table]=[sha(encode([dict(blob_sha256=sha(v),bytes=len(v)) if isinstance(v,bytes) else v for v in row]))
|
|
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')]
|
|
result[name]=dict(schema=tables,rows=rows,identity=identity)
|
|
if path.name=='journal.sqlite':
|
|
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'no running job')
|
|
if path.parent.name=='rmm-helper':
|
|
current=db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
|
|
exact={r[0]:sha(json.dumps([dict(sha256=sha(v),size=len(v)) if isinstance(v,bytes) else v for v in r],separators=(',',':')).encode()) for r in current}
|
|
require(exact==p['preflight']['root_rows'] and len(exact)==18 and all(r[5] is not None for r in current),'exact retained Root18')
|
|
finally:db.close()
|
|
return result
|
|
def description(path):
|
|
if not os.path.lexists(path):return dict(present=False)
|
|
m=path.lstat();require(not path.is_symlink() and stat.S_ISREG(m.st_mode),'fixed regular public file')
|
|
return dict(present=True,device=m.st_dev,inode=m.st_ino,uid=m.st_uid,gid=m.st_gid,
|
|
mode=stat.S_IMODE(m.st_mode),bytes=m.st_size,symlink=False,regular=True,sha256=sha(path.read_bytes()))
|
|
def receipt(path):
|
|
safe(path);require(path.stat().st_size<=8*1024*1024,'bounded private receipt')
|
|
return json.loads(path.read_bytes())
|
|
def services():
|
|
value={}
|
|
for name in ('rmm-helper-cloud.service','rmm-full-maintenance-helper.service','rmm-agent-remote.service','rmm-apt-helper.service'):
|
|
reply=subprocess.run(['systemctl','show',name,'-p','LoadState','-p','ActiveState','-p','SubState','-p','MainPID','-p','UnitFileState'],capture_output=True,timeout=25)
|
|
value[name]=dict(exit_code=reply.returncode,properties=dict(line.split('=',1) for line in reply.stdout.decode().splitlines() if '=' in line))
|
|
return value
|
|
safe(case,True);safe(case/'intent.json')
|
|
require((case/'intent.json').read_bytes()==encode(p),'complete original remote intent')
|
|
marker=case/'continuation-start-intent.json';result_path=case/'continuation-result.json'
|
|
if request['action']=='read':
|
|
return dict(id=recovery,readback_only=True,original_case=ident,services=services(),
|
|
original_report_historical=receipt(case/'report.json'),
|
|
original_restart_intent_present=os.path.lexists(case/'restart-intent.json'),
|
|
continuation_entered=os.path.lexists(marker),continuation_intent=receipt(marker) if os.path.lexists(marker) else None,
|
|
historical_continuation_result=receipt(result_path) if os.path.lexists(result_path) else None,
|
|
current_files={name:description(Path(path)) for name,path in (('agent','/usr/local/libexec/rmm-agent-remote'),('helper','/usr/local/libexec/rmm-privileged-helper'))},
|
|
state_writes=False,rmm_jobs=False,approvals=False,retry_allowed=False)
|
|
gate=request['fresh_server_gate']
|
|
require(gate['closed'] is True and gate['counts']==dict(jobs=30,outbox=30,permits=26,unresolved=0)
|
|
and gate['policy_sha256']==p['central_policy_sha256'],'fresh central scope still closed')
|
|
observed_at=dt.datetime.fromisoformat(gate['observed_at'])
|
|
require(observed_at.utcoffset() is not None and -5<=(dt.datetime.now(dt.timezone.utc)-observed_at).total_seconds()<=60,'fresh bounded server gate')
|
|
require(not os.path.lexists(case/'restart-intent.json') and not os.path.lexists(marker)
|
|
and not os.path.lexists(result_path),'no prior original or continuation start')
|
|
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same boot before continuation')
|
|
observed=request['original_readback']
|
|
for name,expected in observed['markers'].items():
|
|
current=description(case/(name+'.json'))
|
|
require(current=={k:v for k,v in expected.items() if k!='historical_content'},'original markers retained exactly')
|
|
require(description(case/'report.json')=={k:v for k,v in observed['historical_report'].items() if k!='historical_content'},'original failed report retained exactly')
|
|
report=receipt(case/'report.json')
|
|
require(report['stage']=='incomplete_manual_review' and report['preserved_before_restart'] is True
|
|
and report['completed_preserving_bootstrap'] is False and report['full_job_accepted'] is False,'exact installed-before-first-start phase')
|
|
states=services()
|
|
for name,item in states.items():
|
|
require(item['exit_code']==0 and item['properties']==observed['services'][name]['properties']
|
|
and item['properties']['ActiveState']=='inactive' and item['properties']['MainPID']=='0','all four services still inactive')
|
|
binaries={'agent':Path('/usr/local/libexec/rmm-agent-remote'),'helper':Path('/usr/local/libexec/rmm-privileged-helper')}
|
|
for key,path in binaries.items():
|
|
require(description(path)==observed['files'][key+'_current'] and sha(path.read_bytes())==p['artifacts'][key]['sha256'],'same qualified installed image')
|
|
require(description(case/(key+'-previous'))==observed['files'][key+'_previous'],'same preserved old image')
|
|
require(not os.path.lexists(case/(key+'-next')),'no leftover image exchange')
|
|
for name,expected in p['baseline']['controls'].items():
|
|
if name in [str(x) for x in binaries.values()]:continue
|
|
path=Path(name);m=path.lstat()
|
|
require(not path.is_symlink() and dict(device=m.st_dev,inode=m.st_ino,mode=stat.S_IMODE(m.st_mode),uid=m.st_uid,gid=m.st_gid,bytes=m.st_size,sha256=sha(path.read_bytes()))==expected,'all General/APT/package controls unchanged')
|
|
# GeneralPin/Identity serde declaration order is the original inspector's
|
|
# order. The historical report uses sorted object keys and is not the raw
|
|
# pin-file encoding. Reconstruct the original exact compact bytes explicitly.
|
|
full_pin={k:report['full_pin'][k] for k in ('schema_version','profile','identity','policy_sha256','helper_binary_sha256')}
|
|
full_pin['identity']={k:full_pin['identity'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version','signing_key_id')}
|
|
new_files={
|
|
'/etc/rmm-remote/helper-full-maintenance.json':(base64.b64decode(p['baseline']['full_config_candidate']['bytes_b64'],validate=True),0o600,0),
|
|
'/etc/rmm-remote/full-maintenance-helper-proof.json':(json.dumps(full_pin,separators=(',',':')).encode(),0o640,984),
|
|
'/etc/systemd/system/rmm-full-maintenance-helper.service':(base64.b64decode(p['unit_bytes_b64'],validate=True),0o644,0),
|
|
'/etc/systemd/system/rmm-agent-remote.service.d/60-full-maintenance.conf':(b'[Service]\nEnvironment=RMM_FULL_MAINTENANCE_HELPER_SOCKET=/run/rmm-helper-maintenance/control.sock\nEnvironment=RMM_FULL_MAINTENANCE_POLICY_FILE=/etc/rmm-remote/full-maintenance-helper-proof.json\n',0o644,0)}
|
|
for name,(raw,mode,gid) in new_files.items():
|
|
path=Path(name);m=safe(path)
|
|
require(m.st_gid==gid and stat.S_IMODE(m.st_mode)==mode and path.read_bytes()==raw,'exact new Full control bytes/modes')
|
|
env={'PATH':'/usr/sbin:/usr/bin:/sbin:/bin','LANG':'C','LC_ALL':'C','RMM_HELPER_CONFIG':'/etc/rmm-remote/helper-full-maintenance.json'}
|
|
pin=json.loads(run([str(binaries['helper']),'--inspect-full-maintenance-profile'],env=env))
|
|
require(pin==report['full_pin'] and pin['helper_binary_sha256']==p['artifacts']['helper']['sha256']
|
|
and pin['identity']==p['general_pin']['identity'],'actual protected Full image/policy/identity')
|
|
env['RMM_HELPER_CONFIG']='/etc/rmm-remote/helper-general.json'
|
|
require(json.loads(run([str(binaries['helper']),'--inspect-general-profile'],env=env))==p['general_pin'],'preserved General profile')
|
|
names=('User','Group','NoNewPrivileges','PrivateNetwork','PrivateTmp','ProtectSystem','ProtectHome',
|
|
'ProtectKernelTunables','ProtectKernelModules','ProtectControlGroups','RestrictSUIDSGID',
|
|
'RestrictAddressFamilies','SystemCallFilter','RootDirectory','RootImage','NeedDaemonReload','FragmentPath','DropInPaths')
|
|
effective=dict(line.split('=',1) for line in run(['systemctl','show','rmm-full-maintenance-helper.service',*[a for n in names for a in ('-p',n)]]).decode().splitlines())
|
|
require(effective['User']=='root' and effective['Group']=='rmm-agent'
|
|
and all(effective[n]=='no' for n in names[2:11])
|
|
and all(effective[n] in ('','~') for n in names[11:13])
|
|
and effective['RootDirectory']==effective['RootImage']==effective['DropInPaths']==''
|
|
and effective['NeedDaemonReload']=='no' and effective['FragmentPath']=='/etc/systemd/system/rmm-full-maintenance-helper.service','exact unrestricted loaded Fullunit; empty allow/deny lists only')
|
|
locks=[]
|
|
for path in (Path('/var/lib/rmm-agent-remote/agent.lock'),Path('/var/lib/rmm-helper/executor.lock')):
|
|
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(fd)
|
|
try:
|
|
before_sql=receipt(case/'sql-before.json');before_files=receipt(case/'files-before.json')
|
|
require(sql_witness()==before_sql and retained_files()==before_files,'all original SQL rows/inodes/PKI metadata preserved before first start')
|
|
require(services()==states and not os.path.lexists(case/'restart-intent.json'),'fresh final stopped-state CAS')
|
|
first_start_not_before=dt.datetime.now(dt.timezone.utc).isoformat()
|
|
create(marker,encode(dict(id=recovery,original_case=ident,runner_sha256=request['runner_sha256'],first_start_not_before=first_start_not_before,
|
|
original_report_sha256=description(case/'report.json')['sha256'],sql_before_sha256=sha((case/'sql-before.json').read_bytes()),
|
|
files_before_sha256=sha((case/'files-before.json').read_bytes()),first_start_once=True,no_retry=True,
|
|
fresh_server_gate=gate,effective_full_unit=effective)))
|
|
finally:
|
|
for fd in reversed(locks):fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
|
|
outcome=dict(id=recovery,original_case=ident,first_start_entered=True,completed=False,first_start_not_before=first_start_not_before,
|
|
preserved_before_first_start=True,signed_sync_verified=False,full_job_accepted=False,
|
|
direct_product_state_writes=False,rmm_jobs=False,approvals=False,package_effects=False,reboot_requested=False,started=[])
|
|
try:
|
|
for name,key,uid in (('rmm-helper-cloud.service','general',0),('rmm-full-maintenance-helper.service','full',0),('rmm-agent-remote.service','agent',993)):
|
|
run(['systemctl','start',name]);state=svc(name);require(state['ActiveState']=='active' and int(state['MainPID'])>0,'exact first service start active')
|
|
pid=int(state['MainPID']);proc=Path('/proc',str(pid));ticks=(proc/'stat').read_text().rsplit(')',1)[1].split()[19]
|
|
status=(proc/'status').read_text().splitlines();uids=next(x.split()[1:] for x in status if x.startswith('Uid:'));gids=next(x.split()[1:] for x in status if x.startswith('Gid:'))
|
|
image=sha((proc/'exe').read_bytes())
|
|
require(uids==[str(uid)]*4 and gids==['984']*4 and image==p['artifacts']['agent' if key=='agent' else 'helper']['sha256']
|
|
and (proc/'stat').read_text().rsplit(')',1)[1].split()[19]==ticks and svc(name)['MainPID']==str(pid),'actual same new process/code/account')
|
|
evidence=dict(name=name,pid=pid,start_ticks=ticks,sha256=image,uids=uids,gids=gids)
|
|
create(case/('continuation-'+key+'-started.json'),encode(evidence));outcome['started'].append(evidence)
|
|
require(svc('rmm-apt-helper.service')['ActiveState']=='inactive' and svc('rmm-apt-helper.service')['UnitFileState']=='disabled','APT remains closed')
|
|
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'same final boot')
|
|
outcome['completed']=True
|
|
except Exception as e:outcome.update(error_class=type(e).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(e.__traceback__)])
|
|
create(result_path,encode(outcome));return outcome
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('mode',choices=('prepare-local','continue','read'));parser.add_argument('--case');args=parser.parse_args()
|
|
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True);guards()
|
|
if args.mode=='prepare-local':
|
|
require(args.case is None,'new local-only continuation intent');case=str(uuid.uuid4());folder=STATE/case;folder.mkdir(mode=0o700)
|
|
intent=dict(id=case,runner_sha256=support.sha(__file__),install_sha256=INSTALL_SHA,original=support.load(ORIGINAL/'intent.json'),original_readback=support.load(READBACK))
|
|
support.save(folder/'intent.json',intent)
|
|
else:
|
|
case=str(uuid.UUID(args.case));require(case==args.case,'original canonical recovery case');folder=STATE/case;intent=support.load(folder/'intent.json')
|
|
require(intent==dict(id=case,runner_sha256=support.sha(__file__),install_sha256=INSTALL_SHA,original=support.load(ORIGINAL/'intent.json'),original_readback=support.load(READBACK)),'exact original same-source recovery JSON')
|
|
guards()
|
|
if args.mode=='continue':
|
|
support.save(folder/'continue-entered-local.json',dict(start_once=True,no_retry=True))
|
|
gate=old.rpc(old.server_prefix(),old.server_gate,intent['original'],folder);support.save(folder/'server-gate.json',gate)
|
|
guards();result=old.rpc(support.access_settings(old.pre.ACCESS_ROOT,'linux'),continue_vm,dict(intent,action='continue',fresh_server_gate=gate),folder)
|
|
support.save(folder/'continue-reply.json',result)
|
|
require(result['completed'] is True and result['preserved_before_first_start'] is True
|
|
and result['signed_sync_verified'] is False and result['full_job_accepted'] is False
|
|
and result['rmm_jobs'] is False and result['approvals'] is False,'continuation incomplete; readback only')
|
|
else:
|
|
result=old.rpc(support.access_settings(old.pre.ACCESS_ROOT,'linux'),continue_vm,dict(intent,action='read'),folder)
|
|
support.save(folder/('readback-'+str(uuid.uuid4())+'.json'),result)
|
|
print(json.dumps(dict(case_id=case,mode=args.mode,intent=str(folder/'intent.json'),intent_sha256=support.sha(folder/'intent.json'),target_called=args.mode!='prepare-local',rmm_jobs=False,approvals=False)))
|
|
|
|
|
|
if __name__=='__main__':main()
|