263 lines
21 KiB
Python
263 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Preserve the single consumed reboot while extending finite journal capacity.
|
|
|
|
No reset, deletion, package action, approval or additional reboot. The agent
|
|
remains stopped after installation; normal signed reconciliation is separate.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import uuid
|
|
|
|
import cloud_reboot_acceptance as reboot
|
|
|
|
cloud=reboot.cloud
|
|
require=cloud.contract.require
|
|
SESSION='d015c3ef-f3aa-4522-a3c2-fcb6099df17b'
|
|
GRANT='25a4bf3f-772c-487d-825d-73bcbbc418fd'
|
|
|
|
|
|
def capacity_maintenance(payload):
|
|
import contextlib
|
|
import fcntl
|
|
import sqlite3
|
|
import stat
|
|
import traceback
|
|
os.umask(0o077)
|
|
state=Path('/var/lib/rmm-agent-remote');root=Path('/var/lib/rmm-helper')
|
|
ledger=root/'consumption.sqlite';apt=Path('/var/lib/rmm-apt-transaction')
|
|
agent=Path('/usr/local/libexec/rmm-agent-remote');helper=Path('/usr/local/libexec/rmm-privileged-helper')
|
|
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
|
|
config=Path('/etc/rmm-remote/helper-general.json');pin=Path('/etc/rmm-remote/general-helper-proof.json')
|
|
def sha(raw):return hashlib.sha256(raw).hexdigest()
|
|
def run(args):return subprocess.run(args,capture_output=True,check=True,timeout=15).stdout
|
|
def fsync_dir(path):
|
|
fd=os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
|
try:os.fsync(fd)
|
|
finally:os.close(fd)
|
|
def create(path,raw,mode=0o600):
|
|
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
|
|
with os.fdopen(fd,'wb') as f:f.write(raw);f.flush();os.fchmod(f.fileno(),mode);os.fsync(f.fileno())
|
|
fsync_dir(path.parent)
|
|
def file_identity(path):
|
|
s=path.lstat();require(not path.is_symlink() and path.is_file(),'regular preserved file')
|
|
return dict(device=s.st_dev,inode=s.st_ino,size=s.st_size,mode=s.st_mode,uid=s.st_uid,gid=s.st_gid,sha256=sha(path.read_bytes()))
|
|
def manifest(path):
|
|
if not path.exists():return {'absent':True}
|
|
out={}
|
|
for p in [path,*sorted(path.rglob('*'))]:
|
|
s=p.lstat();require(not p.is_symlink() and (p.is_dir() or p.is_file()),'regular preserved tree')
|
|
out[str(p.relative_to(path))]=dict(device=s.st_dev,inode=s.st_ino,mode=s.st_mode,uid=s.st_uid,gid=s.st_gid,
|
|
sha256=sha(p.read_bytes()) if p.is_file() else None)
|
|
return out
|
|
def fingerprint(db):
|
|
h=hashlib.sha256()
|
|
for name,sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"):
|
|
require(name.replace('_','').isalnum(),'fixed table identifier')
|
|
h.update(json.dumps([name,sql],separators=(',',':')).encode())
|
|
for row in db.execute('SELECT * FROM "'+name+'" ORDER BY rowid'):
|
|
cells=[dict(blob_sha256=sha(v),length=len(v)) if isinstance(v,bytes) else v for v in row]
|
|
h.update(json.dumps(cells,separators=(',',':'),ensure_ascii=True).encode())
|
|
return h.hexdigest()
|
|
@contextlib.contextmanager
|
|
def agent_readonly(path):
|
|
owner=path.lstat();require(stat.S_ISREG(owner.st_mode) and owner.st_uid==993 and owner.st_nlink==1,'original agent database owner')
|
|
sidecars=[Path(str(path)+suffix) for suffix in ('-wal','-shm')]
|
|
existing={}
|
|
for p in sidecars:
|
|
if p.exists() or p.is_symlink():
|
|
s=p.lstat();require(stat.S_ISREG(s.st_mode) and s.st_nlink==1 and s.st_uid==owner.st_uid,'existing fixed agent sidecar owner')
|
|
existing[p]=(s.st_dev,s.st_ino,s.st_uid,s.st_gid)
|
|
try:
|
|
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db:yield db
|
|
finally:
|
|
require((path.stat().st_dev,path.stat().st_ino)==(owner.st_dev,owner.st_ino),'agent database inode unchanged')
|
|
for p in sidecars:
|
|
if p in existing:
|
|
s=p.lstat();require((s.st_dev,s.st_ino,s.st_uid,s.st_gid)==existing[p],'existing sidecar unchanged')
|
|
elif p.exists() or p.is_symlink():
|
|
observed=p.lstat()
|
|
fd=os.open(p,os.O_RDWR|os.O_NOFOLLOW)
|
|
try:
|
|
s=os.fstat(fd);require(stat.S_ISREG(s.st_mode) and s.st_nlink==1 and s.st_uid in (0,owner.st_uid) and
|
|
(s.st_dev,s.st_ino)==(observed.st_dev,observed.st_ino),'only own newly created fixed sidecar')
|
|
if s.st_uid==0:os.fchown(fd,owner.st_uid,owner.st_gid)
|
|
finally:os.close(fd)
|
|
def guard():
|
|
require(payload['grant_id']=='25a4bf3f-772c-487d-825d-73bcbbc418fd' and len(payload['root_rows'])==18,'exact original reboot and 18 retained grants')
|
|
require(json.loads((state/'identity.json').read_bytes())['agent_id']==payload['agent_id'],'permanent Cloud identity')
|
|
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==payload['new_boot_id']!=payload['old_boot_id'],'same one completed boot')
|
|
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
|
require('RMM_APT_HELPER_' not in dropin.read_text() and
|
|
run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip()==b'inactive','APT endpoint remains closed')
|
|
require(run(['systemctl','show','rmm-agent-remote.service','-p','ActiveState','--value']).strip()==b'inactive','agent must already be quiesced')
|
|
require(sha(provider.read_bytes())==payload['provider_sha256'] and sha(config.read_bytes())==payload['config_sha256'] and
|
|
sha(pin.read_bytes())==payload['pin_sha256'],'original provider and General policy/pin')
|
|
with contextlib.closing(sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True)) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','Root integrity')
|
|
rows=db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
|
|
actual={r[0]:sha(json.dumps([dict(sha256=sha(v),size=len(v)) if isinstance(v,bytes) else v for v in r],separators=(',',':')).encode()) for r in rows}
|
|
require(actual==payload['root_rows'],'all 18 original consumption/start/outcome bytes retained')
|
|
pending=[r for r in rows if r[5] is None]
|
|
require(len(pending)==1 and pending[0][0]==payload['grant_id'] and pending[0][3:5]==(payload['job_id'],payload['attempt_id']) and
|
|
json.loads(pending[0][7])['state']=='awaiting_reboot','only original consumed reboot awaits proof')
|
|
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'no helper fault')
|
|
for name,inode in payload['database_inodes'].items():
|
|
p=Path(name);s=p.stat();require([s.st_dev,s.st_ino]==inode,'original database inode')
|
|
if p==ledger:continue
|
|
with agent_readonly(p) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent database integrity')
|
|
if p.name=='journal.sqlite':
|
|
require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0]==0 and
|
|
db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(payload['job_id'],payload['attempt_id'])).fetchone()[0]==0,'no unrelated agent unresolved/fault')
|
|
require(payload['mode'] in ('copy','preflight','install'),'fixed maintenance mode')
|
|
require(str(uuid.UUID(payload['id']))==payload['id'],'canonical maintenance ID')
|
|
for name,path in (('agent',agent),('helper',helper)):
|
|
require(file_identity(path)['sha256']==payload['old_'+name+'_sha256'],'qualified old binary')
|
|
guard()
|
|
if payload['mode']=='preflight':return dict(preflight_only=True,maintenance_mutations=False,root_consumed=18,root_unresolved=1,apt_endpoint_enabled=False)
|
|
folder=Path('/var/lib/rmm-bootstrap-history')/('reboot-capacity-'+payload['id']);folder.mkdir(mode=0o700)
|
|
if payload['mode']=='copy':
|
|
before=file_identity(ledger)
|
|
with contextlib.closing(sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True)) as original:
|
|
expected=fingerprint(original);copy=folder/'ledger-copy.sqlite';create(copy,b'')
|
|
with contextlib.closing(sqlite3.connect(copy)) as duplicate:
|
|
original.backup(duplicate)
|
|
require(fingerprint(duplicate)==expected,'exact consistent copied ledger')
|
|
size=duplicate.execute('PRAGMA page_size').fetchone()[0];pages=duplicate.execute('PRAGMA page_count').fetchone()[0]
|
|
require(size==4096 and duplicate.execute('PRAGMA max_page_count=65536').fetchone()[0]==65536 and pages<=65536,'old finite capacity')
|
|
code=None
|
|
try:
|
|
duplicate.execute('BEGIN IMMEDIATE');duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
|
|
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
|
|
except sqlite3.DatabaseError as error:code=getattr(error,'sqlite_errorcode',None)
|
|
finally:duplicate.rollback()
|
|
require(code==sqlite3.SQLITE_FULL and fingerprint(duplicate)==expected,'copied old cap SQLITE_FULL13 with full rollback')
|
|
require(duplicate.execute('PRAGMA max_page_count=262144').fetchone()[0]==262144,'new finite 1GiB cap')
|
|
duplicate.execute('BEGIN IMMEDIATE');duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
|
|
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
|
|
require(duplicate.execute('SELECT length(payload) FROM journal_capacity_probe').fetchone()[0]==524288,'same insertion succeeds at finite new cap')
|
|
duplicate.rollback()
|
|
require(fingerprint(duplicate)==expected and duplicate.execute('PRAGMA integrity_check').fetchone()[0]=='ok','copied rows/schema unchanged')
|
|
require(file_identity(ledger)==before,'original ledger bytes/inode unchanged');guard()
|
|
report=dict(verified=True,backup=str(folder),old_error_code=code,new_insert_passed=True,old_cap_pages=65536,new_cap_pages=262144,
|
|
page_size=size,page_count=pages,probe_bytes=524288,original_before=before,original_unchanged=True,rows_and_schema_rolled_back=True,
|
|
original_sql_mutations=False,package_effects=False,reboot_requested=False)
|
|
create(folder/'report.json',json.dumps(report,indent=2).encode());return report
|
|
raw={n:base64.b64decode(payload[n],validate=True) for n in ('agent','helper')}
|
|
for name in raw:
|
|
require(sha(raw[name])==payload[name+'_sha256'] and raw[name][:6]==b'\x7fELF\x02\x01' and raw[name][18:20]==b'\x3e\x00','qualified Linux x64 artifact')
|
|
report=dict(completed=False,backup=str(folder),stage='validated',agent_stopped=True,apt_endpoint_enabled=False,
|
|
session_id=payload['session_id'],target=payload['target'],job_id=payload['job_id'],attempt_id=payload['attempt_id'],
|
|
grant_id=payload['grant_id'],new_boot_id=payload['new_boot_id'],database_inodes_before=payload['database_inodes'],root_rows_before=payload['root_rows'],
|
|
jobs_created=False,grants_created=False,
|
|
apt_runtime_ready=False,apt_runtime_note='Historical closed APT measurements retained; requalification required before any future APT job',
|
|
package_effects=False,reboot_requested=False,backups={})
|
|
def save():
|
|
create(folder/'.report-next.json',json.dumps(report,indent=2).encode());os.replace(folder/'.report-next.json',folder/'report.json');fsync_dir(folder)
|
|
lock=None
|
|
try:
|
|
save()
|
|
for name in raw:create(folder/(name+'-next'),raw[name],0o755);run(['/lib64/ld-linux-x86-64.so.2','--list',str(folder/(name+'-next'))])
|
|
run(['systemctl','stop','rmm-helper-cloud.service'])
|
|
lock=os.open(root/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
guard()
|
|
trees={str(p):manifest(p) for p in (state,root,Path('/var/lib/rmm-agent'),apt)}
|
|
controls={str(p):file_identity(p) for p in (config,pin,provider,Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'),
|
|
Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf'),Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'))}
|
|
for name,path in (('helper',helper),('agent',agent)):
|
|
previous=folder/(name+'-previous');require(path.stat().st_dev==folder.stat().st_dev,'same-filesystem versioned replacement')
|
|
os.rename(path,previous);fsync_dir(path.parent);fsync_dir(folder);report['backups'][str(path)]=str(previous);save()
|
|
os.rename(folder/(name+'-next'),path);fsync_dir(path.parent);fsync_dir(folder)
|
|
inspected=json.loads(subprocess.run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config)),capture_output=True,check=True,timeout=10).stdout)
|
|
require(inspected==json.loads(pin.read_bytes()),'General identity/policy pin unchanged across binary upgrade')
|
|
require(all(manifest(Path(p))==v for p,v in trees.items()) and all(file_identity(Path(p))==v for p,v in controls.items()),'all state/identities/journals/APT histories/markers and file inodes preserved')
|
|
guard()
|
|
report.update(stage='installed_preserved',preserved_state=True,journals_preserved_before_restart=True,root_consumed=18,root_unresolved=1,
|
|
old_agent_sha256=payload['old_agent_sha256'],old_helper_sha256=payload['old_helper_sha256'],agent_sha256=payload['agent_sha256'],
|
|
helper_sha256=payload['helper_sha256'],provider_sha256=payload['provider_sha256'],general_pin=inspected,
|
|
journal_page_size=4096,journal_max_page_count=262144,journal_capacity_bytes=1073741824)
|
|
save();fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
|
run(['systemctl','start','rmm-helper-cloud.service'])
|
|
require(run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value']).strip()==b'active','General helper starts')
|
|
require(all(manifest(Path(p))==v for p,v in trees.items()) and all(file_identity(Path(p))==v for p,v in controls.items()),'helper startup preserves every journal and APT witness')
|
|
guard();report.update(stage='helper_started_agent_quiesced',completed=True,
|
|
database_inodes_after=payload['database_inodes'],root_rows_after=payload['root_rows']);save()
|
|
except Exception as error:
|
|
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]);save()
|
|
finally:
|
|
if lock is not None:fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
|
|
return report
|
|
|
|
|
|
def build_payload(mode):
|
|
m=cloud.contract;folder=reboot.STATE/SESSION
|
|
session=m.protected_json(folder/'before.json');cp=m.protected_json(folder/'checkpoint.json')
|
|
frozen=m.protected_json(folder/'proof-upgrade-before.json')['native'];enabled=m.protected_json(folder/'helper-enabled.json')
|
|
job=cp['job'];require(job['job_id']=='8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa' and len(frozen['root_rows'])==18,'one exact actual approved reboot')
|
|
payload=dict(mode=mode,id=str(uuid.uuid4()),session_id=SESSION,target=session['target'],agent_id=session['target']['agent_id'],old_boot_id=session['old_boot_id'],new_boot_id=frozen['boot_id'],
|
|
grant_id=GRANT,job_id=job['job_id'],attempt_id=job['attempt_id'],root_rows=frozen['root_rows'],database_inodes=frozen['database_inodes'],
|
|
old_agent_sha256=session['pins']['agent_sha256'],old_helper_sha256=session['pins']['helper_sha256'],
|
|
provider_sha256='1dc0237dd8df9c9d22568bb1f6b047ede73bc2a5fca5c7fcedb7f24ab2a58b1e',config_sha256=enabled['config_sha256'],pin_sha256=enabled['pin_sha256'])
|
|
if mode=='install':
|
|
probe=m.protected_json(folder/'capacity-copy.json')
|
|
require(probe['verified'] and probe['old_error_code']==13 and probe['original_unchanged'] and probe['new_insert_passed'],'copied actual RootDB proof required first')
|
|
for name in ('agent','helper'):
|
|
path=folder/(name+'-reboot-capacity');qualification=m.protected_json(path.with_name(path.name+'.json'));raw=path.read_bytes()
|
|
require(qualification['release_build'] is True and qualification['architecture']=='x86_64-unknown-linux-gnu' and
|
|
qualification['provider_sha256']==payload['provider_sha256'] and qualification['binary_sha256']==hashlib.sha256(raw).hexdigest(),'qualified new release artifact')
|
|
require('crates/privileged-helper/src/lib.rs' in qualification['source_files'] and
|
|
(name!='agent' or 'crates/agent/src/reconcile.rs' in qualification['source_files']),
|
|
'capacity and original reboot reconciliation source bound by release')
|
|
require(all(hashlib.sha256((cloud.ROOT/p).read_bytes()).hexdigest()==h for p,h in qualification['source_files'].items()),'qualified source unchanged')
|
|
payload[name]=base64.b64encode(raw).decode();payload[name+'_sha256']=qualification['binary_sha256']
|
|
return payload
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(description=__doc__);p.add_argument('mode',choices=('copy','preflight','install'));args=p.parse_args();os.umask(0o077)
|
|
folder=reboot.STATE/SESSION
|
|
if args.mode=='install':require(not (folder/'capacity-runtime.json').exists(),'completed/incomplete previous installation retained; inspect, never repeat blindly')
|
|
payload=build_payload(args.mode)
|
|
target=cloud.contract.protected_json(folder/'before.json')['target']
|
|
require(reboot.evidence(target,'gate')==dict(jobs=30,outbox=30,permits=26,unresolved=1),'only original consumed reboot unresolved')
|
|
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(action='state'));require(policy['closed'],'central Root scopes closed')
|
|
cloud.contract.write_private(folder/('capacity-'+args.mode+'-intent.json'),{k:v for k,v in payload.items() if k not in ('agent','helper')})
|
|
report=cloud.bootstrap.rpc('linux-vm.conf',capacity_maintenance,payload)
|
|
path=folder/('capacity-'+('runtime' if args.mode=='install' else args.mode)+'.json');cloud.contract.write_private(path,report)
|
|
require((report.get('completed') if args.mode=='install' else report.get('verified') if args.mode=='copy' else report.get('preflight_only')) is True,'maintenance incomplete; inspect saved report and preserve original effect')
|
|
if args.mode=='install':record_upgrade_index(report)
|
|
print('PASS bounded capacity '+args.mode+'; original reboot retained, no package effect, agent remains stopped')
|
|
|
|
|
|
def record_upgrade_index(report):
|
|
"""Create only after an independently returned/read-back completed report."""
|
|
m=cloud.contract;folder=reboot.STATE/SESSION;session=m.protected_json(folder/'before.json')
|
|
frozen=m.protected_json(folder/'proof-upgrade-before.json')['native'];enabled=m.protected_json(folder/'helper-enabled.json')
|
|
job=m.protected_json(folder/'checkpoint.json')['job']
|
|
require(report['completed'] and report['preserved_state'] and report['journals_preserved_before_restart'] and
|
|
report['root_consumed']==18 and report['root_unresolved']==1 and report['agent_stopped'] and
|
|
report['apt_endpoint_enabled'] is False and report['package_effects'] is False and report['reboot_requested'] is False and
|
|
report['jobs_created'] is False and report['grants_created'] is False and
|
|
report['general_pin']==enabled['general_pin'] and report['root_rows_before']==report['root_rows_after']==frozen['root_rows'] and
|
|
report['database_inodes_before']==report['database_inodes_after']==frozen['database_inodes'],'complete original-state preserving native report')
|
|
bindings=dict(session_id=SESSION,target=session['target'],job_id=job['job_id'],attempt_id=job['attempt_id'],grant_id=GRANT,new_boot_id=frozen['boot_id'])
|
|
require(all(report[k]==v for k,v in bindings.items()),'same approved reboot receipt binding')
|
|
def ref(path):return dict(path=str(path.resolve()),sha256=hashlib.sha256(path.read_bytes()).hexdigest())
|
|
value=dict(schema_version=1,type='rmm.reboot.storage-capacity-upgrade',verified=True,**bindings,
|
|
**{k:report[k] for k in ('old_agent_sha256','old_helper_sha256','agent_sha256','helper_sha256','provider_sha256')},
|
|
session_before_sha256=hashlib.sha256((folder/'before.json').read_bytes()).hexdigest(),
|
|
checkpoint_sha256=hashlib.sha256((folder/'checkpoint.json').read_bytes()).hexdigest(),preservation_witness=ref(folder/'proof-upgrade-before.json'),
|
|
runtime_report=ref(folder/'capacity-runtime.json'),qualification_reports={n:ref(folder/(n+'-reboot-capacity.json')) for n in ('agent','helper')})
|
|
m.write_private(folder/'capacity-upgrade.json',value)
|
|
|
|
|
|
if __name__=='__main__':
|
|
try:main()
|
|
except Exception as error:
|
|
message=str(error);print(message if message.startswith('remote bootstrap incomplete: ') else 'REBOOT_CAPACITY_INCOMPLETE:'+type(error).__name__)
|
|
raise SystemExit(2) from None
|