Files
rust-discovery/deploy/cloud_root_clock_upgrade.py
2026-11-13 13:49:11 +01:00

226 lines
16 KiB
Python

#!/usr/bin/env python3
"""One guarded upgrade of the unused cloud helper; no jobs or package effects.
Retains the previous probe, binary, profiles and reports. A successful systemctl
start is insufficient: both daemons must remain running and bind the shared
ledger before this upgrade reports readiness. Never clears consumed/history.
"""
import base64
import hashlib
import json
import os
import uuid
import cloud_root_bootstrap as initial
transport=initial.transport
ROOT=initial.ROOT
require=transport.require
def upgrade(payload):
import sqlite3
import time
os.umask(0o077)
require(os.geteuid()==0,'root required')
require(str(uuid.UUID(payload['id']))==payload['id'],'upgrade id')
previous=Path(payload['previous']['backup'])
require(previous.parent==Path('/var/lib/rmm-bootstrap-history') and not previous.is_symlink(),'known previous receipt')
receipt=json.loads((previous/'report.json').read_text())
require(receipt.get('stage')=='helpers_started_no_jobs' and receipt.get('preserved_before_start') is True,'previous stage mismatch')
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');state=Path('/var/lib/rmm-agent-remote')
base=Path('/var/lib/rmm-apt-transaction')
def run(args,**kwargs):return subprocess.run(args,capture_output=True,check=True,timeout=90,**kwargs).stdout
def sha(data):return hashlib.sha256(data).hexdigest()
def count():
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity')
return db.execute('SELECT count(*) FROM consumed').fetchone()[0]
def manifest(path):
result={}
for item in path.rglob('*'):
require(not item.is_symlink(),'preserved state symlink')
if item.is_file():result[str(item.relative_to(path))]=sha(item.read_bytes())
return result
def create(path,data,mode=0o600,gid=0):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as f:
f.write(data);f.flush();os.fsync(f.fileno());os.fchown(f.fileno(),0,gid);os.fchmod(f.fileno(),mode)
require(count()==0,'no upgrade after consumed root attempts')
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
require('helper_sync_meta' not in tables or db.execute('SELECT count(*) FROM helper_sync_meta').fetchone()[0]==0,'already bound ledger')
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('bootstrap-attempt-*.json'))
and not (base/'active').exists(),'existing effects require reconciliation')
for name in ('rmm-helper-cloud.service','rmm-apt-helper.service','rmm-helper-dev-test.service','rmm-agent.service'):
active=run(['systemctl','show',name,'-p','ActiveState','--value']).strip()
require(active in (b'inactive',b'failed') and
run(['systemctl','show',name,'-p','MainPID','--value']).strip()==b'0','helpers must be stopped')
require(json.loads((state/'binding.json').read_text())==payload['binding'],'cloud binding drift')
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'unresolved agent job')
runtime=receipt['runtime']
for path,pin in runtime['binaries'].items():
p=Path(path).resolve(strict=True)
require(p.stat().st_uid==0 and p.stat().st_mode&0o022==0 and sha(p.read_bytes())==pin['sha256'],'installed runtime drift')
apt_unit=Path('/etc/systemd/system/rmm-apt-helper.service')
general_unit=Path('/etc/systemd/system/rmm-helper-cloud.service')
require(sha(apt_unit.read_bytes())==runtime['unit_profile_sha256'],'APT unit drift')
require(general_unit.read_text()==payload['old_general_unit'],'general unit drift')
config=Path('/etc/rmm/apt-helper.json');old_config=json.loads(config.read_text())
require(old_config['apt_provider']['runtime_sha256']==receipt['runtime_sha256'],'old runtime pin drift')
metadata=base/'install-metadata.json';old_meta=json.loads(metadata.read_text())
require(old_meta==dict(schema_version=2,provider_sha256=payload['provider_sha256'],hook_sha256=payload['hook_sha256'],
unit_sha256=runtime['unit_profile_sha256']),'metadata drift')
proof=Path('/etc/rmm-remote/apt-helper-proof.json')
require(json.loads(proof.read_text())==payload['previous']['pins']['apt'],'independent proof drift')
binary=base64.b64decode(payload['binary'],validate=True)
require(sha(binary)==payload['helper_sha256'] and binary[:6]==b'\x7fELF\x02\x01' and binary[18:20]==b'\x3e\x00','binary mismatch')
backup=previous.parent/('cloud-clock-'+payload['id']);backup.mkdir(mode=0o700)
report=dict(stage='validated',completed=False,package_effects=False,rmm_jobs_executed=False,backups={})
def save():
p=backup/'.next.json';create(p,json.dumps(report,indent=2).encode());os.replace(p,backup/'report.json')
def retain(p):
require(not p.is_symlink() and p.stat().st_uid==0 and p.stat().st_mode&0o022==0,'unsafe retained path')
target=backup/str(p).lstrip('/').replace('/','__')
require(not target.exists() and p.stat().st_dev==backup.stat().st_dev,'exclusive same filesystem backup')
os.rename(p,target);report['backups'][str(p)]=str(target);save()
stopped=False
try:
save();staged=backup/'helper-next';create(staged,binary,0o755)
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
run(['systemctl','stop','rmm-agent-remote.service']);stopped=True
preserved={str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
inode=(ledger.stat().st_dev,ledger.stat().st_ino)
for p in (Path('/usr/local/libexec/rmm-privileged-helper'),apt_unit,general_unit,config,metadata,proof,base/'native-readiness.json'):retain(p)
os.rename(staged,'/usr/local/libexec/rmm-privileged-helper')
create(apt_unit,payload['apt_unit'].encode(),0o644);create(general_unit,payload['general_unit'].encode(),0o644)
create(metadata,json.dumps(dict(old_meta,unit_sha256=sha(payload['apt_unit'].encode()))).encode())
cmd=['/usr/bin/python3','-I','/usr/lib/rmm-apt-transaction/apt_transaction.py']
new_runtime=json.loads(run(cmd+['inspect-runtime']))
runtime_sha=sha(json.dumps(new_runtime,sort_keys=True,separators=(',',':'),ensure_ascii=True).encode())
old_config['apt_provider']['runtime_sha256']=runtime_sha
create(config,json.dumps(old_config).encode())
data=run(['/usr/local/libexec/rmm-privileged-helper','--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config)))
pin=json.loads(data);require(pin['identity']==payload['previous']['pins']['apt']['identity'],'authority changed')
create(proof,data,0o640,984)
require(all(manifest(Path(p))==v for p,v in preserved.items()) and (ledger.stat().st_dev,ledger.stat().st_ino)==inode,'preserved journals changed')
report.update(stage='installed_preserved',preserved_before_start=True,pins=dict(payload['previous']['pins'],apt=pin),
runtime_sha256=runtime_sha,runtime=new_runtime,apt_config_sha256=receipt['apt_config_sha256']);save()
for p in (apt_unit,general_unit):run(['systemd-analyze','verify',str(p)])
run(['systemctl','daemon-reload'])
for name in ('rmm-helper-cloud.service','rmm-apt-helper.service'):
if run(['systemctl','show',name,'-p','ActiveState','--value']).strip()==b'failed':
run(['systemctl','reset-failed',name])
report['stage']='native_probe_and_stable_start';save()
run(['systemctl','start','rmm-apt-helper.service','rmm-helper-cloud.service'])
for _ in range(10):
for name in ('rmm-helper-cloud.service','rmm-apt-helper.service'):
status=run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status,'daemon not stably ready')
time.sleep(1)
require((base/'native-readiness.json').is_file() and count()==0,'native readiness or zero effects absent')
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('SELECT count(*) FROM helper_sync_meta').fetchone()[0]==1,'helper ledger not bound')
report.update(stage='stable_helpers_no_jobs',completed=True,native_readiness=True);save()
except Exception as error:
report.update(error_class=type(error).__name__,completed=False)
if count()==0:
for name in ('rmm-apt-helper.service','rmm-helper-cloud.service'):subprocess.run(['systemctl','stop',name],capture_output=True,timeout=30)
save()
finally:
if stopped:run(['systemctl','start','rmm-agent-remote.service'])
return dict(report,backup=str(backup))
def main():
os.umask(0o077)
previous=initial.acceptance.contract.protected_json(ROOT/'devstate/cloud-root/install-121352fd-8078-48cd-9185-439e29420c56.json')
identity=previous['pins']['general']['identity']
gate=transport.rpc('server.conf',initial.acceptance.server,dict(kind='gate',tenant_id=identity['tenant_id'],agent_id=identity['agent_id']))
require(gate['unresolved']==0,'central unresolved job')
binding=transport.rpc('server.conf',transport.server,dict(action='observe',agent_id=identity['agent_id']))
material=transport.rpc('server.conf',transport.server,dict(action='material'))
binding={k:binding[k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version')}
binding.update(tenant_id=identity['tenant_id'],public_key_hex=material['key']['public_key_hex'])
general=(ROOT/'deploy/linux/rmm-helper-cloud.service').read_text()
old=general.replace('# Read-only trusted clock query; glibc may use clock_adjtime. No CAP_SYS_TIME.\nSystemCallFilter=adjtimex clock_adjtime\n',
'SystemCallFilter=adjtimex\n')
binary=(ROOT/'devstate/cloud-root/rmm-privileged-helper-clock-fixed').read_bytes()
payload=dict(id=str(uuid.uuid4()),previous=previous,binding=binding,old_general_unit=old,general_unit=general,
apt_unit=(ROOT/'deploy/linux/rmm-apt-helper.service').read_text(),binary=base64.b64encode(binary).decode(),
helper_sha256=hashlib.sha256(binary).hexdigest(),
provider_sha256=hashlib.sha256((ROOT/'tools/apt-transaction/apt_transaction.py').read_bytes()).hexdigest(),
hook_sha256=hashlib.sha256((ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest())
result=transport.rpc('linux-vm.conf',upgrade,payload)
initial.acceptance.contract.write_private(ROOT/'devstate/cloud-root'/('clock-'+payload['id']+'.json'),result)
print(json.dumps({k:result.get(k) for k in ('completed','stage','error_class','native_readiness','preserved_before_start','runtime_sha256')}))
require(result['completed'],'upgrade incomplete; inspect retained report, never reset')
def resume_start(payload):
"""Resume only the installed, unused profile; never reinstall or erase proof."""
import sqlite3
import time
import traceback
backup=Path(payload['backup'])
require(backup.parent==Path('/var/lib/rmm-bootstrap-history') and backup.name.startswith('cloud-clock-'),'fixed retained upgrade')
report=json.loads((backup/'report.json').read_text())
require((report['stage']=='installed_preserved' or report['stage']=='stable_start_resume' and report.get('step')=='start_helpers')
and report.get('completed') is False and report.get('preserved_before_start') is True,'not a resumable start')
def run(args):return subprocess.run(args,capture_output=True,check=True,timeout=90).stdout
def count():
with sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True) as db:
return db.execute('SELECT count(*) FROM consumed').fetchone()[0]
def save():
p=backup/'.start-next.json'
fd=os.open(p,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as f:json.dump(report,f,indent=2);f.flush();os.fsync(f.fileno())
os.replace(p,backup/'report.json')
require(count()==0,'no start recovery after consumed attempt')
base=Path('/var/lib/rmm-apt-transaction')
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('bootstrap-attempt-*.json'))
and not (base/'active').exists() and not (base/'native-readiness.json').exists(),'existing evidence requires inspection')
for kind in ('apt','general'):
unit=Path('/etc/systemd/system/rmm-'+('apt-helper' if kind=='apt' else 'helper-cloud')+'.service')
require(unit.read_text()==payload[kind+'_unit'],'unit differs from reviewed source')
cmd=['/usr/bin/python3','-I','/usr/lib/rmm-apt-transaction/apt_transaction.py']
runtime=json.loads(run(cmd+['inspect-runtime']))
require(runtime==report['runtime'],'installed runtime drift')
pin=json.loads(Path('/etc/rmm-remote/apt-helper-proof.json').read_text())
require(pin==report['pins']['apt'],'independent proof drift')
try:
report['stage']='stable_start_resume';save()
run(['systemctl','daemon-reload'])
for name in ('rmm-apt-helper.service','rmm-helper-cloud.service'):
report['step']='verify_'+name;save()
run(['systemd-analyze','verify','/etc/systemd/system/'+name])
report['step']='start_helpers';save()
for name in ('rmm-apt-helper.service','rmm-helper-cloud.service'):
if run(['systemctl','show',name,'-p','ActiveState','--value']).strip()==b'failed':
run(['systemctl','reset-failed',name])
run(['systemctl','start','rmm-apt-helper.service','rmm-helper-cloud.service'])
for _ in range(10):
for name in ('rmm-apt-helper.service','rmm-helper-cloud.service'):
status=run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status,'daemon failed stable start')
time.sleep(1)
with sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True) as db:
require(db.execute('SELECT count(*) FROM helper_sync_meta').fetchone()[0]==1,'helper ledger not bound')
require((base/'native-readiness.json').is_file() and count()==0,'missing native readiness or unexpected root effect')
previous_error={k:report.pop(k) for k in ('error_class','frames') if k in report}
if previous_error:report['previous_start_failure']=previous_error
report.update(stage='stable_helpers_no_jobs',completed=True,native_readiness=True);save()
run(['systemctl','restart','rmm-agent-remote.service'])
except Exception as error:
report.update(completed=False,error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
if count()==0:
for name in ('rmm-apt-helper.service','rmm-helper-cloud.service'):subprocess.run(['systemctl','stop',name],capture_output=True,timeout=30)
save()
return dict(report,backup=str(backup))
if __name__=='__main__':
try:main()
except Exception:
print('CLOUD_CLOCK_UPGRADE_INCOMPLETE; no automatic repeat');raise SystemExit(2) from None