Files
rust-discovery/deploy/cloud_apt_reconcile_upgrade.py
2026-11-13 17:47:22 +01:00

177 lines
12 KiB
Python

#!/usr/bin/env python3
"""Preserving helper upgrade for the original APT bound readonly recheck.
APT execution endpoint stays stopped. General helper snapshots may prove the
existing failed partial postcondition; only existing signed Sync closes it.
No package command, new grant, approval or consumption reset exists here.
"""
import base64
import hashlib
import json
import os
from pathlib import Path
import uuid
import cloud_apt_retry as apt
cloud=apt.cloud
require=cloud.require
STATE=cloud.ROOT/'devstate/cloud-apt-recovery'
def upgrade(payload):
import fcntl
import sqlite3
os.umask(0o077)
base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote')
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper')
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json'
agent_binary=Path('/usr/local/libexec/rmm-agent-remote')
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
reduced=''.join(line+'\n' for line in full.splitlines() if not line.startswith('Environment=RMM_APT_HELPER_'))
def run(args,**kw):return subprocess.run(args,capture_output=True,check=True,timeout=20,**kw).stdout
def sha(data):return hashlib.sha256(data).hexdigest()
def manifest(path):
result={}
for p in path.rglob('*'):
require(not p.is_symlink(),'preserved state symlink')
if p.is_file():result[str(p.relative_to(path))]=sha(p.read_bytes())
return result
def guard():
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity')
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')}==set(payload['expected_grants']),'exact preserved grants')
rows=db.execute('SELECT c.grant_id,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows)==1 and rows[0][0]==payload['unknown_grant'],'one exact existing unknown')
outcome=json.loads(rows[0][1]);require(outcome['state']=='unknown' and outcome['error_code']=='DPKG_INCONSISTENT','exact known partial outcome')
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'no sticky faults')
return sha(rows[0][1])
guard()
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved')
require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required')
require(sha(agent_binary.read_bytes())==payload['old_agent_sha256'],'reviewed permanent agent required')
require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required')
original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json'
retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json'
require(sha(original.read_bytes())=='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053' and
sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required')
require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected')
raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True)
agent_raw=base64.b64decode(payload['agent_binary'],validate=True)
require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts')
require(sha(agent_raw)==payload['agent_sha256'],'qualified sync-prevention agent')
backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700)
def create(path,data,mode=0o600):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode)
report=dict(stage='validated',completed=False,package_effects=False,backup=str(backup))
def save():
next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json')
save();staged=backup/'helper-next';create(staged,raw,0o755)
staged_agent=backup/'agent-next';create(staged_agent,agent_raw,0o755)
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged_agent)])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
try:
original_unknown=guard();inode=(ledger.stat().st_dev,ledger.stat().st_ino)
preserved={str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent,base/'active',base/'history',base/'archives',base/'baselines',base/'log',base/'state')}
witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))}
old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata')
general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
for p in (binary,agent_binary,provider,metadata,dropin):
target=backup/str(p).lstrip('/').replace('/','__')
require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup')
os.rename(p,target)
os.rename(staged,binary);os.rename(staged_agent,agent_binary);create(provider,code,0o755)
create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644)
inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(inspected==json.loads(general),'general policy unchanged')
require(all(manifest(Path(p))==values for p,values in preserved.items()) and
all(sha(Path(p).read_bytes())==value for p,value in witnesses.items()) and
(ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift')
report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown,
helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save()
report['agent_sha256']=sha(agent_raw);save()
finally:
fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
report.update(stage='started_bound_readonly_recheck',completed=True);save()
return report
def result_evidence(payload):
import sqlite3
ledger=Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
row=db.execute('SELECT result FROM consumed WHERE grant_id=?',(payload['grant_id'],)).fetchone()
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
proof=[]
if 'apt_partial_reconciliations' in tables:
columns=[r[1] for r in db.execute('PRAGMA table_info(apt_partial_reconciliations)')]
proof=[dict(zip(columns,r)) for r in db.execute('SELECT * FROM apt_partial_reconciliations')]
# Opaque witness bytes remain private, JSON-safe for protected file.
for item in proof:
for k,v in list(item.items()):
if isinstance(v,bytes):item[k]=dict(sha256=hashlib.sha256(v).hexdigest(),size=len(v))
return dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0],
fault_codes=[r[0] for r in db.execute('SELECT fault FROM helper_sync_meta WHERE fault IS NOT NULL')],
original_result=json.loads(row[0]) if row and row[0] else None,proof=proof,
apt_active=subprocess.run(['systemctl','is-active','rmm-apt-helper.service'],capture_output=True).stdout.strip().decode())
def installation_status(payload):
root=Path('/var/lib/rmm-bootstrap-history')
reports=[]
for p in sorted(root.glob('cloud-readonly-reconcile-*/report.json')):
reports.append(json.loads(p.read_text()))
services={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True).stdout.strip().decode()
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')}
hashes={name:hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in
('/usr/local/libexec/rmm-agent-remote','/usr/local/libexec/rmm-privileged-helper','/usr/lib/rmm-apt-transaction/apt_transaction.py')}
return dict(reports=reports,services=services,hashes=hashes)
def record_agent_upgrade(report,installed):
m=cloud.contract;path=cloud.ROOT/'devstate/remote-vm.json'
require(report['completed'] and report['preserved_state'] and not report['apt_endpoint_enabled'],'qualified preserving installation')
require(m.protected_json(path)==installed,'local deployment record drift')
ident=Path(report['backup']).name
m.write_private(STATE/(ident+'-agent-before.json'),installed)
next_path=STATE/(ident+'-agent-next.json')
m.write_private(next_path,dict(installed,binary_sha256=report['agent_sha256']))
os.replace(next_path,path)
def main():
m=cloud.contract;os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
checkpoint=m.protected_json(apt.STATE/'apt_install_retry/checkpoint.json');grant=checkpoint['result']['result']['grant_id']
require(apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact closed-policy unknown gate')
folders=[cloud.ROOT/'devstate/cloud-root/cases'/name for name in ('service_setup','service_start','service_stop','apt_native_install')]
folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant]
ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes()
agent_raw=(STATE/'agent-sync-prevention').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant,
old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c',
old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945',
binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(),
agent_binary=base64.b64encode(agent_raw).decode(),agent_sha256=hashlib.sha256(agent_raw).hexdigest(),old_agent_sha256=installed['binary_sha256'],
helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest())
report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report)
require(report['completed'] and report['agent_sha256']==payload['agent_sha256'],'qualified preserving installation')
record_agent_upgrade(report,installed)
print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')})
if __name__=='__main__':
try:main()
except Exception as error:
print('CLOUD_APT_RECONCILE_UPGRADE_INCOMPLETE:'+type(error).__name__);raise SystemExit(2) from None