177 lines
12 KiB
Python
177 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Preserving helper upgrade for the original APT bound readonly recheck.
|
|
|
|
APT execution endpoint stays stopped. General helper snapshots may prove the
|
|
existing failed partial postcondition; only existing signed Sync closes it.
|
|
No package command, new grant, approval or consumption reset exists here.
|
|
"""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import uuid
|
|
|
|
import cloud_apt_retry as apt
|
|
cloud=apt.cloud
|
|
require=cloud.require
|
|
STATE=cloud.ROOT/'devstate/cloud-apt-recovery'
|
|
|
|
|
|
def upgrade(payload):
|
|
import fcntl
|
|
import sqlite3
|
|
os.umask(0o077)
|
|
base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote')
|
|
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper')
|
|
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json'
|
|
agent_binary=Path('/usr/local/libexec/rmm-agent-remote')
|
|
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
|
full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
|
|
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
|
|
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
|
|
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
|
|
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
|
|
reduced=''.join(line+'\n' for line in full.splitlines() if not line.startswith('Environment=RMM_APT_HELPER_'))
|
|
def run(args,**kw):return subprocess.run(args,capture_output=True,check=True,timeout=20,**kw).stdout
|
|
def sha(data):return hashlib.sha256(data).hexdigest()
|
|
def manifest(path):
|
|
result={}
|
|
for p in path.rglob('*'):
|
|
require(not p.is_symlink(),'preserved state symlink')
|
|
if p.is_file():result[str(p.relative_to(path))]=sha(p.read_bytes())
|
|
return result
|
|
def guard():
|
|
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity')
|
|
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')}==set(payload['expected_grants']),'exact preserved grants')
|
|
rows=db.execute('SELECT c.grant_id,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
|
|
require(len(rows)==1 and rows[0][0]==payload['unknown_grant'],'one exact existing unknown')
|
|
outcome=json.loads(rows[0][1]);require(outcome['state']=='unknown' and outcome['error_code']=='DPKG_INCONSISTENT','exact known partial outcome')
|
|
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'no sticky faults')
|
|
return sha(rows[0][1])
|
|
guard()
|
|
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved')
|
|
require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required')
|
|
require(sha(agent_binary.read_bytes())==payload['old_agent_sha256'],'reviewed permanent agent required')
|
|
require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required')
|
|
original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json'
|
|
retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json'
|
|
require(sha(original.read_bytes())=='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053' and
|
|
sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required')
|
|
require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected')
|
|
raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True)
|
|
agent_raw=base64.b64decode(payload['agent_binary'],validate=True)
|
|
require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts')
|
|
require(sha(agent_raw)==payload['agent_sha256'],'qualified sync-prevention agent')
|
|
backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700)
|
|
def create(path,data,mode=0o600):
|
|
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
|
|
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode)
|
|
report=dict(stage='validated',completed=False,package_effects=False,backup=str(backup))
|
|
def save():
|
|
next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json')
|
|
save();staged=backup/'helper-next';create(staged,raw,0o755)
|
|
staged_agent=backup/'agent-next';create(staged_agent,agent_raw,0o755)
|
|
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
|
|
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged_agent)])
|
|
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
|
|
fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
try:
|
|
original_unknown=guard();inode=(ledger.stat().st_dev,ledger.stat().st_ino)
|
|
preserved={str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent,base/'active',base/'history',base/'archives',base/'baselines',base/'log',base/'state')}
|
|
witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))}
|
|
old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata')
|
|
general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
|
|
for p in (binary,agent_binary,provider,metadata,dropin):
|
|
target=backup/str(p).lstrip('/').replace('/','__')
|
|
require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup')
|
|
os.rename(p,target)
|
|
os.rename(staged,binary);os.rename(staged_agent,agent_binary);create(provider,code,0o755)
|
|
create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644)
|
|
inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
|
|
require(inspected==json.loads(general),'general policy unchanged')
|
|
require(all(manifest(Path(p))==values for p,values in preserved.items()) and
|
|
all(sha(Path(p).read_bytes())==value for p,value in witnesses.items()) and
|
|
(ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift')
|
|
report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown,
|
|
helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save()
|
|
report['agent_sha256']=sha(agent_raw);save()
|
|
finally:
|
|
fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
|
|
run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
|
|
report.update(stage='started_bound_readonly_recheck',completed=True);save()
|
|
return report
|
|
|
|
|
|
def result_evidence(payload):
|
|
import sqlite3
|
|
ledger=Path('/var/lib/rmm-helper/consumption.sqlite')
|
|
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
|
|
row=db.execute('SELECT result FROM consumed WHERE grant_id=?',(payload['grant_id'],)).fetchone()
|
|
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
proof=[]
|
|
if 'apt_partial_reconciliations' in tables:
|
|
columns=[r[1] for r in db.execute('PRAGMA table_info(apt_partial_reconciliations)')]
|
|
proof=[dict(zip(columns,r)) for r in db.execute('SELECT * FROM apt_partial_reconciliations')]
|
|
# Opaque witness bytes remain private, JSON-safe for protected file.
|
|
for item in proof:
|
|
for k,v in list(item.items()):
|
|
if isinstance(v,bytes):item[k]=dict(sha256=hashlib.sha256(v).hexdigest(),size=len(v))
|
|
return dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
|
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
|
|
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0],
|
|
fault_codes=[r[0] for r in db.execute('SELECT fault FROM helper_sync_meta WHERE fault IS NOT NULL')],
|
|
original_result=json.loads(row[0]) if row and row[0] else None,proof=proof,
|
|
apt_active=subprocess.run(['systemctl','is-active','rmm-apt-helper.service'],capture_output=True).stdout.strip().decode())
|
|
|
|
|
|
def installation_status(payload):
|
|
root=Path('/var/lib/rmm-bootstrap-history')
|
|
reports=[]
|
|
for p in sorted(root.glob('cloud-readonly-reconcile-*/report.json')):
|
|
reports.append(json.loads(p.read_text()))
|
|
services={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True).stdout.strip().decode()
|
|
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')}
|
|
hashes={name:hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in
|
|
('/usr/local/libexec/rmm-agent-remote','/usr/local/libexec/rmm-privileged-helper','/usr/lib/rmm-apt-transaction/apt_transaction.py')}
|
|
return dict(reports=reports,services=services,hashes=hashes)
|
|
|
|
|
|
def record_agent_upgrade(report,installed):
|
|
m=cloud.contract;path=cloud.ROOT/'devstate/remote-vm.json'
|
|
require(report['completed'] and report['preserved_state'] and not report['apt_endpoint_enabled'],'qualified preserving installation')
|
|
require(m.protected_json(path)==installed,'local deployment record drift')
|
|
ident=Path(report['backup']).name
|
|
m.write_private(STATE/(ident+'-agent-before.json'),installed)
|
|
next_path=STATE/(ident+'-agent-next.json')
|
|
m.write_private(next_path,dict(installed,binary_sha256=report['agent_sha256']))
|
|
os.replace(next_path,path)
|
|
|
|
|
|
def main():
|
|
m=cloud.contract;os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
|
|
checkpoint=m.protected_json(apt.STATE/'apt_install_retry/checkpoint.json');grant=checkpoint['result']['result']['grant_id']
|
|
require(apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact closed-policy unknown gate')
|
|
folders=[cloud.ROOT/'devstate/cloud-root/cases'/name for name in ('service_setup','service_start','service_stop','apt_native_install')]
|
|
folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
|
|
expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant]
|
|
ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes()
|
|
agent_raw=(STATE/'agent-sync-prevention').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
|
payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant,
|
|
old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c',
|
|
old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945',
|
|
binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(),
|
|
agent_binary=base64.b64encode(agent_raw).decode(),agent_sha256=hashlib.sha256(agent_raw).hexdigest(),old_agent_sha256=installed['binary_sha256'],
|
|
helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest())
|
|
report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report)
|
|
require(report['completed'] and report['agent_sha256']==payload['agent_sha256'],'qualified preserving installation')
|
|
record_agent_upgrade(report,installed)
|
|
print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')})
|
|
|
|
|
|
if __name__=='__main__':
|
|
try:main()
|
|
except Exception as error:
|
|
print('CLOUD_APT_RECONCILE_UPGRADE_INCOMPLETE:'+type(error).__name__);raise SystemExit(2) from None
|