Files
rust-discovery/deploy/cloud_configured_reconcile_upgrade.py

407 lines
32 KiB
Python

#!/usr/bin/env python3
"""Preserve UNKNOWN337 while installing its narrowly bound readonly verifier.
Only bootstrap runtime/probe maintenance; no package effect, new grant, retry,
acceptance record, identity reset or direct consumed/result update. General and
agent restart only after immutable state and copied-dpkg probe qualification.
"""
import argparse
import base64
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import sys
import uuid
import cloud_apt_recovery_upgrade as previous
cloud = previous.cloud
require = cloud.require
STATE = previous.STATE
UNKNOWN = '33775478-15b0-46ec-b059-44f500dbb0b6'
OLD = dict(helper='332eae004d9416234eabf7ed3d5c9ece5440d1e0c764ea63357961c16b673b6b',
agent='2ce885cf5e6881e77383482b7d3a87a02c6847aa26b82e5fd70d2ebd8a45360d',
provider='5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327')
def upgrade_configured(payload):
import ast
import fcntl
import sqlite3
import time
import traceback
os.umask(0o077)
base = Path('/var/lib/rmm-apt-transaction'); state = Path('/var/lib/rmm-agent-remote')
ledger = Path('/var/lib/rmm-helper/consumption.sqlite')
helper = Path('/usr/local/libexec/rmm-privileged-helper'); agent = Path('/usr/local/libexec/rmm-agent-remote')
provider = Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
config = Path('/etc/rmm/apt-helper.json'); proof_file = Path('/etc/rmm-remote/apt-helper-proof.json')
metadata = base/'install-metadata.json'; readiness_file = base/'native-readiness.json'
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
reduced = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
full = reduced.replace('Environment=RMM_LOG_UNITS_FILE=',
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=')
def run(args, **kwargs):
return subprocess.run(args, capture_output=True, check=True, timeout=90, **kwargs).stdout
def sha(raw): return hashlib.sha256(raw).hexdigest()
def canonical(value): return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()
def fsync_dir(path):
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
def create(path, raw, mode=0o600, gid=0):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw); stream.flush(); os.fchown(stream.fileno(), 0, gid)
os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
fsync_dir(path.parent)
def manifest(path):
require(not path.is_symlink(), 'preserved directory symlink')
if not path.exists(): return {'__absent__':True}
require(path.is_dir(), 'preserved directory required')
result = {}
for p in [path, *sorted(path.rglob('*'))]:
s = p.lstat(); require(not p.is_symlink() and (p.is_dir() or p.is_file()), 'preserved regular state required')
result[str(p.relative_to(path))] = dict(dev=s.st_dev, inode=s.st_ino, mode=s.st_mode,
uid=s.st_uid, gid=s.st_gid, sha256=sha(p.read_bytes()) if p.is_file() else None)
return result
def guard():
require(len(payload['expected_grants']) == 11 and payload['unknown_grant'] ==
'33775478-15b0-46ec-b059-44f500dbb0b6', 'exact reviewed eleven Root grants')
with sqlite3.connect(ledger.as_uri()+'?mode=ro', uri=True) as db:
db.row_factory = sqlite3.Row
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
{r[0] for r in db.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants']) and
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0,
'exact preserved Root history without faults')
rows = db.execute('SELECT c.grant_id,c.result,e.outcome,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows) == 1 and rows[0]['grant_id'] == payload['unknown_grant'], 'only known configure unknown may remain')
outcome = json.loads(rows[0]['outcome'])
require(outcome == payload['unknown_outcome'] and outcome['state'] == 'unknown' and
outcome['error_code'] == 'FIXTURE_UNEXPECTED_REBOOT_REQUIRED', 'original unknown bytes must retain their meaning')
saved = json.loads(rows[0]['start']); request = saved['request']
job = json.loads(bytes(request['job_bytes']))
require(job['job_id'] == payload['context']['job_id'] and
job['attempt_id'] == payload['context']['attempt_id'] and
job['agent_id'] == payload['agent_id'], 'original saved request binding')
unknown_witness = dict(outcome_sha256=sha(rows[0]['outcome']),start_sha256=sha(rows[0]['start']))
rows = db.execute('SELECT * FROM apt_partial_reconciliations').fetchall()
require(len(rows) == 1, 'one immutable old partial proof required')
row = dict(rows[0]); expected = payload['proof_row']
for key in ('grant_id','original_request_sha256','previous_outcome_sha256','proof_sha256','checked_at_ms'):
require(row[key] == expected[key], 'old partial binding drift')
for key in ('previous_outcome','proof','classified_outcome'):
require(sha(row[key]) == expected[key]['sha256'] and len(row[key]) == expected[key]['size'], 'old partial bytes drift')
require(row['grant_id'] == payload['partial_grant'] and sha(row['proof']) == payload['partial_proof_sha256'], 'original partial proof unchanged')
old = db.execute('SELECT c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(payload['partial_grant'],)).fetchone()
require(old['result'] is not None and json.loads(old['result']) == json.loads(old['outcome']) == json.loads(row['classified_outcome']), 'old partial stays signed-Sync closed')
if db.execute("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='apt_effect_reconciliations'").fetchone()[0]:
require(db.execute('SELECT count(*) FROM apt_effect_reconciliations').fetchone()[0] == 0, 'configure classification must not already exist')
journal = state/'journal.sqlite'; sidecars = [Path(str(journal)+s) for s in ('-wal','-shm')]
existing_sidecars = {p for p in sidecars if p.exists()}
owner = journal.stat()
with sqlite3.connect(journal.as_uri()+'?mode=ro', uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'agent journal integrity without fault')
rows = db.execute("SELECT job,attempt,state,grant_id FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchall()
require(rows == [(payload['context']['job_id'],payload['context']['attempt_id'],'unknown',payload['unknown_grant'])], 'only bound known agent unknown may remain')
# Read-only SQLite can create WAL sidecars. Correct only our newly
# created fixed-name files, never existing state or recursive ownership.
for p in sidecars:
if p not in existing_sidecars and p.exists():
require(not p.is_symlink() and p.stat().st_uid == 0, 'new fixed SQLite sidecar owner')
os.chown(p,owner.st_uid,owner.st_gid)
if p.exists(): require(not p.is_symlink() and p.stat().st_uid == owner.st_uid, 'agent SQLite sidecar ownership preserved')
active = base/'active'
for name,expected in payload['file_witnesses'].items():
p = (base if name.startswith(('bootstrap-','recovery-','pending-recovery-')) else active)/name
require(not p.is_symlink(), 'historical witness symlink')
require(p.exists() == expected['exists'], 'historical file existence changed')
if expected['exists']:
s = p.stat()
require(s.st_ino == expected['inode'] and s.st_size == expected['size']
and s.st_mtime_ns == expected['mtime_ns'] and s.st_ctime_ns == expected['ctime_ns']
and sha(p.read_bytes()) == expected['sha256'], 'historical exact witness bytes/inode changed')
pending = Path('/run/reboot-required'); expected = payload['host_marker_witness']
require(not pending.is_symlink() and pending.exists() == expected['exists'], 'host pending reboot marker changed')
if expected['exists']:
s = pending.stat()
require(s.st_ino == expected['inode'] and s.st_size == expected['size']
and s.st_mtime_ns == expected['mtime_ns'] and s.st_ctime_ns == expected['ctime_ns']
and sha(pending.read_bytes()) == expected['sha256'], 'host pending reboot marker must remain unchanged')
ctx = json.loads((active/'context.json').read_bytes())
require(ctx == payload['context'] and ctx['grant_id'] == payload['unknown_grant'], 'original active configure context')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == ctx['boot_id'], 'original configure boot must remain current')
for name,key in (('signed-action.json','job_payload_sha256'),('expectation.json','transaction_sha256')):
require(sha((active/name).read_bytes()) == ctx[key], 'original active signed bytes')
for name,expected in payload['records'].items():
require(json.loads((active/name).read_bytes()) == expected, 'original configure native record unchanged')
require(payload['records']['sandbox-result.json'] == dict(transaction_sha256=ctx['transaction_sha256'],apt_exit_code=0,reboot_required=False), 'completed native sandbox APT0 required')
action = json.loads((active/'signed-action.json').read_bytes())['action']['apt_manifest']
tx = json.loads((active/'expectation.json').read_bytes())
ref = action['native_recovery']
require(action['schema_version'] == 4 and action['action'] == 'apt-native-acceptance' and
tx['schema_version'] == 2 and tx['operation'] == 'configure_only' and
tx['recipe_id'] == 'rmm-apt-fixture-install-1.0' and len(tx['packages']) == 1 and
tx['packages'][0]['old_version'] == tx['packages'][0]['new_version'] == '1.0' and
ref == dict(schema_version=1,previous_grant_id=payload['partial_grant'],previous_attempt_id=payload['partial_attempt'],partial_proof_sha256=payload['partial_proof_sha256']), 'exact historical schema4 configure-only reference')
require(sha((active/'baseline.json').read_bytes()) == tx['baseline_sha256'] and
sha((active/'partial-proof.json').read_bytes()) == payload['partial_proof_sha256'], 'original partial baseline and staged proof')
marker = base/('recovery-attempt-'+payload['partial_grant']+'.json')
require(json.loads(marker.read_bytes()) == dict(transaction_sha256=ctx['transaction_sha256'],job_payload_sha256=ctx['job_payload_sha256'],grant_id=ctx['grant_id'],attempt_id=ctx['attempt_id'],native_recovery=ref), 'original recovery once marker required')
require(Path('/usr/share/rmm-mvp-test/version').read_bytes() == b'1.0\n', 'existing fixture payload remains1.0')
fixture = []
for paragraph in Path('/var/lib/dpkg/status').read_text().split('\n\n'):
fields = dict(line.split(': ',1) for line in paragraph.splitlines() if ': ' in line and not line[0].isspace())
if fields.get('Package') == 'rmm-mvp-test': fixture.append(fields)
require(len(fixture) == 1 and all(fixture[0].get(k) == v for k,v in
dict(Architecture='all',Version='1.0',Status='install ok installed').items()), 'existing installed1.0 status required for copied probe')
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('pending-acceptance-*.json')), 'no fabricated normal acceptance')
require({p.name for p in base.glob('recovery-*.json')} == {marker.name} and
not list(base.glob('pending-recovery-*.json')), 'only original recovery attempt, no recovery acceptance')
return unknown_witness
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'permanent Cloud identity required')
require(not dropin.is_symlink() and dropin.read_text() in (full,reduced), 'only reviewed endpoint env allowed')
old_endpoint = dropin.read_text() == full
active_state = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip()
main_pid = int(run(['systemctl','show','rmm-apt-helper.service','-p','MainPID','--value']).strip())
require((active_state == b'inactive' and main_pid == 0) or (active_state == b'active' and main_pid > 0), 'known APT helper state required')
cgroup = Path('/sys/fs/cgroup/system.slice/rmm-apt-helper.service')
pids = {int(pid) for p in cgroup.rglob('cgroup.procs') for pid in p.read_text().split()} if cgroup.exists() else set()
require(pids == ({main_pid} if main_pid else set()), 'no native sandbox/package child may remain')
for name, pin in payload['old_files'].items():
p = Path(name); require(not p.is_symlink() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0 and sha(p.read_bytes()) == pin, 'installed file drift')
require(Path('/etc/systemd/system/rmm-apt-helper.service').read_text() == payload['unit'], 'exact reviewed APT service required')
markers = {base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json':'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053',
base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json':'a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3'}
require({p.name for p in base.glob('bootstrap-*.json')} == {p.name for p in markers} and
all(sha(p.read_bytes()) == pin for p,pin in markers.items()), 'original/retry tombstones required')
original_unknown = guard()
raw = {name:base64.b64decode(payload[name], validate=True) for name in ('helper','agent','provider')}
pins = {name:payload[name+'_sha256'] for name in ('helper','agent','provider')}
require(all(sha(raw[name]) == pin == payload[name+'_sha256'] for name,pin in pins.items()), 'qualified exact recovery runtime required')
for name in ('helper','agent'):
require(raw[name][:6] == b'\x7fELF\x02\x01' and raw[name][18:20] == b'\x3e\x00', 'Linux x64 binary required')
ast.parse(raw['provider'])
if payload.get('preflight_only') is True:
return dict(stage='preflight_validated', completed=False, preflight_only=True,
package_effects=False, maintenance_mutations=False, expected_grants=len(payload['expected_grants']),
partial_proof_sha256=payload['partial_proof_sha256'], apt_endpoint_enabled=old_endpoint,
unknown_grant=payload['unknown_grant'],original_unknown=original_unknown)
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-configured-reconcile-'+payload['id']); backup.mkdir(mode=0o700)
report = dict(stage='validated', completed=False, backup=str(backup), package_effects=False,
apt_endpoint_enabled=old_endpoint, partial_proof_sha256=payload['partial_proof_sha256'], backups={})
def save():
next_path = backup/'.report-next.json'; create(next_path,json.dumps(report,indent=2).encode())
os.replace(next_path,backup/'report.json'); fsync_dir(backup)
def replace(path, staged=None):
target = backup/str(path).lstrip('/').replace('/','__')
require(not path.is_symlink() and path.stat().st_dev == backup.stat().st_dev and not target.exists(), 'exclusive same-filesystem versioned replacement')
os.rename(path,target); fsync_dir(path.parent); fsync_dir(backup)
report['backups'][str(path)] = str(target); save()
if staged is not None:
os.rename(staged,path); fsync_dir(path.parent); fsync_dir(backup)
lock = None
try:
save()
for name in ('helper','agent','provider'): create(backup/(name+'-next'), raw[name], 0o755)
for name in ('helper','agent'): run(['/lib64/ld-linux-x86-64.so.2','--list',str(backup/(name+'-next'))])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
require(guard() == original_unknown, 'unknown outcome/start bytes unchanged')
if old_endpoint:
create(backup/'dropin-next',reduced.encode(),0o644); replace(dropin,backup/'dropin-next')
run(['systemctl','daemon-reload'])
report['apt_endpoint_enabled'] = False; save()
preserved = {str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
preserved_base = manifest(base)
witnesses = {}
for p in (Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'),dropin,
Path('/etc/systemd/system/rmm-apt-helper.service'),Path('/usr/lib/rmm-apt-transaction/hook'),
Path('/etc/rmm-remote/general-helper-proof.json'),Path('/etc/rmm-remote/helper-general.json')):
s = p.stat(); witnesses[str(p)] = (s.st_dev,s.st_ino,sha(p.read_bytes()))
general_bytes = Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
general_config = Path('/etc/rmm-remote/helper-general.json').read_bytes()
old_metadata = json.loads(metadata.read_text()); old_config = json.loads(config.read_text())
require(old_metadata['schema_version'] == 2 and old_metadata['provider_sha256'] == sha(provider.read_bytes()) and
old_metadata['hook_sha256'] == sha(Path('/usr/lib/rmm-apt-transaction/hook').read_bytes()) and
old_metadata['unit_sha256'] == sha(payload['unit'].encode()), 'old install metadata required')
for path,name in ((helper,'helper'),(agent,'agent'),(provider,'provider')): replace(path,backup/(name+'-next'))
create(backup/'metadata-next',json.dumps(dict(old_metadata,provider_sha256=sha(raw['provider']))).encode()); replace(metadata,backup/'metadata-next')
command = ['/usr/bin/python3','-I',str(provider)]
config_bytes = run(command+['print-config']); config_sha = sha(config_bytes)
require(config_sha == payload['reviewed_config_sha256'] and all(line in config_bytes for line in (
b'DPkg::ConfigurePending "false";', b'DPkg::TriggersPending "false";', b'dpkg::selection::remove::approved "false";')), 'reviewed exact configure/no-pending settings')
runtime = json.loads(run(command+['inspect-runtime'])); runtime_sha = sha(canonical(runtime))
old_config['apt_provider'].update(runtime_sha256=runtime_sha,apt_config_sha256=config_sha)
create(backup/'config-next',json.dumps(old_config).encode()); replace(config,backup/'config-next')
pin = json.loads(run([str(helper),'--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config))))
require(pin['identity'] == json.loads(general_bytes)['identity'], 'same helper identity required')
create(backup/'proof-next',json.dumps(pin).encode(),0o640,984); replace(proof_file,backup/'proof-next')
general = json.loads(run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(general == json.loads(general_bytes) and Path('/etc/rmm-remote/helper-general.json').read_bytes() == general_config, 'General helper policy unchanged')
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'identity/journal/Root ledger bytes or inode changed')
def preserved_original_base():
current = manifest(base)
require(all(current.get(name) == value for name,value in preserved_base.items() if name not in ('install-metadata.json','native-readiness.json')), 'original APT state/archive/hook/probe changed')
for name,value in witnesses.items():
p=Path(name);s=p.stat();require((s.st_dev,s.st_ino,sha(p.read_bytes())) == value, 'original package/control file bytes or inode changed')
return current
preserved_original_base(); require(guard() == original_unknown, 'unknown outcome/start bytes unchanged')
report.update(stage='installed_preserved',preserved_state=True,helper_sha256=pins['helper'],agent_sha256=pins['agent'],
unknown_grant=payload['unknown_grant'],original_unknown=original_unknown,provider_sha256=pins['provider'],runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general)
save()
# No client can reach this endpoint; it exists only for the service's
# ExecStartPre copied-database probe, then is stopped before restart.
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
replace(readiness_file)
# Daemon startup itself takes executor.lock to open the existing DB.
# Installation is complete; agent/General remain stopped during probe.
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
run(['systemctl','start','rmm-apt-helper.service'])
for _ in range(3):
status = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status, 'stable native probe helper required')
time.sleep(1)
run(['systemctl','stop','rmm-apt-helper.service'])
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
ready = json.loads(readiness_file.read_text())
require(ready['runtime'] == runtime and ready['readiness_only'] is True and ready['native_acceptance'] is False and
ready['host_status_sha256'] == sha(Path('/var/lib/dpkg/status').read_bytes()), 'exact new copied-database probe required')
workspace = base/'probes'/str(uuid.UUID(ready['probe_id']))
require((workspace/'dpkg/status').read_bytes() == Path('/var/lib/dpkg/status').read_bytes() and
(workspace/'apt.conf').read_bytes().count(b'dpkg::selection::remove::approved "false";') == 1,
'installed copied status and configure fix required')
current = preserved_original_base()
require(all(name in preserved_base or name.startswith('probes/'+ready['probe_id']+'/') or name == 'probes/'+ready['probe_id']
for name in current), 'only generated readiness probe metadata may be added')
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'journal/ledger bytes or inode changed during probe')
require(guard() == original_unknown, 'unknown outcome/start bytes unchanged')
require(dropin.read_text() == reduced and Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() == general_bytes, 'endpoint/General proof unchanged')
report.update(stage='native_probe_preserved',readiness_only=True,native_acceptance=False,
journals_preserved_before_restart=True,readiness_sha256=sha(readiness_file.read_bytes()),probe_id=ready['probe_id'])
save()
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT endpoint must remain stopped')
report.update(stage='qualified_runtime_started',completed=True);save()
except Exception as error:
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
save()
finally:
if lock is not None: fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,timeout=30)
return report
def build_payload(args):
m = cloud.contract
installed = m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
old = m.protected_json(STATE/'journal-capacity-runtime.json')
require(old['completed'] and all(old[n+'_sha256'] == pin for n,pin in OLD.items())
and installed['binary_sha256'] == OLD['agent'], 'known qualified capacity runtime required')
stored = m.protected_json(STATE/'configure-attempt-stored.json')
witnesses = m.protected_json(STATE/'configure-attempt-witnesses.json')
require(witnesses['same_boot'] is True and witnesses['original_state_changed'] is False,
'unchanged same-boot historical witnesses required')
context = stored['context']; outcome = stored['ledger']['outcome']
require(context['grant_id'] == UNKNOWN and stored['ledger']['consumed'] == 11
and stored['ledger']['unresolved'] == 1 and stored['ledger']['result'] is None
and outcome['state'] == 'unknown' and outcome['error_code'] == 'FIXTURE_UNEXPECTED_REBOOT_REQUIRED',
'exact original persisted configure unknown')
checkpoint = m.protected_json(STATE/'capacity-recovery/cases/apt_configure_recovery/checkpoint.json')
job = checkpoint['plan']['jobs'][0]
require(checkpoint['result']['state'] == 'unknown' and checkpoint['result']['result']['grant_id'] == UNKNOWN
and job['job_id'] == context['job_id']
and job['attempt_id'] == context['attempt_id'], 'bound actual UI configure attempt')
readonly = m.protected_json(STATE/'readonly-result-first.json')
require(readonly['unresolved'] == readonly['faults'] == 0 and len(readonly['proof']) == 1
and readonly['proof'][0]['grant_id'] == previous.PARTIAL_GRANT
and readonly['proof'][0]['proof_sha256'] == previous.PARTIAL_PROOF, 'original closed partial proof')
folders = [cloud.ROOT/'devstate/cloud-root/cases'/n for n in ('service_setup','service_start','service_stop','apt_native_install','query_logs')]
folders += [previous.previous.apt.prior.STATE/n for n in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
grants = [str(uuid.UUID(m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'])) for p in folders]
grants += [previous.PARTIAL_GRANT, UNKNOWN]
require(len(grants) == len(set(grants)) == 11, 'exact original ten grants plus configure337')
raw = {}; pins = {}; qualifications = {}
for name in ('helper','agent','provider'):
pin = getattr(args,name+'_sha256')
require(isinstance(pin,str) and re.fullmatch('[0-9a-f]{64}',pin) and pin != OLD[name], 'independently supplied new qualified artifact pin')
path = STATE/(name+'-configured-reconcile'+('.py' if name == 'provider' else ''))
raw[name] = path.read_bytes(); pins[name] = pin
require(hashlib.sha256(raw[name]).hexdigest() == pin, 'qualified artifact bytes drift')
if name != 'provider':
qualification = m.protected_json(path.with_name(path.name+'.json'))
require(qualification['binary_sha256'] == pin and qualification['release_build'] is True
and qualification['architecture'] == 'x86_64-unknown-linux-gnu'
and qualification['provider_sha256'] == args.provider_sha256
and qualification['source_files'], 'bounded release qualification required')
for filename,sha in qualification['source_files'].items():
source = (cloud.ROOT/filename).resolve()
require(source.is_relative_to(cloud.ROOT.resolve()) and
hashlib.sha256(source.read_bytes()).hexdigest() == sha, 'qualified source snapshot drift')
qualifications[name] = qualification
require(raw['provider'] == (cloud.ROOT/'tools/apt-transaction/apt_transaction.py').read_bytes(), 'qualified provider/source bytes mismatch')
spec = importlib.util.spec_from_file_location('reviewed_configured_reconcile_provider',cloud.ROOT/'tools/apt-transaction/apt_transaction.py')
module = importlib.util.module_from_spec(spec); sys.modules[spec.name] = module; spec.loader.exec_module(module)
return dict(id=str(uuid.uuid4()),agent_id=installed['identity']['agent_id'],expected_grants=grants,
unknown_grant=UNKNOWN,unknown_outcome=outcome,context=context,records=stored['records'],
file_witnesses=witnesses['files'],host_marker_witness=witnesses['marker_current'],
partial_grant=previous.PARTIAL_GRANT,partial_proof_sha256=previous.PARTIAL_PROOF,
partial_attempt=checkpoint['request']['parameters']['apt_manifest']['native_recovery']['previous_attempt_id'],
proof_row=readonly['proof'][0],unit=(cloud.ROOT/'deploy/linux/rmm-apt-helper.service').read_text(),
reviewed_config_sha256=hashlib.sha256(module.apt_config()).hexdigest(),
qualifications=qualifications,
**{name:base64.b64encode(data).decode() for name,data in raw.items()},
**{name+'_sha256':pin for name,pin in pins.items()}), installed
def main():
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--preflight-only',action='store_true')
mode.add_argument('--install-reviewed-artifacts',action='store_true')
for name in ('helper','agent','provider'): parser.add_argument('--'+name+'-sha256',required=True)
args = parser.parse_args(); os.umask(0o077)
require(not (STATE/'configured-reconcile-runtime.json').exists(), 'existing runtime pointer retained; never repeat maintenance automatically')
payload,installed = build_payload(args)
policy = cloud.bootstrap.rpc('server.conf',previous.closed_policy,{})
expected_policy = cloud.contract.protected_json(STATE/'capacity-recovery/recovery-policy-closed-unknown.json')
require(expected_policy['closed'] is True and expected_policy['policy_sha256'] ==
'06e4484cd95b88c77bf1674c84d867f852ee71b3083d3697f6609867b54893f7'
and policy['policy_sha256'] == expected_policy['policy_sha256'], 'exact reviewed deny-only policy must remain closed')
gate = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,
agent_id=payload['agent_id'],job_id=payload['context']['job_id']))
require(gate == dict(jobs=1,outbox=1,permits=1,unresolved=1), 'only exact approved configure unknown may remain')
old = cloud.bootstrap.rpc('linux-vm.conf',previous.installed_metadata,OLD)
payload.update(old_files=old['files'],preflight_only=args.preflight_only)
report = cloud.bootstrap.rpc('linux-vm.conf',upgrade_configured,payload)
cloud.contract.write_private(STATE/('configured-reconcile-'+('preflight-' if args.preflight_only else 'install-')+payload['id']+'.json'),dict(report,closed_policy=policy,central_gate=gate,qualification=payload['qualifications']))
if args.preflight_only:
require(report.get('preflight_only') is True and report.get('maintenance_mutations') is False, 'read-only preflight response required')
print('PASS reviewed configured-reconciliation preflight; original unknown retained'); return
require(report['completed'] and report['preserved_state'] and report['journals_preserved_before_restart']
and report['unknown_grant'] == UNKNOWN and report['provider_sha256'] == args.provider_sha256,
'preserving installation incomplete; retain report, never repeat automatically')
previous.previous.record_agent_upgrade(report,installed)
cloud.contract.write_private(STATE/'configured-reconcile-runtime.json',report)
print('PASS preserving configured-reconciliation runtime; APT endpoint stopped, normal signed RootSync still required')
if __name__ == '__main__':
try:
main()
except Exception as error:
print('CLOUD_CONFIGURED_RECONCILE_INCOMPLETE:'+type(error).__name__)
if str(error).startswith('remote bootstrap incomplete: '): print(str(error))
raise SystemExit(2) from None