From 2bf0183e68b127ace905dd6285775334a0225657 Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Sat, 14 Nov 2026 03:06:06 +0100 Subject: [PATCH] Verify preserved Linux shared-core deployment and retain failed verifier receipts --- deploy/cloud_shared_core_failed_restore.py | 176 +++++++ deploy/cloud_shared_core_upgrade.py | 389 +++++++++++++++ .../cloud_shared_core_upgrade_attestation.py | 399 +++++++++++++++ deploy/cloud_shared_core_upgrade_v2.py | 453 ++++++++++++++++++ deploy/shared-core-upgrade.md | 108 +++++ docs/handover.md | 12 + docs/implementation-status.md | 38 ++ 7 files changed, 1575 insertions(+) create mode 100644 deploy/cloud_shared_core_failed_restore.py create mode 100644 deploy/cloud_shared_core_upgrade.py create mode 100644 deploy/cloud_shared_core_upgrade_attestation.py create mode 100644 deploy/cloud_shared_core_upgrade_v2.py create mode 100644 deploy/shared-core-upgrade.md diff --git a/deploy/cloud_shared_core_failed_restore.py b/deploy/cloud_shared_core_failed_restore.py new file mode 100644 index 0000000..f670dfa --- /dev/null +++ b/deploy/cloud_shared_core_failed_restore.py @@ -0,0 +1,176 @@ +"""Exact failed shared-core case: metadata diagnosis and one old-runtime restart. + +No installation retry, binary replacement, state reset, package action or reboot. +The original upgrader and its failed receipt remain immutable. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import os +import uuid + +import cloud_shared_core_upgrade as upgrade + +CASE = '2f8de8b9-2abe-4125-aa12-ae38332a32c6' + + +def metadata(payload): + import stat + require(payload['id'] == '2f8de8b9-2abe-4125-aa12-ae38332a32c6', 'exact failed case') + result = {} + for root in (Path('/var/lib/rmm-agent-remote'), Path('/var/lib/rmm-helper'), + Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction')): + entries = [] + if root.exists(): + for path in [root, *sorted(root.rglob('*'))]: + value = path.lstat() + if not stat.S_ISDIR(value.st_mode) and (not stat.S_ISREG(value.st_mode) or value.st_nlink != 1): + require(len(entries) < 100, 'bounded special metadata') + entries.append(dict(path=str(path), mode=value.st_mode, uid=value.st_uid, + gid=value.st_gid, links=value.st_nlink, device=value.st_dev, inode=value.st_ino, + socket=stat.S_ISSOCK(value.st_mode), symlink=stat.S_ISLNK(value.st_mode), + regular=stat.S_ISREG(value.st_mode))) + result[str(root)] = entries + units = {} + for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service', 'rmm-apt-helper.service'): + run = subprocess.run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID'], + capture_output=True, timeout=10) + units[name] = dict(exit_code=run.returncode, properties=dict(line.split('=', 1) for line in run.stdout.decode().splitlines())) + return dict(id=payload['id'], special_entries=result, units=units, file_contents_read=False, mutations=False) + + +def restore(payload): + import contextlib + import fcntl + import sqlite3 + import stat + import traceback + os.umask(0o077) + require(os.geteuid() == 0 and payload['id'] == '2f8de8b9-2abe-4125-aa12-ae38332a32c6', 'exact failed bootstrap case') + action = payload['action']; require(action in ('preflight', 'start_old', 'read'), 'fixed old-runtime recovery') + folder = Path('/var/lib/rmm-bootstrap-history/shared-core-2f8de8b9-2abe-4125-aa12-ae38332a32c6') + intent = payload['intent']; state = Path('/var/lib/rmm-agent-remote'); root = Path('/var/lib/rmm-helper') + def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + def sha(raw): return hashlib.sha256(raw).hexdigest() + def run(args): return subprocess.run(args, capture_output=True, check=True, timeout=12).stdout + def sync_dir(): + fd = os.open(folder, os.O_DIRECTORY|os.O_RDONLY|os.O_NOFOLLOW) + try: os.fsync(fd) + finally: os.close(fd) + def create(path, raw): + fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as stream: stream.write(raw); stream.flush(); os.fsync(stream.fileno()) + sync_dir() + def owned(path, owner=0): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_uid == owner + and not value.st_mode&0o022, 'fixed owned recovery file') + return value + def units(): + return {name: dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID']).decode().splitlines()) + for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service', 'rmm-apt-helper.service')} + def guard(): + require((folder/'intent.json').read_bytes() == encode(intent), 'unchanged original intent') + failed = json.loads((folder/'report.json').read_bytes()) + require(sha(encode(failed)) == payload['failed_report_sha256'] and failed['completed'] is False + and failed['stage'] == 'validated' and failed['backups'] == {} and failed['error_class'] == 'RuntimeError', 'exact original failure before binary exchange') + require(all(not (folder/name).exists() and not (folder/name).is_symlink() for name in + ('agent-previous', 'helper-previous', 'sql-before.json', 'files-before.json', 'controls-before.json', 'backups-before.json')), 'installation never reached exchange/witness stage') + require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'same original boot') + identity = json.loads((state/'identity.json').read_bytes()) + require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment') + for name, path in (('agent', Path('/usr/local/libexec/rmm-agent-remote')), ('helper', Path('/usr/local/libexec/rmm-privileged-helper'))): + owned(path); require(sha(path.read_bytes()) == intent['old_'+name+'_sha256'], 'only unchanged original binaries may restart') + staged = folder/(name+'-next.upload'); owned(staged) + require(sha(staged.read_bytes()) == intent['new_'+name+'_sha256'], 'staged new artifact remains separate/unused') + for key, path in (('config_sha256', Path('/etc/rmm-remote/helper-general.json')), + ('pin_sha256', Path('/etc/rmm-remote/general-helper-proof.json')), + ('provider_sha256', Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')), + ('general_unit_sha256', Path('/etc/systemd/system/rmm-helper-cloud.service')), + ('agent_unit_sha256', Path('/etc/systemd/system/rmm-agent-remote.service'))): + owned(path); require(sha(path.read_bytes()) == intent[key], 'unchanged policy/unit/provider') + require(json.loads(Path('/etc/rmm-remote/helper-general.json').read_bytes())['allow_reboot'] is False + and b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes() + and units()['rmm-apt-helper.service']['ActiveState'] == 'inactive', 'APT endpoint and reboot remain closed') + out = {} + for name, inode in intent['database_inodes'].items(): + path = Path(name); value = owned(path, 993 if path.parent == state else 0) + require([value.st_dev, value.st_ino] == inode and value.st_nlink == 1, 'original database inode') + require(all(not Path(name+suffix).exists() and not Path(name+suffix).is_symlink() for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal') + with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db: + db.execute('PRAGMA query_only=ON'); db.execute('BEGIN') + require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'original journal integrity') + if path == root/'consumption.sqlite': + rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall() + observed = {r[0]: sha(json.dumps([dict(sha256=sha(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in rows} + require(observed == intent['root_rows'] and len(rows) == 18 and all(r[5] is not None for r in rows) + and db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'same closed Root18 without faults') + elif path.name == 'journal.sqlite': + require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 + and db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no unfinished agent effect/fault') + else: + require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0, 'unprivileged consumption closed') + out[name] = dict(device=value.st_dev, inode=value.st_ino, sha256=sha(path.read_bytes())) + return out + s = folder.lstat(); require(stat.S_ISDIR(s.st_mode) and s.st_uid == 0 and not s.st_mode&0o077, 'private retained case') + databases = guard() + if action == 'preflight': return dict(id=payload['id'], preflight=True, original_binaries=True, root_closed=18, databases=databases, units=units(), mutations=False) + path = folder/'restore-old-report.json' + if action == 'read': + result = json.loads(path.read_bytes()); result['units_now'] = units(); result['readback_verified'] = True; return result + require(all(value['ActiveState'] == 'inactive' and value['MainPID'] == '0' for value in units().values()), 'all three owned services stopped') + require(not path.exists() and not (folder/'restore-old-intent.json').exists(), 'old-runtime restart is single-use; readback only') + locks = []; report = dict(id=payload['id'], completed=False, stage='old_runtime_verified', databases_before=databases, + old_agent_sha256=intent['old_agent_sha256'], old_helper_sha256=intent['old_helper_sha256'], + jobs_created=False, grants_created=False, binaries_replaced=False, package_effects=False, reboot_requested=False) + def save(): + create(folder/'.restore-old-next.json', encode(report)); os.replace(folder/'.restore-old-next.json', path); sync_dir() + try: + for p in (state/'agent.lock', root/'executor.lock'): + fd = os.open(p, os.O_RDWR|os.O_NOFOLLOW); locks.append(fd); fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB) + require(guard() == databases, 'old data bytes unchanged before restart') + create(folder/'restore-old-intent.json', encode(dict(id=payload['id'], intent_sha256=sha(encode(intent)), failed_report_sha256=payload['failed_report_sha256']))) + save() + for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd) + locks.clear() + run(['systemctl', 'start', 'rmm-helper-cloud.service']); report['stage'] = 'old_general_started'; save() + run(['systemctl', 'start', 'rmm-agent-remote.service']) + require(all(units()[name]['ActiveState'] == 'active' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')), 'both original daemons active') + guard(); report.update(stage='original_daemons_restarted', completed=True); save() + except Exception as error: + report.update(error_class=type(error).__name__, frames=[dict(function=f.name, line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]); save() + finally: + for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd) + return report + + +def build_payload(action): + folder, intent = upgrade.existing(CASE) + failed = upgrade.cloud.contract.protected_json(folder/'install-reply.json') + require = upgrade.require + require(hashlib.sha256((upgrade.ROOT/'deploy/cloud_shared_core_upgrade.py').read_bytes()).hexdigest() + == '09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873', 'frozen original installer unchanged') + require(failed['completed'] is False and failed['stage'] == 'validated' and failed['backups'] == {}, 'same original failure') + upgrade.server_gate(intent) + return dict(id=CASE, intent=intent, action=action, + failed_report_sha256=hashlib.sha256(json.dumps(failed, sort_keys=True, separators=(',', ':')).encode()).hexdigest()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('preflight', 'start_old', 'read')) + action = parser.parse_args().action; os.umask(0o077) + payload = build_payload(action); folder, _ = upgrade.existing(CASE) + if action == 'start_old': upgrade.cloud.contract.write_private(folder/'restore-old-started.json', dict(id=CASE, failed_report_sha256=payload['failed_report_sha256'])) + result = upgrade.cloud.bootstrap.rpc('linux-vm.conf', restore, payload) + upgrade.cloud.contract.write_private(folder/('restore-old-'+action+'-'+str(uuid.uuid4())+'.json'), result) + upgrade.require(result.get('preflight') if action == 'preflight' else result.get('completed'), 'retained recovery incomplete; readback only') + print('PASS original-runtime '+action+'; no installation retry or RMM job') + + +if __name__ == '__main__': + try: main() + except Exception as error: + text = str(error); print(text if text.startswith('remote bootstrap incomplete: ') else 'OLD_RUNTIME_RECOVERY_INCOMPLETE:'+type(error).__name__) + raise SystemExit(2) from None diff --git a/deploy/cloud_shared_core_upgrade.py b/deploy/cloud_shared_core_upgrade.py new file mode 100644 index 0000000..1d1675f --- /dev/null +++ b/deploy/cloud_shared_core_upgrade.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Preserving Linux shared-core daemon upgrade, without jobs or APT effects. + +Planned, not executed: prepare; install CASE; finish CASE. Install is single-use; +an interrupted/lost reply permits finish/readback only, never an automatic retry. +The caller separately qualifies actual mTLS/Sync/MCP after the retained report. +""" +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import subprocess +import uuid + +import cloud_reboot_acceptance as reboot + +cloud = reboot.cloud +require = cloud.contract.require +ROOT = cloud.ROOT +STATE = ROOT/'devstate/shared-core/runtime-upgrade' +ARTIFACTS = ROOT/'devstate/shared-core/durable-outcome-v1' +REMOTE = '/var/lib/rmm-bootstrap-history/shared-core-' +EXPECTED_NEW = { + 'agent': 'c9c09de0ecb52b556d707cf45accb69186c73b8ec9baeeabfaeff220fafb1d4e', + 'helper': '51c3142ab332216d6a94ff64ff15cbc37ee1d5a99896c14c87bb6f62e63bb946', +} + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def vm(payload): + """Fixed bootstrap operations; source is sent by the pinned existing SSH RPC.""" + import contextlib + import fcntl + import sqlite3 + import stat + import traceback + os.umask(0o077) + ident = str(uuid.UUID(payload['id'])) + require(ident == payload['id'] and os.geteuid() == 0, 'canonical root bootstrap case') + action = payload['action'] + require(action in ('prepare', 'install', 'read'), 'fixed shared-core operation') + folder = Path('/var/lib/rmm-bootstrap-history')/('shared-core-'+ident) + state = Path('/var/lib/rmm-agent-remote'); helper_state = Path('/var/lib/rmm-helper') + agent = Path('/usr/local/libexec/rmm-agent-remote'); helper = Path('/usr/local/libexec/rmm-privileged-helper') + db_paths = (state/'journal.sqlite', state/'consumption.sqlite', helper_state/'consumption.sqlite') + paths = dict(agent=agent, helper=helper) + intent = payload['intent'] + def digest(raw): return hashlib.sha256(raw).hexdigest() + def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + def run(args, **kwargs): + return subprocess.run(args, capture_output=True, check=True, timeout=15, **kwargs).stdout + def fsync_dir(path): + fd = os.open(path, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + try: os.fsync(fd) + finally: os.close(fd) + def create(path, raw, mode=0o600): + fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, mode) + with os.fdopen(fd, 'wb') as stream: + stream.write(raw); stream.flush(); os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno()) + fsync_dir(path.parent) + def owned(path, directory=False): + value = path.lstat() + require(not path.is_symlink() and value.st_uid == 0 and not value.st_mode&0o022 + and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'fixed root-owned bootstrap path') + return value + def file(path, secret=False): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1, 'preserved regular file') + result = dict(device=value.st_dev, inode=value.st_ino, mode=value.st_mode, uid=value.st_uid, + gid=value.st_gid, size=value.st_size, mtime_ns=value.st_mtime_ns, ctime_ns=value.st_ctime_ns) + # Device private keys are never opened or exported by this upgrader. + if not secret: result['sha256'] = digest(path.read_bytes()) + return result + def tree(path, skip_databases=False, metadata_only=False): + if not path.exists(): return {'absent': True} + result = {} + for item in [path, *sorted(path.rglob('*'))]: + value = item.lstat(); require(not item.is_symlink(), 'no symlinks in preserved state') + if item in db_paths or str(item).endswith(('-journal', '-wal', '-shm')): + if skip_databases: continue + if stat.S_ISDIR(value.st_mode): + result[str(item.relative_to(path))] = dict(device=value.st_dev, inode=value.st_ino, + mode=value.st_mode, uid=value.st_uid, gid=value.st_gid) + else: + require(stat.S_ISREG(value.st_mode), 'no unexpected state object') + secret = metadata_only or item.suffix == '.key' or 'private' in item.name.lower() or 'token' in item.name.lower() + result[str(item.relative_to(path))] = file(item, secret) + return result + @contextlib.contextmanager + def database(path): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1 + and value.st_uid == (993 if path.parent == state else 0), 'fixed database owner/inode') + # These qualified Linux journals use DELETE mode. Refuse WAL state rather + # than create root-owned sidecars while opening an unprivileged journal. + require(all(not Path(str(path)+suffix).exists() and not Path(str(path)+suffix).is_symlink() + for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required') + with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db: + db.execute('PRAGMA query_only=ON'); db.execute('BEGIN') + require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'qualified DELETE journal') + require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity') + yield db + require((path.stat().st_dev, path.stat().st_ino) == (value.st_dev, value.st_ino), 'database inode preserved') + def cells(row): + return [dict(blob_sha256=digest(v), length=len(v)) if isinstance(v, bytes) else v for v in row] + def rows(db, table): + require(table.replace('_', '').isalnum(), 'SQLite identifier') + return {str(row[0]): digest(encode(cells(row[1:]))) + for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')} + def sql_witness(): + out = {} + mutable = {'counter', 'pending', 'reconcile_meta', 'helper_sync_meta', 'helper_sync_rows', 'reconcile_requests', 'job_events'} + for path in db_paths: + with database(path) as db: + schemas = {name: sql for name, sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name")} + all_rows = {name: rows(db, name) for name in schemas} + value = dict(schema=schemas, rows=all_rows, + immutable={name: value for name, value in all_rows.items() if name not in mutable}) + if path.name == 'journal.sqlite': + require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0, 'no unfinished agent job') + require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault') + value['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0] + meta = db.execute('SELECT journal_id,checkpoint,high_sequence,generation FROM reconcile_meta WHERE id=1').fetchone() + value['meta'] = list(meta) + # Completed signed requests are immutable; a pending request + # may acquire its response, but its original bytes must remain. + value['requests'] = {r[0]: dict(request=digest(r[1]), envelope=digest(r[2]), generation=r[3], + response=digest(r[4]) if r[4] is not None else None) + for r in db.execute('SELECT id,request,envelope,generation,response FROM reconcile_requests')} + if path.parent == helper_state: + root_rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall() + actual = {r[0]: digest(json.dumps([dict(sha256=digest(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in root_rows} + require(actual == intent['root_rows'] and len(actual) == 18 and all(r[5] is not None for r in root_rows), 'exact closed original Root18') + require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault') + value['meta'] = list(db.execute('SELECT journal_id,checkpoint,generation,binding_sha256 FROM helper_sync_meta WHERE id=1').fetchone()) + value['identity'] = [path.stat().st_dev, path.stat().st_ino] + require(value['identity'] == intent['database_inodes'][str(path)], 'closeout database identity') + out[str(path)] = value + return out + def service(name): + return dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'UnitFileState']).decode().splitlines()) + def controls(): + fixed = (Path('/etc/rmm-remote'), Path('/etc/systemd/system/rmm-agent-remote.service.d'), + Path('/etc/systemd/system/rmm-helper-cloud.service.d')) + result = {str(path): tree(path) for path in fixed} + for path in (Path('/etc/systemd/system/rmm-agent-remote.service'), Path('/etc/systemd/system/rmm-helper-cloud.service'), + Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'), + Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'), Path('/var/lib/dpkg/status'), + Path('/usr/share/rmm-mvp-test/version')): + result[str(path)] = file(path) + return result + def previous_backups(): + # Prior bootstrap folders are never opened for content or modified. + # Bind their complete file identities/metadata, excluding only this case. + return {str(path): tree(path, metadata_only=True) for path in sorted(folder.parent.iterdir()) if path != folder} + def guard(new=False): + require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'no reboot/identity transition') + identity = json.loads((state/'identity.json').read_bytes()) + require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment') + config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json') + require(digest(config.read_bytes()) == intent['config_sha256'] and digest(pin.read_bytes()) == intent['pin_sha256'] + and json.loads(config.read_bytes())['allow_reboot'] is False, 'closed unchanged General policy') + require(digest(Path('/etc/systemd/system/rmm-helper-cloud.service').read_bytes()) == intent['general_unit_sha256'] + and digest(Path('/etc/systemd/system/rmm-agent-remote.service').read_bytes()) == intent['agent_unit_sha256'], 'unchanged qualified service units') + binding = json.loads((state/'binding.json').read_bytes()) + cached = json.loads((state/'job-signing-key.json').read_bytes()) + helper_key = json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_bytes()) + raw_key = bytes.fromhex(cached['public_key_hex']) + require(cached == helper_key and len(raw_key) == 32 and cached['algorithm'] == 'Ed25519' + and cached['schema_version'] == 1 and digest(raw_key) == cached['key_id'] == intent['general_pin']['identity']['signing_key_id'] + and binding['public_key_hex'] == cached['public_key_hex'] + and all(binding[k] == intent['general_pin']['identity'][k] for k in + ('tenant_id', 'asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')), 'same pinned public signing identity') + require(digest(Path('/usr/lib/rmm-apt-transaction/apt_transaction.py').read_bytes()) == intent['provider_sha256'], 'provider unchanged') + require(b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes(), 'APT endpoint absent') + for name in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service'): + value = service(name); require(value['ActiveState'] == 'inactive' and value['UnitFileState'] == 'disabled', 'APT/legacy daemons remain disabled') + for name, path in paths.items(): + owned(path); require(digest(path.read_bytes()) == intent[('new_' if new else 'old_')+name+'_sha256'], 'binary compare-and-swap') + require(intent['id'] == ident and len(intent['root_rows']) == 18 and intent['jobs_created'] is False, 'fixed maintenance intent') + if action == 'prepare': + guard(); sql_witness() + for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service'): + require(service(name)['ActiveState'] == 'active', 'known original daemon active') + folder.mkdir(mode=0o700); create(folder/'intent.json', encode(intent)) + return dict(prepared=True, id=ident, intent_sha256=digest(encode(intent)), daemon_changes=False, jobs_created=False) + owned(folder, True) + require((folder/'intent.json').read_bytes() == encode(intent), 'same original retained intent') + if action == 'read': + if not (folder/'report.json').exists(): + return dict(id=ident, completed=False, stage='prepared_without_install_report', install_entered=False) + report = json.loads((folder/'report.json').read_bytes()) + if report.get('completed'): + guard(True); current = sql_witness(); before = json.loads((folder/'sql-before.json').read_bytes()) + require(all(file(folder/(name+'-previous')) == report['backups'][name] + and report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'] for name in paths), 'versioned original binary backups intact') + for path, previous in before.items(): + after = current[path] + require(after['schema'] == previous['schema'] and after['identity'] == previous['identity'], 'unchanged schema/database inode') + for table, old_rows in previous['immutable'].items(): + require(all(after['immutable'][table].get(k) == v for k, v in old_rows.items()), 'all original immutable SQL rows retained') + if 'sequence' in previous: + require(after['sequence'] >= previous['sequence'] and after['meta'][0] == previous['meta'][0] + and all(a >= b for a, b in zip(after['meta'][1:], previous['meta'][1:])), 'same journal with monotonic signed Sync/counter') + for key, old in previous['requests'].items(): + value = after['requests'].get(key) + require(value is not None and all(value[k] == old[k] for k in ('request', 'envelope', 'generation')) + and (old['response'] is None or value['response'] == old['response']), 'original signed request/response bytes retained') + elif 'meta' in previous: + require(after['meta'][0] == previous['meta'][0] and after['meta'][3] == previous['meta'][3] + and all(after['meta'][i] >= previous['meta'][i] for i in (1, 2)), 'same helper identity/monotonic checkpoint') + require(controls() == json.loads((folder/'controls-before.json').read_bytes()), 'bootstrap/policy/APT/package files unchanged') + for path, old in json.loads((folder/'files-before.json').read_bytes()).items(): + require(tree(Path(path), True) == old, 'all non-database state, private-key metadata and PKI unchanged') + old_backups = json.loads((folder/'backups-before.json').read_bytes()) + current_backups = previous_backups() + require(all(current_backups.get(path) == value for path, value in old_backups.items()), 'all earlier bootstrap backups retained') + report['readback_verified'] = True + report['services_now'] = {name: service(name) for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')} + return report + # Single-use installation: a retained report, even incomplete, forbids rerun. + require(not (folder/'report.json').exists(), 'installation already entered; readback only') + guard(); sql_witness() + for name in paths: + upload = folder/(name+'-next.upload'); value = owned(upload) + require(value.st_nlink == 1 and value.st_mode&0o777 == 0o600, 'private SFTP artifact') + raw = upload.read_bytes() + require(digest(raw) == intent['new_'+name+'_sha256'] and raw[:6] == b'\x7fELF\x02\x01' + and raw[18:20] == b'\x3e\x00', 'qualified x86-64 ELF') + upload.chmod(0o755); run(['/lib64/ld-linux-x86-64.so.2', '--list', str(upload)]) + report = dict(id=ident, completed=False, stage='validated', jobs_created=False, grants_created=False, + package_effects=False, reboot_requested=False, apt_endpoint_enabled=False, apt_pins_stale=True, + intent_sha256=digest(encode(intent)), backups={}, old_agent_sha256=intent['old_agent_sha256'], + old_helper_sha256=intent['old_helper_sha256'], agent_sha256=intent['new_agent_sha256'], + helper_sha256=intent['new_helper_sha256'], provider_sha256=intent['provider_sha256']) + def save(): + create(folder/'.report-next.json', encode(report)); os.replace(folder/'.report-next.json', folder/'report.json'); fsync_dir(folder) + locks = [] + try: + save() + run(['systemctl', 'stop', 'rmm-agent-remote.service', 'rmm-helper-cloud.service']) + for path in (state/'agent.lock', helper_state/'executor.lock'): + fd = os.open(path, os.O_RDWR|os.O_NOFOLLOW); locks.append(fd); fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB) + guard(); frozen = sql_witness() + require(all(service(name)['ActiveState'] == 'inactive' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')), 'both exact daemons stopped') + snapshots = {str(p): tree(p, True) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))} + files_full = {str(p): tree(p) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))} + control_before = controls() + backups_before = previous_backups() + create(folder/'sql-before.json', encode(frozen)); create(folder/'files-before.json', encode(snapshots)); create(folder/'controls-before.json', encode(control_before)) + create(folder/'backups-before.json', encode(backups_before)) + for name, path in paths.items(): + require(path.stat().st_dev == folder.stat().st_dev, 'same-filesystem versioned exchange') + previous = folder/(name+'-previous'); os.rename(path, previous); fsync_dir(path.parent); fsync_dir(folder) + report['backups'][name] = file(previous) + require(report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'], 'retained exact previous binary') + report['stage'] = name+'_original_retained'; save() + os.rename(folder/(name+'-next.upload'), path); fsync_dir(path.parent); fsync_dir(folder) + inspected = json.loads(run([str(helper), '--inspect-general-profile'], env=dict(os.environ, RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json'))) + require(inspected == intent['general_pin'], 'same General policy-based profile pin') + require(sql_witness() == frozen and controls() == control_before and previous_backups() == backups_before + and all(tree(Path(p)) == v for p, v in files_full.items()), 'all data bytes/rows/inodes/backups unchanged before restart') + guard(True); report.update(stage='installed_preserved', preserved_before_restart=True, + root_consumed=18, root_unresolved=0, general_pin=inspected, database_inodes=intent['database_inodes']) + save() + for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd) + locks.clear() + run(['systemctl', 'start', 'rmm-helper-cloud.service']) + require(service('rmm-helper-cloud.service')['ActiveState'] == 'active', 'qualified General helper starts') + run(['systemctl', 'start', 'rmm-agent-remote.service']) + require(service('rmm-agent-remote.service')['ActiveState'] == 'active', 'qualified unprivileged agent starts') + pid = int(service('rmm-agent-remote.service')['MainPID']) + process = Path('/proc/'+str(pid)+'/status').read_text().splitlines() + require(next(line for line in process if line.startswith('Uid:')).split()[1:] == ['993']*4 + and next(line for line in process if line.startswith('Gid:')).split()[1:] == [str(pwd.getpwnam('rmm-agent').pw_gid)]*4, + 'actual agent process remains unprivileged') + report.update(stage='qualified_daemons_started', completed=True, boot_id=intent['boot_id']); save() + except Exception as error: + report.update(error_class=type(error).__name__, frames=[dict(function=f.name, line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]) + save() + finally: + for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd) + return report + + +def qualification(): + out = {} + for name, filename in (('agent', 'rmm-agent'), ('helper', 'rmm-privileged-helper')): + path = ARTIFACTS/filename; metadata = cloud.contract.protected_json(path.with_name(filename+'.json')) + require(metadata['binary_sha256'] == EXPECTED_NEW[name] == sha(path.read_bytes()) + and metadata['architecture'] == 'x86_64-unknown-linux-gnu' and metadata['release_build'] is True + and metadata['qualification_profile'] == 'durable-outcome-v1' and metadata['source_before_after_equal'] is True + and metadata['glibc_max'] == '2.34', 'fixed qualified Bookworm release') + require(all(sha((ROOT/p).read_bytes()) == value for p, value in metadata['source_files'].items()), 'release source bindings unchanged') + out[name] = dict(path=str(path.relative_to(ROOT)), sha256=metadata['binary_sha256'], qualification_sha256=sha(path.with_name(filename+'.json').read_bytes())) + native = ROOT/'devstate/shared-core/acceptance/08a9be72-dd7c-46b7-a2c6-7d17424533c2/linux-passed.json' + result = cloud.contract.protected_json(native) + require(result['passed'] is True and result['qualification']['manifest_sha256'] == sha((ARTIFACTS/'manifest.json').read_bytes()), 'existing real Linux76 shared-core qualification') + return dict(artifacts=out, native_linux_evidence_sha256=sha(native.read_bytes())) + + +def server_gate(intent): + require(reboot.evidence(intent['target'], 'gate') == dict(jobs=30, outbox=30, permits=26, unresolved=0), 'same central30/30/26/0') + policy = cloud.bootstrap.rpc('server.conf', reboot.server_policy, dict(action='state')) + require(policy == dict(policy_sha256=intent['central_policy_sha256'], closed=True), 'same closed central Root policy') + + +def prepare(): + metadata = qualification(); closeout = cloud.contract.protected_json(ROOT/'devstate/cloud-mvp-final-closeout.json') + require(closeout['verified'] is True and closeout['central_counts'] == dict(jobs=30, outbox=30, permits=26, unresolved=0) + and closeout['policy']['closed'] is True and closeout['native']['root_unresolved'] == 0, 'completed Linux MVP closeout') + observation = closeout['observation']; require(sha((ROOT/observation['path']).read_bytes()) == observation['sha256'], 'verified closeout source') + ident = str(uuid.uuid4()); folder = STATE/ident; folder.mkdir(mode=0o700, parents=True) + native = closeout['native']; old = closeout['vm']['artifact_sha256'] + intent = dict(schema_version=1, type='rmm.linux.shared-core-upgrade', id=ident, target=closeout['target'], + boot_id=native['boot_id'], root_rows=native['root_rows'], database_inodes=native['database_inodes'], + old_agent_sha256=old['agent'], old_helper_sha256=old['helper'], new_agent_sha256=EXPECTED_NEW['agent'], + new_helper_sha256=EXPECTED_NEW['helper'], provider_sha256=old['provider'], config_sha256=native['config_sha256'], + pin_sha256=native['pin_sha256'], general_pin=native['general_pin'], central_policy_sha256=closeout['policy']['policy_sha256'], + general_unit_sha256=closeout['vm']['general_unit_sha256'], + agent_unit_sha256=sha((ROOT/'deploy/linux/rmm-agent-remote.service').read_bytes()), + closeout_sha256=sha((ROOT/'devstate/cloud-mvp-final-closeout.json').read_bytes()), qualification=metadata, + jobs_created=False, grants_created=False, reboot_requested=False) + cloud.contract.write_private(folder/'intent.json', intent); server_gate(intent) + result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='prepare', id=ident, intent=intent)) + cloud.contract.write_private(folder/'prepared.json', result); require(result.get('prepared') is True, 'retained remote intent incomplete') + print('PREPARED '+ident+'; no daemon change or RMM job') + + +def existing(ident): + folder = STATE/str(uuid.UUID(ident)); intent = cloud.contract.protected_json(folder/'intent.json') + require(intent['id'] == ident, 'same retained case'); return folder, intent + + +def install(ident): + folder, intent = existing(ident); require(qualification() == intent['qualification'], 'original qualified source/artifacts') + require(cloud.contract.protected_json(folder/'prepared.json')['prepared'] is True, 'remote preparation first') + cloud.contract.write_private(folder/'install-started.json', dict(id=ident, intent_sha256=sha((folder/'intent.json').read_bytes()))) + server_gate(intent) + cfg = cloud.bootstrap.access.load_config(ROOT/'credentials/linux-vm.conf') + args = ['sftp', '-F', '/dev/null', '-P', cfg.get('port', '22'), '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes', + '-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()), + '-b', '-', cfg['username']+'@'+cfg['address']] + for name, item in intent['qualification']['artifacts'].items(): + batch = 'put '+json.dumps(str(ROOT/item['path']))+' '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\nchmod 600 '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\n' + result = subprocess.run(args, input=batch.encode(), capture_output=True, timeout=120) + cloud.contract.write_private(folder/(name+'-sftp.json'), dict(returncode=result.returncode, + stdout_b64=base64.b64encode(result.stdout).decode(), stderr_b64=base64.b64encode(result.stderr).decode())) + require(result.returncode == 0, 'private SFTP error; retain case and inspect, never repeat install') + result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='install', id=ident, intent=intent)) + cloud.contract.write_private(folder/'install-reply.json', result) + require(result.get('completed') is True, 'retained incomplete report; readback only') + print('INSTALLED '+ident+'; finish/readback and independent native Sync qualification still required') + + +def finish(ident): + folder, intent = existing(ident) + cloud.contract.protected_json(folder/'install-started.json') + result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='read', id=ident, intent=intent)) + cloud.contract.write_private(folder/('readback-'+str(uuid.uuid4())+'.json'), result) + require(result.get('completed') is True and result.get('readback_verified') is True, 'remote maintenance incomplete; do not reinstall') + require(all(v['ActiveState'] == 'active' for v in result['services_now'].values()), 'both qualified daemons active') + server_gate(intent) + cloud.contract.write_private(folder/'result.json', dict(verified_preservation=True, native_sync_accepted=False, + id=ident, intent_sha256=sha((folder/'intent.json').read_bytes()), report=result)) + print('PASS '+ident+' preserving Linux daemon upgrade; APT closed/stale; native Sync/MCP acceptance remains separate') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('prepare', 'install', 'finish')); parser.add_argument('id', nargs='?') + args = parser.parse_args(); os.umask(0o077) + require((args.id is None) == (args.action == 'prepare'), 'prepare creates a case; install/finish require its exact ID') + if args.action == 'prepare': prepare() + elif args.action == 'install': install(args.id) + else: finish(args.id) + + +if __name__ == '__main__': + try: main() + except Exception as error: + text = str(error) + print(text if text.startswith('remote bootstrap incomplete: ') else 'SHARED_CORE_UPGRADE_INCOMPLETE:'+type(error).__name__) + raise SystemExit(2) from None diff --git a/deploy/cloud_shared_core_upgrade_attestation.py b/deploy/cloud_shared_core_upgrade_attestation.py new file mode 100644 index 0000000..98cf08c --- /dev/null +++ b/deploy/cloud_shared_core_upgrade_attestation.py @@ -0,0 +1,399 @@ +"""Read-only supplement for exact V2 UID/GID-attestation failure. + +Original runner d4f857b8 and original failed report remain immutable. Literal +readonly witness code is copied from that frozen runner, without install/start +paths or runtime evaluation. This never changes a service, file or SQL row. +""" +import hashlib +import json +import uuid +import argparse + +import cloud_shared_core_upgrade_v2 as upgrade + +CASE = 'a2423612-721f-4a6c-a832-b9eef549c41e' + + +def diagnose(payload): + """Fixed metadata-only differences; no relaxed preservation classification.""" + import contextlib + import sqlite3 + import stat + require(payload['id'] == 'a2423612-721f-4a6c-a832-b9eef549c41e' and os.geteuid() == 0, 'exact read-only diagnosis') + folder = Path('/var/lib/rmm-bootstrap-history/shared-core-v2-a2423612-721f-4a6c-a832-b9eef549c41e') + intent = payload['intent'] + require((folder/'intent.json').read_bytes() == json.dumps(intent,sort_keys=True,separators=(',',':')).encode(), 'original intent') + before = json.loads((folder/'sql-before.json').read_bytes()); changes = [] + def sha(raw): return hashlib.sha256(raw).hexdigest() + def row_hash(row): + cells=[dict(blob_sha256=sha(v),length=len(v)) if isinstance(v,bytes) else v for v in row] + return sha(json.dumps(cells,sort_keys=True,separators=(',',':')).encode()) + for name, previous in before.items(): + path=Path(name); value=path.lstat() + require(stat.S_ISREG(value.st_mode) and not path.is_symlink() and [value.st_dev,value.st_ino]==previous['identity'], 'original database') + require(all(not Path(name+suffix).exists() for suffix in ('-wal','-shm','-journal')), 'settled DELETE mode') + with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db: + db.execute('PRAGMA query_only=ON');db.execute('BEGIN') + require(db.execute('PRAGMA journal_mode').fetchone()[0]=='delete','readonly DELETE journal') + for table, old_rows in previous['immutable'].items(): + require(table.replace('_','').isalnum(),'fixed table identifier') + for key, old_sha in old_rows.items(): + require(key.isdecimal(),'original numeric rowid') + row=db.execute('SELECT * FROM "'+table+'" WHERE rowid=?',[int(key)]).fetchone() + new_sha=row_hash(row) if row is not None else None + if old_sha != new_sha: + require(len(changes)<128,'bounded changed rows') + changes.append(dict(database=str(path),table=table,rowkey=key,old_sha256=old_sha,new_sha256=new_sha)) + return dict(id=payload['id'],diagnostic_only=True,changes=changes,mutations=False,raw_rows_returned=False) + + +def attest(payload): + import contextlib + import sqlite3 + import stat + os.umask(0o077) + require(payload['id'] == 'a2423612-721f-4a6c-a832-b9eef549c41e' and payload['action'] == 'read', 'exact readonly post-start case') + ident = str(uuid.UUID(payload['id'])) + require(ident == payload['id'] and os.geteuid() == 0, 'canonical root bootstrap case') + action = payload['action'] + require(action in ('prepare', 'install', 'read'), 'fixed shared-core operation') + folder = Path('/var/lib/rmm-bootstrap-history')/('shared-core-v2-'+ident) + state = Path('/var/lib/rmm-agent-remote'); helper_state = Path('/var/lib/rmm-helper') + agent = Path('/usr/local/libexec/rmm-agent-remote'); helper = Path('/usr/local/libexec/rmm-privileged-helper') + db_paths = (state/'journal.sqlite', state/'consumption.sqlite', helper_state/'consumption.sqlite') + paths = dict(agent=agent, helper=helper) + intent = payload['intent'] + def digest(raw): return hashlib.sha256(raw).hexdigest() + def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + def run(args, **kwargs): + return subprocess.run(args, capture_output=True, check=True, timeout=15, **kwargs).stdout + def owned(path, directory=False): + value = path.lstat() + require(not path.is_symlink() and value.st_uid == 0 and not value.st_mode&0o022 + and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'fixed root-owned bootstrap path') + return value + aliases = tuple(Path('/var/lib/rmm-apt-transaction')/name for name in ( + 'acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json')) + def check_aliases(): + # Exactly the preexisting, durably linked successful publication. Never + # unlink/rewrite either name, and never admit another hardlinked object. + for path in aliases: + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_mode == 0o100600 + and (value.st_dev, value.st_ino, value.st_uid, value.st_gid, value.st_nlink) == (2049, 1028362, 0, 984, 2), + 'exact known APT publication alias identity') + require(digest(path.read_bytes()) == 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506', + 'original signed-result APT alias bytes') + def file(path, secret=False): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) + and (value.st_nlink == 1 or path in aliases), 'preserved regular file or exact known alias') + if path in aliases: check_aliases() + result = dict(device=value.st_dev, inode=value.st_ino, mode=value.st_mode, uid=value.st_uid, + gid=value.st_gid, size=value.st_size, mtime_ns=value.st_mtime_ns, ctime_ns=value.st_ctime_ns) + # Device private keys are never opened or exported by this upgrader. + if not secret: result['sha256'] = digest(path.read_bytes()) + return result + def tree(path, skip_databases=False, metadata_only=False): + if not path.exists(): return {'absent': True} + result = {} + for item in [path, *sorted(path.rglob('*'))]: + value = item.lstat(); require(not item.is_symlink(), 'no symlinks in preserved state') + if item in db_paths or str(item).endswith(('-journal', '-wal', '-shm')): + if skip_databases: continue + if stat.S_ISDIR(value.st_mode): + result[str(item.relative_to(path))] = dict(device=value.st_dev, inode=value.st_ino, + mode=value.st_mode, uid=value.st_uid, gid=value.st_gid) + else: + require(stat.S_ISREG(value.st_mode), 'no unexpected state object') + secret = metadata_only or item.suffix == '.key' or 'private' in item.name.lower() or 'token' in item.name.lower() + result[str(item.relative_to(path))] = file(item, secret) + return result + @contextlib.contextmanager + def database(path): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1 + and value.st_uid == (993 if path.parent == state else 0), 'fixed database owner/inode') + # These qualified Linux journals use DELETE mode. Refuse WAL state rather + # than create root-owned sidecars while opening an unprivileged journal. + require(all(not Path(str(path)+suffix).exists() and not Path(str(path)+suffix).is_symlink() + for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required') + with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db: + db.execute('PRAGMA query_only=ON'); db.execute('BEGIN') + require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'qualified DELETE journal') + require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity') + yield db + require((path.stat().st_dev, path.stat().st_ino) == (value.st_dev, value.st_ino), 'database inode preserved') + def cells(row): + return [dict(blob_sha256=digest(v), length=len(v)) if isinstance(v, bytes) else v for v in row] + def rows(db, table): + require(table.replace('_', '').isalnum(), 'SQLite identifier') + return {str(row[0]): digest(encode(cells(row[1:]))) + for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')} + def sql_witness(): + out = {} + mutable = {'counter', 'pending', 'reconcile_meta', 'helper_sync_meta', 'helper_sync_rows', 'reconcile_requests', 'job_events'} + for path in db_paths: + with database(path) as db: + schemas = {name: sql for name, sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name")} + all_rows = {name: rows(db, name) for name in schemas} + value = dict(schema=schemas, rows=all_rows, + immutable={name: value for name, value in all_rows.items() if name not in mutable}) + if path.name == 'journal.sqlite': + require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0, 'no unfinished agent job') + require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault') + value['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0] + meta = db.execute('SELECT journal_id,checkpoint,high_sequence,generation FROM reconcile_meta WHERE id=1').fetchone() + value['meta'] = list(meta) + # Completed signed requests are immutable; a pending request + # may acquire its response, but its original bytes must remain. + value['requests'] = {r[0]: dict(request=digest(r[1]), envelope=digest(r[2]), generation=r[3], + response=digest(r[4]) if r[4] is not None else None) + for r in db.execute('SELECT id,request,envelope,generation,response FROM reconcile_requests')} + if path.parent == helper_state: + root_rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall() + actual = {r[0]: digest(json.dumps([dict(sha256=digest(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in root_rows} + require(actual == intent['root_rows'] and len(actual) == 18 and all(r[5] is not None for r in root_rows), 'exact closed original Root18') + require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault') + value['meta'] = list(db.execute('SELECT journal_id,checkpoint,generation,binding_sha256 FROM helper_sync_meta WHERE id=1').fetchone()) + value['identity'] = [path.stat().st_dev, path.stat().st_ino] + require(value['identity'] == intent['database_inodes'][str(path)], 'closeout database identity') + out[str(path)] = value + return out + def service(name): + return dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'UnitFileState']).decode().splitlines()) + def controls(): + fixed = (Path('/etc/rmm-remote'), Path('/etc/systemd/system/rmm-agent-remote.service.d'), + Path('/etc/systemd/system/rmm-helper-cloud.service.d')) + result = {str(path): tree(path) for path in fixed} + for path in (Path('/etc/systemd/system/rmm-agent-remote.service'), Path('/etc/systemd/system/rmm-helper-cloud.service'), + Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'), + Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'), Path('/var/lib/dpkg/status'), + Path('/usr/share/rmm-mvp-test/version')): + result[str(path)] = file(path) + return result + def previous_backups(): + # Prior bootstrap folders are never opened for content or modified. + # Bind their complete file identities/metadata, excluding only this case. + return {str(path): tree(path, metadata_only=True) for path in sorted(folder.parent.iterdir()) if path != folder} + def guard(new=False): + check_aliases() + require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'no reboot/identity transition') + identity = json.loads((state/'identity.json').read_bytes()) + require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment') + config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json') + require(digest(config.read_bytes()) == intent['config_sha256'] and digest(pin.read_bytes()) == intent['pin_sha256'] + and json.loads(config.read_bytes())['allow_reboot'] is False, 'closed unchanged General policy') + require(digest(Path('/etc/systemd/system/rmm-helper-cloud.service').read_bytes()) == intent['general_unit_sha256'] + and digest(Path('/etc/systemd/system/rmm-agent-remote.service').read_bytes()) == intent['agent_unit_sha256'], 'unchanged qualified service units') + binding = json.loads((state/'binding.json').read_bytes()) + cached = json.loads((state/'job-signing-key.json').read_bytes()) + helper_key = json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_bytes()) + raw_key = bytes.fromhex(cached['public_key_hex']) + require(cached == helper_key and len(raw_key) == 32 and cached['algorithm'] == 'Ed25519' + and cached['schema_version'] == 1 and digest(raw_key) == cached['key_id'] == intent['general_pin']['identity']['signing_key_id'] + and binding['public_key_hex'] == cached['public_key_hex'] + and all(binding[k] == intent['general_pin']['identity'][k] for k in + ('tenant_id', 'asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')), 'same pinned public signing identity') + require(digest(Path('/usr/lib/rmm-apt-transaction/apt_transaction.py').read_bytes()) == intent['provider_sha256'], 'provider unchanged') + require(b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes(), 'APT endpoint absent') + for name in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service'): + value = service(name); require(value['ActiveState'] == 'inactive' and value['UnitFileState'] == 'disabled', 'APT/legacy daemons remain disabled') + for name, path in paths.items(): + owned(path); require(digest(path.read_bytes()) == intent[('new_' if new else 'old_')+name+'_sha256'], 'binary compare-and-swap') + owned(folder, True) + require((folder/'intent.json').read_bytes() == encode(intent), 'same retained V2 intent') + report_raw = (folder/'report.json').read_bytes() + report = json.loads(report_raw) + require(digest(encode(report)) == payload['original_report_sha256'] and report['completed'] is False + and report['stage'] == 'installed_preserved' and report['preserved_before_restart'] is True + and set(report['backups']) == {'agent', 'helper'} and report['root_consumed'] == 18 + and report['root_unresolved'] == 0 and report['error_class'] == 'RuntimeError' + and report['frames'] == [{'function':'vm','line':275},{'function':'require','line':5}], + 'exact last UID/GID assertion failure after retained installation') + guard(True); current = sql_witness(); before = json.loads((folder/'sql-before.json').read_bytes()) + require(all(file(folder/(name+'-previous')) == report['backups'][name] + and report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'] for name in paths), 'versioned original binary backups intact') + # reconcile.rs:468 intentionally renews ONLY the applied decision reference. + # Preserve the original complete row hash; prove the same job/attempt/outcome + # with its original signed reference, then independently verify the new one. + decision_cache = {}; reference_changes = [] + old_decisions = before[str(helper_state/'consumption.sqlite')]['immutable']['helper_sync_decisions'] + with database(helper_state/'consumption.sqlite') as db: + candidates = [(str(rowid), decision_id) for rowid,decision_id in db.execute( + 'SELECT rowid,decision_id FROM helper_sync_decisions ORDER BY rowid DESC') if str(rowid) in old_decisions] + def row_hash(row): return digest(encode(cells(row))) + def signed_decision(decision_id, original=False): + if decision_id in decision_cache: + value = decision_cache[decision_id] + require(not original or value['original'], 'original signed decision existed before restart') + return value['decision'] + with database(helper_state/'consumption.sqlite') as db: + row = db.execute('SELECT rowid,* FROM helper_sync_decisions WHERE decision_id=?', [decision_id]).fetchone() + require(row is not None, 'retained helper-applied signed decision') + protected = old_decisions.get(str(row[0])) == row_hash(row[1:]) + require(not original or protected, 'byte-identical original signed decision witness') + signed = json.loads(row[3]); raw = bytes(signed['decision_bytes']); signature = bytes(signed['signature']) + require(0 < len(raw) <= 2*1024*1024 and len(signature) == 64, 'bounded signed response') + public = bytes.fromhex(json.loads((state/'binding.json').read_bytes())['public_key_hex']) + require(len(public) == 32, 'exact pinned Ed25519 public key') + descriptors = [] + try: + # Public key, signed payload and signature only; anonymous seekable + # memory, no state/temporary-file writes or private-key reads. + for value in (bytes.fromhex('302a300506032b6570032100')+public, raw, signature): + fd = os.memfd_create('rmm-public-signature-check', os.MFD_CLOEXEC); descriptors.append(fd) + require(os.write(fd,value) == len(value), 'complete anonymous verification input');os.lseek(fd,0,0) + verified = subprocess.run(['/usr/bin/openssl','pkeyutl','-verify','-pubin','-keyform','DER', + '-inkey','/proc/self/fd/'+str(descriptors[0]),'-rawin','-in','/proc/self/fd/'+str(descriptors[1]), + '-sigfile','/proc/self/fd/'+str(descriptors[2])], pass_fds=tuple(descriptors),capture_output=True,timeout=5) + require(verified.returncode == 0, 'historical original Ed25519 signature') + finally: + for fd in descriptors: os.close(fd) + decision = json.loads(raw) + require(decision['domain'] == 'rmm.reconciliation-decision' and decision['schema_version'] == 1 + and decision['decision_id'] == decision_id and decision['key_id'] == digest(public) + and all(decision[k] == intent['general_pin']['identity'][k] for k in + ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')), + 'original tenant/device/key/signed decision domain') + import datetime + issued = datetime.datetime.fromisoformat(decision['issued_at'].replace('Z','+00:00')) + until = datetime.datetime.fromisoformat(decision['valid_until'].replace('Z','+00:00')) + require(0 < (until-issued).total_seconds() <= 60 and len(decision['attempts']) <= 256, + 'historical bounded signed decision, never a new authorization') + def length(value): + out=bytearray() + while value >= 128: out.append((value&127)|128);value >>= 7 + out.append(value);return bytes(out) + # Canonical prost SyncResponse: exactly bytes fields1/2 (proto/rmm.proto). + protobuf = b'\x0a'+length(len(raw))+raw+b'\x12\x40'+signature + with database(state/'journal.sqlite') as db: + request = db.execute('SELECT request,generation,response FROM reconcile_requests WHERE id=?', [decision['request_id']]).fetchone() + require(request is not None and request[2] == protobuf and request[0] == row[4] + and digest(request[0]) == decision['request_hash'] and request[1] == decision['connection_generation'], + 'exact agent-applied original request and helper-applied signed response') + decision_cache[decision_id] = dict(decision=decision,original=protected) + return decision + def renewed_outcome(path, key, original_sha): + require(path == str(state/'journal.sqlite'), 'only agent cached-outcome reference can renew') + with database(state/'journal.sqlite') as db: + current_row = db.execute('SELECT job,attempt,outcome,decision_id FROM reconcile_outcomes WHERE rowid=?', [int(key)]).fetchone() + require(current_row is not None, 'original cached outcome row retained') + original_id = next((decision_id for _,decision_id in candidates + if row_hash((*current_row[:3],decision_id)) == original_sha), None) + require(original_id is not None, 'original full row hash proves same job/attempt/outcome bytes') + old = signed_decision(original_id,True);new = signed_decision(current_row[3]) + selected = [] + for decision in (old,new): + found = [a for a in decision['attempts'] if a['job_id'] == current_row[0] and a['attempt_id'] == current_row[1]] + require(len(found) == 1 and found[0]['outcome_sha256'] == digest(current_row[2]) + and (not found[0]['outcome_bytes'] or bytes(found[0]['outcome_bytes']) == current_row[2]), + 'signed original outcome bytes/hash and exact job/attempt') + selected.append(found[0]) + require(all(selected[0][k] == selected[1][k] for k in ('job_payload_hash','grant_id','authorization_id','plan_hash', + 'action','execution_identity','outcome_state','outcome_sha256','original_boot_id')) + and new['journal_id'] == old['journal_id'] and new['helper_journal_id'] == old['helper_journal_id'] + and new['checkpoint'] >= old['checkpoint'] and new['connection_generation'] >= old['connection_generation'], + 'same authorized attempt/outcome with a monotonic applied signed reference') + reference_changes.append(dict(rowkey=key,original_row_sha256=original_sha,old_decision_id=original_id,new_decision_id=current_row[3], + outcome_sha256=digest(current_row[2]))) + for path, previous in before.items(): + after = current[path] + require(after['schema'] == previous['schema'] and after['identity'] == previous['identity'], 'unchanged schema/database inode') + for table, old_rows in previous['immutable'].items(): + for key, original_sha in old_rows.items(): + if after['immutable'][table].get(key) == original_sha: continue + require(table == 'reconcile_outcomes', 'all other original immutable SQL rows retained') + renewed_outcome(path,key,original_sha) + if 'sequence' in previous: + require(after['sequence'] >= previous['sequence'] and after['meta'][0] == previous['meta'][0] + and all(a >= b for a, b in zip(after['meta'][1:], previous['meta'][1:])), 'same journal with monotonic signed Sync/counter') + for key, old in previous['requests'].items(): + value = after['requests'].get(key) + require(value is not None and all(value[k] == old[k] for k in ('request', 'envelope', 'generation')) + and (old['response'] is None or value['response'] == old['response']), 'original signed request/response bytes retained') + elif 'meta' in previous: + require(after['meta'][0] == previous['meta'][0] and after['meta'][3] == previous['meta'][3] + and all(after['meta'][i] >= previous['meta'][i] for i in (1, 2)), 'same helper identity/monotonic checkpoint') + require(controls() == json.loads((folder/'controls-before.json').read_bytes()), 'bootstrap/policy/APT/package files unchanged') + for path, old in json.loads((folder/'files-before.json').read_bytes()).items(): + require(tree(Path(path), True) == old, 'all non-database state, private-key metadata and PKI unchanged') + old_backups = json.loads((folder/'backups-before.json').read_bytes()) + current_backups = previous_backups() + require(all(current_backups.get(path) == value for path, value in old_backups.items()), 'all earlier bootstrap backups retained') + observed_units = {name:service(name) for name in ('rmm-agent-remote.service','rmm-helper-cloud.service')} + require(all(value['ActiveState'] == 'active' for value in observed_units.values()), 'same two daemons active') + pid = int(observed_units['rmm-agent-remote.service']['MainPID']) + process = Path('/proc/'+str(pid)+'/status').read_text().splitlines() + actual = {label:[int(v) for v in next(line for line in process if line.startswith(label+':')).split()[1:]] for label in ('Uid','Gid')} + require(payload['expected_uids'] == [993]*4 and payload['expected_gids'] == [984]*4 + and actual['Uid'] == payload['expected_uids'] and actual['Gid'] == payload['expected_gids'], + 'actual process identity matches independently verified closeout') + sync = payload['native_sync'] + with database(helper_state/'consumption.sqlite') as db: + stored = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?', [sync['decision_id']]).fetchone() + require(stored is not None, 'exact native new-core signed decision retained') + signed = json.loads(stored[0]); raw = bytes(signed['decision_bytes']); decision = json.loads(raw) + require(digest(raw) == sync['payload_sha256'] and decision['decision_id'] == sync['decision_id'] + and decision['request_hash'] == sync['request_sha256'] and decision['boot_id'] == intent['boot_id'] + and decision['connection_generation'] == sync['generation'] == 114 + and decision['ready'] is True and decision['helper_ready'] is True, + 'actual helper-applied signed Sync matches stored server/VM evidence') + with database(state/'journal.sqlite') as db: + stored = db.execute('SELECT request,response FROM reconcile_requests WHERE id=?', [sync['request_id']]).fetchone() + require(stored is not None and digest(stored[0]) == sync['request_sha256'] and stored[1] is not None + and current[str(state/'journal.sqlite')]['meta'][3] >= sync['generation'] + and current[str(helper_state/'consumption.sqlite')]['meta'][2] >= sync['generation'], 'new-core request/response and monotonic generation retained') + require((folder/'report.json').read_bytes() == report_raw, 'original failed report remains byte-identical') + return dict(id=ident, verified_preservation=True, original_report_completed=False, + original_report_retained=True, original_report_sha256=digest(report_raw), + actual_agent_process=dict(pid=pid,uids=actual['Uid'],gids=actual['Gid']), units=observed_units, + root_consumed=18,root_unresolved=0, boot_id=intent['boot_id'], + database_inodes=intent['database_inodes'],agent_sha256=intent['new_agent_sha256'],helper_sha256=intent['new_helper_sha256'], + apt_endpoint_enabled=False,apt_pins_stale=True,jobs_created=False,grants_created=False, + package_effects=False,reboot_requested=False,services_changed=False, + native_sync_evidence_bound=True,native_sync=sync,native_sync_evidence=payload['native_sync_evidence'], + verified_outcome_reference_changes=reference_changes) + + +def main(): + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--diagnose',action='store_true');args=parser.parse_args() + folder,intent=upgrade.existing(CASE) + failed=upgrade.cloud.contract.protected_json(folder/'install-reply.json') + closeout=upgrade.cloud.contract.protected_json(upgrade.ROOT/'devstate/cloud-mvp-final-closeout.json') + upgrade.require(closeout['verified'] is True and closeout['vm']['artifact_sha256']['agent']==intent['old_agent_sha256'], 'original verified runtime closeout') + vm_path=folder/'new-core-vm-sync.json';server_path=folder/'new-core-server-sync.json' + vm=upgrade.cloud.contract.protected_json(vm_path);server=upgrade.cloud.contract.protected_json(server_path) + agent=vm['agent']['meta'];helper=vm['helper']['meta'];head=server['head'];installation=server['installation'] + upgrade.require(agent['fault'] is None and helper['fault'] is None and agent['last_decision']==helper['last_decision']==head['decision_id'] + and agent['generation']==helper['generation']==installation['connection_generation']==114 + and agent['journal_id']==head['journal_id']==helper['agent_journal_id'] + and helper['journal_id']==head['helper_journal_id']==agent['helper_id'] + and installation['connection_boot_id']==intent['boot_id'] and installation['sync_ready'] is True + and installation['helper_sync_ready'] is True and installation['restore_blocked'] is False + and installation['sequence_regressed'] is False and installation['sync_recovery_required'] is False, 'actual matching post-upgrade native Sync proofs') + decision=next(v for v in server['decisions'] if v['decision_id']==head['decision_id']) + request=next(v for v in vm['agent']['requests'] if v.get('decision',{}).get('decision_id')==head['decision_id']) + upgrade.require(decision['payload_sha256']==request['decision']['payload_sha256'] + and decision['request_hash']==request['request_sha256'] and decision['decision']['ready'] is True + and decision['decision']['helper_ready'] is True, 'same exact stored signed decision payload') + native_sync=dict(decision_id=head['decision_id'],request_id=decision['request_id'],generation=114, + payload_sha256=decision['payload_sha256'],request_sha256=decision['request_hash']) + payload=dict(id=CASE,action='read',intent=intent, + original_report_sha256=hashlib.sha256(json.dumps(failed,sort_keys=True,separators=(',',':')).encode()).hexdigest(), + expected_uids=closeout['vm']['agent_process']['uids'],expected_gids=closeout['vm']['agent_process']['gids'], + native_sync=native_sync,native_sync_evidence={str(p.relative_to(upgrade.ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (vm_path,server_path)}) + upgrade.server_gate(intent) + result=upgrade.cloud.bootstrap.rpc('linux-vm.conf',diagnose if args.diagnose else attest,payload) + upgrade.cloud.contract.write_private(folder/(('readonly-diagnostic-' if args.diagnose else 'readonly-attestation-')+str(uuid.uuid4())+'.json'),result) + if args.diagnose: + upgrade.require(result.get('diagnostic_only') is True,'diagnostic incomplete');print('PASS read-only row-hash diagnosis retained');return + upgrade.require(result.get('verified_preservation') is True, 'supplemental preservation not proven') + print('PASS readonly supplemental preservation/UID993/GID984; native Sync114 evidence bound; original failed receipt retained') + + +if __name__=='__main__': + try:main() + except Exception as error: + text=str(error);print(text if text.startswith('remote bootstrap incomplete: ') else 'SHARED_CORE_ATTESTATION_INCOMPLETE:'+type(error).__name__) + raise SystemExit(2) from None diff --git a/deploy/cloud_shared_core_upgrade_v2.py b/deploy/cloud_shared_core_upgrade_v2.py new file mode 100644 index 0000000..ae8631e --- /dev/null +++ b/deploy/cloud_shared_core_upgrade_v2.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +"""V2 preserving Linux upgrade; admits only the proven immutable APT alias pair. + +Source copy of cloud_shared_core_upgrade.py SHA09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873. +V1 and failed case2f8de8b9 remain immutable. V2 requires its actual old-runtime +restore readback before creating a NEW independent case. Alias SHA comes from +the original verified UI-approved apt_native_update checkpoint, not new effects. + +Planned, not executed: prepare --restore-readback FILE; install CASE; finish CASE. +Install is single-use; +an interrupted/lost reply permits finish/readback only, never an automatic retry. +The caller separately qualifies actual mTLS/Sync/MCP after the retained report. +""" +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import subprocess +import uuid + +import cloud_reboot_acceptance as reboot + +cloud = reboot.cloud +require = cloud.contract.require +ROOT = cloud.ROOT +STATE = ROOT/'devstate/shared-core/runtime-upgrade-v2' +ARTIFACTS = ROOT/'devstate/shared-core/durable-outcome-v1' +REMOTE = '/var/lib/rmm-bootstrap-history/shared-core-v2-' +ORIGINAL_RUNNER_SHA256 = '09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873' +FAILED_CASE = '2f8de8b9-2abe-4125-aa12-ae38332a32c6' +ALIAS_SHA256 = 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506' +EXPECTED_NEW = { + 'agent': 'c9c09de0ecb52b556d707cf45accb69186c73b8ec9baeeabfaeff220fafb1d4e', + 'helper': '51c3142ab332216d6a94ff64ff15cbc37ee1d5a99896c14c87bb6f62e63bb946', +} + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def vm(payload): + """Fixed bootstrap operations; source is sent by the pinned existing SSH RPC.""" + import contextlib + import fcntl + import sqlite3 + import stat + import traceback + os.umask(0o077) + ident = str(uuid.UUID(payload['id'])) + require(ident == payload['id'] and os.geteuid() == 0, 'canonical root bootstrap case') + action = payload['action'] + require(action in ('prepare', 'install', 'read'), 'fixed shared-core operation') + folder = Path('/var/lib/rmm-bootstrap-history')/('shared-core-v2-'+ident) + state = Path('/var/lib/rmm-agent-remote'); helper_state = Path('/var/lib/rmm-helper') + agent = Path('/usr/local/libexec/rmm-agent-remote'); helper = Path('/usr/local/libexec/rmm-privileged-helper') + db_paths = (state/'journal.sqlite', state/'consumption.sqlite', helper_state/'consumption.sqlite') + paths = dict(agent=agent, helper=helper) + intent = payload['intent'] + def digest(raw): return hashlib.sha256(raw).hexdigest() + def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + def run(args, **kwargs): + return subprocess.run(args, capture_output=True, check=True, timeout=15, **kwargs).stdout + def fsync_dir(path): + fd = os.open(path, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + try: os.fsync(fd) + finally: os.close(fd) + def create(path, raw, mode=0o600): + fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, mode) + with os.fdopen(fd, 'wb') as stream: + stream.write(raw); stream.flush(); os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno()) + fsync_dir(path.parent) + def owned(path, directory=False): + value = path.lstat() + require(not path.is_symlink() and value.st_uid == 0 and not value.st_mode&0o022 + and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'fixed root-owned bootstrap path') + return value + aliases = tuple(Path('/var/lib/rmm-apt-transaction')/name for name in ( + 'acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json')) + def check_aliases(): + # Exactly the preexisting, durably linked successful publication. Never + # unlink/rewrite either name, and never admit another hardlinked object. + for path in aliases: + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_mode == 0o100600 + and (value.st_dev, value.st_ino, value.st_uid, value.st_gid, value.st_nlink) == (2049, 1028362, 0, 984, 2), + 'exact known APT publication alias identity') + require(digest(path.read_bytes()) == 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506', + 'original signed-result APT alias bytes') + def file(path, secret=False): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) + and (value.st_nlink == 1 or path in aliases), 'preserved regular file or exact known alias') + if path in aliases: check_aliases() + result = dict(device=value.st_dev, inode=value.st_ino, mode=value.st_mode, uid=value.st_uid, + gid=value.st_gid, size=value.st_size, mtime_ns=value.st_mtime_ns, ctime_ns=value.st_ctime_ns) + # Device private keys are never opened or exported by this upgrader. + if not secret: result['sha256'] = digest(path.read_bytes()) + return result + def tree(path, skip_databases=False, metadata_only=False): + if not path.exists(): return {'absent': True} + result = {} + for item in [path, *sorted(path.rglob('*'))]: + value = item.lstat(); require(not item.is_symlink(), 'no symlinks in preserved state') + if item in db_paths or str(item).endswith(('-journal', '-wal', '-shm')): + if skip_databases: continue + if stat.S_ISDIR(value.st_mode): + result[str(item.relative_to(path))] = dict(device=value.st_dev, inode=value.st_ino, + mode=value.st_mode, uid=value.st_uid, gid=value.st_gid) + else: + require(stat.S_ISREG(value.st_mode), 'no unexpected state object') + secret = metadata_only or item.suffix == '.key' or 'private' in item.name.lower() or 'token' in item.name.lower() + result[str(item.relative_to(path))] = file(item, secret) + return result + @contextlib.contextmanager + def database(path): + value = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1 + and value.st_uid == (993 if path.parent == state else 0), 'fixed database owner/inode') + # These qualified Linux journals use DELETE mode. Refuse WAL state rather + # than create root-owned sidecars while opening an unprivileged journal. + require(all(not Path(str(path)+suffix).exists() and not Path(str(path)+suffix).is_symlink() + for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required') + with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db: + db.execute('PRAGMA query_only=ON'); db.execute('BEGIN') + require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'qualified DELETE journal') + require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity') + yield db + require((path.stat().st_dev, path.stat().st_ino) == (value.st_dev, value.st_ino), 'database inode preserved') + def cells(row): + return [dict(blob_sha256=digest(v), length=len(v)) if isinstance(v, bytes) else v for v in row] + def rows(db, table): + require(table.replace('_', '').isalnum(), 'SQLite identifier') + return {str(row[0]): digest(encode(cells(row[1:]))) + for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')} + def sql_witness(): + out = {} + mutable = {'counter', 'pending', 'reconcile_meta', 'helper_sync_meta', 'helper_sync_rows', 'reconcile_requests', 'job_events'} + for path in db_paths: + with database(path) as db: + schemas = {name: sql for name, sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name")} + all_rows = {name: rows(db, name) for name in schemas} + value = dict(schema=schemas, rows=all_rows, + immutable={name: value for name, value in all_rows.items() if name not in mutable}) + if path.name == 'journal.sqlite': + require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0, 'no unfinished agent job') + require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault') + value['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0] + meta = db.execute('SELECT journal_id,checkpoint,high_sequence,generation FROM reconcile_meta WHERE id=1').fetchone() + value['meta'] = list(meta) + # Completed signed requests are immutable; a pending request + # may acquire its response, but its original bytes must remain. + value['requests'] = {r[0]: dict(request=digest(r[1]), envelope=digest(r[2]), generation=r[3], + response=digest(r[4]) if r[4] is not None else None) + for r in db.execute('SELECT id,request,envelope,generation,response FROM reconcile_requests')} + if path.parent == helper_state: + root_rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall() + actual = {r[0]: digest(json.dumps([dict(sha256=digest(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in root_rows} + require(actual == intent['root_rows'] and len(actual) == 18 and all(r[5] is not None for r in root_rows), 'exact closed original Root18') + require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault') + value['meta'] = list(db.execute('SELECT journal_id,checkpoint,generation,binding_sha256 FROM helper_sync_meta WHERE id=1').fetchone()) + value['identity'] = [path.stat().st_dev, path.stat().st_ino] + require(value['identity'] == intent['database_inodes'][str(path)], 'closeout database identity') + out[str(path)] = value + return out + def service(name): + return dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'UnitFileState']).decode().splitlines()) + def controls(): + fixed = (Path('/etc/rmm-remote'), Path('/etc/systemd/system/rmm-agent-remote.service.d'), + Path('/etc/systemd/system/rmm-helper-cloud.service.d')) + result = {str(path): tree(path) for path in fixed} + for path in (Path('/etc/systemd/system/rmm-agent-remote.service'), Path('/etc/systemd/system/rmm-helper-cloud.service'), + Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'), + Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'), Path('/var/lib/dpkg/status'), + Path('/usr/share/rmm-mvp-test/version')): + result[str(path)] = file(path) + return result + def previous_backups(): + # Prior bootstrap folders are never opened for content or modified. + # Bind their complete file identities/metadata, excluding only this case. + return {str(path): tree(path, metadata_only=True) for path in sorted(folder.parent.iterdir()) if path != folder} + def guard(new=False): + check_aliases() + require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'no reboot/identity transition') + identity = json.loads((state/'identity.json').read_bytes()) + require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment') + config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json') + require(digest(config.read_bytes()) == intent['config_sha256'] and digest(pin.read_bytes()) == intent['pin_sha256'] + and json.loads(config.read_bytes())['allow_reboot'] is False, 'closed unchanged General policy') + require(digest(Path('/etc/systemd/system/rmm-helper-cloud.service').read_bytes()) == intent['general_unit_sha256'] + and digest(Path('/etc/systemd/system/rmm-agent-remote.service').read_bytes()) == intent['agent_unit_sha256'], 'unchanged qualified service units') + binding = json.loads((state/'binding.json').read_bytes()) + cached = json.loads((state/'job-signing-key.json').read_bytes()) + helper_key = json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_bytes()) + raw_key = bytes.fromhex(cached['public_key_hex']) + require(cached == helper_key and len(raw_key) == 32 and cached['algorithm'] == 'Ed25519' + and cached['schema_version'] == 1 and digest(raw_key) == cached['key_id'] == intent['general_pin']['identity']['signing_key_id'] + and binding['public_key_hex'] == cached['public_key_hex'] + and all(binding[k] == intent['general_pin']['identity'][k] for k in + ('tenant_id', 'asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')), 'same pinned public signing identity') + require(digest(Path('/usr/lib/rmm-apt-transaction/apt_transaction.py').read_bytes()) == intent['provider_sha256'], 'provider unchanged') + require(b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes(), 'APT endpoint absent') + for name in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service'): + value = service(name); require(value['ActiveState'] == 'inactive' and value['UnitFileState'] == 'disabled', 'APT/legacy daemons remain disabled') + for name, path in paths.items(): + owned(path); require(digest(path.read_bytes()) == intent[('new_' if new else 'old_')+name+'_sha256'], 'binary compare-and-swap') + require(intent['id'] == ident and intent['type'] == 'rmm.linux.shared-core-upgrade-v2' + and len(intent['root_rows']) == 18 and intent['jobs_created'] is False + and intent['alias_sha256'] == 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506', 'fixed maintenance intent') + if action == 'prepare': + guard(); sql_witness() + # Catch unsupported historical objects while the old daemons still run. + for path in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction')): + tree(path, True) + controls(); previous_backups() + for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service'): + require(service(name)['ActiveState'] == 'active', 'known original daemon active') + folder.mkdir(mode=0o700); create(folder/'intent.json', encode(intent)) + return dict(prepared=True, id=ident, intent_sha256=digest(encode(intent)), daemon_changes=False, jobs_created=False) + owned(folder, True) + require((folder/'intent.json').read_bytes() == encode(intent), 'same original retained intent') + if action == 'read': + if not (folder/'report.json').exists(): + return dict(id=ident, completed=False, stage='prepared_without_install_report', install_entered=False) + report = json.loads((folder/'report.json').read_bytes()) + if report.get('completed'): + guard(True); current = sql_witness(); before = json.loads((folder/'sql-before.json').read_bytes()) + require(all(file(folder/(name+'-previous')) == report['backups'][name] + and report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'] for name in paths), 'versioned original binary backups intact') + for path, previous in before.items(): + after = current[path] + require(after['schema'] == previous['schema'] and after['identity'] == previous['identity'], 'unchanged schema/database inode') + for table, old_rows in previous['immutable'].items(): + require(all(after['immutable'][table].get(k) == v for k, v in old_rows.items()), 'all original immutable SQL rows retained') + if 'sequence' in previous: + require(after['sequence'] >= previous['sequence'] and after['meta'][0] == previous['meta'][0] + and all(a >= b for a, b in zip(after['meta'][1:], previous['meta'][1:])), 'same journal with monotonic signed Sync/counter') + for key, old in previous['requests'].items(): + value = after['requests'].get(key) + require(value is not None and all(value[k] == old[k] for k in ('request', 'envelope', 'generation')) + and (old['response'] is None or value['response'] == old['response']), 'original signed request/response bytes retained') + elif 'meta' in previous: + require(after['meta'][0] == previous['meta'][0] and after['meta'][3] == previous['meta'][3] + and all(after['meta'][i] >= previous['meta'][i] for i in (1, 2)), 'same helper identity/monotonic checkpoint') + require(controls() == json.loads((folder/'controls-before.json').read_bytes()), 'bootstrap/policy/APT/package files unchanged') + for path, old in json.loads((folder/'files-before.json').read_bytes()).items(): + require(tree(Path(path), True) == old, 'all non-database state, private-key metadata and PKI unchanged') + old_backups = json.loads((folder/'backups-before.json').read_bytes()) + current_backups = previous_backups() + require(all(current_backups.get(path) == value for path, value in old_backups.items()), 'all earlier bootstrap backups retained') + report['readback_verified'] = True + report['services_now'] = {name: service(name) for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')} + return report + # Single-use installation: a retained report, even incomplete, forbids rerun. + require(not (folder/'report.json').exists(), 'installation already entered; readback only') + guard(); sql_witness() + for path in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction')): + tree(path, True) + controls(); previous_backups() + for name in paths: + upload = folder/(name+'-next.upload'); value = owned(upload) + require(value.st_nlink == 1 and value.st_mode&0o777 == 0o600, 'private SFTP artifact') + raw = upload.read_bytes() + require(digest(raw) == intent['new_'+name+'_sha256'] and raw[:6] == b'\x7fELF\x02\x01' + and raw[18:20] == b'\x3e\x00', 'qualified x86-64 ELF') + upload.chmod(0o755); run(['/lib64/ld-linux-x86-64.so.2', '--list', str(upload)]) + report = dict(id=ident, completed=False, stage='validated', jobs_created=False, grants_created=False, + package_effects=False, reboot_requested=False, apt_endpoint_enabled=False, apt_pins_stale=True, + intent_sha256=digest(encode(intent)), backups={}, old_agent_sha256=intent['old_agent_sha256'], + old_helper_sha256=intent['old_helper_sha256'], agent_sha256=intent['new_agent_sha256'], + helper_sha256=intent['new_helper_sha256'], provider_sha256=intent['provider_sha256']) + def save(): + create(folder/'.report-next.json', encode(report)); os.replace(folder/'.report-next.json', folder/'report.json'); fsync_dir(folder) + locks = [] + try: + save() + run(['systemctl', 'stop', 'rmm-agent-remote.service', 'rmm-helper-cloud.service']) + for path in (state/'agent.lock', helper_state/'executor.lock'): + fd = os.open(path, os.O_RDWR|os.O_NOFOLLOW); locks.append(fd); fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB) + guard(); frozen = sql_witness() + require(all(service(name)['ActiveState'] == 'inactive' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')), 'both exact daemons stopped') + snapshots = {str(p): tree(p, True) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))} + files_full = {str(p): tree(p) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))} + control_before = controls() + backups_before = previous_backups() + create(folder/'sql-before.json', encode(frozen)); create(folder/'files-before.json', encode(snapshots)); create(folder/'controls-before.json', encode(control_before)) + create(folder/'backups-before.json', encode(backups_before)) + for name, path in paths.items(): + require(path.stat().st_dev == folder.stat().st_dev, 'same-filesystem versioned exchange') + previous = folder/(name+'-previous'); os.rename(path, previous); fsync_dir(path.parent); fsync_dir(folder) + report['backups'][name] = file(previous) + require(report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'], 'retained exact previous binary') + report['stage'] = name+'_original_retained'; save() + os.rename(folder/(name+'-next.upload'), path); fsync_dir(path.parent); fsync_dir(folder) + inspected = json.loads(run([str(helper), '--inspect-general-profile'], env=dict(os.environ, RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json'))) + require(inspected == intent['general_pin'], 'same General policy-based profile pin') + require(sql_witness() == frozen and controls() == control_before and previous_backups() == backups_before + and all(tree(Path(p)) == v for p, v in files_full.items()), 'all data bytes/rows/inodes/backups unchanged before restart') + guard(True); report.update(stage='installed_preserved', preserved_before_restart=True, + root_consumed=18, root_unresolved=0, general_pin=inspected, database_inodes=intent['database_inodes']) + save() + for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd) + locks.clear() + run(['systemctl', 'start', 'rmm-helper-cloud.service']) + require(service('rmm-helper-cloud.service')['ActiveState'] == 'active', 'qualified General helper starts') + run(['systemctl', 'start', 'rmm-agent-remote.service']) + require(service('rmm-agent-remote.service')['ActiveState'] == 'active', 'qualified unprivileged agent starts') + pid = int(service('rmm-agent-remote.service')['MainPID']) + process = Path('/proc/'+str(pid)+'/status').read_text().splitlines() + require(next(line for line in process if line.startswith('Uid:')).split()[1:] == ['993']*4 + and next(line for line in process if line.startswith('Gid:')).split()[1:] == [str(pwd.getpwnam('rmm-agent').pw_gid)]*4, + 'actual agent process remains unprivileged') + report.update(stage='qualified_daemons_started', completed=True, boot_id=intent['boot_id']); save() + except Exception as error: + report.update(error_class=type(error).__name__, frames=[dict(function=f.name, line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]) + save() + finally: + for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd) + return report + + +def qualification(): + out = {} + for name, filename in (('agent', 'rmm-agent'), ('helper', 'rmm-privileged-helper')): + path = ARTIFACTS/filename; metadata = cloud.contract.protected_json(path.with_name(filename+'.json')) + require(metadata['binary_sha256'] == EXPECTED_NEW[name] == sha(path.read_bytes()) + and metadata['architecture'] == 'x86_64-unknown-linux-gnu' and metadata['release_build'] is True + and metadata['qualification_profile'] == 'durable-outcome-v1' and metadata['source_before_after_equal'] is True + and metadata['glibc_max'] == '2.34', 'fixed qualified Bookworm release') + require(all(sha((ROOT/p).read_bytes()) == value for p, value in metadata['source_files'].items()), 'release source bindings unchanged') + out[name] = dict(path=str(path.relative_to(ROOT)), sha256=metadata['binary_sha256'], qualification_sha256=sha(path.with_name(filename+'.json').read_bytes())) + native = ROOT/'devstate/shared-core/acceptance/08a9be72-dd7c-46b7-a2c6-7d17424533c2/linux-passed.json' + result = cloud.contract.protected_json(native) + require(result['passed'] is True and result['qualification']['manifest_sha256'] == sha((ARTIFACTS/'manifest.json').read_bytes()), 'existing real Linux76 shared-core qualification') + return dict(artifacts=out, native_linux_evidence_sha256=sha(native.read_bytes())) + + +def server_gate(intent): + require(reboot.evidence(intent['target'], 'gate') == dict(jobs=30, outbox=30, permits=26, unresolved=0), 'same central30/30/26/0') + policy = cloud.bootstrap.rpc('server.conf', reboot.server_policy, dict(action='state')) + require(policy == dict(policy_sha256=intent['central_policy_sha256'], closed=True), 'same closed central Root policy') + + +def prepare(restore_readback): + metadata = qualification(); closeout = cloud.contract.protected_json(ROOT/'devstate/cloud-mvp-final-closeout.json') + failed = ROOT/'devstate/shared-core/runtime-upgrade'/FAILED_CASE + restored_path = Path(restore_readback).resolve() + require(restored_path.parent == failed.resolve() and restored_path.name.startswith('restore-old-read-') + and restored_path.suffix == '.json', 'explicit original-case restore readback') + restored = cloud.contract.protected_json(restored_path) + require(restored['id'] == FAILED_CASE and restored['completed'] is True and restored['readback_verified'] is True + and restored['stage'] == 'original_daemons_restarted' + and all(restored['units_now'][name]['ActiveState'] == 'active' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')) + and restored['units_now']['rmm-apt-helper.service']['ActiveState'] == 'inactive', 'original runtime actually restored first') + original = ROOT/'deploy/cloud_shared_core_upgrade.py' + require(sha(original.read_bytes()) == ORIGINAL_RUNNER_SHA256, 'historical original runner unchanged') + metadata_path = failed/'special-metadata.json'; special = cloud.contract.protected_json(metadata_path) + names = ('acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json') + entries = special['special_entries'] + require(all(not value for key, value in entries.items() if key != '/var/lib/rmm-apt-transaction') + and {value['path'] for value in entries['/var/lib/rmm-apt-transaction']} == {'/var/lib/rmm-apt-transaction/'+name for name in names} + and all((value['device'], value['inode'], value['uid'], value['gid'], value['mode'], value['links']) + == (2049, 1028362, 0, 984, 0o100600, 2) and value['regular'] and not value['socket'] and not value['symlink'] + for value in entries['/var/lib/rmm-apt-transaction']), 'exact actually observed alias pair') + checkpoint_path = ROOT/'devstate/cloud-apt-recovery/configured-reconcile-runtime/cases/apt_native_update/checkpoint.json' + checkpoint = cloud.contract.protected_json(checkpoint_path) + require(checkpoint['verified'] is True and checkpoint['stage'] == 'succeeded' + and checkpoint['result']['result']['evidence']['apt']['native_record_sha256'] == ALIAS_SHA256, + 'original UI-approved native update bound alias SHA') + require(closeout['verified'] is True and closeout['central_counts'] == dict(jobs=30, outbox=30, permits=26, unresolved=0) + and closeout['policy']['closed'] is True and closeout['native']['root_unresolved'] == 0, 'completed Linux MVP closeout') + observation = closeout['observation']; require(sha((ROOT/observation['path']).read_bytes()) == observation['sha256'], 'verified closeout source') + ident = str(uuid.uuid4()); folder = STATE/ident; folder.mkdir(mode=0o700, parents=True) + native = closeout['native']; old = closeout['vm']['artifact_sha256'] + intent = dict(schema_version=1, type='rmm.linux.shared-core-upgrade-v2', id=ident, target=closeout['target'], + boot_id=native['boot_id'], root_rows=native['root_rows'], database_inodes=native['database_inodes'], + old_agent_sha256=old['agent'], old_helper_sha256=old['helper'], new_agent_sha256=EXPECTED_NEW['agent'], + new_helper_sha256=EXPECTED_NEW['helper'], provider_sha256=old['provider'], config_sha256=native['config_sha256'], + pin_sha256=native['pin_sha256'], general_pin=native['general_pin'], central_policy_sha256=closeout['policy']['policy_sha256'], + general_unit_sha256=closeout['vm']['general_unit_sha256'], + agent_unit_sha256=sha((ROOT/'deploy/linux/rmm-agent-remote.service').read_bytes()), + alias_sha256=ALIAS_SHA256, alias_metadata_sha256=sha(metadata_path.read_bytes()), alias_original_checkpoint_sha256=sha(checkpoint_path.read_bytes()), + previous_failed_case=FAILED_CASE, restored_evidence_path=str(restored_path.relative_to(ROOT)), restored_evidence_sha256=sha(restored_path.read_bytes()), + runner_sha256=sha(Path(__file__).read_bytes()), original_runner_sha256=ORIGINAL_RUNNER_SHA256, + closeout_sha256=sha((ROOT/'devstate/cloud-mvp-final-closeout.json').read_bytes()), qualification=metadata, + jobs_created=False, grants_created=False, reboot_requested=False) + cloud.contract.write_private(folder/'intent.json', intent); server_gate(intent) + result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='prepare', id=ident, intent=intent)) + cloud.contract.write_private(folder/'prepared.json', result); require(result.get('prepared') is True, 'retained remote intent incomplete') + print('PREPARED '+ident+'; no daemon change or RMM job') + + +def existing(ident): + folder = STATE/str(uuid.UUID(ident)); intent = cloud.contract.protected_json(folder/'intent.json') + require(intent['id'] == ident and intent['runner_sha256'] == sha(Path(__file__).read_bytes()) + and intent['original_runner_sha256'] == sha((ROOT/'deploy/cloud_shared_core_upgrade.py').read_bytes()), 'same retained case/source versions') + return folder, intent + + +def install(ident): + folder, intent = existing(ident); require(qualification() == intent['qualification'], 'original qualified source/artifacts') + require(cloud.contract.protected_json(folder/'prepared.json')['prepared'] is True, 'remote preparation first') + cloud.contract.write_private(folder/'install-started.json', dict(id=ident, intent_sha256=sha((folder/'intent.json').read_bytes()))) + server_gate(intent) + cfg = cloud.bootstrap.access.load_config(ROOT/'credentials/linux-vm.conf') + args = ['sftp', '-F', '/dev/null', '-P', cfg.get('port', '22'), '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes', + '-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()), + '-b', '-', cfg['username']+'@'+cfg['address']] + for name, item in intent['qualification']['artifacts'].items(): + batch = 'put '+json.dumps(str(ROOT/item['path']))+' '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\nchmod 600 '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\n' + result = subprocess.run(args, input=batch.encode(), capture_output=True, timeout=120) + cloud.contract.write_private(folder/(name+'-sftp.json'), dict(returncode=result.returncode, + stdout_b64=base64.b64encode(result.stdout).decode(), stderr_b64=base64.b64encode(result.stderr).decode())) + require(result.returncode == 0, 'private SFTP error; retain case and inspect, never repeat install') + result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='install', id=ident, intent=intent)) + cloud.contract.write_private(folder/'install-reply.json', result) + require(result.get('completed') is True, 'retained incomplete report; readback only') + print('INSTALLED '+ident+'; finish/readback and independent native Sync qualification still required') + + +def finish(ident): + folder, intent = existing(ident) + cloud.contract.protected_json(folder/'install-started.json') + result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='read', id=ident, intent=intent)) + cloud.contract.write_private(folder/('readback-'+str(uuid.uuid4())+'.json'), result) + require(result.get('completed') is True and result.get('readback_verified') is True, 'remote maintenance incomplete; do not reinstall') + require(all(v['ActiveState'] == 'active' for v in result['services_now'].values()), 'both qualified daemons active') + server_gate(intent) + cloud.contract.write_private(folder/'result.json', dict(verified_preservation=True, native_sync_accepted=False, + id=ident, intent_sha256=sha((folder/'intent.json').read_bytes()), report=result)) + print('PASS '+ident+' preserving Linux daemon upgrade; APT closed/stale; native Sync/MCP acceptance remains separate') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('prepare', 'install', 'finish')); parser.add_argument('id', nargs='?') + parser.add_argument('--restore-readback') + args = parser.parse_args(); os.umask(0o077) + require((args.id is None) == (args.action == 'prepare'), 'prepare creates a case; install/finish require its exact ID') + require(bool(args.restore_readback) == (args.action == 'prepare'), 'prepare requires exact old-runtime restore readback') + if args.action == 'prepare': prepare(args.restore_readback) + elif args.action == 'install': install(args.id) + else: finish(args.id) + + +if __name__ == '__main__': + try: main() + except Exception as error: + text = str(error) + print(text if text.startswith('remote bootstrap incomplete: ') else 'SHARED_CORE_UPGRADE_INCOMPLETE:'+type(error).__name__) + raise SystemExit(2) from None diff --git a/deploy/shared-core-upgrade.md b/deploy/shared-core-upgrade.md new file mode 100644 index 0000000..7747568 --- /dev/null +++ b/deploy/shared-core-upgrade.md @@ -0,0 +1,108 @@ +# Preserving Linux shared-core runtime qualification + +The notebook only builds and orchestrates. The fixed Cloud Linux VM remains the +client; `discovery.ease-technologies.de` remains the server. These bootstrap +tools never create or approve an RMM job, enroll, reset a journal, install a +package or reboot the guest. + +The original `cloud_shared_core_upgrade.py` remains frozen at SHA-256 +`09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873`. +Its case `2f8de8b9-2abe-4125-aa12-ae38332a32c6` stopped before any binary exchange: +the generic `nlink == 1` check rejected an existing valid APT publication pair. +The original failed report, staged binaries and APT records remain retained. +`cloud_shared_core_failed_restore.py` restarts only the verified original +runtime, once; its saved actual readback is required before a new V2 case. + +V2 is a source copy with a narrow alias admission and earlier object checks: +`cloud_shared_core_upgrade_v2.py`, SHA-256 +`d4f857b8ff6e7d9802f863a120a1576ffac5b36facb6a259149409ef25bfbc91`. +The only permitted alias paths are under `/var/lib/rmm-apt-transaction/`: + +- `acceptance-rmm-apt-fixture-update-1.1.json` +- `pending-acceptance-rmm-apt-fixture-update-1.1.json` + +Both must retain device `2049`, inode `1028362`, UID `0`, GID `984`, regular-file +mode `0600`, link count `2`, and SHA-256 +`a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506`. +That content hash comes from the original verified, UI-approved native APT +update result. The mode is the actual stored `lstat` value `33152` (`0100600`). +Neither name is rewritten, unlinked or chmodded. Other hardlinks, symlinks and +unexpected nonregular state objects remain rejected. + +The qualified `durable-outcome-v1` release artifacts remain: + +| Artifact | SHA-256 | +|---|---| +| Linux agent | `c9c09de0ecb52b556d707cf45accb69186c73b8ec9baeeabfaeff220fafb1d4e` | +| Linux helper | `51c3142ab332216d6a94ff64ff15cbc37ee1d5a99896c14c87bb6f62e63bb946` | +| Unchanged APT provider | `1dc0237dd8df9c9d22568bb1f6b047ede73bc2a5fca5c7fcedb7f24ab2a58b1e` | + +Planned, not executed by this preparation: + +```text +python3 deploy/cloud_shared_core_upgrade_v2.py prepare --restore-readback +python3 deploy/cloud_shared_core_upgrade_v2.py install +python3 deploy/cloud_shared_core_upgrade_v2.py finish +``` + +Each new intent binds both runner hashes, the restore readback, actual alias +metadata, original APT checkpoint, release qualifications and the Linux-MVP +closeout. Local cases use `devstate/shared-core/runtime-upgrade-v2/`; remote +cases use `/var/lib/rmm-bootstrap-history/shared-core-v2-`. A durable +installation receipt forbids repeating installation, including after a lost +reply; only readback/finish of that same case is permitted. + +The pre-restart preservation checks run under both existing locks. Old +consumptions, outcomes, signatures, database identities, trust and backup +witnesses remain bound. Private key contents are never read or exported. +General policy/pin remain unchanged, `allow_reboot=false`; the APT endpoint +stays closed and its historical runtime pins remain stale. Native mTLS/Sync, +MCP and central-count qualification are separate final acceptance steps. + +V2 preparation checks passed locally: Python syntax, fixed RPC globals, source +format, all 65 qualified release source bindings and the existing native Linux76 +core-unit receipt. This document does not claim native V2 installation success. + +## Retained native bootstrap assertion failures + +The original V1 failure above was a bootstrap verifier error: the existing +legitimate APT hardlink publication was rejected. Neither alias was repaired or +rewritten. Its original-runtime restart and actual readback were separately +recorded before the new V2 case. + +V2 case `a2423612-721f-4a6c-a832-b9eef549c41e` subsequently reached the persisted +`installed_preserved` stage: both new binaries were installed, all pre-restart +data witnesses matched, and both daemons started. Its last verifier assertion +incorrectly compared process GID with the account's primary passwd GID. Actual +readback showed the expected service identity, UID993/GID984, matching the +independently verified Linux-MVP closeout. This was a second bootstrap verifier +error; the report remains `completed=false` and is never rewritten as success. + +`cloud_shared_core_upgrade_attestation.py` is a separate read-only supplement +for that exact case. It rechecks preserved rows/schema/inodes/trust/backups and +the actual process identity, and binds the stored native VM/server Sync114 +evidence to the retained exact signed decision and request. It does not start +services, retry installation, change SQL, alter the original report, or create +jobs. Its prepared SHA-256 is +`59b20ecd2d32ea3b37f2840868eb67ce677ffdb967e079a667e53c7907fce86b`. + +## Actual read-only closeout, 2026-11-14 + +The supplemental attestation was actually executed and passed; its protected +receipt is `readonly-attestation-63730c38-327e-4635-802f-8289c28081de.json` +inside the V2 case. A prior strict row-hash readback rejected28 cached outcome +references that normal signed Sync had renewed. Only those reference changes +were then independently proven: reconstruct the original complete row hash with +identical job/attempt/outcome bytes and the preserved original decision ID; +verify both original Ed25519 decisions and the exact applied request/response, +grant/plan/outcome binding and monotonic journal history. All other historical +rows remain byte-identical. No general outcome exemption or state rewrite. + +Native MCP subsequently read the existing Linux device online, rejected the +foreign tenant, and confirmed unchanged30/30/26/0 central counts. Agent/helper +and server share the ready signed Sync114; original18 Root consumptions remain +closed. `devstate/cloud-shared-core-current-runtime.json` binds both actual +attestation and MCP receipts. It supplements the preserved historical MVP +closeout and does not requalify APT for the new binary hashes. All original +failure reports remain unchanged, and no installation or service start was +repeated to obtain this read-only closeout. diff --git a/docs/handover.md b/docs/handover.md index bfabefb..731102d 100644 --- a/docs/handover.md +++ b/docs/handover.md @@ -52,6 +52,18 @@ Zugänge und private Rohbelege bleiben gitignoriert. ## Aktuelle Windowsfortsetzung +Aktueller Linux-Dienststand nach Shared-Core-Fortsetzung: Agent `c9c09de0…`, +Helper `51c3142a…`, Provider/Units/Policies unverändert. Eigener Fall `a2423612…` +hat beide Binärdateien erhaltend installiert; ein getrennter Nur-Lese-Nachbeleg +bestätigt UID993/GID984, Root18/0, historische DB-/PKI-/Outcomeerhaltung und +normalen signierten Sync114. Zentral30/30/26/0, MCP online/fremder Tenant +verweigert; keine neuen Jobs oder Reboots. Originale Fehlerberichte bleiben +erhalten (zu strenger Hardlinkcheck, falscher letzter Gruppenvergleich und +zunächst nicht berücksichtigte signierte Cache-Referenzerneuerung). +`devstate/cloud-shared-core-current-runtime.json` bindet die tatsächlichen +Zusatzbelege. Historischer Abschluss und alte Paketabnahmen bleiben unverändert; +APT ist weiter geschlossen und für die neuen Hashes nicht erneut qualifiziert. + Neuester passiver Windowsstand: Agent `f64e8115…`, eigener erhaltender Fall `c4f9b280-e83d-4b8e-a1c2-1cd103f73eda`, tatsächlicher LocalService/Auto/WSS, gleiche Identität/Schlüsselmetadaten/ACLs und ursprüngliche Telemetrie-DB. diff --git a/docs/implementation-status.md b/docs/implementation-status.md index 61d1569..3ab21dd 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -1,5 +1,43 @@ # Tatsächlicher Implementierungsstand +## Gemeinsamer Kern im tatsächlichen Linux-Dienstbetrieb nachgewiesen — 2026-11-14 + +Agent `c9c09de0…` und Generalhelper `51c3142a…` laufen jetzt auf der Cloud- +Linux-VM mit dem zuvor nativ qualifizierten gemeinsamen Kern. Tatsächlicher +Agent weiterhin UID993/GID984, beide Dienste aktiv, gleiche Geräteidentität, +Bootkennung, DB-Inodes, PKI und ursprüngliche18 geschlossenen Root-Verbräuche. +Die alten Binärdateien sind versioniert erhalten. Neuer normaler signierter +Sync Generation114: derselbe Entscheid in Agent/Helper/Server, ready/helper_ready, +kein Restoreblock/Fault. MCP liest das gespeicherte Gerät online; fremder Tenant +verweigert. Zentral weiterhin30 Jobs/30 Outbox/26 Grants/0 ungeklärt, Rootscope +geschlossen, keine neue Genehmigung oder RMM-Wirkung. + +Die erste Installationsprüfung `2f8de8b9…` brach vor Binärtausch an zwei gültigen +APT-Hardlink-Abnahmedateien ab. Beide alten Dienste wurden unter derselben +erhaltenen Intention einmalig wieder gestartet, ohne Dateien/Verbräuche zu +ändern. Ein eigener zweiter Fall `a2423612…` bewahrte anschließend alle Daten +vor dem Tausch und startete beide neuen Dienste; nur sein letzter Check +verwechselte primäre Benutzergruppe und festgelegte Dienstgruppe. Der originale +Fehlerbericht bleibt `completed=false`; keine Wiederholung des Installers. + +Ein separater tatsächlicher Nur-Lese-Nachbeleg bestätigt die korrekte Dienst- +identität und alle erhaltenen Originale.28 lokale Outcome-Caches erneuern im +normalen Sync ihre Entscheidungsreferenz; auch das wurde zunächst als +Byteabweichung gemeldet. Danach ausschließlich diese Referenzerneuerung streng +nachgewiesen: ursprünglicher vollständiger Zeilenhash aus unveränderten Job-/ +Versuchs-/Outcome-Bytes rekonstruiert, alte/neue Ed25519-Entscheide unabhängig +verifiziert, ursprüngliche Anträge/Responses und Plan-/Grant-/Outcomebindungen +belegt. Alle übrigen historischen Zeilen unverändert. Kein pauschales Ausnehmen +von Outcomes, kein Reset oder nachträgliches Umschreiben des Fehlerberichts. + +Geschützte Fälle unter `devstate/shared-core/runtime-upgrade{,-v2}/`; separater +aktueller Beleg `devstate/cloud-shared-core-current-runtime.json`. Historischer +MVP-Abschluss bleibt unverändert. APT bleibt inaktiv, Endpunkt entfernt und +General `allow_reboot=false`. Die früheren APT-Paketabnahmen gelten weiter für +ihre ursprünglichen Artefakte; neue Agent-/Helperhashes ergeben ausdrücklich +keine neue APT-Readiness-/Paketqualifikation. Runner und Erhaltungspins: +[Shared-Core-Dienstwechsel](../deploy/shared-core-upgrade.md). + ## Gebundenes Unknown-Folgeergebnis wird dauerhaft bestätigt — lokal geprüft Ein nachträgliches authentisiertes JobResult mit höherer Revision darf jetzt