Verify preserved Linux shared-core deployment and retain failed verifier receipts
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-14 03:06:06 +01:00
parent c80f4d4a6e
commit 2bf0183e68
7 changed files with 1575 additions and 0 deletions
+176
View File
@@ -0,0 +1,176 @@
"""Exact failed shared-core case: metadata diagnosis and one old-runtime restart.
No installation retry, binary replacement, state reset, package action or reboot.
The original upgrader and its failed receipt remain immutable.
"""
import argparse
import hashlib
import json
from pathlib import Path
import os
import uuid
import cloud_shared_core_upgrade as upgrade
CASE = '2f8de8b9-2abe-4125-aa12-ae38332a32c6'
def metadata(payload):
import stat
require(payload['id'] == '2f8de8b9-2abe-4125-aa12-ae38332a32c6', 'exact failed case')
result = {}
for root in (Path('/var/lib/rmm-agent-remote'), Path('/var/lib/rmm-helper'),
Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction')):
entries = []
if root.exists():
for path in [root, *sorted(root.rglob('*'))]:
value = path.lstat()
if not stat.S_ISDIR(value.st_mode) and (not stat.S_ISREG(value.st_mode) or value.st_nlink != 1):
require(len(entries) < 100, 'bounded special metadata')
entries.append(dict(path=str(path), mode=value.st_mode, uid=value.st_uid,
gid=value.st_gid, links=value.st_nlink, device=value.st_dev, inode=value.st_ino,
socket=stat.S_ISSOCK(value.st_mode), symlink=stat.S_ISLNK(value.st_mode),
regular=stat.S_ISREG(value.st_mode)))
result[str(root)] = entries
units = {}
for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service', 'rmm-apt-helper.service'):
run = subprocess.run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID'],
capture_output=True, timeout=10)
units[name] = dict(exit_code=run.returncode, properties=dict(line.split('=', 1) for line in run.stdout.decode().splitlines()))
return dict(id=payload['id'], special_entries=result, units=units, file_contents_read=False, mutations=False)
def restore(payload):
import contextlib
import fcntl
import sqlite3
import stat
import traceback
os.umask(0o077)
require(os.geteuid() == 0 and payload['id'] == '2f8de8b9-2abe-4125-aa12-ae38332a32c6', 'exact failed bootstrap case')
action = payload['action']; require(action in ('preflight', 'start_old', 'read'), 'fixed old-runtime recovery')
folder = Path('/var/lib/rmm-bootstrap-history/shared-core-2f8de8b9-2abe-4125-aa12-ae38332a32c6')
intent = payload['intent']; state = Path('/var/lib/rmm-agent-remote'); root = Path('/var/lib/rmm-helper')
def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def sha(raw): return hashlib.sha256(raw).hexdigest()
def run(args): return subprocess.run(args, capture_output=True, check=True, timeout=12).stdout
def sync_dir():
fd = os.open(folder, os.O_DIRECTORY|os.O_RDONLY|os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
def create(path, raw):
fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'wb') as stream: stream.write(raw); stream.flush(); os.fsync(stream.fileno())
sync_dir()
def owned(path, owner=0):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_uid == owner
and not value.st_mode&0o022, 'fixed owned recovery file')
return value
def units():
return {name: dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID']).decode().splitlines())
for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service', 'rmm-apt-helper.service')}
def guard():
require((folder/'intent.json').read_bytes() == encode(intent), 'unchanged original intent')
failed = json.loads((folder/'report.json').read_bytes())
require(sha(encode(failed)) == payload['failed_report_sha256'] and failed['completed'] is False
and failed['stage'] == 'validated' and failed['backups'] == {} and failed['error_class'] == 'RuntimeError', 'exact original failure before binary exchange')
require(all(not (folder/name).exists() and not (folder/name).is_symlink() for name in
('agent-previous', 'helper-previous', 'sql-before.json', 'files-before.json', 'controls-before.json', 'backups-before.json')), 'installation never reached exchange/witness stage')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'same original boot')
identity = json.loads((state/'identity.json').read_bytes())
require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment')
for name, path in (('agent', Path('/usr/local/libexec/rmm-agent-remote')), ('helper', Path('/usr/local/libexec/rmm-privileged-helper'))):
owned(path); require(sha(path.read_bytes()) == intent['old_'+name+'_sha256'], 'only unchanged original binaries may restart')
staged = folder/(name+'-next.upload'); owned(staged)
require(sha(staged.read_bytes()) == intent['new_'+name+'_sha256'], 'staged new artifact remains separate/unused')
for key, path in (('config_sha256', Path('/etc/rmm-remote/helper-general.json')),
('pin_sha256', Path('/etc/rmm-remote/general-helper-proof.json')),
('provider_sha256', Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')),
('general_unit_sha256', Path('/etc/systemd/system/rmm-helper-cloud.service')),
('agent_unit_sha256', Path('/etc/systemd/system/rmm-agent-remote.service'))):
owned(path); require(sha(path.read_bytes()) == intent[key], 'unchanged policy/unit/provider')
require(json.loads(Path('/etc/rmm-remote/helper-general.json').read_bytes())['allow_reboot'] is False
and b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes()
and units()['rmm-apt-helper.service']['ActiveState'] == 'inactive', 'APT endpoint and reboot remain closed')
out = {}
for name, inode in intent['database_inodes'].items():
path = Path(name); value = owned(path, 993 if path.parent == state else 0)
require([value.st_dev, value.st_ino] == inode and value.st_nlink == 1, 'original database inode')
require(all(not Path(name+suffix).exists() and not Path(name+suffix).is_symlink() for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal')
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'original journal integrity')
if path == root/'consumption.sqlite':
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
observed = {r[0]: sha(json.dumps([dict(sha256=sha(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in rows}
require(observed == intent['root_rows'] and len(rows) == 18 and all(r[5] is not None for r in rows)
and db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'same closed Root18 without faults')
elif path.name == 'journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0
and db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no unfinished agent effect/fault')
else:
require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0, 'unprivileged consumption closed')
out[name] = dict(device=value.st_dev, inode=value.st_ino, sha256=sha(path.read_bytes()))
return out
s = folder.lstat(); require(stat.S_ISDIR(s.st_mode) and s.st_uid == 0 and not s.st_mode&0o077, 'private retained case')
databases = guard()
if action == 'preflight': return dict(id=payload['id'], preflight=True, original_binaries=True, root_closed=18, databases=databases, units=units(), mutations=False)
path = folder/'restore-old-report.json'
if action == 'read':
result = json.loads(path.read_bytes()); result['units_now'] = units(); result['readback_verified'] = True; return result
require(all(value['ActiveState'] == 'inactive' and value['MainPID'] == '0' for value in units().values()), 'all three owned services stopped')
require(not path.exists() and not (folder/'restore-old-intent.json').exists(), 'old-runtime restart is single-use; readback only')
locks = []; report = dict(id=payload['id'], completed=False, stage='old_runtime_verified', databases_before=databases,
old_agent_sha256=intent['old_agent_sha256'], old_helper_sha256=intent['old_helper_sha256'],
jobs_created=False, grants_created=False, binaries_replaced=False, package_effects=False, reboot_requested=False)
def save():
create(folder/'.restore-old-next.json', encode(report)); os.replace(folder/'.restore-old-next.json', path); sync_dir()
try:
for p in (state/'agent.lock', root/'executor.lock'):
fd = os.open(p, os.O_RDWR|os.O_NOFOLLOW); locks.append(fd); fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
require(guard() == databases, 'old data bytes unchanged before restart')
create(folder/'restore-old-intent.json', encode(dict(id=payload['id'], intent_sha256=sha(encode(intent)), failed_report_sha256=payload['failed_report_sha256'])))
save()
for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd)
locks.clear()
run(['systemctl', 'start', 'rmm-helper-cloud.service']); report['stage'] = 'old_general_started'; save()
run(['systemctl', 'start', 'rmm-agent-remote.service'])
require(all(units()[name]['ActiveState'] == 'active' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')), 'both original daemons active')
guard(); report.update(stage='original_daemons_restarted', completed=True); save()
except Exception as error:
report.update(error_class=type(error).__name__, frames=[dict(function=f.name, line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]); save()
finally:
for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd)
return report
def build_payload(action):
folder, intent = upgrade.existing(CASE)
failed = upgrade.cloud.contract.protected_json(folder/'install-reply.json')
require = upgrade.require
require(hashlib.sha256((upgrade.ROOT/'deploy/cloud_shared_core_upgrade.py').read_bytes()).hexdigest()
== '09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873', 'frozen original installer unchanged')
require(failed['completed'] is False and failed['stage'] == 'validated' and failed['backups'] == {}, 'same original failure')
upgrade.server_gate(intent)
return dict(id=CASE, intent=intent, action=action,
failed_report_sha256=hashlib.sha256(json.dumps(failed, sort_keys=True, separators=(',', ':')).encode()).hexdigest())
def main():
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('preflight', 'start_old', 'read'))
action = parser.parse_args().action; os.umask(0o077)
payload = build_payload(action); folder, _ = upgrade.existing(CASE)
if action == 'start_old': upgrade.cloud.contract.write_private(folder/'restore-old-started.json', dict(id=CASE, failed_report_sha256=payload['failed_report_sha256']))
result = upgrade.cloud.bootstrap.rpc('linux-vm.conf', restore, payload)
upgrade.cloud.contract.write_private(folder/('restore-old-'+action+'-'+str(uuid.uuid4())+'.json'), result)
upgrade.require(result.get('preflight') if action == 'preflight' else result.get('completed'), 'retained recovery incomplete; readback only')
print('PASS original-runtime '+action+'; no installation retry or RMM job')
if __name__ == '__main__':
try: main()
except Exception as error:
text = str(error); print(text if text.startswith('remote bootstrap incomplete: ') else 'OLD_RUNTIME_RECOVERY_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+389
View File
@@ -0,0 +1,389 @@
#!/usr/bin/env python3
"""Preserving Linux shared-core daemon upgrade, without jobs or APT effects.
Planned, not executed: prepare; install CASE; finish CASE. Install is single-use;
an interrupted/lost reply permits finish/readback only, never an automatic retry.
The caller separately qualifies actual mTLS/Sync/MCP after the retained report.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import subprocess
import uuid
import cloud_reboot_acceptance as reboot
cloud = reboot.cloud
require = cloud.contract.require
ROOT = cloud.ROOT
STATE = ROOT/'devstate/shared-core/runtime-upgrade'
ARTIFACTS = ROOT/'devstate/shared-core/durable-outcome-v1'
REMOTE = '/var/lib/rmm-bootstrap-history/shared-core-'
EXPECTED_NEW = {
'agent': 'c9c09de0ecb52b556d707cf45accb69186c73b8ec9baeeabfaeff220fafb1d4e',
'helper': '51c3142ab332216d6a94ff64ff15cbc37ee1d5a99896c14c87bb6f62e63bb946',
}
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def vm(payload):
"""Fixed bootstrap operations; source is sent by the pinned existing SSH RPC."""
import contextlib
import fcntl
import sqlite3
import stat
import traceback
os.umask(0o077)
ident = str(uuid.UUID(payload['id']))
require(ident == payload['id'] and os.geteuid() == 0, 'canonical root bootstrap case')
action = payload['action']
require(action in ('prepare', 'install', 'read'), 'fixed shared-core operation')
folder = Path('/var/lib/rmm-bootstrap-history')/('shared-core-'+ident)
state = Path('/var/lib/rmm-agent-remote'); helper_state = Path('/var/lib/rmm-helper')
agent = Path('/usr/local/libexec/rmm-agent-remote'); helper = Path('/usr/local/libexec/rmm-privileged-helper')
db_paths = (state/'journal.sqlite', state/'consumption.sqlite', helper_state/'consumption.sqlite')
paths = dict(agent=agent, helper=helper)
intent = payload['intent']
def digest(raw): return hashlib.sha256(raw).hexdigest()
def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def run(args, **kwargs):
return subprocess.run(args, capture_output=True, check=True, timeout=15, **kwargs).stdout
def fsync_dir(path):
fd = os.open(path, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
def create(path, raw, mode=0o600):
fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw); stream.flush(); os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
fsync_dir(path.parent)
def owned(path, directory=False):
value = path.lstat()
require(not path.is_symlink() and value.st_uid == 0 and not value.st_mode&0o022
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'fixed root-owned bootstrap path')
return value
def file(path, secret=False):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1, 'preserved regular file')
result = dict(device=value.st_dev, inode=value.st_ino, mode=value.st_mode, uid=value.st_uid,
gid=value.st_gid, size=value.st_size, mtime_ns=value.st_mtime_ns, ctime_ns=value.st_ctime_ns)
# Device private keys are never opened or exported by this upgrader.
if not secret: result['sha256'] = digest(path.read_bytes())
return result
def tree(path, skip_databases=False, metadata_only=False):
if not path.exists(): return {'absent': True}
result = {}
for item in [path, *sorted(path.rglob('*'))]:
value = item.lstat(); require(not item.is_symlink(), 'no symlinks in preserved state')
if item in db_paths or str(item).endswith(('-journal', '-wal', '-shm')):
if skip_databases: continue
if stat.S_ISDIR(value.st_mode):
result[str(item.relative_to(path))] = dict(device=value.st_dev, inode=value.st_ino,
mode=value.st_mode, uid=value.st_uid, gid=value.st_gid)
else:
require(stat.S_ISREG(value.st_mode), 'no unexpected state object')
secret = metadata_only or item.suffix == '.key' or 'private' in item.name.lower() or 'token' in item.name.lower()
result[str(item.relative_to(path))] = file(item, secret)
return result
@contextlib.contextmanager
def database(path):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1
and value.st_uid == (993 if path.parent == state else 0), 'fixed database owner/inode')
# These qualified Linux journals use DELETE mode. Refuse WAL state rather
# than create root-owned sidecars while opening an unprivileged journal.
require(all(not Path(str(path)+suffix).exists() and not Path(str(path)+suffix).is_symlink()
for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required')
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'qualified DELETE journal')
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity')
yield db
require((path.stat().st_dev, path.stat().st_ino) == (value.st_dev, value.st_ino), 'database inode preserved')
def cells(row):
return [dict(blob_sha256=digest(v), length=len(v)) if isinstance(v, bytes) else v for v in row]
def rows(db, table):
require(table.replace('_', '').isalnum(), 'SQLite identifier')
return {str(row[0]): digest(encode(cells(row[1:])))
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')}
def sql_witness():
out = {}
mutable = {'counter', 'pending', 'reconcile_meta', 'helper_sync_meta', 'helper_sync_rows', 'reconcile_requests', 'job_events'}
for path in db_paths:
with database(path) as db:
schemas = {name: sql for name, sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name")}
all_rows = {name: rows(db, name) for name in schemas}
value = dict(schema=schemas, rows=all_rows,
immutable={name: value for name, value in all_rows.items() if name not in mutable})
if path.name == 'journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0, 'no unfinished agent job')
require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault')
value['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
meta = db.execute('SELECT journal_id,checkpoint,high_sequence,generation FROM reconcile_meta WHERE id=1').fetchone()
value['meta'] = list(meta)
# Completed signed requests are immutable; a pending request
# may acquire its response, but its original bytes must remain.
value['requests'] = {r[0]: dict(request=digest(r[1]), envelope=digest(r[2]), generation=r[3],
response=digest(r[4]) if r[4] is not None else None)
for r in db.execute('SELECT id,request,envelope,generation,response FROM reconcile_requests')}
if path.parent == helper_state:
root_rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
actual = {r[0]: digest(json.dumps([dict(sha256=digest(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in root_rows}
require(actual == intent['root_rows'] and len(actual) == 18 and all(r[5] is not None for r in root_rows), 'exact closed original Root18')
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault')
value['meta'] = list(db.execute('SELECT journal_id,checkpoint,generation,binding_sha256 FROM helper_sync_meta WHERE id=1').fetchone())
value['identity'] = [path.stat().st_dev, path.stat().st_ino]
require(value['identity'] == intent['database_inodes'][str(path)], 'closeout database identity')
out[str(path)] = value
return out
def service(name):
return dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'UnitFileState']).decode().splitlines())
def controls():
fixed = (Path('/etc/rmm-remote'), Path('/etc/systemd/system/rmm-agent-remote.service.d'),
Path('/etc/systemd/system/rmm-helper-cloud.service.d'))
result = {str(path): tree(path) for path in fixed}
for path in (Path('/etc/systemd/system/rmm-agent-remote.service'), Path('/etc/systemd/system/rmm-helper-cloud.service'),
Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'),
Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'), Path('/var/lib/dpkg/status'),
Path('/usr/share/rmm-mvp-test/version')):
result[str(path)] = file(path)
return result
def previous_backups():
# Prior bootstrap folders are never opened for content or modified.
# Bind their complete file identities/metadata, excluding only this case.
return {str(path): tree(path, metadata_only=True) for path in sorted(folder.parent.iterdir()) if path != folder}
def guard(new=False):
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'no reboot/identity transition')
identity = json.loads((state/'identity.json').read_bytes())
require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment')
config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json')
require(digest(config.read_bytes()) == intent['config_sha256'] and digest(pin.read_bytes()) == intent['pin_sha256']
and json.loads(config.read_bytes())['allow_reboot'] is False, 'closed unchanged General policy')
require(digest(Path('/etc/systemd/system/rmm-helper-cloud.service').read_bytes()) == intent['general_unit_sha256']
and digest(Path('/etc/systemd/system/rmm-agent-remote.service').read_bytes()) == intent['agent_unit_sha256'], 'unchanged qualified service units')
binding = json.loads((state/'binding.json').read_bytes())
cached = json.loads((state/'job-signing-key.json').read_bytes())
helper_key = json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_bytes())
raw_key = bytes.fromhex(cached['public_key_hex'])
require(cached == helper_key and len(raw_key) == 32 and cached['algorithm'] == 'Ed25519'
and cached['schema_version'] == 1 and digest(raw_key) == cached['key_id'] == intent['general_pin']['identity']['signing_key_id']
and binding['public_key_hex'] == cached['public_key_hex']
and all(binding[k] == intent['general_pin']['identity'][k] for k in
('tenant_id', 'asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')), 'same pinned public signing identity')
require(digest(Path('/usr/lib/rmm-apt-transaction/apt_transaction.py').read_bytes()) == intent['provider_sha256'], 'provider unchanged')
require(b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes(), 'APT endpoint absent')
for name in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service'):
value = service(name); require(value['ActiveState'] == 'inactive' and value['UnitFileState'] == 'disabled', 'APT/legacy daemons remain disabled')
for name, path in paths.items():
owned(path); require(digest(path.read_bytes()) == intent[('new_' if new else 'old_')+name+'_sha256'], 'binary compare-and-swap')
require(intent['id'] == ident and len(intent['root_rows']) == 18 and intent['jobs_created'] is False, 'fixed maintenance intent')
if action == 'prepare':
guard(); sql_witness()
for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service'):
require(service(name)['ActiveState'] == 'active', 'known original daemon active')
folder.mkdir(mode=0o700); create(folder/'intent.json', encode(intent))
return dict(prepared=True, id=ident, intent_sha256=digest(encode(intent)), daemon_changes=False, jobs_created=False)
owned(folder, True)
require((folder/'intent.json').read_bytes() == encode(intent), 'same original retained intent')
if action == 'read':
if not (folder/'report.json').exists():
return dict(id=ident, completed=False, stage='prepared_without_install_report', install_entered=False)
report = json.loads((folder/'report.json').read_bytes())
if report.get('completed'):
guard(True); current = sql_witness(); before = json.loads((folder/'sql-before.json').read_bytes())
require(all(file(folder/(name+'-previous')) == report['backups'][name]
and report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'] for name in paths), 'versioned original binary backups intact')
for path, previous in before.items():
after = current[path]
require(after['schema'] == previous['schema'] and after['identity'] == previous['identity'], 'unchanged schema/database inode')
for table, old_rows in previous['immutable'].items():
require(all(after['immutable'][table].get(k) == v for k, v in old_rows.items()), 'all original immutable SQL rows retained')
if 'sequence' in previous:
require(after['sequence'] >= previous['sequence'] and after['meta'][0] == previous['meta'][0]
and all(a >= b for a, b in zip(after['meta'][1:], previous['meta'][1:])), 'same journal with monotonic signed Sync/counter')
for key, old in previous['requests'].items():
value = after['requests'].get(key)
require(value is not None and all(value[k] == old[k] for k in ('request', 'envelope', 'generation'))
and (old['response'] is None or value['response'] == old['response']), 'original signed request/response bytes retained')
elif 'meta' in previous:
require(after['meta'][0] == previous['meta'][0] and after['meta'][3] == previous['meta'][3]
and all(after['meta'][i] >= previous['meta'][i] for i in (1, 2)), 'same helper identity/monotonic checkpoint')
require(controls() == json.loads((folder/'controls-before.json').read_bytes()), 'bootstrap/policy/APT/package files unchanged')
for path, old in json.loads((folder/'files-before.json').read_bytes()).items():
require(tree(Path(path), True) == old, 'all non-database state, private-key metadata and PKI unchanged')
old_backups = json.loads((folder/'backups-before.json').read_bytes())
current_backups = previous_backups()
require(all(current_backups.get(path) == value for path, value in old_backups.items()), 'all earlier bootstrap backups retained')
report['readback_verified'] = True
report['services_now'] = {name: service(name) for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')}
return report
# Single-use installation: a retained report, even incomplete, forbids rerun.
require(not (folder/'report.json').exists(), 'installation already entered; readback only')
guard(); sql_witness()
for name in paths:
upload = folder/(name+'-next.upload'); value = owned(upload)
require(value.st_nlink == 1 and value.st_mode&0o777 == 0o600, 'private SFTP artifact')
raw = upload.read_bytes()
require(digest(raw) == intent['new_'+name+'_sha256'] and raw[:6] == b'\x7fELF\x02\x01'
and raw[18:20] == b'\x3e\x00', 'qualified x86-64 ELF')
upload.chmod(0o755); run(['/lib64/ld-linux-x86-64.so.2', '--list', str(upload)])
report = dict(id=ident, completed=False, stage='validated', jobs_created=False, grants_created=False,
package_effects=False, reboot_requested=False, apt_endpoint_enabled=False, apt_pins_stale=True,
intent_sha256=digest(encode(intent)), backups={}, old_agent_sha256=intent['old_agent_sha256'],
old_helper_sha256=intent['old_helper_sha256'], agent_sha256=intent['new_agent_sha256'],
helper_sha256=intent['new_helper_sha256'], provider_sha256=intent['provider_sha256'])
def save():
create(folder/'.report-next.json', encode(report)); os.replace(folder/'.report-next.json', folder/'report.json'); fsync_dir(folder)
locks = []
try:
save()
run(['systemctl', 'stop', 'rmm-agent-remote.service', 'rmm-helper-cloud.service'])
for path in (state/'agent.lock', helper_state/'executor.lock'):
fd = os.open(path, os.O_RDWR|os.O_NOFOLLOW); locks.append(fd); fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
guard(); frozen = sql_witness()
require(all(service(name)['ActiveState'] == 'inactive' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')), 'both exact daemons stopped')
snapshots = {str(p): tree(p, True) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))}
files_full = {str(p): tree(p) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))}
control_before = controls()
backups_before = previous_backups()
create(folder/'sql-before.json', encode(frozen)); create(folder/'files-before.json', encode(snapshots)); create(folder/'controls-before.json', encode(control_before))
create(folder/'backups-before.json', encode(backups_before))
for name, path in paths.items():
require(path.stat().st_dev == folder.stat().st_dev, 'same-filesystem versioned exchange')
previous = folder/(name+'-previous'); os.rename(path, previous); fsync_dir(path.parent); fsync_dir(folder)
report['backups'][name] = file(previous)
require(report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'], 'retained exact previous binary')
report['stage'] = name+'_original_retained'; save()
os.rename(folder/(name+'-next.upload'), path); fsync_dir(path.parent); fsync_dir(folder)
inspected = json.loads(run([str(helper), '--inspect-general-profile'], env=dict(os.environ, RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(inspected == intent['general_pin'], 'same General policy-based profile pin')
require(sql_witness() == frozen and controls() == control_before and previous_backups() == backups_before
and all(tree(Path(p)) == v for p, v in files_full.items()), 'all data bytes/rows/inodes/backups unchanged before restart')
guard(True); report.update(stage='installed_preserved', preserved_before_restart=True,
root_consumed=18, root_unresolved=0, general_pin=inspected, database_inodes=intent['database_inodes'])
save()
for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd)
locks.clear()
run(['systemctl', 'start', 'rmm-helper-cloud.service'])
require(service('rmm-helper-cloud.service')['ActiveState'] == 'active', 'qualified General helper starts')
run(['systemctl', 'start', 'rmm-agent-remote.service'])
require(service('rmm-agent-remote.service')['ActiveState'] == 'active', 'qualified unprivileged agent starts')
pid = int(service('rmm-agent-remote.service')['MainPID'])
process = Path('/proc/'+str(pid)+'/status').read_text().splitlines()
require(next(line for line in process if line.startswith('Uid:')).split()[1:] == ['993']*4
and next(line for line in process if line.startswith('Gid:')).split()[1:] == [str(pwd.getpwnam('rmm-agent').pw_gid)]*4,
'actual agent process remains unprivileged')
report.update(stage='qualified_daemons_started', completed=True, boot_id=intent['boot_id']); save()
except Exception as error:
report.update(error_class=type(error).__name__, frames=[dict(function=f.name, line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
save()
finally:
for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd)
return report
def qualification():
out = {}
for name, filename in (('agent', 'rmm-agent'), ('helper', 'rmm-privileged-helper')):
path = ARTIFACTS/filename; metadata = cloud.contract.protected_json(path.with_name(filename+'.json'))
require(metadata['binary_sha256'] == EXPECTED_NEW[name] == sha(path.read_bytes())
and metadata['architecture'] == 'x86_64-unknown-linux-gnu' and metadata['release_build'] is True
and metadata['qualification_profile'] == 'durable-outcome-v1' and metadata['source_before_after_equal'] is True
and metadata['glibc_max'] == '2.34', 'fixed qualified Bookworm release')
require(all(sha((ROOT/p).read_bytes()) == value for p, value in metadata['source_files'].items()), 'release source bindings unchanged')
out[name] = dict(path=str(path.relative_to(ROOT)), sha256=metadata['binary_sha256'], qualification_sha256=sha(path.with_name(filename+'.json').read_bytes()))
native = ROOT/'devstate/shared-core/acceptance/08a9be72-dd7c-46b7-a2c6-7d17424533c2/linux-passed.json'
result = cloud.contract.protected_json(native)
require(result['passed'] is True and result['qualification']['manifest_sha256'] == sha((ARTIFACTS/'manifest.json').read_bytes()), 'existing real Linux76 shared-core qualification')
return dict(artifacts=out, native_linux_evidence_sha256=sha(native.read_bytes()))
def server_gate(intent):
require(reboot.evidence(intent['target'], 'gate') == dict(jobs=30, outbox=30, permits=26, unresolved=0), 'same central30/30/26/0')
policy = cloud.bootstrap.rpc('server.conf', reboot.server_policy, dict(action='state'))
require(policy == dict(policy_sha256=intent['central_policy_sha256'], closed=True), 'same closed central Root policy')
def prepare():
metadata = qualification(); closeout = cloud.contract.protected_json(ROOT/'devstate/cloud-mvp-final-closeout.json')
require(closeout['verified'] is True and closeout['central_counts'] == dict(jobs=30, outbox=30, permits=26, unresolved=0)
and closeout['policy']['closed'] is True and closeout['native']['root_unresolved'] == 0, 'completed Linux MVP closeout')
observation = closeout['observation']; require(sha((ROOT/observation['path']).read_bytes()) == observation['sha256'], 'verified closeout source')
ident = str(uuid.uuid4()); folder = STATE/ident; folder.mkdir(mode=0o700, parents=True)
native = closeout['native']; old = closeout['vm']['artifact_sha256']
intent = dict(schema_version=1, type='rmm.linux.shared-core-upgrade', id=ident, target=closeout['target'],
boot_id=native['boot_id'], root_rows=native['root_rows'], database_inodes=native['database_inodes'],
old_agent_sha256=old['agent'], old_helper_sha256=old['helper'], new_agent_sha256=EXPECTED_NEW['agent'],
new_helper_sha256=EXPECTED_NEW['helper'], provider_sha256=old['provider'], config_sha256=native['config_sha256'],
pin_sha256=native['pin_sha256'], general_pin=native['general_pin'], central_policy_sha256=closeout['policy']['policy_sha256'],
general_unit_sha256=closeout['vm']['general_unit_sha256'],
agent_unit_sha256=sha((ROOT/'deploy/linux/rmm-agent-remote.service').read_bytes()),
closeout_sha256=sha((ROOT/'devstate/cloud-mvp-final-closeout.json').read_bytes()), qualification=metadata,
jobs_created=False, grants_created=False, reboot_requested=False)
cloud.contract.write_private(folder/'intent.json', intent); server_gate(intent)
result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='prepare', id=ident, intent=intent))
cloud.contract.write_private(folder/'prepared.json', result); require(result.get('prepared') is True, 'retained remote intent incomplete')
print('PREPARED '+ident+'; no daemon change or RMM job')
def existing(ident):
folder = STATE/str(uuid.UUID(ident)); intent = cloud.contract.protected_json(folder/'intent.json')
require(intent['id'] == ident, 'same retained case'); return folder, intent
def install(ident):
folder, intent = existing(ident); require(qualification() == intent['qualification'], 'original qualified source/artifacts')
require(cloud.contract.protected_json(folder/'prepared.json')['prepared'] is True, 'remote preparation first')
cloud.contract.write_private(folder/'install-started.json', dict(id=ident, intent_sha256=sha((folder/'intent.json').read_bytes())))
server_gate(intent)
cfg = cloud.bootstrap.access.load_config(ROOT/'credentials/linux-vm.conf')
args = ['sftp', '-F', '/dev/null', '-P', cfg.get('port', '22'), '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
'-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()),
'-b', '-', cfg['username']+'@'+cfg['address']]
for name, item in intent['qualification']['artifacts'].items():
batch = 'put '+json.dumps(str(ROOT/item['path']))+' '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\nchmod 600 '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\n'
result = subprocess.run(args, input=batch.encode(), capture_output=True, timeout=120)
cloud.contract.write_private(folder/(name+'-sftp.json'), dict(returncode=result.returncode,
stdout_b64=base64.b64encode(result.stdout).decode(), stderr_b64=base64.b64encode(result.stderr).decode()))
require(result.returncode == 0, 'private SFTP error; retain case and inspect, never repeat install')
result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='install', id=ident, intent=intent))
cloud.contract.write_private(folder/'install-reply.json', result)
require(result.get('completed') is True, 'retained incomplete report; readback only')
print('INSTALLED '+ident+'; finish/readback and independent native Sync qualification still required')
def finish(ident):
folder, intent = existing(ident)
cloud.contract.protected_json(folder/'install-started.json')
result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='read', id=ident, intent=intent))
cloud.contract.write_private(folder/('readback-'+str(uuid.uuid4())+'.json'), result)
require(result.get('completed') is True and result.get('readback_verified') is True, 'remote maintenance incomplete; do not reinstall')
require(all(v['ActiveState'] == 'active' for v in result['services_now'].values()), 'both qualified daemons active')
server_gate(intent)
cloud.contract.write_private(folder/'result.json', dict(verified_preservation=True, native_sync_accepted=False,
id=ident, intent_sha256=sha((folder/'intent.json').read_bytes()), report=result))
print('PASS '+ident+' preserving Linux daemon upgrade; APT closed/stale; native Sync/MCP acceptance remains separate')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('prepare', 'install', 'finish')); parser.add_argument('id', nargs='?')
args = parser.parse_args(); os.umask(0o077)
require((args.id is None) == (args.action == 'prepare'), 'prepare creates a case; install/finish require its exact ID')
if args.action == 'prepare': prepare()
elif args.action == 'install': install(args.id)
else: finish(args.id)
if __name__ == '__main__':
try: main()
except Exception as error:
text = str(error)
print(text if text.startswith('remote bootstrap incomplete: ') else 'SHARED_CORE_UPGRADE_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
@@ -0,0 +1,399 @@
"""Read-only supplement for exact V2 UID/GID-attestation failure.
Original runner d4f857b8 and original failed report remain immutable. Literal
readonly witness code is copied from that frozen runner, without install/start
paths or runtime evaluation. This never changes a service, file or SQL row.
"""
import hashlib
import json
import uuid
import argparse
import cloud_shared_core_upgrade_v2 as upgrade
CASE = 'a2423612-721f-4a6c-a832-b9eef549c41e'
def diagnose(payload):
"""Fixed metadata-only differences; no relaxed preservation classification."""
import contextlib
import sqlite3
import stat
require(payload['id'] == 'a2423612-721f-4a6c-a832-b9eef549c41e' and os.geteuid() == 0, 'exact read-only diagnosis')
folder = Path('/var/lib/rmm-bootstrap-history/shared-core-v2-a2423612-721f-4a6c-a832-b9eef549c41e')
intent = payload['intent']
require((folder/'intent.json').read_bytes() == json.dumps(intent,sort_keys=True,separators=(',',':')).encode(), 'original intent')
before = json.loads((folder/'sql-before.json').read_bytes()); changes = []
def sha(raw): return hashlib.sha256(raw).hexdigest()
def row_hash(row):
cells=[dict(blob_sha256=sha(v),length=len(v)) if isinstance(v,bytes) else v for v in row]
return sha(json.dumps(cells,sort_keys=True,separators=(',',':')).encode())
for name, previous in before.items():
path=Path(name); value=path.lstat()
require(stat.S_ISREG(value.st_mode) and not path.is_symlink() and [value.st_dev,value.st_ino]==previous['identity'], 'original database')
require(all(not Path(name+suffix).exists() for suffix in ('-wal','-shm','-journal')), 'settled DELETE mode')
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db:
db.execute('PRAGMA query_only=ON');db.execute('BEGIN')
require(db.execute('PRAGMA journal_mode').fetchone()[0]=='delete','readonly DELETE journal')
for table, old_rows in previous['immutable'].items():
require(table.replace('_','').isalnum(),'fixed table identifier')
for key, old_sha in old_rows.items():
require(key.isdecimal(),'original numeric rowid')
row=db.execute('SELECT * FROM "'+table+'" WHERE rowid=?',[int(key)]).fetchone()
new_sha=row_hash(row) if row is not None else None
if old_sha != new_sha:
require(len(changes)<128,'bounded changed rows')
changes.append(dict(database=str(path),table=table,rowkey=key,old_sha256=old_sha,new_sha256=new_sha))
return dict(id=payload['id'],diagnostic_only=True,changes=changes,mutations=False,raw_rows_returned=False)
def attest(payload):
import contextlib
import sqlite3
import stat
os.umask(0o077)
require(payload['id'] == 'a2423612-721f-4a6c-a832-b9eef549c41e' and payload['action'] == 'read', 'exact readonly post-start case')
ident = str(uuid.UUID(payload['id']))
require(ident == payload['id'] and os.geteuid() == 0, 'canonical root bootstrap case')
action = payload['action']
require(action in ('prepare', 'install', 'read'), 'fixed shared-core operation')
folder = Path('/var/lib/rmm-bootstrap-history')/('shared-core-v2-'+ident)
state = Path('/var/lib/rmm-agent-remote'); helper_state = Path('/var/lib/rmm-helper')
agent = Path('/usr/local/libexec/rmm-agent-remote'); helper = Path('/usr/local/libexec/rmm-privileged-helper')
db_paths = (state/'journal.sqlite', state/'consumption.sqlite', helper_state/'consumption.sqlite')
paths = dict(agent=agent, helper=helper)
intent = payload['intent']
def digest(raw): return hashlib.sha256(raw).hexdigest()
def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def run(args, **kwargs):
return subprocess.run(args, capture_output=True, check=True, timeout=15, **kwargs).stdout
def owned(path, directory=False):
value = path.lstat()
require(not path.is_symlink() and value.st_uid == 0 and not value.st_mode&0o022
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'fixed root-owned bootstrap path')
return value
aliases = tuple(Path('/var/lib/rmm-apt-transaction')/name for name in (
'acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json'))
def check_aliases():
# Exactly the preexisting, durably linked successful publication. Never
# unlink/rewrite either name, and never admit another hardlinked object.
for path in aliases:
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_mode == 0o100600
and (value.st_dev, value.st_ino, value.st_uid, value.st_gid, value.st_nlink) == (2049, 1028362, 0, 984, 2),
'exact known APT publication alias identity')
require(digest(path.read_bytes()) == 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506',
'original signed-result APT alias bytes')
def file(path, secret=False):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode)
and (value.st_nlink == 1 or path in aliases), 'preserved regular file or exact known alias')
if path in aliases: check_aliases()
result = dict(device=value.st_dev, inode=value.st_ino, mode=value.st_mode, uid=value.st_uid,
gid=value.st_gid, size=value.st_size, mtime_ns=value.st_mtime_ns, ctime_ns=value.st_ctime_ns)
# Device private keys are never opened or exported by this upgrader.
if not secret: result['sha256'] = digest(path.read_bytes())
return result
def tree(path, skip_databases=False, metadata_only=False):
if not path.exists(): return {'absent': True}
result = {}
for item in [path, *sorted(path.rglob('*'))]:
value = item.lstat(); require(not item.is_symlink(), 'no symlinks in preserved state')
if item in db_paths or str(item).endswith(('-journal', '-wal', '-shm')):
if skip_databases: continue
if stat.S_ISDIR(value.st_mode):
result[str(item.relative_to(path))] = dict(device=value.st_dev, inode=value.st_ino,
mode=value.st_mode, uid=value.st_uid, gid=value.st_gid)
else:
require(stat.S_ISREG(value.st_mode), 'no unexpected state object')
secret = metadata_only or item.suffix == '.key' or 'private' in item.name.lower() or 'token' in item.name.lower()
result[str(item.relative_to(path))] = file(item, secret)
return result
@contextlib.contextmanager
def database(path):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1
and value.st_uid == (993 if path.parent == state else 0), 'fixed database owner/inode')
# These qualified Linux journals use DELETE mode. Refuse WAL state rather
# than create root-owned sidecars while opening an unprivileged journal.
require(all(not Path(str(path)+suffix).exists() and not Path(str(path)+suffix).is_symlink()
for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required')
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'qualified DELETE journal')
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity')
yield db
require((path.stat().st_dev, path.stat().st_ino) == (value.st_dev, value.st_ino), 'database inode preserved')
def cells(row):
return [dict(blob_sha256=digest(v), length=len(v)) if isinstance(v, bytes) else v for v in row]
def rows(db, table):
require(table.replace('_', '').isalnum(), 'SQLite identifier')
return {str(row[0]): digest(encode(cells(row[1:])))
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')}
def sql_witness():
out = {}
mutable = {'counter', 'pending', 'reconcile_meta', 'helper_sync_meta', 'helper_sync_rows', 'reconcile_requests', 'job_events'}
for path in db_paths:
with database(path) as db:
schemas = {name: sql for name, sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name")}
all_rows = {name: rows(db, name) for name in schemas}
value = dict(schema=schemas, rows=all_rows,
immutable={name: value for name, value in all_rows.items() if name not in mutable})
if path.name == 'journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0, 'no unfinished agent job')
require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault')
value['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
meta = db.execute('SELECT journal_id,checkpoint,high_sequence,generation FROM reconcile_meta WHERE id=1').fetchone()
value['meta'] = list(meta)
# Completed signed requests are immutable; a pending request
# may acquire its response, but its original bytes must remain.
value['requests'] = {r[0]: dict(request=digest(r[1]), envelope=digest(r[2]), generation=r[3],
response=digest(r[4]) if r[4] is not None else None)
for r in db.execute('SELECT id,request,envelope,generation,response FROM reconcile_requests')}
if path.parent == helper_state:
root_rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
actual = {r[0]: digest(json.dumps([dict(sha256=digest(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in root_rows}
require(actual == intent['root_rows'] and len(actual) == 18 and all(r[5] is not None for r in root_rows), 'exact closed original Root18')
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault')
value['meta'] = list(db.execute('SELECT journal_id,checkpoint,generation,binding_sha256 FROM helper_sync_meta WHERE id=1').fetchone())
value['identity'] = [path.stat().st_dev, path.stat().st_ino]
require(value['identity'] == intent['database_inodes'][str(path)], 'closeout database identity')
out[str(path)] = value
return out
def service(name):
return dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'UnitFileState']).decode().splitlines())
def controls():
fixed = (Path('/etc/rmm-remote'), Path('/etc/systemd/system/rmm-agent-remote.service.d'),
Path('/etc/systemd/system/rmm-helper-cloud.service.d'))
result = {str(path): tree(path) for path in fixed}
for path in (Path('/etc/systemd/system/rmm-agent-remote.service'), Path('/etc/systemd/system/rmm-helper-cloud.service'),
Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'),
Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'), Path('/var/lib/dpkg/status'),
Path('/usr/share/rmm-mvp-test/version')):
result[str(path)] = file(path)
return result
def previous_backups():
# Prior bootstrap folders are never opened for content or modified.
# Bind their complete file identities/metadata, excluding only this case.
return {str(path): tree(path, metadata_only=True) for path in sorted(folder.parent.iterdir()) if path != folder}
def guard(new=False):
check_aliases()
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'no reboot/identity transition')
identity = json.loads((state/'identity.json').read_bytes())
require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment')
config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json')
require(digest(config.read_bytes()) == intent['config_sha256'] and digest(pin.read_bytes()) == intent['pin_sha256']
and json.loads(config.read_bytes())['allow_reboot'] is False, 'closed unchanged General policy')
require(digest(Path('/etc/systemd/system/rmm-helper-cloud.service').read_bytes()) == intent['general_unit_sha256']
and digest(Path('/etc/systemd/system/rmm-agent-remote.service').read_bytes()) == intent['agent_unit_sha256'], 'unchanged qualified service units')
binding = json.loads((state/'binding.json').read_bytes())
cached = json.loads((state/'job-signing-key.json').read_bytes())
helper_key = json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_bytes())
raw_key = bytes.fromhex(cached['public_key_hex'])
require(cached == helper_key and len(raw_key) == 32 and cached['algorithm'] == 'Ed25519'
and cached['schema_version'] == 1 and digest(raw_key) == cached['key_id'] == intent['general_pin']['identity']['signing_key_id']
and binding['public_key_hex'] == cached['public_key_hex']
and all(binding[k] == intent['general_pin']['identity'][k] for k in
('tenant_id', 'asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')), 'same pinned public signing identity')
require(digest(Path('/usr/lib/rmm-apt-transaction/apt_transaction.py').read_bytes()) == intent['provider_sha256'], 'provider unchanged')
require(b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes(), 'APT endpoint absent')
for name in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service'):
value = service(name); require(value['ActiveState'] == 'inactive' and value['UnitFileState'] == 'disabled', 'APT/legacy daemons remain disabled')
for name, path in paths.items():
owned(path); require(digest(path.read_bytes()) == intent[('new_' if new else 'old_')+name+'_sha256'], 'binary compare-and-swap')
owned(folder, True)
require((folder/'intent.json').read_bytes() == encode(intent), 'same retained V2 intent')
report_raw = (folder/'report.json').read_bytes()
report = json.loads(report_raw)
require(digest(encode(report)) == payload['original_report_sha256'] and report['completed'] is False
and report['stage'] == 'installed_preserved' and report['preserved_before_restart'] is True
and set(report['backups']) == {'agent', 'helper'} and report['root_consumed'] == 18
and report['root_unresolved'] == 0 and report['error_class'] == 'RuntimeError'
and report['frames'] == [{'function':'vm','line':275},{'function':'require','line':5}],
'exact last UID/GID assertion failure after retained installation')
guard(True); current = sql_witness(); before = json.loads((folder/'sql-before.json').read_bytes())
require(all(file(folder/(name+'-previous')) == report['backups'][name]
and report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'] for name in paths), 'versioned original binary backups intact')
# reconcile.rs:468 intentionally renews ONLY the applied decision reference.
# Preserve the original complete row hash; prove the same job/attempt/outcome
# with its original signed reference, then independently verify the new one.
decision_cache = {}; reference_changes = []
old_decisions = before[str(helper_state/'consumption.sqlite')]['immutable']['helper_sync_decisions']
with database(helper_state/'consumption.sqlite') as db:
candidates = [(str(rowid), decision_id) for rowid,decision_id in db.execute(
'SELECT rowid,decision_id FROM helper_sync_decisions ORDER BY rowid DESC') if str(rowid) in old_decisions]
def row_hash(row): return digest(encode(cells(row)))
def signed_decision(decision_id, original=False):
if decision_id in decision_cache:
value = decision_cache[decision_id]
require(not original or value['original'], 'original signed decision existed before restart')
return value['decision']
with database(helper_state/'consumption.sqlite') as db:
row = db.execute('SELECT rowid,* FROM helper_sync_decisions WHERE decision_id=?', [decision_id]).fetchone()
require(row is not None, 'retained helper-applied signed decision')
protected = old_decisions.get(str(row[0])) == row_hash(row[1:])
require(not original or protected, 'byte-identical original signed decision witness')
signed = json.loads(row[3]); raw = bytes(signed['decision_bytes']); signature = bytes(signed['signature'])
require(0 < len(raw) <= 2*1024*1024 and len(signature) == 64, 'bounded signed response')
public = bytes.fromhex(json.loads((state/'binding.json').read_bytes())['public_key_hex'])
require(len(public) == 32, 'exact pinned Ed25519 public key')
descriptors = []
try:
# Public key, signed payload and signature only; anonymous seekable
# memory, no state/temporary-file writes or private-key reads.
for value in (bytes.fromhex('302a300506032b6570032100')+public, raw, signature):
fd = os.memfd_create('rmm-public-signature-check', os.MFD_CLOEXEC); descriptors.append(fd)
require(os.write(fd,value) == len(value), 'complete anonymous verification input');os.lseek(fd,0,0)
verified = subprocess.run(['/usr/bin/openssl','pkeyutl','-verify','-pubin','-keyform','DER',
'-inkey','/proc/self/fd/'+str(descriptors[0]),'-rawin','-in','/proc/self/fd/'+str(descriptors[1]),
'-sigfile','/proc/self/fd/'+str(descriptors[2])], pass_fds=tuple(descriptors),capture_output=True,timeout=5)
require(verified.returncode == 0, 'historical original Ed25519 signature')
finally:
for fd in descriptors: os.close(fd)
decision = json.loads(raw)
require(decision['domain'] == 'rmm.reconciliation-decision' and decision['schema_version'] == 1
and decision['decision_id'] == decision_id and decision['key_id'] == digest(public)
and all(decision[k] == intent['general_pin']['identity'][k] for k in
('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')),
'original tenant/device/key/signed decision domain')
import datetime
issued = datetime.datetime.fromisoformat(decision['issued_at'].replace('Z','+00:00'))
until = datetime.datetime.fromisoformat(decision['valid_until'].replace('Z','+00:00'))
require(0 < (until-issued).total_seconds() <= 60 and len(decision['attempts']) <= 256,
'historical bounded signed decision, never a new authorization')
def length(value):
out=bytearray()
while value >= 128: out.append((value&127)|128);value >>= 7
out.append(value);return bytes(out)
# Canonical prost SyncResponse: exactly bytes fields1/2 (proto/rmm.proto).
protobuf = b'\x0a'+length(len(raw))+raw+b'\x12\x40'+signature
with database(state/'journal.sqlite') as db:
request = db.execute('SELECT request,generation,response FROM reconcile_requests WHERE id=?', [decision['request_id']]).fetchone()
require(request is not None and request[2] == protobuf and request[0] == row[4]
and digest(request[0]) == decision['request_hash'] and request[1] == decision['connection_generation'],
'exact agent-applied original request and helper-applied signed response')
decision_cache[decision_id] = dict(decision=decision,original=protected)
return decision
def renewed_outcome(path, key, original_sha):
require(path == str(state/'journal.sqlite'), 'only agent cached-outcome reference can renew')
with database(state/'journal.sqlite') as db:
current_row = db.execute('SELECT job,attempt,outcome,decision_id FROM reconcile_outcomes WHERE rowid=?', [int(key)]).fetchone()
require(current_row is not None, 'original cached outcome row retained')
original_id = next((decision_id for _,decision_id in candidates
if row_hash((*current_row[:3],decision_id)) == original_sha), None)
require(original_id is not None, 'original full row hash proves same job/attempt/outcome bytes')
old = signed_decision(original_id,True);new = signed_decision(current_row[3])
selected = []
for decision in (old,new):
found = [a for a in decision['attempts'] if a['job_id'] == current_row[0] and a['attempt_id'] == current_row[1]]
require(len(found) == 1 and found[0]['outcome_sha256'] == digest(current_row[2])
and (not found[0]['outcome_bytes'] or bytes(found[0]['outcome_bytes']) == current_row[2]),
'signed original outcome bytes/hash and exact job/attempt')
selected.append(found[0])
require(all(selected[0][k] == selected[1][k] for k in ('job_payload_hash','grant_id','authorization_id','plan_hash',
'action','execution_identity','outcome_state','outcome_sha256','original_boot_id'))
and new['journal_id'] == old['journal_id'] and new['helper_journal_id'] == old['helper_journal_id']
and new['checkpoint'] >= old['checkpoint'] and new['connection_generation'] >= old['connection_generation'],
'same authorized attempt/outcome with a monotonic applied signed reference')
reference_changes.append(dict(rowkey=key,original_row_sha256=original_sha,old_decision_id=original_id,new_decision_id=current_row[3],
outcome_sha256=digest(current_row[2])))
for path, previous in before.items():
after = current[path]
require(after['schema'] == previous['schema'] and after['identity'] == previous['identity'], 'unchanged schema/database inode')
for table, old_rows in previous['immutable'].items():
for key, original_sha in old_rows.items():
if after['immutable'][table].get(key) == original_sha: continue
require(table == 'reconcile_outcomes', 'all other original immutable SQL rows retained')
renewed_outcome(path,key,original_sha)
if 'sequence' in previous:
require(after['sequence'] >= previous['sequence'] and after['meta'][0] == previous['meta'][0]
and all(a >= b for a, b in zip(after['meta'][1:], previous['meta'][1:])), 'same journal with monotonic signed Sync/counter')
for key, old in previous['requests'].items():
value = after['requests'].get(key)
require(value is not None and all(value[k] == old[k] for k in ('request', 'envelope', 'generation'))
and (old['response'] is None or value['response'] == old['response']), 'original signed request/response bytes retained')
elif 'meta' in previous:
require(after['meta'][0] == previous['meta'][0] and after['meta'][3] == previous['meta'][3]
and all(after['meta'][i] >= previous['meta'][i] for i in (1, 2)), 'same helper identity/monotonic checkpoint')
require(controls() == json.loads((folder/'controls-before.json').read_bytes()), 'bootstrap/policy/APT/package files unchanged')
for path, old in json.loads((folder/'files-before.json').read_bytes()).items():
require(tree(Path(path), True) == old, 'all non-database state, private-key metadata and PKI unchanged')
old_backups = json.loads((folder/'backups-before.json').read_bytes())
current_backups = previous_backups()
require(all(current_backups.get(path) == value for path, value in old_backups.items()), 'all earlier bootstrap backups retained')
observed_units = {name:service(name) for name in ('rmm-agent-remote.service','rmm-helper-cloud.service')}
require(all(value['ActiveState'] == 'active' for value in observed_units.values()), 'same two daemons active')
pid = int(observed_units['rmm-agent-remote.service']['MainPID'])
process = Path('/proc/'+str(pid)+'/status').read_text().splitlines()
actual = {label:[int(v) for v in next(line for line in process if line.startswith(label+':')).split()[1:]] for label in ('Uid','Gid')}
require(payload['expected_uids'] == [993]*4 and payload['expected_gids'] == [984]*4
and actual['Uid'] == payload['expected_uids'] and actual['Gid'] == payload['expected_gids'],
'actual process identity matches independently verified closeout')
sync = payload['native_sync']
with database(helper_state/'consumption.sqlite') as db:
stored = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?', [sync['decision_id']]).fetchone()
require(stored is not None, 'exact native new-core signed decision retained')
signed = json.loads(stored[0]); raw = bytes(signed['decision_bytes']); decision = json.loads(raw)
require(digest(raw) == sync['payload_sha256'] and decision['decision_id'] == sync['decision_id']
and decision['request_hash'] == sync['request_sha256'] and decision['boot_id'] == intent['boot_id']
and decision['connection_generation'] == sync['generation'] == 114
and decision['ready'] is True and decision['helper_ready'] is True,
'actual helper-applied signed Sync matches stored server/VM evidence')
with database(state/'journal.sqlite') as db:
stored = db.execute('SELECT request,response FROM reconcile_requests WHERE id=?', [sync['request_id']]).fetchone()
require(stored is not None and digest(stored[0]) == sync['request_sha256'] and stored[1] is not None
and current[str(state/'journal.sqlite')]['meta'][3] >= sync['generation']
and current[str(helper_state/'consumption.sqlite')]['meta'][2] >= sync['generation'], 'new-core request/response and monotonic generation retained')
require((folder/'report.json').read_bytes() == report_raw, 'original failed report remains byte-identical')
return dict(id=ident, verified_preservation=True, original_report_completed=False,
original_report_retained=True, original_report_sha256=digest(report_raw),
actual_agent_process=dict(pid=pid,uids=actual['Uid'],gids=actual['Gid']), units=observed_units,
root_consumed=18,root_unresolved=0, boot_id=intent['boot_id'],
database_inodes=intent['database_inodes'],agent_sha256=intent['new_agent_sha256'],helper_sha256=intent['new_helper_sha256'],
apt_endpoint_enabled=False,apt_pins_stale=True,jobs_created=False,grants_created=False,
package_effects=False,reboot_requested=False,services_changed=False,
native_sync_evidence_bound=True,native_sync=sync,native_sync_evidence=payload['native_sync_evidence'],
verified_outcome_reference_changes=reference_changes)
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--diagnose',action='store_true');args=parser.parse_args()
folder,intent=upgrade.existing(CASE)
failed=upgrade.cloud.contract.protected_json(folder/'install-reply.json')
closeout=upgrade.cloud.contract.protected_json(upgrade.ROOT/'devstate/cloud-mvp-final-closeout.json')
upgrade.require(closeout['verified'] is True and closeout['vm']['artifact_sha256']['agent']==intent['old_agent_sha256'], 'original verified runtime closeout')
vm_path=folder/'new-core-vm-sync.json';server_path=folder/'new-core-server-sync.json'
vm=upgrade.cloud.contract.protected_json(vm_path);server=upgrade.cloud.contract.protected_json(server_path)
agent=vm['agent']['meta'];helper=vm['helper']['meta'];head=server['head'];installation=server['installation']
upgrade.require(agent['fault'] is None and helper['fault'] is None and agent['last_decision']==helper['last_decision']==head['decision_id']
and agent['generation']==helper['generation']==installation['connection_generation']==114
and agent['journal_id']==head['journal_id']==helper['agent_journal_id']
and helper['journal_id']==head['helper_journal_id']==agent['helper_id']
and installation['connection_boot_id']==intent['boot_id'] and installation['sync_ready'] is True
and installation['helper_sync_ready'] is True and installation['restore_blocked'] is False
and installation['sequence_regressed'] is False and installation['sync_recovery_required'] is False, 'actual matching post-upgrade native Sync proofs')
decision=next(v for v in server['decisions'] if v['decision_id']==head['decision_id'])
request=next(v for v in vm['agent']['requests'] if v.get('decision',{}).get('decision_id')==head['decision_id'])
upgrade.require(decision['payload_sha256']==request['decision']['payload_sha256']
and decision['request_hash']==request['request_sha256'] and decision['decision']['ready'] is True
and decision['decision']['helper_ready'] is True, 'same exact stored signed decision payload')
native_sync=dict(decision_id=head['decision_id'],request_id=decision['request_id'],generation=114,
payload_sha256=decision['payload_sha256'],request_sha256=decision['request_hash'])
payload=dict(id=CASE,action='read',intent=intent,
original_report_sha256=hashlib.sha256(json.dumps(failed,sort_keys=True,separators=(',',':')).encode()).hexdigest(),
expected_uids=closeout['vm']['agent_process']['uids'],expected_gids=closeout['vm']['agent_process']['gids'],
native_sync=native_sync,native_sync_evidence={str(p.relative_to(upgrade.ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in (vm_path,server_path)})
upgrade.server_gate(intent)
result=upgrade.cloud.bootstrap.rpc('linux-vm.conf',diagnose if args.diagnose else attest,payload)
upgrade.cloud.contract.write_private(folder/(('readonly-diagnostic-' if args.diagnose else 'readonly-attestation-')+str(uuid.uuid4())+'.json'),result)
if args.diagnose:
upgrade.require(result.get('diagnostic_only') is True,'diagnostic incomplete');print('PASS read-only row-hash diagnosis retained');return
upgrade.require(result.get('verified_preservation') is True, 'supplemental preservation not proven')
print('PASS readonly supplemental preservation/UID993/GID984; native Sync114 evidence bound; original failed receipt retained')
if __name__=='__main__':
try:main()
except Exception as error:
text=str(error);print(text if text.startswith('remote bootstrap incomplete: ') else 'SHARED_CORE_ATTESTATION_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+453
View File
@@ -0,0 +1,453 @@
#!/usr/bin/env python3
"""V2 preserving Linux upgrade; admits only the proven immutable APT alias pair.
Source copy of cloud_shared_core_upgrade.py SHA09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873.
V1 and failed case2f8de8b9 remain immutable. V2 requires its actual old-runtime
restore readback before creating a NEW independent case. Alias SHA comes from
the original verified UI-approved apt_native_update checkpoint, not new effects.
Planned, not executed: prepare --restore-readback FILE; install CASE; finish CASE.
Install is single-use;
an interrupted/lost reply permits finish/readback only, never an automatic retry.
The caller separately qualifies actual mTLS/Sync/MCP after the retained report.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import subprocess
import uuid
import cloud_reboot_acceptance as reboot
cloud = reboot.cloud
require = cloud.contract.require
ROOT = cloud.ROOT
STATE = ROOT/'devstate/shared-core/runtime-upgrade-v2'
ARTIFACTS = ROOT/'devstate/shared-core/durable-outcome-v1'
REMOTE = '/var/lib/rmm-bootstrap-history/shared-core-v2-'
ORIGINAL_RUNNER_SHA256 = '09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873'
FAILED_CASE = '2f8de8b9-2abe-4125-aa12-ae38332a32c6'
ALIAS_SHA256 = 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506'
EXPECTED_NEW = {
'agent': 'c9c09de0ecb52b556d707cf45accb69186c73b8ec9baeeabfaeff220fafb1d4e',
'helper': '51c3142ab332216d6a94ff64ff15cbc37ee1d5a99896c14c87bb6f62e63bb946',
}
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def vm(payload):
"""Fixed bootstrap operations; source is sent by the pinned existing SSH RPC."""
import contextlib
import fcntl
import sqlite3
import stat
import traceback
os.umask(0o077)
ident = str(uuid.UUID(payload['id']))
require(ident == payload['id'] and os.geteuid() == 0, 'canonical root bootstrap case')
action = payload['action']
require(action in ('prepare', 'install', 'read'), 'fixed shared-core operation')
folder = Path('/var/lib/rmm-bootstrap-history')/('shared-core-v2-'+ident)
state = Path('/var/lib/rmm-agent-remote'); helper_state = Path('/var/lib/rmm-helper')
agent = Path('/usr/local/libexec/rmm-agent-remote'); helper = Path('/usr/local/libexec/rmm-privileged-helper')
db_paths = (state/'journal.sqlite', state/'consumption.sqlite', helper_state/'consumption.sqlite')
paths = dict(agent=agent, helper=helper)
intent = payload['intent']
def digest(raw): return hashlib.sha256(raw).hexdigest()
def encode(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def run(args, **kwargs):
return subprocess.run(args, capture_output=True, check=True, timeout=15, **kwargs).stdout
def fsync_dir(path):
fd = os.open(path, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
def create(path, raw, mode=0o600):
fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw); stream.flush(); os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
fsync_dir(path.parent)
def owned(path, directory=False):
value = path.lstat()
require(not path.is_symlink() and value.st_uid == 0 and not value.st_mode&0o022
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'fixed root-owned bootstrap path')
return value
aliases = tuple(Path('/var/lib/rmm-apt-transaction')/name for name in (
'acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json'))
def check_aliases():
# Exactly the preexisting, durably linked successful publication. Never
# unlink/rewrite either name, and never admit another hardlinked object.
for path in aliases:
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_mode == 0o100600
and (value.st_dev, value.st_ino, value.st_uid, value.st_gid, value.st_nlink) == (2049, 1028362, 0, 984, 2),
'exact known APT publication alias identity')
require(digest(path.read_bytes()) == 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506',
'original signed-result APT alias bytes')
def file(path, secret=False):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode)
and (value.st_nlink == 1 or path in aliases), 'preserved regular file or exact known alias')
if path in aliases: check_aliases()
result = dict(device=value.st_dev, inode=value.st_ino, mode=value.st_mode, uid=value.st_uid,
gid=value.st_gid, size=value.st_size, mtime_ns=value.st_mtime_ns, ctime_ns=value.st_ctime_ns)
# Device private keys are never opened or exported by this upgrader.
if not secret: result['sha256'] = digest(path.read_bytes())
return result
def tree(path, skip_databases=False, metadata_only=False):
if not path.exists(): return {'absent': True}
result = {}
for item in [path, *sorted(path.rglob('*'))]:
value = item.lstat(); require(not item.is_symlink(), 'no symlinks in preserved state')
if item in db_paths or str(item).endswith(('-journal', '-wal', '-shm')):
if skip_databases: continue
if stat.S_ISDIR(value.st_mode):
result[str(item.relative_to(path))] = dict(device=value.st_dev, inode=value.st_ino,
mode=value.st_mode, uid=value.st_uid, gid=value.st_gid)
else:
require(stat.S_ISREG(value.st_mode), 'no unexpected state object')
secret = metadata_only or item.suffix == '.key' or 'private' in item.name.lower() or 'token' in item.name.lower()
result[str(item.relative_to(path))] = file(item, secret)
return result
@contextlib.contextmanager
def database(path):
value = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_nlink == 1
and value.st_uid == (993 if path.parent == state else 0), 'fixed database owner/inode')
# These qualified Linux journals use DELETE mode. Refuse WAL state rather
# than create root-owned sidecars while opening an unprivileged journal.
require(all(not Path(str(path)+suffix).exists() and not Path(str(path)+suffix).is_symlink()
for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required')
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete', 'qualified DELETE journal')
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'journal integrity')
yield db
require((path.stat().st_dev, path.stat().st_ino) == (value.st_dev, value.st_ino), 'database inode preserved')
def cells(row):
return [dict(blob_sha256=digest(v), length=len(v)) if isinstance(v, bytes) else v for v in row]
def rows(db, table):
require(table.replace('_', '').isalnum(), 'SQLite identifier')
return {str(row[0]): digest(encode(cells(row[1:])))
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')}
def sql_witness():
out = {}
mutable = {'counter', 'pending', 'reconcile_meta', 'helper_sync_meta', 'helper_sync_rows', 'reconcile_requests', 'job_events'}
for path in db_paths:
with database(path) as db:
schemas = {name: sql for name, sql in db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name")}
all_rows = {name: rows(db, name) for name in schemas}
value = dict(schema=schemas, rows=all_rows,
immutable={name: value for name, value in all_rows.items() if name not in mutable})
if path.name == 'journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0, 'no unfinished agent job')
require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault')
value['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
meta = db.execute('SELECT journal_id,checkpoint,high_sequence,generation FROM reconcile_meta WHERE id=1').fetchone()
value['meta'] = list(meta)
# Completed signed requests are immutable; a pending request
# may acquire its response, but its original bytes must remain.
value['requests'] = {r[0]: dict(request=digest(r[1]), envelope=digest(r[2]), generation=r[3],
response=digest(r[4]) if r[4] is not None else None)
for r in db.execute('SELECT id,request,envelope,generation,response FROM reconcile_requests')}
if path.parent == helper_state:
root_rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
actual = {r[0]: digest(json.dumps([dict(sha256=digest(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in root_rows}
require(actual == intent['root_rows'] and len(actual) == 18 and all(r[5] is not None for r in root_rows), 'exact closed original Root18')
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault')
value['meta'] = list(db.execute('SELECT journal_id,checkpoint,generation,binding_sha256 FROM helper_sync_meta WHERE id=1').fetchone())
value['identity'] = [path.stat().st_dev, path.stat().st_ino]
require(value['identity'] == intent['database_inodes'][str(path)], 'closeout database identity')
out[str(path)] = value
return out
def service(name):
return dict(line.split('=', 1) for line in run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'UnitFileState']).decode().splitlines())
def controls():
fixed = (Path('/etc/rmm-remote'), Path('/etc/systemd/system/rmm-agent-remote.service.d'),
Path('/etc/systemd/system/rmm-helper-cloud.service.d'))
result = {str(path): tree(path) for path in fixed}
for path in (Path('/etc/systemd/system/rmm-agent-remote.service'), Path('/etc/systemd/system/rmm-helper-cloud.service'),
Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'),
Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'), Path('/var/lib/dpkg/status'),
Path('/usr/share/rmm-mvp-test/version')):
result[str(path)] = file(path)
return result
def previous_backups():
# Prior bootstrap folders are never opened for content or modified.
# Bind their complete file identities/metadata, excluding only this case.
return {str(path): tree(path, metadata_only=True) for path in sorted(folder.parent.iterdir()) if path != folder}
def guard(new=False):
check_aliases()
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == intent['boot_id'], 'no reboot/identity transition')
identity = json.loads((state/'identity.json').read_bytes())
require(all(identity[k] == intent['target'][k] for k in ('agent_id', 'asset_id')), 'same existing enrollment')
config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json')
require(digest(config.read_bytes()) == intent['config_sha256'] and digest(pin.read_bytes()) == intent['pin_sha256']
and json.loads(config.read_bytes())['allow_reboot'] is False, 'closed unchanged General policy')
require(digest(Path('/etc/systemd/system/rmm-helper-cloud.service').read_bytes()) == intent['general_unit_sha256']
and digest(Path('/etc/systemd/system/rmm-agent-remote.service').read_bytes()) == intent['agent_unit_sha256'], 'unchanged qualified service units')
binding = json.loads((state/'binding.json').read_bytes())
cached = json.loads((state/'job-signing-key.json').read_bytes())
helper_key = json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_bytes())
raw_key = bytes.fromhex(cached['public_key_hex'])
require(cached == helper_key and len(raw_key) == 32 and cached['algorithm'] == 'Ed25519'
and cached['schema_version'] == 1 and digest(raw_key) == cached['key_id'] == intent['general_pin']['identity']['signing_key_id']
and binding['public_key_hex'] == cached['public_key_hex']
and all(binding[k] == intent['general_pin']['identity'][k] for k in
('tenant_id', 'asset_id', 'agent_id', 'enrollment_generation', 'assignment_version')), 'same pinned public signing identity')
require(digest(Path('/usr/lib/rmm-apt-transaction/apt_transaction.py').read_bytes()) == intent['provider_sha256'], 'provider unchanged')
require(b'RMM_APT_HELPER_' not in Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf').read_bytes(), 'APT endpoint absent')
for name in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service'):
value = service(name); require(value['ActiveState'] == 'inactive' and value['UnitFileState'] == 'disabled', 'APT/legacy daemons remain disabled')
for name, path in paths.items():
owned(path); require(digest(path.read_bytes()) == intent[('new_' if new else 'old_')+name+'_sha256'], 'binary compare-and-swap')
require(intent['id'] == ident and intent['type'] == 'rmm.linux.shared-core-upgrade-v2'
and len(intent['root_rows']) == 18 and intent['jobs_created'] is False
and intent['alias_sha256'] == 'a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506', 'fixed maintenance intent')
if action == 'prepare':
guard(); sql_witness()
# Catch unsupported historical objects while the old daemons still run.
for path in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction')):
tree(path, True)
controls(); previous_backups()
for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service'):
require(service(name)['ActiveState'] == 'active', 'known original daemon active')
folder.mkdir(mode=0o700); create(folder/'intent.json', encode(intent))
return dict(prepared=True, id=ident, intent_sha256=digest(encode(intent)), daemon_changes=False, jobs_created=False)
owned(folder, True)
require((folder/'intent.json').read_bytes() == encode(intent), 'same original retained intent')
if action == 'read':
if not (folder/'report.json').exists():
return dict(id=ident, completed=False, stage='prepared_without_install_report', install_entered=False)
report = json.loads((folder/'report.json').read_bytes())
if report.get('completed'):
guard(True); current = sql_witness(); before = json.loads((folder/'sql-before.json').read_bytes())
require(all(file(folder/(name+'-previous')) == report['backups'][name]
and report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'] for name in paths), 'versioned original binary backups intact')
for path, previous in before.items():
after = current[path]
require(after['schema'] == previous['schema'] and after['identity'] == previous['identity'], 'unchanged schema/database inode')
for table, old_rows in previous['immutable'].items():
require(all(after['immutable'][table].get(k) == v for k, v in old_rows.items()), 'all original immutable SQL rows retained')
if 'sequence' in previous:
require(after['sequence'] >= previous['sequence'] and after['meta'][0] == previous['meta'][0]
and all(a >= b for a, b in zip(after['meta'][1:], previous['meta'][1:])), 'same journal with monotonic signed Sync/counter')
for key, old in previous['requests'].items():
value = after['requests'].get(key)
require(value is not None and all(value[k] == old[k] for k in ('request', 'envelope', 'generation'))
and (old['response'] is None or value['response'] == old['response']), 'original signed request/response bytes retained')
elif 'meta' in previous:
require(after['meta'][0] == previous['meta'][0] and after['meta'][3] == previous['meta'][3]
and all(after['meta'][i] >= previous['meta'][i] for i in (1, 2)), 'same helper identity/monotonic checkpoint')
require(controls() == json.loads((folder/'controls-before.json').read_bytes()), 'bootstrap/policy/APT/package files unchanged')
for path, old in json.loads((folder/'files-before.json').read_bytes()).items():
require(tree(Path(path), True) == old, 'all non-database state, private-key metadata and PKI unchanged')
old_backups = json.loads((folder/'backups-before.json').read_bytes())
current_backups = previous_backups()
require(all(current_backups.get(path) == value for path, value in old_backups.items()), 'all earlier bootstrap backups retained')
report['readback_verified'] = True
report['services_now'] = {name: service(name) for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')}
return report
# Single-use installation: a retained report, even incomplete, forbids rerun.
require(not (folder/'report.json').exists(), 'installation already entered; readback only')
guard(); sql_witness()
for path in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction')):
tree(path, True)
controls(); previous_backups()
for name in paths:
upload = folder/(name+'-next.upload'); value = owned(upload)
require(value.st_nlink == 1 and value.st_mode&0o777 == 0o600, 'private SFTP artifact')
raw = upload.read_bytes()
require(digest(raw) == intent['new_'+name+'_sha256'] and raw[:6] == b'\x7fELF\x02\x01'
and raw[18:20] == b'\x3e\x00', 'qualified x86-64 ELF')
upload.chmod(0o755); run(['/lib64/ld-linux-x86-64.so.2', '--list', str(upload)])
report = dict(id=ident, completed=False, stage='validated', jobs_created=False, grants_created=False,
package_effects=False, reboot_requested=False, apt_endpoint_enabled=False, apt_pins_stale=True,
intent_sha256=digest(encode(intent)), backups={}, old_agent_sha256=intent['old_agent_sha256'],
old_helper_sha256=intent['old_helper_sha256'], agent_sha256=intent['new_agent_sha256'],
helper_sha256=intent['new_helper_sha256'], provider_sha256=intent['provider_sha256'])
def save():
create(folder/'.report-next.json', encode(report)); os.replace(folder/'.report-next.json', folder/'report.json'); fsync_dir(folder)
locks = []
try:
save()
run(['systemctl', 'stop', 'rmm-agent-remote.service', 'rmm-helper-cloud.service'])
for path in (state/'agent.lock', helper_state/'executor.lock'):
fd = os.open(path, os.O_RDWR|os.O_NOFOLLOW); locks.append(fd); fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
guard(); frozen = sql_witness()
require(all(service(name)['ActiveState'] == 'inactive' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')), 'both exact daemons stopped')
snapshots = {str(p): tree(p, True) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))}
files_full = {str(p): tree(p) for p in (state, helper_state, Path('/var/lib/rmm-agent'), Path('/var/lib/rmm-apt-transaction'))}
control_before = controls()
backups_before = previous_backups()
create(folder/'sql-before.json', encode(frozen)); create(folder/'files-before.json', encode(snapshots)); create(folder/'controls-before.json', encode(control_before))
create(folder/'backups-before.json', encode(backups_before))
for name, path in paths.items():
require(path.stat().st_dev == folder.stat().st_dev, 'same-filesystem versioned exchange')
previous = folder/(name+'-previous'); os.rename(path, previous); fsync_dir(path.parent); fsync_dir(folder)
report['backups'][name] = file(previous)
require(report['backups'][name]['sha256'] == intent['old_'+name+'_sha256'], 'retained exact previous binary')
report['stage'] = name+'_original_retained'; save()
os.rename(folder/(name+'-next.upload'), path); fsync_dir(path.parent); fsync_dir(folder)
inspected = json.loads(run([str(helper), '--inspect-general-profile'], env=dict(os.environ, RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(inspected == intent['general_pin'], 'same General policy-based profile pin')
require(sql_witness() == frozen and controls() == control_before and previous_backups() == backups_before
and all(tree(Path(p)) == v for p, v in files_full.items()), 'all data bytes/rows/inodes/backups unchanged before restart')
guard(True); report.update(stage='installed_preserved', preserved_before_restart=True,
root_consumed=18, root_unresolved=0, general_pin=inspected, database_inodes=intent['database_inodes'])
save()
for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd)
locks.clear()
run(['systemctl', 'start', 'rmm-helper-cloud.service'])
require(service('rmm-helper-cloud.service')['ActiveState'] == 'active', 'qualified General helper starts')
run(['systemctl', 'start', 'rmm-agent-remote.service'])
require(service('rmm-agent-remote.service')['ActiveState'] == 'active', 'qualified unprivileged agent starts')
pid = int(service('rmm-agent-remote.service')['MainPID'])
process = Path('/proc/'+str(pid)+'/status').read_text().splitlines()
require(next(line for line in process if line.startswith('Uid:')).split()[1:] == ['993']*4
and next(line for line in process if line.startswith('Gid:')).split()[1:] == [str(pwd.getpwnam('rmm-agent').pw_gid)]*4,
'actual agent process remains unprivileged')
report.update(stage='qualified_daemons_started', completed=True, boot_id=intent['boot_id']); save()
except Exception as error:
report.update(error_class=type(error).__name__, frames=[dict(function=f.name, line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
save()
finally:
for fd in reversed(locks): fcntl.flock(fd, fcntl.LOCK_UN); os.close(fd)
return report
def qualification():
out = {}
for name, filename in (('agent', 'rmm-agent'), ('helper', 'rmm-privileged-helper')):
path = ARTIFACTS/filename; metadata = cloud.contract.protected_json(path.with_name(filename+'.json'))
require(metadata['binary_sha256'] == EXPECTED_NEW[name] == sha(path.read_bytes())
and metadata['architecture'] == 'x86_64-unknown-linux-gnu' and metadata['release_build'] is True
and metadata['qualification_profile'] == 'durable-outcome-v1' and metadata['source_before_after_equal'] is True
and metadata['glibc_max'] == '2.34', 'fixed qualified Bookworm release')
require(all(sha((ROOT/p).read_bytes()) == value for p, value in metadata['source_files'].items()), 'release source bindings unchanged')
out[name] = dict(path=str(path.relative_to(ROOT)), sha256=metadata['binary_sha256'], qualification_sha256=sha(path.with_name(filename+'.json').read_bytes()))
native = ROOT/'devstate/shared-core/acceptance/08a9be72-dd7c-46b7-a2c6-7d17424533c2/linux-passed.json'
result = cloud.contract.protected_json(native)
require(result['passed'] is True and result['qualification']['manifest_sha256'] == sha((ARTIFACTS/'manifest.json').read_bytes()), 'existing real Linux76 shared-core qualification')
return dict(artifacts=out, native_linux_evidence_sha256=sha(native.read_bytes()))
def server_gate(intent):
require(reboot.evidence(intent['target'], 'gate') == dict(jobs=30, outbox=30, permits=26, unresolved=0), 'same central30/30/26/0')
policy = cloud.bootstrap.rpc('server.conf', reboot.server_policy, dict(action='state'))
require(policy == dict(policy_sha256=intent['central_policy_sha256'], closed=True), 'same closed central Root policy')
def prepare(restore_readback):
metadata = qualification(); closeout = cloud.contract.protected_json(ROOT/'devstate/cloud-mvp-final-closeout.json')
failed = ROOT/'devstate/shared-core/runtime-upgrade'/FAILED_CASE
restored_path = Path(restore_readback).resolve()
require(restored_path.parent == failed.resolve() and restored_path.name.startswith('restore-old-read-')
and restored_path.suffix == '.json', 'explicit original-case restore readback')
restored = cloud.contract.protected_json(restored_path)
require(restored['id'] == FAILED_CASE and restored['completed'] is True and restored['readback_verified'] is True
and restored['stage'] == 'original_daemons_restarted'
and all(restored['units_now'][name]['ActiveState'] == 'active' for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service'))
and restored['units_now']['rmm-apt-helper.service']['ActiveState'] == 'inactive', 'original runtime actually restored first')
original = ROOT/'deploy/cloud_shared_core_upgrade.py'
require(sha(original.read_bytes()) == ORIGINAL_RUNNER_SHA256, 'historical original runner unchanged')
metadata_path = failed/'special-metadata.json'; special = cloud.contract.protected_json(metadata_path)
names = ('acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json')
entries = special['special_entries']
require(all(not value for key, value in entries.items() if key != '/var/lib/rmm-apt-transaction')
and {value['path'] for value in entries['/var/lib/rmm-apt-transaction']} == {'/var/lib/rmm-apt-transaction/'+name for name in names}
and all((value['device'], value['inode'], value['uid'], value['gid'], value['mode'], value['links'])
== (2049, 1028362, 0, 984, 0o100600, 2) and value['regular'] and not value['socket'] and not value['symlink']
for value in entries['/var/lib/rmm-apt-transaction']), 'exact actually observed alias pair')
checkpoint_path = ROOT/'devstate/cloud-apt-recovery/configured-reconcile-runtime/cases/apt_native_update/checkpoint.json'
checkpoint = cloud.contract.protected_json(checkpoint_path)
require(checkpoint['verified'] is True and checkpoint['stage'] == 'succeeded'
and checkpoint['result']['result']['evidence']['apt']['native_record_sha256'] == ALIAS_SHA256,
'original UI-approved native update bound alias SHA')
require(closeout['verified'] is True and closeout['central_counts'] == dict(jobs=30, outbox=30, permits=26, unresolved=0)
and closeout['policy']['closed'] is True and closeout['native']['root_unresolved'] == 0, 'completed Linux MVP closeout')
observation = closeout['observation']; require(sha((ROOT/observation['path']).read_bytes()) == observation['sha256'], 'verified closeout source')
ident = str(uuid.uuid4()); folder = STATE/ident; folder.mkdir(mode=0o700, parents=True)
native = closeout['native']; old = closeout['vm']['artifact_sha256']
intent = dict(schema_version=1, type='rmm.linux.shared-core-upgrade-v2', id=ident, target=closeout['target'],
boot_id=native['boot_id'], root_rows=native['root_rows'], database_inodes=native['database_inodes'],
old_agent_sha256=old['agent'], old_helper_sha256=old['helper'], new_agent_sha256=EXPECTED_NEW['agent'],
new_helper_sha256=EXPECTED_NEW['helper'], provider_sha256=old['provider'], config_sha256=native['config_sha256'],
pin_sha256=native['pin_sha256'], general_pin=native['general_pin'], central_policy_sha256=closeout['policy']['policy_sha256'],
general_unit_sha256=closeout['vm']['general_unit_sha256'],
agent_unit_sha256=sha((ROOT/'deploy/linux/rmm-agent-remote.service').read_bytes()),
alias_sha256=ALIAS_SHA256, alias_metadata_sha256=sha(metadata_path.read_bytes()), alias_original_checkpoint_sha256=sha(checkpoint_path.read_bytes()),
previous_failed_case=FAILED_CASE, restored_evidence_path=str(restored_path.relative_to(ROOT)), restored_evidence_sha256=sha(restored_path.read_bytes()),
runner_sha256=sha(Path(__file__).read_bytes()), original_runner_sha256=ORIGINAL_RUNNER_SHA256,
closeout_sha256=sha((ROOT/'devstate/cloud-mvp-final-closeout.json').read_bytes()), qualification=metadata,
jobs_created=False, grants_created=False, reboot_requested=False)
cloud.contract.write_private(folder/'intent.json', intent); server_gate(intent)
result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='prepare', id=ident, intent=intent))
cloud.contract.write_private(folder/'prepared.json', result); require(result.get('prepared') is True, 'retained remote intent incomplete')
print('PREPARED '+ident+'; no daemon change or RMM job')
def existing(ident):
folder = STATE/str(uuid.UUID(ident)); intent = cloud.contract.protected_json(folder/'intent.json')
require(intent['id'] == ident and intent['runner_sha256'] == sha(Path(__file__).read_bytes())
and intent['original_runner_sha256'] == sha((ROOT/'deploy/cloud_shared_core_upgrade.py').read_bytes()), 'same retained case/source versions')
return folder, intent
def install(ident):
folder, intent = existing(ident); require(qualification() == intent['qualification'], 'original qualified source/artifacts')
require(cloud.contract.protected_json(folder/'prepared.json')['prepared'] is True, 'remote preparation first')
cloud.contract.write_private(folder/'install-started.json', dict(id=ident, intent_sha256=sha((folder/'intent.json').read_bytes())))
server_gate(intent)
cfg = cloud.bootstrap.access.load_config(ROOT/'credentials/linux-vm.conf')
args = ['sftp', '-F', '/dev/null', '-P', cfg.get('port', '22'), '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
'-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()),
'-b', '-', cfg['username']+'@'+cfg['address']]
for name, item in intent['qualification']['artifacts'].items():
batch = 'put '+json.dumps(str(ROOT/item['path']))+' '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\nchmod 600 '+json.dumps(REMOTE+ident+'/'+name+'-next.upload')+'\n'
result = subprocess.run(args, input=batch.encode(), capture_output=True, timeout=120)
cloud.contract.write_private(folder/(name+'-sftp.json'), dict(returncode=result.returncode,
stdout_b64=base64.b64encode(result.stdout).decode(), stderr_b64=base64.b64encode(result.stderr).decode()))
require(result.returncode == 0, 'private SFTP error; retain case and inspect, never repeat install')
result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='install', id=ident, intent=intent))
cloud.contract.write_private(folder/'install-reply.json', result)
require(result.get('completed') is True, 'retained incomplete report; readback only')
print('INSTALLED '+ident+'; finish/readback and independent native Sync qualification still required')
def finish(ident):
folder, intent = existing(ident)
cloud.contract.protected_json(folder/'install-started.json')
result = cloud.bootstrap.rpc('linux-vm.conf', vm, dict(action='read', id=ident, intent=intent))
cloud.contract.write_private(folder/('readback-'+str(uuid.uuid4())+'.json'), result)
require(result.get('completed') is True and result.get('readback_verified') is True, 'remote maintenance incomplete; do not reinstall')
require(all(v['ActiveState'] == 'active' for v in result['services_now'].values()), 'both qualified daemons active')
server_gate(intent)
cloud.contract.write_private(folder/'result.json', dict(verified_preservation=True, native_sync_accepted=False,
id=ident, intent_sha256=sha((folder/'intent.json').read_bytes()), report=result))
print('PASS '+ident+' preserving Linux daemon upgrade; APT closed/stale; native Sync/MCP acceptance remains separate')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('prepare', 'install', 'finish')); parser.add_argument('id', nargs='?')
parser.add_argument('--restore-readback')
args = parser.parse_args(); os.umask(0o077)
require((args.id is None) == (args.action == 'prepare'), 'prepare creates a case; install/finish require its exact ID')
require(bool(args.restore_readback) == (args.action == 'prepare'), 'prepare requires exact old-runtime restore readback')
if args.action == 'prepare': prepare(args.restore_readback)
elif args.action == 'install': install(args.id)
else: finish(args.id)
if __name__ == '__main__':
try: main()
except Exception as error:
text = str(error)
print(text if text.startswith('remote bootstrap incomplete: ') else 'SHARED_CORE_UPGRADE_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+108
View File
@@ -0,0 +1,108 @@
# Preserving Linux shared-core runtime qualification
The notebook only builds and orchestrates. The fixed Cloud Linux VM remains the
client; `discovery.ease-technologies.de` remains the server. These bootstrap
tools never create or approve an RMM job, enroll, reset a journal, install a
package or reboot the guest.
The original `cloud_shared_core_upgrade.py` remains frozen at SHA-256
`09b60e99e1bf92dbaa60f0b0a64f9b94e3cefebb110d558b14c381293cbde873`.
Its case `2f8de8b9-2abe-4125-aa12-ae38332a32c6` stopped before any binary exchange:
the generic `nlink == 1` check rejected an existing valid APT publication pair.
The original failed report, staged binaries and APT records remain retained.
`cloud_shared_core_failed_restore.py` restarts only the verified original
runtime, once; its saved actual readback is required before a new V2 case.
V2 is a source copy with a narrow alias admission and earlier object checks:
`cloud_shared_core_upgrade_v2.py`, SHA-256
`d4f857b8ff6e7d9802f863a120a1576ffac5b36facb6a259149409ef25bfbc91`.
The only permitted alias paths are under `/var/lib/rmm-apt-transaction/`:
- `acceptance-rmm-apt-fixture-update-1.1.json`
- `pending-acceptance-rmm-apt-fixture-update-1.1.json`
Both must retain device `2049`, inode `1028362`, UID `0`, GID `984`, regular-file
mode `0600`, link count `2`, and SHA-256
`a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506`.
That content hash comes from the original verified, UI-approved native APT
update result. The mode is the actual stored `lstat` value `33152` (`0100600`).
Neither name is rewritten, unlinked or chmodded. Other hardlinks, symlinks and
unexpected nonregular state objects remain rejected.
The qualified `durable-outcome-v1` release artifacts remain:
| Artifact | SHA-256 |
|---|---|
| Linux agent | `c9c09de0ecb52b556d707cf45accb69186c73b8ec9baeeabfaeff220fafb1d4e` |
| Linux helper | `51c3142ab332216d6a94ff64ff15cbc37ee1d5a99896c14c87bb6f62e63bb946` |
| Unchanged APT provider | `1dc0237dd8df9c9d22568bb1f6b047ede73bc2a5fca5c7fcedb7f24ab2a58b1e` |
Planned, not executed by this preparation:
```text
python3 deploy/cloud_shared_core_upgrade_v2.py prepare --restore-readback <exact-private-original-case-readback.json>
python3 deploy/cloud_shared_core_upgrade_v2.py install <new-case-id>
python3 deploy/cloud_shared_core_upgrade_v2.py finish <same-new-case-id>
```
Each new intent binds both runner hashes, the restore readback, actual alias
metadata, original APT checkpoint, release qualifications and the Linux-MVP
closeout. Local cases use `devstate/shared-core/runtime-upgrade-v2/`; remote
cases use `/var/lib/rmm-bootstrap-history/shared-core-v2-<id>`. A durable
installation receipt forbids repeating installation, including after a lost
reply; only readback/finish of that same case is permitted.
The pre-restart preservation checks run under both existing locks. Old
consumptions, outcomes, signatures, database identities, trust and backup
witnesses remain bound. Private key contents are never read or exported.
General policy/pin remain unchanged, `allow_reboot=false`; the APT endpoint
stays closed and its historical runtime pins remain stale. Native mTLS/Sync,
MCP and central-count qualification are separate final acceptance steps.
V2 preparation checks passed locally: Python syntax, fixed RPC globals, source
format, all 65 qualified release source bindings and the existing native Linux76
core-unit receipt. This document does not claim native V2 installation success.
## Retained native bootstrap assertion failures
The original V1 failure above was a bootstrap verifier error: the existing
legitimate APT hardlink publication was rejected. Neither alias was repaired or
rewritten. Its original-runtime restart and actual readback were separately
recorded before the new V2 case.
V2 case `a2423612-721f-4a6c-a832-b9eef549c41e` subsequently reached the persisted
`installed_preserved` stage: both new binaries were installed, all pre-restart
data witnesses matched, and both daemons started. Its last verifier assertion
incorrectly compared process GID with the account's primary passwd GID. Actual
readback showed the expected service identity, UID993/GID984, matching the
independently verified Linux-MVP closeout. This was a second bootstrap verifier
error; the report remains `completed=false` and is never rewritten as success.
`cloud_shared_core_upgrade_attestation.py` is a separate read-only supplement
for that exact case. It rechecks preserved rows/schema/inodes/trust/backups and
the actual process identity, and binds the stored native VM/server Sync114
evidence to the retained exact signed decision and request. It does not start
services, retry installation, change SQL, alter the original report, or create
jobs. Its prepared SHA-256 is
`59b20ecd2d32ea3b37f2840868eb67ce677ffdb967e079a667e53c7907fce86b`.
## Actual read-only closeout, 2026-11-14
The supplemental attestation was actually executed and passed; its protected
receipt is `readonly-attestation-63730c38-327e-4635-802f-8289c28081de.json`
inside the V2 case. A prior strict row-hash readback rejected28 cached outcome
references that normal signed Sync had renewed. Only those reference changes
were then independently proven: reconstruct the original complete row hash with
identical job/attempt/outcome bytes and the preserved original decision ID;
verify both original Ed25519 decisions and the exact applied request/response,
grant/plan/outcome binding and monotonic journal history. All other historical
rows remain byte-identical. No general outcome exemption or state rewrite.
Native MCP subsequently read the existing Linux device online, rejected the
foreign tenant, and confirmed unchanged30/30/26/0 central counts. Agent/helper
and server share the ready signed Sync114; original18 Root consumptions remain
closed. `devstate/cloud-shared-core-current-runtime.json` binds both actual
attestation and MCP receipts. It supplements the preserved historical MVP
closeout and does not requalify APT for the new binary hashes. All original
failure reports remain unchanged, and no installation or service start was
repeated to obtain this read-only closeout.
+12
View File
@@ -52,6 +52,18 @@ Zugänge und private Rohbelege bleiben gitignoriert.
## Aktuelle Windowsfortsetzung
Aktueller Linux-Dienststand nach Shared-Core-Fortsetzung: Agent `c9c09de0…`,
Helper `51c3142a…`, Provider/Units/Policies unverändert. Eigener Fall `a2423612…`
hat beide Binärdateien erhaltend installiert; ein getrennter Nur-Lese-Nachbeleg
bestätigt UID993/GID984, Root18/0, historische DB-/PKI-/Outcomeerhaltung und
normalen signierten Sync114. Zentral30/30/26/0, MCP online/fremder Tenant
verweigert; keine neuen Jobs oder Reboots. Originale Fehlerberichte bleiben
erhalten (zu strenger Hardlinkcheck, falscher letzter Gruppenvergleich und
zunächst nicht berücksichtigte signierte Cache-Referenzerneuerung).
`devstate/cloud-shared-core-current-runtime.json` bindet die tatsächlichen
Zusatzbelege. Historischer Abschluss und alte Paketabnahmen bleiben unverändert;
APT ist weiter geschlossen und für die neuen Hashes nicht erneut qualifiziert.
Neuester passiver Windowsstand: Agent `f64e8115…`, eigener erhaltender Fall
`c4f9b280-e83d-4b8e-a1c2-1cd103f73eda`, tatsächlicher LocalService/Auto/WSS,
gleiche Identität/Schlüsselmetadaten/ACLs und ursprüngliche Telemetrie-DB.
+38
View File
@@ -1,5 +1,43 @@
# Tatsächlicher Implementierungsstand
## Gemeinsamer Kern im tatsächlichen Linux-Dienstbetrieb nachgewiesen — 2026-11-14
Agent `c9c09de0…` und Generalhelper `51c3142a…` laufen jetzt auf der Cloud-
Linux-VM mit dem zuvor nativ qualifizierten gemeinsamen Kern. Tatsächlicher
Agent weiterhin UID993/GID984, beide Dienste aktiv, gleiche Geräteidentität,
Bootkennung, DB-Inodes, PKI und ursprüngliche18 geschlossenen Root-Verbräuche.
Die alten Binärdateien sind versioniert erhalten. Neuer normaler signierter
Sync Generation114: derselbe Entscheid in Agent/Helper/Server, ready/helper_ready,
kein Restoreblock/Fault. MCP liest das gespeicherte Gerät online; fremder Tenant
verweigert. Zentral weiterhin30 Jobs/30 Outbox/26 Grants/0 ungeklärt, Rootscope
geschlossen, keine neue Genehmigung oder RMM-Wirkung.
Die erste Installationsprüfung `2f8de8b9…` brach vor Binärtausch an zwei gültigen
APT-Hardlink-Abnahmedateien ab. Beide alten Dienste wurden unter derselben
erhaltenen Intention einmalig wieder gestartet, ohne Dateien/Verbräuche zu
ändern. Ein eigener zweiter Fall `a2423612…` bewahrte anschließend alle Daten
vor dem Tausch und startete beide neuen Dienste; nur sein letzter Check
verwechselte primäre Benutzergruppe und festgelegte Dienstgruppe. Der originale
Fehlerbericht bleibt `completed=false`; keine Wiederholung des Installers.
Ein separater tatsächlicher Nur-Lese-Nachbeleg bestätigt die korrekte Dienst-
identität und alle erhaltenen Originale.28 lokale Outcome-Caches erneuern im
normalen Sync ihre Entscheidungsreferenz; auch das wurde zunächst als
Byteabweichung gemeldet. Danach ausschließlich diese Referenzerneuerung streng
nachgewiesen: ursprünglicher vollständiger Zeilenhash aus unveränderten Job-/
Versuchs-/Outcome-Bytes rekonstruiert, alte/neue Ed25519-Entscheide unabhängig
verifiziert, ursprüngliche Anträge/Responses und Plan-/Grant-/Outcomebindungen
belegt. Alle übrigen historischen Zeilen unverändert. Kein pauschales Ausnehmen
von Outcomes, kein Reset oder nachträgliches Umschreiben des Fehlerberichts.
Geschützte Fälle unter `devstate/shared-core/runtime-upgrade{,-v2}/`; separater
aktueller Beleg `devstate/cloud-shared-core-current-runtime.json`. Historischer
MVP-Abschluss bleibt unverändert. APT bleibt inaktiv, Endpunkt entfernt und
General `allow_reboot=false`. Die früheren APT-Paketabnahmen gelten weiter für
ihre ursprünglichen Artefakte; neue Agent-/Helperhashes ergeben ausdrücklich
keine neue APT-Readiness-/Paketqualifikation. Runner und Erhaltungspins:
[Shared-Core-Dienstwechsel](../deploy/shared-core-upgrade.md).
## Gebundenes Unknown-Folgeergebnis wird dauerhaft bestätigt — lokal geprüft
Ein nachträgliches authentisiertes JobResult mit höherer Revision darf jetzt