From a2ce8f5e8a6689b76a015f10b69e26b5f3b3bf2d Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Fri, 13 Nov 2026 22:14:44 +0100 Subject: [PATCH] Verify automatic General helper startup recovery without another guest reboot --- deploy/cloud_general_startup_retry.py | 188 ++++++++++++++++++++++++++ deploy/linux/rmm-helper-cloud.service | 3 +- docs/implementation-status.md | 12 ++ docs/reboot-acceptance.md | 20 ++- 4 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 deploy/cloud_general_startup_retry.py diff --git a/deploy/cloud_general_startup_retry.py b/deploy/cloud_general_startup_retry.py new file mode 100644 index 0000000..3deb2c0 --- /dev/null +++ b/deploy/cloud_general_startup_retry.py @@ -0,0 +1,188 @@ +"""Configuration-only General startup retry; no OS reboot or RMM job. + +The existing 40s SSH transport can time out before this bounded maintenance +returns. Read its deterministic durable report; never repeat the operation. +""" + + +def qualify_startup_retry(payload): + import contextlib + import fcntl + import sqlite3 + import stat + import time + import traceback + os.umask(0o077) + require(os.geteuid() == 0 and type(payload['apply']) is bool, 'explicit bootstrap preflight/apply') + identifier = str(uuid.UUID(payload['id'])) + state = Path('/var/lib/rmm-agent-remote'); root = Path('/var/lib/rmm-helper') + account = pwd.getpwnam('rmm-agent') + unit = Path('/etc/systemd/system/rmm-helper-cloud.service') + config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json') + def digest(data):return hashlib.sha256(data).hexdigest() + def owned(path,owner,limit): + require(not path.is_symlink(), 'owned fixed file symlink') + s = path.stat() + require(stat.S_ISREG(s.st_mode) and s.st_uid == owner and not s.st_mode&0o022 and s.st_size <= limit, 'owned bounded fixed file') + return path.read_bytes() + old_unit = owned(unit,0,16384) + require(digest(old_unit) == payload['old_unit_sha256'] and old_unit.count(b'Restart=no\n') == 1 + and b'RestartSec=' not in old_unit, 'exact original General unit') + new_unit = old_unit.replace(b'Restart=no\n',b'Restart=on-failure\nRestartSec=5\n') + require(digest(new_unit) == payload['new_unit_sha256'], 'independently reviewed two-line unit delta') + files = {Path('/usr/local/libexec/rmm-privileged-helper'):payload['helper_sha256'], + Path('/usr/local/libexec/rmm-agent-remote'):payload['agent_sha256'], + Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'):payload['provider_sha256']} + require(all(digest(owned(path,0,32*1024*1024)) == sha for path,sha in files.items()), 'qualified helper/agent/provider unchanged') + binding = json.loads(owned(state/'binding.json',account.pw_uid,16384)) + cached = json.loads(owned(state/'job-signing-key.json',account.pw_uid,4096)) + public = json.loads(owned(Path('/etc/rmm-remote/helper-signing-key.json'),0,4096)) + general = json.loads(owned(config,0,16384)); policy_pin = json.loads(owned(pin,0,16384)) + raw_key = bytes.fromhex(public['public_key_hex']) + require(set(public) == {'algorithm','schema_version','public_key_hex','key_id'} and cached == public + and public['algorithm'] == 'Ed25519' and type(public['schema_version']) is int and public['schema_version'] == 1 + and len(raw_key) == 32 and public['key_id'] == digest(raw_key) + and binding['public_key_hex'] == public['public_key_hex'] + and binding['agent_id'] == payload['agent_id'] and binding['asset_id'] == payload['asset_id'] + and general['allow_reboot'] is False and general['profile'] == policy_pin['profile'] == 'general' + and general['signing_key_file'] == '/etc/rmm-remote/helper-signing-key.json' + and general['binding']['public_key_hex'] in ('',public['public_key_hex']) + and policy_pin['identity']['signing_key_id'] == public['key_id'] + and all(binding[k] == general['binding'][k] == policy_pin['identity'][k] for k in + ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')), 'same non-reboot General public identity/policy') + require(b'RMM_APT_HELPER_' not in owned(Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf'),0,4096), 'APT endpoint remains closed') + def run(args,timeout=10,check=True):return subprocess.run(args,capture_output=True,check=check,timeout=timeout) + def services(): + result = {} + for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'): + raw = run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts','-p','Restart','-p','RestartUSec','-p','Result','-p','ExecMainStatus','-p','UnitFileState']).stdout.decode() + result[name] = dict(line.split('=',1) for line in raw.splitlines()) + return result + def settled(): + result = dict(boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),inodes={}) + for path in (state/'journal.sqlite',state/'consumption.sqlite',root/'consumption.sqlite'): + require(not path.is_symlink(), 'fixed database symlink') + s = path.stat(); result['inodes'][str(path)] = [s.st_dev,s.st_ino] + sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')] + existing = {p for p in sidecars if p.exists()} + try: + with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db: + require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'database integrity') + if path == state/'journal.sqlite': + require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 + and db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'all agent work/faults closed') + else:require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0, 'all effects closed') + if path == root/'consumption.sqlite': + require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no Root fault') + rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall() + serial = [[dict(sha256=digest(v),size=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows] + result.update(root_consumed=len(rows),root_rows_sha256=digest(json.dumps(serial,separators=(',',':')).encode())) + finally: + for p in sidecars: + if p not in existing and p.exists(): + fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW) + try: + side = os.fstat(fd);require(stat.S_ISREG(side.st_mode) and side.st_nlink == 1 and side.st_uid in (0,s.st_uid), 'only own new fixed SQLite sidecar') + if side.st_uid == 0 and s.st_uid != 0:os.fchown(fd,s.st_uid,s.st_gid) + finally:os.close(fd) + return result + before = services(); original = settled() + require(all(before[n]['ActiveState'] == 'active' and before[n]['UnitFileState'] == 'enabled' for n in + ('rmm-agent-remote.service','rmm-helper-cloud.service')) + and before['rmm-apt-helper.service']['ActiveState'] == 'inactive' + and before['rmm-helper-cloud.service']['Restart'] == 'no', 'healthy enabled services with exact startup gap') + folder = Path('/var/lib/rmm-bootstrap-history')/('cloud-general-startup-retry-'+identifier) + probe_dir = unit.parent/'rmm-helper-cloud.service.d'; probe = probe_dir/'96-cloud-startup-retry-probe.conf' + require(not folder.exists() and not folder.is_symlink() and not probe.exists() and not probe.is_symlink() + and not probe_dir.is_symlink(), 'new own versioned test only') + report = dict(preflight=True,completed=False,unit_delta_only=True,old_unit_sha256=payload['old_unit_sha256'], + new_unit_sha256=payload['new_unit_sha256'],retained=original, + jobs_created=False,reboot_requested=False,package_effects=False,binaries_changed=False) + if not payload['apply']:return report + folder.mkdir(mode=0o700); probe_dir.mkdir(mode=0o755,exist_ok=True) + require(probe_dir.stat().st_uid == 0 and not probe_dir.stat().st_mode&0o022, 'root-owned helper dropin directory') + def sync_dir(path): + fd = os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + try:os.fsync(fd) + finally:os.close(fd) + def create(path,raw,mode=0o600): + fd = os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode) + with os.fdopen(fd,'wb') as stream:stream.write(raw);stream.flush();os.fchmod(stream.fileno(),mode);os.fsync(stream.fileno()) + sync_dir(path.parent) + def save(): + path = folder/'.report-next.json';create(path,json.dumps(report).encode());os.replace(path,folder/'report.json');sync_dir(folder) + report.update(preflight=False,stage='validated_before_service_stop',backup=str(folder),rollback_armed=False);save() + try: + missing = folder/'intentionally-absent-helper-config.json' + require(not missing.exists(), 'fixed nonexistent configuration fault') + fault = ('[Service]\nEnvironment=RMM_HELPER_CONFIG='+str(missing)+'\n').encode() + # Independent cleanup only removes our exact temporary config override. + # The permanent on-failure policy then performs the recovery itself. + rollback = ('from pathlib import Path\nimport os,subprocess\np=Path('+repr(str(probe))+')\n' + 'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(fault)+':\n' + ' p.unlink()\n fd=os.open(p.parent,os.O_RDONLY|os.O_DIRECTORY);os.fsync(fd);os.close(fd)\n' + ' subprocess.run(["systemctl","daemon-reload"],check=True)\n') + create(folder/'rollback.py',rollback.encode()) + run(['systemd-run','--quiet','--unit=rmm-general-startup-rollback-'+identifier,'--on-active=60s','--property=Type=oneshot','/usr/bin/python3',str(folder/'rollback.py')]) + report.update(stage='rollback_armed',rollback_armed=True);save() + run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service']) + report['stage']='services_stopped';save() + lock = os.open(root/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + try: + require(settled() == original, 'no boot/inode/effect change before unit transition') + preserved = [state/'binding.json',state/'job-signing-key.json',config,pin, + Path('/etc/rmm-remote/helper-signing-key.json'),Path('/etc/rmm/apt-helper.json'), + Path('/etc/rmm-remote/apt-helper-proof.json'),Path('/var/lib/rmm-apt-transaction/install-metadata.json'),*files] + def retained_files(): + result = {} + for p in preserved: + require(not p.is_symlink(), 'preserved pin/artifact symlink') + s = p.stat();result[str(p)] = [s.st_dev,s.st_ino,digest(p.read_bytes())] + return result + pins = retained_files(); root_db_sha = digest((root/'consumption.sqlite').read_bytes()) + create(folder/'unit-next',new_unit,0o644) + require(unit.stat().st_dev == folder.stat().st_dev, 'same-filesystem unit versioning') + os.rename(unit,folder/'unit-before.service');os.rename(folder/'unit-next',unit) + sync_dir(folder);sync_dir(unit.parent) + report['stage']='unit_versioned';save() + run(['systemd-analyze','verify',str(unit)]) + create(probe,fault,0o644);run(['systemctl','daemon-reload']) + report.update(preflight=False,stage='fault_armed',backup=str(folder));save() + finally:fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock) + run(['systemctl','start','rmm-helper-cloud.service'],check=False) + deadline = time.monotonic()+24 + while True: + failed = services()['rmm-helper-cloud.service'] + if int(failed['NRestarts']) >= 1 and failed['Result'] == 'exit-code' and int(failed['ExecMainStatus']) != 0:break + require(time.monotonic() < deadline, 'actual automatic retry not observed');time.sleep(0.5) + require(not missing.exists() and probe.read_bytes() == fault and retained_files() == pins + and digest((root/'consumption.sqlite').read_bytes()) == root_db_sha + and settled() == original, 'failure before ledger open must preserve all original state') + report.update(stage='automatic_failure_verified',failed=failed,failed_before_ledger_open=True);save() + probe.unlink();sync_dir(probe_dir);run(['systemctl','daemon-reload']) + # No start/restart/reset-failed command here: actual policy must recover. + while True: + recovered = services()['rmm-helper-cloud.service'] + if recovered['ActiveState'] == 'active' and recovered['SubState'] == 'running' and int(recovered['MainPID']) > 1:break + require(time.monotonic() < deadline, 'automatic helper recovery missing');time.sleep(0.5) + time.sleep(1) + stable = services()['rmm-helper-cloud.service'] + require(stable['ActiveState'] == 'active' and stable['MainPID'] == recovered['MainPID'] + and int(stable['NRestarts']) >= 1 and stable['Restart'] == 'on-failure' and stable['RestartUSec'] == '5s' + and settled() == original and retained_files() == pins + and digest(unit.read_bytes()) == payload['new_unit_sha256'] and not probe.exists(), 'native automatic recovery with preserved identity/pins/Root rows') + run(['systemctl','stop','rmm-general-startup-rollback-'+identifier+'.timer'],check=False) + report.update(stage='automatic_recovery_verified',recovered=stable,state_preserved=True, + temporary_fault_removed=True,automatic_recovery=True);save() + run(['systemctl','start','rmm-agent-remote.service']) + report.update(stage='qualified_runtime_started',completed=True);save() + return report + except Exception as error: + report.update(completed=False,error_phase=report.get('stage'),error_class=type(error).__name__, + frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]) + report['exception_receipt_persisted']=True + try: + save() + except Exception: + report['exception_receipt_persisted']=False + return report diff --git a/deploy/linux/rmm-helper-cloud.service b/deploy/linux/rmm-helper-cloud.service index 567c0a4..cfa1ea1 100644 --- a/deploy/linux/rmm-helper-cloud.service +++ b/deploy/linux/rmm-helper-cloud.service @@ -13,7 +13,8 @@ RuntimeDirectoryMode=0750 UMask=0077 Environment=RMM_HELPER_CONFIG=/etc/rmm-remote/helper-general.json ExecStart=/usr/local/libexec/rmm-privileged-helper -Restart=no +Restart=on-failure +RestartSec=5 NoNewPrivileges=true PrivateTmp=true PrivateNetwork=true diff --git a/docs/implementation-status.md b/docs/implementation-status.md index abd169b..df3e8ce 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -1,5 +1,17 @@ # Tatsächlicher Implementierungsstand +## General-Startretry nativ nachgewiesen — 2026-11-13 + +Einzige dauerhafte UnitänderungRestart=on-failure/RestartSec=5, keinBinary-/ +Policy-/Privilegienwechsel. Kontrollierter früher Konfigurationsfehler erzeugte +Exit1 und automatischeWiederholungNRestarts1; nach exakterOverrideentfernung +automatische stabileRecoveryNRestarts2 ohne zweitenStartbefehl. AlleRoot18-/ +DBinodes/-bytes, Schlüssel und Policies erhalten, keinRMMjob/OSreboot. +Native private startup-retry-result.jsoncompleted/automatic_recovery/ +state_preservedtrue. Quelledeploy/linux/rmm-helper-cloud.service entspricht +nativerUnitSHA104a7d6f. Rebootrechte weiterfalse/zentralgeschlossen, Agentwiederaktiv. +Noch APT-Runtime-Messungsrequalifikation, MCP-Endbericht und DoD-Endsnapshot. + ## Einziger Reboot vollständig nativ abgeglichen und Rechte geschlossen Original8ffd/25a jetztSucceeded/revision3. Rechtzeitiger Originalinventarbeleg diff --git a/docs/reboot-acceptance.md b/docs/reboot-acceptance.md index a9b1396..78a5210 100644 --- a/docs/reboot-acceptance.md +++ b/docs/reboot-acceptance.md @@ -22,10 +22,22 @@ Checkpoint und ursprüngliche Artefaktpins blieben unverändert; separater hashgebundener capacity-upgrade.json qualifiziert lediglich den erhaltenden storage-onlyArtefaktaustausch auf9e9f/d6fb. [Kapazitätsnachweis](journal-capacity.md). -BootfehlerCLOCK_UNTRUSTED/Restart=no bleibt realer Befund: keine Behauptung, -dass die ursprüngliche Unit automatisch wieder anlief. Erforderliche enge -Startretrykorrektur wird separat ohne weiteren OSreboot nativ geprüft. Noch -kein vollständiger MVP-Abschluss; aktuelle restliche Gates im Abschlussaudit. +Der ursprüngliche BootfehlerCLOCK_UNTRUSTED/Restart=no bleibt realer Befund. +Danach enge Konfigurationskorrektur nativ bestanden: nur Restart=no durch +Restart=on-failure/RestartSec=5 ersetzt. Kontrollierte fehlende Konfigurationsdatei +ließ Helper vor Ledgeröffnung mitExit1 scheitern, tatsächlichesNRestarts1 +belegt. Exakter temporärer Override entfernt; **ohne zweiten Start-/Restart- +befehl** lief derselbe Helper automatisch stabil (NRestarts2/active). Agent +während dieser Bootstrapprobe gestoppt; alleRoot18-Verbrauchszeilen/DBinodes/ +RootDBbytes, Identitäten, Schlüssel, Policies und Binärdateien erhalten. Keine +Job-/Paketwirkung, keinOSreboot; eigenesRollbacktimer entfernt nur exakten +Probeoverride und wurde nach Erfolg gestoppt. Private startup-retry-{intent, +preflight,apply-intent,result}.json, Ergebniscompleted/automatic_recovery/ +state_preservedtrue. Tatsächliche GeneralUnitSHA jetzt +104a7d6f01ec79aa2d6730f098e433418ce3bd04fc180fc4847dc94d873bb5a7. +Dies belegt automatische Retry-/Recoverywirkung nach Startfehler; kein weiterer +OSboottest nach Unitkorrektur wurde ausgeführt. Noch APT-Messungsrequalifikation, +MCP-Endbericht und Endaudit erforderlich. ## Historischer Zwischenstand vor endgültigem Abschluss