"""Configuration-only General startup retry; no OS reboot or RMM job. The existing 40s SSH transport can time out before this bounded maintenance returns. Read its deterministic durable report; never repeat the operation. """ def qualify_startup_retry(payload): import contextlib import fcntl import sqlite3 import stat import time import traceback os.umask(0o077) require(os.geteuid() == 0 and type(payload['apply']) is bool, 'explicit bootstrap preflight/apply') identifier = str(uuid.UUID(payload['id'])) state = Path('/var/lib/rmm-agent-remote'); root = Path('/var/lib/rmm-helper') account = pwd.getpwnam('rmm-agent') unit = Path('/etc/systemd/system/rmm-helper-cloud.service') config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json') def digest(data):return hashlib.sha256(data).hexdigest() def owned(path,owner,limit): require(not path.is_symlink(), 'owned fixed file symlink') s = path.stat() require(stat.S_ISREG(s.st_mode) and s.st_uid == owner and not s.st_mode&0o022 and s.st_size <= limit, 'owned bounded fixed file') return path.read_bytes() old_unit = owned(unit,0,16384) require(digest(old_unit) == payload['old_unit_sha256'] and old_unit.count(b'Restart=no\n') == 1 and b'RestartSec=' not in old_unit, 'exact original General unit') new_unit = old_unit.replace(b'Restart=no\n',b'Restart=on-failure\nRestartSec=5\n') require(digest(new_unit) == payload['new_unit_sha256'], 'independently reviewed two-line unit delta') files = {Path('/usr/local/libexec/rmm-privileged-helper'):payload['helper_sha256'], Path('/usr/local/libexec/rmm-agent-remote'):payload['agent_sha256'], Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'):payload['provider_sha256']} require(all(digest(owned(path,0,32*1024*1024)) == sha for path,sha in files.items()), 'qualified helper/agent/provider unchanged') binding = json.loads(owned(state/'binding.json',account.pw_uid,16384)) cached = json.loads(owned(state/'job-signing-key.json',account.pw_uid,4096)) public = json.loads(owned(Path('/etc/rmm-remote/helper-signing-key.json'),0,4096)) general = json.loads(owned(config,0,16384)); policy_pin = json.loads(owned(pin,0,16384)) raw_key = bytes.fromhex(public['public_key_hex']) require(set(public) == {'algorithm','schema_version','public_key_hex','key_id'} and cached == public and public['algorithm'] == 'Ed25519' and type(public['schema_version']) is int and public['schema_version'] == 1 and len(raw_key) == 32 and public['key_id'] == digest(raw_key) and binding['public_key_hex'] == public['public_key_hex'] and binding['agent_id'] == payload['agent_id'] and binding['asset_id'] == payload['asset_id'] and general['allow_reboot'] is False and general['profile'] == policy_pin['profile'] == 'general' and general['signing_key_file'] == '/etc/rmm-remote/helper-signing-key.json' and general['binding']['public_key_hex'] in ('',public['public_key_hex']) and policy_pin['identity']['signing_key_id'] == public['key_id'] and all(binding[k] == general['binding'][k] == policy_pin['identity'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')), 'same non-reboot General public identity/policy') require(b'RMM_APT_HELPER_' not in owned(Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf'),0,4096), 'APT endpoint remains closed') def run(args,timeout=10,check=True):return subprocess.run(args,capture_output=True,check=check,timeout=timeout) def services(): result = {} for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'): raw = run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts','-p','Restart','-p','RestartUSec','-p','Result','-p','ExecMainStatus','-p','UnitFileState']).stdout.decode() result[name] = dict(line.split('=',1) for line in raw.splitlines()) return result def settled(): result = dict(boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),inodes={}) for path in (state/'journal.sqlite',state/'consumption.sqlite',root/'consumption.sqlite'): require(not path.is_symlink(), 'fixed database symlink') s = path.stat(); result['inodes'][str(path)] = [s.st_dev,s.st_ino] sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')] existing = {p for p in sidecars if p.exists()} try: with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'database integrity') if path == state/'journal.sqlite': require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 and db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'all agent work/faults closed') else:require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0, 'all effects closed') if path == root/'consumption.sqlite': require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no Root fault') rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall() serial = [[dict(sha256=digest(v),size=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows] result.update(root_consumed=len(rows),root_rows_sha256=digest(json.dumps(serial,separators=(',',':')).encode())) finally: for p in sidecars: if p not in existing and p.exists(): fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW) try: side = os.fstat(fd);require(stat.S_ISREG(side.st_mode) and side.st_nlink == 1 and side.st_uid in (0,s.st_uid), 'only own new fixed SQLite sidecar') if side.st_uid == 0 and s.st_uid != 0:os.fchown(fd,s.st_uid,s.st_gid) finally:os.close(fd) return result before = services(); original = settled() require(all(before[n]['ActiveState'] == 'active' and before[n]['UnitFileState'] == 'enabled' for n in ('rmm-agent-remote.service','rmm-helper-cloud.service')) and before['rmm-apt-helper.service']['ActiveState'] == 'inactive' and before['rmm-helper-cloud.service']['Restart'] == 'no', 'healthy enabled services with exact startup gap') folder = Path('/var/lib/rmm-bootstrap-history')/('cloud-general-startup-retry-'+identifier) probe_dir = unit.parent/'rmm-helper-cloud.service.d'; probe = probe_dir/'96-cloud-startup-retry-probe.conf' require(not folder.exists() and not folder.is_symlink() and not probe.exists() and not probe.is_symlink() and not probe_dir.is_symlink(), 'new own versioned test only') report = dict(preflight=True,completed=False,unit_delta_only=True,old_unit_sha256=payload['old_unit_sha256'], new_unit_sha256=payload['new_unit_sha256'],retained=original, jobs_created=False,reboot_requested=False,package_effects=False,binaries_changed=False) if not payload['apply']:return report folder.mkdir(mode=0o700); probe_dir.mkdir(mode=0o755,exist_ok=True) require(probe_dir.stat().st_uid == 0 and not probe_dir.stat().st_mode&0o022, 'root-owned helper dropin directory') def sync_dir(path): fd = os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) try:os.fsync(fd) finally:os.close(fd) def create(path,raw,mode=0o600): fd = os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode) with os.fdopen(fd,'wb') as stream:stream.write(raw);stream.flush();os.fchmod(stream.fileno(),mode);os.fsync(stream.fileno()) sync_dir(path.parent) def save(): path = folder/'.report-next.json';create(path,json.dumps(report).encode());os.replace(path,folder/'report.json');sync_dir(folder) report.update(preflight=False,stage='validated_before_service_stop',backup=str(folder),rollback_armed=False);save() try: missing = folder/'intentionally-absent-helper-config.json' require(not missing.exists(), 'fixed nonexistent configuration fault') fault = ('[Service]\nEnvironment=RMM_HELPER_CONFIG='+str(missing)+'\n').encode() # Independent cleanup only removes our exact temporary config override. # The permanent on-failure policy then performs the recovery itself. rollback = ('from pathlib import Path\nimport os,subprocess\np=Path('+repr(str(probe))+')\n' 'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(fault)+':\n' ' p.unlink()\n fd=os.open(p.parent,os.O_RDONLY|os.O_DIRECTORY);os.fsync(fd);os.close(fd)\n' ' subprocess.run(["systemctl","daemon-reload"],check=True)\n') create(folder/'rollback.py',rollback.encode()) run(['systemd-run','--quiet','--unit=rmm-general-startup-rollback-'+identifier,'--on-active=60s','--property=Type=oneshot','/usr/bin/python3',str(folder/'rollback.py')]) report.update(stage='rollback_armed',rollback_armed=True);save() run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service']) report['stage']='services_stopped';save() lock = os.open(root/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) try: require(settled() == original, 'no boot/inode/effect change before unit transition') preserved = [state/'binding.json',state/'job-signing-key.json',config,pin, Path('/etc/rmm-remote/helper-signing-key.json'),Path('/etc/rmm/apt-helper.json'), Path('/etc/rmm-remote/apt-helper-proof.json'),Path('/var/lib/rmm-apt-transaction/install-metadata.json'),*files] def retained_files(): result = {} for p in preserved: require(not p.is_symlink(), 'preserved pin/artifact symlink') s = p.stat();result[str(p)] = [s.st_dev,s.st_ino,digest(p.read_bytes())] return result pins = retained_files(); root_db_sha = digest((root/'consumption.sqlite').read_bytes()) create(folder/'unit-next',new_unit,0o644) require(unit.stat().st_dev == folder.stat().st_dev, 'same-filesystem unit versioning') os.rename(unit,folder/'unit-before.service');os.rename(folder/'unit-next',unit) sync_dir(folder);sync_dir(unit.parent) report['stage']='unit_versioned';save() run(['systemd-analyze','verify',str(unit)]) create(probe,fault,0o644);run(['systemctl','daemon-reload']) report.update(preflight=False,stage='fault_armed',backup=str(folder));save() finally:fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock) run(['systemctl','start','rmm-helper-cloud.service'],check=False) deadline = time.monotonic()+24 while True: failed = services()['rmm-helper-cloud.service'] if int(failed['NRestarts']) >= 1 and failed['Result'] == 'exit-code' and int(failed['ExecMainStatus']) != 0:break require(time.monotonic() < deadline, 'actual automatic retry not observed');time.sleep(0.5) require(not missing.exists() and probe.read_bytes() == fault and retained_files() == pins and digest((root/'consumption.sqlite').read_bytes()) == root_db_sha and settled() == original, 'failure before ledger open must preserve all original state') report.update(stage='automatic_failure_verified',failed=failed,failed_before_ledger_open=True);save() probe.unlink();sync_dir(probe_dir);run(['systemctl','daemon-reload']) # No start/restart/reset-failed command here: actual policy must recover. while True: recovered = services()['rmm-helper-cloud.service'] if recovered['ActiveState'] == 'active' and recovered['SubState'] == 'running' and int(recovered['MainPID']) > 1:break require(time.monotonic() < deadline, 'automatic helper recovery missing');time.sleep(0.5) time.sleep(1) stable = services()['rmm-helper-cloud.service'] require(stable['ActiveState'] == 'active' and stable['MainPID'] == recovered['MainPID'] and int(stable['NRestarts']) >= 1 and stable['Restart'] == 'on-failure' and stable['RestartUSec'] == '5s' and settled() == original and retained_files() == pins and digest(unit.read_bytes()) == payload['new_unit_sha256'] and not probe.exists(), 'native automatic recovery with preserved identity/pins/Root rows') run(['systemctl','stop','rmm-general-startup-rollback-'+identifier+'.timer'],check=False) report.update(stage='automatic_recovery_verified',recovered=stable,state_preserved=True, temporary_fault_removed=True,automatic_recovery=True);save() run(['systemctl','start','rmm-agent-remote.service']) report.update(stage='qualified_runtime_started',completed=True);save() return report except Exception as error: report.update(completed=False,error_phase=report.get('stage'),error_class=type(error).__name__, frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)]) report['exception_receipt_persisted']=True try: save() except Exception: report['exception_receipt_persisted']=False return report