Files
rust-discovery/deploy/cloud_general_startup_retry.py
2026-11-13 22:14:44 +01:00

189 lines
14 KiB
Python

"""Configuration-only General startup retry; no OS reboot or RMM job.
The existing 40s SSH transport can time out before this bounded maintenance
returns. Read its deterministic durable report; never repeat the operation.
"""
def qualify_startup_retry(payload):
import contextlib
import fcntl
import sqlite3
import stat
import time
import traceback
os.umask(0o077)
require(os.geteuid() == 0 and type(payload['apply']) is bool, 'explicit bootstrap preflight/apply')
identifier = str(uuid.UUID(payload['id']))
state = Path('/var/lib/rmm-agent-remote'); root = Path('/var/lib/rmm-helper')
account = pwd.getpwnam('rmm-agent')
unit = Path('/etc/systemd/system/rmm-helper-cloud.service')
config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json')
def digest(data):return hashlib.sha256(data).hexdigest()
def owned(path,owner,limit):
require(not path.is_symlink(), 'owned fixed file symlink')
s = path.stat()
require(stat.S_ISREG(s.st_mode) and s.st_uid == owner and not s.st_mode&0o022 and s.st_size <= limit, 'owned bounded fixed file')
return path.read_bytes()
old_unit = owned(unit,0,16384)
require(digest(old_unit) == payload['old_unit_sha256'] and old_unit.count(b'Restart=no\n') == 1
and b'RestartSec=' not in old_unit, 'exact original General unit')
new_unit = old_unit.replace(b'Restart=no\n',b'Restart=on-failure\nRestartSec=5\n')
require(digest(new_unit) == payload['new_unit_sha256'], 'independently reviewed two-line unit delta')
files = {Path('/usr/local/libexec/rmm-privileged-helper'):payload['helper_sha256'],
Path('/usr/local/libexec/rmm-agent-remote'):payload['agent_sha256'],
Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'):payload['provider_sha256']}
require(all(digest(owned(path,0,32*1024*1024)) == sha for path,sha in files.items()), 'qualified helper/agent/provider unchanged')
binding = json.loads(owned(state/'binding.json',account.pw_uid,16384))
cached = json.loads(owned(state/'job-signing-key.json',account.pw_uid,4096))
public = json.loads(owned(Path('/etc/rmm-remote/helper-signing-key.json'),0,4096))
general = json.loads(owned(config,0,16384)); policy_pin = json.loads(owned(pin,0,16384))
raw_key = bytes.fromhex(public['public_key_hex'])
require(set(public) == {'algorithm','schema_version','public_key_hex','key_id'} and cached == public
and public['algorithm'] == 'Ed25519' and type(public['schema_version']) is int and public['schema_version'] == 1
and len(raw_key) == 32 and public['key_id'] == digest(raw_key)
and binding['public_key_hex'] == public['public_key_hex']
and binding['agent_id'] == payload['agent_id'] and binding['asset_id'] == payload['asset_id']
and general['allow_reboot'] is False and general['profile'] == policy_pin['profile'] == 'general'
and general['signing_key_file'] == '/etc/rmm-remote/helper-signing-key.json'
and general['binding']['public_key_hex'] in ('',public['public_key_hex'])
and policy_pin['identity']['signing_key_id'] == public['key_id']
and all(binding[k] == general['binding'][k] == policy_pin['identity'][k] for k in
('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')), 'same non-reboot General public identity/policy')
require(b'RMM_APT_HELPER_' not in owned(Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf'),0,4096), 'APT endpoint remains closed')
def run(args,timeout=10,check=True):return subprocess.run(args,capture_output=True,check=check,timeout=timeout)
def services():
result = {}
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
raw = run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts','-p','Restart','-p','RestartUSec','-p','Result','-p','ExecMainStatus','-p','UnitFileState']).stdout.decode()
result[name] = dict(line.split('=',1) for line in raw.splitlines())
return result
def settled():
result = dict(boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),inodes={})
for path in (state/'journal.sqlite',state/'consumption.sqlite',root/'consumption.sqlite'):
require(not path.is_symlink(), 'fixed database symlink')
s = path.stat(); result['inodes'][str(path)] = [s.st_dev,s.st_ino]
sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')]
existing = {p for p in sidecars if p.exists()}
try:
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'database integrity')
if path == state/'journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0
and db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'all agent work/faults closed')
else:require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0, 'all effects closed')
if path == root/'consumption.sqlite':
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no Root fault')
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
serial = [[dict(sha256=digest(v),size=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
result.update(root_consumed=len(rows),root_rows_sha256=digest(json.dumps(serial,separators=(',',':')).encode()))
finally:
for p in sidecars:
if p not in existing and p.exists():
fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
try:
side = os.fstat(fd);require(stat.S_ISREG(side.st_mode) and side.st_nlink == 1 and side.st_uid in (0,s.st_uid), 'only own new fixed SQLite sidecar')
if side.st_uid == 0 and s.st_uid != 0:os.fchown(fd,s.st_uid,s.st_gid)
finally:os.close(fd)
return result
before = services(); original = settled()
require(all(before[n]['ActiveState'] == 'active' and before[n]['UnitFileState'] == 'enabled' for n in
('rmm-agent-remote.service','rmm-helper-cloud.service'))
and before['rmm-apt-helper.service']['ActiveState'] == 'inactive'
and before['rmm-helper-cloud.service']['Restart'] == 'no', 'healthy enabled services with exact startup gap')
folder = Path('/var/lib/rmm-bootstrap-history')/('cloud-general-startup-retry-'+identifier)
probe_dir = unit.parent/'rmm-helper-cloud.service.d'; probe = probe_dir/'96-cloud-startup-retry-probe.conf'
require(not folder.exists() and not folder.is_symlink() and not probe.exists() and not probe.is_symlink()
and not probe_dir.is_symlink(), 'new own versioned test only')
report = dict(preflight=True,completed=False,unit_delta_only=True,old_unit_sha256=payload['old_unit_sha256'],
new_unit_sha256=payload['new_unit_sha256'],retained=original,
jobs_created=False,reboot_requested=False,package_effects=False,binaries_changed=False)
if not payload['apply']:return report
folder.mkdir(mode=0o700); probe_dir.mkdir(mode=0o755,exist_ok=True)
require(probe_dir.stat().st_uid == 0 and not probe_dir.stat().st_mode&0o022, 'root-owned helper dropin directory')
def sync_dir(path):
fd = os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try:os.fsync(fd)
finally:os.close(fd)
def create(path,raw,mode=0o600):
fd = os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as stream:stream.write(raw);stream.flush();os.fchmod(stream.fileno(),mode);os.fsync(stream.fileno())
sync_dir(path.parent)
def save():
path = folder/'.report-next.json';create(path,json.dumps(report).encode());os.replace(path,folder/'report.json');sync_dir(folder)
report.update(preflight=False,stage='validated_before_service_stop',backup=str(folder),rollback_armed=False);save()
try:
missing = folder/'intentionally-absent-helper-config.json'
require(not missing.exists(), 'fixed nonexistent configuration fault')
fault = ('[Service]\nEnvironment=RMM_HELPER_CONFIG='+str(missing)+'\n').encode()
# Independent cleanup only removes our exact temporary config override.
# The permanent on-failure policy then performs the recovery itself.
rollback = ('from pathlib import Path\nimport os,subprocess\np=Path('+repr(str(probe))+')\n'
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(fault)+':\n'
' p.unlink()\n fd=os.open(p.parent,os.O_RDONLY|os.O_DIRECTORY);os.fsync(fd);os.close(fd)\n'
' subprocess.run(["systemctl","daemon-reload"],check=True)\n')
create(folder/'rollback.py',rollback.encode())
run(['systemd-run','--quiet','--unit=rmm-general-startup-rollback-'+identifier,'--on-active=60s','--property=Type=oneshot','/usr/bin/python3',str(folder/'rollback.py')])
report.update(stage='rollback_armed',rollback_armed=True);save()
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service'])
report['stage']='services_stopped';save()
lock = os.open(root/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
try:
require(settled() == original, 'no boot/inode/effect change before unit transition')
preserved = [state/'binding.json',state/'job-signing-key.json',config,pin,
Path('/etc/rmm-remote/helper-signing-key.json'),Path('/etc/rmm/apt-helper.json'),
Path('/etc/rmm-remote/apt-helper-proof.json'),Path('/var/lib/rmm-apt-transaction/install-metadata.json'),*files]
def retained_files():
result = {}
for p in preserved:
require(not p.is_symlink(), 'preserved pin/artifact symlink')
s = p.stat();result[str(p)] = [s.st_dev,s.st_ino,digest(p.read_bytes())]
return result
pins = retained_files(); root_db_sha = digest((root/'consumption.sqlite').read_bytes())
create(folder/'unit-next',new_unit,0o644)
require(unit.stat().st_dev == folder.stat().st_dev, 'same-filesystem unit versioning')
os.rename(unit,folder/'unit-before.service');os.rename(folder/'unit-next',unit)
sync_dir(folder);sync_dir(unit.parent)
report['stage']='unit_versioned';save()
run(['systemd-analyze','verify',str(unit)])
create(probe,fault,0o644);run(['systemctl','daemon-reload'])
report.update(preflight=False,stage='fault_armed',backup=str(folder));save()
finally:fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
run(['systemctl','start','rmm-helper-cloud.service'],check=False)
deadline = time.monotonic()+24
while True:
failed = services()['rmm-helper-cloud.service']
if int(failed['NRestarts']) >= 1 and failed['Result'] == 'exit-code' and int(failed['ExecMainStatus']) != 0:break
require(time.monotonic() < deadline, 'actual automatic retry not observed');time.sleep(0.5)
require(not missing.exists() and probe.read_bytes() == fault and retained_files() == pins
and digest((root/'consumption.sqlite').read_bytes()) == root_db_sha
and settled() == original, 'failure before ledger open must preserve all original state')
report.update(stage='automatic_failure_verified',failed=failed,failed_before_ledger_open=True);save()
probe.unlink();sync_dir(probe_dir);run(['systemctl','daemon-reload'])
# No start/restart/reset-failed command here: actual policy must recover.
while True:
recovered = services()['rmm-helper-cloud.service']
if recovered['ActiveState'] == 'active' and recovered['SubState'] == 'running' and int(recovered['MainPID']) > 1:break
require(time.monotonic() < deadline, 'automatic helper recovery missing');time.sleep(0.5)
time.sleep(1)
stable = services()['rmm-helper-cloud.service']
require(stable['ActiveState'] == 'active' and stable['MainPID'] == recovered['MainPID']
and int(stable['NRestarts']) >= 1 and stable['Restart'] == 'on-failure' and stable['RestartUSec'] == '5s'
and settled() == original and retained_files() == pins
and digest(unit.read_bytes()) == payload['new_unit_sha256'] and not probe.exists(), 'native automatic recovery with preserved identity/pins/Root rows')
run(['systemctl','stop','rmm-general-startup-rollback-'+identifier+'.timer'],check=False)
report.update(stage='automatic_recovery_verified',recovered=stable,state_preserved=True,
temporary_fault_removed=True,automatic_recovery=True);save()
run(['systemctl','start','rmm-agent-remote.service'])
report.update(stage='qualified_runtime_started',completed=True);save()
return report
except Exception as error:
report.update(completed=False,error_phase=report.get('stage'),error_class=type(error).__name__,
frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
report['exception_receipt_persisted']=True
try:
save()
except Exception:
report['exception_receipt_persisted']=False
return report