189 lines
14 KiB
Python
189 lines
14 KiB
Python
"""Configuration-only General startup retry; no OS reboot or RMM job.
|
|
|
|
The existing 40s SSH transport can time out before this bounded maintenance
|
|
returns. Read its deterministic durable report; never repeat the operation.
|
|
"""
|
|
|
|
|
|
def qualify_startup_retry(payload):
|
|
import contextlib
|
|
import fcntl
|
|
import sqlite3
|
|
import stat
|
|
import time
|
|
import traceback
|
|
os.umask(0o077)
|
|
require(os.geteuid() == 0 and type(payload['apply']) is bool, 'explicit bootstrap preflight/apply')
|
|
identifier = str(uuid.UUID(payload['id']))
|
|
state = Path('/var/lib/rmm-agent-remote'); root = Path('/var/lib/rmm-helper')
|
|
account = pwd.getpwnam('rmm-agent')
|
|
unit = Path('/etc/systemd/system/rmm-helper-cloud.service')
|
|
config = Path('/etc/rmm-remote/helper-general.json'); pin = Path('/etc/rmm-remote/general-helper-proof.json')
|
|
def digest(data):return hashlib.sha256(data).hexdigest()
|
|
def owned(path,owner,limit):
|
|
require(not path.is_symlink(), 'owned fixed file symlink')
|
|
s = path.stat()
|
|
require(stat.S_ISREG(s.st_mode) and s.st_uid == owner and not s.st_mode&0o022 and s.st_size <= limit, 'owned bounded fixed file')
|
|
return path.read_bytes()
|
|
old_unit = owned(unit,0,16384)
|
|
require(digest(old_unit) == payload['old_unit_sha256'] and old_unit.count(b'Restart=no\n') == 1
|
|
and b'RestartSec=' not in old_unit, 'exact original General unit')
|
|
new_unit = old_unit.replace(b'Restart=no\n',b'Restart=on-failure\nRestartSec=5\n')
|
|
require(digest(new_unit) == payload['new_unit_sha256'], 'independently reviewed two-line unit delta')
|
|
files = {Path('/usr/local/libexec/rmm-privileged-helper'):payload['helper_sha256'],
|
|
Path('/usr/local/libexec/rmm-agent-remote'):payload['agent_sha256'],
|
|
Path('/usr/lib/rmm-apt-transaction/apt_transaction.py'):payload['provider_sha256']}
|
|
require(all(digest(owned(path,0,32*1024*1024)) == sha for path,sha in files.items()), 'qualified helper/agent/provider unchanged')
|
|
binding = json.loads(owned(state/'binding.json',account.pw_uid,16384))
|
|
cached = json.loads(owned(state/'job-signing-key.json',account.pw_uid,4096))
|
|
public = json.loads(owned(Path('/etc/rmm-remote/helper-signing-key.json'),0,4096))
|
|
general = json.loads(owned(config,0,16384)); policy_pin = json.loads(owned(pin,0,16384))
|
|
raw_key = bytes.fromhex(public['public_key_hex'])
|
|
require(set(public) == {'algorithm','schema_version','public_key_hex','key_id'} and cached == public
|
|
and public['algorithm'] == 'Ed25519' and type(public['schema_version']) is int and public['schema_version'] == 1
|
|
and len(raw_key) == 32 and public['key_id'] == digest(raw_key)
|
|
and binding['public_key_hex'] == public['public_key_hex']
|
|
and binding['agent_id'] == payload['agent_id'] and binding['asset_id'] == payload['asset_id']
|
|
and general['allow_reboot'] is False and general['profile'] == policy_pin['profile'] == 'general'
|
|
and general['signing_key_file'] == '/etc/rmm-remote/helper-signing-key.json'
|
|
and general['binding']['public_key_hex'] in ('',public['public_key_hex'])
|
|
and policy_pin['identity']['signing_key_id'] == public['key_id']
|
|
and all(binding[k] == general['binding'][k] == policy_pin['identity'][k] for k in
|
|
('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')), 'same non-reboot General public identity/policy')
|
|
require(b'RMM_APT_HELPER_' not in owned(Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf'),0,4096), 'APT endpoint remains closed')
|
|
def run(args,timeout=10,check=True):return subprocess.run(args,capture_output=True,check=check,timeout=timeout)
|
|
def services():
|
|
result = {}
|
|
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
|
raw = run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts','-p','Restart','-p','RestartUSec','-p','Result','-p','ExecMainStatus','-p','UnitFileState']).stdout.decode()
|
|
result[name] = dict(line.split('=',1) for line in raw.splitlines())
|
|
return result
|
|
def settled():
|
|
result = dict(boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),inodes={})
|
|
for path in (state/'journal.sqlite',state/'consumption.sqlite',root/'consumption.sqlite'):
|
|
require(not path.is_symlink(), 'fixed database symlink')
|
|
s = path.stat(); result['inodes'][str(path)] = [s.st_dev,s.st_ino]
|
|
sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')]
|
|
existing = {p for p in sidecars if p.exists()}
|
|
try:
|
|
with contextlib.closing(sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'database integrity')
|
|
if path == state/'journal.sqlite':
|
|
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0
|
|
and db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'all agent work/faults closed')
|
|
else:require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0, 'all effects closed')
|
|
if path == root/'consumption.sqlite':
|
|
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no Root fault')
|
|
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
|
|
serial = [[dict(sha256=digest(v),size=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
|
|
result.update(root_consumed=len(rows),root_rows_sha256=digest(json.dumps(serial,separators=(',',':')).encode()))
|
|
finally:
|
|
for p in sidecars:
|
|
if p not in existing and p.exists():
|
|
fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
|
try:
|
|
side = os.fstat(fd);require(stat.S_ISREG(side.st_mode) and side.st_nlink == 1 and side.st_uid in (0,s.st_uid), 'only own new fixed SQLite sidecar')
|
|
if side.st_uid == 0 and s.st_uid != 0:os.fchown(fd,s.st_uid,s.st_gid)
|
|
finally:os.close(fd)
|
|
return result
|
|
before = services(); original = settled()
|
|
require(all(before[n]['ActiveState'] == 'active' and before[n]['UnitFileState'] == 'enabled' for n in
|
|
('rmm-agent-remote.service','rmm-helper-cloud.service'))
|
|
and before['rmm-apt-helper.service']['ActiveState'] == 'inactive'
|
|
and before['rmm-helper-cloud.service']['Restart'] == 'no', 'healthy enabled services with exact startup gap')
|
|
folder = Path('/var/lib/rmm-bootstrap-history')/('cloud-general-startup-retry-'+identifier)
|
|
probe_dir = unit.parent/'rmm-helper-cloud.service.d'; probe = probe_dir/'96-cloud-startup-retry-probe.conf'
|
|
require(not folder.exists() and not folder.is_symlink() and not probe.exists() and not probe.is_symlink()
|
|
and not probe_dir.is_symlink(), 'new own versioned test only')
|
|
report = dict(preflight=True,completed=False,unit_delta_only=True,old_unit_sha256=payload['old_unit_sha256'],
|
|
new_unit_sha256=payload['new_unit_sha256'],retained=original,
|
|
jobs_created=False,reboot_requested=False,package_effects=False,binaries_changed=False)
|
|
if not payload['apply']:return report
|
|
folder.mkdir(mode=0o700); probe_dir.mkdir(mode=0o755,exist_ok=True)
|
|
require(probe_dir.stat().st_uid == 0 and not probe_dir.stat().st_mode&0o022, 'root-owned helper dropin directory')
|
|
def sync_dir(path):
|
|
fd = os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
|
try:os.fsync(fd)
|
|
finally:os.close(fd)
|
|
def create(path,raw,mode=0o600):
|
|
fd = os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
|
|
with os.fdopen(fd,'wb') as stream:stream.write(raw);stream.flush();os.fchmod(stream.fileno(),mode);os.fsync(stream.fileno())
|
|
sync_dir(path.parent)
|
|
def save():
|
|
path = folder/'.report-next.json';create(path,json.dumps(report).encode());os.replace(path,folder/'report.json');sync_dir(folder)
|
|
report.update(preflight=False,stage='validated_before_service_stop',backup=str(folder),rollback_armed=False);save()
|
|
try:
|
|
missing = folder/'intentionally-absent-helper-config.json'
|
|
require(not missing.exists(), 'fixed nonexistent configuration fault')
|
|
fault = ('[Service]\nEnvironment=RMM_HELPER_CONFIG='+str(missing)+'\n').encode()
|
|
# Independent cleanup only removes our exact temporary config override.
|
|
# The permanent on-failure policy then performs the recovery itself.
|
|
rollback = ('from pathlib import Path\nimport os,subprocess\np=Path('+repr(str(probe))+')\n'
|
|
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(fault)+':\n'
|
|
' p.unlink()\n fd=os.open(p.parent,os.O_RDONLY|os.O_DIRECTORY);os.fsync(fd);os.close(fd)\n'
|
|
' subprocess.run(["systemctl","daemon-reload"],check=True)\n')
|
|
create(folder/'rollback.py',rollback.encode())
|
|
run(['systemd-run','--quiet','--unit=rmm-general-startup-rollback-'+identifier,'--on-active=60s','--property=Type=oneshot','/usr/bin/python3',str(folder/'rollback.py')])
|
|
report.update(stage='rollback_armed',rollback_armed=True);save()
|
|
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service'])
|
|
report['stage']='services_stopped';save()
|
|
lock = os.open(root/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
try:
|
|
require(settled() == original, 'no boot/inode/effect change before unit transition')
|
|
preserved = [state/'binding.json',state/'job-signing-key.json',config,pin,
|
|
Path('/etc/rmm-remote/helper-signing-key.json'),Path('/etc/rmm/apt-helper.json'),
|
|
Path('/etc/rmm-remote/apt-helper-proof.json'),Path('/var/lib/rmm-apt-transaction/install-metadata.json'),*files]
|
|
def retained_files():
|
|
result = {}
|
|
for p in preserved:
|
|
require(not p.is_symlink(), 'preserved pin/artifact symlink')
|
|
s = p.stat();result[str(p)] = [s.st_dev,s.st_ino,digest(p.read_bytes())]
|
|
return result
|
|
pins = retained_files(); root_db_sha = digest((root/'consumption.sqlite').read_bytes())
|
|
create(folder/'unit-next',new_unit,0o644)
|
|
require(unit.stat().st_dev == folder.stat().st_dev, 'same-filesystem unit versioning')
|
|
os.rename(unit,folder/'unit-before.service');os.rename(folder/'unit-next',unit)
|
|
sync_dir(folder);sync_dir(unit.parent)
|
|
report['stage']='unit_versioned';save()
|
|
run(['systemd-analyze','verify',str(unit)])
|
|
create(probe,fault,0o644);run(['systemctl','daemon-reload'])
|
|
report.update(preflight=False,stage='fault_armed',backup=str(folder));save()
|
|
finally:fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
|
|
run(['systemctl','start','rmm-helper-cloud.service'],check=False)
|
|
deadline = time.monotonic()+24
|
|
while True:
|
|
failed = services()['rmm-helper-cloud.service']
|
|
if int(failed['NRestarts']) >= 1 and failed['Result'] == 'exit-code' and int(failed['ExecMainStatus']) != 0:break
|
|
require(time.monotonic() < deadline, 'actual automatic retry not observed');time.sleep(0.5)
|
|
require(not missing.exists() and probe.read_bytes() == fault and retained_files() == pins
|
|
and digest((root/'consumption.sqlite').read_bytes()) == root_db_sha
|
|
and settled() == original, 'failure before ledger open must preserve all original state')
|
|
report.update(stage='automatic_failure_verified',failed=failed,failed_before_ledger_open=True);save()
|
|
probe.unlink();sync_dir(probe_dir);run(['systemctl','daemon-reload'])
|
|
# No start/restart/reset-failed command here: actual policy must recover.
|
|
while True:
|
|
recovered = services()['rmm-helper-cloud.service']
|
|
if recovered['ActiveState'] == 'active' and recovered['SubState'] == 'running' and int(recovered['MainPID']) > 1:break
|
|
require(time.monotonic() < deadline, 'automatic helper recovery missing');time.sleep(0.5)
|
|
time.sleep(1)
|
|
stable = services()['rmm-helper-cloud.service']
|
|
require(stable['ActiveState'] == 'active' and stable['MainPID'] == recovered['MainPID']
|
|
and int(stable['NRestarts']) >= 1 and stable['Restart'] == 'on-failure' and stable['RestartUSec'] == '5s'
|
|
and settled() == original and retained_files() == pins
|
|
and digest(unit.read_bytes()) == payload['new_unit_sha256'] and not probe.exists(), 'native automatic recovery with preserved identity/pins/Root rows')
|
|
run(['systemctl','stop','rmm-general-startup-rollback-'+identifier+'.timer'],check=False)
|
|
report.update(stage='automatic_recovery_verified',recovered=stable,state_preserved=True,
|
|
temporary_fault_removed=True,automatic_recovery=True);save()
|
|
run(['systemctl','start','rmm-agent-remote.service'])
|
|
report.update(stage='qualified_runtime_started',completed=True);save()
|
|
return report
|
|
except Exception as error:
|
|
report.update(completed=False,error_phase=report.get('stage'),error_class=type(error).__name__,
|
|
frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
|
|
report['exception_receipt_persisted']=True
|
|
try:
|
|
save()
|
|
except Exception:
|
|
report['exception_receipt_persisted']=False
|
|
return report
|