fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:11:45 +01:00
parent da7eb8109c
commit c512bd6fb2
36 changed files with 2556 additions and 38 deletions
+40
View File
@@ -3,6 +3,27 @@
def service_processes(payload):
result={}
# Same read-only kernel clock-readiness query required by the runtime;
# modes stays zero. No time correction or trust override exists here.
import ctypes
class TimexPrefix(ctypes.Structure):
_fields_=[('modes',ctypes.c_uint),('offset',ctypes.c_long),('freq',ctypes.c_long),
('maxerror',ctypes.c_long),('esterror',ctypes.c_long),('status',ctypes.c_int),('remaining',ctypes.c_byte*256)]
require(ctypes.sizeof(ctypes.c_long)==8 and TimexPrefix.maxerror.offset==24 and TimexPrefix.status.offset==40,'reviewed Linux x64 timex layout')
clock=TimexPrefix();libc=ctypes.CDLL(None,use_errno=True);query=libc.adjtimex
query.argtypes=[ctypes.POINTER(TimexPrefix)];query.restype=ctypes.c_int
status=query(ctypes.byref(clock))
result['kernel_clock']=dict(status=status,unsynchronized=bool(clock.status&0x40),maxerror_us=clock.maxerror,
trusted=status>=0 and status!=5 and not clock.status&0x40 and 0<=clock.maxerror<=5_000_000,modes=clock.modes)
storage=os.statvfs('/var/lib/rmm-helper')
result['bootstrap_storage']=dict(available_bytes=storage.f_bavail*storage.f_frsize,available_inodes=storage.f_favail,
helper_database_bytes=Path('/var/lib/rmm-helper/consumption.sqlite').stat().st_size,
agent_database_bytes=Path('/var/lib/rmm-agent-remote/journal.sqlite').stat().st_size)
lock=Path('/var/lib/rmm-helper/executor.lock').stat()
result['executor_lock_pids']=[]
for line in Path('/proc/locks').read_text().splitlines():
parts=line.split()
if len(parts)>5 and parts[5].split(':')[-1]==str(lock.st_ino):result['executor_lock_pids'].append(parts[4])
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
values=subprocess.run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts'],
capture_output=True,check=True,timeout=10).stdout.decode()
@@ -23,3 +44,22 @@ def service_processes(payload):
if children.exists():pending.extend(int(s) for s in children.read_text().split())
result[name]=dict(service=fields,processes=processes)
return result
def stored_bootstrap_error_codes(payload):
"""Only allowlisted stored daemon errors during this runtime deployment."""
known=('CLOCK_UNTRUSTED','HELPER_VALIDATION_OR_IO_FAILURE','HELPER_PIN_PROFILE_MISMATCH',
'SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE','HELPER_CHECKPOINTS_DID_NOT_CONVERGE',
'HELPER_MANUAL_RECOVERY_REQUIRED','LOCAL_CHECKPOINT_ROLLBACK','SYNC_MANUAL_RECOVERY_REQUIRED')
result={}
for unit in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
process=subprocess.run(['/usr/bin/journalctl','--no-pager','--quiet','--output=json','--unit',unit,
'--since','2026-11-13 17:34:00 UTC','--lines','100'],capture_output=True,check=True,timeout=10)
require(len(process.stdout)<=256*1024,'bounded deployment error metadata')
counts={k:0 for k in known};times=[]
for line in process.stdout.splitlines():
row=json.loads(line);message=row.get('MESSAGE','')
for code in known:
if code in message:counts[code]+=1;times.append(row.get('__REALTIME_TIMESTAMP'))
result[unit]=dict(counts={k:v for k,v in counts.items() if v},first=min(times) if times else None,last=max(times) if times else None)
return result