fix: retain operational offline preflight evidence and close readonly SQLite handles
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 20:24:01 +01:00
parent 75e4412b12
commit 1074767255
3 changed files with 146 additions and 5 deletions
+39 -5
View File
@@ -66,6 +66,7 @@ def server_evidence(payload):
def vm_network(payload): def vm_network(payload):
"""One operational transport fault, plus reads of existing durable evidence.""" """One operational transport fault, plus reads of existing durable evidence."""
import contextlib
import datetime as dt import datetime as dt
import errno import errno
import fcntl import fcntl
@@ -85,9 +86,40 @@ def vm_network(payload):
def properties(): def properties():
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode() raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
return dict(line.split('=',1) for line in raw.splitlines()) return dict(line.split('=',1) for line in raw.splitlines())
@contextlib.contextmanager
def agent_read(path):
require(path in (state/'journal.sqlite',state/'consumption.sqlite') and not path.is_symlink(), 'fixed agent database')
owner = path.stat()
require(stat.S_ISREG(owner.st_mode) and owner.st_uid == account.pw_uid
and owner.st_gid == account.pw_gid, 'agent database ownership')
sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')]
existing = set()
for p in sidecars:
require(not p.is_symlink(), 'SQLite sidecar symlink')
if p.exists():
require(p.stat().st_uid == owner.st_uid, 'existing SQLite sidecar owner')
existing.add(p)
db = None
try:
db = sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
yield db
finally:
if db is not None: db.close()
# sqlite3's transaction context alone does NOT close a connection.
# A root readonly opener can create these two fixed WAL files;
# correct only such newly created files, even on a query failure.
for p in sidecars:
if p not in existing and p.exists():
fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
try:
s = os.fstat(fd)
require(stat.S_ISREG(s.st_mode) and s.st_nlink == 1
and s.st_uid in (0,owner.st_uid), 'new fixed SQLite sidecar')
if s.st_uid == 0: os.fchown(fd,owner.st_uid,owner.st_gid)
finally: os.close(fd)
def snapshot(): def snapshot():
result = dict(agent_service=properties()) result = dict(agent_service=properties())
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db: with agent_read(state/'journal.sqlite') as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity') require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0] result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0] result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
@@ -98,9 +130,9 @@ def vm_network(payload):
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0] result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
else: else:
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db: with agent_read(state/'consumption.sqlite') as db:
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall() result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
with sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db: with contextlib.closing(sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity') require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0] result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall() rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
@@ -182,7 +214,7 @@ def vm_network(payload):
# attempts TCP only. No agent credential, TLS override or appdata is used. # attempts TCP only. No agent credential, TLS override or appdata is used.
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n' probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n' 'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
'os.setgroups([]);os.setgid(993);os.setuid(993)\n' 'os.setgroups([]);os.setgid('+str(account.pw_gid)+');os.setuid('+str(account.pw_uid)+')\n'
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n' 'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
'except OSError as e: result={"connected":False,"errno":e.errno}\n' 'except OSError as e: result={"connected":False,"errno":e.errno}\n'
'print(json.dumps(result))\n') 'print(json.dumps(result))\n')
@@ -304,5 +336,7 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
try: main() try: main()
except Exception as error: except Exception as error:
print('CLOUD_OFFLINE_INCOMPLETE:'+ (str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__)) message = str(error)
print('CLOUD_OFFLINE_INCOMPLETE:'+ (message if isinstance(error,cloud.contract.GuardError)
or message.startswith('remote bootstrap incomplete: ') else type(error).__name__))
raise SystemExit(2) from None raise SystemExit(2) from None
+81
View File
@@ -0,0 +1,81 @@
"""One bounded operational transport probe; no RMM job or application data."""
def network_status(payload):
import grp
import stat
account = pwd.getpwnam('rmm-agent')
require(os.geteuid() == 0 and account.pw_uid == 993 and account.pw_gid > 0
and account.pw_gid == grp.getgrnam('rmm-agent').gr_gid, 'named unprivileged Cloud agent')
state = Path('/var/lib/rmm-agent-remote')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent Cloud identity')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['binary_sha256'], 'qualified agent artifact')
identifier = str(uuid.UUID(payload['id']))
phase = payload.get('phase','preflight'); require(phase in ('preflight','window'), 'fixed network phase')
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/96-cloud-offline-acceptance.conf')
raw = None
if dropin.exists() and not dropin.is_symlink():
s = dropin.stat()
if stat.S_ISREG(s.st_mode) and s.st_uid == 0 and s.st_size <= 4096: raw = dropin.read_bytes()
own_rule = raw == ('# Cloud MVP offline '+identifier+' '+phase+'\n[Service]\nIPAddressDeny=88.99.80.202/32\n').encode()
properties = ('ActiveState','SubState','MainPID','ControlGroup','IPAddressDeny','IPAddressAllow',
'Result','ExecMainCode','ExecMainStatus','NRestarts')
command = ['systemctl','show','rmm-agent-remote.service']
for name in properties: command.extend(('-p',name))
response = subprocess.run(command,capture_output=True,check=True,timeout=10).stdout.decode()
service = dict(line.split('=',1) for line in response.splitlines())
group = service['ControlGroup']
files = {}
for name in ('journal.sqlite','consumption.sqlite'):
for suffix in ('','-wal','-shm'):
path = state/(name+suffix)
require(not path.is_symlink(), 'fixed SQLite metadata symlink')
if not path.exists(): files[path.name] = dict(exists=False); continue
s = path.stat(); require(stat.S_ISREG(s.st_mode), 'fixed SQLite regular file')
files[path.name] = dict(exists=True,uid=s.st_uid,gid=s.st_gid,mode=oct(stat.S_IMODE(s.st_mode)),
inode=s.st_ino,size=s.st_size)
summary = dict(service=service,dropin_exists=dropin.exists(),own_rule=own_rule,
dropin_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None,sqlite_files=files,
agent_uid=account.pw_uid,agent_gid=account.pw_gid,probe_cgroup=group,
server_ip='88.99.80.202',port=443,application_data_sent=False,
job_created=False,network_rule_changed=False)
if service['ActiveState'] != 'active' or int(service['MainPID']) <= 1:
return dict(summary,probe=dict(not_run='agent_inactive'))
if not own_rule: return dict(summary,probe=dict(not_run='own_network_rule_absent_or_different'))
require((group == '/system.slice/rmm-agent-remote.service' or group.startswith('/system.slice/rmm-agent-remote.service/'))
and '..' not in group, 'actual fixed agent cgroup')
cgroup = Path('/sys/fs/cgroup'+group)/'cgroup.procs'
require(cgroup.is_file(), 'unified actual cgroup')
# This child moves only itself and sends no application bytes. Numeric TCP
# is a transport probe, never a TLS-verification exception for the agent.
probe = ('import os,socket,json\nfrom pathlib import Path\nstage="cgroup_move"\n'
'try:\n Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
' stage="drop_groups";os.setgroups([])\n'
' stage="drop_gid";os.setgid('+str(account.pw_gid)+')\n'
' stage="drop_uid";os.setuid('+str(account.pw_uid)+')\n'
' stage="connect";s=socket.create_connection(("88.99.80.202",443),2);s.close()\n'
' r={"stage":stage,"connected":True,"timeout":False,"errno":None,"error_class":None}\n'
'except Exception as e:\n'
' name=type(e).__name__;number=getattr(e,"errno",None)\n'
' r={"stage":stage,"connected":False,"timeout":isinstance(e,TimeoutError),'
'"errno":number if type(number) is int else None,'
'"error_class":name if name in ("OSError","PermissionError","ProcessLookupError","FileNotFoundError","TimeoutError","ValueError","RuntimeError") else "OtherError"}\n'
'print(json.dumps(r))\n')
result = dict(stage='probe_process',connected=False,timeout=False,errno=None,error_class='InvalidProbeOutput',exit_code=None)
try:
answer = subprocess.run(['/usr/bin/python3','-I','-c',probe],capture_output=True,check=False,timeout=4)
result['exit_code'] = answer.returncode
if len(answer.stdout) <= 1024:
try: parsed = json.loads(answer.stdout)
except (ValueError,UnicodeError): parsed = None
if (isinstance(parsed,dict) and set(parsed) == {'stage','connected','timeout','errno','error_class'}
and parsed['stage'] in ('cgroup_move','drop_groups','drop_gid','drop_uid','connect')
and type(parsed['connected']) is bool and type(parsed['timeout']) is bool
and (parsed['errno'] is None or type(parsed['errno']) is int)
and parsed['error_class'] in (None,'OSError','PermissionError','ProcessLookupError','FileNotFoundError','TimeoutError','ValueError','RuntimeError','OtherError')):
result.update(parsed)
except subprocess.TimeoutExpired:
result.update(timeout=True,error_class='TimeoutExpired')
except OSError as error:
result.update(errno=error.errno,error_class='OSError')
return dict(summary,probe=result)
+26
View File
@@ -1,5 +1,31 @@
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug # Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
## Erster tatsächlicher Preflight, noch kein Job
Fall`d9663783-9c8c-441a-808e-768bd72bd69a` brach bereits vor Planung/Antrag ab.
Die private Phase000 enthält nur den gesunden Ausgangszustand; kein Job wurde
zur Offlineabnahme angelegt. Tatsächliche operative Metadaten zeigten später
Agent-Autorestarts unter der Netzregel, unveränderte Agent-DB-Eigentümer993:984
und keine WAL/SHM-Dateien. Ein SQLite-Eigentümerproblem ist damit **nicht als
Ursache nachgewiesen**. Der normale Startpfad mit aktivem einmaligem
Online-Signierschlüsselbootstrap wird vor Fortsetzung geprüft/finalisiert.
Eigener420s-Rollback entfernte die Testregel; anschließend tatsächlicher
WSS-Kanal Generation86, signierter Sync bereit, `restore_blocked=false`,
Heartbeat19:22:22UTC. Zentrale weiterhin24Jobs/24Outbox/22Grants/0offen.
Private tatsächliche Belege im oben genannten Fallordner:
`preflight-complete-diagnostic.json`, `preflight-exit-metadata.json` und
`restored-central-state.json`. Fehlgeschlagene Probe ist kein bestandener
aktiver Offlinejob; die drei Abnahmefälle bleiben offen.
Der Probeadapter wurde zusätzlich so korrigiert, dass auch inaktive Dienste
und fehlgeschlagene Unterprozesse ausschließlich begrenzte Zustands-/Owner-/
Phasen-/Errnometadaten zurückliefern. Keine Rohstderr-/Credentialausgabe.
Feste numerische Probe-GID wird durch die geprüfte tatsächliche Agentgruppe
ersetzt; eigene neue SQLite-Sidecars werden defensiv korrekt zugeordnet und
Verbindungen ausdrücklich geschlossen. Diese Sourcekorrekturen sind noch
kein erfolgreicher vollständiger Offline-Nachweis.
Stand: Adapter implementiert und lokal auf Python-/Node-Syntax sowie identische Stand: Adapter implementiert und lokal auf Python-/Node-Syntax sowie identische
Skriptbytes zwischen Orchestrator und Browser geprüft. **Noch kein nativer Skriptbytes zwischen Orchestrator und Browser geprüft. **Noch kein nativer
VM-Nachweis aus diesem Adapter.** Die folgenden Befehle sind geplant, nicht VM-Nachweis aus diesem Adapter.** Die folgenden Befehle sind geplant, nicht