fix: retain operational offline preflight evidence and close readonly SQLite handles
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -66,6 +66,7 @@ def server_evidence(payload):
|
||||
|
||||
def vm_network(payload):
|
||||
"""One operational transport fault, plus reads of existing durable evidence."""
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import errno
|
||||
import fcntl
|
||||
@@ -85,9 +86,40 @@ def vm_network(payload):
|
||||
def properties():
|
||||
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
return dict(line.split('=',1) for line in raw.splitlines())
|
||||
@contextlib.contextmanager
|
||||
def agent_read(path):
|
||||
require(path in (state/'journal.sqlite',state/'consumption.sqlite') and not path.is_symlink(), 'fixed agent database')
|
||||
owner = path.stat()
|
||||
require(stat.S_ISREG(owner.st_mode) and owner.st_uid == account.pw_uid
|
||||
and owner.st_gid == account.pw_gid, 'agent database ownership')
|
||||
sidecars = [Path(str(path)+suffix) for suffix in ('-wal','-shm')]
|
||||
existing = set()
|
||||
for p in sidecars:
|
||||
require(not p.is_symlink(), 'SQLite sidecar symlink')
|
||||
if p.exists():
|
||||
require(p.stat().st_uid == owner.st_uid, 'existing SQLite sidecar owner')
|
||||
existing.add(p)
|
||||
db = None
|
||||
try:
|
||||
db = sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
|
||||
yield db
|
||||
finally:
|
||||
if db is not None: db.close()
|
||||
# sqlite3's transaction context alone does NOT close a connection.
|
||||
# A root readonly opener can create these two fixed WAL files;
|
||||
# correct only such newly created files, even on a query failure.
|
||||
for p in sidecars:
|
||||
if p not in existing and p.exists():
|
||||
fd = os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
try:
|
||||
s = os.fstat(fd)
|
||||
require(stat.S_ISREG(s.st_mode) and s.st_nlink == 1
|
||||
and s.st_uid in (0,owner.st_uid), 'new fixed SQLite sidecar')
|
||||
if s.st_uid == 0: os.fchown(fd,owner.st_uid,owner.st_gid)
|
||||
finally: os.close(fd)
|
||||
def snapshot():
|
||||
result = dict(agent_service=properties())
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
with agent_read(state/'journal.sqlite') as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
|
||||
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
|
||||
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
|
||||
@@ -98,9 +130,9 @@ def vm_network(payload):
|
||||
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
|
||||
else:
|
||||
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
||||
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
with agent_read(state/'consumption.sqlite') as db:
|
||||
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
|
||||
with sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
with contextlib.closing(sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True)) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
|
||||
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
|
||||
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
|
||||
@@ -182,7 +214,7 @@ def vm_network(payload):
|
||||
# attempts TCP only. No agent credential, TLS override or appdata is used.
|
||||
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
|
||||
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
|
||||
'os.setgroups([]);os.setgid(993);os.setuid(993)\n'
|
||||
'os.setgroups([]);os.setgid('+str(account.pw_gid)+');os.setuid('+str(account.pw_uid)+')\n'
|
||||
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
|
||||
'except OSError as e: result={"connected":False,"errno":e.errno}\n'
|
||||
'print(json.dumps(result))\n')
|
||||
@@ -304,5 +336,7 @@ def main():
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except Exception as error:
|
||||
print('CLOUD_OFFLINE_INCOMPLETE:'+ (str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
message = str(error)
|
||||
print('CLOUD_OFFLINE_INCOMPLETE:'+ (message if isinstance(error,cloud.contract.GuardError)
|
||||
or message.startswith('remote bootstrap incomplete: ') else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
|
||||
Reference in New Issue
Block a user