Files
rust-discovery/deploy/cloud_sync_evidence.py
T
2026-11-13 21:06:27 +01:00

206 lines
14 KiB
Python

#!/usr/bin/env python3
"""Bounded stored Sync diagnostics only; no socket, repair, job or DB write."""
import hashlib
import json
from pathlib import Path
import subprocess
import uuid
import cloud_mvp_acceptance as cloud
require = cloud.contract.require
def server(payload):
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"'"
query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),"
"'installation',(SELECT json_build_object('connection_generation',connection_generation,'connection_boot_id',connection_boot_id,"
"'sync_ready',sync_ready,'helper_sync_ready',helper_sync_ready,'restore_blocked',restore_blocked,"
"'sequence_regressed',sequence_regressed,'sync_recovery_required',sync_recovery_required) FROM agent_installations WHERE "+scope+"),"
"'head',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
"'decisions',(SELECT COALESCE(json_agg(s),'[]'::json) FROM (SELECT request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,created_at,"
"encode(payload,'hex') AS payload_hex FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC,decision_id DESC LIMIT 8) s)); COMMIT;")
root = Path('/opt/rust-discovery')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint',
'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason')
for row in data['decisions']:
raw = bytes.fromhex(row.pop('payload_hex')); value = json.loads(raw)
row['payload_sha256'] = hashlib.sha256(raw).hexdigest()
row['decision'] = {k:value.get(k) for k in keys}
row['attempts'] = [{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')}
for a in value.get('attempts',[])]
return data
def vm(payload):
import sqlite3
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity')
def wire(raw):
fields = {}; index = 0
def number():
nonlocal index
value = 0
for shift in range(0,70,7):
require(index < len(raw), 'bounded stored protobuf')
b = raw[index]; index += 1; value |= (b & 127) << shift
if b < 128:return value
raise ValueError('stored protobuf varint')
while index < len(raw):
tag = number(); field = tag >> 3; kind = tag & 7
if kind == 0:value = number()
elif kind == 2:
size = number(); require(index+size <= len(raw), 'stored protobuf length')
value = raw[index:index+size]; index += size
else:raise ValueError('unexpected stored protobuf wire type')
fields.setdefault(field,[]).append(value)
return fields
def decision(raw):
value = json.loads(raw)
keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint',
'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason')
return dict(payload_sha256=hashlib.sha256(raw).hexdigest(),**{k:value.get(k) for k in keys},
attempts=[{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')}
for a in value.get('attempts',[])])
result = {}
for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'),
('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')):
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
meta = dict(db.execute('SELECT * FROM '+table+' WHERE id=1').fetchone())
result[name] = dict(meta=meta,decisions=[])
if name == 'agent':
result[name]['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
result[name]['requests'] = []
for row in db.execute('SELECT id,request,generation,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 8'):
fields = wire(row['request']); item = dict(id=row['id'],generation=row['generation'],request_sha256=hashlib.sha256(row['request']).hexdigest())
for number,key in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
item[key] = fields.get(number,[b''])[0].decode('ascii')
for number,key in ((3,'checkpoint'),(5,'assignment_version'),(8,'journal_complete'),(10,'helper_checkpoint')):
item[key] = fields.get(number,[0])[0]
item['attempt_count'] = len(fields.get(6,[])); item['durable_message_count'] = len(fields.get(7,[]))
item['response_present'] = row['response'] is not None
if row['response'] is not None:
response = wire(row['response']); item['decision'] = decision(response[1][0])
result[name]['requests'].append(item)
else:
for row in db.execute('SELECT decision_id,checkpoint,response FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 8'):
response = json.loads(row['response']); raw = bytes(response['decision_bytes'])
result[name]['decisions'].append(dict(id=row['decision_id'],checkpoint=row['checkpoint'],decision=decision(raw)))
result[name]['latest_capture'] = dict(db.execute('SELECT checkpoint,fingerprint,captured_at FROM helper_sync_captures ORDER BY checkpoint DESC LIMIT 1').fetchone() or {})
return result
def server_attempt(payload):
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
root = Path('/opt/rust-discovery')
query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),'job',"
"(SELECT json_build_object('job_id',j.job_id,'attempt_id',j.attempt_id,'state',j.state,'revision',j.revision,"
"'start_reserved',j.start_reserved,'result_present',j.result IS NOT NULL,'result_error_code',j.result->>'error_code',"
"'payload_hash',j.payload_hash,'payload_sha256_hex_input',encode(j.payload,'hex'),'pending_reason',j.pending_reason,"
"'outbox_acknowledged_at',b.acknowledged_at) FROM jobs j LEFT JOIN outbox b USING(tenant_id,job_id) "
"WHERE j.tenant_id='"+tenant+"' AND j.agent_id='"+agent+"' AND j.job_id='"+job+"' AND j.attempt_id='"+attempt+"'),"
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'start_audits',(SELECT count(*) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')); COMMIT;")
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
if data['job']:
raw = bytes.fromhex(data['job'].pop('payload_sha256_hex_input')); value = json.loads(raw)
data['job']['actual_payload_sha256'] = hashlib.sha256(raw).hexdigest()
data['job']['execution_identity'] = value.get('execution_identity')
data['job']['start_before'] = value.get('start_before')
return data
def vm_attempt(payload):
import sqlite3
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity')
result = {}
for name,path in (('journal','/var/lib/rmm-agent-remote/journal.sqlite'),
('agent_ledger','/var/lib/rmm-agent-remote/consumption.sqlite'),
('root_ledger','/var/lib/rmm-helper/consumption.sqlite')):
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
if name == 'journal':
rows = [dict(r) for r in db.execute('SELECT job,attempt,state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))]
outcomes = [dict(state=json.loads(r[0])['state'],error_code=json.loads(r[0])['error_code'],sha256=hashlib.sha256(r[0]).hexdigest())
for r in db.execute('SELECT outcome FROM reconcile_outcomes WHERE job=? AND attempt=?',(job,attempt))]
result[name] = dict(attempts=rows,reconciled_outcomes=outcomes)
else:
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
return result
# Retain the original stored-checkpoint reader entry points.
def vm_chain(payload):
import sqlite3
def wire(raw):
result={};i=0
def number():
nonlocal i
value=0;shift=0
while True:
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
if not b&128:return value
shift+=7
while i<len(raw):
key=number();field=key>>3;kind=key&7
if kind==0:value=number()
elif kind==2:
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
else:raise RuntimeError('stored protobuf wire type')
result.setdefault(field,[]).append(value)
return result
def request(raw):
fields=wire(raw);result={}
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
result[k]=fields.get(n,[b''])[0].decode('ascii')
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
return result
def decision(value):
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
return {k:value.get(k) for k in names}
result={}
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
requests=[]
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
requests.append(item)
result['agent_requests']=requests
path=Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
decisions=[]
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
result['helper_decisions']=decisions
return result
def server_chain(payload):
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
return json.loads(raw)