#!/usr/bin/env python3 """Bounded stored Sync diagnostics only; no socket, repair, job or DB write.""" import hashlib import json from pathlib import Path import subprocess import uuid import cloud_mvp_acceptance as cloud require = cloud.contract.require def server(payload): tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id'])) require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant') scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"'" query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now()," "'installation',(SELECT json_build_object('connection_generation',connection_generation,'connection_boot_id',connection_boot_id," "'sync_ready',sync_ready,'helper_sync_ready',helper_sync_ready,'restore_blocked',restore_blocked," "'sequence_regressed',sequence_regressed,'sync_recovery_required',sync_recovery_required) FROM agent_installations WHERE "+scope+")," "'head',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+")," "'decisions',(SELECT COALESCE(json_agg(s),'[]'::json) FROM (SELECT request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,created_at," "encode(payload,'hex') AS payload_hex FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC,decision_id DESC LIMIT 8) s)); COMMIT;") root = Path('/opt/rust-discovery') cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), 'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'] data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout) keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint', 'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason') for row in data['decisions']: raw = bytes.fromhex(row.pop('payload_hex')); value = json.loads(raw) row['payload_sha256'] = hashlib.sha256(raw).hexdigest() row['decision'] = {k:value.get(k) for k in keys} row['attempts'] = [{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')} for a in value.get('attempts',[])] return data def vm(payload): import sqlite3 require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity') def wire(raw): fields = {}; index = 0 def number(): nonlocal index value = 0 for shift in range(0,70,7): require(index < len(raw), 'bounded stored protobuf') b = raw[index]; index += 1; value |= (b & 127) << shift if b < 128:return value raise ValueError('stored protobuf varint') while index < len(raw): tag = number(); field = tag >> 3; kind = tag & 7 if kind == 0:value = number() elif kind == 2: size = number(); require(index+size <= len(raw), 'stored protobuf length') value = raw[index:index+size]; index += size else:raise ValueError('unexpected stored protobuf wire type') fields.setdefault(field,[]).append(value) return fields def decision(raw): value = json.loads(raw) keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint', 'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason') return dict(payload_sha256=hashlib.sha256(raw).hexdigest(),**{k:value.get(k) for k in keys}, attempts=[{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')} for a in value.get('attempts',[])]) result = {} for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'), ('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')): with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db: db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN') meta = dict(db.execute('SELECT * FROM '+table+' WHERE id=1').fetchone()) result[name] = dict(meta=meta,decisions=[]) if name == 'agent': result[name]['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0] result[name]['requests'] = [] for row in db.execute('SELECT id,request,generation,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 8'): fields = wire(row['request']); item = dict(id=row['id'],generation=row['generation'],request_sha256=hashlib.sha256(row['request']).hexdigest()) for number,key in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')): item[key] = fields.get(number,[b''])[0].decode('ascii') for number,key in ((3,'checkpoint'),(5,'assignment_version'),(8,'journal_complete'),(10,'helper_checkpoint')): item[key] = fields.get(number,[0])[0] item['attempt_count'] = len(fields.get(6,[])); item['durable_message_count'] = len(fields.get(7,[])) item['response_present'] = row['response'] is not None if row['response'] is not None: response = wire(row['response']); item['decision'] = decision(response[1][0]) result[name]['requests'].append(item) else: for row in db.execute('SELECT decision_id,checkpoint,response FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 8'): response = json.loads(row['response']); raw = bytes(response['decision_bytes']) result[name]['decisions'].append(dict(id=row['decision_id'],checkpoint=row['checkpoint'],decision=decision(raw))) result[name]['latest_capture'] = dict(db.execute('SELECT checkpoint,fingerprint,captured_at FROM helper_sync_captures ORDER BY checkpoint DESC LIMIT 1').fetchone() or {}) return result def server_attempt(payload): tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id'])) job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id'])) root = Path('/opt/rust-discovery') query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),'job'," "(SELECT json_build_object('job_id',j.job_id,'attempt_id',j.attempt_id,'state',j.state,'revision',j.revision," "'start_reserved',j.start_reserved,'result_present',j.result IS NOT NULL,'result_error_code',j.result->>'error_code'," "'payload_hash',j.payload_hash,'payload_sha256_hex_input',encode(j.payload,'hex'),'pending_reason',j.pending_reason," "'outbox_acknowledged_at',b.acknowledged_at) FROM jobs j LEFT JOIN outbox b USING(tenant_id,job_id) " "WHERE j.tenant_id='"+tenant+"' AND j.agent_id='"+agent+"' AND j.job_id='"+job+"' AND j.attempt_id='"+attempt+"')," "'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"')," "'start_audits',(SELECT count(*) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')); COMMIT;") cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), 'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'] data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout) if data['job']: raw = bytes.fromhex(data['job'].pop('payload_sha256_hex_input')); value = json.loads(raw) data['job']['actual_payload_sha256'] = hashlib.sha256(raw).hexdigest() data['job']['execution_identity'] = value.get('execution_identity') data['job']['start_before'] = value.get('start_before') return data def vm_attempt(payload): import sqlite3 job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id'])) require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity') result = {} for name,path in (('journal','/var/lib/rmm-agent-remote/journal.sqlite'), ('agent_ledger','/var/lib/rmm-agent-remote/consumption.sqlite'), ('root_ledger','/var/lib/rmm-helper/consumption.sqlite')): with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db: db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN') if name == 'journal': rows = [dict(r) for r in db.execute('SELECT job,attempt,state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))] outcomes = [dict(state=json.loads(r[0])['state'],error_code=json.loads(r[0])['error_code'],sha256=hashlib.sha256(r[0]).hexdigest()) for r in db.execute('SELECT outcome FROM reconcile_outcomes WHERE job=? AND attempt=?',(job,attempt))] result[name] = dict(attempts=rows,reconciled_outcomes=outcomes) else: result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))]) return result # Retain the original stored-checkpoint reader entry points. def vm_chain(payload): import sqlite3 def wire(raw): result={};i=0 def number(): nonlocal i value=0;shift=0 while True: require(i>3;kind=key&7 if kind==0:value=number() elif kind==2: length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length else:raise RuntimeError('stored protobuf wire type') result.setdefault(field,[]).append(value) return result def request(raw): fields=wire(raw);result={} for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')): result[k]=fields.get(n,[b''])[0].decode('ascii') for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0] return result def decision(value): names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation') return {k:value.get(k) for k in names} result={} path=Path('/var/lib/rmm-agent-remote/journal.sqlite') with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity') names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')] result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone())) requests=[] for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'): item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None) if response:item['decision']=decision(json.loads(wire(response)[1][0])) requests.append(item) result['agent_requests']=requests path=Path('/var/lib/rmm-helper/consumption.sqlite') with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db: require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity') names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')] result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone())) decisions=[] for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'): signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes'])) decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value))) result['helper_decisions']=decisions return result def server_chain(payload): root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id'])) cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'), 'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'] scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'" query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+")," "'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at," "convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready," "convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));") raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip() return json.loads(raw)