217 lines
15 KiB
Python
217 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded stored Sync diagnostics only; no socket, repair, job or DB write."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import uuid
|
|
|
|
import cloud_mvp_acceptance as cloud
|
|
|
|
require = cloud.contract.require
|
|
|
|
|
|
def reboot_receipts(payload):
|
|
tenant=str(uuid.UUID(payload['tenant_id']));agent=str(uuid.UUID(payload['agent_id']));job=str(uuid.UUID(payload['job_id']))
|
|
require(tenant=='11111111-1111-4111-8111-111111111111','fixed development tenant')
|
|
query=("BEGIN READ ONLY; SELECT json_build_object('observed_at',now(),'installation',(SELECT json_build_object('boot_id',connection_boot_id,'generation',connection_generation,'last_contact_at',last_contact_at,'sync_ready',sync_ready,'helper_ready',helper_sync_ready) FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'),"
|
|
"'job',(SELECT json_build_object('state',state,'revision',revision,'result',result) FROM jobs WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
|
"'grant',(SELECT json_build_object('grant_id',grant_id,'created_at',created_at) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
|
"'inventories',(SELECT COALESCE(json_agg(x),'[]'::json) FROM (SELECT message_id,boot_id,observed_at,received_at,source_id FROM observations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' ORDER BY received_at DESC LIMIT 20) x)); COMMIT;")
|
|
root=Path('/opt/rust-discovery');cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
|
|
|
|
|
def server(payload):
|
|
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
|
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
|
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"'"
|
|
query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),"
|
|
"'installation',(SELECT json_build_object('connection_generation',connection_generation,'connection_boot_id',connection_boot_id,"
|
|
"'sync_ready',sync_ready,'helper_sync_ready',helper_sync_ready,'restore_blocked',restore_blocked,"
|
|
"'sequence_regressed',sequence_regressed,'sync_recovery_required',sync_recovery_required) FROM agent_installations WHERE "+scope+"),"
|
|
"'head',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
|
|
"'decisions',(SELECT COALESCE(json_agg(s),'[]'::json) FROM (SELECT request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,created_at,"
|
|
"encode(payload,'hex') AS payload_hex FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC,decision_id DESC LIMIT 8) s)); COMMIT;")
|
|
root = Path('/opt/rust-discovery')
|
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
|
keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint',
|
|
'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason')
|
|
for row in data['decisions']:
|
|
raw = bytes.fromhex(row.pop('payload_hex')); value = json.loads(raw)
|
|
row['payload_sha256'] = hashlib.sha256(raw).hexdigest()
|
|
row['decision'] = {k:value.get(k) for k in keys}
|
|
row['attempts'] = [{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')}
|
|
for a in value.get('attempts',[])]
|
|
return data
|
|
|
|
|
|
def vm(payload):
|
|
import sqlite3
|
|
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity')
|
|
def wire(raw):
|
|
fields = {}; index = 0
|
|
def number():
|
|
nonlocal index
|
|
value = 0
|
|
for shift in range(0,70,7):
|
|
require(index < len(raw), 'bounded stored protobuf')
|
|
b = raw[index]; index += 1; value |= (b & 127) << shift
|
|
if b < 128:return value
|
|
raise ValueError('stored protobuf varint')
|
|
while index < len(raw):
|
|
tag = number(); field = tag >> 3; kind = tag & 7
|
|
if kind == 0:value = number()
|
|
elif kind == 2:
|
|
size = number(); require(index+size <= len(raw), 'stored protobuf length')
|
|
value = raw[index:index+size]; index += size
|
|
else:raise ValueError('unexpected stored protobuf wire type')
|
|
fields.setdefault(field,[]).append(value)
|
|
return fields
|
|
def decision(raw):
|
|
value = json.loads(raw)
|
|
keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint',
|
|
'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason')
|
|
return dict(payload_sha256=hashlib.sha256(raw).hexdigest(),**{k:value.get(k) for k in keys},
|
|
attempts=[{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')}
|
|
for a in value.get('attempts',[])])
|
|
result = {}
|
|
for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'),
|
|
('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')):
|
|
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
|
|
db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
|
|
meta = dict(db.execute('SELECT * FROM '+table+' WHERE id=1').fetchone())
|
|
result[name] = dict(meta=meta,decisions=[])
|
|
if name == 'agent':
|
|
result[name]['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
|
|
result[name]['requests'] = []
|
|
for row in db.execute('SELECT id,request,generation,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 8'):
|
|
fields = wire(row['request']); item = dict(id=row['id'],generation=row['generation'],request_sha256=hashlib.sha256(row['request']).hexdigest())
|
|
for number,key in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
|
|
item[key] = fields.get(number,[b''])[0].decode('ascii')
|
|
for number,key in ((3,'checkpoint'),(5,'assignment_version'),(8,'journal_complete'),(10,'helper_checkpoint')):
|
|
item[key] = fields.get(number,[0])[0]
|
|
item['attempt_count'] = len(fields.get(6,[])); item['durable_message_count'] = len(fields.get(7,[]))
|
|
item['response_present'] = row['response'] is not None
|
|
if row['response'] is not None:
|
|
response = wire(row['response']); item['decision'] = decision(response[1][0])
|
|
result[name]['requests'].append(item)
|
|
else:
|
|
for row in db.execute('SELECT decision_id,checkpoint,response FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 8'):
|
|
response = json.loads(row['response']); raw = bytes(response['decision_bytes'])
|
|
result[name]['decisions'].append(dict(id=row['decision_id'],checkpoint=row['checkpoint'],decision=decision(raw)))
|
|
result[name]['latest_capture'] = dict(db.execute('SELECT checkpoint,fingerprint,captured_at FROM helper_sync_captures ORDER BY checkpoint DESC LIMIT 1').fetchone() or {})
|
|
return result
|
|
|
|
|
|
def server_attempt(payload):
|
|
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
|
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
|
root = Path('/opt/rust-discovery')
|
|
query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),'job',"
|
|
"(SELECT json_build_object('job_id',j.job_id,'attempt_id',j.attempt_id,'state',j.state,'revision',j.revision,"
|
|
"'start_reserved',j.start_reserved,'result_present',j.result IS NOT NULL,'result_error_code',j.result->>'error_code',"
|
|
"'payload_hash',j.payload_hash,'payload_sha256_hex_input',encode(j.payload,'hex'),'pending_reason',j.pending_reason,"
|
|
"'outbox_acknowledged_at',b.acknowledged_at) FROM jobs j LEFT JOIN outbox b USING(tenant_id,job_id) "
|
|
"WHERE j.tenant_id='"+tenant+"' AND j.agent_id='"+agent+"' AND j.job_id='"+job+"' AND j.attempt_id='"+attempt+"'),"
|
|
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
|
"'start_audits',(SELECT count(*) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')); COMMIT;")
|
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
|
if data['job']:
|
|
raw = bytes.fromhex(data['job'].pop('payload_sha256_hex_input')); value = json.loads(raw)
|
|
data['job']['actual_payload_sha256'] = hashlib.sha256(raw).hexdigest()
|
|
data['job']['execution_identity'] = value.get('execution_identity')
|
|
data['job']['start_before'] = value.get('start_before')
|
|
return data
|
|
|
|
|
|
def vm_attempt(payload):
|
|
import sqlite3
|
|
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
|
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity')
|
|
result = {}
|
|
for name,path in (('journal','/var/lib/rmm-agent-remote/journal.sqlite'),
|
|
('agent_ledger','/var/lib/rmm-agent-remote/consumption.sqlite'),
|
|
('root_ledger','/var/lib/rmm-helper/consumption.sqlite')):
|
|
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
|
|
db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
|
|
if name == 'journal':
|
|
rows = [dict(r) for r in db.execute('SELECT job,attempt,state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))]
|
|
outcomes = [dict(state=json.loads(r[0])['state'],error_code=json.loads(r[0])['error_code'],sha256=hashlib.sha256(r[0]).hexdigest())
|
|
for r in db.execute('SELECT outcome FROM reconcile_outcomes WHERE job=? AND attempt=?',(job,attempt))]
|
|
result[name] = dict(attempts=rows,reconciled_outcomes=outcomes)
|
|
else:
|
|
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
|
|
return result
|
|
|
|
|
|
# Retain the original stored-checkpoint reader entry points.
|
|
def vm_chain(payload):
|
|
import sqlite3
|
|
def wire(raw):
|
|
result={};i=0
|
|
def number():
|
|
nonlocal i
|
|
value=0;shift=0
|
|
while True:
|
|
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
|
|
if not b&128:return value
|
|
shift+=7
|
|
while i<len(raw):
|
|
key=number();field=key>>3;kind=key&7
|
|
if kind==0:value=number()
|
|
elif kind==2:
|
|
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
|
|
else:raise RuntimeError('stored protobuf wire type')
|
|
result.setdefault(field,[]).append(value)
|
|
return result
|
|
def request(raw):
|
|
fields=wire(raw);result={}
|
|
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
|
|
result[k]=fields.get(n,[b''])[0].decode('ascii')
|
|
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
|
|
return result
|
|
def decision(value):
|
|
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
|
|
return {k:value.get(k) for k in names}
|
|
result={}
|
|
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
|
|
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
|
|
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
|
|
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
|
|
requests=[]
|
|
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
|
|
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
|
|
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
|
|
requests.append(item)
|
|
result['agent_requests']=requests
|
|
path=Path('/var/lib/rmm-helper/consumption.sqlite')
|
|
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
|
|
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
|
|
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
|
|
decisions=[]
|
|
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
|
|
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
|
|
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
|
|
result['helper_decisions']=decisions
|
|
return result
|
|
|
|
|
|
def server_chain(payload):
|
|
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
|
|
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
|
|
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
|
|
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
|
|
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
|
|
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
|
|
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
|
|
return json.loads(raw)
|