docs: diagnose exact expired nonstart sync block without changing authority or effects
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
+135
-64
@@ -1,67 +1,138 @@
|
|||||||
"""Read-only stored checkpoint metadata; no effects or trust-state reset."""
|
#!/usr/bin/env python3
|
||||||
|
"""Bounded stored Sync diagnostics only; no socket, repair, job or DB write."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import cloud_mvp_acceptance as cloud
|
||||||
|
|
||||||
|
require = cloud.contract.require
|
||||||
|
|
||||||
|
|
||||||
def vm_chain(payload):
|
def server(payload):
|
||||||
import sqlite3
|
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||||
def wire(raw):
|
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
||||||
result={};i=0
|
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"'"
|
||||||
def number():
|
query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),"
|
||||||
nonlocal i
|
"'installation',(SELECT json_build_object('connection_generation',connection_generation,'connection_boot_id',connection_boot_id,"
|
||||||
value=0;shift=0
|
"'sync_ready',sync_ready,'helper_sync_ready',helper_sync_ready,'restore_blocked',restore_blocked,"
|
||||||
while True:
|
"'sequence_regressed',sequence_regressed,'sync_recovery_required',sync_recovery_required) FROM agent_installations WHERE "+scope+"),"
|
||||||
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
|
"'head',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
|
||||||
if not b&128:return value
|
"'decisions',(SELECT COALESCE(json_agg(s),'[]'::json) FROM (SELECT request_id,decision_id,request_hash,journal_id,checkpoint,connection_generation,created_at,"
|
||||||
shift+=7
|
"encode(payload,'hex') AS payload_hex FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC,decision_id DESC LIMIT 8) s)); COMMIT;")
|
||||||
while i<len(raw):
|
root = Path('/opt/rust-discovery')
|
||||||
key=number();field=key>>3;kind=key&7
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||||
if kind==0:value=number()
|
|
||||||
elif kind==2:
|
|
||||||
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
|
|
||||||
else:raise RuntimeError('stored protobuf wire type')
|
|
||||||
result.setdefault(field,[]).append(value)
|
|
||||||
return result
|
|
||||||
def request(raw):
|
|
||||||
fields=wire(raw);result={}
|
|
||||||
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
|
|
||||||
result[k]=fields.get(n,[b''])[0].decode('ascii')
|
|
||||||
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
|
|
||||||
return result
|
|
||||||
def decision(value):
|
|
||||||
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
|
|
||||||
return {k:value.get(k) for k in names}
|
|
||||||
result={}
|
|
||||||
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
|
|
||||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
|
||||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
|
|
||||||
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
|
|
||||||
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
|
|
||||||
requests=[]
|
|
||||||
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
|
|
||||||
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
|
|
||||||
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
|
|
||||||
requests.append(item)
|
|
||||||
result['agent_requests']=requests
|
|
||||||
path=Path('/var/lib/rmm-helper/consumption.sqlite')
|
|
||||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
|
||||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
|
|
||||||
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
|
|
||||||
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
|
|
||||||
decisions=[]
|
|
||||||
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
|
|
||||||
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
|
|
||||||
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
|
|
||||||
result['helper_decisions']=decisions
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def server_chain(payload):
|
|
||||||
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
|
|
||||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
||||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||||
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
|
data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
||||||
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
|
keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint',
|
||||||
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
|
'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason')
|
||||||
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
|
for row in data['decisions']:
|
||||||
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
|
raw = bytes.fromhex(row.pop('payload_hex')); value = json.loads(raw)
|
||||||
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
|
row['payload_sha256'] = hashlib.sha256(raw).hexdigest()
|
||||||
return json.loads(raw)
|
row['decision'] = {k:value.get(k) for k in keys}
|
||||||
|
row['attempts'] = [{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')}
|
||||||
|
for a in value.get('attempts',[])]
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def vm(payload):
|
||||||
|
import sqlite3
|
||||||
|
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity')
|
||||||
|
def wire(raw):
|
||||||
|
fields = {}; index = 0
|
||||||
|
def number():
|
||||||
|
nonlocal index
|
||||||
|
value = 0
|
||||||
|
for shift in range(0,70,7):
|
||||||
|
require(index < len(raw), 'bounded stored protobuf')
|
||||||
|
b = raw[index]; index += 1; value |= (b & 127) << shift
|
||||||
|
if b < 128:return value
|
||||||
|
raise ValueError('stored protobuf varint')
|
||||||
|
while index < len(raw):
|
||||||
|
tag = number(); field = tag >> 3; kind = tag & 7
|
||||||
|
if kind == 0:value = number()
|
||||||
|
elif kind == 2:
|
||||||
|
size = number(); require(index+size <= len(raw), 'stored protobuf length')
|
||||||
|
value = raw[index:index+size]; index += size
|
||||||
|
else:raise ValueError('unexpected stored protobuf wire type')
|
||||||
|
fields.setdefault(field,[]).append(value)
|
||||||
|
return fields
|
||||||
|
def decision(raw):
|
||||||
|
value = json.loads(raw)
|
||||||
|
keys = ('decision_id','request_id','request_hash','connection_generation','boot_id','journal_id','checkpoint',
|
||||||
|
'helper_journal_id','helper_checkpoint','issued_at','valid_until','ready','helper_ready','reason')
|
||||||
|
return dict(payload_sha256=hashlib.sha256(raw).hexdigest(),**{k:value.get(k) for k in keys},
|
||||||
|
attempts=[{k:a.get(k) for k in ('job_id','attempt_id','grant_id','revision','outcome_state','unlock','reason')}
|
||||||
|
for a in value.get('attempts',[])])
|
||||||
|
result = {}
|
||||||
|
for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'),
|
||||||
|
('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')):
|
||||||
|
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
|
||||||
|
db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
|
||||||
|
meta = dict(db.execute('SELECT * FROM '+table+' WHERE id=1').fetchone())
|
||||||
|
result[name] = dict(meta=meta,decisions=[])
|
||||||
|
if name == 'agent':
|
||||||
|
result[name]['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
|
||||||
|
result[name]['requests'] = []
|
||||||
|
for row in db.execute('SELECT id,request,generation,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 8'):
|
||||||
|
fields = wire(row['request']); item = dict(id=row['id'],generation=row['generation'],request_sha256=hashlib.sha256(row['request']).hexdigest())
|
||||||
|
for number,key in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
|
||||||
|
item[key] = fields.get(number,[b''])[0].decode('ascii')
|
||||||
|
for number,key in ((3,'checkpoint'),(5,'assignment_version'),(8,'journal_complete'),(10,'helper_checkpoint')):
|
||||||
|
item[key] = fields.get(number,[0])[0]
|
||||||
|
item['attempt_count'] = len(fields.get(6,[])); item['durable_message_count'] = len(fields.get(7,[]))
|
||||||
|
item['response_present'] = row['response'] is not None
|
||||||
|
if row['response'] is not None:
|
||||||
|
response = wire(row['response']); item['decision'] = decision(response[1][0])
|
||||||
|
result[name]['requests'].append(item)
|
||||||
|
else:
|
||||||
|
for row in db.execute('SELECT decision_id,checkpoint,response FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 8'):
|
||||||
|
response = json.loads(row['response']); raw = bytes(response['decision_bytes'])
|
||||||
|
result[name]['decisions'].append(dict(id=row['decision_id'],checkpoint=row['checkpoint'],decision=decision(raw)))
|
||||||
|
result[name]['latest_capture'] = dict(db.execute('SELECT checkpoint,fingerprint,captured_at FROM helper_sync_captures ORDER BY checkpoint DESC LIMIT 1').fetchone() or {})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def server_attempt(payload):
|
||||||
|
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||||
|
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
||||||
|
root = Path('/opt/rust-discovery')
|
||||||
|
query = ("BEGIN TRANSACTION READ ONLY; SELECT json_build_object('observed_at',now(),'job',"
|
||||||
|
"(SELECT json_build_object('job_id',j.job_id,'attempt_id',j.attempt_id,'state',j.state,'revision',j.revision,"
|
||||||
|
"'start_reserved',j.start_reserved,'result_present',j.result IS NOT NULL,'result_error_code',j.result->>'error_code',"
|
||||||
|
"'payload_hash',j.payload_hash,'payload_sha256_hex_input',encode(j.payload,'hex'),'pending_reason',j.pending_reason,"
|
||||||
|
"'outbox_acknowledged_at',b.acknowledged_at) FROM jobs j LEFT JOIN outbox b USING(tenant_id,job_id) "
|
||||||
|
"WHERE j.tenant_id='"+tenant+"' AND j.agent_id='"+agent+"' AND j.job_id='"+job+"' AND j.attempt_id='"+attempt+"'),"
|
||||||
|
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||||
|
"'start_audits',(SELECT count(*) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')); COMMIT;")
|
||||||
|
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||||
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||||
|
data = json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
||||||
|
if data['job']:
|
||||||
|
raw = bytes.fromhex(data['job'].pop('payload_sha256_hex_input')); value = json.loads(raw)
|
||||||
|
data['job']['actual_payload_sha256'] = hashlib.sha256(raw).hexdigest()
|
||||||
|
data['job']['execution_identity'] = value.get('execution_identity')
|
||||||
|
data['job']['start_before'] = value.get('start_before')
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def vm_attempt(payload):
|
||||||
|
import sqlite3
|
||||||
|
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
||||||
|
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent identity')
|
||||||
|
result = {}
|
||||||
|
for name,path in (('journal','/var/lib/rmm-agent-remote/journal.sqlite'),
|
||||||
|
('agent_ledger','/var/lib/rmm-agent-remote/consumption.sqlite'),
|
||||||
|
('root_ledger','/var/lib/rmm-helper/consumption.sqlite')):
|
||||||
|
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
|
||||||
|
db.row_factory = sqlite3.Row; db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
|
||||||
|
if name == 'journal':
|
||||||
|
rows = [dict(r) for r in db.execute('SELECT job,attempt,state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))]
|
||||||
|
outcomes = [dict(state=json.loads(r[0])['state'],error_code=json.loads(r[0])['error_code'],sha256=hashlib.sha256(r[0]).hexdigest())
|
||||||
|
for r in db.execute('SELECT outcome FROM reconcile_outcomes WHERE job=? AND attempt=?',(job,attempt))]
|
||||||
|
result[name] = dict(attempts=rows,reconciled_outcomes=outcomes)
|
||||||
|
else:
|
||||||
|
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
|
||||||
|
return result
|
||||||
|
|||||||
@@ -1,5 +1,27 @@
|
|||||||
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
|
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
|
||||||
|
|
||||||
|
## Nach Ablauf gefundene Sync-Klassifikationslücke
|
||||||
|
|
||||||
|
Vor dem nächsten Fall stoppte der Bereitschaftsguard; Fall09fc228c-e339-409a-
|
||||||
|
af1c-afe1a9fa9aaa hat **keinen** Plan/Antrag/Job erzeugt. Gespeicherter signierter
|
||||||
|
Sync zeigt genau den vorherigen Ablaufjob40edcd4b-99c5-43e7-8562-ccdda86c4031
|
||||||
|
als`GRANT_OR_EFFECT_UNRESOLVED`. Tatsächlich zentral und lokal expired/revision0,
|
||||||
|
Grantleer, keine Resultbehauptung; zentral`start_reserved=false`,0Permits und
|
||||||
|
0Startaudits, beide Consumptionledgers ohne diesen Versuch. Alle15Rootwirkungen
|
||||||
|
geschlossen/faultfrei, Journalintegrität und Kerneluhr geprüft, kein Platzmangel.
|
||||||
|
|
||||||
|
Ursache: Nach der ersten signierten Nichtstart-Expiry bezeichnete der Agent
|
||||||
|
denselben grantlosen terminalen Nichtstart beim nächsten Snapshot als
|
||||||
|
`uncertain`; nur`accepted` bekam das Label`unconsumed`. Der Server verweigert
|
||||||
|
dieses widersprüchliche Bild korrekt. Eine reine Serverlockerung genügt zudem
|
||||||
|
nicht, weil auch der Agent terminale beweislose Ergebnisse bei`uncertain`
|
||||||
|
verweigert. Notwendige Korrektur bleibt auf die richtige Agentklassifikation
|
||||||
|
des belegten Nichtstarts begrenzt; vorhandene Verbrauchszeilen haben weiterhin
|
||||||
|
Vorrang, requesting/running/unknown bleiben gesperrt. Noch nicht als native
|
||||||
|
Korrektur abgenommen; keine Datenbank-/Freigabe-/Unlockabkürzung.
|
||||||
|
Private tatsächliche Readbacks`09fc.../sync-{server,vm}.json` und
|
||||||
|
`expired-attempt-{server,vm}.json`; zentrale Historie26/26/23/0 unverändert.
|
||||||
|
|
||||||
## Eigenständiger Ablauf bei aktivem Offlineagent bestanden
|
## Eigenständiger Ablauf bei aktivem Offlineagent bestanden
|
||||||
|
|
||||||
Fall`5869480b-d06c-450c-8163-838bf2dda6bc` tatsächlich bestanden: eigener fester
|
Fall`5869480b-d06c-450c-8163-838bf2dda6bc` tatsächlich bestanden: eigener fester
|
||||||
|
|||||||
@@ -1,5 +1,18 @@
|
|||||||
# Tatsächlicher Implementierungsstand
|
# Tatsächlicher Implementierungsstand
|
||||||
|
|
||||||
|
## Nichtstart nach Ablauf beim Folgesync falsch klassifiziert — 2026-11-13
|
||||||
|
|
||||||
|
Vor UI-Entzugsplan blockiert korrekter Readinessguard, noch kein neuer Antrag.
|
||||||
|
Tatsächlicher gespeicherter Sync und beide Ledgers bestätigen für den
|
||||||
|
abgelaufenen vorherigen Versuch0Grant/0Startreservierung/0Startaudit/0Verbrauch,
|
||||||
|
expired/revision0. Agent kennzeichnet den grantlosen terminalen Zustand jedoch
|
||||||
|
beim Folgesnapshot als uncertain und verursacht dadurch die Verweigerung
|
||||||
|
`ATTEMPT_RECONCILIATION_REQUIRED`. Keine ungeklärte tatsächliche Wirkung,
|
||||||
|
beide Journale faultfrei, Root15geschlossen, Zentrale26/26/23/0. Keine
|
||||||
|
Umgehung/DBreparatur; minimale Agentklassifikation wird gezielt korrigiert,
|
||||||
|
Helper/Provider/M4Runtime bleiben unverändert. Vollständiger M2/MVP-Abschluss
|
||||||
|
weiter offen. Details:[Offlineabnahme](cloud-offline-acceptance.md).
|
||||||
|
|
||||||
## Aktiver Offline-Ablauf ohne nachträgliche Wirkung bestanden — 2026-11-13
|
## Aktiver Offline-Ablauf ohne nachträgliche Wirkung bestanden — 2026-11-13
|
||||||
|
|
||||||
Eigener Fall5869480b-d06c-450c-8163-838bf2dda6bc mit echtem WebUI-Nullgate und
|
Eigener Fall5869480b-d06c-450c-8163-838bf2dda6bc mit echtem WebUI-Nullgate und
|
||||||
|
|||||||
Reference in New Issue
Block a user