fix: preserve signed terminal nonstarts across repeated and restarted sync
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 21:06:27 +01:00
parent 6d91988bec
commit 6715add024
6 changed files with 233 additions and 1 deletions
+7 -1
View File
@@ -758,7 +758,13 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)
.optional()?;
let mut consumption = if grant_id.is_empty() && state == "accepted" {
// A signed definitive nonstart can make an accepted no-grant row
// terminal without a JobResult. Preserve that fact on the next Sync;
// requesting/running/unknown still cannot claim non-consumption. An
// actual durable consumption row below always overrides this label.
let mut consumption = if grant_id.is_empty()
&& matches!(state.as_str(), "accepted" | "expired" | "cancelled")
{
"unconsumed"
} else {
"uncertain"
+117
View File
@@ -456,6 +456,123 @@ fn definite_nonstart_result_is_unconsumed_not_ambiguous() {
assert!(c.can_start(&f.channel, false, &f.clock));
}
#[test]
fn signed_terminal_nonstart_stays_unconsumed_across_sync_and_pending_denial() {
for terminal in ["expired", "cancelled"] {
let mut f = Fixture::new();
let (offer, _, _) = f.add("accepted", false, None, script());
let mut client = f.client();
let first = f.prepare(&mut client);
assert_eq!(first.request.attempts[0].consumption, "unconsumed");
let mut decision = f.decision(&first, true);
decision.attempts[0].outcome_state = terminal.into();
decision.attempts[0].reason = "DEFINITIVE_NONSTART".into();
f.apply(&mut client, &decision).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
let second = f.prepare(&mut client);
let local = &second.request.attempts[0];
assert_eq!(local.state, terminal);
assert_eq!(local.consumption, "unconsumed");
assert!(local.grant_id.is_empty() && local.outcome_bytes.is_empty());
let mut repeated = f.decision(&second, true);
repeated.attempts[0].reason = "DEFINITIVE_NONSTART".into();
f.apply(&mut client, &repeated).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
// Reproduce the old server's rejected terminal->unknown response.
// Keep its pending request and the last applied signed anchor, then
// recover through a fresh channel/request with the ordinary protocol.
let pending = f.prepare(&mut client);
let mut denied = f.decision(&pending, false);
denied.attempts[0].outcome_state = "unknown".into();
denied.attempts[0].reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
assert!(f.apply(&mut client, &denied).is_err());
assert!(!client.can_start(&f.channel, false, &f.clock));
client = f.client();
f.channel.connection_generation += 1;
let fresh = f.prepare(&mut client);
assert_eq!(
fresh.request.previous_request_id,
pending.request.request_id
);
assert_eq!(
fresh.request.previous_decision_id,
repeated.decision_id.to_string()
);
assert_eq!(fresh.request.attempts[0].consumption, "unconsumed");
let mut recovered = f.decision(&fresh, true);
recovered.attempts[0].reason = "DEFINITIVE_NONSTART".into();
f.apply(&mut client, &recovered).unwrap();
assert!(client.can_start(&f.channel, false, &f.clock));
let stored: (String, String, String) =
f.db.query_row("SELECT job,attempt,state FROM job_attempts", [], |r| {
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
})
.unwrap();
assert_eq!(stored, (offer.job_id, offer.attempt_id, terminal.into()));
assert_eq!(
f.ledger
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
.unwrap(),
0
);
assert_eq!(
f.db.query_row("SELECT count(*) FROM reconcile_outcomes", [], |r| r
.get::<_, i64>(0))
.unwrap(),
0
);
}
}
#[test]
fn terminal_labels_never_override_grant_consumption_or_unclear_attempts() {
for state in ["requesting", "running", "unknown"] {
let mut f = Fixture::new();
f.add(state, false, None, script());
let mut client = f.client();
let prepared = f.prepare(&mut client);
assert_eq!(prepared.request.attempts[0].consumption, "uncertain");
let decision = f.decision(&prepared, true);
f.apply(&mut client, &decision).unwrap();
assert!(!client.can_start(&f.channel, false, &f.clock));
}
for terminal in ["expired", "cancelled"] {
for consumed in [false, true] {
let mut f = Fixture::new();
f.add(terminal, consumed, None, script());
// A surviving grant ID also forbids the nonstart shortcut when
// no consumption row exists. A real row always remains consumed.
if !consumed {
f.db.execute(
"UPDATE job_attempts SET grant_id=?1",
[Uuid::new_v4().to_string()],
)
.unwrap();
}
let mut client = f.client();
let prepared = f.prepare(&mut client);
assert_eq!(
prepared.request.attempts[0].consumption,
if consumed { "consumed" } else { "uncertain" }
);
let mut decision = f.decision(&prepared, true);
decision.attempts[0].unlock = false;
assert!(f.apply(&mut client, &decision).is_err());
assert!(!client.can_start(&f.channel, false, &f.clock));
assert_eq!(
f.ledger
.query_row(
"SELECT count(*) FROM consumed WHERE result IS NULL",
[],
|r| r.get::<_, i64>(0)
)
.unwrap(),
i64::from(consumed)
);
}
}
}
#[test]
fn negative_server_revision_never_rolls_back_local_unknown_revision() {
let mut f = Fixture::new();
f.add("running", true, None, script());
+67
View File
@@ -136,3 +136,70 @@ def vm_attempt(payload):
else:
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
return result
# Retain the original stored-checkpoint reader entry points.
def vm_chain(payload):
import sqlite3
def wire(raw):
result={};i=0
def number():
nonlocal i
value=0;shift=0
while True:
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
if not b&128:return value
shift+=7
while i<len(raw):
key=number();field=key>>3;kind=key&7
if kind==0:value=number()
elif kind==2:
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
else:raise RuntimeError('stored protobuf wire type')
result.setdefault(field,[]).append(value)
return result
def request(raw):
fields=wire(raw);result={}
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
result[k]=fields.get(n,[b''])[0].decode('ascii')
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
return result
def decision(value):
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
return {k:value.get(k) for k in names}
result={}
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
requests=[]
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
requests.append(item)
result['agent_requests']=requests
path=Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
decisions=[]
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
result['helper_decisions']=decisions
return result
def server_chain(payload):
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
return json.loads(raw)
+19
View File
@@ -1,5 +1,24 @@
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
## Wiederholter Nichtstart-Sync nativ korrigiert
Gezielter Agentfix qualifiziert und auf der Cloud-VM ausgeführt.20 echte
SQLite-/Signatur-Reconciletests ohne Skips, Format und Agent-Alltargets-Clippy
bestanden. Grantfreie accepted/expired/cancelled-Zeilen behalten das Nichtstart-
Label; tatsächlich vorhandene Verbrauchszeilen überschreiben es weiterhin.
requesting/running/unknown und überlebende Grants bleiben gesperrt. Keine
Serverlockerung und keine Änderung von Helperbbd/Provider1dc/M4Runtime3bac.
Agent702bd7b5… auf vorhandener Identität und unveränderten Journalbytes vor
Neustart installiert. Normaler ausgehender Kanal Generation93 und signierter
Sync wieder bereit, restore_blockedfalse. Derselbe ursprüngliche Ablaufjob
40edcd4b… jetzt korrekt`expired/DEFINITIVE_NONSTART`, Grantnull, revision0,
unlockfalse;26Jobs/26Outbox/23Grants/0offen unverändert. Keine Repair-CLI,
manuelle DB-/Outcomeänderung oder erneute Ablaufwirkung.
Private Belege`devstate/cloud-apt-recovery/nonstart-agent-install.json`,
`agent-nonstart-sync.json` und`nonstart-sync-native.json`.
UI-Entzug, Crash und Reboot bleiben offen.
## Nach Ablauf gefundene Sync-Klassifikationslücke
Vor dem nächsten Fall stoppte der Bereitschaftsguard; Fall09fc228c-e339-409a-
+9
View File
@@ -1,5 +1,14 @@
# Übergabe / Entwicklungspause — 2026-11-12
**Aktuellster nativer Stand:** Agent702bd7b5 korrigiert wiederholten Nichtstart-
Sync nach abgelaufenem grantlosen Auftrag. Erhaltend installiert; normaler
Generation93-Sync ready/restore_blockedfalse, gleicher Ablaufjob korrekt
expired/DEFINITIVE_NONSTART, kein Grant/Unlock. Zentrale26/26/23/0, Root15closed.
Keine Server-/Helper-/Provideränderung, M4Runtime3bac bleibt gültig. Aktiver
Offline-Warte- und Ablauffall sowie MCP-Vorbericht bestanden; Generalautostart
nachgewiesen aktiviert, öffentlicher Pinbootstrapfinalisiert. Noch UI-Entzug,
Crash und letzter einzelner Reboot plus Endaudit. Ältere Sperrstände historisch.
**Neuester abgeschlossener Stand:** M4-APT im begrenzten Fixtureumfang bestanden:
echtes1.0→1.1-Upgrade plus eigene ensure_package-/apply_updates-Noops, feste
WebUI-Grenze jeweils0/0/0→1/1/1/0. Tatsächliche Paketnachbedingungen, true→true
+14
View File
@@ -1,5 +1,19 @@
# Tatsächlicher Implementierungsstand
## Nichtstart beim Folgesync nativ stabil erhalten — 2026-11-13
Minimaler Agentfix für grantfreie expired/cancelled-Nichtstarts,20 echte
SQLite-/Signatur-Reconciletests/0Skips, Format und Alltargets-Clippy bestanden.
Vorhandene Verbrauchszeilen haben Vorrang; unklare Wirkungen bleiben gesperrt.
Neuer Bookworm-Agent702bd7b5293a38b3d910d9b38401a155dd5bccb88d43fa3af807d2094c70caa3
erhaltend auf Cloud-VM installiert. Gewöhnlicher frischer Kanal Generation93
übernahm die vorhandenen letzten gültigen/pending Anker ohne Reparatur/Reset.
Signierter Sync wieder ready, restore_blockedfalse; ursprünglicher Ablaufjob
korrekt expired/DEFINITIVE_NONSTART mitnullGrant/revision0/unlockfalse.
Zentrale26/26/23/0 unverändert; keine erneute Wirkung, kein neuer Plan. Helper,
Provider und native1.1-Acceptanceruntime3bac unverändert. Nächste Arbeit eigener
UI-Entzugsfall, Crash, Reboot zuletzt und Endaudit.
## Nichtstart nach Ablauf beim Folgesync falsch klassifiziert — 2026-11-13
Vor UI-Entzugsplan blockiert korrekter Readinessguard, noch kein neuer Antrag.