fix: preserve signed terminal nonstarts across repeated and restarted sync
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -758,7 +758,13 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|
|||||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
let mut consumption = if grant_id.is_empty() && state == "accepted" {
|
// A signed definitive nonstart can make an accepted no-grant row
|
||||||
|
// terminal without a JobResult. Preserve that fact on the next Sync;
|
||||||
|
// requesting/running/unknown still cannot claim non-consumption. An
|
||||||
|
// actual durable consumption row below always overrides this label.
|
||||||
|
let mut consumption = if grant_id.is_empty()
|
||||||
|
&& matches!(state.as_str(), "accepted" | "expired" | "cancelled")
|
||||||
|
{
|
||||||
"unconsumed"
|
"unconsumed"
|
||||||
} else {
|
} else {
|
||||||
"uncertain"
|
"uncertain"
|
||||||
|
|||||||
@@ -456,6 +456,123 @@ fn definite_nonstart_result_is_unconsumed_not_ambiguous() {
|
|||||||
assert!(c.can_start(&f.channel, false, &f.clock));
|
assert!(c.can_start(&f.channel, false, &f.clock));
|
||||||
}
|
}
|
||||||
#[test]
|
#[test]
|
||||||
|
fn signed_terminal_nonstart_stays_unconsumed_across_sync_and_pending_denial() {
|
||||||
|
for terminal in ["expired", "cancelled"] {
|
||||||
|
let mut f = Fixture::new();
|
||||||
|
let (offer, _, _) = f.add("accepted", false, None, script());
|
||||||
|
let mut client = f.client();
|
||||||
|
let first = f.prepare(&mut client);
|
||||||
|
assert_eq!(first.request.attempts[0].consumption, "unconsumed");
|
||||||
|
let mut decision = f.decision(&first, true);
|
||||||
|
decision.attempts[0].outcome_state = terminal.into();
|
||||||
|
decision.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
||||||
|
f.apply(&mut client, &decision).unwrap();
|
||||||
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||||
|
|
||||||
|
let second = f.prepare(&mut client);
|
||||||
|
let local = &second.request.attempts[0];
|
||||||
|
assert_eq!(local.state, terminal);
|
||||||
|
assert_eq!(local.consumption, "unconsumed");
|
||||||
|
assert!(local.grant_id.is_empty() && local.outcome_bytes.is_empty());
|
||||||
|
let mut repeated = f.decision(&second, true);
|
||||||
|
repeated.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
||||||
|
f.apply(&mut client, &repeated).unwrap();
|
||||||
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||||
|
|
||||||
|
// Reproduce the old server's rejected terminal->unknown response.
|
||||||
|
// Keep its pending request and the last applied signed anchor, then
|
||||||
|
// recover through a fresh channel/request with the ordinary protocol.
|
||||||
|
let pending = f.prepare(&mut client);
|
||||||
|
let mut denied = f.decision(&pending, false);
|
||||||
|
denied.attempts[0].outcome_state = "unknown".into();
|
||||||
|
denied.attempts[0].reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
|
||||||
|
assert!(f.apply(&mut client, &denied).is_err());
|
||||||
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||||
|
client = f.client();
|
||||||
|
f.channel.connection_generation += 1;
|
||||||
|
let fresh = f.prepare(&mut client);
|
||||||
|
assert_eq!(
|
||||||
|
fresh.request.previous_request_id,
|
||||||
|
pending.request.request_id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fresh.request.previous_decision_id,
|
||||||
|
repeated.decision_id.to_string()
|
||||||
|
);
|
||||||
|
assert_eq!(fresh.request.attempts[0].consumption, "unconsumed");
|
||||||
|
let mut recovered = f.decision(&fresh, true);
|
||||||
|
recovered.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
||||||
|
f.apply(&mut client, &recovered).unwrap();
|
||||||
|
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||||
|
let stored: (String, String, String) =
|
||||||
|
f.db.query_row("SELECT job,attempt,state FROM job_attempts", [], |r| {
|
||||||
|
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stored, (offer.job_id, offer.attempt_id, terminal.into()));
|
||||||
|
assert_eq!(
|
||||||
|
f.ledger
|
||||||
|
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
|
||||||
|
.unwrap(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f.db.query_row("SELECT count(*) FROM reconcile_outcomes", [], |r| r
|
||||||
|
.get::<_, i64>(0))
|
||||||
|
.unwrap(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn terminal_labels_never_override_grant_consumption_or_unclear_attempts() {
|
||||||
|
for state in ["requesting", "running", "unknown"] {
|
||||||
|
let mut f = Fixture::new();
|
||||||
|
f.add(state, false, None, script());
|
||||||
|
let mut client = f.client();
|
||||||
|
let prepared = f.prepare(&mut client);
|
||||||
|
assert_eq!(prepared.request.attempts[0].consumption, "uncertain");
|
||||||
|
let decision = f.decision(&prepared, true);
|
||||||
|
f.apply(&mut client, &decision).unwrap();
|
||||||
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||||
|
}
|
||||||
|
for terminal in ["expired", "cancelled"] {
|
||||||
|
for consumed in [false, true] {
|
||||||
|
let mut f = Fixture::new();
|
||||||
|
f.add(terminal, consumed, None, script());
|
||||||
|
// A surviving grant ID also forbids the nonstart shortcut when
|
||||||
|
// no consumption row exists. A real row always remains consumed.
|
||||||
|
if !consumed {
|
||||||
|
f.db.execute(
|
||||||
|
"UPDATE job_attempts SET grant_id=?1",
|
||||||
|
[Uuid::new_v4().to_string()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let mut client = f.client();
|
||||||
|
let prepared = f.prepare(&mut client);
|
||||||
|
assert_eq!(
|
||||||
|
prepared.request.attempts[0].consumption,
|
||||||
|
if consumed { "consumed" } else { "uncertain" }
|
||||||
|
);
|
||||||
|
let mut decision = f.decision(&prepared, true);
|
||||||
|
decision.attempts[0].unlock = false;
|
||||||
|
assert!(f.apply(&mut client, &decision).is_err());
|
||||||
|
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||||
|
assert_eq!(
|
||||||
|
f.ledger
|
||||||
|
.query_row(
|
||||||
|
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||||
|
[],
|
||||||
|
|r| r.get::<_, i64>(0)
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
i64::from(consumed)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
fn negative_server_revision_never_rolls_back_local_unknown_revision() {
|
fn negative_server_revision_never_rolls_back_local_unknown_revision() {
|
||||||
let mut f = Fixture::new();
|
let mut f = Fixture::new();
|
||||||
f.add("running", true, None, script());
|
f.add("running", true, None, script());
|
||||||
|
|||||||
@@ -136,3 +136,70 @@ def vm_attempt(payload):
|
|||||||
else:
|
else:
|
||||||
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
|
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# Retain the original stored-checkpoint reader entry points.
|
||||||
|
def vm_chain(payload):
|
||||||
|
import sqlite3
|
||||||
|
def wire(raw):
|
||||||
|
result={};i=0
|
||||||
|
def number():
|
||||||
|
nonlocal i
|
||||||
|
value=0;shift=0
|
||||||
|
while True:
|
||||||
|
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
|
||||||
|
if not b&128:return value
|
||||||
|
shift+=7
|
||||||
|
while i<len(raw):
|
||||||
|
key=number();field=key>>3;kind=key&7
|
||||||
|
if kind==0:value=number()
|
||||||
|
elif kind==2:
|
||||||
|
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
|
||||||
|
else:raise RuntimeError('stored protobuf wire type')
|
||||||
|
result.setdefault(field,[]).append(value)
|
||||||
|
return result
|
||||||
|
def request(raw):
|
||||||
|
fields=wire(raw);result={}
|
||||||
|
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
|
||||||
|
result[k]=fields.get(n,[b''])[0].decode('ascii')
|
||||||
|
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
|
||||||
|
return result
|
||||||
|
def decision(value):
|
||||||
|
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
|
||||||
|
return {k:value.get(k) for k in names}
|
||||||
|
result={}
|
||||||
|
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
|
||||||
|
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||||
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
|
||||||
|
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
|
||||||
|
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
|
||||||
|
requests=[]
|
||||||
|
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
|
||||||
|
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
|
||||||
|
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
|
||||||
|
requests.append(item)
|
||||||
|
result['agent_requests']=requests
|
||||||
|
path=Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||||
|
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||||
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
|
||||||
|
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
|
||||||
|
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
|
||||||
|
decisions=[]
|
||||||
|
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
|
||||||
|
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
|
||||||
|
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
|
||||||
|
result['helper_decisions']=decisions
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def server_chain(payload):
|
||||||
|
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
|
||||||
|
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||||
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||||
|
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
|
||||||
|
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
|
||||||
|
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
|
||||||
|
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
|
||||||
|
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
|
||||||
|
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
|
||||||
|
return json.loads(raw)
|
||||||
|
|||||||
@@ -1,5 +1,24 @@
|
|||||||
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
|
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
|
||||||
|
|
||||||
|
## Wiederholter Nichtstart-Sync nativ korrigiert
|
||||||
|
|
||||||
|
Gezielter Agentfix qualifiziert und auf der Cloud-VM ausgeführt.20 echte
|
||||||
|
SQLite-/Signatur-Reconciletests ohne Skips, Format und Agent-Alltargets-Clippy
|
||||||
|
bestanden. Grantfreie accepted/expired/cancelled-Zeilen behalten das Nichtstart-
|
||||||
|
Label; tatsächlich vorhandene Verbrauchszeilen überschreiben es weiterhin.
|
||||||
|
requesting/running/unknown und überlebende Grants bleiben gesperrt. Keine
|
||||||
|
Serverlockerung und keine Änderung von Helperbbd/Provider1dc/M4Runtime3bac.
|
||||||
|
|
||||||
|
Agent702bd7b5… auf vorhandener Identität und unveränderten Journalbytes vor
|
||||||
|
Neustart installiert. Normaler ausgehender Kanal Generation93 und signierter
|
||||||
|
Sync wieder bereit, restore_blockedfalse. Derselbe ursprüngliche Ablaufjob
|
||||||
|
40edcd4b… jetzt korrekt`expired/DEFINITIVE_NONSTART`, Grantnull, revision0,
|
||||||
|
unlockfalse;26Jobs/26Outbox/23Grants/0offen unverändert. Keine Repair-CLI,
|
||||||
|
manuelle DB-/Outcomeänderung oder erneute Ablaufwirkung.
|
||||||
|
Private Belege`devstate/cloud-apt-recovery/nonstart-agent-install.json`,
|
||||||
|
`agent-nonstart-sync.json` und`nonstart-sync-native.json`.
|
||||||
|
UI-Entzug, Crash und Reboot bleiben offen.
|
||||||
|
|
||||||
## Nach Ablauf gefundene Sync-Klassifikationslücke
|
## Nach Ablauf gefundene Sync-Klassifikationslücke
|
||||||
|
|
||||||
Vor dem nächsten Fall stoppte der Bereitschaftsguard; Fall09fc228c-e339-409a-
|
Vor dem nächsten Fall stoppte der Bereitschaftsguard; Fall09fc228c-e339-409a-
|
||||||
|
|||||||
@@ -1,5 +1,14 @@
|
|||||||
# Übergabe / Entwicklungspause — 2026-11-12
|
# Übergabe / Entwicklungspause — 2026-11-12
|
||||||
|
|
||||||
|
**Aktuellster nativer Stand:** Agent702bd7b5 korrigiert wiederholten Nichtstart-
|
||||||
|
Sync nach abgelaufenem grantlosen Auftrag. Erhaltend installiert; normaler
|
||||||
|
Generation93-Sync ready/restore_blockedfalse, gleicher Ablaufjob korrekt
|
||||||
|
expired/DEFINITIVE_NONSTART, kein Grant/Unlock. Zentrale26/26/23/0, Root15closed.
|
||||||
|
Keine Server-/Helper-/Provideränderung, M4Runtime3bac bleibt gültig. Aktiver
|
||||||
|
Offline-Warte- und Ablauffall sowie MCP-Vorbericht bestanden; Generalautostart
|
||||||
|
nachgewiesen aktiviert, öffentlicher Pinbootstrapfinalisiert. Noch UI-Entzug,
|
||||||
|
Crash und letzter einzelner Reboot plus Endaudit. Ältere Sperrstände historisch.
|
||||||
|
|
||||||
**Neuester abgeschlossener Stand:** M4-APT im begrenzten Fixtureumfang bestanden:
|
**Neuester abgeschlossener Stand:** M4-APT im begrenzten Fixtureumfang bestanden:
|
||||||
echtes1.0→1.1-Upgrade plus eigene ensure_package-/apply_updates-Noops, feste
|
echtes1.0→1.1-Upgrade plus eigene ensure_package-/apply_updates-Noops, feste
|
||||||
WebUI-Grenze jeweils0/0/0→1/1/1/0. Tatsächliche Paketnachbedingungen, true→true
|
WebUI-Grenze jeweils0/0/0→1/1/1/0. Tatsächliche Paketnachbedingungen, true→true
|
||||||
|
|||||||
@@ -1,5 +1,19 @@
|
|||||||
# Tatsächlicher Implementierungsstand
|
# Tatsächlicher Implementierungsstand
|
||||||
|
|
||||||
|
## Nichtstart beim Folgesync nativ stabil erhalten — 2026-11-13
|
||||||
|
|
||||||
|
Minimaler Agentfix für grantfreie expired/cancelled-Nichtstarts,20 echte
|
||||||
|
SQLite-/Signatur-Reconciletests/0Skips, Format und Alltargets-Clippy bestanden.
|
||||||
|
Vorhandene Verbrauchszeilen haben Vorrang; unklare Wirkungen bleiben gesperrt.
|
||||||
|
Neuer Bookworm-Agent702bd7b5293a38b3d910d9b38401a155dd5bccb88d43fa3af807d2094c70caa3
|
||||||
|
erhaltend auf Cloud-VM installiert. Gewöhnlicher frischer Kanal Generation93
|
||||||
|
übernahm die vorhandenen letzten gültigen/pending Anker ohne Reparatur/Reset.
|
||||||
|
Signierter Sync wieder ready, restore_blockedfalse; ursprünglicher Ablaufjob
|
||||||
|
korrekt expired/DEFINITIVE_NONSTART mitnullGrant/revision0/unlockfalse.
|
||||||
|
Zentrale26/26/23/0 unverändert; keine erneute Wirkung, kein neuer Plan. Helper,
|
||||||
|
Provider und native1.1-Acceptanceruntime3bac unverändert. Nächste Arbeit eigener
|
||||||
|
UI-Entzugsfall, Crash, Reboot zuletzt und Endaudit.
|
||||||
|
|
||||||
## Nichtstart nach Ablauf beim Folgesync falsch klassifiziert — 2026-11-13
|
## Nichtstart nach Ablauf beim Folgesync falsch klassifiziert — 2026-11-13
|
||||||
|
|
||||||
Vor UI-Entzugsplan blockiert korrekter Readinessguard, noch kein neuer Antrag.
|
Vor UI-Entzugsplan blockiert korrekter Readinessguard, noch kein neuer Antrag.
|
||||||
|
|||||||
Reference in New Issue
Block a user