fix: preserve signed terminal nonstarts across repeated and restarted sync
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -758,7 +758,13 @@ fn attempts(db: &Connection, ledger: &Connection, binding: &Binding) -> Result<V
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let mut consumption = if grant_id.is_empty() && state == "accepted" {
|
||||
// A signed definitive nonstart can make an accepted no-grant row
|
||||
// terminal without a JobResult. Preserve that fact on the next Sync;
|
||||
// requesting/running/unknown still cannot claim non-consumption. An
|
||||
// actual durable consumption row below always overrides this label.
|
||||
let mut consumption = if grant_id.is_empty()
|
||||
&& matches!(state.as_str(), "accepted" | "expired" | "cancelled")
|
||||
{
|
||||
"unconsumed"
|
||||
} else {
|
||||
"uncertain"
|
||||
|
||||
@@ -456,6 +456,123 @@ fn definite_nonstart_result_is_unconsumed_not_ambiguous() {
|
||||
assert!(c.can_start(&f.channel, false, &f.clock));
|
||||
}
|
||||
#[test]
|
||||
fn signed_terminal_nonstart_stays_unconsumed_across_sync_and_pending_denial() {
|
||||
for terminal in ["expired", "cancelled"] {
|
||||
let mut f = Fixture::new();
|
||||
let (offer, _, _) = f.add("accepted", false, None, script());
|
||||
let mut client = f.client();
|
||||
let first = f.prepare(&mut client);
|
||||
assert_eq!(first.request.attempts[0].consumption, "unconsumed");
|
||||
let mut decision = f.decision(&first, true);
|
||||
decision.attempts[0].outcome_state = terminal.into();
|
||||
decision.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
||||
f.apply(&mut client, &decision).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
|
||||
let second = f.prepare(&mut client);
|
||||
let local = &second.request.attempts[0];
|
||||
assert_eq!(local.state, terminal);
|
||||
assert_eq!(local.consumption, "unconsumed");
|
||||
assert!(local.grant_id.is_empty() && local.outcome_bytes.is_empty());
|
||||
let mut repeated = f.decision(&second, true);
|
||||
repeated.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
||||
f.apply(&mut client, &repeated).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
|
||||
// Reproduce the old server's rejected terminal->unknown response.
|
||||
// Keep its pending request and the last applied signed anchor, then
|
||||
// recover through a fresh channel/request with the ordinary protocol.
|
||||
let pending = f.prepare(&mut client);
|
||||
let mut denied = f.decision(&pending, false);
|
||||
denied.attempts[0].outcome_state = "unknown".into();
|
||||
denied.attempts[0].reason = "GRANT_OR_EFFECT_UNRESOLVED".into();
|
||||
assert!(f.apply(&mut client, &denied).is_err());
|
||||
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||
client = f.client();
|
||||
f.channel.connection_generation += 1;
|
||||
let fresh = f.prepare(&mut client);
|
||||
assert_eq!(
|
||||
fresh.request.previous_request_id,
|
||||
pending.request.request_id
|
||||
);
|
||||
assert_eq!(
|
||||
fresh.request.previous_decision_id,
|
||||
repeated.decision_id.to_string()
|
||||
);
|
||||
assert_eq!(fresh.request.attempts[0].consumption, "unconsumed");
|
||||
let mut recovered = f.decision(&fresh, true);
|
||||
recovered.attempts[0].reason = "DEFINITIVE_NONSTART".into();
|
||||
f.apply(&mut client, &recovered).unwrap();
|
||||
assert!(client.can_start(&f.channel, false, &f.clock));
|
||||
let stored: (String, String, String) =
|
||||
f.db.query_row("SELECT job,attempt,state FROM job_attempts", [], |r| {
|
||||
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(stored, (offer.job_id, offer.attempt_id, terminal.into()));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row("SELECT count(*) FROM consumed", [], |r| r.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
f.db.query_row("SELECT count(*) FROM reconcile_outcomes", [], |r| r
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn terminal_labels_never_override_grant_consumption_or_unclear_attempts() {
|
||||
for state in ["requesting", "running", "unknown"] {
|
||||
let mut f = Fixture::new();
|
||||
f.add(state, false, None, script());
|
||||
let mut client = f.client();
|
||||
let prepared = f.prepare(&mut client);
|
||||
assert_eq!(prepared.request.attempts[0].consumption, "uncertain");
|
||||
let decision = f.decision(&prepared, true);
|
||||
f.apply(&mut client, &decision).unwrap();
|
||||
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||
}
|
||||
for terminal in ["expired", "cancelled"] {
|
||||
for consumed in [false, true] {
|
||||
let mut f = Fixture::new();
|
||||
f.add(terminal, consumed, None, script());
|
||||
// A surviving grant ID also forbids the nonstart shortcut when
|
||||
// no consumption row exists. A real row always remains consumed.
|
||||
if !consumed {
|
||||
f.db.execute(
|
||||
"UPDATE job_attempts SET grant_id=?1",
|
||||
[Uuid::new_v4().to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let mut client = f.client();
|
||||
let prepared = f.prepare(&mut client);
|
||||
assert_eq!(
|
||||
prepared.request.attempts[0].consumption,
|
||||
if consumed { "consumed" } else { "uncertain" }
|
||||
);
|
||||
let mut decision = f.decision(&prepared, true);
|
||||
decision.attempts[0].unlock = false;
|
||||
assert!(f.apply(&mut client, &decision).is_err());
|
||||
assert!(!client.can_start(&f.channel, false, &f.clock));
|
||||
assert_eq!(
|
||||
f.ledger
|
||||
.query_row(
|
||||
"SELECT count(*) FROM consumed WHERE result IS NULL",
|
||||
[],
|
||||
|r| r.get::<_, i64>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
i64::from(consumed)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn negative_server_revision_never_rolls_back_local_unknown_revision() {
|
||||
let mut f = Fixture::new();
|
||||
f.add("running", true, None, script());
|
||||
|
||||
@@ -136,3 +136,70 @@ def vm_attempt(payload):
|
||||
else:
|
||||
result[name] = dict(consumed=[dict(r) for r in db.execute('SELECT grant_id,agent,generation,result IS NOT NULL AS result_present FROM consumed WHERE job=? AND attempt=?',(job,attempt))])
|
||||
return result
|
||||
|
||||
|
||||
# Retain the original stored-checkpoint reader entry points.
|
||||
def vm_chain(payload):
|
||||
import sqlite3
|
||||
def wire(raw):
|
||||
result={};i=0
|
||||
def number():
|
||||
nonlocal i
|
||||
value=0;shift=0
|
||||
while True:
|
||||
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
|
||||
if not b&128:return value
|
||||
shift+=7
|
||||
while i<len(raw):
|
||||
key=number();field=key>>3;kind=key&7
|
||||
if kind==0:value=number()
|
||||
elif kind==2:
|
||||
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
|
||||
else:raise RuntimeError('stored protobuf wire type')
|
||||
result.setdefault(field,[]).append(value)
|
||||
return result
|
||||
def request(raw):
|
||||
fields=wire(raw);result={}
|
||||
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
|
||||
result[k]=fields.get(n,[b''])[0].decode('ascii')
|
||||
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
|
||||
return result
|
||||
def decision(value):
|
||||
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
|
||||
return {k:value.get(k) for k in names}
|
||||
result={}
|
||||
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
|
||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
|
||||
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
|
||||
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
|
||||
requests=[]
|
||||
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
|
||||
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
|
||||
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
|
||||
requests.append(item)
|
||||
result['agent_requests']=requests
|
||||
path=Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
|
||||
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
|
||||
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
|
||||
decisions=[]
|
||||
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
|
||||
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
|
||||
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
|
||||
result['helper_decisions']=decisions
|
||||
return result
|
||||
|
||||
|
||||
def server_chain(payload):
|
||||
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
|
||||
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
|
||||
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
|
||||
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
|
||||
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
|
||||
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
|
||||
return json.loads(raw)
|
||||
|
||||
@@ -1,5 +1,24 @@
|
||||
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
|
||||
|
||||
## Wiederholter Nichtstart-Sync nativ korrigiert
|
||||
|
||||
Gezielter Agentfix qualifiziert und auf der Cloud-VM ausgeführt.20 echte
|
||||
SQLite-/Signatur-Reconciletests ohne Skips, Format und Agent-Alltargets-Clippy
|
||||
bestanden. Grantfreie accepted/expired/cancelled-Zeilen behalten das Nichtstart-
|
||||
Label; tatsächlich vorhandene Verbrauchszeilen überschreiben es weiterhin.
|
||||
requesting/running/unknown und überlebende Grants bleiben gesperrt. Keine
|
||||
Serverlockerung und keine Änderung von Helperbbd/Provider1dc/M4Runtime3bac.
|
||||
|
||||
Agent702bd7b5… auf vorhandener Identität und unveränderten Journalbytes vor
|
||||
Neustart installiert. Normaler ausgehender Kanal Generation93 und signierter
|
||||
Sync wieder bereit, restore_blockedfalse. Derselbe ursprüngliche Ablaufjob
|
||||
40edcd4b… jetzt korrekt`expired/DEFINITIVE_NONSTART`, Grantnull, revision0,
|
||||
unlockfalse;26Jobs/26Outbox/23Grants/0offen unverändert. Keine Repair-CLI,
|
||||
manuelle DB-/Outcomeänderung oder erneute Ablaufwirkung.
|
||||
Private Belege`devstate/cloud-apt-recovery/nonstart-agent-install.json`,
|
||||
`agent-nonstart-sync.json` und`nonstart-sync-native.json`.
|
||||
UI-Entzug, Crash und Reboot bleiben offen.
|
||||
|
||||
## Nach Ablauf gefundene Sync-Klassifikationslücke
|
||||
|
||||
Vor dem nächsten Fall stoppte der Bereitschaftsguard; Fall09fc228c-e339-409a-
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
# Übergabe / Entwicklungspause — 2026-11-12
|
||||
|
||||
**Aktuellster nativer Stand:** Agent702bd7b5 korrigiert wiederholten Nichtstart-
|
||||
Sync nach abgelaufenem grantlosen Auftrag. Erhaltend installiert; normaler
|
||||
Generation93-Sync ready/restore_blockedfalse, gleicher Ablaufjob korrekt
|
||||
expired/DEFINITIVE_NONSTART, kein Grant/Unlock. Zentrale26/26/23/0, Root15closed.
|
||||
Keine Server-/Helper-/Provideränderung, M4Runtime3bac bleibt gültig. Aktiver
|
||||
Offline-Warte- und Ablauffall sowie MCP-Vorbericht bestanden; Generalautostart
|
||||
nachgewiesen aktiviert, öffentlicher Pinbootstrapfinalisiert. Noch UI-Entzug,
|
||||
Crash und letzter einzelner Reboot plus Endaudit. Ältere Sperrstände historisch.
|
||||
|
||||
**Neuester abgeschlossener Stand:** M4-APT im begrenzten Fixtureumfang bestanden:
|
||||
echtes1.0→1.1-Upgrade plus eigene ensure_package-/apply_updates-Noops, feste
|
||||
WebUI-Grenze jeweils0/0/0→1/1/1/0. Tatsächliche Paketnachbedingungen, true→true
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
# Tatsächlicher Implementierungsstand
|
||||
|
||||
## Nichtstart beim Folgesync nativ stabil erhalten — 2026-11-13
|
||||
|
||||
Minimaler Agentfix für grantfreie expired/cancelled-Nichtstarts,20 echte
|
||||
SQLite-/Signatur-Reconciletests/0Skips, Format und Alltargets-Clippy bestanden.
|
||||
Vorhandene Verbrauchszeilen haben Vorrang; unklare Wirkungen bleiben gesperrt.
|
||||
Neuer Bookworm-Agent702bd7b5293a38b3d910d9b38401a155dd5bccb88d43fa3af807d2094c70caa3
|
||||
erhaltend auf Cloud-VM installiert. Gewöhnlicher frischer Kanal Generation93
|
||||
übernahm die vorhandenen letzten gültigen/pending Anker ohne Reparatur/Reset.
|
||||
Signierter Sync wieder ready, restore_blockedfalse; ursprünglicher Ablaufjob
|
||||
korrekt expired/DEFINITIVE_NONSTART mitnullGrant/revision0/unlockfalse.
|
||||
Zentrale26/26/23/0 unverändert; keine erneute Wirkung, kein neuer Plan. Helper,
|
||||
Provider und native1.1-Acceptanceruntime3bac unverändert. Nächste Arbeit eigener
|
||||
UI-Entzugsfall, Crash, Reboot zuletzt und Endaudit.
|
||||
|
||||
## Nichtstart nach Ablauf beim Folgesync falsch klassifiziert — 2026-11-13
|
||||
|
||||
Vor UI-Entzugsplan blockiert korrekter Readinessguard, noch kein neuer Antrag.
|
||||
|
||||
Reference in New Issue
Block a user