Preserve timely reboot evidence and document blocked original reconciliation
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 22:03:24 +01:00
parent 80906a3eb6
commit 4102b97fb9
9 changed files with 387 additions and 13 deletions
+14 -1
View File
@@ -1,6 +1,6 @@
# Rust RMM: Architektur und Implementierungsspezifikation # Rust RMM: Architektur und Implementierungsspezifikation
Version 1.13 · 2026-11-13 · Sprache: Deutsch Version 1.14 · 2026-11-13 · Sprache: Deutsch
**Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam. **Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam.
@@ -43,6 +43,19 @@ Version 1.13 · 2026-11-13 · Sprache: Deutsch
## 0 Auftrag und verbindliche Produktanforderungen ## 0 Auftrag und verbindliche Produktanforderungen
**Präzisierung Version1.14, zeitgebundener Rebootnachweis:** Die ursprüngliche
Wiederverbindungsfrist ab serverseitiger Startreservierung bleibt unverändert.
Ein tatsächlich vor Fristende unabhängig empfangener neuer Boot-Inventarbeleg
bleibt als unveränderlicher rechtzeitiger Nachweis verwendbar. Erfolgt seine
Erstverifikation später als fünf Minuten nach Empfang, braucht sie zusätzlich
einen unabhängig empfangenen, höchstens fünf Minuten alten Inventarbeleg
derselben aktuellen Boot-ID sowie den aktuell authentisierten, nicht
widerrufenen Geräte-/Mandanten-/Generationskontext. Beide konkreten Beleg-IDs
und Serverempfangszeiten sind im signierten Ergebnis und Audit zu binden.
Fehlender ursprünglicher Fristbeleg bleibt unknown; ein erst nach Fristende
empfangener Beleg wird durch spätere Aktualität niemals rechtzeitig. Keine
Verlängerung, Umdatierung, neue Wirkung oder Wiederholungserlaubnis.
**Präzisierung Version1.13, begrenzte MVP-Nachbedingungen:** Ein bereits vor **Präzisierung Version1.13, begrenzte MVP-Nachbedingungen:** Ein bereits vor
einer Paketwirkung vorhandener Host-Neustartbedarf verhindert deren Erfolg einer Paketwirkung vorhandener Host-Neustartbedarf verhindert deren Erfolg
nicht, wenn sämtliche Paketnachbedingungen unabhängig vollständig nachgewiesen nicht, wenn sämtliche Paketnachbedingungen unabhängig vollständig nachgewiesen
+175 -8
View File
@@ -92,6 +92,18 @@ struct RebootWindow {
now: chrono::DateTime<Utc>, now: chrono::DateTime<Utc>,
require_fresh: bool, require_fresh: bool,
} }
async fn reboot_fresh_inventory(
tx: &mut Transaction<'_, Postgres>,
tenant: Uuid,
agent: Uuid,
boot: Uuid,
now: chrono::DateTime<Utc>,
receipt: Option<Uuid>,
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt)
.fetch_optional(&mut **tx).await?)
}
async fn reboot_inventory( async fn reboot_inventory(
tx: &mut Transaction<'_, Postgres>, tx: &mut Transaction<'_, Postgres>,
tenant: Uuid, tenant: Uuid,
@@ -103,9 +115,21 @@ async fn reboot_inventory(
// Already-persisted definitive success keeps its original receipt. It need // Already-persisted definitive success keeps its original receipt. It need
// not be younger than five minutes again, but its original server receipt // not be younger than five minutes again, but its original server receipt
// time must still prove the approved reconnect window and current boot. // time must still prove the approved reconnect window and current boot.
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND (NOT $9::boolean OR received_at>$6) AND received_at<=$7 AND ($8::uuid IS NULL OR message_id=$8) ORDER BY received_at DESC LIMIT 1") let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1")
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now-Duration::minutes(5)).bind(window.now).bind(existing_receipt).bind(window.require_fresh) .bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt)
.fetch_optional(&mut **tx).await?) .fetch_optional(&mut **tx).await?;
// Verification may be delayed by local bootstrap readiness. Never move
// the original reconnect cutoff: retain the actual timely receipt and
// independently demand fresh evidence of the SAME current boot.
if timely.is_some()
&& window.require_fresh
&& reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None)
.await?
.is_none()
{
return Ok(None);
}
Ok(timely)
} }
/// First ingress of success needs the same independent, timely receipt as an /// First ingress of success needs the same independent, timely receipt as an
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing. /// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
@@ -120,6 +144,7 @@ pub(crate) async fn verify_reboot_success(
) -> Result<()> { ) -> Result<()> {
let boot = Uuid::parse_str(&envelope.boot_id)?; let boot = Uuid::parse_str(&envelope.boot_id)?;
let grant = Uuid::parse_str(&result.grant_id)?; let grant = Uuid::parse_str(&result.grant_id)?;
let now = Utc::now();
ensure!( ensure!(
job.tenant_id == tenant job.tenant_id == tenant
&& job.agent_id == agent && job.agent_id == agent
@@ -130,6 +155,9 @@ pub(crate) async fn verify_reboot_success(
&& evidence["new_boot_id"] == envelope.boot_id, && evidence["new_boot_id"] == envelope.boot_id,
"REBOOT_RESULT_BINDING" "REBOOT_RESULT_BINDING"
); );
let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)")
.bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?;
ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED");
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4") let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?; .bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
let deadline = let deadline =
@@ -156,13 +184,41 @@ pub(crate) async fn verify_reboot_success(
&RebootWindow { &RebootWindow {
reserved_at: reserved, reserved_at: reserved,
deadline, deadline,
now: Utc::now(), now,
require_fresh: true, require_fresh: true,
}, },
Some(receipt), Some(receipt),
) )
.await? .await?
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?; .context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
let verification_id = evidence
.get("verification_inventory_message_id")
.and_then(Value::as_str)
.map(Uuid::parse_str)
.transpose()?;
let verification_time = evidence.get("verification_inventory_received_at");
let verification = if verification_id.is_some()
|| verification_time.is_some()
|| received <= now - Duration::minutes(5)
{
let id = verification_id
.filter(|id| !id.is_nil())
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id))
.await?
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
ensure!(
serde_json::from_value::<chrono::DateTime<Utc>>(
verification_time.cloned().unwrap_or(Value::Null)
)
.ok()
== Some(time),
"REBOOT_VERIFICATION_TIME_BINDING"
);
(id, time)
} else {
(receipt, received)
};
for (key, actual) in [ for (key, actual) in [
("reboot_reserved_at", reserved), ("reboot_reserved_at", reserved),
("reconnect_deadline", deadline), ("reconnect_deadline", deadline),
@@ -174,6 +230,11 @@ pub(crate) async fn verify_reboot_success(
"REBOOT_SERVER_TIME_BINDING" "REBOOT_SERVER_TIME_BINDING"
); );
} }
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())")
.bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot,
"connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline,
"inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1}))
.execute(&mut **tx).await?;
Ok(()) Ok(())
} }
pub async fn reconcile( pub async fn reconcile(
@@ -587,8 +648,10 @@ pub async fn reconcile(
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?) && decision.original_boot_id == Some(Uuid::parse_str(&original)?)
{ {
reboot_witness = Some((original, inventory)); reboot_witness = Some((original, inventory));
let verification =
reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?;
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1; let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at}).to_string(),error_code:String::new()}; let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()};
let mut evidence = e.clone(); let mut evidence = e.clone();
evidence.message_id = Uuid::new_v4().to_string(); evidence.message_id = Uuid::new_v4().to_string();
evidence.payload = Some(Payload::JobResult(result)); evidence.payload = Some(Payload::JobResult(result));
@@ -877,6 +940,61 @@ mod tests {
.map(|v| v.0), .map(|v| v.0),
Some(timely) Some(timely)
); );
let fresh = Uuid::new_v4();
let future = Uuid::new_v4();
for (sequence, message, received) in [
(4, fresh, later.now - Duration::seconds(1)),
(5, future, later.now + Duration::days(1)),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap();
}
// Delayed first verification keeps the original cutoff-bound ID and
// uses a DIFFERENT fresh same-boot receipt; late-only never suffices.
assert_eq!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
.await
.unwrap()
.map(|v| v.0),
Some(timely)
);
assert!(
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
.await
.unwrap()
.is_none()
);
assert_eq!(
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None)
.await
.unwrap()
.map(|v| v.0),
Some(fresh)
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future))
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None)
.await
.unwrap()
.is_none()
);
assert!(
reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None)
.await
.unwrap()
.is_none()
);
let before_return = RebootWindow { let before_return = RebootWindow {
now: deadline - Duration::microseconds(1), now: deadline - Duration::microseconds(1),
..window ..window
@@ -915,7 +1033,7 @@ mod tests {
let authorization = Uuid::new_v4(); let authorization = Uuid::new_v4();
let grant = Uuid::new_v4(); let grant = Uuid::new_v4();
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap(); let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
let reserved = now - Duration::seconds(200); let reserved = now - Duration::seconds(600);
let deadline = reserved + Duration::seconds(180); let deadline = reserved + Duration::seconds(180);
let payload = JobPayload { let payload = JobPayload {
schema_version: 1, schema_version: 1,
@@ -949,7 +1067,7 @@ mod tests {
.execute(&admin) .execute(&admin)
.await .await
.unwrap(); .unwrap();
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
@@ -984,14 +1102,26 @@ mod tests {
.unwrap(); .unwrap();
let timely = Uuid::new_v4(); let timely = Uuid::new_v4();
let late = Uuid::new_v4(); let late = Uuid::new_v4();
let fresh = Uuid::new_v4();
let wrong_boot = Uuid::new_v4();
let future = Uuid::new_v4();
for (sequence, message, received) in [ for (sequence, message, received) in [
(1, timely, deadline), (1, timely, deadline),
(2, late, deadline + Duration::microseconds(1)), (2, late, deadline + Duration::microseconds(1)),
] { ] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap(); sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
} }
for (sequence, message, received, observed_boot) in [
(3, fresh, now - Duration::seconds(1), boot),
(4, wrong_boot, now - Duration::seconds(1), old),
(5, future, now + Duration::days(1), boot),
] {
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
.bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
}
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late, let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline}); "reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline,
"verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)});
let mut result = JobResult { let mut result = JobResult {
job_id: job.to_string(), job_id: job.to_string(),
attempt_id: attempt.to_string(), attempt_id: attempt.to_string(),
@@ -1004,6 +1134,7 @@ mod tests {
let mut envelope = Envelope { let mut envelope = Envelope {
message_id: Uuid::new_v4().to_string(), message_id: Uuid::new_v4().to_string(),
agent_id: agent.to_string(), agent_id: agent.to_string(),
connection_generation: 1,
boot_id: boot.to_string(), boot_id: boot.to_string(),
payload: Some(Payload::JobResult(result.clone())), payload: Some(Payload::JobResult(result.clone())),
..Default::default() ..Default::default()
@@ -1031,6 +1162,32 @@ mod tests {
); );
let mut good = evidence.clone(); let mut good = evidence.clone();
good["inventory_message_id"] = json!(timely); good["inventory_message_id"] = json!(timely);
for (id, time) in [
(Some(wrong_boot), Some(now - Duration::seconds(1))),
(Some(future), Some(now + Duration::days(1))),
(None, None),
(Some(fresh), Some(now)),
] {
let mut bad = good.clone();
bad["verification_inventory_message_id"] = json!(id);
bad["verification_inventory_received_at"] = json!(time);
result.evidence_json = bad.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
}
envelope.connection_generation = 2;
result.evidence_json = good.to_string();
envelope.payload = Some(Payload::JobResult(result.clone()));
assert!(
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
.await
.is_err()
);
envelope.connection_generation = 1;
let mut false_clock = good.clone(); let mut false_clock = good.clone();
false_clock["reconnect_deadline"] = json!(now + Duration::days(1)); false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
result.evidence_json = false_clock.to_string(); result.evidence_json = false_clock.to_string();
@@ -1085,6 +1242,16 @@ mod tests {
.unwrap(), .unwrap(),
1 1
); );
let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'")
.bind(tenant).fetch_one(&mut *tx).await.unwrap();
assert_eq!(audit["inventory_message_id"], json!(timely));
assert_eq!(audit["inventory_received_at"], json!(deadline));
assert_eq!(audit["verification_inventory_message_id"], json!(fresh));
assert_eq!(
audit["verification_inventory_received_at"],
json!(now - Duration::seconds(1))
);
assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1);
tx.rollback().await.unwrap(); tx.rollback().await.unwrap();
} }
#[test] #[test]
+24
View File
@@ -243,6 +243,30 @@ def legacy_helper_unit_metadata(payload):
clock_query_filters=[line for line in lines if line in ('SystemCallFilter=adjtimex clock_adjtime','SystemCallFilter=adjtimex')]) clock_query_filters=[line for line in lines if line in ('SystemCallFilter=adjtimex clock_adjtime','SystemCallFilter=adjtimex')])
def start_helper_after_boot_clock(payload):
"""Start the existing helper after its clock guard failed; never a job."""
import contextlib
import sqlite3
unit=Path('/etc/systemd/system/rmm-helper-cloud.service')
require(hashlib.sha256(unit.read_bytes()).hexdigest()==payload['unit_sha256']
and hashlib.sha256(Path('/usr/local/libexec/rmm-privileged-helper').read_bytes()).hexdigest()==payload['helper_sha256'],'same qualified helper/unit')
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
require(boot!=payload['old_boot_id'],'actual guest already rebooted; never request another reboot')
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy ledger')
rows=db.execute('SELECT c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows)==1 and rows[0][:2]==(payload['job_id'],payload['attempt_id'])
and json.loads(rows[0][2])['state']=='awaiting_reboot','only exact existing reboot may await reconciliation')
before=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()
require(before==b'failed','known boot-start failure only')
subprocess.run(['systemctl','start','rmm-helper-cloud.service'],capture_output=True,check=True,timeout=20)
active=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()==b'active'
return dict(started_existing_helper=active,new_boot_id=boot,ledger_reset=False,
jobs_created=False,grants_created=False,reboot_requested=False)
def disable_legacy_helper_autostart(payload): def disable_legacy_helper_autostart(payload):
"""Disable only the stopped historical dev helper; never stop/start a unit.""" """Disable only the stopped historical dev helper; never stop/start a unit."""
import contextlib import contextlib
+5 -1
View File
@@ -254,7 +254,11 @@ def close(ident):
m=cloud.contract;folder=STATE/str(uuid.UUID(ident));require(m.protected_json(folder/'result.json')['verified'],'normal verified reconnect required before rollback') m=cloud.contract;folder=STATE/str(uuid.UUID(ident));require(m.protected_json(folder/'result.json')['verified'],'normal verified reconnect required before rollback')
session=m.protected_json(folder/'before.json');target=session['target'];opened=m.protected_json(folder/'policy-open.json');enabled=m.protected_json(folder/'helper-enabled.json') session=m.protected_json(folder/'before.json');target=session['target'];opened=m.protected_json(folder/'policy-open.json');enabled=m.protected_json(folder/'helper-enabled.json')
require(evidence(target,'gate')['unresolved']==0,'all effects closed') require(evidence(target,'gate')['unresolved']==0,'all effects closed')
closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256'])) earlier=folder/'policy-closed-for-upgrade.json'
if earlier.exists():
closed=m.protected_json(earlier);current=cloud.bootstrap.rpc('server.conf',server_policy,dict(action='state'))
require(current['closed'] and current['policy_sha256']==closed['policy_sha256'],'exact earlier deny-only closure retained')
else:closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
m.write_private(folder/'policy-closed.json',closed) m.write_private(folder/'policy-closed.json',closed)
restored=cloud.bootstrap.rpc('linux-vm.conf',vm_policy,dict(session['pins'],id=ident,enabled=False,config_sha256=enabled['config_sha256'],pin_sha256=enabled['pin_sha256'],original_backup=enabled['backup'])) restored=cloud.bootstrap.rpc('linux-vm.conf',vm_policy,dict(session['pins'],id=ident,enabled=False,config_sha256=enabled['config_sha256'],pin_sha256=enabled['pin_sha256'],original_backup=enabled['backup']))
require(restored['general_pin']==session['before']['general_pin'] and restored['config_sha256']==session['before']['config_sha256'] and restored['pin_sha256']==session['before']['pin_sha256'] and restored['preserved_state'] and not restored['reboot_requested'],'original non-reboot policy/pin bytes restored without another reboot') require(restored['general_pin']==session['before']['general_pin'] and restored['config_sha256']==session['before']['config_sha256'] and restored['pin_sha256']==session['before']['pin_sha256'] and restored['preserved_state'] and not restored['reboot_requested'],'original non-reboot policy/pin bytes restored without another reboot')
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Preserve the one consumed reboot while updating its read-only verifier."""
import argparse
import base64
import hashlib
import io
import json
from pathlib import Path
import tarfile
import cloud_reboot_acceptance as reboot
import cloud_server_upgrade as upgrade
cloud=reboot.cloud
require=cloud.contract.require
def agent_transport(payload):
import contextlib
import sqlite3
require(payload['action'] in ('stop','start'),'own agent transport direction')
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'qualified agent')
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
require(boot==payload['new_boot_id'] and boot!=payload['old_boot_id'],'already completed one guest boot')
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy Root ledger')
rows=db.execute('SELECT c.grant_id,c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows)==1 and rows[0][:3]==(payload['grant_id'],payload['job_id'],payload['attempt_id']) and json.loads(rows[0][3])['state']=='awaiting_reboot','only original consumed reboot may await proof')
subprocess.run(['systemctl',payload['action'],'rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
return dict(action=payload['action'],new_boot_id=boot,jobs_created=False,grants_created=False,reboot_requested=False)
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--id',required=True);p.add_argument('--reboot-id',required=True);args=p.parse_args()
import uuid
candidate=str(uuid.UUID(args.id));sid=str(uuid.UUID(args.reboot_id));folder=reboot.STATE/sid
session=cloud.contract.protected_json(folder/'before.json');checkpoint=cloud.contract.protected_json(folder/'checkpoint.json');job=checkpoint['job'];target=session['target']
require(sid=='d015c3ef-f3aa-4522-a3c2-fcb6099df17b' and job['job_id']=='8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa','one exact already approved reboot')
gate=lambda:reboot.evidence(target,'gate')
require(gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1),'only known consumed reboot unresolved')
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
grant='25a4bf3f-772c-487d-825d-73bcbbc418fd'
require(current['state'] in ('unknown','awaiting_reboot') and current['result']['grant_id']==grant,'same immutable reboot attempt')
scope=dict(session['pins'],job_id=job['job_id'],attempt_id=job['attempt_id'])
before=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
require(before['root_unresolved']==1 and before['other_unresolved']==0 and before['selected_root_grants']==[grant]
and before['boot_id']!=session['old_boot_id'] and len(before['root_rows'])==len(session['before']['root_rows'])+1
and all(before['root_rows'].get(k)==v for k,v in session['before']['root_rows'].items()),'all old Root rows and exact consumed reboot preserved')
audit=reboot.evidence(target,'audit',operation_id=checkpoint['operation_id'])
require(audit==dict(origin='dev_test_browser_simulation',simulation=True,principal='simulated-testprincipal-a',plan_hash=checkpoint['plan']['plan_hash']),'original actual UI approval')
status=cloud.bootstrap.rpc('server.conf',upgrade.status,dict(id=candidate))
require(status['finished'] and status['exit_code']==0 and status['source_unchanged'] and status['runtime_tag_unchanged'],'qualified separate server image')
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(action='state'))
if not policy['closed']:
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(id=sid,action='close',previous_sha256=policy['policy_sha256']))
cloud.contract.write_private(folder/'policy-closed-for-upgrade.json',policy)
transport=dict(agent_id=target['agent_id'],agent_sha256=session['pins']['agent_sha256'],old_boot_id=session['old_boot_id'],new_boot_id=before['boot_id'],grant_id=grant,job_id=job['job_id'],attempt_id=job['attempt_id'])
stopped=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='stop'))
frozen=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
require(frozen['root_rows']==before['root_rows'] and frozen['database_inodes']==before['database_inodes'] and frozen['services']['rmm-agent-remote.service']['ActiveState']=='inactive','quiesced agent and original evidence retained')
cloud.contract.write_private(folder/'proof-upgrade-before.json',dict(candidate=candidate,gate=gate(),current=current,native=frozen,transport=stopped,policy=policy))
stream=io.BytesIO()
with tarfile.open(fileobj=stream,mode='w:') as archive:
for path in sorted((cloud.ROOT/'web/dist').rglob('*')):
if path.is_file():
require(not path.is_symlink(),'built UI symlink');archive.add(path,arcname=str(path.relative_to(cloud.ROOT/'web/dist')),recursive=False)
raw=stream.getvalue()
result=cloud.bootstrap.rpc('server.conf',upgrade.deploy,dict(id=candidate,closed_policy_sha256=policy['policy_sha256'],previous_ui_index_sha256='a3513f928e20ea11d53f1cd27e8735d911555e1c66d5d97ca8bdefaff60e71fa',web_tar=base64.b64encode(raw).decode(),web_sha256=hashlib.sha256(raw).hexdigest()))
after=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
require(result['deployed'] and gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1) and after['root_rows']==frozen['root_rows'] and after['database_inodes']==frozen['database_inodes'],'deployment preserves exact unresolved effect before normal resumption')
cloud.contract.write_private(folder/'proof-upgrade-deployed.json',dict(verified=True,candidate=candidate,result=result,native=after,counts=gate()))
resumed=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='start'))
cloud.contract.write_private(folder/'proof-upgrade-resumed.json',resumed)
print('PASS exact reboot evidence preserved during qualified server upgrade; ordinary signed resumption required; no second reboot')
if __name__=='__main__':
try:main()
except Exception as error:
message=str(error);print(message if message.startswith('remote bootstrap incomplete: ') else 'REBOOT_PROOF_UPGRADE_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+11
View File
@@ -11,6 +11,17 @@ import cloud_mvp_acceptance as cloud
require = cloud.contract.require require = cloud.contract.require
def reboot_receipts(payload):
tenant=str(uuid.UUID(payload['tenant_id']));agent=str(uuid.UUID(payload['agent_id']));job=str(uuid.UUID(payload['job_id']))
require(tenant=='11111111-1111-4111-8111-111111111111','fixed development tenant')
query=("BEGIN READ ONLY; SELECT json_build_object('observed_at',now(),'installation',(SELECT json_build_object('boot_id',connection_boot_id,'generation',connection_generation,'last_contact_at',last_contact_at,'sync_ready',sync_ready,'helper_ready',helper_sync_ready) FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'),"
"'job',(SELECT json_build_object('state',state,'revision',revision,'result',result) FROM jobs WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'grant',(SELECT json_build_object('grant_id',grant_id,'created_at',created_at) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'inventories',(SELECT COALESCE(json_agg(x),'[]'::json) FROM (SELECT message_id,boot_id,observed_at,received_at,source_id FROM observations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' ORDER BY received_at DESC LIMIT 20) x)); COMMIT;")
root=Path('/opt/rust-discovery');cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
def server(payload): def server(payload):
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id'])) tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant') require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
+29
View File
@@ -1,5 +1,34 @@
# Tatsächlicher Implementierungsstand # Tatsächlicher Implementierungsstand
## Original-Reboot erfolgt; Abschluss noch gesperrt — 2026-11-13
Der feste WebUI-genehmigte Job `8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa`
verbrauchte genau Grant `25a4bf3f-772c-487d-825d-73bcbbc418fd`. Tatsächlicher
neuer Boot und mTLS-Inventare nach etwa88/148Sekunden liegen innerhalb der
unveränderten180Sekunden ab zentraler Reservierung20:26:12.430245UTC.
General startete zunächst mit `CLOCK_UNTRUSTED` nicht; die bestehende Unit
`Restart=no` konnte nicht selbst wieder anlaufen. Nach nachgewiesen vertrauter
Kerneluhr wurde ausschließlich derselbe Helper gestartet. Keine zweite
Rebootwirkung, keine Journal-/Grant-/Identitätsrücksetzung.
Servercandidate `7be1f330-4bbd-4801-b72a-c35ce7ac479e` nativ ARM64 gebaut und
erhaltend deployt. Spec1.14 präzisiert spätere Erstverifikation: unveränderlicher
rechtzeitiger Originalbeleg **und** separater frischer Sameboot-Inventarbeleg
unter aktuellem authentisierten Kanal. Drei isolierte echte PostgreSQL-Tests,
Format und Clippy bestanden. Alle30Jobs/30Outbox/26Grants, Root18 und alten
DBinodes/-Verbrauchszeilen beim Deployment erhalten; UI unverändert. Dies ist
noch keine vollständige Rebootabnahme: Abschluss weiterhin unknown.
Danach kein neuer SyncRequest, obwohl Heartbeats/Inventare fortliefen.
Rootjournal tatsächlich268414976Bytes nahe256MiB-Grenze; Meta blieb1378,
Agent meldet nur generischen Syncabhängigkeitsfehler. Sourcepfad schreibt
Helpercapture vor Agentrequest. Agent exakt an denselben Originalversuch
gebunden administrativ gestoppt, Helper aktiv/Rootfaultfrei. Kapazitätsursache
wird an eigener DBkopie geprüft; keine Historylöschung oder Wiederholung.
Erneuter `finish`-Leseversuch endete ohne Erfolgsnachweis. Noch offen sind
Abschluss desselben signierten Rebootabgleichs, Startretrykorrektur, MCP-Endbericht
und Endaudit. Details:[reboot-acceptance.md](reboot-acceptance.md).
## Kontrollierter Agentabbruch nativ vollständig abgeglichen — 2026-11-13 ## Kontrollierter Agentabbruch nativ vollständig abgeglichen — 2026-11-13
Eigener Cloudfall644c06eb-24b0-41b0-9a90-f60ea1fe38a9 bestanden. Tatsächlicher Eigener Cloudfall644c06eb-24b0-41b0-9a90-f60ea1fe38a9 bestanden. Tatsächlicher
+16 -1
View File
@@ -1,6 +1,21 @@
# Verbindliche Projektentscheidungen # Verbindliche Projektentscheidungen
Stand: **2026-11-13**, Spezifikation **1.13** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung. Stand: **2026-11-13**, Spezifikation **1.14** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
## Rechtzeitiger Rebootkontakt und spätere frische Verifikation (Version1.14)
Die feste180s-MVP-Frist wird nicht verändert. Ein rechtzeitig gespeicherter
Originalbeleg wird bei späterer Erstverifikation zusammen mit einem zweiten,
frischen Beleg derselben aktuellen Boot-ID und dem tatsächlichen aktuellen
authentisierten Gerätekontext geprüft. Signiertes Ergebnis bindet Original-
`inventory_message_id`/`inventory_received_at` und zusätzliche
`verification_inventory_message_id`/`verification_inventory_received_at`;
Audit bindet Job/Versuch/Grant/Boot/Generation und beide Datenbankbelege.
Verspäteter Originalkontakt, falscher Boot/Kunde/Agent/Generation oder fehlende
aktuelle Verifikation bleiben verweigert. Grundlage: tatsächlicher rechtzeitiger
Rebootkontakt bei zunächst clock-bedingt ausgefallenem Helper, keine neue
Rebootfreigabe oder erfundene rückwirkende Fristeinhaltung. Details:
[Rebootabnahme](reboot-acceptance.md).
## Gebundene Paket- und Rebootnachbedingungen (Version1.13) ## Gebundene Paket- und Rebootnachbedingungen (Version1.13)
+32 -2
View File
@@ -1,5 +1,31 @@
# Letzte Linux-Cloud-VM-Rebootabnahme # Letzte Linux-Cloud-VM-Rebootabnahme
## Aktueller tatsächlicher Stand2026-11-13
Der einzelne feste WebUI-Job8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa mit
Grant25a4bf3f-772c-487d-825d-73bcbbc418fd wurde tatsächlich einmal ausgeführt.
Neue Boot-ID45aba743-be1d-4ae5-ba00-139fb9363bfe statt
603e9280-801d-403e-bbfe-7340784e000a, zentrale Reservierung20:26:12.430245UTC,
Originalfrist20:29:12.430245UTC. Tatsächliche neue Bootinventare
6f5bd8a6-a33b-4e45-a102-63d43e12c23d um20:27:40.878097UTC und
1cf6146c-15fb-491f-a35a-8c4a0bb8f679 um20:28:40.877951UTC sind rechtzeitig.
Kein zweiter Neustart ist autorisiert oder vorgesehen.
Noch **unknown**, nicht abgenommen: General scheiterte beim Boot an
CLOCK_UNTRUSTED/Restart=no. Nach realer Uhrbereitschaft derselbe Helper
operativ gestartet. Spec1.14/Zwei-Beleg-Serverfix unter Candidate7be1f330
nativ erhaltend deployt, drei isolierte PostgreSQL-Tests/Clippy/Format bestanden.
Originalfrist, alte Daten/Grants/DBinodes und UI blieben erhalten. Erneuter
Finishleselauf konnte noch keinen signierten Erfolg belegen. Seitdem
Rootjournal268414976Bytes nahe256MiB, keine neuen Helpercaptures/Agentrequests.
Agent für begrenzte Kopiediagnose angehalten; keine Ledgerbereinigung.
MVP bleibt offen. Private Belege im unten genannten festen Fallordner,
zusätzlich proof-upgrade-{before,deployed,resumed}.json und
reconcile-diagnostic-quiesced.json.
Die folgenden Vorbereitungsabschnitte und Formulierungen „noch nicht ausgeführt“
sind historische Vorstufen, kein aktueller Wirkungsstand.
Vor dem tatsächlichen Reboot neun bereits verifizierte frühere Gates an feste Vor dem tatsächlichen Reboot neun bereits verifizierte frühere Gates an feste
JSON-/SHA-Referenzen gebunden. Root17/0offen/faultfrei, Zentrale29/29/25/0, JSON-/SHA-Referenzen gebunden. Root17/0offen/faultfrei, Zentrale29/29/25/0,
Agent/General aktiv+enabled, APTinactive/disabled und Endpunkt entfernt. Agent/General aktiv+enabled, APTinactive/disabled und Endpunkt entfernt.
@@ -34,8 +60,12 @@ neue Verbindungsversuche und neue Inventare verlängern sie ebenfalls nicht.
Erstmaliger Erfolg braucht einen tatsächlich gespeicherten Inventarbeleg Erstmaliger Erfolg braucht einen tatsächlich gespeicherten Inventarbeleg
derselben Identität und des aktuellen neuen Boot-IDs. Dessen serverseitige derselben Identität und des aktuellen neuen Boot-IDs. Dessen serverseitige
Empfangszeit muss **nach** der Startreservierung und **spätestens zur Frist** Empfangszeit muss **nach** der Startreservierung und **spätestens zur Frist**
liegen. Der Beleg muss bei erstmaliger Abnahme außerdem jünger als fünf Minuten liegen. Bei später als fünf Minuten erfolgender Erstverifikation braucht der
sein; die alte Boot-ID muss aus dem zum gleichen Grant gehörenden unveränderliche rechtzeitige Originalbeleg einen zusätzlichen, unabhängig
empfangenen aktuellen Sameboot-Beleg jünger als fünf Minuten und den aktuellen
authentisierten Geräte-/Generationskontext. Beide tatsächlichen IDs und
Empfangszeiten werden im signierten Ergebnis und Audit gebunden. Ein erst
verspätet empfangener Originalbeleg bleibt verweigert. Die alte Boot-ID muss aus dem zum gleichen Grant gehörenden
`start_reserved`-Audit stammen und vom neuen Boot abweichen. Fehlende oder zu `start_reserved`-Audit stammen und vom neuen Boot abweichen. Fehlende oder zu
späte Belege bleiben `unknown`; daraus entsteht weder ein geratenes `failed` späte Belege bleiben `unknown`; daraus entsteht weder ein geratenes `failed`
noch eine Wiederholungserlaubnis. noch eine Wiederholungserlaubnis.