Preserve timely reboot evidence and document blocked original reconciliation
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Rust RMM: Architektur und Implementierungsspezifikation
|
||||
|
||||
Version 1.13 · 2026-11-13 · Sprache: Deutsch
|
||||
Version 1.14 · 2026-11-13 · Sprache: Deutsch
|
||||
|
||||
**Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam.
|
||||
|
||||
@@ -43,6 +43,19 @@ Version 1.13 · 2026-11-13 · Sprache: Deutsch
|
||||
|
||||
## 0 Auftrag und verbindliche Produktanforderungen
|
||||
|
||||
**Präzisierung Version1.14, zeitgebundener Rebootnachweis:** Die ursprüngliche
|
||||
Wiederverbindungsfrist ab serverseitiger Startreservierung bleibt unverändert.
|
||||
Ein tatsächlich vor Fristende unabhängig empfangener neuer Boot-Inventarbeleg
|
||||
bleibt als unveränderlicher rechtzeitiger Nachweis verwendbar. Erfolgt seine
|
||||
Erstverifikation später als fünf Minuten nach Empfang, braucht sie zusätzlich
|
||||
einen unabhängig empfangenen, höchstens fünf Minuten alten Inventarbeleg
|
||||
derselben aktuellen Boot-ID sowie den aktuell authentisierten, nicht
|
||||
widerrufenen Geräte-/Mandanten-/Generationskontext. Beide konkreten Beleg-IDs
|
||||
und Serverempfangszeiten sind im signierten Ergebnis und Audit zu binden.
|
||||
Fehlender ursprünglicher Fristbeleg bleibt unknown; ein erst nach Fristende
|
||||
empfangener Beleg wird durch spätere Aktualität niemals rechtzeitig. Keine
|
||||
Verlängerung, Umdatierung, neue Wirkung oder Wiederholungserlaubnis.
|
||||
|
||||
**Präzisierung Version1.13, begrenzte MVP-Nachbedingungen:** Ein bereits vor
|
||||
einer Paketwirkung vorhandener Host-Neustartbedarf verhindert deren Erfolg
|
||||
nicht, wenn sämtliche Paketnachbedingungen unabhängig vollständig nachgewiesen
|
||||
|
||||
@@ -92,6 +92,18 @@ struct RebootWindow {
|
||||
now: chrono::DateTime<Utc>,
|
||||
require_fresh: bool,
|
||||
}
|
||||
async fn reboot_fresh_inventory(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
boot: Uuid,
|
||||
now: chrono::DateTime<Utc>,
|
||||
receipt: Option<Uuid>,
|
||||
) -> Result<Option<(Uuid, chrono::DateTime<Utc>)>> {
|
||||
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND ($6::uuid IS NULL OR message_id=$6) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(now-Duration::minutes(5)).bind(now).bind(receipt)
|
||||
.fetch_optional(&mut **tx).await?)
|
||||
}
|
||||
async fn reboot_inventory(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
tenant: Uuid,
|
||||
@@ -103,9 +115,21 @@ async fn reboot_inventory(
|
||||
// Already-persisted definitive success keeps its original receipt. It need
|
||||
// not be younger than five minutes again, but its original server receipt
|
||||
// time must still prove the approved reconnect window and current boot.
|
||||
Ok(sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND (NOT $9::boolean OR received_at>$6) AND received_at<=$7 AND ($8::uuid IS NULL OR message_id=$8) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now-Duration::minutes(5)).bind(window.now).bind(existing_receipt).bind(window.require_fresh)
|
||||
.fetch_optional(&mut **tx).await?)
|
||||
let timely = sqlx::query_as("SELECT message_id,received_at FROM observations WHERE tenant_id=$1 AND agent_id=$2 AND boot_id=$3 AND received_at>$4 AND received_at<=$5 AND received_at<=$6 AND ($7::uuid IS NULL OR message_id=$7) ORDER BY received_at DESC LIMIT 1")
|
||||
.bind(tenant).bind(agent).bind(boot).bind(window.reserved_at).bind(window.deadline).bind(window.now).bind(existing_receipt)
|
||||
.fetch_optional(&mut **tx).await?;
|
||||
// Verification may be delayed by local bootstrap readiness. Never move
|
||||
// the original reconnect cutoff: retain the actual timely receipt and
|
||||
// independently demand fresh evidence of the SAME current boot.
|
||||
if timely.is_some()
|
||||
&& window.require_fresh
|
||||
&& reboot_fresh_inventory(tx, tenant, agent, boot, window.now, None)
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(timely)
|
||||
}
|
||||
/// First ingress of success needs the same independent, timely receipt as an
|
||||
/// authoritative Sync decision. Claimed deadlines/boot IDs alone grant nothing.
|
||||
@@ -120,6 +144,7 @@ pub(crate) async fn verify_reboot_success(
|
||||
) -> Result<()> {
|
||||
let boot = Uuid::parse_str(&envelope.boot_id)?;
|
||||
let grant = Uuid::parse_str(&result.grant_id)?;
|
||||
let now = Utc::now();
|
||||
ensure!(
|
||||
job.tenant_id == tenant
|
||||
&& job.agent_id == agent
|
||||
@@ -130,6 +155,9 @@ pub(crate) async fn verify_reboot_success(
|
||||
&& evidence["new_boot_id"] == envelope.boot_id,
|
||||
"REBOOT_RESULT_BINDING"
|
||||
);
|
||||
let current:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND enrollment_generation=$4 AND assignment_version=$5 AND connection_generation=$6 AND connection_boot_id=$7 AND revoked_at IS NULL)")
|
||||
.bind(tenant).bind(agent).bind(job.asset_id).bind(i64::try_from(job.enrollment_generation)?).bind(i64::try_from(job.assignment_version)?).bind(i64::try_from(envelope.connection_generation)?).bind(boot).fetch_one(&mut **tx).await?;
|
||||
ensure!(current, "REBOOT_CURRENT_AUTHENTICATED_CHANNEL_REQUIRED");
|
||||
let reserved:chrono::DateTime<Utc>=sqlx::query_scalar("SELECT created_at FROM execution_permits WHERE tenant_id=$1 AND grant_id=$2 AND job_id=$3 AND attempt_id=$4")
|
||||
.bind(tenant).bind(grant).bind(job.job_id).bind(job.attempt_id).fetch_optional(&mut **tx).await?.context("REBOOT_RESERVED_GRANT_REQUIRED")?;
|
||||
let deadline =
|
||||
@@ -156,13 +184,41 @@ pub(crate) async fn verify_reboot_success(
|
||||
&RebootWindow {
|
||||
reserved_at: reserved,
|
||||
deadline,
|
||||
now: Utc::now(),
|
||||
now,
|
||||
require_fresh: true,
|
||||
},
|
||||
Some(receipt),
|
||||
)
|
||||
.await?
|
||||
.context("REBOOT_TIMELY_INVENTORY_REQUIRED")?;
|
||||
let verification_id = evidence
|
||||
.get("verification_inventory_message_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(Uuid::parse_str)
|
||||
.transpose()?;
|
||||
let verification_time = evidence.get("verification_inventory_received_at");
|
||||
let verification = if verification_id.is_some()
|
||||
|| verification_time.is_some()
|
||||
|| received <= now - Duration::minutes(5)
|
||||
{
|
||||
let id = verification_id
|
||||
.filter(|id| !id.is_nil())
|
||||
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
|
||||
let (id, time) = reboot_fresh_inventory(tx, tenant, agent, boot, now, Some(id))
|
||||
.await?
|
||||
.context("REBOOT_FRESH_VERIFICATION_REQUIRED")?;
|
||||
ensure!(
|
||||
serde_json::from_value::<chrono::DateTime<Utc>>(
|
||||
verification_time.cloned().unwrap_or(Value::Null)
|
||||
)
|
||||
.ok()
|
||||
== Some(time),
|
||||
"REBOOT_VERIFICATION_TIME_BINDING"
|
||||
);
|
||||
(id, time)
|
||||
} else {
|
||||
(receipt, received)
|
||||
};
|
||||
for (key, actual) in [
|
||||
("reboot_reserved_at", reserved),
|
||||
("reconnect_deadline", deadline),
|
||||
@@ -174,6 +230,11 @@ pub(crate) async fn verify_reboot_success(
|
||||
"REBOOT_SERVER_TIME_BINDING"
|
||||
);
|
||||
}
|
||||
sqlx::query("INSERT INTO audit_events VALUES($1,$2,'job-service','reboot_postcondition_verified',$3,now())")
|
||||
.bind(tenant).bind(Uuid::new_v4()).bind(json!({"job_id":job.job_id,"attempt_id":job.attempt_id,"grant_id":grant,"new_boot_id":boot,
|
||||
"connection_generation":envelope.connection_generation,"reboot_reserved_at":reserved,"reconnect_deadline":deadline,
|
||||
"inventory_message_id":receipt,"inventory_received_at":received,"verification_inventory_message_id":verification.0,"verification_inventory_received_at":verification.1}))
|
||||
.execute(&mut **tx).await?;
|
||||
Ok(())
|
||||
}
|
||||
pub async fn reconcile(
|
||||
@@ -587,8 +648,10 @@ pub async fn reconcile(
|
||||
&& decision.original_boot_id == Some(Uuid::parse_str(&original)?)
|
||||
{
|
||||
reboot_witness = Some((original, inventory));
|
||||
let verification =
|
||||
reboot_fresh_inventory(tx, tenant, agent, boot, Utc::now(), None).await?;
|
||||
let revision = (final_row.get::<i64, _>("revision") as u64).max(local.revision) + 1;
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at}).to_string(),error_code:String::new()};
|
||||
let result=JobResult{job_id:local.job_id.clone(),attempt_id:local.attempt_id.clone(),revision,state:"succeeded".into(),grant_id:local.grant_id.clone(),stdout:vec![],stderr:vec![],output_truncated:false,reboot_required:false,exit_code:None,evidence_json:json!({"postconditions_passed":true,"new_boot_id":boot,"inventory_message_id":inventory,"reconciled_after_reboot":true,"reboot_reserved_at":reserved,"reconnect_deadline":until,"inventory_received_at":received_at,"verification_inventory_message_id":verification.map(|v|v.0),"verification_inventory_received_at":verification.map(|v|v.1)}).to_string(),error_code:String::new()};
|
||||
let mut evidence = e.clone();
|
||||
evidence.message_id = Uuid::new_v4().to_string();
|
||||
evidence.payload = Some(Payload::JobResult(result));
|
||||
@@ -877,6 +940,61 @@ mod tests {
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
let fresh = Uuid::new_v4();
|
||||
let future = Uuid::new_v4();
|
||||
for (sequence, message, received) in [
|
||||
(4, fresh, later.now - Duration::seconds(1)),
|
||||
(5, future, later.now + Duration::days(1)),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
||||
.bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("ab".repeat(32)).execute(&mut *tx).await.unwrap();
|
||||
}
|
||||
// Delayed first verification keeps the original cutoff-bound ID and
|
||||
// uses a DIFFERENT fresh same-boot receipt; late-only never suffices.
|
||||
assert_eq!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(timely)
|
||||
);
|
||||
assert!(
|
||||
reboot_inventory(&mut tx, tenant, agent, boot, &later, Some(late))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.map(|v| v.0),
|
||||
Some(fresh)
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, agent, boot, later.now, Some(future))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, agent, Uuid::new_v4(), later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, tenant, Uuid::new_v4(), boot, later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
reboot_fresh_inventory(&mut tx, Uuid::new_v4(), agent, boot, later.now, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let before_return = RebootWindow {
|
||||
now: deadline - Duration::microseconds(1),
|
||||
..window
|
||||
@@ -915,7 +1033,7 @@ mod tests {
|
||||
let authorization = Uuid::new_v4();
|
||||
let grant = Uuid::new_v4();
|
||||
let now = chrono::DateTime::from_timestamp_micros(Utc::now().timestamp_micros()).unwrap();
|
||||
let reserved = now - Duration::seconds(200);
|
||||
let reserved = now - Duration::seconds(600);
|
||||
let deadline = reserved + Duration::seconds(180);
|
||||
let payload = JobPayload {
|
||||
schema_version: 1,
|
||||
@@ -949,7 +1067,7 @@ mod tests {
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256) VALUES($1,$2,$3,$4)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,connection_generation,connection_boot_id) VALUES($1,$2,$3,$4,1,$5)").bind(tenant).bind(agent).bind(asset).bind(hash(agent.as_bytes())).bind(boot).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,'isolated-test',$3,$4,$5)").bind(tenant).bind(plan).bind(b"{}".as_slice()).bind("ab".repeat(32)).bind(now+Duration::minutes(10)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash,approval_state) VALUES($1,$2,$3,'isolated-test',$4,$5,'approved')").bind(tenant).bind(operation).bind(plan).bind(operation.to_string()).bind("cd".repeat(32)).execute(&admin).await.unwrap();
|
||||
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,'isolated-test',$6,'dev_test_browser_simulation',true,1)").bind(tenant).bind(authorization).bind(operation).bind(plan).bind("ab".repeat(32)).bind("ef".repeat(32)).execute(&admin).await.unwrap();
|
||||
@@ -984,14 +1102,26 @@ mod tests {
|
||||
.unwrap();
|
||||
let timely = Uuid::new_v4();
|
||||
let late = Uuid::new_v4();
|
||||
let fresh = Uuid::new_v4();
|
||||
let wrong_boot = Uuid::new_v4();
|
||||
let future = Uuid::new_v4();
|
||||
for (sequence, message, received) in [
|
||||
(1, timely, deadline),
|
||||
(2, late, deadline + Duration::microseconds(1)),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)").bind(tenant).bind(agent).bind(message).bind(boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
||||
}
|
||||
for (sequence, message, received, observed_boot) in [
|
||||
(3, fresh, now - Duration::seconds(1), boot),
|
||||
(4, wrong_boot, now - Duration::seconds(1), old),
|
||||
(5, future, now + Duration::days(1), boot),
|
||||
] {
|
||||
sqlx::query("INSERT INTO observations(tenant_id,agent_id,message_id,boot_id,sequence,observed_at,received_at,source_id,quality,completeness,snapshot_id,data,payload_sha256) VALUES($1,$2,$3,$4,$5,$6,$6,'linux.base','valid','partial',$7,'{}',$8)")
|
||||
.bind(tenant).bind(agent).bind(message).bind(observed_boot).bind(sequence).bind(received).bind(Uuid::new_v4()).bind("aa".repeat(32)).execute(&admin).await.unwrap();
|
||||
}
|
||||
let evidence = json!({"postconditions_passed":true,"reconciled_after_reboot":true,"new_boot_id":boot,"inventory_message_id":late,
|
||||
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline});
|
||||
"reboot_reserved_at":reserved,"reconnect_deadline":deadline,"inventory_received_at":deadline,
|
||||
"verification_inventory_message_id":fresh,"verification_inventory_received_at":now-Duration::seconds(1)});
|
||||
let mut result = JobResult {
|
||||
job_id: job.to_string(),
|
||||
attempt_id: attempt.to_string(),
|
||||
@@ -1004,6 +1134,7 @@ mod tests {
|
||||
let mut envelope = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
agent_id: agent.to_string(),
|
||||
connection_generation: 1,
|
||||
boot_id: boot.to_string(),
|
||||
payload: Some(Payload::JobResult(result.clone())),
|
||||
..Default::default()
|
||||
@@ -1031,6 +1162,32 @@ mod tests {
|
||||
);
|
||||
let mut good = evidence.clone();
|
||||
good["inventory_message_id"] = json!(timely);
|
||||
for (id, time) in [
|
||||
(Some(wrong_boot), Some(now - Duration::seconds(1))),
|
||||
(Some(future), Some(now + Duration::days(1))),
|
||||
(None, None),
|
||||
(Some(fresh), Some(now)),
|
||||
] {
|
||||
let mut bad = good.clone();
|
||||
bad["verification_inventory_message_id"] = json!(id);
|
||||
bad["verification_inventory_received_at"] = json!(time);
|
||||
result.evidence_json = bad.to_string();
|
||||
envelope.payload = Some(Payload::JobResult(result.clone()));
|
||||
assert!(
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
envelope.connection_generation = 2;
|
||||
result.evidence_json = good.to_string();
|
||||
envelope.payload = Some(Payload::JobResult(result.clone()));
|
||||
assert!(
|
||||
crate::job_gateway::event(&mut tx, tenant, agent, &envelope)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
envelope.connection_generation = 1;
|
||||
let mut false_clock = good.clone();
|
||||
false_clock["reconnect_deadline"] = json!(now + Duration::days(1));
|
||||
result.evidence_json = false_clock.to_string();
|
||||
@@ -1085,6 +1242,16 @@ mod tests {
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let audit:Value=sqlx::query_scalar("SELECT data FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'")
|
||||
.bind(tenant).fetch_one(&mut *tx).await.unwrap();
|
||||
assert_eq!(audit["inventory_message_id"], json!(timely));
|
||||
assert_eq!(audit["inventory_received_at"], json!(deadline));
|
||||
assert_eq!(audit["verification_inventory_message_id"], json!(fresh));
|
||||
assert_eq!(
|
||||
audit["verification_inventory_received_at"],
|
||||
json!(now - Duration::seconds(1))
|
||||
);
|
||||
assert_eq!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM audit_events WHERE tenant_id=$1 AND event_type='reboot_postcondition_verified'").bind(tenant).fetch_one(&mut *tx).await.unwrap(),1);
|
||||
tx.rollback().await.unwrap();
|
||||
}
|
||||
#[test]
|
||||
|
||||
@@ -243,6 +243,30 @@ def legacy_helper_unit_metadata(payload):
|
||||
clock_query_filters=[line for line in lines if line in ('SystemCallFilter=adjtimex clock_adjtime','SystemCallFilter=adjtimex')])
|
||||
|
||||
|
||||
def start_helper_after_boot_clock(payload):
|
||||
"""Start the existing helper after its clock guard failed; never a job."""
|
||||
import contextlib
|
||||
import sqlite3
|
||||
unit=Path('/etc/systemd/system/rmm-helper-cloud.service')
|
||||
require(hashlib.sha256(unit.read_bytes()).hexdigest()==payload['unit_sha256']
|
||||
and hashlib.sha256(Path('/usr/local/libexec/rmm-privileged-helper').read_bytes()).hexdigest()==payload['helper_sha256'],'same qualified helper/unit')
|
||||
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
|
||||
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
require(boot!=payload['old_boot_id'],'actual guest already rebooted; never request another reboot')
|
||||
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and
|
||||
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy ledger')
|
||||
rows=db.execute('SELECT c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
|
||||
require(len(rows)==1 and rows[0][:2]==(payload['job_id'],payload['attempt_id'])
|
||||
and json.loads(rows[0][2])['state']=='awaiting_reboot','only exact existing reboot may await reconciliation')
|
||||
before=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()
|
||||
require(before==b'failed','known boot-start failure only')
|
||||
subprocess.run(['systemctl','start','rmm-helper-cloud.service'],capture_output=True,check=True,timeout=20)
|
||||
active=subprocess.run(['systemctl','show','rmm-helper-cloud.service','-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip()==b'active'
|
||||
return dict(started_existing_helper=active,new_boot_id=boot,ledger_reset=False,
|
||||
jobs_created=False,grants_created=False,reboot_requested=False)
|
||||
|
||||
|
||||
def disable_legacy_helper_autostart(payload):
|
||||
"""Disable only the stopped historical dev helper; never stop/start a unit."""
|
||||
import contextlib
|
||||
|
||||
@@ -254,7 +254,11 @@ def close(ident):
|
||||
m=cloud.contract;folder=STATE/str(uuid.UUID(ident));require(m.protected_json(folder/'result.json')['verified'],'normal verified reconnect required before rollback')
|
||||
session=m.protected_json(folder/'before.json');target=session['target'];opened=m.protected_json(folder/'policy-open.json');enabled=m.protected_json(folder/'helper-enabled.json')
|
||||
require(evidence(target,'gate')['unresolved']==0,'all effects closed')
|
||||
closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
|
||||
earlier=folder/'policy-closed-for-upgrade.json'
|
||||
if earlier.exists():
|
||||
closed=m.protected_json(earlier);current=cloud.bootstrap.rpc('server.conf',server_policy,dict(action='state'))
|
||||
require(current['closed'] and current['policy_sha256']==closed['policy_sha256'],'exact earlier deny-only closure retained')
|
||||
else:closed=cloud.bootstrap.rpc('server.conf',server_policy,dict(id=ident,action='close',previous_sha256=opened['policy_sha256']))
|
||||
m.write_private(folder/'policy-closed.json',closed)
|
||||
restored=cloud.bootstrap.rpc('linux-vm.conf',vm_policy,dict(session['pins'],id=ident,enabled=False,config_sha256=enabled['config_sha256'],pin_sha256=enabled['pin_sha256'],original_backup=enabled['backup']))
|
||||
require(restored['general_pin']==session['before']['general_pin'] and restored['config_sha256']==session['before']['config_sha256'] and restored['pin_sha256']==session['before']['pin_sha256'] and restored['preserved_state'] and not restored['reboot_requested'],'original non-reboot policy/pin bytes restored without another reboot')
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserve the one consumed reboot while updating its read-only verifier."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import cloud_reboot_acceptance as reboot
|
||||
import cloud_server_upgrade as upgrade
|
||||
|
||||
cloud=reboot.cloud
|
||||
require=cloud.contract.require
|
||||
|
||||
|
||||
def agent_transport(payload):
|
||||
import contextlib
|
||||
import sqlite3
|
||||
require(payload['action'] in ('stop','start'),'own agent transport direction')
|
||||
require(json.loads(Path('/var/lib/rmm-agent-remote/identity.json').read_text())['agent_id']==payload['agent_id'],'same device')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'qualified agent')
|
||||
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
require(boot==payload['new_boot_id'] and boot!=payload['old_boot_id'],'already completed one guest boot')
|
||||
with contextlib.closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' and db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'preserved healthy Root ledger')
|
||||
rows=db.execute('SELECT c.grant_id,c.job,c.attempt,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
|
||||
require(len(rows)==1 and rows[0][:3]==(payload['grant_id'],payload['job_id'],payload['attempt_id']) and json.loads(rows[0][3])['state']=='awaiting_reboot','only original consumed reboot may await proof')
|
||||
subprocess.run(['systemctl',payload['action'],'rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
|
||||
return dict(action=payload['action'],new_boot_id=boot,jobs_created=False,grants_created=False,reboot_requested=False)
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--id',required=True);p.add_argument('--reboot-id',required=True);args=p.parse_args()
|
||||
import uuid
|
||||
candidate=str(uuid.UUID(args.id));sid=str(uuid.UUID(args.reboot_id));folder=reboot.STATE/sid
|
||||
session=cloud.contract.protected_json(folder/'before.json');checkpoint=cloud.contract.protected_json(folder/'checkpoint.json');job=checkpoint['job'];target=session['target']
|
||||
require(sid=='d015c3ef-f3aa-4522-a3c2-fcb6099df17b' and job['job_id']=='8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa','one exact already approved reboot')
|
||||
gate=lambda:reboot.evidence(target,'gate')
|
||||
require(gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1),'only known consumed reboot unresolved')
|
||||
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
|
||||
grant='25a4bf3f-772c-487d-825d-73bcbbc418fd'
|
||||
require(current['state'] in ('unknown','awaiting_reboot') and current['result']['grant_id']==grant,'same immutable reboot attempt')
|
||||
scope=dict(session['pins'],job_id=job['job_id'],attempt_id=job['attempt_id'])
|
||||
before=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
|
||||
require(before['root_unresolved']==1 and before['other_unresolved']==0 and before['selected_root_grants']==[grant]
|
||||
and before['boot_id']!=session['old_boot_id'] and len(before['root_rows'])==len(session['before']['root_rows'])+1
|
||||
and all(before['root_rows'].get(k)==v for k,v in session['before']['root_rows'].items()),'all old Root rows and exact consumed reboot preserved')
|
||||
audit=reboot.evidence(target,'audit',operation_id=checkpoint['operation_id'])
|
||||
require(audit==dict(origin='dev_test_browser_simulation',simulation=True,principal='simulated-testprincipal-a',plan_hash=checkpoint['plan']['plan_hash']),'original actual UI approval')
|
||||
status=cloud.bootstrap.rpc('server.conf',upgrade.status,dict(id=candidate))
|
||||
require(status['finished'] and status['exit_code']==0 and status['source_unchanged'] and status['runtime_tag_unchanged'],'qualified separate server image')
|
||||
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(action='state'))
|
||||
if not policy['closed']:
|
||||
policy=cloud.bootstrap.rpc('server.conf',reboot.server_policy,dict(id=sid,action='close',previous_sha256=policy['policy_sha256']))
|
||||
cloud.contract.write_private(folder/'policy-closed-for-upgrade.json',policy)
|
||||
transport=dict(agent_id=target['agent_id'],agent_sha256=session['pins']['agent_sha256'],old_boot_id=session['old_boot_id'],new_boot_id=before['boot_id'],grant_id=grant,job_id=job['job_id'],attempt_id=job['attempt_id'])
|
||||
stopped=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='stop'))
|
||||
frozen=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
|
||||
require(frozen['root_rows']==before['root_rows'] and frozen['database_inodes']==before['database_inodes'] and frozen['services']['rmm-agent-remote.service']['ActiveState']=='inactive','quiesced agent and original evidence retained')
|
||||
cloud.contract.write_private(folder/'proof-upgrade-before.json',dict(candidate=candidate,gate=gate(),current=current,native=frozen,transport=stopped,policy=policy))
|
||||
stream=io.BytesIO()
|
||||
with tarfile.open(fileobj=stream,mode='w:') as archive:
|
||||
for path in sorted((cloud.ROOT/'web/dist').rglob('*')):
|
||||
if path.is_file():
|
||||
require(not path.is_symlink(),'built UI symlink');archive.add(path,arcname=str(path.relative_to(cloud.ROOT/'web/dist')),recursive=False)
|
||||
raw=stream.getvalue()
|
||||
result=cloud.bootstrap.rpc('server.conf',upgrade.deploy,dict(id=candidate,closed_policy_sha256=policy['policy_sha256'],previous_ui_index_sha256='a3513f928e20ea11d53f1cd27e8735d911555e1c66d5d97ca8bdefaff60e71fa',web_tar=base64.b64encode(raw).decode(),web_sha256=hashlib.sha256(raw).hexdigest()))
|
||||
after=cloud.bootstrap.rpc('linux-vm.conf',reboot.vm_state,scope)
|
||||
require(result['deployed'] and gate()==dict(jobs=30,outbox=30,permits=26,unresolved=1) and after['root_rows']==frozen['root_rows'] and after['database_inodes']==frozen['database_inodes'],'deployment preserves exact unresolved effect before normal resumption')
|
||||
cloud.contract.write_private(folder/'proof-upgrade-deployed.json',dict(verified=True,candidate=candidate,result=result,native=after,counts=gate()))
|
||||
resumed=cloud.bootstrap.rpc('linux-vm.conf',agent_transport,dict(transport,action='start'))
|
||||
cloud.contract.write_private(folder/'proof-upgrade-resumed.json',resumed)
|
||||
print('PASS exact reboot evidence preserved during qualified server upgrade; ordinary signed resumption required; no second reboot')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
message=str(error);print(message if message.startswith('remote bootstrap incomplete: ') else 'REBOOT_PROOF_UPGRADE_INCOMPLETE:'+type(error).__name__)
|
||||
raise SystemExit(2) from None
|
||||
@@ -11,6 +11,17 @@ import cloud_mvp_acceptance as cloud
|
||||
require = cloud.contract.require
|
||||
|
||||
|
||||
def reboot_receipts(payload):
|
||||
tenant=str(uuid.UUID(payload['tenant_id']));agent=str(uuid.UUID(payload['agent_id']));job=str(uuid.UUID(payload['job_id']))
|
||||
require(tenant=='11111111-1111-4111-8111-111111111111','fixed development tenant')
|
||||
query=("BEGIN READ ONLY; SELECT json_build_object('observed_at',now(),'installation',(SELECT json_build_object('boot_id',connection_boot_id,'generation',connection_generation,'last_contact_at',last_contact_at,'sync_ready',sync_ready,'helper_ready',helper_sync_ready) FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'),"
|
||||
"'job',(SELECT json_build_object('state',state,'revision',revision,'result',result) FROM jobs WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||
"'grant',(SELECT json_build_object('grant_id',grant_id,'created_at',created_at) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||
"'inventories',(SELECT COALESCE(json_agg(x),'[]'::json) FROM (SELECT message_id,boot_id,observed_at,received_at,source_id FROM observations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' ORDER BY received_at DESC LIMIT 20) x)); COMMIT;")
|
||||
root=Path('/opt/rust-discovery');cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
||||
|
||||
|
||||
def server(payload):
|
||||
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
||||
|
||||
@@ -1,5 +1,34 @@
|
||||
# Tatsächlicher Implementierungsstand
|
||||
|
||||
## Original-Reboot erfolgt; Abschluss noch gesperrt — 2026-11-13
|
||||
|
||||
Der feste WebUI-genehmigte Job `8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa`
|
||||
verbrauchte genau Grant `25a4bf3f-772c-487d-825d-73bcbbc418fd`. Tatsächlicher
|
||||
neuer Boot und mTLS-Inventare nach etwa88/148Sekunden liegen innerhalb der
|
||||
unveränderten180Sekunden ab zentraler Reservierung20:26:12.430245UTC.
|
||||
General startete zunächst mit `CLOCK_UNTRUSTED` nicht; die bestehende Unit
|
||||
`Restart=no` konnte nicht selbst wieder anlaufen. Nach nachgewiesen vertrauter
|
||||
Kerneluhr wurde ausschließlich derselbe Helper gestartet. Keine zweite
|
||||
Rebootwirkung, keine Journal-/Grant-/Identitätsrücksetzung.
|
||||
|
||||
Servercandidate `7be1f330-4bbd-4801-b72a-c35ce7ac479e` nativ ARM64 gebaut und
|
||||
erhaltend deployt. Spec1.14 präzisiert spätere Erstverifikation: unveränderlicher
|
||||
rechtzeitiger Originalbeleg **und** separater frischer Sameboot-Inventarbeleg
|
||||
unter aktuellem authentisierten Kanal. Drei isolierte echte PostgreSQL-Tests,
|
||||
Format und Clippy bestanden. Alle30Jobs/30Outbox/26Grants, Root18 und alten
|
||||
DBinodes/-Verbrauchszeilen beim Deployment erhalten; UI unverändert. Dies ist
|
||||
noch keine vollständige Rebootabnahme: Abschluss weiterhin unknown.
|
||||
|
||||
Danach kein neuer SyncRequest, obwohl Heartbeats/Inventare fortliefen.
|
||||
Rootjournal tatsächlich268414976Bytes nahe256MiB-Grenze; Meta blieb1378,
|
||||
Agent meldet nur generischen Syncabhängigkeitsfehler. Sourcepfad schreibt
|
||||
Helpercapture vor Agentrequest. Agent exakt an denselben Originalversuch
|
||||
gebunden administrativ gestoppt, Helper aktiv/Rootfaultfrei. Kapazitätsursache
|
||||
wird an eigener DBkopie geprüft; keine Historylöschung oder Wiederholung.
|
||||
Erneuter `finish`-Leseversuch endete ohne Erfolgsnachweis. Noch offen sind
|
||||
Abschluss desselben signierten Rebootabgleichs, Startretrykorrektur, MCP-Endbericht
|
||||
und Endaudit. Details:[reboot-acceptance.md](reboot-acceptance.md).
|
||||
|
||||
## Kontrollierter Agentabbruch nativ vollständig abgeglichen — 2026-11-13
|
||||
|
||||
Eigener Cloudfall644c06eb-24b0-41b0-9a90-f60ea1fe38a9 bestanden. Tatsächlicher
|
||||
|
||||
@@ -1,6 +1,21 @@
|
||||
# Verbindliche Projektentscheidungen
|
||||
|
||||
Stand: **2026-11-13**, Spezifikation **1.13** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
|
||||
Stand: **2026-11-13**, Spezifikation **1.14** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
|
||||
|
||||
## Rechtzeitiger Rebootkontakt und spätere frische Verifikation (Version1.14)
|
||||
|
||||
Die feste180s-MVP-Frist wird nicht verändert. Ein rechtzeitig gespeicherter
|
||||
Originalbeleg wird bei späterer Erstverifikation zusammen mit einem zweiten,
|
||||
frischen Beleg derselben aktuellen Boot-ID und dem tatsächlichen aktuellen
|
||||
authentisierten Gerätekontext geprüft. Signiertes Ergebnis bindet Original-
|
||||
`inventory_message_id`/`inventory_received_at` und zusätzliche
|
||||
`verification_inventory_message_id`/`verification_inventory_received_at`;
|
||||
Audit bindet Job/Versuch/Grant/Boot/Generation und beide Datenbankbelege.
|
||||
Verspäteter Originalkontakt, falscher Boot/Kunde/Agent/Generation oder fehlende
|
||||
aktuelle Verifikation bleiben verweigert. Grundlage: tatsächlicher rechtzeitiger
|
||||
Rebootkontakt bei zunächst clock-bedingt ausgefallenem Helper, keine neue
|
||||
Rebootfreigabe oder erfundene rückwirkende Fristeinhaltung. Details:
|
||||
[Rebootabnahme](reboot-acceptance.md).
|
||||
|
||||
## Gebundene Paket- und Rebootnachbedingungen (Version1.13)
|
||||
|
||||
|
||||
@@ -1,5 +1,31 @@
|
||||
# Letzte Linux-Cloud-VM-Rebootabnahme
|
||||
|
||||
## Aktueller tatsächlicher Stand2026-11-13
|
||||
|
||||
Der einzelne feste WebUI-Job8ffd0ce5-187e-4227-9fd4-374ac5fdf4aa mit
|
||||
Grant25a4bf3f-772c-487d-825d-73bcbbc418fd wurde tatsächlich einmal ausgeführt.
|
||||
Neue Boot-ID45aba743-be1d-4ae5-ba00-139fb9363bfe statt
|
||||
603e9280-801d-403e-bbfe-7340784e000a, zentrale Reservierung20:26:12.430245UTC,
|
||||
Originalfrist20:29:12.430245UTC. Tatsächliche neue Bootinventare
|
||||
6f5bd8a6-a33b-4e45-a102-63d43e12c23d um20:27:40.878097UTC und
|
||||
1cf6146c-15fb-491f-a35a-8c4a0bb8f679 um20:28:40.877951UTC sind rechtzeitig.
|
||||
Kein zweiter Neustart ist autorisiert oder vorgesehen.
|
||||
|
||||
Noch **unknown**, nicht abgenommen: General scheiterte beim Boot an
|
||||
CLOCK_UNTRUSTED/Restart=no. Nach realer Uhrbereitschaft derselbe Helper
|
||||
operativ gestartet. Spec1.14/Zwei-Beleg-Serverfix unter Candidate7be1f330
|
||||
nativ erhaltend deployt, drei isolierte PostgreSQL-Tests/Clippy/Format bestanden.
|
||||
Originalfrist, alte Daten/Grants/DBinodes und UI blieben erhalten. Erneuter
|
||||
Finishleselauf konnte noch keinen signierten Erfolg belegen. Seitdem
|
||||
Rootjournal268414976Bytes nahe256MiB, keine neuen Helpercaptures/Agentrequests.
|
||||
Agent für begrenzte Kopiediagnose angehalten; keine Ledgerbereinigung.
|
||||
MVP bleibt offen. Private Belege im unten genannten festen Fallordner,
|
||||
zusätzlich proof-upgrade-{before,deployed,resumed}.json und
|
||||
reconcile-diagnostic-quiesced.json.
|
||||
|
||||
Die folgenden Vorbereitungsabschnitte und Formulierungen „noch nicht ausgeführt“
|
||||
sind historische Vorstufen, kein aktueller Wirkungsstand.
|
||||
|
||||
Vor dem tatsächlichen Reboot neun bereits verifizierte frühere Gates an feste
|
||||
JSON-/SHA-Referenzen gebunden. Root17/0offen/faultfrei, Zentrale29/29/25/0,
|
||||
Agent/General aktiv+enabled, APTinactive/disabled und Endpunkt entfernt.
|
||||
@@ -34,8 +60,12 @@ neue Verbindungsversuche und neue Inventare verlängern sie ebenfalls nicht.
|
||||
Erstmaliger Erfolg braucht einen tatsächlich gespeicherten Inventarbeleg
|
||||
derselben Identität und des aktuellen neuen Boot-IDs. Dessen serverseitige
|
||||
Empfangszeit muss **nach** der Startreservierung und **spätestens zur Frist**
|
||||
liegen. Der Beleg muss bei erstmaliger Abnahme außerdem jünger als fünf Minuten
|
||||
sein; die alte Boot-ID muss aus dem zum gleichen Grant gehörenden
|
||||
liegen. Bei später als fünf Minuten erfolgender Erstverifikation braucht der
|
||||
unveränderliche rechtzeitige Originalbeleg einen zusätzlichen, unabhängig
|
||||
empfangenen aktuellen Sameboot-Beleg jünger als fünf Minuten und den aktuellen
|
||||
authentisierten Geräte-/Generationskontext. Beide tatsächlichen IDs und
|
||||
Empfangszeiten werden im signierten Ergebnis und Audit gebunden. Ein erst
|
||||
verspätet empfangener Originalbeleg bleibt verweigert. Die alte Boot-ID muss aus dem zum gleichen Grant gehörenden
|
||||
`start_reserved`-Audit stammen und vom neuen Boot abweichen. Fehlende oder zu
|
||||
späte Belege bleiben `unknown`; daraus entsteht weder ein geratenes `failed`
|
||||
noch eine Wiederholungserlaubnis.
|
||||
|
||||
Reference in New Issue
Block a user