fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Rust RMM: Architektur und Implementierungsspezifikation
|
||||
|
||||
Version 1.11 · 2026-11-13 · Sprache: Deutsch
|
||||
Version 1.12 · 2026-11-13 · Sprache: Deutsch
|
||||
|
||||
**Zweck:** Vollständige, eigenständig nutzbare Grundlage für die Implementierung einer mandantenfähigen Plattform zur Inventarisierung, Überwachung und Fernadministration von Kundengeräten durch eine Coding-KI und ein Entwicklungsteam.
|
||||
|
||||
@@ -66,6 +66,17 @@ Deploymentdaten, offene Konfiguration und reale Nachweise:
|
||||
|
||||
### 0.1 Auftrag an die implementierende KI
|
||||
|
||||
**Technische Präzisierung Version 1.12 — endliche Journalbudgets:**
|
||||
Agentjournal sowie Agent-/Helper-Verbrauchsledger erhalten pro SQLite-Datei
|
||||
maximal256MiB bei geprüften4096-Byte-Seiten. Der bisherige Implementierungswert
|
||||
64MiB blockierte auf der tatsächlichen VM bereits zehn abgeschlossene Rootjobs
|
||||
wegen vollständig erhaltener Sync-Historien. Die Anpassung erhält alle
|
||||
Identitäten, Journale, Signaturen und Tombstones; kein Reset oder Löschen zur
|
||||
Abnahme. Volle/inkonsistente Journale sperren weiterhin neue Wirkungen;
|
||||
`SQLITE_FULL` wird ausdrücklich als Kapazitätssperre gemeldet und nie ignoriert.
|
||||
Sichere Kompaktierung bleibt späterer eigener Umfang, keine automatische
|
||||
Löschung oder unbeschränkte Datenbank in diesem MVP.
|
||||
|
||||
**Technische Präzisierung Version 1.11 — neuer Configure-only-Plan:**
|
||||
Nach unabhängig belegtem und durch normalen signierten RootSync abgeschlossenem
|
||||
Fehler „Fixture1.0 ausschließlich entpackt“ darf ein neuer fester WebUI-Plan
|
||||
|
||||
@@ -92,6 +92,92 @@ fn hex_hash(s: &str) -> bool {
|
||||
s.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}
|
||||
|
||||
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
|
||||
/// executor thread from the previous process can still deliver its outcome.
|
||||
/// This does not execute, release consumption or invent a successful outcome.
|
||||
pub fn interrupted(
|
||||
db: &Connection,
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
) -> Result<Vec<(JobOffer, String)>> {
|
||||
let mut statement = db.prepare(
|
||||
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
|
||||
)?;
|
||||
let rows = statement.query_map([], |r| {
|
||||
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
|
||||
})?;
|
||||
let mut interrupted = Vec::new();
|
||||
for row in rows {
|
||||
let (bytes, grant) = row?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
if !grant.is_empty() {
|
||||
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
|
||||
.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[&grant],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (agent, generation, job_id, attempt, start) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Verify the historical signed request at its original valid time;
|
||||
// this is evidence only, never permission to start it again.
|
||||
let (_, verified) = binding.verify(&request, permit.not_before)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
} else {
|
||||
// A request may have committed remotely without its reply arriving.
|
||||
// It can only become local unknown, never a fabricated grant/result.
|
||||
let state: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
|
||||
}
|
||||
interrupted.push((offer, grant));
|
||||
}
|
||||
Ok(interrupted)
|
||||
}
|
||||
|
||||
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
)? == 1,
|
||||
"INTERRUPTED_REQUEST_STATE"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
|
||||
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
||||
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
|
||||
|
||||
@@ -209,6 +209,29 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
/// New process only. Persist honest uncertainty before any helper snapshot
|
||||
/// can block; the ordinary first-channel replay publishes it before Sync.
|
||||
fn recover_interrupted(&mut self) -> Result<()> {
|
||||
let Some(binding) = &self.binding else {
|
||||
return Ok(());
|
||||
};
|
||||
let interrupted = jobs::interrupted(&self.db, &self.ledger, binding)?;
|
||||
for (offer, grant) in interrupted {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(&self.db, &offer)?;
|
||||
} else {
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
// No finish/unlock here. Only normal bound reconciliation can release
|
||||
// a consumed attempt once independent definitive evidence is available.
|
||||
Ok(())
|
||||
}
|
||||
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
// A large replay backlog must not monopolize heartbeat/cancel handling.
|
||||
// On backpressure timeout, discard this channel rather than continuing
|
||||
@@ -789,6 +812,7 @@ async fn main() -> Result<()> {
|
||||
certificates,
|
||||
rotation_policy,
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1_u64;
|
||||
loop {
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -18,6 +18,181 @@ use std::{
|
||||
|
||||
struct Clock(bool);
|
||||
#[test]
|
||||
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
|
||||
use openssl::sign::Signer;
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
|
||||
use rmm_privileged_helper::{Binding, Request};
|
||||
use rusqlite::params;
|
||||
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&dir).unwrap();
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let public = key.raw_public_key().unwrap();
|
||||
let binding = Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
};
|
||||
let now = Utc::now();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::RunScript {
|
||||
interpreter: "posix_sh".into(),
|
||||
script_utf8: "printf 'fixture only; never executed\\n'".into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: Some("fixture only; never executed\n".into()),
|
||||
},
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::minutes(5),
|
||||
max_runtime_seconds: 120,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let permit = ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: Uuid::new_v4(),
|
||||
authorization_id: Uuid::new_v4(),
|
||||
tenant_id: job.tenant_id,
|
||||
asset_id: job.asset_id,
|
||||
agent_id: job.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
plan_hash: hash(b"fixed test plan; no native approval"),
|
||||
job_payload_hash: hash(&encode(&job).unwrap()),
|
||||
action: "run_script".into(),
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(59),
|
||||
max_runtime_seconds: 120,
|
||||
key_id: hash(&public),
|
||||
};
|
||||
let mut request = Request {
|
||||
job_bytes: encode(&job).unwrap(),
|
||||
permit_bytes: encode(&permit).unwrap(),
|
||||
signature: vec![],
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
authorization_id: permit.authorization_id,
|
||||
};
|
||||
request.signature = Signer::new_without_digest(&key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&request.permit_bytes)
|
||||
.unwrap();
|
||||
let offer = JobOffer {
|
||||
job_id: job.job_id.to_string(),
|
||||
attempt_id: job.attempt_id.to_string(),
|
||||
authorization_id: permit.authorization_id.to_string(),
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
job_payload_hash: permit.job_payload_hash.clone(),
|
||||
payload_bytes: request.job_bytes.clone(),
|
||||
};
|
||||
let db = storage::open(&dir).unwrap();
|
||||
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
|
||||
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
|
||||
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
|
||||
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
|
||||
let mut queued_job = job.clone();
|
||||
queued_job.job_id = Uuid::new_v4();
|
||||
queued_job.attempt_id = Uuid::new_v4();
|
||||
let mut queued = offer.clone();
|
||||
queued.job_id = queued_job.job_id.to_string();
|
||||
queued.attempt_id = queued_job.attempt_id.to_string();
|
||||
queued.payload_bytes = encode(&queued_job).unwrap();
|
||||
queued.job_payload_hash = hash(&queued.payload_bytes);
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
|
||||
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
let (itx, irx) = tokio::sync::mpsc::channel(1);
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&dir.join("certificates"),
|
||||
&[],
|
||||
binding.agent_id,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
let mut agent = Agent {
|
||||
address: "127.0.0.1:1".parse().unwrap(),
|
||||
ca: vec![],
|
||||
id: binding.agent_id.to_string(),
|
||||
boot: rmm_agent_linux::boot_id().unwrap(),
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db,
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding: Some(binding.clone()),
|
||||
ledger,
|
||||
completed_tx: tx,
|
||||
completed_rx: rx,
|
||||
inventory_tx: itx,
|
||||
inventory_rx: irx,
|
||||
inventory_inflight: false,
|
||||
sync: Some(reconcile::ClientSync::new(binding.clone())),
|
||||
sync_clock: Clock(true),
|
||||
helpers: None,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode: wire_channel::Mode::Wss,
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
};
|
||||
// A forged local consumption/start cannot become a purported grant result.
|
||||
agent
|
||||
.ledger
|
||||
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
|
||||
.unwrap();
|
||||
assert!(agent.recover_interrupted().is_err());
|
||||
assert!(jobs::events(&agent.db).unwrap().is_empty());
|
||||
agent
|
||||
.ledger
|
||||
.execute(
|
||||
"UPDATE consumed SET agent=?1",
|
||||
[binding.agent_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
agent.recover_interrupted().unwrap();
|
||||
let original = jobs::events(&agent.db).unwrap();
|
||||
assert_eq!(original.len(), 1);
|
||||
let Some(Payload::JobResult(result)) = &original[0].payload else {
|
||||
panic!("real durable result required")
|
||||
};
|
||||
assert_eq!(result.state, "unknown");
|
||||
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
|
||||
assert_eq!(result.grant_id, permit.grant_id.to_string());
|
||||
assert_eq!(result.revision, 2);
|
||||
assert!(result.stdout.is_empty() && result.exit_code.is_none());
|
||||
assert!(!agent.fence.allows(1, true));
|
||||
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
|
||||
assert_eq!(jobs::events(&reopened).unwrap(), original);
|
||||
let requesting: (String, u64) = reopened
|
||||
.query_row(
|
||||
"SELECT state,revision FROM job_attempts WHERE job=?1",
|
||||
[&queued.job_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(requesting, ("unknown".into(), 1));
|
||||
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(retained, (1, 1, 1));
|
||||
// The same process/restart cannot invent another revision or message ID.
|
||||
agent.recover_interrupted().unwrap();
|
||||
assert_eq!(jobs::events(&agent.db).unwrap(), original);
|
||||
drop(reopened);
|
||||
drop(agent);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
|
||||
let gate = EffectFence::default();
|
||||
assert!(!gate.allows(1, false));
|
||||
|
||||
@@ -65,8 +65,9 @@ pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
rmm_privileged_helper::private_path(&path, uid, false)?;
|
||||
let db = rusqlite::Connection::open(path)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("PRAGMA max_page_count=16384;
|
||||
rmm_privileged_helper::bounded_journal_capacity(&db)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("
|
||||
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
|
||||
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
|
||||
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
|
||||
@@ -106,6 +107,29 @@ pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod passive_pressure_tests {
|
||||
#[test]
|
||||
fn agent_journal_capacity_is_finite_and_preserves_sequence_on_reopen() {
|
||||
let dir = std::env::temp_dir().join(format!("rmm-capacity-{}", uuid::Uuid::new_v4()));
|
||||
super::prepare_dir(&dir).unwrap();
|
||||
let mut db = super::open(&dir).unwrap();
|
||||
assert_eq!(super::next_sequence(&mut db).unwrap(), 1);
|
||||
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
|
||||
drop(db);
|
||||
let mut db = super::open(&dir).unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4096
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
65536
|
||||
);
|
||||
assert_eq!(super::next_sequence(&mut db).unwrap(), 2);
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn pending_durable_traffic_backpressures_collection_without_deletion() {
|
||||
let db = rusqlite::Connection::open_in_memory().unwrap();
|
||||
|
||||
@@ -204,6 +204,17 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
f.try_lock_exclusive()?;
|
||||
Ok(f)
|
||||
}
|
||||
/// Finite 256 MiB bound for durable agent/helper history. Unexpected page sizes
|
||||
/// are refused rather than silently multiplying the byte limit. No compaction
|
||||
/// or tombstone deletion occurs here; SQLITE_FULL must still fail closed.
|
||||
pub fn bounded_journal_capacity(db: &rusqlite::Connection) -> Result<()> {
|
||||
let page_size: u32 = db.query_row("PRAGMA page_size", [], |r| r.get(0))?;
|
||||
ensure!(page_size == 4096, "JOURNAL_PAGE_SIZE_UNSUPPORTED");
|
||||
let maximum: u32 = db.query_row("PRAGMA max_page_count=65536", [], |r| r.get(0))?;
|
||||
ensure!(maximum == 65536, "JOURNAL_STORAGE_CAPACITY_BLOCKED");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
let path = dir.join("consumption.sqlite");
|
||||
@@ -218,7 +229,8 @@ pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
f.sync_all()?;
|
||||
private_path(&path, uid, false)?;
|
||||
let db = rusqlite::Connection::open(path)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=16384;
|
||||
bounded_journal_capacity(&db)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE;
|
||||
CREATE TABLE IF NOT EXISTS consumed(grant_id TEXT PRIMARY KEY, agent TEXT NOT NULL, generation INTEGER NOT NULL,
|
||||
job TEXT NOT NULL, attempt TEXT NOT NULL, result BLOB, UNIQUE(agent,generation,job,attempt));
|
||||
CREATE TABLE IF NOT EXISTS execution_evidence(grant_id TEXT PRIMARY KEY, start BLOB NOT NULL, outcome BLOB);")?;
|
||||
|
||||
@@ -472,7 +472,30 @@ fn handle(
|
||||
stream.write_all(&bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
fn safe_error_code(error: &anyhow::Error) -> String {
|
||||
if error.chain().any(|cause| {
|
||||
matches!(cause.downcast_ref::<rusqlite::Error>(),
|
||||
Some(rusqlite::Error::SqliteFailure(code, _)) if code.code == rusqlite::ErrorCode::DiskFull)
|
||||
}) {
|
||||
return "JOURNAL_STORAGE_CAPACITY_BLOCKED".into();
|
||||
}
|
||||
let text = error.to_string();
|
||||
if text.len() <= 96
|
||||
&& text
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
{
|
||||
text
|
||||
} else {
|
||||
"HELPER_VALIDATION_OR_IO_FAILURE".into()
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
run().map_err(|error| anyhow::anyhow!(safe_error_code(&error)))
|
||||
}
|
||||
|
||||
fn run() -> Result<()> {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
if args == [std::ffi::OsString::from("--help")] {
|
||||
println!(
|
||||
@@ -607,16 +630,7 @@ fn main() -> Result<()> {
|
||||
&mut sync,
|
||||
&sync_clock,
|
||||
) {
|
||||
let text = error.to_string();
|
||||
let code = if text.len() <= 96
|
||||
&& text
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
||||
{
|
||||
text.as_str()
|
||||
} else {
|
||||
"HELPER_VALIDATION_OR_IO_FAILURE"
|
||||
};
|
||||
let code = safe_error_code(&error);
|
||||
eprintln!("helper request rejected: {code}");
|
||||
}
|
||||
}
|
||||
@@ -627,6 +641,41 @@ fn main() -> Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn sqlite_full_is_sanitized_and_failed_transaction_preserves_history() {
|
||||
let db = rusqlite::Connection::open_in_memory().unwrap();
|
||||
rmm_privileged_helper::bounded_journal_capacity(&db).unwrap();
|
||||
db.execute_batch("CREATE TABLE witness(value BLOB);INSERT INTO witness VALUES('retained');PRAGMA max_page_count=4;").unwrap();
|
||||
db.execute_batch("BEGIN;INSERT INTO witness VALUES('must roll back');")
|
||||
.unwrap();
|
||||
let error = db
|
||||
.execute("INSERT INTO witness VALUES(zeroblob(1048576))", [])
|
||||
.unwrap_err();
|
||||
let wrapped = anyhow::Error::new(error).context("private SQL context must never leak");
|
||||
assert_eq!(
|
||||
safe_error_code(&wrapped),
|
||||
"JOURNAL_STORAGE_CAPACITY_BLOCKED"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM witness", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA integrity_check", [], |r| r.get::<_, String>(0))
|
||||
.unwrap(),
|
||||
"ok"
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4
|
||||
);
|
||||
assert_eq!(
|
||||
safe_error_code(&anyhow::anyhow!("private SQL text")),
|
||||
"HELPER_VALIDATION_OR_IO_FAILURE"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn apt_only_profile_denies_scripts_services_reboot_and_wrong_shared_state() {
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let mut config:Config=serde_json::from_value(serde_json::json!({"profile":"apt_only",
|
||||
|
||||
@@ -85,6 +85,57 @@ fn directory() -> std::path::PathBuf {
|
||||
path
|
||||
}
|
||||
#[test]
|
||||
fn journal_capacity_is_finite_and_retains_original_consumption_on_reopen() {
|
||||
let dir = directory();
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
let mut db = open_ledger(&dir, uid).unwrap();
|
||||
let (binding, request, _, now) = fixture();
|
||||
let (_, permit) = binding.verify(&request, now).unwrap();
|
||||
consume(&mut db, &permit).unwrap();
|
||||
save_start(&db, &permit, &request).unwrap();
|
||||
let original: Vec<u8> = db
|
||||
.query_row("SELECT start FROM execution_evidence", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
|
||||
drop(db);
|
||||
let mut db = open_ledger(&dir, uid).unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4096
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
65536
|
||||
);
|
||||
let preserved: (String, Option<Vec<u8>>, Vec<u8>) = db.query_row("SELECT c.grant_id,c.result,e.start FROM consumed c JOIN execution_evidence e USING(grant_id)", [], |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(preserved, (permit.grant_id.to_string(), None, original));
|
||||
assert!(consume(&mut db, &permit).is_err());
|
||||
drop(db);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_capacity_rejects_unexpected_page_size_without_rewriting_history() {
|
||||
let dir = directory();
|
||||
let db = rusqlite::Connection::open(dir.join("odd.sqlite")).unwrap();
|
||||
db.execute_batch("PRAGMA page_size=8192;CREATE TABLE witness(value TEXT);INSERT INTO witness VALUES('retained');").unwrap();
|
||||
assert!(bounded_journal_capacity(&db).is_err());
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
8192
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT value FROM witness", [], |r| r.get::<_, String>(0))
|
||||
.unwrap(),
|
||||
"retained"
|
||||
);
|
||||
drop(db);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn exact_signature_and_every_permit_binding() {
|
||||
let (binding, request, key, now) = fixture();
|
||||
assert!(binding.verify(&request, now).is_ok());
|
||||
|
||||
@@ -157,6 +157,7 @@ fn load() -> Result<Option<Loaded>> {
|
||||
}))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
super::dev_transport_faults::validate_environment()?;
|
||||
load().map(|_| ())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Development-only, exact-device WSS handshake rejection after normal mTLS
|
||||
//! binding. HTTPS and every execution/approval path remain untouched.
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use rmm_domain::jobs::hash;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::PgPool;
|
||||
use std::{fs, os::unix::fs::MetadataExt, path::PathBuf};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Config {
|
||||
schema_version: u32,
|
||||
fault_id: Uuid,
|
||||
tenant_id: Uuid,
|
||||
agent_id: Uuid,
|
||||
issued_at: DateTime<Utc>,
|
||||
expires_at: DateTime<Utc>,
|
||||
}
|
||||
impl Config {
|
||||
fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == 1
|
||||
&& !self.fault_id.is_nil()
|
||||
&& !self.tenant_id.is_nil()
|
||||
&& !self.agent_id.is_nil()
|
||||
&& self.issued_at.timestamp() > 0
|
||||
&& self.expires_at > self.issued_at
|
||||
&& self.expires_at - self.issued_at <= Duration::minutes(15),
|
||||
"DEV_TRANSPORT_FAULT_CONFIG"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn active(&self, tenant: Uuid, agent: Uuid, now: DateTime<Utc>) -> bool {
|
||||
self.tenant_id == tenant
|
||||
&& self.agent_id == agent
|
||||
&& now >= self.issued_at
|
||||
&& now < self.expires_at
|
||||
}
|
||||
}
|
||||
fn configured_path(
|
||||
flag: Option<&str>,
|
||||
path: Option<&str>,
|
||||
profile: &str,
|
||||
dev: &str,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
match (flag, path) {
|
||||
(None | Some("false"), None) => Ok(None),
|
||||
(Some("true"), Some("/root-policy/transport-fault.json")) => {
|
||||
ensure!(
|
||||
profile == "development" && dev == "true",
|
||||
"DEV_TRANSPORT_FAULT_PROFILE"
|
||||
);
|
||||
Ok(Some(PathBuf::from("/root-policy/transport-fault.json")))
|
||||
}
|
||||
_ => anyhow::bail!("DEV_TRANSPORT_FAULT_EXPLICIT_OPT_IN"),
|
||||
}
|
||||
}
|
||||
fn load() -> Result<Option<(Config, String)>> {
|
||||
let flag = std::env::var("RMM_DEV_TRANSPORT_FAULTS").ok();
|
||||
let path = std::env::var("RMM_DEV_TRANSPORT_FAULT_FILE").ok();
|
||||
let profile = std::env::var("RMM_PROFILE").unwrap_or_default();
|
||||
let dev = std::env::var("RMM_DEV_AUTH").unwrap_or_default();
|
||||
let Some(path) = configured_path(flag.as_deref(), path.as_deref(), &profile, &dev)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = fs::symlink_metadata(path.parent().context("DEV_TRANSPORT_FAULT_PARENT")?)?;
|
||||
let metadata = fs::symlink_metadata(&path)?;
|
||||
ensure!(
|
||||
directory.is_dir()
|
||||
&& directory.uid() == 10001
|
||||
&& directory.mode() & 0o777 == 0o700
|
||||
&& metadata.is_file()
|
||||
&& metadata.uid() == 10001
|
||||
&& metadata.mode() & 0o777 == 0o600
|
||||
&& metadata.nlink() == 1
|
||||
&& metadata.len() <= 4096,
|
||||
"DEV_TRANSPORT_FAULT_PRIVATE_CONFIG"
|
||||
);
|
||||
let raw = fs::read(path)?;
|
||||
ensure!(raw.len() <= 4096, "DEV_TRANSPORT_FAULT_CONFIG_SIZE");
|
||||
let config: Config = serde_json::from_slice(&raw)?;
|
||||
config.validate()?;
|
||||
Ok(Some((config, hash(&raw))))
|
||||
}
|
||||
pub(crate) fn validate_environment() -> Result<()> {
|
||||
load().map(|_| ())
|
||||
}
|
||||
fn slot_id(fault: Uuid, slot: u8) -> Uuid {
|
||||
use sha2::{Digest, Sha256};
|
||||
let digest = Sha256::digest(format!("rmm.dev-transport-fault.v1:{fault}:{slot}"));
|
||||
Uuid::from_slice(&digest[..16]).expect("fixed digest prefix")
|
||||
}
|
||||
fn same_configuration(value: &Value, sha256: &str) -> Result<()> {
|
||||
ensure!(
|
||||
value.get("config_sha256") == Some(&json!(sha256)),
|
||||
"DEV_TRANSPORT_FAULT_ID_REUSED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Caller has already checked certificate expiry and authoritative device_binding.
|
||||
/// No upgrade/ClientHello/JobOffer occurs on this path. At most two narrow durable
|
||||
/// observations are inserted; every matching WSS request stays denied until expiry.
|
||||
pub(crate) async fn reject_wss(
|
||||
pool: &PgPool,
|
||||
tenant: Uuid,
|
||||
agent: Uuid,
|
||||
certificate_sha256: &str,
|
||||
) -> Result<bool> {
|
||||
let Some((config, sha256)) = load()? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !config.active(tenant, agent, Utc::now()) {
|
||||
return Ok(false);
|
||||
}
|
||||
ensure!(
|
||||
certificate_sha256.len() == 64
|
||||
&& certificate_sha256
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
|
||||
"DEV_TRANSPORT_FAULT_PEER_FINGERPRINT"
|
||||
);
|
||||
let mut tx = rmm_persistence::tenant_tx(pool, tenant).await?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))")
|
||||
.bind(format!("dev-transport-fault:{}", config.fault_id))
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
for slot in 1..=2 {
|
||||
let id = slot_id(config.fault_id, slot);
|
||||
let existing: Option<Value> = sqlx::query_scalar(
|
||||
"SELECT data FROM audit_events WHERE event_id=$1 AND principal='dev_transport_fault_adapter'",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
if let Some(value) = existing {
|
||||
same_configuration(&value, &sha256)?;
|
||||
} else {
|
||||
let value = json!({"schema_version":1,"testsimulation":true,"noProductionEvidence":true,
|
||||
"fault_id":config.fault_id,"agent_id":agent,"tenant_id":tenant,"slot":slot,
|
||||
"config_sha256":sha256,"certificate_sha256":certificate_sha256,
|
||||
"handshake_id":Uuid::new_v4(),"status":503,
|
||||
"detail":"authenticated_mtls_device_wss_rejected_before_upgrade"});
|
||||
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,'dev_transport_fault_adapter','dev_transport_fault.wss_rejected',$3) ON CONFLICT(tenant_id,event_id) DO NOTHING")
|
||||
.bind(tenant).bind(id).bind(value).execute(&mut *tx).await?;
|
||||
break;
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn opt_in_is_explicit_and_production_activation_is_rejected() {
|
||||
assert!(
|
||||
configured_path(None, None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
configured_path(Some("false"), None, "production", "false")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for (flag, path, profile, dev) in [
|
||||
(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"production",
|
||||
"true",
|
||||
),
|
||||
(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"false",
|
||||
),
|
||||
(
|
||||
Some("false"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"true",
|
||||
),
|
||||
(Some("true"), None, "development", "true"),
|
||||
(Some("true"), Some("/etc/passwd"), "development", "true"),
|
||||
] {
|
||||
assert!(configured_path(flag, path, profile, dev).is_err());
|
||||
}
|
||||
assert!(
|
||||
configured_path(
|
||||
Some("true"),
|
||||
Some("/root-policy/transport-fault.json"),
|
||||
"development",
|
||||
"true"
|
||||
)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn exact_scope_and_bounded_window_never_affect_another_device() {
|
||||
let now = Utc::now();
|
||||
let mut config = Config {
|
||||
schema_version: 1,
|
||||
fault_id: Uuid::new_v4(),
|
||||
tenant_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
issued_at: now,
|
||||
expires_at: now + Duration::minutes(15),
|
||||
};
|
||||
config.validate().unwrap();
|
||||
assert!(config.active(config.tenant_id, config.agent_id, now));
|
||||
assert!(!config.active(Uuid::new_v4(), config.agent_id, now));
|
||||
assert!(!config.active(config.tenant_id, Uuid::new_v4(), now));
|
||||
assert!(!config.active(
|
||||
config.tenant_id,
|
||||
config.agent_id,
|
||||
now - Duration::seconds(1)
|
||||
));
|
||||
assert!(!config.active(config.tenant_id, config.agent_id, config.expires_at));
|
||||
config.expires_at += Duration::milliseconds(1);
|
||||
assert!(config.validate().is_err());
|
||||
config.expires_at = now;
|
||||
assert!(config.validate().is_err());
|
||||
config.expires_at = now + Duration::minutes(1);
|
||||
config.agent_id = Uuid::nil();
|
||||
assert!(config.validate().is_err());
|
||||
}
|
||||
#[test]
|
||||
fn slots_are_bounded_stable_and_cannot_change_configuration() {
|
||||
let fault = Uuid::new_v4();
|
||||
assert_eq!(slot_id(fault, 1), slot_id(fault, 1));
|
||||
assert_ne!(slot_id(fault, 1), slot_id(fault, 2));
|
||||
assert_ne!(slot_id(fault, 1), slot_id(Uuid::new_v4(), 1));
|
||||
let value = json!({"config_sha256":"a".repeat(64)});
|
||||
same_configuration(&value, &"a".repeat(64)).unwrap();
|
||||
assert!(same_configuration(&value, &"b".repeat(64)).is_err());
|
||||
assert!(same_configuration(&json!({}), &"a".repeat(64)).is_err());
|
||||
}
|
||||
}
|
||||
@@ -92,6 +92,12 @@ pub async fn connect(
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?;
|
||||
let (tenant, agent) = binding.ok_or(StatusCode::UNAUTHORIZED)?;
|
||||
if dev_transport_faults::reject_wss(&state.pool, tenant, agent, &fingerprint)
|
||||
.await
|
||||
.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?
|
||||
{
|
||||
return Err(StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
Ok(ws
|
||||
.max_message_size(MAX_MESSAGE)
|
||||
.max_frame_size(MAX_MESSAGE)
|
||||
@@ -226,3 +232,6 @@ async fn channel(
|
||||
}
|
||||
#[path = "dev_delivery_faults.rs"]
|
||||
pub(crate) mod dev_delivery_faults;
|
||||
|
||||
#[path = "dev_transport_faults.rs"]
|
||||
pub(crate) mod dev_transport_faults;
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Actual visible dev/test UI only; no API/DB approval or injected requests.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const { isDeepStrictEqual: equal } = require('node:util');
|
||||
const { chromium } = require('../../web/node_modules/playwright');
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const assert = value => { if (!value) throw new Error('Crash UI contract mismatch'); };
|
||||
function readPrivate(file) {
|
||||
const metadata = fs.lstatSync(file);
|
||||
assert(metadata.isFile() && !metadata.isSymbolicLink() && metadata.uid === process.getuid()
|
||||
&& (metadata.mode & 0o777) === 0o600 && metadata.size < 1048576);
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
function checkJob(plan, expected, script) {
|
||||
assert(plan.jobs.length === 1 && plan.resolved_targets.length === 1
|
||||
&& crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
|
||||
const job = plan.jobs[0], p = job.payload;
|
||||
assert(p.tenant_id === expected.tenant_id && p.agent_id === expected.agent_id && p.asset_id === expected.asset_id
|
||||
&& p.execution_identity === 'root' && p.max_runtime_seconds === 120 && p.max_output_bytes === 65536
|
||||
&& Date.parse(p.start_before) > Date.now());
|
||||
assert(equal(p.action, { action: 'run_script', interpreter: 'posix_sh', script_utf8: script,
|
||||
arguments: [], expected_stdout: 'RMM_CRASH_COUNT=1\n' }));
|
||||
}
|
||||
(async () => {
|
||||
const file = path.resolve(process.argv[2]), expected = readPrivate(file);
|
||||
assert(expected.profile === 'dev/test' && expected.case === 'agent_crash'
|
||||
&& expected.expected_principal === 'simulated-testprincipal-a' && typeof expected.read_only === 'boolean'
|
||||
&& /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(expected.id));
|
||||
const source = fs.readFileSync(path.join(root, 'deploy/linux/agent-crash-counter.py'), 'utf8');
|
||||
const script = 'exec /usr/bin/python3 -I - ' + (expected.read_only ? 'read' : 'effect') + ' ' + expected.id
|
||||
+ " <<'RMM_AGENT_CRASH_COUNTER'\n" + source + '\nRMM_AGENT_CRASH_COUNTER\n';
|
||||
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
|
||||
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
|
||||
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
|
||||
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
|
||||
const browser = await chromium.launch({ headless: true, executablePath,
|
||||
env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
|
||||
try {
|
||||
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' });page.setDefaultTimeout(20000);
|
||||
let approvals = 0;
|
||||
page.on('request', r => { if (new URL(r.url()).pathname.endsWith('/authorizations')) approvals++; });
|
||||
await page.goto('https://localhost:18443/');
|
||||
await page.getByLabel('Benutzername').fill(login.username);
|
||||
await page.getByLabel('Passwort', { exact: true }).fill(login.password);
|
||||
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
|
||||
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
|
||||
const session = await (await response).json();
|
||||
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
|
||||
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
|
||||
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
|
||||
if (expected.phase === 'prepare') {
|
||||
assert(expected.request.parameters.variants.length === 1 && expected.request.parameters.variants[0].script_utf8 === script
|
||||
&& expected.request.parameters.expected_stdout === 'RMM_CRASH_COUNT=1\n');
|
||||
await page.getByRole('checkbox', { name: 'Ziel ' + expected.asset_id, exact: true }).check();
|
||||
await page.getByLabel('Aktion', { exact: true }).selectOption('run_script');
|
||||
await page.getByRole('combobox', { name: 'Interpreter', exact: true }).selectOption('posix_sh');
|
||||
await page.getByLabel('Exakte UTF-8 Skriptbytes', { exact: false }).fill(script);
|
||||
await page.getByLabel('Erwartete stdout-Ausgabe', { exact: false }).fill('RMM_CRASH_COUNT=1\n');
|
||||
await page.getByLabel('Ausführungsidentität', { exact: true }).selectOption('root');
|
||||
await page.getByLabel('Laufzeitlimit (s)', { exact: true }).fill('120');
|
||||
await page.getByLabel('Ausgabelimit (Bytes)', { exact: true }).fill('65536');
|
||||
const start = new Date(Math.ceil(Date.now() / 60000) * 60000);
|
||||
await page.getByLabel('Frühester Start (lokale Zeit)', { exact: true }).fill(start.toISOString().slice(0, 16));
|
||||
await page.getByLabel('Startfrist / Ablauf (lokale Zeit)', { exact: true }).fill(new Date(start.getTime() + 15 * 60000).toISOString().slice(0, 16));
|
||||
await page.getByLabel('Maximal parallel', { exact: true }).fill('1');
|
||||
await page.getByLabel('Stopp nach Fehlern', { exact: true }).fill('1');
|
||||
await page.getByLabel('Begründung', { exact: true }).fill(expected.request.reason);
|
||||
await page.getByRole('button', { name: 'Plan erstellen · noch keine Ausführung' }).click();
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());checkJob(plan, expected, script);
|
||||
await detail.getByRole('button', { name: 'Antrag absenden · pending_approval' }).click();
|
||||
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: /^Antrag \/ Ausführung / }) });
|
||||
await operation.locator('pre').first().filter({ hasText: '"job_count": 0' }).waitFor();
|
||||
const pending = JSON.parse(await operation.locator('pre').first().innerText());
|
||||
assert(pending.approval_state === 'pending_approval' && pending.execution_state === null && pending.job_count === 0 && approvals === 0);
|
||||
assert(path.dirname(path.resolve(expected.output_file)) === path.dirname(file));
|
||||
fs.writeFileSync(expected.output_file, JSON.stringify({ plan, operation: pending }), { flag: 'wx', mode: 0o600 });
|
||||
} else {
|
||||
assert(expected.phase === 'approve');
|
||||
await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
|
||||
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
|
||||
const pending = JSON.parse(await operation.locator('pre').first().innerText());
|
||||
assert(pending.approval_state === 'pending_approval' && pending.execution_state === null && pending.job_count === 0 && approvals === 0);
|
||||
await operation.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen', exact: true }).click();
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());
|
||||
const { approval_state: currentState, ...currentPlan } = plan;
|
||||
const { approval_state: preparedState, ...preparedPlan } = expected.plan;
|
||||
assert(currentState === 'pending_approval' && preparedState === null && equal(currentPlan, preparedPlan));
|
||||
checkJob(plan, expected, script);assert(equal(plan.jobs[0], expected.job));
|
||||
const approve = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true });
|
||||
assert(await approve.isDisabled());
|
||||
await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check();
|
||||
await approve.click();
|
||||
await operation.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
|
||||
assert(approvals === 1);
|
||||
}
|
||||
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
|
||||
} finally { await browser.close(); }
|
||||
})().catch(error => {
|
||||
const line = error.stack?.match(/cloud_crash_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
|
||||
console.log('CLOUD_CRASH_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line);process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
// Actual development UI approval/revocation only; no API client or injected fetch.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const { isDeepStrictEqual: equal } = require('node:util');
|
||||
const { chromium } = require('../../web/node_modules/playwright');
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const assert = value => { if (!value) throw new Error('Offline UI contract mismatch'); };
|
||||
function readPrivate(file) {
|
||||
const s = fs.lstatSync(file);
|
||||
assert(s.isFile() && !s.isSymbolicLink() && s.uid === process.getuid() && (s.mode & 0o777) === 0o600 && s.size < 1048576);
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
function counterScript(id) {
|
||||
assert(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(id));
|
||||
return 'set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-' + id + '\n'
|
||||
+ 'test ! -e "$d"\nmkdir "$d"\nprintf \'%s\\n\' \'1\' > "$d/count"\n'
|
||||
+ 'test "$(cat "$d/count")" = 1\nprintf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n';
|
||||
}
|
||||
(async () => {
|
||||
const expected = readPrivate(path.resolve(process.argv[2]));
|
||||
assert(expected.profile === 'dev/test' && ['offline_wait', 'expired_offline', 'revoked_before_start'].includes(expected.case)
|
||||
&& ['approve', 'revoke'].includes(expected.phase) && expected.expected_principal === 'simulated-testprincipal-a');
|
||||
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
|
||||
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
|
||||
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
|
||||
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
|
||||
const browser = await chromium.launch({ headless: true, executablePath, env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
|
||||
try {
|
||||
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' }); page.setDefaultTimeout(15000);
|
||||
let approvals = 0, revocations = 0;
|
||||
page.on('request', r => { const p = new URL(r.url()).pathname; if (p.endsWith('/authorizations')) approvals++; if (p.endsWith('/revocation') && r.method() === 'POST') revocations++; });
|
||||
await page.goto('https://localhost:18443/');
|
||||
await page.getByLabel('Benutzername').fill(login.username); await page.getByLabel('Passwort', { exact: true }).fill(login.password);
|
||||
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
|
||||
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
|
||||
const session = await (await response).json();
|
||||
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
|
||||
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
|
||||
assert(await page.getByLabel('Mandant', { exact: true }).inputValue() === expected.tenant_id);
|
||||
await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
|
||||
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
|
||||
const state = JSON.parse(await operation.locator('pre').first().innerText());
|
||||
assert(state.approval_state === (expected.phase === 'approve' ? 'pending_approval' : 'approved'));
|
||||
if (expected.phase === 'approve') assert(state.job_count === 0 && state.execution_state === null);
|
||||
await operation.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen', exact: true }).click();
|
||||
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());
|
||||
const { approval_state: actualApproval, ...actual } = plan;
|
||||
const { approval_state: plannedApproval, ...planned } = expected.plan;
|
||||
assert(equal(actual, planned) && plan.jobs.length === 1 && plan.resolved_targets.length === 1
|
||||
&& crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
|
||||
const job = plan.jobs[0], p = job.payload;
|
||||
assert(equal(job, expected.job) && p.agent_id === expected.agent_id && p.asset_id === expected.asset_id && p.tenant_id === expected.tenant_id
|
||||
&& p.execution_identity === 'agent_service' && p.max_runtime_seconds === 60 && p.max_output_bytes === 65536);
|
||||
assert(equal(p.action, { action: 'run_script', interpreter: 'posix_sh', script_utf8: counterScript(expected.id), arguments: [], expected_stdout: 'RMM_OFFLINE_COUNT=1\n' }));
|
||||
assert(Date.parse(p.start_before) - Date.parse(p.not_before) === 130000 && Date.parse(p.start_before) > Date.now());
|
||||
if (expected.phase === 'approve') {
|
||||
assert(actualApproval === 'pending_approval' && plannedApproval === null && Date.parse(p.not_before) > Date.now() + 90000);
|
||||
const approve = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true });
|
||||
assert(await approve.isDisabled()); await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check(); await approve.click();
|
||||
await operation.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
|
||||
assert(approvals === 1 && revocations === 0);
|
||||
} else {
|
||||
assert(expected.case === 'revoked_before_start' && actualApproval === 'approved' && session.permissions.includes('plans:revoke') && Date.parse(p.not_before) > Date.now() + 1000);
|
||||
const control = detail.locator('details').filter({ has: page.locator('summary').filter({ hasText: /^Planberechtigung widerrufen$/ }) });
|
||||
await control.locator('summary').click();
|
||||
await control.getByLabel('Grund für Planberechtigung widerrufen', { exact: true }).fill('Explicit dev/test offline acceptance: revoke this exact future plan before any start');
|
||||
const nonce = await control.locator('code').innerText();
|
||||
await control.getByLabel('Bestätigungskennung eingeben', { exact: true }).fill(nonce);
|
||||
await control.getByRole('checkbox', { name: 'Ich bestätige diese konkrete administrative Wirkung ausdrücklich.' }).check();
|
||||
await control.getByRole('button', { name: 'Planberechtigung widerrufen · ausdrücklich ausführen', exact: true }).click();
|
||||
await control.locator('pre').filter({ hasText: '"approval_state": "revoked"' }).waitFor();
|
||||
assert(approvals === 0 && revocations === 1);
|
||||
}
|
||||
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
|
||||
} finally { await browser.close(); }
|
||||
})().catch(error => {
|
||||
const line = error.stack?.match(/cloud_offline_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
|
||||
console.log('CLOUD_OFFLINE_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line); process.exitCode = 1;
|
||||
});
|
||||
@@ -19,6 +19,15 @@ STATE=BASE/'cases'
|
||||
CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_updates')
|
||||
|
||||
|
||||
def runtime_session():
|
||||
latest=BASE/'journal-capacity-runtime.json'
|
||||
if latest.exists():
|
||||
report=cloud.contract.protected_json(latest)
|
||||
require(report['completed'] and report['preserved_state'],'qualified preserved capacity runtime')
|
||||
return report,BASE/'capacity-recovery'
|
||||
return cloud.contract.protected_json(BASE/'configure-runtime.json'),BASE
|
||||
|
||||
|
||||
def recovery_request(device,target):
|
||||
m=cloud.contract;data=prior.adapters.planning_from_device(device,recovery=True)
|
||||
closed=m.protected_json(BASE/'readonly-central-result.json')['result']
|
||||
@@ -49,14 +58,14 @@ def recovery_request(device,target):
|
||||
|
||||
def enable():
|
||||
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready=m.protected_json(BASE/'configure-runtime.json')
|
||||
ready,session=runtime_session();session.mkdir(mode=0o700,exist_ok=True)
|
||||
require(ready['completed'] and ready['readiness_only'] and ready['preserved_state'] and not ready['native_acceptance'],
|
||||
'qualified native probe, no invented package acceptance')
|
||||
target=prior.prior.identity();pin=ready['apt_pin'];identity=pin['identity']
|
||||
require(all(identity[k]==target[k] for k in target),'same permanent Cloud VM')
|
||||
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=True,
|
||||
agent_id=target['agent_id'],binary_sha256=ready['helper_sha256']))
|
||||
m.write_private(BASE/'recovery-endpoint-enabled.json',endpoint)
|
||||
m.write_private(session/'recovery-endpoint-enabled.json',endpoint)
|
||||
peers=cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,target)
|
||||
require(peers['ledger']['faults']==0 and peers['apt']['provider_stack_sha256']==pin['provider_stack_sha256'],
|
||||
'actual fixed helper peer and stack')
|
||||
@@ -67,15 +76,43 @@ def enable():
|
||||
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
|
||||
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
|
||||
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
|
||||
old=m.protected_json(cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
|
||||
m.write_private(BASE/'recovery-policy.json',dict(record,policy=policy))
|
||||
m.write_private(session/'recovery-policy.json',dict(record,policy=policy))
|
||||
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
|
||||
|
||||
|
||||
def accept(case):
|
||||
def unsubmitted_preparation(payload):
|
||||
import re
|
||||
root=Path('/opt/rust-discovery');ident=payload['test_result_id']
|
||||
require(isinstance(ident,str) and re.fullmatch('[A-Za-z0-9_-]{1,128}',ident),'fixed preparation test ID')
|
||||
query="SELECT count(*) FROM operation_plans WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND position('"+ident+"' in convert_from(payload,'UTF8'))>0;"
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
count=int(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip())
|
||||
return dict(persisted_plans=count)
|
||||
|
||||
|
||||
def accept(case,resume_unsubmitted=False):
|
||||
m=cloud.contract;target=prior.prior.identity()
|
||||
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
|
||||
if session!=BASE and case=='apt_configure_recovery':
|
||||
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
|
||||
old_job=original['plan']['jobs'][0]
|
||||
old_result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=old_job['job_id']))
|
||||
counts=prior.prior.evidence('gate',job_id=old_job['job_id'])
|
||||
require(original['not_started_capacity_block'] and old_result['state'] in ('expired','cancelled') and counts['permits']==0,
|
||||
'preserve original approved no-start; new attempt requires fresh UI approval')
|
||||
folder=cases/case
|
||||
if resume_unsubmitted:
|
||||
require(folder.is_dir() and {p.name for p in folder.iterdir()}=={'prepare.json'},'only an unsubmitted UI preparation may be archived')
|
||||
old=m.protected_json(folder/'prepare.json')
|
||||
require(old['case']==case and old['phase']=='prepare' and all(old[k]==target[k] for k in target),'same fixed prior preparation target')
|
||||
untouched=cloud.bootstrap.rpc('server.conf',unsubmitted_preparation,dict(test_result_id=old['request']['parameters']['apt_manifest']['test_result_id']))
|
||||
require(untouched==dict(persisted_plans=0),'existing plan must be reviewed, never silently replaced')
|
||||
archived=cases/(case+'.unsubmitted-ui-'+uuid.uuid4().hex)
|
||||
os.rename(folder,archived);m.write_private(archived/'not-submitted.json',untouched)
|
||||
# A maintenance restart only refreshes passive inventory from the preserved
|
||||
# agent; it never issues a package diagnostic or executes a job.
|
||||
deadline=time.monotonic()+90
|
||||
@@ -87,28 +124,30 @@ def accept(case):
|
||||
except m.GuardError:
|
||||
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
|
||||
request=recovery_request(device,target) if case=='apt_configure_recovery' else None
|
||||
prior.accept(case,request=request,folder_root=STATE)
|
||||
if request is not None and session!=BASE:
|
||||
request['reason']+='; prior fixed plan expired with zero grants at the old journal capacity; preserve it and authorize this fresh plan separately'
|
||||
prior.accept(case,request=request,folder_root=cases)
|
||||
|
||||
|
||||
def close():
|
||||
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
record=m.protected_json(BASE/'recovery-policy.json')
|
||||
ready,session=runtime_session();record=m.protected_json(session/'recovery-policy.json')
|
||||
closed=cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=record['policy_sha256'],backup=record['backup']))
|
||||
m.write_private(BASE/'recovery-policy-closed.json',closed)
|
||||
ready=m.protected_json(BASE/'configure-runtime.json')
|
||||
m.write_private(session/'recovery-policy-closed.json',closed)
|
||||
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=False,
|
||||
agent_id=prior.prior.identity()['agent_id'],binary_sha256=ready['helper_sha256']))
|
||||
m.write_private(BASE/'recovery-endpoint-disabled.json',endpoint)
|
||||
m.write_private(session/'recovery-endpoint-disabled.json',endpoint)
|
||||
print('APT test limits closed and endpoint stopped; original histories retained')
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
|
||||
p.add_argument('--resume-unsubmitted-ui',action='store_true',help='Preserve a failed preparation only after proving no plan was persisted')
|
||||
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
|
||||
if args.action in ('accept','finish'):
|
||||
require(args.case is not None,'one fixed case required')
|
||||
if args.action=='accept':accept(args.case)
|
||||
else:prior.finish(args.case,folder_root=STATE)
|
||||
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui)
|
||||
else:prior.finish(args.case,folder_root=runtime_session()[1]/'cases')
|
||||
else:(enable if args.action=='enable' else close)()
|
||||
|
||||
|
||||
@@ -116,5 +155,5 @@ if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
message=str(error)
|
||||
print(message if message.startswith('remote bootstrap incomplete: ') else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
||||
print(message if message.startswith(('remote bootstrap incomplete: ','APT_UI_INCOMPLETE:source_lines=')) else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
||||
raise SystemExit(2) from None
|
||||
|
||||
@@ -205,7 +205,10 @@ def accept(case, *, request=None, folder_root=STATE):
|
||||
cloud.prepare_browser()
|
||||
def browser(file):
|
||||
result = subprocess.run(['node', str(cloud.ROOT/'deploy/browser/cloud_apt_acceptance.cjs'), str(file)], capture_output=True, timeout=120)
|
||||
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint, never repeat automatically')
|
||||
if result.returncode!=0:
|
||||
import re
|
||||
lines=re.findall(rb'CLOUD_APT_UI_INCOMPLETE:[A-Za-z]+:source_line=([0-9]+|unknown)',result.stdout)
|
||||
raise RuntimeError('APT_UI_INCOMPLETE:source_lines='+','.join(x.decode('ascii') for x in lines))
|
||||
print(case+': actual UI preparation', flush=True); browser(prepare)
|
||||
prepared = m.protected_json(output); plan = prepared['plan']; operation = prepared['operation']; job = plan['jobs'][0]
|
||||
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'immutable single job')
|
||||
|
||||
@@ -119,6 +119,13 @@ def status_vm(payload):
|
||||
result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0],
|
||||
receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0,
|
||||
integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok')
|
||||
if payload.get('job_id'):
|
||||
job=str(uuid.UUID(payload['job_id']));attempt=str(uuid.UUID(payload['attempt_id']))
|
||||
if name=='agent':
|
||||
result[name]['attempts']=[dict(state=s,revision=r,grant_id=g) for s,r,g in db.execute(
|
||||
'SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))]
|
||||
else:
|
||||
result[name]['attempt_consumed']=db.execute('SELECT count(*) FROM consumed WHERE job=? AND attempt=?',(job,attempt)).fetchone()[0]
|
||||
if name=='helper':
|
||||
result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
||||
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One bounded Cloud agent SIGKILL; independent root helper closes its outcome.
|
||||
|
||||
Requires fixed dev/test UI approval twice, a qualified agent and known closed
|
||||
root-policy hash. No API/DB approval, ledger edits, retry or unsigned recovery.
|
||||
Do not execute alongside another VM/runtime/approval operation.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import cloud_delivery_acceptance as delivery
|
||||
import cloud_apt_followup as prior
|
||||
import cloud_root_evidence
|
||||
import cloud_root_policy
|
||||
|
||||
cloud = delivery.cloud
|
||||
require = cloud.contract.require
|
||||
STATE = cloud.ROOT/'devstate/cloud-crash'
|
||||
|
||||
|
||||
def script(identifier, read_only=False):
|
||||
identifier = str(uuid.UUID(identifier))
|
||||
source = (cloud.ROOT/'deploy/linux/agent-crash-counter.py').read_text()
|
||||
return ("exec /usr/bin/python3 -I - "+('read' if read_only else 'effect')+' '+identifier+
|
||||
" <<'RMM_AGENT_CRASH_COUNTER'\n"+source+'\nRMM_AGENT_CRASH_COUNTER\n')
|
||||
|
||||
|
||||
def vm(payload):
|
||||
"""Read retained evidence; optionally signal only the exact bound MainPID."""
|
||||
import signal
|
||||
import sqlite3
|
||||
import stat
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'permanent identity')
|
||||
for path,key in ((Path('/usr/local/libexec/rmm-agent-remote'),'agent_sha256'),
|
||||
(Path('/usr/local/libexec/rmm-privileged-helper'),'helper_sha256')):
|
||||
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == payload[key],'qualified pinned artifact')
|
||||
def process(unit, expected_sha):
|
||||
raw = subprocess.run(['systemctl','show',unit,'-p','MainPID','--value'],capture_output=True,check=True,timeout=10).stdout
|
||||
pid = int(raw);require(pid > 1,'active pinned service')
|
||||
require(hashlib.sha256(Path('/proc/'+str(pid)+'/exe').read_bytes()).hexdigest() == expected_sha,'running service artifact')
|
||||
start = Path('/proc/'+str(pid)+'/stat').read_text().rsplit(')',1)[1].split()[19]
|
||||
return dict(pid=pid,start=start)
|
||||
processes = dict(agent=process('rmm-agent-remote.service',payload['agent_sha256']),
|
||||
helper=process('rmm-helper-cloud.service',payload['helper_sha256']))
|
||||
result = dict(processes=processes)
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','native journal integrity')
|
||||
result['native_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
||||
if payload.get('job_id'):
|
||||
job = str(uuid.UUID(payload['job_id']));attempt = str(uuid.UUID(payload['attempt_id']))
|
||||
rows = db.execute('SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
|
||||
result['attempts'] = [dict(state=r[0],revision=r[1],grant_id=r[2]) for r in rows]
|
||||
result['agent_last_decision'] = db.execute('SELECT last_decision FROM reconcile_meta').fetchone()[0]
|
||||
helper = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
with sqlite3.connect(helper.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','root ledger integrity')
|
||||
result['ledger'] = dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
||||
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
|
||||
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0])
|
||||
if payload.get('job_id'):
|
||||
rows = db.execute('SELECT c.grant_id,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.job=? AND c.attempt=?',(job,attempt)).fetchall()
|
||||
result['root_attempts'] = []
|
||||
for grant,finished,start,outcome in rows:
|
||||
stored = json.loads(start);request = stored['request'];permit = json.loads(bytes(request['permit_bytes']))
|
||||
require(permit['job_id'] == job and permit['attempt_id'] == attempt and permit['agent_id'] == payload['agent_id']
|
||||
and permit['grant_id'] == grant and hashlib.sha256(bytes(request['job_bytes'])).hexdigest() == payload['job_payload_sha256']
|
||||
and request['plan_hash'] == payload['plan_sha256'],'exact durable root start binding')
|
||||
result['root_attempts'].append(dict(grant_id=grant,finished=json.loads(finished) if finished else None,
|
||||
outcome=json.loads(outcome) if outcome else None,start_sha256=hashlib.sha256(start).hexdigest()))
|
||||
head = db.execute('SELECT last_decision FROM helper_sync_meta').fetchone()[0]
|
||||
if head:
|
||||
row = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?',(head,)).fetchone()
|
||||
if row:
|
||||
signed = json.loads(row[0]);decision = json.loads(bytes(signed['decision_bytes']))
|
||||
selected = [a for a in decision['attempts'] if a['job_id'] == job and a['attempt_id'] == attempt]
|
||||
result['signed_sync'] = dict(decision_id=head,ready=decision['ready'],helper_ready=decision['helper_ready'],
|
||||
payload_sha256=hashlib.sha256(bytes(signed['decision_bytes'])).hexdigest(),signature_sha256=hashlib.sha256(bytes(signed['signature'])).hexdigest(),
|
||||
selected=selected)
|
||||
if payload.get('job_id'):
|
||||
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
result['agent_consumed'] = [dict(grant_id=g,finished=json.loads(o) if o else None) for g,o in db.execute('SELECT grant_id,result FROM consumed WHERE job=? AND attempt=?',(job,attempt))]
|
||||
identifier = str(uuid.UUID(payload['id']));directory = helper.parent/('crash-'+identifier)
|
||||
require(not directory.is_symlink(),'counter symlink')
|
||||
result['counter'] = None
|
||||
if directory.exists():
|
||||
require(directory.stat().st_uid == 0 and stat.S_IMODE(directory.stat().st_mode) == 0o700,'private root counter')
|
||||
path = directory/'count'
|
||||
if path.exists():
|
||||
fd = os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd,'rb') as stream:
|
||||
metadata = os.fstat(stream.fileno());require(stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0
|
||||
and metadata.st_nlink == 1 and stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16,'fixed root counter file')
|
||||
raw = stream.read();result['counter'] = int(raw)
|
||||
if payload['mode'] == 'kill':
|
||||
require(processes == payload['processes'],'exact original process instance')
|
||||
require(len(result['attempts']) == len(result['root_attempts']) == len(result['agent_consumed']) == 1
|
||||
and result['attempts'][0]['state'] == 'running' and result['root_attempts'][0]['finished'] is None
|
||||
and result['root_attempts'][0]['outcome'] is None and result['counter'] == 1
|
||||
and result['ledger'] == dict(consumed=payload['baseline_consumed']+1,unresolved=1,faults=0),'one actual running root effect; deterministic independent closure')
|
||||
require(result['attempts'][0]['grant_id'] == result['root_attempts'][0]['grant_id'] == result['agent_consumed'][0]['grant_id'],'one consumed grant on both sides')
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-agent-crash-'+identifier)
|
||||
backup.mkdir(mode=0o700)
|
||||
fd = os.open(backup/'before-signal.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:json.dump(result,stream);stream.flush();os.fsync(stream.fileno())
|
||||
# pidfd prevents a PID-reuse race. No unit stop, cgroup/process-tree kill,
|
||||
# helper signal or package-manager intervention exists here.
|
||||
fd = os.pidfd_open(processes['agent']['pid'])
|
||||
try:
|
||||
require(process('rmm-agent-remote.service',payload['agent_sha256']) == processes['agent'],'same agent before signal')
|
||||
signal.pidfd_send_signal(fd,signal.SIGKILL)
|
||||
finally: os.close(fd)
|
||||
require(process('rmm-helper-cloud.service',payload['helper_sha256']) == processes['helper'],'independent helper survived')
|
||||
return dict(before=result,agent_signal='SIGKILL',helper_unchanged=True,backup=str(backup))
|
||||
require(payload['mode'] == 'inspect','fixed fault operation')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
||||
parser.add_argument('--expected-agent-sha256',required=True)
|
||||
parser.add_argument('--expected-helper-sha256',required=True)
|
||||
parser.add_argument('--expected-root-policy-sha256',required=True)
|
||||
args = parser.parse_args();os.umask(0o077)
|
||||
for value in (args.expected_agent_sha256,args.expected_helper_sha256,args.expected_root_policy_sha256):
|
||||
require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified artifact/config pin')
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
||||
cloud.prepare_browser()
|
||||
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
settings = dict(tenant_id=cloud.TENANT,**identity)
|
||||
backend = cloud.Backend()
|
||||
def evidence(kind,**extra):return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind=kind,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**extra))
|
||||
require(evidence('gate')['unresolved'] == 0,'previous central effects must be closed')
|
||||
identifier = str(uuid.uuid4())
|
||||
scope = dict(id=identifier,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,helper_sha256=args.expected_helper_sha256)
|
||||
initial = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(scope,mode='inspect'))
|
||||
require(initial['native_unresolved'] == initial['ledger']['unresolved'] == initial['ledger']['faults'] == 0
|
||||
and initial['counter'] is None,'native/history must be settled; untouched unique counter')
|
||||
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
require(observed['sync_ready'] and not observed['restore_blocked'],'signed sync before new root plan')
|
||||
peers = cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,dict(settings,general_only=True))
|
||||
pin = peers['general'];binding = pin['identity']
|
||||
root_scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
|
||||
root_scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['policy_sha256'],
|
||||
acceptance_evidence_id='cloud-agent-crash-independent-helper:'+identifier,allowed_services=[],allow_root_scripts=True,allow_reboot=False)
|
||||
policy = dict(schema_version=1,profile='development',expires_at=(cloud.contract.now()+dt.timedelta(minutes=15)).isoformat(),scopes=[root_scope])
|
||||
folder = STATE/identifier;folder.mkdir(mode=0o700,parents=True)
|
||||
checkpoint = dict(profile='dev/test',case='agent_crash',expected_principal='simulated-testprincipal-a',**settings,
|
||||
id=identifier,initial=initial,qualified=scope,stage='prepared_no_job',no_retry=True)
|
||||
path = folder/'checkpoint.json'
|
||||
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
||||
save()
|
||||
opened = cloud.bootstrap.rpc('server.conf',prior.narrow_policy,dict(id=identifier,expected_sha256=args.expected_root_policy_sha256,policy=policy))
|
||||
checkpoint['policy_open'] = opened;save()
|
||||
def browser(value, name):
|
||||
file = folder/(name+'.json');cloud.contract.write_private(file,value)
|
||||
process = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_crash_acceptance.cjs'),str(file)],capture_output=True,timeout=120)
|
||||
require(process.returncode == 0,'actual UI incomplete; preserve pending plan; never resubmit/approve again')
|
||||
def authorize(read_only):
|
||||
# A busy helper may have temporarily removed root capabilities from the
|
||||
# new process's hello. Wait for genuine measured readvertisement/Sync;
|
||||
# never plan a new effect against a stale capability or readiness flag.
|
||||
deadline = time.monotonic()+100
|
||||
while True:
|
||||
device = backend.call('inventory.get_device',dict(tenant_id=cloud.TENANT,asset_id=identity['asset_id']))
|
||||
current = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
if (device['agent_id'] == identity['agent_id'] and device['presence'] == 'online'
|
||||
and {'run_script','root:run_script','helper.root.v1','helper.policy.sha256:'+pin['policy_sha256']} <= set(device['capabilities'])
|
||||
and current['sync_ready'] and not current['restore_blocked']):break
|
||||
require(time.monotonic() < deadline,'fresh root capability and signed Sync before new plan');time.sleep(2)
|
||||
request = cloud.contract.case_input('service_setup',settings)
|
||||
request['parameters'].update(variants=[dict(os_family='linux',interpreter='posix_sh',encoding='utf-8',script_utf8=script(identifier,read_only))],
|
||||
expected_stdout='RMM_CRASH_COUNT=1\n',timeout_seconds=120,max_output_bytes=65536)
|
||||
request['reason'] = 'Explicit bounded Cloud agent crash '+('independent final counter read' if read_only else 'one root counter effect, helper remains alive for90s')
|
||||
common = dict(profile='dev/test',case='agent_crash',expected_principal=checkpoint['expected_principal'],**settings,id=identifier,read_only=read_only,request=request)
|
||||
name = 'postcheck' if read_only else 'effect'
|
||||
output = folder/(name+'-prepared.json')
|
||||
browser(dict(common,phase='prepare',output_file=str(output)),name+'-prepare')
|
||||
prepared = cloud.contract.protected_json(output);plan=prepared['plan'];pending=prepared['operation']
|
||||
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'],'one immutable fixed plan')
|
||||
job = plan['jobs'][0];gate = evidence('gate',job_id=job['job_id'])
|
||||
stored = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,operation_id=pending['operation_id']))
|
||||
require(stored['approval_state'] == 'pending_approval' and stored['job_count'] == 0 and stored['execution_state'] is None
|
||||
and stored['plan_hash'] == plan['plan_hash'] and all(gate[k] == 0 for k in ('jobs','outbox','permits')),'pending zero artifacts')
|
||||
record = dict(plan=plan,job=job,operation_id=pending['operation_id'],preapproval=gate,request=request)
|
||||
checkpoint[name] = record;save()
|
||||
browser(dict(common,phase='approve',**record),name+'-approve')
|
||||
return record
|
||||
effect = authorize(False);job=effect['job']
|
||||
fixed = dict(scope,job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],job_payload_sha256=job['job_payload_hash'],plan_sha256=effect['plan']['plan_hash'])
|
||||
deadline = time.monotonic()+90
|
||||
while True:
|
||||
before = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
||||
if before['counter'] == 1 and before['root_attempts'] and before['root_attempts'][0]['outcome'] is None:break
|
||||
require(time.monotonic() < deadline,'counter/running marker missing; never kill outside bound window');time.sleep(1)
|
||||
killed = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='kill',processes=initial['processes'],baseline_consumed=initial['ledger']['consumed']))
|
||||
checkpoint['signal'] = killed;save()
|
||||
deadline = time.monotonic()+100
|
||||
while True:
|
||||
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
if result['state'] == 'unknown':
|
||||
unknown = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
||||
if unknown['root_attempts'][0]['outcome'] is None and unknown['attempts'][0]['state'] == 'unknown':break
|
||||
require(result['state'] not in cloud.contract.TERMINAL and time.monotonic() < deadline,'actual durable and central unknown not observed; preserve same attempt');time.sleep(1)
|
||||
require(result['result']['error_code'] == 'AGENT_PROCESS_INTERRUPTED' and unknown['counter'] == 1
|
||||
and unknown['processes']['agent'] != initial['processes']['agent']
|
||||
and unknown['processes']['helper'] == initial['processes']['helper'],'actual new agent process; independent live helper')
|
||||
checkpoint['unknown'] = dict(central=result,native=unknown);save()
|
||||
deadline = time.monotonic()+180
|
||||
while True:
|
||||
final = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
native = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
||||
sync = native.get('signed_sync',{})
|
||||
if (final['state'] in cloud.contract.TERMINAL and native['ledger']['unresolved'] == 0 and sync.get('ready') and sync.get('helper_ready')
|
||||
and native['agent_last_decision'] == sync.get('decision_id')):break
|
||||
require(time.monotonic() < deadline,'same-attempt definitive reconciliation missing; never unlock/retry');time.sleep(2)
|
||||
require(final['state'] == 'succeeded' and final['result']['exit_code'] == 0 and bytes(final['result']['stdout']) == b'RMM_CRASH_COUNT=1\n'
|
||||
and final['result']['evidence']['postconditions_passed'] and native['counter'] == 1
|
||||
and len(native['root_attempts']) == len(native['agent_consumed']) == len(native['attempts']) == 1
|
||||
and native['ledger'] == dict(consumed=initial['ledger']['consumed']+1,unresolved=0,faults=0),'one actual helper completion and consumption')
|
||||
selected = sync['selected'];grant = killed['before']['root_attempts'][0]['grant_id']
|
||||
require(len(selected) == 1 and selected[0]['grant_id'] == grant and selected[0]['unlock'] is True
|
||||
and selected[0]['outcome_state'] == 'succeeded' and selected[0]['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
|
||||
and selected[0]['revision'] > unknown['attempts'][0]['revision']
|
||||
and selected[0]['job_payload_hash'] == fixed['job_payload_sha256'] and selected[0]['plan_hash'] == fixed['plan_sha256']
|
||||
and native['root_attempts'][0]['grant_id'] == native['agent_consumed'][0]['grant_id'] == grant,'exact existing signed Sync closure')
|
||||
checkpoint.update(final=final,native=native,stage='reconciled');save()
|
||||
def audit(record):
|
||||
record['postapproval'] = evidence('gate',job_id=record['job']['job_id'])
|
||||
record['audit'] = evidence('audit',operation_id=record['operation_id']);save()
|
||||
require([record['postapproval'][k] for k in ('jobs','outbox','permits')] == [1,1,1]
|
||||
and record['audit'] == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=record['plan']['plan_hash']),'one authorized immutable attempt')
|
||||
audit(effect)
|
||||
postcheck = authorize(True);deadline=time.monotonic()+90
|
||||
while True:
|
||||
checked = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=postcheck['job']['job_id']))
|
||||
if checked['state'] in cloud.contract.TERMINAL or checked['state'] == 'unknown':break
|
||||
require(time.monotonic() < deadline,'independent read outcome missing');time.sleep(2)
|
||||
postcheck['result'] = checked;save()
|
||||
require(checked['state'] == 'succeeded' and checked['result']['exit_code'] == 0
|
||||
and checked['result']['evidence']['postconditions_passed'] and bytes(checked['result']['stdout']) == b'RMM_CRASH_COUNT=1\n','independent final counter still one')
|
||||
audit(postcheck)
|
||||
require(evidence('gate')['unresolved'] == 0,'all effects must be closed before policy close')
|
||||
closed = cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=opened['policy_sha256'],backup=opened['backup']))
|
||||
checkpoint.update(policy_closed=closed,verified=True,stage='native_dev_test_accepted');save()
|
||||
print('PASS bounded Cloud agent crash: durable+central unknown, same-attempt root outcome and signed Sync, one effect, separate UI-approved counter1; dev/test only')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_CRASH_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -31,6 +31,9 @@ def vm_pending(payload):
|
||||
events.append(dict(message_id=ident,agent_id=e[4].decode(),snapshot_id=snap[1].decode(),
|
||||
observed_at_ms=e[9],sequence=e[8],data_sha256=hashlib.sha256(snap[6]).hexdigest(),
|
||||
payload_sha256=hashlib.sha256(b''.join(raw for number,raw in parts if number!=6)).hexdigest(),
|
||||
collection_started_at=data.get('collection_started_at'),collection_completed_at=data.get('collection_completed_at'),
|
||||
observations=[dict(scope=o.get('scope'),source_id=o.get('source_id'),quality=o.get('quality'),observed_at=o.get('observed_at'),
|
||||
bytes=len(json.dumps(o,separators=(',',':')).encode())) for o in data.get('observations',[])],
|
||||
recovery_scope_present=any(o.get('scope')=='linux.apt.recovery-planning' for o in data.get('observations',[]))))
|
||||
return dict(events=events,other_unacked=other)
|
||||
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserving Cloud journal-capacity maintenance; never a job/history reset.
|
||||
|
||||
The copy-only mode changes only its own private disposable database copy.
|
||||
Installation requires a separate explicit option and qualified artifact hashes.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import cloud_apt_recovery_upgrade as previous
|
||||
|
||||
cloud = previous.cloud
|
||||
require = cloud.require
|
||||
STATE = previous.STATE
|
||||
|
||||
|
||||
def copy_reproducer(payload):
|
||||
import sqlite3
|
||||
os.umask(0o077)
|
||||
source = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent Cloud identity')
|
||||
require(not source.is_symlink() and source.stat().st_uid == 0 and source.stat().st_mode&0o777 == 0o600, 'private original ledger')
|
||||
def fingerprint(db):
|
||||
h = hashlib.sha256()
|
||||
tables = db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name").fetchall()
|
||||
for name,sql in tables:
|
||||
require(name.replace('_','').isalnum(), 'fixed stored table name')
|
||||
h.update(json.dumps([name,sql],separators=(',',':')).encode())
|
||||
for row in db.execute('SELECT * FROM "'+name+'" ORDER BY rowid'):
|
||||
data = [dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row]
|
||||
h.update(json.dumps(data,separators=(',',':'),ensure_ascii=True).encode())
|
||||
return h.hexdigest()
|
||||
def identity():
|
||||
s = source.stat()
|
||||
return dict(device=s.st_dev,inode=s.st_ino,size=s.st_size,sha256=hashlib.sha256(source.read_bytes()).hexdigest())
|
||||
before = identity()
|
||||
with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as original:
|
||||
require(original.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'original integrity')
|
||||
require({r[0] for r in original.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants'])
|
||||
and len(payload['expected_grants']) == 10
|
||||
and original.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0
|
||||
and original.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0,
|
||||
'exact ten known closed Root consumptions')
|
||||
original_fingerprint = fingerprint(original)
|
||||
folder = Path('/var/lib/rmm-bootstrap-history')/('journal-capacity-copy-'+str(uuid.UUID(payload['id'])))
|
||||
folder.mkdir(mode=0o700)
|
||||
copy = folder/'ledger-copy.sqlite'
|
||||
fd = os.open(copy,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600);os.close(fd)
|
||||
with sqlite3.connect(copy) as duplicate:
|
||||
original.backup(duplicate)
|
||||
require(fingerprint(duplicate) == original_fingerprint, 'consistent private SQLite backup')
|
||||
page_size = duplicate.execute('PRAGMA page_size').fetchone()[0]
|
||||
page_count = duplicate.execute('PRAGMA page_count').fetchone()[0]
|
||||
old = duplicate.execute('PRAGMA max_page_count=16384').fetchone()[0]
|
||||
require(page_size == 4096 and old == 16384 and page_count <= old, 'actual bounded old capacity')
|
||||
failed = None
|
||||
try:
|
||||
duplicate.execute('BEGIN IMMEDIATE')
|
||||
duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
|
||||
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
|
||||
except sqlite3.DatabaseError as error:
|
||||
failed = getattr(error,'sqlite_errorcode',None)
|
||||
finally:
|
||||
duplicate.rollback()
|
||||
require(failed == sqlite3.SQLITE_FULL, 'old limit must fail with SQLITE_FULL13 on the copied actual DB')
|
||||
require(fingerprint(duplicate) == original_fingerprint, 'old-capacity test rolled back every row/schema change')
|
||||
bounded = duplicate.execute('PRAGMA max_page_count=65536').fetchone()[0]
|
||||
require(bounded == 65536, 'bounded 256 MiB new capacity')
|
||||
duplicate.execute('BEGIN IMMEDIATE')
|
||||
duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
|
||||
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
|
||||
require(duplicate.execute('SELECT length(payload) FROM journal_capacity_probe').fetchone()[0] == 524288, 'same copied-DB insertion passes at the new finite cap')
|
||||
duplicate.rollback()
|
||||
require(fingerprint(duplicate) == original_fingerprint and duplicate.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'new-capacity test rolled back every row/schema change')
|
||||
after = identity()
|
||||
require(after == before, 'original DB bytes, size and inode must remain unchanged')
|
||||
report = dict(verified=True,backup=str(folder),old_cap_pages=16384,new_cap_pages=65536,page_size=page_size,page_count=page_count,
|
||||
old_error_code=failed,probe_bytes=524288,new_insert_passed=True,copy_original_content_sha256=original_fingerprint,
|
||||
original_before=before,original_after=after,original_unchanged=True,rows_and_schema_rolled_back=True,
|
||||
original_package_effects=False,original_sql_mutations=False)
|
||||
fd = os.open(folder/'report.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:json.dump(report,stream,indent=2);stream.flush();os.fsync(stream.fileno())
|
||||
return report
|
||||
|
||||
|
||||
|
||||
def install_capacity(payload):
|
||||
import fcntl
|
||||
import sqlite3
|
||||
import time
|
||||
import traceback
|
||||
os.umask(0o077)
|
||||
base = Path('/var/lib/rmm-apt-transaction'); state = Path('/var/lib/rmm-agent-remote')
|
||||
ledger = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
helper = Path('/usr/local/libexec/rmm-privileged-helper'); agent = Path('/usr/local/libexec/rmm-agent-remote')
|
||||
provider = Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
|
||||
config = Path('/etc/rmm/apt-helper.json'); proof_file = Path('/etc/rmm-remote/apt-helper-proof.json')
|
||||
metadata = base/'install-metadata.json'; readiness_file = base/'native-readiness.json'
|
||||
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
||||
reduced = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
|
||||
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
|
||||
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
|
||||
def run(args, **kwargs):
|
||||
return subprocess.run(args, capture_output=True, check=True, timeout=90, **kwargs).stdout
|
||||
def sha(raw): return hashlib.sha256(raw).hexdigest()
|
||||
def canonical(value): return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()
|
||||
def fsync_dir(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
def create(path, raw, mode=0o600, gid=0):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(raw); stream.flush(); os.fchown(stream.fileno(), 0, gid)
|
||||
os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
|
||||
fsync_dir(path.parent)
|
||||
def manifest(path):
|
||||
require(not path.is_symlink(), 'preserved directory symlink')
|
||||
if not path.exists(): return {'__absent__':True}
|
||||
require(path.is_dir(), 'preserved directory required')
|
||||
result = {}
|
||||
for p in [path, *sorted(path.rglob('*'))]:
|
||||
s = p.lstat(); require(not p.is_symlink() and (p.is_dir() or p.is_file()), 'preserved regular state required')
|
||||
result[str(p.relative_to(path))] = dict(dev=s.st_dev, inode=s.st_ino, mode=s.st_mode,
|
||||
uid=s.st_uid, gid=s.st_gid, sha256=sha(p.read_bytes()) if p.is_file() else None)
|
||||
return result
|
||||
def guard():
|
||||
require(len(payload['expected_grants']) == 10, 'exact ten original closed Root grants')
|
||||
require(payload['partial_grant'] == '31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0' and
|
||||
payload['partial_proof_sha256'] == 'dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21', 'one reviewed closed partial only')
|
||||
with sqlite3.connect(ledger.as_uri()+'?mode=ro', uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Root ledger integrity')
|
||||
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants']) and
|
||||
db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0 and
|
||||
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'exact known closed Root attempts')
|
||||
db.row_factory = sqlite3.Row
|
||||
rows = db.execute('SELECT * FROM apt_partial_reconciliations').fetchall()
|
||||
require(len(rows) == 1, 'one immutable partial proof required'); row = dict(rows[0])
|
||||
expected = payload['proof_row']
|
||||
for key in ('grant_id', 'original_request_sha256', 'previous_outcome_sha256', 'proof_sha256', 'checked_at_ms'):
|
||||
require(row[key] == expected[key], 'immutable partial binding drift')
|
||||
for key in ('previous_outcome', 'proof', 'classified_outcome'):
|
||||
require(sha(row[key]) == expected[key]['sha256'] and len(row[key]) == expected[key]['size'], 'immutable partial bytes drift')
|
||||
require(row['grant_id'] == payload['partial_grant'] and sha(row['proof']) == payload['partial_proof_sha256'], 'closed proof hash required')
|
||||
proof = json.loads(row['proof']); result = db.execute('SELECT c.result,e.outcome,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE grant_id=?', (payload['partial_grant'],)).fetchone()
|
||||
classified = json.loads(row['classified_outcome'])
|
||||
require(json.loads(result['result']) == json.loads(result['outcome']) == classified and
|
||||
classified['state'] == 'failed' and classified['error_code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED', 'normal signed RootSync closure required')
|
||||
saved = json.loads(result['start'])
|
||||
# save_start nests Request through serde_json::Value (sorted keys),
|
||||
# whereas the immutable proof hashes serde's Request struct codec.
|
||||
request_fields = ('job_bytes','permit_bytes','signature','plan_hash','authorization_id')
|
||||
require(set(saved['request']) == set(request_fields), 'original Request fields required')
|
||||
request_bytes = json.dumps({key:saved['request'][key] for key in request_fields},
|
||||
separators=(',', ':'), ensure_ascii=False).encode()
|
||||
require(sha(request_bytes) == row['original_request_sha256'], 'original signed request hash required')
|
||||
require(proof['grant_id'] == payload['partial_grant'] and proof['attempt_id'] == payload['partial_attempt'] and
|
||||
proof['snapshot']['rmm-mvp-test:all'] == {'version':'1.0','status':'install ok unpacked'}, 'exact partial fixture required')
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro', uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
|
||||
db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 and
|
||||
db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no unresolved or faulted agent attempt')
|
||||
require(Path('/usr/share/rmm-mvp-test/version').read_bytes() == b'1.0\n', 'existing unpacked fixture payload required')
|
||||
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('pending-acceptance-*.json')) and
|
||||
not list(base.glob('recovery-*.json')) and not list(base.glob('pending-recovery-*.json')), 'no existing acceptance or recovery attempt')
|
||||
return proof
|
||||
def bound_partial(proof):
|
||||
keys = 'transaction_sha256 job_payload_sha256 effect_armed_ledger_id grant_id job_id attempt_id authorization_id plan_sha256'.split()
|
||||
value = json.loads(run(['/usr/bin/python3','-I',str(provider),'verify-partial'], input=canonical({k:proof[k] for k in keys})))
|
||||
require(value['effect_state'] == 'known_failed' and value['code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED' and
|
||||
{k:v for k,v in value['proof'].items() if k != 'checked_at_ms'} ==
|
||||
{k:v for k,v in proof.items() if k != 'checked_at_ms'}, 'fresh original-bound partial state required')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'permanent Cloud identity required')
|
||||
require(not dropin.is_symlink() and dropin.read_text() == reduced, 'APT endpoint must remain absent')
|
||||
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT helper must be stopped')
|
||||
for name, pin in payload['old_files'].items():
|
||||
p = Path(name); require(not p.is_symlink() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0 and sha(p.read_bytes()) == pin, 'installed file drift')
|
||||
require(Path('/etc/systemd/system/rmm-apt-helper.service').read_text() == payload['unit'], 'exact reviewed APT service required')
|
||||
markers = {base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json':'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053',
|
||||
base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json':'a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3'}
|
||||
require({p.name for p in base.glob('bootstrap-*.json')} == {p.name for p in markers} and
|
||||
all(sha(p.read_bytes()) == pin for p,pin in markers.items()), 'original/retry tombstones required')
|
||||
partial = guard(); bound_partial(partial)
|
||||
raw = {name:base64.b64decode(payload[name], validate=True) for name in ('helper','agent')}
|
||||
pins = {'helper':payload['helper_sha256'],'agent':payload['agent_sha256'],
|
||||
'provider':'5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327'}
|
||||
require(all(sha(raw[name]) == pins[name] for name in ('helper','agent')) and
|
||||
sha(provider.read_bytes()) == pins['provider'], 'qualified exact capacity binaries and unchanged provider required')
|
||||
for name in ('helper','agent'):
|
||||
require(raw[name][:6] == b'\x7fELF\x02\x01' and raw[name][18:20] == b'\x3e\x00', 'Linux x64 binary required')
|
||||
if payload.get('preflight_only') is True:
|
||||
return dict(stage='preflight_validated', completed=False, preflight_only=True,
|
||||
package_effects=False, maintenance_mutations=False, expected_grants=len(payload['expected_grants']),
|
||||
partial_proof_sha256=payload['partial_proof_sha256'], apt_endpoint_enabled=False)
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-journal-capacity-'+payload['id']); backup.mkdir(mode=0o700)
|
||||
report = dict(stage='validated', completed=False, backup=str(backup), package_effects=False,
|
||||
apt_endpoint_enabled=False, partial_proof_sha256=payload['partial_proof_sha256'], backups={})
|
||||
def save():
|
||||
next_path = backup/'.report-next.json'; create(next_path,json.dumps(report,indent=2).encode())
|
||||
os.replace(next_path,backup/'report.json'); fsync_dir(backup)
|
||||
def replace(path, staged=None):
|
||||
target = backup/str(path).lstrip('/').replace('/','__')
|
||||
require(not path.is_symlink() and path.stat().st_dev == backup.stat().st_dev and not target.exists(), 'exclusive same-filesystem versioned replacement')
|
||||
os.rename(path,target); fsync_dir(path.parent); fsync_dir(backup)
|
||||
report['backups'][str(path)] = str(target); save()
|
||||
if staged is not None:
|
||||
os.rename(staged,path); fsync_dir(path.parent); fsync_dir(backup)
|
||||
lock = None
|
||||
try:
|
||||
save()
|
||||
for name in ('helper','agent'): create(backup/(name+'-next'), raw[name], 0o755)
|
||||
for name in ('helper','agent'): run(['/lib64/ld-linux-x86-64.so.2','--list',str(backup/(name+'-next'))])
|
||||
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
|
||||
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
partial = guard(); bound_partial(partial)
|
||||
preserved = {str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
|
||||
preserved_base = manifest(base)
|
||||
witnesses = {}
|
||||
for p in (Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'),dropin,
|
||||
Path('/etc/systemd/system/rmm-apt-helper.service'),Path('/usr/lib/rmm-apt-transaction/hook'),
|
||||
Path('/etc/rmm-remote/general-helper-proof.json'),Path('/etc/rmm-remote/helper-general.json'),provider,metadata):
|
||||
s = p.stat(); witnesses[str(p)] = (s.st_dev,s.st_ino,sha(p.read_bytes()))
|
||||
general_bytes = Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
|
||||
general_config = Path('/etc/rmm-remote/helper-general.json').read_bytes()
|
||||
old_metadata = json.loads(metadata.read_text()); old_config = json.loads(config.read_text())
|
||||
require(old_metadata['schema_version'] == 2 and old_metadata['provider_sha256'] == sha(provider.read_bytes()) and
|
||||
old_metadata['hook_sha256'] == sha(Path('/usr/lib/rmm-apt-transaction/hook').read_bytes()) and
|
||||
old_metadata['unit_sha256'] == sha(payload['unit'].encode()), 'old install metadata required')
|
||||
for path,name in ((helper,'helper'),(agent,'agent')): replace(path,backup/(name+'-next'))
|
||||
command = ['/usr/bin/python3','-I',str(provider)]
|
||||
config_bytes = run(command+['print-config']); config_sha = sha(config_bytes)
|
||||
require(config_sha == payload['reviewed_config_sha256'] and all(line in config_bytes for line in (
|
||||
b'DPkg::ConfigurePending "false";', b'DPkg::TriggersPending "false";', b'dpkg::selection::remove::approved "false";')), 'reviewed exact configure/no-pending settings')
|
||||
runtime = json.loads(run(command+['inspect-runtime'])); runtime_sha = sha(canonical(runtime))
|
||||
old_config['apt_provider'].update(runtime_sha256=runtime_sha,apt_config_sha256=config_sha)
|
||||
create(backup/'config-next',json.dumps(old_config).encode()); replace(config,backup/'config-next')
|
||||
pin = json.loads(run([str(helper),'--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config))))
|
||||
require(pin['identity'] == json.loads(general_bytes)['identity'], 'same helper identity required')
|
||||
create(backup/'proof-next',json.dumps(pin).encode(),0o640,984); replace(proof_file,backup/'proof-next')
|
||||
general = json.loads(run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
|
||||
require(general == json.loads(general_bytes) and Path('/etc/rmm-remote/helper-general.json').read_bytes() == general_config, 'General helper policy unchanged')
|
||||
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'identity/journal/Root ledger bytes or inode changed')
|
||||
def preserved_original_base():
|
||||
current = manifest(base)
|
||||
require(all(current.get(name) == value for name,value in preserved_base.items() if name not in ('install-metadata.json','native-readiness.json')), 'original APT state/archive/hook/probe changed')
|
||||
for name,value in witnesses.items():
|
||||
p=Path(name);s=p.stat();require((s.st_dev,s.st_ino,sha(p.read_bytes())) == value, 'original package/control file bytes or inode changed')
|
||||
return current
|
||||
preserved_original_base(); bound_partial(guard())
|
||||
report.update(stage='installed_preserved',preserved_state=True,helper_sha256=pins['helper'],agent_sha256=pins['agent'],journal_page_size=4096,journal_max_page_count=65536,journal_capacity_bytes=268435456,
|
||||
provider_sha256=pins['provider'],runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general)
|
||||
save()
|
||||
# No client can reach this endpoint; it exists only for the service's
|
||||
# ExecStartPre copied-database probe, then is stopped before restart.
|
||||
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
|
||||
replace(readiness_file)
|
||||
# Daemon startup itself takes executor.lock to open the existing DB.
|
||||
# Installation is complete; agent/General remain stopped during probe.
|
||||
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
||||
run(['systemctl','start','rmm-apt-helper.service'])
|
||||
for _ in range(3):
|
||||
status = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
|
||||
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status, 'stable native probe helper required')
|
||||
time.sleep(1)
|
||||
run(['systemctl','stop','rmm-apt-helper.service'])
|
||||
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
ready = json.loads(readiness_file.read_text())
|
||||
require(ready['runtime'] == runtime and ready['readiness_only'] is True and ready['native_acceptance'] is False and
|
||||
ready['host_status_sha256'] == sha(Path('/var/lib/dpkg/status').read_bytes()), 'exact new copied-database probe required')
|
||||
workspace = base/'probes'/str(uuid.UUID(ready['probe_id']))
|
||||
require((workspace/'dpkg/status').read_bytes() == Path('/var/lib/dpkg/status').read_bytes() and
|
||||
(workspace/'apt.conf').read_bytes().count(b'dpkg::selection::remove::approved "false";') == 1,
|
||||
'partial copied status and configure fix required')
|
||||
current = preserved_original_base()
|
||||
require(all(name in preserved_base or name.startswith('probes/'+ready['probe_id']+'/') or name == 'probes/'+ready['probe_id']
|
||||
for name in current), 'only generated readiness probe metadata may be added')
|
||||
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'journal/ledger bytes or inode changed during probe')
|
||||
bound_partial(guard())
|
||||
require(dropin.read_text() == reduced and Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() == general_bytes, 'endpoint/General proof unchanged')
|
||||
report.update(stage='native_probe_preserved',readiness_only=True,native_acceptance=False,
|
||||
journals_preserved_before_restart=True,readiness_sha256=sha(readiness_file.read_bytes()),probe_id=ready['probe_id'])
|
||||
save()
|
||||
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
||||
run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
|
||||
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT endpoint must remain stopped')
|
||||
report.update(stage='qualified_runtime_started',completed=True);save()
|
||||
except Exception as error:
|
||||
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
|
||||
save()
|
||||
finally:
|
||||
if lock is not None: fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
|
||||
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,timeout=30)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--copy-only',action='store_true')
|
||||
mode.add_argument('--preflight-only',action='store_true')
|
||||
mode.add_argument('--install-reviewed-artifacts',action='store_true')
|
||||
parser.add_argument('--agent-sha256');parser.add_argument('--helper-sha256')
|
||||
args = parser.parse_args();os.umask(0o077)
|
||||
require(previous.previous.apt.prior.evidence('gate')['unresolved'] == 0, 'no unsettled central job')
|
||||
payload = previous.build_payload()
|
||||
require(len(payload['expected_grants']) == 10, 'exact completed Root/log history')
|
||||
if args.copy_only:
|
||||
report = cloud.bootstrap.rpc('linux-vm.conf',copy_reproducer,dict(id=str(uuid.uuid4()),agent_id=payload['agent_id'],expected_grants=payload['expected_grants']))
|
||||
cloud.contract.write_private(STATE/('journal-capacity-copy-'+Path(report['backup']).name.removeprefix('journal-capacity-copy-')+'.json'),report)
|
||||
print({k:report[k] for k in ('verified','old_error_code','new_insert_passed','original_unchanged','rows_and_schema_rolled_back','original_sql_mutations')})
|
||||
return
|
||||
require(not (STATE/'journal-capacity-runtime.json').exists(), 'existing capacity runtime retained; no automatic maintenance repeat')
|
||||
reviewed = {'agent':'2ce885cf5e6881e77383482b7d3a87a02c6847aa26b82e5fd70d2ebd8a45360d',
|
||||
'helper':'332eae004d9416234eabf7ed3d5c9ece5440d1e0c764ea63357961c16b673b6b'}
|
||||
for name,pin in reviewed.items():
|
||||
require(getattr(args,name+'_sha256') == pin and re.fullmatch('[0-9a-f]{64}',pin), 'independently reviewed capacity artifact pin required')
|
||||
raw = (STATE/(name+'-journal-capacity')).read_bytes()
|
||||
require(hashlib.sha256(raw).hexdigest() == pin, 'qualified capacity artifact drift')
|
||||
payload[name] = base64.b64encode(raw).decode();payload[name+'_sha256'] = pin
|
||||
payload.pop('provider')
|
||||
previous_runtime = cloud.contract.protected_json(STATE/'configure-runtime.json')
|
||||
require(previous_runtime['completed'] and previous_runtime['helper_sha256'] == previous.PINS['helper']
|
||||
and previous_runtime['agent_sha256'] == previous.PINS['agent'] and previous_runtime['provider_sha256'] == previous.PINS['provider'], 'exact independently qualified previous runtime')
|
||||
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
require(installed['binary_sha256'] == previous.PINS['agent'], 'old permanent agent record required')
|
||||
policy = cloud.bootstrap.rpc('server.conf',previous.closed_policy,{})
|
||||
old = cloud.bootstrap.rpc('linux-vm.conf',previous.installed_metadata,previous.PINS)
|
||||
payload.update(old_files=old['files'],preflight_only=args.preflight_only)
|
||||
report = cloud.bootstrap.rpc('linux-vm.conf',install_capacity,payload)
|
||||
if args.preflight_only:
|
||||
require(report.get('preflight_only') and report.get('maintenance_mutations') is False, 'read-only maintenance preflight required')
|
||||
cloud.contract.write_private(STATE/('journal-capacity-preflight-'+payload['id']+'.json'),dict(report,closed_policy=policy))
|
||||
print(report);return
|
||||
cloud.contract.write_private(STATE/('journal-capacity-install-'+payload['id']+'.json'),dict(report,closed_policy=policy))
|
||||
require(report['completed'] and report['preserved_state'] and report['journals_preserved_before_restart'] and report['provider_sha256'] == previous.PINS['provider'], 'preserving bounded-capacity installation incomplete; retain report and do not repeat')
|
||||
previous.previous.record_agent_upgrade(report,installed)
|
||||
cloud.contract.write_private(STATE/'journal-capacity-runtime.json',report)
|
||||
print({k:report[k] for k in ('completed','stage','preserved_state','journals_preserved_before_restart','readiness_only','apt_endpoint_enabled')})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_JOURNAL_CAPACITY_INCOMPLETE:'+type(error).__name__)
|
||||
if str(error).startswith('remote bootstrap incomplete: '):print(str(error))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cloud-only active-agent offline/start-window/revocation acceptance.
|
||||
|
||||
Requires explicit execution. Plans start pending with zero executable artifacts;
|
||||
approval and plan revocation use actual dev/test WebUI controls. A narrowly
|
||||
scoped systemd BPF network fault affects only the existing Cloud agent unit.
|
||||
SSH is used solely for that transport bootstrap and retained-state evidence.
|
||||
No enrollment, package action, journal reset, approval API or automatic retry.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import cloud_mvp_acceptance as cloud
|
||||
|
||||
CASES = ('offline_wait', 'expired_offline', 'revoked_before_start')
|
||||
SERVER_IP = '88.99.80.202'
|
||||
STATE = cloud.ROOT/'devstate/cloud-offline'
|
||||
require = cloud.contract.require
|
||||
|
||||
|
||||
def counter_script(identifier):
|
||||
identifier = str(uuid.UUID(identifier))
|
||||
return ('set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-'+identifier+'\n'
|
||||
'test ! -e "$d"\nmkdir "$d"\n'
|
||||
'printf \'%s\\n\' \'1\' > "$d/count"\n'
|
||||
'test "$(cat "$d/count")" = 1\n'
|
||||
'printf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n')
|
||||
|
||||
|
||||
def server_evidence(payload):
|
||||
"""Fixed SELECTs only, never creates/revokes/approves an operation."""
|
||||
root = Path('/opt/rust-discovery')
|
||||
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||
job = str(uuid.UUID(payload['job_id'])); operation = str(uuid.UUID(payload['operation_id']))
|
||||
plan = str(uuid.UUID(payload['plan_id']))
|
||||
generation = int(payload['previous_generation']); require(generation > 0, 'previous fenced channel')
|
||||
import datetime as dt
|
||||
after = dt.datetime.fromisoformat(payload['after'].replace('Z','+00:00')).isoformat() if payload.get('after') else '1970-01-01T00:00:00+00:00'
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
||||
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND job_id='"+job+"'"
|
||||
query = ("SELECT json_build_object('observed_at',now(),'state',(SELECT state FROM jobs WHERE "+scope+"),"
|
||||
"'start_reserved',COALESCE((SELECT start_reserved FROM jobs WHERE "+scope+"),false),"
|
||||
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||
"'approval_state',(SELECT approval_state FROM approval_requests WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"'),"
|
||||
"'revoked_authorizations',(SELECT count(*) FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' AND revoked_at IS NOT NULL),"
|
||||
"'starts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data)),'[]'::json) FROM audit_events "
|
||||
"WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"'),"
|
||||
"'fresh_signed_sync_before_start',EXISTS(SELECT 1 FROM sync_checkpoints c WHERE c.tenant_id='"+tenant+"' AND c.agent_id='"+agent+"' "
|
||||
"AND c.connection_generation>"+str(generation)+" AND (convert_from(c.payload,'UTF8')::jsonb->>'ready')::boolean "
|
||||
"AND c.created_at>='"+after+"'::timestamptz AND c.created_at<=(SELECT min(created_at) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')),"
|
||||
"'revocation',(SELECT json_build_object('principal',principal,'origin',data->>'approval_origin',"
|
||||
"'simulation',data->'testsimulation','interactive_session',data->>'session_hash' IS NOT NULL,"
|
||||
"'plan_id',data->'decision'->>'plan_id','operation_id',data->'decision'->>'operation_id') "
|
||||
"FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='plan_revoked' AND data->'decision'->>'plan_id'='"+plan+"' ORDER BY created_at DESC LIMIT 1));")
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
||||
|
||||
|
||||
def vm_network(payload):
|
||||
"""One operational transport fault, plus reads of existing durable evidence."""
|
||||
import datetime as dt
|
||||
import errno
|
||||
import fcntl
|
||||
import sqlite3
|
||||
import stat
|
||||
identifier = str(uuid.UUID(payload['id']))
|
||||
phase = payload['phase']; require(phase in ('preflight','window'), 'fixed network phase')
|
||||
action = payload['action']; require(action in ('state','block','restore'), 'fixed network operation')
|
||||
state = Path('/var/lib/rmm-agent-remote'); helper = Path('/var/lib/rmm-helper')
|
||||
unit = 'rmm-agent-remote.service'; account = pwd.getpwnam('rmm-agent')
|
||||
require(os.geteuid() == 0 and account.pw_uid == 993 and account.pw_gid == 993, 'fixed unprivileged Cloud agent')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed Cloud identity')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed current agent binary')
|
||||
def properties():
|
||||
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
return dict(line.split('=',1) for line in raw.splitlines())
|
||||
def snapshot():
|
||||
result = dict(agent_service=properties())
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
|
||||
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
|
||||
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
|
||||
if payload.get('job_id'):
|
||||
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
||||
rows = db.execute('SELECT state,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
|
||||
result['attempts'] = [dict(state=r[0],grant_id=r[1]) for r in rows]
|
||||
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
|
||||
else:
|
||||
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
||||
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
|
||||
with sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
|
||||
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
|
||||
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
|
||||
serial = [[dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
|
||||
result['helper_ledger_sha256'] = hashlib.sha256(json.dumps(serial,separators=(',',':')).encode()).hexdigest()
|
||||
result['helper_consumed'] = len(rows); result['helper_unresolved'] = sum(row[5] is None for row in rows)
|
||||
effect_directory = state/('offline-'+identifier)
|
||||
if effect_directory.exists():
|
||||
m = effect_directory.lstat();require(stat.S_ISDIR(m.st_mode) and m.st_uid == 993 and m.st_mode&0o777 == 0o700, 'fixed stored effect directory')
|
||||
else: require(not effect_directory.is_symlink(), 'stored effect directory symlink')
|
||||
marker = effect_directory/'count'
|
||||
if marker.exists():
|
||||
m = marker.lstat(); require(stat.S_ISREG(m.st_mode) and m.st_uid == account.pw_uid and m.st_mode&0o777 == 0o600 and m.st_size <= 16, 'fixed stored counter evidence')
|
||||
result['count'] = marker.read_text()
|
||||
else:
|
||||
require(not marker.is_symlink(), 'stored counter symlink'); result['count'] = None
|
||||
return result
|
||||
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
||||
require(not directory.is_symlink(), 'network dropin directory')
|
||||
directory.mkdir(mode=0o755,exist_ok=True)
|
||||
require(directory.stat().st_uid == 0 and directory.stat().st_mode&0o022 == 0, 'network dropin ownership')
|
||||
dropin = directory/'96-cloud-offline-acceptance.conf'
|
||||
expected = ('# Cloud MVP offline '+identifier+' '+phase+'\n[Service]\nIPAddressDeny=88.99.80.202/32\n').encode()
|
||||
require(not dropin.is_symlink(), 'network dropin symlink')
|
||||
timer = 'rmm-offline-rollback-'+identifier+'-'+phase
|
||||
before = snapshot()
|
||||
require(before['agent_fault'] is None and before['helper_fault'] is None and before['helper_unresolved'] == 0 and before['other_unresolved'] == 0, 'unrelated or unknown work remains fenced')
|
||||
if payload.get('helper_ledger_sha256'):
|
||||
require(before['helper_ledger_sha256'] == payload['helper_ledger_sha256'], 'existing Root ledger changed')
|
||||
if action == 'state': return before
|
||||
if action == 'restore':
|
||||
if dropin.exists():
|
||||
require(dropin.stat().st_uid == 0 and dropin.read_bytes() == expected, 'exact network fault ownership')
|
||||
dropin.unlink()
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
|
||||
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
|
||||
subprocess.run(['systemctl','stop',timer+'.timer'],capture_output=True,timeout=10)
|
||||
return dict(restored=True,after=snapshot())
|
||||
require(not dropin.exists() and before['agent_service']['ActiveState'] == 'active' and not before['agent_service']['IPAddressAllow'], 'clean active unit and no allow override')
|
||||
require(before['count'] is None and not before['consumed'], 'no prior effect/consumption')
|
||||
if phase == 'window':
|
||||
start = dt.datetime.fromisoformat(payload['not_before'].replace('Z','+00:00'))
|
||||
require(dt.datetime.now(dt.timezone.utc) < start-dt.timedelta(seconds=60) and before['attempts'] == [dict(state='accepted',grant_id='')], 'fixed future accepted attempt with zero grant')
|
||||
else:
|
||||
require(not payload.get('job_id'), 'preflight has no job')
|
||||
# Arm bounded automatic transport rollback before creating the deny rule.
|
||||
# The timer can remove only these exact bytes and never touches job state.
|
||||
rollback_dir = Path('/var/lib/rmm-offline-bootstrap')
|
||||
require(not rollback_dir.is_symlink(), 'rollback directory symlink')
|
||||
rollback_dir.mkdir(mode=0o700,exist_ok=True)
|
||||
require(rollback_dir.stat().st_uid == 0 and rollback_dir.stat().st_mode&0o777 == 0o700, 'private bootstrap rollback directory')
|
||||
rollback = rollback_dir/(identifier+'-'+phase+'.py')
|
||||
code = ('import os,subprocess\nfrom pathlib import Path\np=Path('+repr(str(dropin))+')\n'
|
||||
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(expected)+':\n'
|
||||
' p.unlink()\n subprocess.run(["systemctl","daemon-reload"],check=True)\n'
|
||||
' subprocess.run(["systemctl","restart","rmm-agent-remote.service"],check=True)\n')
|
||||
fd = os.open(rollback,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream: stream.write(code);stream.flush();os.fsync(stream.fileno())
|
||||
subprocess.run(['systemd-run','--quiet','--unit='+timer,'--on-active=420s','--property=Type=oneshot','/usr/bin/python3',str(rollback)],capture_output=True,check=True,timeout=10)
|
||||
# Quiesce this accepted/unstarted process only while applying the tested
|
||||
# policy. It must be running again BEFORE not_before; a stopped-agent
|
||||
# interval never counts toward the active offline acceptance window.
|
||||
subprocess.run(['systemctl','stop',unit],capture_output=True,check=True,timeout=20)
|
||||
stopped = snapshot()
|
||||
require(stopped['agent_service']['ActiveState'] == 'inactive' and not stopped['consumed'] and stopped['count'] is None, 'quiescent unstarted network setup')
|
||||
fd = os.open(dropin,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
|
||||
with os.fdopen(fd,'wb') as stream: stream.write(expected);stream.flush();os.fsync(stream.fileno())
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
|
||||
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
|
||||
after = snapshot(); require(after['agent_service']['ActiveState'] == 'active' and int(after['agent_service']['MainPID']) > 0 and after['count'] is None and not after['consumed'], 'active offline agent and zero effect')
|
||||
if phase == 'window': require(dt.datetime.now(dt.timezone.utc) < start, 'agent must be active offline before earliest start')
|
||||
group = after['agent_service']['ControlGroup']
|
||||
require((group == '/system.slice/rmm-agent-remote.service' or group.startswith('/system.slice/rmm-agent-remote.service/')) and '..' not in group, 'actual agent cgroup')
|
||||
cgroup = Path('/sys/fs/cgroup'+group)/'cgroup.procs'
|
||||
require(cgroup.is_file(), 'unified cgroup transport enforcement required')
|
||||
# This separate bootstrap child moves ONLY ITS OWN PID, drops privilege and
|
||||
# attempts TCP only. No agent credential, TLS override or appdata is used.
|
||||
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
|
||||
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
|
||||
'os.setgroups([]);os.setgid(993);os.setuid(993)\n'
|
||||
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
|
||||
'except OSError as e: result={"connected":False,"errno":e.errno}\n'
|
||||
'print(json.dumps(result))\n')
|
||||
result = subprocess.run(['/usr/bin/python3','-I','-c',probe],capture_output=True,check=True,timeout=4)
|
||||
blocked = json.loads(result.stdout)
|
||||
require(blocked.get('connected') is False and blocked.get('errno') in (errno.EACCES,errno.EPERM), 'actual same-cgroup BPF denial must be EACCES/EPERM; timeout is not proof')
|
||||
after = snapshot()
|
||||
require(after['helper_ledger_sha256'] == before['helper_ledger_sha256'], 'Root ledger preserved during network bootstrap')
|
||||
return dict(blocked=True,probe=blocked,after=after,rollback_after_seconds=420,unit=unit,server_ip='88.99.80.202',port=443)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
||||
parser.add_argument('--case',choices=CASES,required=True)
|
||||
args = parser.parse_args(); os.umask(0o077)
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(), 'notebook is development only')
|
||||
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
identity = installed['identity']; settings = dict(tenant_id=cloud.TENANT,**identity)
|
||||
backend = cloud.Backend(); identifier = str(uuid.uuid4())
|
||||
folder = STATE/identifier; folder.mkdir(mode=0o700,parents=True)
|
||||
checkpoint = dict(profile='dev/test',case=args.case,id=identifier,expected_principal='simulated-testprincipal-a',**settings,no_retry=True,stage='preflight')
|
||||
serial = 0
|
||||
def save():
|
||||
nonlocal serial
|
||||
cloud.contract.write_private(folder/('%03d-%s.json'%(serial,checkpoint['stage'])),checkpoint);serial += 1
|
||||
def gate(job_id=None):
|
||||
return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**({'job_id':job_id} if job_id else {})))
|
||||
def native(action,phase='window',**extra):
|
||||
p = dict(action=action,phase=phase,id=identifier,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256'],**extra)
|
||||
if 'job' in checkpoint and phase == 'window': p.update(job_id=checkpoint['job']['job_id'],attempt_id=checkpoint['job']['payload']['attempt_id'])
|
||||
return cloud.bootstrap.rpc('linux-vm.conf',vm_network,p)
|
||||
def healthy(previous_generation=None):
|
||||
end = time.monotonic()+100
|
||||
while True:
|
||||
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
if observed.get('sync_ready') and not observed.get('restore_blocked') and (previous_generation is None or observed['connection_generation'] > previous_generation): return observed
|
||||
require(time.monotonic() < end, 'ordinary signed reconnect did not become healthy');time.sleep(2)
|
||||
require(gate()['unresolved'] == 0, 'unresolved central work remains fenced')
|
||||
checkpoint['initial_channel'] = healthy(); initial = native('state','preflight')
|
||||
ledger_sha = initial['helper_ledger_sha256']; checkpoint['initial_native'] = initial;save()
|
||||
checkpoint['network_preflight'] = native('block','preflight',helper_ledger_sha256=ledger_sha);save()
|
||||
checkpoint['preflight_restored'] = native('restore','preflight',helper_ledger_sha256=ledger_sha);save()
|
||||
checkpoint['healthy_before_plan'] = healthy(checkpoint['initial_channel']['connection_generation'])
|
||||
start = cloud.contract.now()+dt.timedelta(seconds=150); end = start+dt.timedelta(seconds=130)
|
||||
request = cloud.contract.case_input('hello',settings,start=start)
|
||||
request['parameters']['variants'][0]['script_utf8'] = counter_script(identifier)
|
||||
request['parameters']['expected_stdout'] = 'RMM_OFFLINE_COUNT=1\n'
|
||||
request['schedule'] = cloud.contract.schedule(start,end)
|
||||
request['reason'] = 'Explicit dev/test active Cloud agent '+args.case+'; one fixed attempt, no implicit retry'
|
||||
plan = backend.call('operations.plan',dict(tenant_id=cloud.TENANT,input=request)); require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'one immutable fixed job')
|
||||
job = plan['jobs'][0]; pending = backend.call('operations.submit',dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier))
|
||||
checkpoint.update(plan=plan,job=job,request=request,operation_id=pending['operation_id'],stage='pending_approval',preapproval=gate(job['job_id']))
|
||||
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and all(checkpoint['preapproval'][k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts');save()
|
||||
def browser(phase):
|
||||
cloud.prepare_browser(); path = folder/('ui-'+phase+'.json'); cloud.contract.write_private(path,dict(checkpoint,phase=phase))
|
||||
result = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_offline_acceptance.cjs'),str(path)],capture_output=True,timeout=90)
|
||||
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint and never repeat approval')
|
||||
browser('approve'); checkpoint['stage'] = 'approved';save()
|
||||
deadline = time.monotonic()+45
|
||||
while True:
|
||||
server = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])); retained = native('state',helper_ledger_sha256=ledger_sha)
|
||||
require(gate(job['job_id'])['permits'] == 0 and not retained['consumed'] and retained['count'] is None, 'future attempt already has a grant/effect')
|
||||
if server['state'] == 'accepted' and retained['attempts'] == [dict(state='accepted',grant_id='')]: break
|
||||
require(time.monotonic() < deadline, 'future offer was not durably accepted');time.sleep(2)
|
||||
checkpoint.update(accepted=server,native_accepted=retained,stage='accepted_unstarted');save()
|
||||
checkpoint['network_fault'] = native('block',not_before=job['payload']['not_before'],helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'active_offline';save()
|
||||
channel = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
resume = cloud.contract.timestamp(job['payload']['start_before'] if args.case == 'expired_offline' else job['payload']['not_before'])+dt.timedelta(seconds=10)
|
||||
if args.case == 'revoked_before_start': browser('revoke');checkpoint['ui_revoked'] = True;save()
|
||||
sequence = checkpoint['network_fault']['after']['sequence']; windows = []
|
||||
while cloud.contract.now() < resume:
|
||||
retained = native('state',helper_ledger_sha256=ledger_sha); counts = gate(job['job_id'])
|
||||
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
require(retained['agent_service']['ActiveState'] == 'active' and retained['agent_service']['MainPID'] == checkpoint['network_fault']['after']['agent_service']['MainPID']
|
||||
and not retained['consumed'] and retained['count'] is None and counts['permits'] == 0 and observed['last_contact_at'] == channel['last_contact_at'], 'active offline window had a grant/effect/contact or agent exited')
|
||||
windows.append(dict(observed_at=cloud.contract.now().isoformat(),sequence=retained['sequence'],grants=counts['permits'],count=retained['count'],active=True))
|
||||
checkpoint['offline_window'] = windows;save();time.sleep(5)
|
||||
require(windows and windows[-1]['sequence'] > sequence, 'running offline agent did not persist reconnect attempts')
|
||||
checkpoint['server_offline_tail'] = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation']))
|
||||
require(checkpoint['server_offline_tail']['grants'] == 0 and not checkpoint['server_offline_tail']['start_reserved'], 'no start immediately before network restoration')
|
||||
checkpoint['network_restored'] = native('restore',helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'network_restored';save()
|
||||
checkpoint['healthy_reconnect'] = healthy(channel['connection_generation'])
|
||||
deadline = time.monotonic()+100
|
||||
while True:
|
||||
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
if result['state'] in cloud.contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
|
||||
require(time.monotonic() < deadline, 'terminal native result missing; no retry');time.sleep(2)
|
||||
expected = {'offline_wait':'succeeded','expired_offline':'expired','revoked_before_start':'cancelled'}[args.case]
|
||||
require(result['state'] == expected, 'actual terminal state differs from bounded case')
|
||||
native_final = native('state',helper_ledger_sha256=ledger_sha)
|
||||
wanted = 1 if args.case == 'offline_wait' else 0
|
||||
final_gate = gate(job['job_id']); require([final_gate[k] for k in ('jobs','outbox','permits')] == [1,1,wanted] and final_gate['unresolved'] == 0, 'exact job/outbox/start-grant counts')
|
||||
require(len(native_final['consumed']) == wanted and native_final['count'] == ('1\n' if wanted else None), 'exact native consumption and stored effect counter')
|
||||
if wanted: require(result['result']['exit_code'] == 0 and bytes(result['result']['stdout']) == b'RMM_OFFLINE_COUNT=1\n' and result['result']['evidence']['postconditions_passed'], 'native one-effect postcondition')
|
||||
audit = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='audit',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],operation_id=pending['operation_id']))
|
||||
require(audit == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']), 'exact interactive approval audit')
|
||||
details = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation'],after=checkpoint['server_offline_tail']['observed_at']))
|
||||
require(details['grants'] == wanted and len(details['starts']) == wanted and details['start_reserved'] == bool(wanted), 'retained start reservation evidence')
|
||||
if wanted: require(details['fresh_signed_sync_before_start'], 'fresh healthy signed reconnect must precede the one start reservation')
|
||||
if args.case == 'revoked_before_start':
|
||||
require(details['approval_state'] == 'revoked' and details['revoked_authorizations'] == 1 and details['revocation'] == dict(principal=checkpoint['expected_principal'],origin='dev_test_browser_simulation',simulation=True,interactive_session=True,plan_id=plan['plan_id'],operation_id=pending['operation_id']), 'actual interactive plan revocation audit')
|
||||
checkpoint.update(result=result,native_final=native_final,postapproval=final_gate,audit=audit,server_evidence=details,stage='native_dev_test_accepted',verified=True);save()
|
||||
print('PASS Cloud '+args.case+': active offline BPF verified, WebUI gate, exact grants/effect='+str(wanted))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except Exception as error:
|
||||
print('CLOUD_OFFLINE_INCOMPLETE:'+ (str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -3,6 +3,27 @@
|
||||
|
||||
def service_processes(payload):
|
||||
result={}
|
||||
# Same read-only kernel clock-readiness query required by the runtime;
|
||||
# modes stays zero. No time correction or trust override exists here.
|
||||
import ctypes
|
||||
class TimexPrefix(ctypes.Structure):
|
||||
_fields_=[('modes',ctypes.c_uint),('offset',ctypes.c_long),('freq',ctypes.c_long),
|
||||
('maxerror',ctypes.c_long),('esterror',ctypes.c_long),('status',ctypes.c_int),('remaining',ctypes.c_byte*256)]
|
||||
require(ctypes.sizeof(ctypes.c_long)==8 and TimexPrefix.maxerror.offset==24 and TimexPrefix.status.offset==40,'reviewed Linux x64 timex layout')
|
||||
clock=TimexPrefix();libc=ctypes.CDLL(None,use_errno=True);query=libc.adjtimex
|
||||
query.argtypes=[ctypes.POINTER(TimexPrefix)];query.restype=ctypes.c_int
|
||||
status=query(ctypes.byref(clock))
|
||||
result['kernel_clock']=dict(status=status,unsynchronized=bool(clock.status&0x40),maxerror_us=clock.maxerror,
|
||||
trusted=status>=0 and status!=5 and not clock.status&0x40 and 0<=clock.maxerror<=5_000_000,modes=clock.modes)
|
||||
storage=os.statvfs('/var/lib/rmm-helper')
|
||||
result['bootstrap_storage']=dict(available_bytes=storage.f_bavail*storage.f_frsize,available_inodes=storage.f_favail,
|
||||
helper_database_bytes=Path('/var/lib/rmm-helper/consumption.sqlite').stat().st_size,
|
||||
agent_database_bytes=Path('/var/lib/rmm-agent-remote/journal.sqlite').stat().st_size)
|
||||
lock=Path('/var/lib/rmm-helper/executor.lock').stat()
|
||||
result['executor_lock_pids']=[]
|
||||
for line in Path('/proc/locks').read_text().splitlines():
|
||||
parts=line.split()
|
||||
if len(parts)>5 and parts[5].split(':')[-1]==str(lock.st_ino):result['executor_lock_pids'].append(parts[4])
|
||||
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
||||
values=subprocess.run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts'],
|
||||
capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
@@ -23,3 +44,22 @@ def service_processes(payload):
|
||||
if children.exists():pending.extend(int(s) for s in children.read_text().split())
|
||||
result[name]=dict(service=fields,processes=processes)
|
||||
return result
|
||||
|
||||
|
||||
def stored_bootstrap_error_codes(payload):
|
||||
"""Only allowlisted stored daemon errors during this runtime deployment."""
|
||||
known=('CLOCK_UNTRUSTED','HELPER_VALIDATION_OR_IO_FAILURE','HELPER_PIN_PROFILE_MISMATCH',
|
||||
'SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE','HELPER_CHECKPOINTS_DID_NOT_CONVERGE',
|
||||
'HELPER_MANUAL_RECOVERY_REQUIRED','LOCAL_CHECKPOINT_ROLLBACK','SYNC_MANUAL_RECOVERY_REQUIRED')
|
||||
result={}
|
||||
for unit in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
||||
process=subprocess.run(['/usr/bin/journalctl','--no-pager','--quiet','--output=json','--unit',unit,
|
||||
'--since','2026-11-13 17:34:00 UTC','--lines','100'],capture_output=True,check=True,timeout=10)
|
||||
require(len(process.stdout)<=256*1024,'bounded deployment error metadata')
|
||||
counts={k:0 for k in known};times=[]
|
||||
for line in process.stdout.splitlines():
|
||||
row=json.loads(line);message=row.get('MESSAGE','')
|
||||
for code in known:
|
||||
if code in message:counts[code]+=1;times.append(row.get('__REALTIME_TIMESTAMP'))
|
||||
result[unit]=dict(counts={k:v for k,v in counts.items() if v},first=min(times) if times else None,last=max(times) if times else None)
|
||||
return result
|
||||
|
||||
@@ -95,6 +95,9 @@ def deploy(payload):
|
||||
require(built['exit_code'] == 0, 'build must pass')
|
||||
require(all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()), 'source drift')
|
||||
require(not (backup/'deploy-result.json').exists(), 'deployment already recorded')
|
||||
if payload.get('previous_ui_index_sha256'):
|
||||
require(hashlib.sha256((root/'web/dist/index.html').read_bytes()).hexdigest()==payload['previous_ui_index_sha256'],
|
||||
'independently known previous UI index required')
|
||||
if payload.get('closed_policy_sha256'):
|
||||
import datetime as dt
|
||||
policy = (root/'devstate/root-policy/policy.json').read_bytes()
|
||||
@@ -142,6 +145,7 @@ def deploy(payload):
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
|
||||
parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path')
|
||||
parser.add_argument('--expected-ui-index-sha256',help='CAS the independently known previous UI build before publication')
|
||||
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if args.action == 'stage':
|
||||
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
|
||||
@@ -161,6 +165,9 @@ def main():
|
||||
ident = str(uuid.UUID(args.id)); cloud.contract.protected_json(STATE/(ident+'.json'))
|
||||
before = None; payload = dict(id=ident)
|
||||
if args.action == 'deploy':
|
||||
if args.expected_ui_index_sha256:
|
||||
require(len(args.expected_ui_index_sha256)==64 and all(c in '0123456789abcdef' for c in args.expected_ui_index_sha256),'fixed previous UI hash')
|
||||
payload['previous_ui_index_sha256']=args.expected_ui_index_sha256
|
||||
target = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
gate = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
|
||||
if args.preserve_known_unknown:
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Passive Cloud auto-WSS→HTTPS fallback with two actual mTLS WSS rejections.
|
||||
|
||||
No RMM job, plan, approval or new identity. Only explicit, bounded dev/test
|
||||
transport fault configuration and preserving agent/service reconnects exist.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import cloud_mvp_acceptance as cloud
|
||||
|
||||
require = cloud.contract.require
|
||||
STATE = cloud.ROOT/'devstate/cloud-transport'
|
||||
|
||||
|
||||
def auto_mode(payload):
|
||||
import sqlite3
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed permanent identity')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['agent_sha256'],'known qualified agent')
|
||||
for path,query in ((state/'journal.sqlite',"SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
|
||||
(Path('/var/lib/rmm-helper/consumption.sqlite'),'SELECT count(*) FROM consumed WHERE result IS NULL')):
|
||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0,'all native effects must be settled')
|
||||
unit = Path('/etc/systemd/system/rmm-agent-remote.service')
|
||||
require(unit.read_text().splitlines().count('Environment=RMM_TRANSPORT=auto') == 1,'known automatic base service')
|
||||
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'no environment-file transport override')
|
||||
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
||||
selected = directory/'95-cloud-acceptance-transport.conf'
|
||||
for path in directory.glob('*.conf'):
|
||||
require(not path.is_symlink(),'transport drop-in symlink')
|
||||
require(path == selected or 'RMM_TRANSPORT=' not in path.read_text(),'unknown transport override')
|
||||
changed = selected.exists()
|
||||
if changed:
|
||||
require(selected.stat().st_uid == 0 and selected.read_text() in
|
||||
('[Service]\nEnvironment=RMM_TRANSPORT=wss\n','[Service]\nEnvironment=RMM_TRANSPORT=https\n'),'known existing test override')
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-auto-transport-'+str(uuid.UUID(payload['id'])))
|
||||
backup.mkdir(mode=0o700)
|
||||
os.rename(selected,backup/'previous-transport.conf')
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
|
||||
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
|
||||
return dict(mode='auto',preserved_identity=True,removed_test_override=changed)
|
||||
|
||||
|
||||
def configure_server(payload):
|
||||
root = Path('/opt/rust-discovery');compose = root/'deploy/server.compose.yaml'
|
||||
directory = root/'devstate/root-policy';path = directory/'transport-fault.json'
|
||||
require(not compose.is_symlink() and hashlib.sha256(compose.read_bytes()).hexdigest() == payload['compose_sha256'],'independently known compose required')
|
||||
require(not directory.is_symlink() and directory.is_dir() and directory.stat().st_uid == 10001
|
||||
and directory.stat().st_mode & 0o777 == 0o700,'existing private read-only mount')
|
||||
require(not path.is_symlink(),'transport config symlink')
|
||||
old = path.read_bytes() if path.exists() else None
|
||||
actual_sha = hashlib.sha256(old).hexdigest() if old is not None else None
|
||||
expected = payload['expected_config_sha256']
|
||||
# Cleanup may accept either of two PREVIOUSLY known hashes when an SSH
|
||||
# response was lost; it never adopts an arbitrary hash read from the host.
|
||||
require(actual_sha in expected if isinstance(expected,list) else actual_sha == expected,'known prior transport config')
|
||||
tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111','fixed development tenant')
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(compose)]
|
||||
query = "SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND state NOT IN ('succeeded','failed','cancelled','expired');"
|
||||
require(subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'],
|
||||
input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip() == b'0','no open Cloud jobs')
|
||||
enabled = payload['enabled'];require(type(enabled) is bool,'explicit opt-in')
|
||||
backup = root/'devstate'/('transport-fault-'+str(uuid.UUID(payload['id'])))
|
||||
backup.mkdir(mode=0o700)
|
||||
def create(target,raw,uid=0):
|
||||
fd = os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'wb') as stream:
|
||||
stream.write(raw);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,uid)
|
||||
if old is not None:create(backup/'transport-fault-before.json',old)
|
||||
if enabled:
|
||||
value = payload['config'];require(value['tenant_id'] == tenant and value['agent_id'] == agent,'exact scope')
|
||||
raw = json.dumps(value,separators=(',',':')).encode()
|
||||
create(directory/'transport-fault.next',raw,10001);os.replace(directory/'transport-fault.next',path)
|
||||
else:raw = old
|
||||
overlay = backup/'transport.compose.yaml'
|
||||
# Literal overlay; no existing .env secret is read, returned or replaced.
|
||||
body = 'services:\n'
|
||||
for role in ('api','gateway'):
|
||||
body += ' '+role+':\n environment:\n RMM_DEV_TRANSPORT_FAULTS: "'+('true' if enabled else 'false')+'"\n'
|
||||
if enabled:body += ' RMM_DEV_TRANSPORT_FAULT_FILE: /root-policy/transport-fault.json\n'
|
||||
create(overlay,body.encode())
|
||||
cmd += ['-f',str(overlay)]
|
||||
subprocess.run(cmd+['config','-q'],capture_output=True,check=True,timeout=15)
|
||||
completed = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],capture_output=True,timeout=25).returncode == 0
|
||||
return dict(deployed=completed,enabled=enabled,backup=str(backup),config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None)
|
||||
|
||||
|
||||
def server_evidence(payload):
|
||||
root = Path('/opt/rust-discovery');tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']));fault = str(uuid.UUID(payload['fault_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111','fixed test tenant')
|
||||
query = ("SELECT json_build_object('receipts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data) ORDER BY created_at),'[]'::json) "
|
||||
"FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_transport_fault_adapter' AND data->>'fault_id'='"+fault+"'),"
|
||||
"'device',(SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,"
|
||||
"'transport',transport,'connection_generation',connection_generation,'last_contact_at',last_contact_at,'last_authenticated_at',last_authenticated_at,"
|
||||
"'certificate_sha256',last_authenticated_certificate_sha256,'sync_ready',sync_ready,'restore_blocked',restore_blocked) "
|
||||
"FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'));")
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout)
|
||||
|
||||
|
||||
def runtime_flags(payload):
|
||||
root = Path('/opt/rust-discovery')
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
|
||||
result = {}
|
||||
for role in ('api','gateway'):
|
||||
ident = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip()
|
||||
require(ident and '\n' not in ident,'one running role')
|
||||
values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',ident],capture_output=True,check=True,timeout=10).stdout)
|
||||
keys = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_TRANSPORT_FAULTS','RMM_DEV_TRANSPORT_FAULT_FILE')
|
||||
result[role] = {v.split('=',1)[0]:v.split('=',1)[1] for v in values if v.split('=',1)[0] in keys}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-transport',action='store_true',required=True)
|
||||
parser.add_argument('--expected-agent-sha256',required=True)
|
||||
parser.add_argument('--expected-compose-sha256',required=True)
|
||||
parser.add_argument('--expected-config-sha256')
|
||||
args = parser.parse_args();os.umask(0o077)
|
||||
for value in (args.expected_agent_sha256,args.expected_compose_sha256):require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified pin')
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
||||
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
scope = dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'])
|
||||
def gate():return cloud.bootstrap.rpc('server.conf',cloud.server,dict(scope,kind='gate'))
|
||||
before = gate();require(before['unresolved'] == 0,'earlier effects must be closed')
|
||||
ident = str(uuid.uuid4());folder = STATE/ident;folder.mkdir(mode=0o700,parents=True)
|
||||
evidence_scope = dict(scope,fault_id=ident)
|
||||
auto = cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=ident,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256))
|
||||
checkpoint = dict(profile='dev/test',testsimulation=True,case='actual_auto_fallback',id=ident,identity=identity,preapproval_counts=before,auto_mode=auto,no_jobs_created=True)
|
||||
path = folder/'checkpoint.json'
|
||||
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
||||
def wait_transport(transport,after=None):
|
||||
end=time.monotonic()+150
|
||||
while True:
|
||||
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);device=observed['device']
|
||||
if (device['transport'] == transport and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at']
|
||||
and (after is None or cloud.contract.timestamp(device['last_contact_at']) > after)):return observed
|
||||
require(time.monotonic() < end,'fresh actual transport heartbeat and signed Sync missing');time.sleep(2)
|
||||
save();checkpoint['initial_wss']=wait_transport('wss');save()
|
||||
now=cloud.contract.now();config=dict(schema_version=1,fault_id=ident,**scope,issued_at=now.isoformat(),expires_at=(now+dt.timedelta(minutes=10)).isoformat())
|
||||
raw=json.dumps(config,separators=(',',':')).encode();checkpoint['config']=config;save()
|
||||
enabled=None;complete=False;attempted=False
|
||||
try:
|
||||
attempted=True
|
||||
enabled=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=ident,enabled=True,**scope,compose_sha256=args.expected_compose_sha256,
|
||||
expected_config_sha256=args.expected_config_sha256,config=config))
|
||||
checkpoint['enabled']=enabled;save();require(enabled['deployed'],'transport opt-in incomplete')
|
||||
end=time.monotonic()+150
|
||||
while True:
|
||||
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);receipts=observed['receipts'];device=observed['device']
|
||||
if (len(receipts) == 2 and device['transport'] == 'https' and device['sync_ready'] and not device['restore_blocked']
|
||||
and device['last_contact_at'] and cloud.contract.timestamp(device['last_contact_at']) > cloud.contract.timestamp(receipts[1]['created_at'])):break
|
||||
require(time.monotonic() < end,'two authenticated WSS503 and actual HTTPS heartbeat missing');time.sleep(2)
|
||||
require({r['data']['slot'] for r in receipts} == {1,2} and len({r['data']['handshake_id'] for r in receipts}) == 2
|
||||
and all(r['data']['status'] == 503 and r['data']['agent_id'] == identity['agent_id'] and r['data']['tenant_id'] == cloud.TENANT
|
||||
and r['data']['config_sha256'] == hashlib.sha256(raw).hexdigest() and r['data']['certificate_sha256'] == device['certificate_sha256']
|
||||
and r['data']['testsimulation'] is True for r in receipts),'actual exact peer fault receipts')
|
||||
require(all(device[k] == identity[k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version')),'permanent identity unchanged')
|
||||
checkpoint['https']=observed;save();complete=True
|
||||
finally:
|
||||
if attempted:
|
||||
closed=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,**scope,compose_sha256=args.expected_compose_sha256,
|
||||
expected_config_sha256=[hashlib.sha256(raw).hexdigest(),args.expected_config_sha256]))
|
||||
checkpoint['disabled']=closed;save();require(closed['deployed'],'transport fault cleanup incomplete')
|
||||
flags=cloud.bootstrap.rpc('server.conf',runtime_flags,{})
|
||||
require(len(flags) == 2 and all(v.get('RMM_DEV_TRANSPORT_FAULTS') in (None,'false') and 'RMM_DEV_TRANSPORT_FAULT_FILE' not in v for v in flags.values()),'both real roles disabled')
|
||||
checkpoint['disabled_runtime']=flags;save()
|
||||
checkpoint['restored_wss']=wait_transport('wss',cloud.contract.now());save()
|
||||
require(complete,'fallback not proven')
|
||||
after=gate();require(after == before,'passive fallback must leave job/outbox/grant counts unchanged')
|
||||
checkpoint.update(post_counts=after,verified=True,stage='native_dev_test_accepted');save()
|
||||
print('PASS passive Cloud auto WSS→HTTPS: two authenticated503, fresh HTTPS heartbeat+Sync, normal WSS restored, identity and job counts preserved')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_TRANSPORT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exact dev/test root script payload. Never installed as a daemon/provider."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
assert os.geteuid() == 0 and len(sys.argv) == 3
|
||||
operation, identifier = sys.argv[1:]
|
||||
assert operation in ('effect', 'read') and str(uuid.UUID(identifier)) == identifier
|
||||
directory = Path('/var/lib/rmm-helper')/('crash-'+identifier)
|
||||
for parent in directory.parents:
|
||||
metadata = parent.lstat()
|
||||
assert stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_mode & 0o022 == 0
|
||||
os.umask(0o077)
|
||||
if operation == 'effect' and not directory.exists(): directory.mkdir(mode=0o700)
|
||||
metadata = directory.lstat()
|
||||
assert stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == 0 and stat.S_IMODE(metadata.st_mode) == 0o700
|
||||
counter = directory/'count'
|
||||
def read_counter():
|
||||
fd = os.open(counter, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
with os.fdopen(fd, 'r') as stream:
|
||||
metadata = os.fstat(stream.fileno())
|
||||
assert stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_nlink == 1
|
||||
assert stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16
|
||||
value = stream.read()
|
||||
assert value.endswith('\n') and value[:-1].isdigit()
|
||||
return int(value)
|
||||
if operation == 'effect':
|
||||
lock = directory/'lock';lock.mkdir(mode=0o700)
|
||||
try:
|
||||
value = read_counter() if counter.exists() else 0
|
||||
assert 0 <= value < 100
|
||||
fd = os.open(directory/'count.next',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:
|
||||
stream.write(str(value+1)+'\n');stream.flush();os.fsync(stream.fileno())
|
||||
os.replace(directory/'count.next',counter)
|
||||
fd = os.open(directory,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
finally: lock.rmdir()
|
||||
# Independent helper remains alive after only the agent process dies.
|
||||
time.sleep(90)
|
||||
assert read_counter() == 1
|
||||
print('RMM_CRASH_COUNT=1')
|
||||
@@ -0,0 +1,102 @@
|
||||
# Agentabbruch nach möglicher Wirkung: vorbereitete Abnahme
|
||||
|
||||
Stand 2026-11-13. **Noch kein nativer Crashversuch ausgeführt.** Dieser Plan
|
||||
erweitert weder Rootrechte noch die WebUI-Ausnahme. Vor einer tatsächlichen
|
||||
Ausführung müssen frühere Wirkungen abgeschlossen und VM-/UI-Fenster frei sein.
|
||||
|
||||
## Gefundene Lücke und minimale Korrektur
|
||||
|
||||
`Agent::start_sync` wartet zuerst auf einen unabhängig authentisierten
|
||||
Helper-Snapshot. Der Roothelper bearbeitet IPC seriell und hält den gemeinsamen
|
||||
Executorlock während der Wirkung; ein Snapshot wartet deshalb oder läuft nach
|
||||
10 Sekunden in einen Timeout. Nach Ende der Wirkung importiert
|
||||
`reconcile::prepare` das gespeicherte Helperergebnis **vor** seiner bisherigen
|
||||
Unknown-Konvertierung. Ein früherer Test hätte deshalb möglicherweise direkt
|
||||
Erfolg statt eines nachweisbar dauerhaften Unknown ergeben.
|
||||
|
||||
Der Agent hält nun genau einmal beim **neuen Prozessstart**, vor der ersten
|
||||
Verbindung und jedem Helperzugriff, unterbrochene Versuche dauerhaft fest:
|
||||
|
||||
- `running`/`verifying` mit bekanntem Grant: unveränderten Offer und ursprünglich
|
||||
signierten Startrequest gegen tatsächlichen Verbrauch und Gerätebindung prüfen;
|
||||
danach regulärer dauerhafter `JobResult=unknown`, eigener Fehlercode
|
||||
`AGENT_PROCESS_INTERRUPTED`, höhere Revision, keine erfundene Wirkung/Ausgabe.
|
||||
- `requesting` ohne bekannten Grant: nur lokal Unknown; keinen Grant oder
|
||||
grantgebundenen Ergebnisnachweis erfinden. Der normale Sync bleibt zuständig.
|
||||
- Verbrauch, Startnachweis und Ressourcensperre bleiben erhalten. Kein `finish`,
|
||||
kein erneuter Permitrequest, keine Änderung erfolgreicher Endzustände.
|
||||
- Normale erste Kanalwiedergabe sendet den dauerhaften Unknown vor dem
|
||||
blockierenden Snapshot. Reconnect desselben Prozesses ruft diese Korrektur
|
||||
nicht auf. Wiederholte Prozessstarts verändern einen vorhandenen Unknown
|
||||
einschließlich Nachrichten-ID und Revision nicht erneut.
|
||||
|
||||
Der Abschluss verwendet ausschließlich den vorhandenen Pfad: unabhängiger
|
||||
Roothelper schreibt `finish` **vor** seiner IPC-Antwort; authentisierter Snapshot
|
||||
importiert denselben ursprünglichen Request und dessen tatsächliches Ergebnis;
|
||||
normaler signierter Sync bindet Revision, Ergebnis-/Plan-/Payloadhash, Agent,
|
||||
Versuch und Grant und öffnet erst danach die Sperre. Eine neue Recovery-API,
|
||||
administrative Ergebnisinjektion oder erneute Skriptausführung existiert nicht.
|
||||
|
||||
Tatsächlich lokal bestanden: gezielter Dispatcher-/SQLite-Test für dauerhaften
|
||||
Unknown vor jedem Helper-I/O, abgewiesene fremde Verbrauchsidentität,
|
||||
grantlosen Antrag, erhaltene Verbrauchssperre und idempotenten Neustart;
|
||||
vorhandener Integrationstest für authentisierten Helperimport als neuen
|
||||
Ergebnisnachweis ohne erneute Wirkung; Agent-Clippy aller Targets mit
|
||||
`-D warnings`. Dies sind keine VM-Nachweise.
|
||||
|
||||
Ein qualifiziert gebautes x86-64-Releaseartefakt liegt geschützt unter
|
||||
`devstate/cloud-apt-recovery/agent-interruption-recovery`, SHA-256
|
||||
`adc4ac0b35f7992300d82b0f263970246fdc0eaec0d2104e4ade22ae3f74d7b6`.
|
||||
ELF-/Abhängigkeitsprüfung: x86-64, höchste benötigte GLIBC-Version 2.34,
|
||||
OpenSSL 3. Noch nicht auf die VM übertragen oder dort gestartet.
|
||||
|
||||
Vorbereitet, **nicht ausgeführt**: `deploy/cloud_crash_acceptance.py` mit
|
||||
echtem zweiphasigem Browserablauf, `deploy/browser/cloud_crash_acceptance.cjs`
|
||||
und exakt sichtbarem Payload `deploy/linux/agent-crash-counter.py`.
|
||||
Python-/Node-Syntax sowie exakte Python-/Browser-Payloadgleichheit und
|
||||
Shell-Parsing ohne Ausführung wurden lokal geprüft. Der Runner verlangt
|
||||
unabhängig bekannte Agent-/Helper-SHAs und den bisherigen geschlossenen
|
||||
Policyhash; er verändert weder Runtimeartefakte noch Providerpins.
|
||||
|
||||
## Geplanter nativer Versuch, noch nicht ausgeführt
|
||||
|
||||
1. Genau die vorhandene Cloud-VM und Geräteidentität verwenden. Zentral und im
|
||||
erhaltenen Journal keine offenen Versuche, qualifizierter neuer Agent,
|
||||
unveränderter gesunder Roothelper. Keine parallele VM-/Imageänderung.
|
||||
2. Fester Root-RunScript-Plan erzeugt einen einmaligen privaten Testzähler unter
|
||||
dem beschreibbaren Helper-StateDirectory, hält danach begrenzt 60–90 Sekunden
|
||||
an und endet mit fest erwarteter Ausgabe bei Laufzeitlimit 120 Sekunden.
|
||||
Alle Bytes, Ziel und Fristen stehen vor der tatsächlichen WebUI-Genehmigung
|
||||
fest; vorher 0 Jobs / 0 Outbox / 0 Grants. Keine Paketmanagerwirkung.
|
||||
3. Nach gespeichertem Verbrauch/Start und dem vom genehmigten Skript dauerhaft
|
||||
geschriebenen Zähler den **exakten Agentprozess** kontrolliert beenden;
|
||||
PID-/Startzeit-/Artefaktbindung prüfen, keinen Prozessbaum oder ganzen
|
||||
systemd-Cgroup beenden. Roothelper und dessen Skript bleiben unbeeinflusst.
|
||||
Die gespeicherten Test-/Ledgerdaten dienen nur der Fehlerorchestrierung.
|
||||
4. Der vorhandene Agentdienst startet automatisch neu. Während der Helper noch
|
||||
arbeitet, lokalen dauerhaften Unknown und dessen tatsächliche zentrale
|
||||
Jobansicht erfassen: gleicher Versuch/Grant, höhere Revision, keine neue
|
||||
Wirkung/Starterlaubnis. Startcapability-Probes können zusätzlich bis zu
|
||||
10 Sekunden Helper-Policy und zweimal 5 Sekunden APT-Probe benötigen; den
|
||||
Zeitpuffer vorab berücksichtigen, keine Wartezeit als Ergebnis erfinden.
|
||||
5. Roothelper darf genau einmal regulär fertig werden. Sein tatsächliches
|
||||
gespeichertes Ergebnis, authentisierten Import, signierten Sync und endgültige
|
||||
identische Ergebnisbindung nachweisen. Beide Journale/Tombstones behalten;
|
||||
kein manueller Unlock und keine Wiederholung nach Fehler.
|
||||
6. Ein **separater** fester WebUI-genehmigter Lesejob prüft abschließend den
|
||||
erhaltenen Zähler exakt 1. Eigenen Antrag/Genehmigungs-/Grantnachweis getrennt
|
||||
vom ursprünglichen Versuch führen. Erst bei keiner offenen Wirkung abnehmen,
|
||||
dokumentieren und pushen.
|
||||
|
||||
Ein frei laufendes `agent_service`-Skript, dessen Prozess vor dauerhafter
|
||||
Ergebnisspeicherung stirbt, hat heute keinen unabhängigen Abschlussbeleg.
|
||||
`expected_stdout` allein rekonstruiert weder Exitstatus noch Wirkung. Diesen
|
||||
Versuch daher **nicht** als Ersatz ausführen. Der vorbereitete Roothelperfall
|
||||
weist genau den Agentprozessabbruch bei unabhängig weiterlaufendem Executor
|
||||
nach; ein gleichzeitiger Helper-/Maschinenabsturz bleibt ein anderer Fehlerfall.
|
||||
|
||||
Nachfolgender Runtime-Stand: die tatsächlich auf Cloud installierte Variante
|
||||
`agent-journal-capacity` (`2ce885cf…`) enthält dieselbe Startupkorrektur plus die
|
||||
endliche Journalbudgetkorrektur. Normaler Sync und kopierte Sandboxprobe mit
|
||||
erhaltener Historie bestanden. Das ist weiterhin **kein** nativer Crashversuch;
|
||||
der oben beschriebene kontrollierte Versuch bleibt ein eigenes Gate.
|
||||
@@ -0,0 +1,62 @@
|
||||
# Cloud-Abnahme: Offlinefenster, Ablauf und Genehmigungsentzug
|
||||
|
||||
Stand: Adapter implementiert und lokal auf Python-/Node-Syntax sowie identische
|
||||
Skriptbytes zwischen Orchestrator und Browser geprüft. **Noch kein nativer
|
||||
VM-Nachweis aus diesem Adapter.** Die folgenden Befehle sind geplant, nicht
|
||||
ausgeführt. Alle Ergebnisse entstehen als private, fortlaufende JSON-Belege in
|
||||
`devstate/cloud-offline/<UUID>/`.
|
||||
|
||||
Der Adapter verwendet ausschließlich die bestehende Cloud-Linux-VM und den
|
||||
festgelegten Cloudserver. Das Notebook betreibt keinen Agenten oder Server.
|
||||
Er setzt einen gesunden signierten Reconnect, keine offenen Wirkungen und den
|
||||
geprüften aktuellen Agentenhash aus `devstate/remote-vm.json` voraus.
|
||||
|
||||
```sh
|
||||
python3 deploy/cloud_offline_acceptance.py --execute-dev-test-ui --case offline_wait
|
||||
python3 deploy/cloud_offline_acceptance.py --execute-dev-test-ui --case expired_offline
|
||||
python3 deploy/cloud_offline_acceptance.py --execute-dev-test-ui --case revoked_before_start
|
||||
```
|
||||
|
||||
Jeder Aufruf erzeugt einen eigenen unveränderlichen Plan mit genau einem
|
||||
zukünftigen `agent_service`-Skriptjob. MCP erstellt Plan und persistenten Antrag;
|
||||
vor der Genehmigung müssen Jobs, Outbox und Startgrants jeweils null sein.
|
||||
`deploy/browser/cloud_offline_acceptance.cjs` prüft alle Planbytes und betätigt
|
||||
den echten WebUI-Genehmigungsknopf mit dem sichtbar simulierten Testprincipal.
|
||||
Der dritte Fall verwendet zusätzlich das vorhandene WebUI-Control
|
||||
„Planberechtigung widerrufen“ mit Bestätigungskennung. Keine Freigabe oder
|
||||
Widerruf wird über einen direkten API-/DB-Aufruf ersetzt.
|
||||
|
||||
Ein Agentstopp allein gilt nicht als Offline-Nachweis. Der Adapter prüft vor
|
||||
dem ersten Job die tatsächliche systemd-BPF-Funktion: Nur für
|
||||
`rmm-agent-remote.service` wird die feste Serveradresse `88.99.80.202/32`
|
||||
gesperrt. Ein separater Bootstrap-Prozess verschiebt ausschließlich seine
|
||||
eigene PID in die tatsächliche Cgroup dieses Dienstes, wechselt zu UID/GID 993
|
||||
und versucht nur einen TCP-Verbindungsaufbau zu Port 443. Ausschließlich
|
||||
`EACCES` oder `EPERM` bestätigt die Sperre; ein Timeout genügt nicht. Der Probe
|
||||
liest keine Agentenidentität oder Zugangsdaten und sendet keine Anwendungsdaten.
|
||||
Die TLS-Prüfungen des echten Agenten bleiben erhalten.
|
||||
|
||||
Nach der dauerhaften Jobannahme werden vor der Sperre erneut null Grants,
|
||||
Consumption-Einträge und Wirkungen geprüft. Während des Offlinefensters muss
|
||||
der Agent aktiv bleiben, seine Reconnect-Versuche dauerhaft zählen und auch
|
||||
über `not_before` hinaus ohne Grant oder Effekt bleiben. SSH und Helper sind
|
||||
von der Netzsperre unabhängig. Die bereits vorhandenen Root-Consumption- und
|
||||
Start-/Outcome-Belege müssen dieselben Hashes behalten.
|
||||
Der kurze operative Stopp zum Einrichten der Regel zählt nicht zum Nachweis;
|
||||
der Agent muss bereits vor `not_before` wieder aktiv und tatsächlich gesperrt sein.
|
||||
|
||||
- `offline_wait`: Netz vor Ablauf wiederherstellen; ein neuer gesunder
|
||||
signierter Sync muss vor genau einer Startreservierung liegen. Erwartet sind
|
||||
ein Grant, ein Consumption-Eintrag und der gespeicherte Effektzähler `1`.
|
||||
- `expired_offline`: Sperre bis hinter die feste Startfrist halten;
|
||||
anschließend `expired`, null Grants und keine Wirkung nachweisen.
|
||||
- `revoked_before_start`: Genehmigung während der Sperre tatsächlich in der
|
||||
WebUI entziehen; nach dem Reconnect `cancelled`, null Grants und keine
|
||||
Wirkung einschließlich interaktivem Widerrufsaudit nachweisen.
|
||||
|
||||
Ein vorab eingerichteter Timer entfernt nach 420 Sekunden ausschließlich die
|
||||
eigenen unveränderten Drop-in-Bytes und stellt den normalen Agententransport
|
||||
wieder her. Diese Frist liegt hinter der festen Jobfrist. Ein fehlgeschlagener
|
||||
Aufruf wird nicht automatisch wiederholt; seine Belege und Jobs bleiben zur
|
||||
Auswertung erhalten. Das Skript besitzt keinen Journal-, Identitäts- oder
|
||||
Consumption-Reset.
|
||||
@@ -1,5 +1,17 @@
|
||||
# Übergabe / Entwicklungspause — 2026-11-12
|
||||
|
||||
**Neuester aktiver Stand2026-11-13:** Scope-Replayfix und TLS-Lifecycle einschließlich
|
||||
nativ abgelaufener Clientcerts bestanden. UI-Schema4 geprüft/publiziert;
|
||||
Configure-only-Plan tatsächlich genehmigt, aber ohneOffer/Grant expired an
|
||||
64MiB-Helperjournalgrenze. Isoliert SQLFull reproduziert, bounded256MiB-Default
|
||||
auf Cloud-VM mit allen Historien/Identitäten/DBinodes erhalten installiert.
|
||||
Agent2ce885cf/Helper332eae00/Provider5ff260f7, normaler signierterSync bereit,
|
||||
Root10/0offen/0faults, Zentrale19/19/17/0, Rootpolicygeschlossen/APTinactive.
|
||||
Neuer eigener UI-Configureplan im Namensraum `capacity-recovery` erforderlich;
|
||||
Paketwirkung noch NICHT ausgeführt. Details oben in implementation-status.md;
|
||||
MVP weiterhin offen, kein Produktionsfreigabenachweis. Ältere Zwischenstände
|
||||
bleiben historisch und werden nicht als aktuelle Runtimepins verwendet.
|
||||
|
||||
**Neuester Stand 2026-11-13, Arbeit ausdrücklich fortgesetzt:** Signierte
|
||||
Kontrollkettenreparatur auf tatsächlicher Cloud-VM bestanden. Anschließender
|
||||
Readonly-APT-Nachweis plus normaler RootSync schloss ursprüngliche Teilwirkung
|
||||
|
||||
@@ -1,5 +1,51 @@
|
||||
# Tatsächlicher Implementierungsstand
|
||||
|
||||
## Journal-Kapazität nativ korrigiert, vollständige Historie erhalten — 2026-11-13
|
||||
|
||||
Der tatsächlich per simulierter WebUI genehmigte Configure-only-Plan wurde
|
||||
zunächst nicht zugestellt und **expired** mit **0 Grants**, keinem lokalen
|
||||
Versuch/Verbrauch und keiner Paketwirkung. Vorherige UI-Schema4-Erweiterung
|
||||
mit elf gezielten Safetytests, Typecheck und Produktionsbuild bestanden;
|
||||
Indexpublication verglich den unabhängig vor dem Build gesicherten alten
|
||||
Indexhash. Browser prüfte tatsächlich feste Schema4-/Planbytes und 0/0/0 vor
|
||||
Genehmigung. Dieser Lauf ist ausdrücklich keine Paketabnahme und bleibt erhalten.
|
||||
|
||||
Ursache des geschlossenen Sync-Gates: Helperledger **67.076.096Bytes /16.376
|
||||
Seiten à4096Bytes** bei festem Connectionlimit16.384Seiten. Datenträger noch
|
||||
rund21GB frei, Kerneluhr vertrauenswürdig, kein Executorlockinhaber. Nur erlaubte
|
||||
gespeicherte Fehlercodezählungen der drei Projektdienste als operative
|
||||
Bootstrapfehlermetadaten gelesen, keine Kundenlogs oder Paketdiagnose ausgeführt.
|
||||
Isolierte tatsächliche VM-Kopie reproduzierte mit512KiB-Testschreiben
|
||||
**SQLITE_FULL13** unter altem Limit und Erfolg unter neuem endlichem Limit.
|
||||
Beide Transaktionen vollständig zurückgerollt; Kopieinhalte und Originalbytes,
|
||||
Größe/Inode unverändert, keine ursprüngliche SQL- oder Paketmutation.
|
||||
|
||||
Minimaler Default auf **256MiB pro Datei** angepasst;4096-Byte-Seiten explizit
|
||||
geprüft. Kein unbegrenzter Modus oder Kompaktierungs-/Resetpfad. Helper meldet
|
||||
echtes DiskFull ohne Rohsql als `JOURNAL_STORAGE_CAPACITY_BLOCKED`.
|
||||
**Vier gezielte Kapazitäts-/Rollback-/Fehlerklassifizierungs-/Sequenztests**,
|
||||
Agent-/Helper-Alltargets-Clippy und Releasebuilds bestanden. Neuer Agent enthält
|
||||
außerdem die lokal geprüfte Startup-Unknown-Korrektur; eigentliche native
|
||||
Abbruchabnahme weiterhin offen und unter `agent-crash-acceptance.md` getrennt.
|
||||
|
||||
Native identitäts-/journalbewahrende Installation von Agent`2ce885cf…` und
|
||||
Helper`332eae00…` bestanden, Provider`5ff260f7…` unverändert. Unabhängige Pins,
|
||||
neue helpergebundene Runtime und kopierte dpkg-Sandboxprobe aktualisiert;
|
||||
alle alten Reports versioniert. Vor normalem Neustart Agent-/Ledgerbytes und
|
||||
Inodes, alle zehn abgeschlossenen Rootverbräuche, ursprünglicher Partial-Proof,
|
||||
Marker/Active-/History-/Archive und Generalpolicy gleich. Danach gewöhnlicher
|
||||
signierter Sync tatsächlich **bereit**, Helper **10 consumed /0offen /0faults**,
|
||||
Zentrale **19 Jobs /19Outbox /17Grants /0offen**. Rootpolicy geschlossen,
|
||||
APT-Endpunkt absent/Helperinactive; keine neue Paketwirkung durch Wartung/Probe.
|
||||
|
||||
Details: [journal-capacity.md](journal-capacity.md). Private Belege
|
||||
`devstate/cloud-apt-recovery/journal-capacity-copy-6e882a74-507a-4af0-8baf-8f9005e4ac08.json`,
|
||||
`journal-capacity-runtime.json`, `journal-capacity-native-ready.json` und
|
||||
ursprüngliches `cases/apt_configure_recovery/checkpoint-capacity-terminal.json`.
|
||||
**MVP weiterhin offen:** neuer fester UI-Plan im getrennten Kapazitätsruntime-
|
||||
Namensraum; ursprüngliche Genehmigung/abgelaufener Versuch werden nicht ersetzt.
|
||||
Danach native Upgrade-/Noop- und übrige Offline-/Abbruch-/Fallback-/Rebootgates.
|
||||
|
||||
## Abgelaufenes Clientzertifikat nativ auf TLS-Ebene verweigert — 2026-11-13
|
||||
|
||||
Isoliertes Schlüsselpaar ausschließlich auf der Cloud-VM, CSR öffentlich;
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# Begrenzte Journalgröße und SQLite-Kapazitätsfehler
|
||||
|
||||
Stand 2026-11-13. **Quellkorrektur implementiert, lokal geprüft und x86-64-
|
||||
Releaseartefakte gebaut; nachfolgende native Kopier-/Installationsbelege stehen unten.** Die
|
||||
erhaltene Cloudhistorie darf weder gelöscht noch kompaktiert/zurückgesetzt werden.
|
||||
|
||||
Der erfasste Cloudbefund zeigte das Rootledger mit **67.076.096 Byte**, also
|
||||
16.376 Seiten zu 4.096 Byte, unmittelbar unter dem bisherigen Limit von
|
||||
16.384 Seiten / 64 MiB. Trotz freiem Datenträger konnten neue größere
|
||||
Checkpointnachweise damit an `SQLITE_FULL` scheitern. Der allgemeine
|
||||
Helperfehler verdeckte diese Ursache. Das Agentjournal lag ebenfalls bereits
|
||||
nahe derselben Begrenzung. Dies ist ein aktueller MVP-Korrektheitsblocker.
|
||||
|
||||
Die bestehende Helperbibliothek setzt nun für gemeinsam genutztes Rootledger,
|
||||
lokales Verbrauchsledger und Agentjournal denselben **endlichen Grenzwert von
|
||||
65.536 Seiten bei zwingend 4.096 Byte**, also **256 MiB**. Die Grenze wird vor
|
||||
Schema-/Checkpointzugriffen eingerichtet und ihr tatsächlicher Rückgabewert
|
||||
geprüft. Eine andere Seitengröße oder bereits nicht einhaltbare Grenze verweigert
|
||||
den Start. Keine Journalbereinigung, kein Löschen von Tombstones, kein unbegrenzter
|
||||
Modus und kein Übergehen fehlgeschlagener Schreibvorgänge.
|
||||
|
||||
`max_page_count` ist eine SQLite-Verbindungseinstellung. Ein isolierter
|
||||
Reproduktionstest muss daher das alte bzw. neue Limit **auf seiner eigenen
|
||||
Testverbindung** einstellen; eine neue administrative Leserverbindung belegt
|
||||
nicht den bisherigen Grenzwert des laufenden Daemons.
|
||||
|
||||
Der Helper klassifiziert tatsächlich verkettete SQLite-`DiskFull`-Fehler jetzt
|
||||
bei Anfrage und Start als **`JOURNAL_STORAGE_CAPACITY_BLOCKED`**. Fehlerausgaben
|
||||
enthalten keine SQLtexte, privaten Kontexte oder rohen Datenbankmeldungen.
|
||||
|
||||
Tatsächlich lokal bestanden:
|
||||
|
||||
- Ursprünglich signierter Verbrauch/Startnachweis bleibt beim Wiederöffnen und
|
||||
erneuten Begrenzen unverändert; derselbe Grant bleibt verbraucht.
|
||||
- Unerwartete 8-KiB-Seitengröße wird verweigert, gespeicherter Inhalt bleibt.
|
||||
- Ein echtes `SQLITE_FULL` auf einem winzigen Vierseiten-Testjournal wird
|
||||
bereinigt klassifiziert; SQLite rollt die fehlgeschlagene Transaktion zurück,
|
||||
erhält früheren Inhalt und Integrität. Kein 256-MiB-Fülltest.
|
||||
- Agentjournal erhält Sequenz und neue endliche Grenze beim Wiederöffnen.
|
||||
- Agent-/Helper-Clippy aller Targets mit `-D warnings` und beide Releasebuilds.
|
||||
|
||||
Geschützte qualifizierte Artefakte (native Verwendung im letzten Abschnitt):
|
||||
|
||||
| Artefakt | SHA-256 |
|
||||
| --- | --- |
|
||||
| `devstate/cloud-apt-recovery/agent-journal-capacity` | `2ce885cf5e6881e77383482b7d3a87a02c6847aa26b82e5fd70d2ebd8a45360d` |
|
||||
| `devstate/cloud-apt-recovery/helper-journal-capacity` | `332eae004d9416234eabf7ed3d5c9ece5440d1e0c764ea63357961c16b673b6b` |
|
||||
|
||||
Beide ELF x86-64, höchste benötigte GLIBC-Version 2.34; private Begleitdateien
|
||||
halten Quellhashes und Prüfungen fest. Der APT-Provider blieb unverändert bei
|
||||
`5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327`.
|
||||
Native Installation muss alle Identitäten, Schlüssel, Ledgerinodes und
|
||||
ursprünglichen Versuchsnachweise erhalten sowie Runtime-/Providerpins neu
|
||||
nachweisen. Lokale Guards und Build sind noch keine native M4-Abnahme.
|
||||
|
||||
## Tatsächliche Cloud-Abnahme danach
|
||||
|
||||
Isolierte VM-Kopie reproduzierte `SQLITE_FULL`13 beim512KiB-Schreiben unter
|
||||
16.384Seiten; derselbe Vorgang unter65.536Seiten gelang, beide Tests rollback,
|
||||
Originalbytes/Größe/Inode unverändert. Privater Kopierbeleg
|
||||
`journal-capacity-copy-6e882a74-507a-4af0-8baf-8f9005e4ac08.json`.
|
||||
|
||||
Qualifizierter Austausch von Agent/Helper und helpergebundenem Runtime-/APTpin
|
||||
nativ bestanden. Provider und Installmetadata unverändert, alte Artefakte/Reports
|
||||
versioniert; vor normalem Restart vollständiger State/DBbytes und Inodes gleich,
|
||||
zehn bestehende Rootgrants geschlossen. Kopierte dpkg-Probe bestanden, keine
|
||||
Paketwirkung. Gewöhnlicher signierter Sync danach tatsächlich bereit,
|
||||
Rootledger10/0offen/0faults, Zentrale19/19/17/0. APT-Endpunkt geschlossen.
|
||||
Private Nachweise `journal-capacity-runtime.json`/`journal-capacity-native-ready.json`.
|
||||
Ein zuvor tatsächlich UI-genehmigter Configure-Plan blieb unzugestellt und
|
||||
expired mit0Grants; er wird erhalten, ein neuer Versuch braucht einen eigenen
|
||||
festen WebUI-Plan. Paket-/Crash-/Offline-MVP-Abnahmen dadurch nicht vorgetäuscht.
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Linux-MVP: Abnahmegates
|
||||
|
||||
Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.11. Die
|
||||
Stand 2026-11-13, ursprüngliche Auditbasis `d38d93d`, Spezifikation 1.12. Die
|
||||
Bestandsaufnahme selbst hat keine Tests, Jobs oder Deployments ausgeführt;
|
||||
anschließende tatsächliche Nachweise sind unten ausdrücklich ergänzt. Einzelbelege und spätere
|
||||
Ergebnisse stehen im [Implementierungsstand](implementation-status.md).
|
||||
@@ -45,7 +45,7 @@ normalem signiertem RootSync bestanden. Beidefaultfrei; Ledger 9/0 unresolved,
|
||||
zentral 13/13/12/0. Derselbe Versuch jetzt ehrlich failed
|
||||
`APT_PARTIAL_UNCONFIGURED_VERIFIED`, Fixture1.0 unpacked, keine Paket-Acceptance.
|
||||
Agent/Generalhelper aktiv, APT-Endpunkt gestoppt, Rootpolicy geschlossen.
|
||||
Neue Configure-only-Runtime samt kopierter nativer Sandboxprobe ist inzwischen qualifiziert; Paketwirkung noch nicht ausgeführt. Nächster Paketpfad ist der neue feste UI-Configure-only-Wiederherstellungsplan.
|
||||
Neue Configure-only-Runtime samt kopierter nativer Sandboxprobe ist inzwischen qualifiziert; Paketwirkung noch nicht ausgeführt. Erster neuer UI-Plan lief ohne Offer/Grant an der alten64MiB-Journalgrenze ab; unverändert erhalten. Neue endliche256MiB-Runtime samt Erhaltungsprobe nativ installiert und Sync wiederbereit. Nächster Paketpfad ist ein separat neu UI-genehmigter Configure-only-Plan.
|
||||
|
||||
Die noch offenen Teile der folgenden Gates sind **geplant, nicht ausgeführt**. Native Teilergebnisse stehen ausdrücklich bei den Gates und im Implementierungsstand.
|
||||
Jeder Ausführungsjob braucht weiterhin seinen eigenen festen WebUI-genehmigten
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ Korrektheitsfehler in einem aktuellen MVP-Pfad bleiben dort sofort relevant.
|
||||
| Dauerhafter Produktivbetrieb (M7) | Ein dedizierter Cloudserver, getrennte API/Gateway-Rollen, interne DB; Wiederanläufe ohne Historienreset belegt. | HA, getestete Backups/zentraler Restore, vereinbarte RTO/RPO, externe Plattformüberwachung, signierte Agentupdates und getrennt gesicherter Auditbestand. Grundlegende aktuelle Wiederholungssperren bleiben schon im MVP Pflicht. |
|
||||
| Gleichzeitiger Build und Deployment | Entdeckt beim Zustellungstest und inzwischen im aktuellen Upgradepfad behoben: eindeutiger Kandidatentag + aufgezeichneter Digest; nur explizites geprüftes Deploy bewegt den Runtimetag. Abgeschlossener nativer Build ließ Runtime unverändert. Historische Buildrunner bleiben getrennt zu prüfen. | Verfahren später auf alle Release-/Rollbacks-/CI-Pfade übertragen. Historische mutable Runner bis dahin nur ausdrücklich koordiniert verwenden. |
|
||||
| Monitoringausbau (M6a) | Präsenz, Quellen-/Datenalter und unbekannte Gesundheit werden getrennt dargestellt. | Sollservice-/Funktionschecks, Alarmdauer/Hysterese, Vorfälle und produktiver Metrikpfad samt nativen AT-28-Nachweisen; keine erfundenen Patchdaten. |
|
||||
| Langzeitjournale und Kompaktierung | Verbrauchs-/Versuchstombstones werden erhalten; volle/inkonsistente Journale sperren neue Wirkungen. | Dokumentierte beidseitige Kompaktierung und Kapazitätsplanung; niemals stille Löschung von Unknown oder unbestätigten Ergebnissen. |
|
||||
| Langzeitjournale und Kompaktierung | Reale Sync-Historien erreichten64MiB nach zehn Rootverbräuchen. Isoliert reproduziert; aktueller endlicher Default256MiB pro Datei, alle Historien erhalten. Volle/inkonsistente Journale sperren weiterhin neue Wirkungen. | Dokumentierte beidseitige Kompaktierung und Kapazitätsplanung; niemals stille Löschung von Unknown oder unbestätigten Ergebnissen. |
|
||||
| Dauerhaft ungültiges passives Inventar vor Reconnect-Sync | Der neue Recovery-Scope wurde zunächst vom zentralen Zeichenvalidator abgewiesen. Das unverändert persistierte Inventar wurde vor dem Sync erneut gesendet und blockierte so den Kanal; der unmittelbare MVP-Fix erlaubt ausschließlich diesen bekannten Linux-Scope. | Einen expliziten Umgang mit dauerhaft abgewiesener passiver Telemetrie untersuchen, der Originalbytes, Nachrichten-ID, Herkunft und Ablehnungsbeleg erhält und keine Bestätigung erfindet. Job-/Ergebnisjournale und unbekannte Wirkungen dürfen dabei niemals übersprungen werden. |
|
||||
| Weitere Plattformen / Skalierung | Ubuntu 24.04 x86-64 als Linux-Testclient; ARM64-Serverbuild ist kein ARM64-Agentnachweis. Nur eine tatsächliche Linux-Zielmaschine im aktuellen Aufbau. | Native Agentmatrix und realistische Last-/Mehrgeräteabnahmen erst zur vereinbarten Ausbaustufe erweitern; keine erfundenen Parallelitätszahlen. |
|
||||
| Proxmox/PBS/Cluster/Backups (M6b) | Kein Hypervisorzugang für eigenständige Gastagenten nötig; bewusst nicht Teil des Linux-MVP. | Separater späterer Integrationsauftrag mit eigenen ACL-/Discovery-/Backup-/Clusterabnahmen. |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Verbindliche Projektentscheidungen
|
||||
|
||||
Stand: **2026-11-13**, Spezifikation **1.11** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
|
||||
Stand: **2026-11-13**, Spezifikation **1.12** (`Rust_RMM_Implementierungsspezifikation.md`). Die ausdrücklichen normativen Entscheidungen in Kapitel 0 haben Vorrang vor früheren Architekturannahmen. Version 1.6 ersetzt die lokale Servertopologie durch den dedizierten Internetserver; Version 1.7 präzisiert das Notebook als reinen Entwicklungsrechner; Version 1.8 erlaubt vorgezogene parallele Windows-Entwicklung.
|
||||
|
||||
## Einzelner signierter APT-Wiederaufnahmeversuch (2026-11-13)
|
||||
|
||||
@@ -136,3 +136,12 @@ normale Baselines bleiben strikt. Eigene Recoverymarker, kein Überschreiben
|
||||
früherer Testmarker oder falsche Freshinstall-Acceptance. Jede Paketwirkung,
|
||||
anschließendes Upgrade und beide Noops erhalten eigene tatsächliche WebUI-
|
||||
Plangenehmigungen. Runtimeprobe allein ist kein Ausführungsnachweis.
|
||||
|
||||
## Endliches Journalbudget nach tatsächlichem Kapazitätsbefund
|
||||
|
||||
Spezifikation1.12 setzt pro Agentjournal/Agent- oder Helperledger die feste
|
||||
Grenze256MiB bei validierter4KiB-Seitengröße. Reale Cloud-Historie erreichte den
|
||||
alten64MiB-Wert bereits ohne offene Wirkung; unveränderte Originaldatenbank
|
||||
auf isolierter Kopie mit `SQLITE_FULL` reproduziert. Keine History-/Marker-
|
||||
löschung oder Ausnahme vom Fail-closed-Verhalten. Quelle, lokale Guards und
|
||||
nativer Erhaltungs-/Sync-Nachweis: `docs/journal-capacity.md`.
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# Tatsächlicher automatischer WSS-/HTTPS-Fallback
|
||||
|
||||
Stand 2026-11-13. **Implementiert und lokal geprüft, noch nicht auf Cloud
|
||||
ausgeführt oder abgenommen.** Ein ausdrücklich gewählter HTTPS-Modus und ein
|
||||
WSS-Reconnect ersetzen diesen Nachweis nicht.
|
||||
|
||||
Der minimale Entwicklungsadapter sitzt nach gültigem mTLS-Peer, Ablaufprüfung
|
||||
und autoritativer Gerätebindung im WSS-Connectpfad, noch vor dem Upgrade.
|
||||
Nur das konfigurierte Tenant-/Agentpaar erhält HTTP 503. HTTPS bleibt normal;
|
||||
es entstehen keine JobOffers, Grants oder Genehmigungen.
|
||||
|
||||
Aktivierung ausschließlich mit `RMM_DEV_TRANSPORT_FAULTS=true`,
|
||||
`RMM_DEV_TRANSPORT_FAULT_FILE=/root-policy/transport-fault.json`,
|
||||
`RMM_PROFILE=development` und `RMM_DEV_AUTH=true`. Standardmäßig deaktiviert;
|
||||
Produktionsaktivierung und mehrdeutige Konfiguration verweigern den Start.
|
||||
Datei/Verzeichnis im bestehenden read-only Rootpolicy-Mount müssen UID 10001,
|
||||
Modi 0600/0700 und sichere reguläre Inodes besitzen. Striktes Konfigurationsschema:
|
||||
Version 1, neue Fault-UUID, exakte Tenant-/Agent-UUIDs, `issued_at` und
|
||||
`expires_at` mit höchstens 15 Minuten. Kein öffentlicher Faultendpunkt.
|
||||
|
||||
Zwei begrenzte append-only Audit-Slots erfassen tatsächliche authentisierte
|
||||
Handshakeabweisungen: Peerzertifikatfingerprint, unterschiedliche Handshake-IDs,
|
||||
Status 503, Zeit sowie exakte Fault-/Scope-/Konfigurationsbindung. Nach zwei
|
||||
Slots bleibt die Abweisung bis Ablauf aktiv, ohne weiteren Auditbestand
|
||||
anzuhäufen. Bestehende Slots dürfen nicht mit anderer Konfiguration wieder-
|
||||
verwendet werden. Keine Job-/Outbox-/Grantänderung und keine neue Migration.
|
||||
|
||||
Tatsächlich lokal bestanden: drei gezielte Guards für explizites Opt-in,
|
||||
Produktionsverweigerung, exakten Scope, Fristen und stabile begrenzte Audit-Slots;
|
||||
Server-Clippy aller Targets mit `-D warnings`; Syntaxprüfung des Cloudrunners.
|
||||
|
||||
`deploy/cloud_transport_acceptance.py` ist **vorbereitet, nicht ausgeführt**:
|
||||
|
||||
1. Nur permanente Cloud-VM und `discovery.ease-technologies.de`; keine offenen
|
||||
Wirkungen. Bekannte Agent-/Compose-/vorherige Konfigurationshashes prüfen.
|
||||
2. Den erhaltenen Agenten im tatsächlichen Automatikmodus prüfen; ausschließlich
|
||||
einen bekannten alten Transport-Testoverride bei Bedarf versioniert entfernen.
|
||||
Standard-WSS, Heartbeat und signierten Sync bestätigen.
|
||||
3. Den begrenzten privaten Hook aktivieren. Der normale ausgehende Agent muss
|
||||
zweimal an authentisiertem WSS scheitern und selbstständig auf HTTPS wechseln.
|
||||
Zwei echte Peer-Auditbelege sowie anschließend gespeicherten frischen
|
||||
**HTTPS-Heartbeat** und `sync_ready` nachweisen. Kein TLS-Bypass.
|
||||
4. Im `finally`-Pfad Hook deaktivieren, tatsächlich laufende Serverrollen ohne
|
||||
aktiven Flag/FILE prüfen und normalen WSS-Heartbeat samt Sync wiederherstellen.
|
||||
Keine geänderte Geräteidentität, keine neuen Jobs/Outbox/Grants. Auch bei einer
|
||||
verlorenen SSH-Antwort akzeptiert Cleanup ausschließlich vorher bekannte alte
|
||||
oder vorgesehene neue Hashes, keinen ungeprüften Hostzustand.
|
||||
|
||||
Diese passive Transportabnahme benötigt keine Jobgenehmigung. Sie darf keine
|
||||
laufenden RMM-Wirkungen oder einen anderen VM-/Deploymenttest unterbrechen.
|
||||
Fehlende Nativebelege bleiben offen; erst tatsächliche Ergebnisse hier ergänzen.
|
||||
@@ -146,3 +146,14 @@ asset hashes match the current `web/dist` build.
|
||||
|
||||
Build/unit checks are local development evidence only. Real backend/agent execution,
|
||||
VM feature evidence, browser acceptance and production approval remain separate gates.
|
||||
|
||||
### Gebundene Configure-only-Wiederherstellung (Schema4)
|
||||
|
||||
Frontend prüft native_recovery mit ursprünglichem Grant/Versuch/Partial-Proof,
|
||||
Transaktionsschema2 und ausschließlich configure_only/Fixture1.0→1.0. Normale
|
||||
Schemas2/3 erlauben keine Operationserweiterung; Nichtnative-, Altversions-,
|
||||
Hash- und Bindungsmanipulationen bleiben verweigert. Elf gezielte Safetytests,
|
||||
Typecheck und Produktionsbuild bestanden. Tatsächliche separate Browser-
|
||||
Vorbereitung/0-Jobgate/Genehmigung mit exakten Bytes ist belegt; der erste Plan
|
||||
lief ohne Offer/Grant an der alten Journalgrenze ab. Paketabnahme bleibt separat:
|
||||
`../docs/implementation-status.md`. Keine API-/DB-Freigabeabkürzung.
|
||||
|
||||
@@ -10,6 +10,13 @@ async function manifest(acceptance = true) {
|
||||
const raw = new TextEncoder().encode(JSON.stringify(transaction));
|
||||
return { type: 'rmm.apt.helper-action', schema_version: 2, action: acceptance ? 'apt-native-acceptance' : 'apt-execute', execution_profile: 'dev-test', transaction_sha256: await sha256(raw), transaction_bytes: Array.from(raw), resolved_apt_transaction: transaction, provider_sha256: 'c'.repeat(64), hook_sha256: 'd'.repeat(64), apt_config_sha256: 'e'.repeat(64), runtime_sha256: 'f'.repeat(64), test_result_id: 'unit-test-only-not-native-evidence' };
|
||||
}
|
||||
async function recoveryManifest() {
|
||||
const base = await manifest();
|
||||
const transaction = { ...base.resolved_apt_transaction, schema_version: 2, operation: 'configure_only', packages: base.resolved_apt_transaction.packages.map(p => ({ ...p, old_version: '1.0' })) };
|
||||
const raw = new TextEncoder().encode(JSON.stringify(transaction));
|
||||
return { ...base, schema_version: 4, resolved_apt_transaction: transaction, transaction_bytes: Array.from(raw), transaction_sha256: await sha256(raw),
|
||||
native_recovery: { schema_version: 1, previous_grant_id: '11111111-1111-4111-8111-111111111111', previous_attempt_id: '22222222-2222-4222-8222-222222222222', partial_proof_sha256: 'a'.repeat(64) } };
|
||||
}
|
||||
function input(apt_manifest: unknown): PlanInput { return { tenant_id: 't', action: 'apt_native_acceptance', targets: { asset_ids: ['asset'] }, parameters: { execution_identity: 'root', recipe_id: 'rmm-apt-fixture-install-1.0', recipe_version: '1', apt_manifest }, schedule: { not_before: '', start_before: '', offline_policy: 'wait_until_deadline' }, rollout: { max_parallel: 1, halt_after_failures: 1 }, reason: 'unit only' }; }
|
||||
describe('immutable catalog and strict import contracts', () => {
|
||||
it('only accepts full immutable exact APT package metadata, never URLs or unknown fields', () => {
|
||||
@@ -60,6 +67,29 @@ describe('root and dedicated dev-test APT actions', () => {
|
||||
expect(() => validateAptManifest({ ...m, native_retry: { ...native_retry, bootstrap_marker_sha256: '0'.repeat(64) } }, true)).toThrow();
|
||||
expect(() => validateAptManifest({ ...m, action: 'apt-execute' }, false)).toThrow('eigener nativer');
|
||||
});
|
||||
it('accepts only an explicit schema4 configure-only partial binding and preserves all byte/plan checks', async () => {
|
||||
const m = await recoveryManifest(), before = JSON.stringify(m);
|
||||
await validateSensitiveInput(input(m), auth); expect(JSON.stringify(m)).toBe(before);
|
||||
expect(() => validateAptManifest({ ...m, action: 'apt-execute' }, false)).toThrow('eigenen nativen');
|
||||
expect(() => validateAptManifest({ ...m, native_recovery: { ...m.native_recovery, previous_grant_id: '00000000-0000-0000-0000-000000000000' } }, true)).toThrow();
|
||||
expect(() => validateAptManifest({ ...m, native_recovery: { ...m.native_recovery, partial_proof_sha256: '0'.repeat(64) } }, true)).toThrow();
|
||||
expect(() => validateAptManifest({ ...m, native_retry: {} }, true)).toThrow();
|
||||
const wrongVersion = { ...m.resolved_apt_transaction, packages: m.resolved_apt_transaction.packages.map(p => ({ ...p, old_version: null })) };
|
||||
const wrongRaw = new TextEncoder().encode(JSON.stringify(wrongVersion));
|
||||
expect(() => validateAptManifest({ ...m, resolved_apt_transaction: wrongVersion, transaction_bytes: Array.from(wrongRaw) }, true)).toThrow('Version');
|
||||
const normal = await manifest();
|
||||
expect(() => validateAptManifest({ ...normal, resolved_apt_transaction: { ...normal.resolved_apt_transaction, operation: 'configure_only' } }, true)).toThrow('Schemafelder');
|
||||
expect(() => validateAptManifest({ ...m, resolved_apt_transaction: { ...m.resolved_apt_transaction, operation: 'install' } }, true)).toThrow('Transaktionsschema');
|
||||
await expect(verifyAptManifest({ ...m, transaction_sha256: 'b'.repeat(64) }, true)).rejects.toThrow('Transaktionshash');
|
||||
const payload = { jobs: [{ payload: { action: { action: 'apt_native_acceptance', apt_manifest: m } } }] };
|
||||
const raw = new TextEncoder().encode(JSON.stringify(payload));
|
||||
const plan = { plan_payload: payload, plan_payload_bytes: Array.from(raw), plan_hash: await sha256(raw), jobs: payload.jobs };
|
||||
await verifyPlanIntegrity(plan);
|
||||
const changed = structuredClone(payload), changedJob = changed.jobs[0];
|
||||
if (!changedJob) throw new Error('One fixture job required');
|
||||
changedJob.payload.action.apt_manifest.native_recovery.partial_proof_sha256 = 'b'.repeat(64);
|
||||
await expect(verifyPlanIntegrity({ ...plan, plan_payload: changed })).rejects.toThrow('widerspricht');
|
||||
});
|
||||
it('binds update package names to the exact versions in the unchanged signed manifest', async () => {
|
||||
const m = await manifest(false); const value = input(m); value.action = 'apply_updates';
|
||||
value.parameters = { execution_identity: 'root', provider: 'apt', packages: ['rmm-mvp-test'], reboot_policy: 'never', apt_manifest: m };
|
||||
|
||||
+14
-6
@@ -27,8 +27,8 @@ export async function sha256(value: Uint8Array<ArrayBuffer>): Promise<string> {
|
||||
function canonical(v: unknown): string { if (Array.isArray(v)) return `[${v.map(canonical).join(',')}]`; if (v !== null && typeof v === 'object') return `{${Object.entries(v).sort(([a], [b]) => a.localeCompare(b)).map(([k, x]) => `${JSON.stringify(k)}:${canonical(x)}`).join(',')}}`; return JSON.stringify(v); }
|
||||
export function validateAptManifest(value: unknown, acceptance: boolean): RecordValue {
|
||||
const m = record(value, 'apt_manifest');
|
||||
exact(m, ['type', 'schema_version', 'action', 'execution_profile', 'transaction_sha256', 'resolved_apt_transaction', 'transaction_bytes', 'provider_sha256', 'hook_sha256', 'apt_config_sha256', 'runtime_sha256', 'test_result_id', ...(m.schema_version === 3 ? ['native_retry'] : [])]);
|
||||
if (m.type !== 'rmm.apt.helper-action' || ![2, 3].includes(Number(m.schema_version)) || typeof m.schema_version !== 'number' || m.execution_profile !== 'dev-test' || m.action !== (acceptance ? 'apt-native-acceptance' : 'apt-execute')) throw new Error('APT Schema 2/3 / explizites dev-test-Profil erforderlich');
|
||||
exact(m, ['type', 'schema_version', 'action', 'execution_profile', 'transaction_sha256', 'resolved_apt_transaction', 'transaction_bytes', 'provider_sha256', 'hook_sha256', 'apt_config_sha256', 'runtime_sha256', 'test_result_id', ...(m.schema_version === 3 ? ['native_retry'] : []), ...(m.schema_version === 4 ? ['native_recovery'] : [])]);
|
||||
if (m.type !== 'rmm.apt.helper-action' || ![2, 3, 4].includes(Number(m.schema_version)) || typeof m.schema_version !== 'number' || m.execution_profile !== 'dev-test' || m.action !== (acceptance ? 'apt-native-acceptance' : 'apt-execute')) throw new Error('APT Schema 2/3/4 / explizites dev-test-Profil erforderlich');
|
||||
if (m.schema_version === 3) {
|
||||
if (!acceptance) throw new Error('Wiederaufnahme nur als eigener nativer Abnahmeplan');
|
||||
const retry = record(m.native_retry); exact(retry, ['schema_version', 'previous_grant_id', 'previous_attempt_id', 'bootstrap_marker_sha256']);
|
||||
@@ -36,11 +36,19 @@ export function validateAptManifest(value: unknown, acceptance: boolean): Record
|
||||
if (retry.schema_version !== 1 || ['previous_grant_id', 'previous_attempt_id'].some(k => typeof retry[k] !== 'string' || !uuid.test(retry[k] as string) || retry[k] === '00000000-0000-0000-0000-000000000000')) throw new Error('Exakte ursprüngliche Grant-/Versuchsbindung erforderlich');
|
||||
hash(retry.bootstrap_marker_sha256);
|
||||
}
|
||||
const recovery = m.schema_version === 4;
|
||||
if (recovery) {
|
||||
if (!acceptance) throw new Error('Configure-only benötigt einen eigenen nativen Abnahmeplan');
|
||||
const ref = record(m.native_recovery); exact(ref, ['schema_version', 'previous_grant_id', 'previous_attempt_id', 'partial_proof_sha256']);
|
||||
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
if (ref.schema_version !== 1 || ['previous_grant_id', 'previous_attempt_id'].some(k => typeof ref[k] !== 'string' || !uuid.test(ref[k] as string) || ref[k] === '00000000-0000-0000-0000-000000000000')) throw new Error('Exakte geschlossene Partial-Grant-/Versuchsbindung erforderlich');
|
||||
hash(ref.partial_proof_sha256);
|
||||
}
|
||||
for (const k of ['transaction_sha256', 'provider_sha256', 'hook_sha256', 'apt_config_sha256', 'runtime_sha256']) hash(m[k]);
|
||||
if (typeof m.test_result_id !== 'string' || !/^[A-Za-z0-9_.-]{1,128}$/.test(m.test_result_id)) throw new Error('Exakte Testnachweis-ID erforderlich');
|
||||
const transaction = record(m.resolved_apt_transaction);
|
||||
exact(transaction, ['schema_version', 'recipe_id', 'recipe_version', 'platform', 'resolver_config_version', 'no_reboot', 'status_fingerprint', 'baseline_sha256', 'packages']);
|
||||
if (transaction.schema_version !== 1 || transaction.recipe_version !== '1' || transaction.platform !== 'ubuntu-24.04-amd64' || transaction.resolver_config_version !== 'rmm-apt-local-v1' || transaction.no_reboot !== true) throw new Error('APT Transaktionsschema/Plattform/No-Reboot ungültig');
|
||||
exact(transaction, ['schema_version', 'recipe_id', 'recipe_version', 'platform', 'resolver_config_version', 'no_reboot', 'status_fingerprint', 'baseline_sha256', 'packages', ...(recovery ? ['operation'] : [])]);
|
||||
if (transaction.schema_version !== (recovery ? 2 : 1) || (recovery && transaction.operation !== 'configure_only') || transaction.recipe_version !== '1' || transaction.platform !== 'ubuntu-24.04-amd64' || transaction.resolver_config_version !== 'rmm-apt-local-v1' || transaction.no_reboot !== true) throw new Error('APT Transaktionsschema/Plattform/No-Reboot ungültig');
|
||||
text(transaction.recipe_id, 'recipe_id'); hash(transaction.status_fingerprint); hash(transaction.baseline_sha256);
|
||||
if (!Array.isArray(transaction.packages) || !transaction.packages.length || transaction.packages.length > 128) throw new Error('Vollständige Paketclosure erforderlich');
|
||||
const identities = new Set<string>();
|
||||
@@ -55,9 +63,9 @@ export function validateAptManifest(value: unknown, acceptance: boolean): Record
|
||||
if (canonical(parsed) !== canonical(transaction)) throw new Error('Exakte Transaktionsbytes widersprechen Manifestobjekt');
|
||||
if (acceptance) {
|
||||
const install = transaction.recipe_id === 'rmm-apt-fixture-install-1.0', update = transaction.recipe_id === 'rmm-apt-fixture-update-1.1';
|
||||
if ((!install && !update) || transaction.packages.length !== 1) throw new Error('Nur genaue native Fixture-Rezepte erlaubt');
|
||||
if ((!install && !update) || (recovery && !install) || transaction.packages.length !== 1) throw new Error('Nur genaue native Fixture-Rezepte erlaubt');
|
||||
const p = record(transaction.packages[0]);
|
||||
if (p.name !== 'rmm-mvp-test' || p.architecture !== 'all' || p.origin !== 'rmm-test' || p.size !== 20672 || p.old_version !== (install ? null : '1.0') || p.new_version !== (install ? '1.0' : '1.1') || p.sha256 !== (install ? '74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867' : 'ef67c9fe0ba1443fcfe465768e1204763063c369a2a1da04af6027d8a9c00491')) throw new Error('Native Fixture-Bytes/Version/Origin stimmen nicht überein');
|
||||
if (p.name !== 'rmm-mvp-test' || p.architecture !== 'all' || p.origin !== 'rmm-test' || p.size !== 20672 || p.old_version !== (recovery ? '1.0' : install ? null : '1.0') || p.new_version !== (install ? '1.0' : '1.1') || p.sha256 !== (install ? '74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867' : 'ef67c9fe0ba1443fcfe465768e1204763063c369a2a1da04af6027d8a9c00491')) throw new Error('Native Fixture-Bytes/Version/Origin stimmen nicht überein');
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user