From 6195e2b4b943463a974e57ea07399aeb0980243d Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Mon, 16 Nov 2026 15:44:48 +0100 Subject: [PATCH] Bound helper initialization lock contention and record native Linux proof --- crates/privileged-helper/src/lib.rs | 65 ++++++++++++++- crates/privileged-helper/src/main.rs | 4 +- crates/privileged-helper/src/tests.rs | 82 +++++++++++++++++++ deploy/linux_updater_helper_start_build.py | 41 ++++++++++ .../linux-helper-startup-lock-native.json | 16 ++++ docs/linux-updater-compatible-controllers.md | 46 +++++++++++ 6 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 deploy/linux_updater_helper_start_build.py create mode 100644 docs/evidence/linux-helper-startup-lock-native.json diff --git a/crates/privileged-helper/src/lib.rs b/crates/privileged-helper/src/lib.rs index af42b7a..807023e 100644 --- a/crates/privileged-helper/src/lib.rs +++ b/crates/privileged-helper/src/lib.rs @@ -66,8 +66,7 @@ pub fn private_path(path: &Path, uid: u32, directory: bool) -> Result<()> { ); Ok(()) } -pub fn lock_state(dir: &Path, uid: u32) -> Result { - use fs2::FileExt; +fn open_state_lock(dir: &Path, uid: u32) -> Result { use std::os::unix::fs::OpenOptionsExt; private_path(dir, uid, true)?; let f = fs::OpenOptions::new() @@ -79,9 +78,69 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result { .custom_flags(libc::O_NOFOLLOW) .open(dir.join("executor.lock"))?; private_path(&dir.join("executor.lock"), uid, false)?; - f.try_lock_exclusive()?; Ok(f) } +pub fn lock_state(dir: &Path, uid: u32) -> Result { + let file = open_state_lock(dir, uid)?; + fs2::FileExt::try_lock_exclusive(&file)?; + Ok(file) +} +/// Daemon initialization only. General and Full may start concurrently against +/// the same ledger. Wait for that existing exclusive lock, never reopen its +/// pathname or retry a job, grant, ledger transaction or other I/O failure. +pub fn startup_lock_state(dir: &Path, uid: u32) -> Result { + use std::time::{Duration, Instant}; + let directory = fs::File::open(dir)?; + let file = open_state_lock(dir, uid)?; + let identity = |metadata: &fs::Metadata| { + ( + metadata.dev(), + metadata.ino(), + metadata.uid(), + metadata.gid(), + metadata.mode(), + metadata.nlink(), + ) + }; + let parent = identity(&directory.metadata()?); + let original = identity(&file.metadata()?); + let started = Instant::now(); + let deadline = Duration::from_secs(5); + loop { + ensure!( + started.elapsed() < deadline, + "HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT" + ); + private_path(dir, uid, true)?; + private_path(&dir.join("executor.lock"), uid, false)?; + ensure!( + identity(&fs::symlink_metadata(dir)?) == parent + && identity(&directory.metadata()?) == parent + && identity(&fs::symlink_metadata(dir.join("executor.lock"))?) == original + && identity(&file.metadata()?) == original, + "HELPER_STARTUP_EXECUTOR_LOCK_REPLACED" + ); + match fs2::FileExt::try_lock_exclusive(&file) { + Ok(()) => { + ensure!( + started.elapsed() < deadline, + "HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT" + ); + return Ok(file); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + ensure!( + started.elapsed() < deadline, + "HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT" + ); + std::thread::sleep( + Duration::from_millis(20).min(deadline.saturating_sub(started.elapsed())), + ); + } + Err(error) => return Err(error).context("HELPER_STARTUP_EXECUTOR_LOCK_IO"), + } + } +} pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish, persist_outcome}; pub fn open_ledger(dir: &Path, uid: u32) -> Result { diff --git a/crates/privileged-helper/src/main.rs b/crates/privileged-helper/src/main.rs index 0d36917..a108a97 100644 --- a/crates/privileged-helper/src/main.rs +++ b/crates/privileged-helper/src/main.rs @@ -2,7 +2,7 @@ mod image; use anyhow::{Result, ensure}; use rmm_domain::jobs::Action; use rmm_privileged_helper::{ - Binding, Outcome, Request, control, finish, lock_state, open_ledger, private_path, reconcile, + Binding, Outcome, Request, control, finish, open_ledger, private_path, reconcile, }; use serde::Deserialize; use std::{ @@ -891,7 +891,7 @@ fn run() -> Result<()> { "CONFIG_BINDING" ); let mut db = { - let _initialization_lock = lock_state(&config.state, 0)?; + let _initialization_lock = rmm_privileged_helper::startup_lock_state(&config.state, 0)?; open_ledger(&config.state, 0)? }; let update_clock = if config.profile == HelperProfile::FullMaintenanceV1 diff --git a/crates/privileged-helper/src/tests.rs b/crates/privileged-helper/src/tests.rs index 634238a..c61511d 100644 --- a/crates/privileged-helper/src/tests.rs +++ b/crates/privileged-helper/src/tests.rs @@ -2,6 +2,88 @@ use super::*; use openssl::sign::Signer; use rmm_domain::jobs::{Action, encode}; +#[test] +fn startup_lock_waits_only_for_original_executor_and_times_out_without_effects() { + use std::io::{BufRead, Write}; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + const TEST: &str = + "tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects"; + let uid = unsafe { libc::geteuid() }; + if let Ok(path) = std::env::var("RMM_STARTUP_LOCK_CONTRACT_PATH") { + let mode = std::env::var("RMM_STARTUP_LOCK_CONTRACT_MODE").unwrap(); + println!("STARTUP_LOCK_CHILD_ATTEMPT"); + std::io::stdout().flush().unwrap(); + let started = Instant::now(); + let result = startup_lock_state(Path::new(&path), uid); + match mode.as_str() { + "release" => { + let held = result.unwrap(); + assert!(started.elapsed() >= Duration::from_millis(150)); + assert!(started.elapsed() < Duration::from_secs(5)); + assert_eq!( + held.metadata().unwrap().ino(), + fs::symlink_metadata(Path::new(&path).join("executor.lock")) + .unwrap() + .ino() + ); + } + "timeout" => { + assert_eq!( + result.unwrap_err().to_string(), + "HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT" + ); + assert!(started.elapsed() >= Duration::from_secs(5)); + assert!(started.elapsed() < Duration::from_secs(7)); + } + _ => panic!("closed contract mode"), + } + return; + } + let path = std::env::temp_dir().join(format!("rmm-startup-lock-contract-{}", Uuid::new_v4())); + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + for mode in ["release", "timeout"] { + let original = lock_state(&path, uid).unwrap(); + let metadata = original.metadata().unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST, "--nocapture", "--test-threads=1"]) + .env("RMM_STARTUP_LOCK_CONTRACT_PATH", &path) + .env("RMM_STARTUP_LOCK_CONTRACT_MODE", mode) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut reader = std::io::BufReader::new(child.stdout.take().unwrap()); + loop { + let mut line = String::new(); + assert!(reader.read_line(&mut line).unwrap() > 0); + if line.contains("STARTUP_LOCK_CHILD_ATTEMPT") { + break; + } + } + std::thread::sleep(Duration::from_millis(200)); + if mode == "release" { + drop(original); + } + let result = child.wait_with_output().unwrap(); + assert!( + result.status.success(), + "native child contract failed: {}", + String::from_utf8_lossy(&result.stderr) + ); + let after = fs::symlink_metadata(path.join("executor.lock")).unwrap(); + assert_eq!( + (after.dev(), after.ino(), after.len()), + (metadata.dev(), metadata.ino(), 0) + ); + assert_eq!(fs::read_dir(&path).unwrap().count(), 1); + } + fs::remove_file(path.join("executor.lock")).unwrap(); + fs::remove_dir(path).unwrap(); +} + fn fixture() -> ( Binding, Request, diff --git a/deploy/linux_updater_helper_start_build.py b/deploy/linux_updater_helper_start_build.py new file mode 100644 index 0000000..d624dbf --- /dev/null +++ b/deploy/linux_updater_helper_start_build.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Freeze only the bounded initialization lock fix; no target operations.""" +import json +import os +from pathlib import Path +import shutil +import uuid +import linux_updater_qualification as q +ROOT=Path(__file__).resolve().parents[1] +BASE=ROOT/'devstate/linux-updater-capability-build/9923a97f-4c8c-4e0a-bbad-4f1d88f0d43a' +BASE_SHA='a41ad700b4b7527bf911bf00f2745914188760c631af0c3255364c05f1ef4fbc' +CHANGED=['crates/privileged-helper/src/lib.rs','crates/privileged-helper/src/main.rs','crates/privileged-helper/src/tests.rs'] +TEST='tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects' +def main(): + os.umask(0o077);q.require(q.sha((BASE/'manifest.json').read_bytes())==BASE_SHA,'exact original capability/tick derivative') + old=json.loads((BASE/'manifest.json').read_text());case=ROOT/'devstate/linux-updater-helper-start-build'/str(uuid.uuid4());case.mkdir(mode=0o700,parents=True) + fixed={n:(ROOT/n).read_bytes()for n in CHANGED};cache=case/'cache';cache.mkdir(mode=0o700);shutil.copytree(BASE/'cache/release',cache/'release') + artifacts={};inputs={};unit=None + for version in ('source0','source1'): + expected=json.loads((BASE/(version+'-inputs.json')).read_text());names=sorted(expected);q.require(q.capture(BASE/version,names)==expected,'original snapshot unchanged') + source=case/version;shutil.copytree(BASE/version,source) + for name,raw in fixed.items(): + target=source/name;target.chmod(0o600);target.write_bytes(raw);target.chmod(0o444) + measured=q.capture(source,names);q.require({n for n in names if measured[n]!=expected[n]}==set(CHANGED),'exact three-file lock-only derivative');q.readonly(source);q.write(case/(version+'-inputs.json'),q.encode(measured));inputs[version]=q.sha((case/(version+'-inputs.json')).read_bytes()) + q.run(case,source,cache,version+'-helper-clippy',['cargo','clippy','--locked','--offline','--release','--jobs','2','-p','rmm-privileged-helper','--all-targets','--','-D','warnings']) + q.run(case,source,cache,version+'-helper-release',['cargo','build','--locked','--offline','--release','--jobs','2','-p','rmm-privileged-helper','--bin','rmm-privileged-helper']) + helper=q.copy_artifact(cache/'release/rmm-privileged-helper',case/'artifacts'/version/'rmm-privileged-helper');artifacts[version]={**old['artifacts'][version],'helper':helper} + if version=='source0': + output=q.run(case,source,cache,'source0-startup-lock-unit-build',['cargo','test','--locked','--offline','--release','--jobs','2','-p','rmm-privileged-helper','--lib','--no-run','--message-format=json']) + candidates=[] + for line in output.splitlines(): + try:item=json.loads(line) + except ValueError:continue + if item.get('reason')=='compiler-artifact'and item.get('executable')and item.get('target',{}).get('name')=='rmm_privileged_helper':candidates.append(item['executable']) + q.require(len(candidates)==1,'one actual helper lib test executable');unit=q.copy_artifact(cache/Path(candidates[0]).relative_to('/target'),case/'artifacts/helper-startup-lock-unit');unit['filter']=TEST + q.require(q.capture(source,names)==measured,'frozen helper source unchanged') + for item in [old['stable_updater'],old['artifacts']['source0']['agent'],old['artifacts']['source1']['agent']]:q.require(q.sha(Path(item['path']).read_bytes())==item['sha256'],'original Agents/stable updater unchanged') + q.require(artifacts['source0']['helper']['sha256']!=artifacts['source1']['helper']['sha256'],'real different-version helper bytes') + manifest={'schema_version':1,'profile':'linux_compatible_helper_start_lock_derivative_v1','base_manifest_sha256':BASE_SHA,'base_manifest':str(BASE/'manifest.json'),'changed_modules':{n:q.sha(raw)for n,raw in fixed.items()},'source_input_hashes':inputs,'source0_version':'0.1.0','source1_version':'0.1.1','builder':q.BUILDER,'artifacts':artifacts,'stable_updater':old['stable_updater'],'units':old['units'],'native_startup_lock_unit':unit,'only_helpers_rebuilt':True,'original_artifacts_unchanged':True,'clippy_all_targets_passed':True,'native_vm_tested':False,'bootstrap_installed':False,'apply_executed':False} + q.write(case/'manifest.json',q.encode(manifest));print(json.dumps({'manifest':str(case/'manifest.json'),'manifest_sha256':q.sha((case/'manifest.json').read_bytes()),'artifacts':artifacts,'native_unit':unit})) +if __name__=='__main__':main() diff --git a/docs/evidence/linux-helper-startup-lock-native.json b/docs/evidence/linux-helper-startup-lock-native.json new file mode 100644 index 0000000..48153af --- /dev/null +++ b/docs/evidence/linux-helper-startup-lock-native.json @@ -0,0 +1,16 @@ +{ + "build_manifest_sha256": "3520d0cfdda856b2c9c075197059047e47a24dab42270f4676416dd1ec5871cf", + "case_id": "c337357c-4eb5-4266-a723-b58b838ac7db", + "executable_sha256": "3d22b6831d3922ec1c6b92de930b7af4dc7b44ad13d6e44759d72f504ff18859", + "exit_code": 0, + "filter": "tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects", + "job_authority": false, + "native_runtime_installed": false, + "operation": "linux_helper_startup_lock_native_contract_v1", + "original_db_paths_used": false, + "rmm_services_changed": false, + "source0_helper_sha256": "eaee85d7d9d7d9592928448186d883ccb8384cdad967b5d98f47e925b9a8ae8d", + "source1_helper_sha256": "1a65e2d9ca7f862578c12fb3d3bff9f1a381a44ce9e8bccd47271a129e64e6e7", + "stderr": "", + "stdout": "\nrunning 1 test\ntest tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 42 filtered out; finished in 5.22s\n\n" +} diff --git a/docs/linux-updater-compatible-controllers.md b/docs/linux-updater-compatible-controllers.md index 52f5aed..c25b557 100644 --- a/docs/linux-updater-compatible-controllers.md +++ b/docs/linux-updater-compatible-controllers.md @@ -220,3 +220,49 @@ hatten vor/nach dem Build exakt dieselben Container-IDs, Startzeiten und Restartzähler. **Nur gebaut: kein Deploy, keine Migration, keine Mount-/Env-/ Keyänderung.** Der öffentliche [Buildbeleg](evidence/server-updater-platform-keyrings-build.json) bindet Source-Manifest `376b744896e3ca7845c4824ae41286e1fb0c486118e491993435724446a3ef31`. + +## Begrenzter Initialisierungslock: tatsächlich nativ geprüfter Folgefix + +Der Produktstart verwendet jetzt ausschließlich für die Ledgerinitialisierung +`startup_lock_state`: Ein einziger geöffneter Original-Lock-FD und der gehaltene +Stateverzeichnis-FD bleiben über die Wartezeit erhalten. Nur `WouldBlock` auf +demselben exklusiven Lock wird für höchstens fünf Sekunden abgewartet. Pfad- und +FD-Identitäten werden vor jeder Probe verglichen; andere I/O-Fehler verweigern +sofort. Fristüberschreitung meldet `HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT`. +Normale Wirkungslocks, Grants, Jobs und Sync werden dadurch nicht wiederholt. +Das ändert die noch unbewiesene Ursache des ursprünglichen 96-ms-Fehlers nicht. + +Die separate Source0/Source1-Helperderivation liegt unter +`devstate/linux-updater-helper-start-build/d674db1c-ebca-48ed-b04c-8f7ec9e45ca1`, +Manifest-SHA256 `3520d0cfdda856b2c9c075197059047e47a24dab42270f4676416dd1ec5871cf`. +Geändert wurden ausschließlich Helperlib, Helpermain und der gezielte native +Prozessvertrag. Beide Helper bestanden all-targets Clippy mit `-D warnings` +und Releasebuild. Neue Helperhashes: + +| Version | SHA256 | Bytes | +|---|---|---:| +| 0.1.0 | `eaee85d7d9d7d9592928448186d883ccb8384cdad967b5d98f47e925b9a8ae8d` | 13323816 | +| 0.1.1 | `1a65e2d9ca7f862578c12fb3d3bff9f1a381a44ce9e8bccd47271a129e64e6e7` | 13328416 | + +Die beiden Agenten bleiben exakt `7fef…`/`4d21…`, der stabile Updater exakt +`dec6…`. Frühere Artefakte/Snapshots wurden nicht überschrieben. Der erste +erhaltene Buildversuch verweigerte einen nun unbenutzten Import; nach dessen +Entfernung entstand der erfolgreiche neue Fall, ohne Warningunterdrückung. + +Der eine gezielte Zweiprozessvertrag lief zuerst auf der tatsächlichen +Cloud-Linux-VM: **1 passed, 0 failed, 42 filtered, 5.22 Sekunden**. Ein Child +wartete auf den vom Parent gehaltenen Original-Lock und erhielt nach Freigabe +denselben Inode; ein weiteres Child verweigerte nach fünf Sekunden den weiter +gehaltenen Lock. Im temporären Root-Fall entstand ausschließlich die leere +Lockdatei. Keine ursprüngliche DB und kein Produktdienst wurde dabei geöffnet +beziehungsweise verändert. Executable-SHA256 +`3d22b6831d3922ec1c6b92de930b7af4dc7b44ad13d6e44759d72f504ff18859`. +[Originalbeleg](evidence/linux-helper-startup-lock-native.json). + +Diese neue Helperderivation ist zu diesem Zeitpunkt **noch nicht im laufenden +Agent/General/Full installiert**. Der anschließend freigegebene eigene +Followupbootstrap muss den bekannten gestoppten Fullzustand und alle +Originalhistorien erneut prüfen, die drei betroffenen Dateien mit erhaltenen +Vorgängern ersetzen und vor Fullstart eine tatsächlich authentisierte +General-Policyprobe bestehen. Erst die folgende normale signierte Synchronisierung +kann die neue Baseline qualifizieren.