Bound helper initialization lock contention and record native Linux proof
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -66,8 +66,7 @@ pub fn private_path(path: &Path, uid: u32, directory: bool) -> Result<()> {
|
|||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
fn open_state_lock(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||||
use fs2::FileExt;
|
|
||||||
use std::os::unix::fs::OpenOptionsExt;
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
private_path(dir, uid, true)?;
|
private_path(dir, uid, true)?;
|
||||||
let f = fs::OpenOptions::new()
|
let f = fs::OpenOptions::new()
|
||||||
@@ -79,9 +78,69 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
|||||||
.custom_flags(libc::O_NOFOLLOW)
|
.custom_flags(libc::O_NOFOLLOW)
|
||||||
.open(dir.join("executor.lock"))?;
|
.open(dir.join("executor.lock"))?;
|
||||||
private_path(&dir.join("executor.lock"), uid, false)?;
|
private_path(&dir.join("executor.lock"), uid, false)?;
|
||||||
f.try_lock_exclusive()?;
|
|
||||||
Ok(f)
|
Ok(f)
|
||||||
}
|
}
|
||||||
|
pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||||
|
let file = open_state_lock(dir, uid)?;
|
||||||
|
fs2::FileExt::try_lock_exclusive(&file)?;
|
||||||
|
Ok(file)
|
||||||
|
}
|
||||||
|
/// Daemon initialization only. General and Full may start concurrently against
|
||||||
|
/// the same ledger. Wait for that existing exclusive lock, never reopen its
|
||||||
|
/// pathname or retry a job, grant, ledger transaction or other I/O failure.
|
||||||
|
pub fn startup_lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
let directory = fs::File::open(dir)?;
|
||||||
|
let file = open_state_lock(dir, uid)?;
|
||||||
|
let identity = |metadata: &fs::Metadata| {
|
||||||
|
(
|
||||||
|
metadata.dev(),
|
||||||
|
metadata.ino(),
|
||||||
|
metadata.uid(),
|
||||||
|
metadata.gid(),
|
||||||
|
metadata.mode(),
|
||||||
|
metadata.nlink(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let parent = identity(&directory.metadata()?);
|
||||||
|
let original = identity(&file.metadata()?);
|
||||||
|
let started = Instant::now();
|
||||||
|
let deadline = Duration::from_secs(5);
|
||||||
|
loop {
|
||||||
|
ensure!(
|
||||||
|
started.elapsed() < deadline,
|
||||||
|
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||||
|
);
|
||||||
|
private_path(dir, uid, true)?;
|
||||||
|
private_path(&dir.join("executor.lock"), uid, false)?;
|
||||||
|
ensure!(
|
||||||
|
identity(&fs::symlink_metadata(dir)?) == parent
|
||||||
|
&& identity(&directory.metadata()?) == parent
|
||||||
|
&& identity(&fs::symlink_metadata(dir.join("executor.lock"))?) == original
|
||||||
|
&& identity(&file.metadata()?) == original,
|
||||||
|
"HELPER_STARTUP_EXECUTOR_LOCK_REPLACED"
|
||||||
|
);
|
||||||
|
match fs2::FileExt::try_lock_exclusive(&file) {
|
||||||
|
Ok(()) => {
|
||||||
|
ensure!(
|
||||||
|
started.elapsed() < deadline,
|
||||||
|
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||||
|
);
|
||||||
|
return Ok(file);
|
||||||
|
}
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||||
|
ensure!(
|
||||||
|
started.elapsed() < deadline,
|
||||||
|
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||||
|
);
|
||||||
|
std::thread::sleep(
|
||||||
|
Duration::from_millis(20).min(deadline.saturating_sub(started.elapsed())),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error).context("HELPER_STARTUP_EXECUTOR_LOCK_IO"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish, persist_outcome};
|
pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish, persist_outcome};
|
||||||
|
|
||||||
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ mod image;
|
|||||||
use anyhow::{Result, ensure};
|
use anyhow::{Result, ensure};
|
||||||
use rmm_domain::jobs::Action;
|
use rmm_domain::jobs::Action;
|
||||||
use rmm_privileged_helper::{
|
use rmm_privileged_helper::{
|
||||||
Binding, Outcome, Request, control, finish, lock_state, open_ledger, private_path, reconcile,
|
Binding, Outcome, Request, control, finish, open_ledger, private_path, reconcile,
|
||||||
};
|
};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::{
|
use std::{
|
||||||
@@ -891,7 +891,7 @@ fn run() -> Result<()> {
|
|||||||
"CONFIG_BINDING"
|
"CONFIG_BINDING"
|
||||||
);
|
);
|
||||||
let mut db = {
|
let mut db = {
|
||||||
let _initialization_lock = lock_state(&config.state, 0)?;
|
let _initialization_lock = rmm_privileged_helper::startup_lock_state(&config.state, 0)?;
|
||||||
open_ledger(&config.state, 0)?
|
open_ledger(&config.state, 0)?
|
||||||
};
|
};
|
||||||
let update_clock = if config.profile == HelperProfile::FullMaintenanceV1
|
let update_clock = if config.profile == HelperProfile::FullMaintenanceV1
|
||||||
|
|||||||
@@ -2,6 +2,88 @@ use super::*;
|
|||||||
use openssl::sign::Signer;
|
use openssl::sign::Signer;
|
||||||
use rmm_domain::jobs::{Action, encode};
|
use rmm_domain::jobs::{Action, encode};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn startup_lock_waits_only_for_original_executor_and_times_out_without_effects() {
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
const TEST: &str =
|
||||||
|
"tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects";
|
||||||
|
let uid = unsafe { libc::geteuid() };
|
||||||
|
if let Ok(path) = std::env::var("RMM_STARTUP_LOCK_CONTRACT_PATH") {
|
||||||
|
let mode = std::env::var("RMM_STARTUP_LOCK_CONTRACT_MODE").unwrap();
|
||||||
|
println!("STARTUP_LOCK_CHILD_ATTEMPT");
|
||||||
|
std::io::stdout().flush().unwrap();
|
||||||
|
let started = Instant::now();
|
||||||
|
let result = startup_lock_state(Path::new(&path), uid);
|
||||||
|
match mode.as_str() {
|
||||||
|
"release" => {
|
||||||
|
let held = result.unwrap();
|
||||||
|
assert!(started.elapsed() >= Duration::from_millis(150));
|
||||||
|
assert!(started.elapsed() < Duration::from_secs(5));
|
||||||
|
assert_eq!(
|
||||||
|
held.metadata().unwrap().ino(),
|
||||||
|
fs::symlink_metadata(Path::new(&path).join("executor.lock"))
|
||||||
|
.unwrap()
|
||||||
|
.ino()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
"timeout" => {
|
||||||
|
assert_eq!(
|
||||||
|
result.unwrap_err().to_string(),
|
||||||
|
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||||
|
);
|
||||||
|
assert!(started.elapsed() >= Duration::from_secs(5));
|
||||||
|
assert!(started.elapsed() < Duration::from_secs(7));
|
||||||
|
}
|
||||||
|
_ => panic!("closed contract mode"),
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let path = std::env::temp_dir().join(format!("rmm-startup-lock-contract-{}", Uuid::new_v4()));
|
||||||
|
fs::create_dir(&path).unwrap();
|
||||||
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
for mode in ["release", "timeout"] {
|
||||||
|
let original = lock_state(&path, uid).unwrap();
|
||||||
|
let metadata = original.metadata().unwrap();
|
||||||
|
let mut child = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args(["--exact", TEST, "--nocapture", "--test-threads=1"])
|
||||||
|
.env("RMM_STARTUP_LOCK_CONTRACT_PATH", &path)
|
||||||
|
.env("RMM_STARTUP_LOCK_CONTRACT_MODE", mode)
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = std::io::BufReader::new(child.stdout.take().unwrap());
|
||||||
|
loop {
|
||||||
|
let mut line = String::new();
|
||||||
|
assert!(reader.read_line(&mut line).unwrap() > 0);
|
||||||
|
if line.contains("STARTUP_LOCK_CHILD_ATTEMPT") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(200));
|
||||||
|
if mode == "release" {
|
||||||
|
drop(original);
|
||||||
|
}
|
||||||
|
let result = child.wait_with_output().unwrap();
|
||||||
|
assert!(
|
||||||
|
result.status.success(),
|
||||||
|
"native child contract failed: {}",
|
||||||
|
String::from_utf8_lossy(&result.stderr)
|
||||||
|
);
|
||||||
|
let after = fs::symlink_metadata(path.join("executor.lock")).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
(after.dev(), after.ino(), after.len()),
|
||||||
|
(metadata.dev(), metadata.ino(), 0)
|
||||||
|
);
|
||||||
|
assert_eq!(fs::read_dir(&path).unwrap().count(), 1);
|
||||||
|
}
|
||||||
|
fs::remove_file(path.join("executor.lock")).unwrap();
|
||||||
|
fs::remove_dir(path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
fn fixture() -> (
|
fn fixture() -> (
|
||||||
Binding,
|
Binding,
|
||||||
Request,
|
Request,
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Freeze only the bounded initialization lock fix; no target operations."""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import uuid
|
||||||
|
import linux_updater_qualification as q
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
BASE=ROOT/'devstate/linux-updater-capability-build/9923a97f-4c8c-4e0a-bbad-4f1d88f0d43a'
|
||||||
|
BASE_SHA='a41ad700b4b7527bf911bf00f2745914188760c631af0c3255364c05f1ef4fbc'
|
||||||
|
CHANGED=['crates/privileged-helper/src/lib.rs','crates/privileged-helper/src/main.rs','crates/privileged-helper/src/tests.rs']
|
||||||
|
TEST='tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects'
|
||||||
|
def main():
|
||||||
|
os.umask(0o077);q.require(q.sha((BASE/'manifest.json').read_bytes())==BASE_SHA,'exact original capability/tick derivative')
|
||||||
|
old=json.loads((BASE/'manifest.json').read_text());case=ROOT/'devstate/linux-updater-helper-start-build'/str(uuid.uuid4());case.mkdir(mode=0o700,parents=True)
|
||||||
|
fixed={n:(ROOT/n).read_bytes()for n in CHANGED};cache=case/'cache';cache.mkdir(mode=0o700);shutil.copytree(BASE/'cache/release',cache/'release')
|
||||||
|
artifacts={};inputs={};unit=None
|
||||||
|
for version in ('source0','source1'):
|
||||||
|
expected=json.loads((BASE/(version+'-inputs.json')).read_text());names=sorted(expected);q.require(q.capture(BASE/version,names)==expected,'original snapshot unchanged')
|
||||||
|
source=case/version;shutil.copytree(BASE/version,source)
|
||||||
|
for name,raw in fixed.items():
|
||||||
|
target=source/name;target.chmod(0o600);target.write_bytes(raw);target.chmod(0o444)
|
||||||
|
measured=q.capture(source,names);q.require({n for n in names if measured[n]!=expected[n]}==set(CHANGED),'exact three-file lock-only derivative');q.readonly(source);q.write(case/(version+'-inputs.json'),q.encode(measured));inputs[version]=q.sha((case/(version+'-inputs.json')).read_bytes())
|
||||||
|
q.run(case,source,cache,version+'-helper-clippy',['cargo','clippy','--locked','--offline','--release','--jobs','2','-p','rmm-privileged-helper','--all-targets','--','-D','warnings'])
|
||||||
|
q.run(case,source,cache,version+'-helper-release',['cargo','build','--locked','--offline','--release','--jobs','2','-p','rmm-privileged-helper','--bin','rmm-privileged-helper'])
|
||||||
|
helper=q.copy_artifact(cache/'release/rmm-privileged-helper',case/'artifacts'/version/'rmm-privileged-helper');artifacts[version]={**old['artifacts'][version],'helper':helper}
|
||||||
|
if version=='source0':
|
||||||
|
output=q.run(case,source,cache,'source0-startup-lock-unit-build',['cargo','test','--locked','--offline','--release','--jobs','2','-p','rmm-privileged-helper','--lib','--no-run','--message-format=json'])
|
||||||
|
candidates=[]
|
||||||
|
for line in output.splitlines():
|
||||||
|
try:item=json.loads(line)
|
||||||
|
except ValueError:continue
|
||||||
|
if item.get('reason')=='compiler-artifact'and item.get('executable')and item.get('target',{}).get('name')=='rmm_privileged_helper':candidates.append(item['executable'])
|
||||||
|
q.require(len(candidates)==1,'one actual helper lib test executable');unit=q.copy_artifact(cache/Path(candidates[0]).relative_to('/target'),case/'artifacts/helper-startup-lock-unit');unit['filter']=TEST
|
||||||
|
q.require(q.capture(source,names)==measured,'frozen helper source unchanged')
|
||||||
|
for item in [old['stable_updater'],old['artifacts']['source0']['agent'],old['artifacts']['source1']['agent']]:q.require(q.sha(Path(item['path']).read_bytes())==item['sha256'],'original Agents/stable updater unchanged')
|
||||||
|
q.require(artifacts['source0']['helper']['sha256']!=artifacts['source1']['helper']['sha256'],'real different-version helper bytes')
|
||||||
|
manifest={'schema_version':1,'profile':'linux_compatible_helper_start_lock_derivative_v1','base_manifest_sha256':BASE_SHA,'base_manifest':str(BASE/'manifest.json'),'changed_modules':{n:q.sha(raw)for n,raw in fixed.items()},'source_input_hashes':inputs,'source0_version':'0.1.0','source1_version':'0.1.1','builder':q.BUILDER,'artifacts':artifacts,'stable_updater':old['stable_updater'],'units':old['units'],'native_startup_lock_unit':unit,'only_helpers_rebuilt':True,'original_artifacts_unchanged':True,'clippy_all_targets_passed':True,'native_vm_tested':False,'bootstrap_installed':False,'apply_executed':False}
|
||||||
|
q.write(case/'manifest.json',q.encode(manifest));print(json.dumps({'manifest':str(case/'manifest.json'),'manifest_sha256':q.sha((case/'manifest.json').read_bytes()),'artifacts':artifacts,'native_unit':unit}))
|
||||||
|
if __name__=='__main__':main()
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"build_manifest_sha256": "3520d0cfdda856b2c9c075197059047e47a24dab42270f4676416dd1ec5871cf",
|
||||||
|
"case_id": "c337357c-4eb5-4266-a723-b58b838ac7db",
|
||||||
|
"executable_sha256": "3d22b6831d3922ec1c6b92de930b7af4dc7b44ad13d6e44759d72f504ff18859",
|
||||||
|
"exit_code": 0,
|
||||||
|
"filter": "tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects",
|
||||||
|
"job_authority": false,
|
||||||
|
"native_runtime_installed": false,
|
||||||
|
"operation": "linux_helper_startup_lock_native_contract_v1",
|
||||||
|
"original_db_paths_used": false,
|
||||||
|
"rmm_services_changed": false,
|
||||||
|
"source0_helper_sha256": "eaee85d7d9d7d9592928448186d883ccb8384cdad967b5d98f47e925b9a8ae8d",
|
||||||
|
"source1_helper_sha256": "1a65e2d9ca7f862578c12fb3d3bff9f1a381a44ce9e8bccd47271a129e64e6e7",
|
||||||
|
"stderr": "",
|
||||||
|
"stdout": "\nrunning 1 test\ntest tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects ... ok\n\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 42 filtered out; finished in 5.22s\n\n"
|
||||||
|
}
|
||||||
@@ -220,3 +220,49 @@ hatten vor/nach dem Build exakt dieselben Container-IDs, Startzeiten und
|
|||||||
Restartzähler. **Nur gebaut: kein Deploy, keine Migration, keine Mount-/Env-/
|
Restartzähler. **Nur gebaut: kein Deploy, keine Migration, keine Mount-/Env-/
|
||||||
Keyänderung.** Der öffentliche [Buildbeleg](evidence/server-updater-platform-keyrings-build.json)
|
Keyänderung.** Der öffentliche [Buildbeleg](evidence/server-updater-platform-keyrings-build.json)
|
||||||
bindet Source-Manifest `376b744896e3ca7845c4824ae41286e1fb0c486118e491993435724446a3ef31`.
|
bindet Source-Manifest `376b744896e3ca7845c4824ae41286e1fb0c486118e491993435724446a3ef31`.
|
||||||
|
|
||||||
|
## Begrenzter Initialisierungslock: tatsächlich nativ geprüfter Folgefix
|
||||||
|
|
||||||
|
Der Produktstart verwendet jetzt ausschließlich für die Ledgerinitialisierung
|
||||||
|
`startup_lock_state`: Ein einziger geöffneter Original-Lock-FD und der gehaltene
|
||||||
|
Stateverzeichnis-FD bleiben über die Wartezeit erhalten. Nur `WouldBlock` auf
|
||||||
|
demselben exklusiven Lock wird für höchstens fünf Sekunden abgewartet. Pfad- und
|
||||||
|
FD-Identitäten werden vor jeder Probe verglichen; andere I/O-Fehler verweigern
|
||||||
|
sofort. Fristüberschreitung meldet `HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT`.
|
||||||
|
Normale Wirkungslocks, Grants, Jobs und Sync werden dadurch nicht wiederholt.
|
||||||
|
Das ändert die noch unbewiesene Ursache des ursprünglichen 96-ms-Fehlers nicht.
|
||||||
|
|
||||||
|
Die separate Source0/Source1-Helperderivation liegt unter
|
||||||
|
`devstate/linux-updater-helper-start-build/d674db1c-ebca-48ed-b04c-8f7ec9e45ca1`,
|
||||||
|
Manifest-SHA256 `3520d0cfdda856b2c9c075197059047e47a24dab42270f4676416dd1ec5871cf`.
|
||||||
|
Geändert wurden ausschließlich Helperlib, Helpermain und der gezielte native
|
||||||
|
Prozessvertrag. Beide Helper bestanden all-targets Clippy mit `-D warnings`
|
||||||
|
und Releasebuild. Neue Helperhashes:
|
||||||
|
|
||||||
|
| Version | SHA256 | Bytes |
|
||||||
|
|---|---|---:|
|
||||||
|
| 0.1.0 | `eaee85d7d9d7d9592928448186d883ccb8384cdad967b5d98f47e925b9a8ae8d` | 13323816 |
|
||||||
|
| 0.1.1 | `1a65e2d9ca7f862578c12fb3d3bff9f1a381a44ce9e8bccd47271a129e64e6e7` | 13328416 |
|
||||||
|
|
||||||
|
Die beiden Agenten bleiben exakt `7fef…`/`4d21…`, der stabile Updater exakt
|
||||||
|
`dec6…`. Frühere Artefakte/Snapshots wurden nicht überschrieben. Der erste
|
||||||
|
erhaltene Buildversuch verweigerte einen nun unbenutzten Import; nach dessen
|
||||||
|
Entfernung entstand der erfolgreiche neue Fall, ohne Warningunterdrückung.
|
||||||
|
|
||||||
|
Der eine gezielte Zweiprozessvertrag lief zuerst auf der tatsächlichen
|
||||||
|
Cloud-Linux-VM: **1 passed, 0 failed, 42 filtered, 5.22 Sekunden**. Ein Child
|
||||||
|
wartete auf den vom Parent gehaltenen Original-Lock und erhielt nach Freigabe
|
||||||
|
denselben Inode; ein weiteres Child verweigerte nach fünf Sekunden den weiter
|
||||||
|
gehaltenen Lock. Im temporären Root-Fall entstand ausschließlich die leere
|
||||||
|
Lockdatei. Keine ursprüngliche DB und kein Produktdienst wurde dabei geöffnet
|
||||||
|
beziehungsweise verändert. Executable-SHA256
|
||||||
|
`3d22b6831d3922ec1c6b92de930b7af4dc7b44ad13d6e44759d72f504ff18859`.
|
||||||
|
[Originalbeleg](evidence/linux-helper-startup-lock-native.json).
|
||||||
|
|
||||||
|
Diese neue Helperderivation ist zu diesem Zeitpunkt **noch nicht im laufenden
|
||||||
|
Agent/General/Full installiert**. Der anschließend freigegebene eigene
|
||||||
|
Followupbootstrap muss den bekannten gestoppten Fullzustand und alle
|
||||||
|
Originalhistorien erneut prüfen, die drei betroffenen Dateien mit erhaltenen
|
||||||
|
Vorgängern ersetzen und vor Fullstart eine tatsächlich authentisierte
|
||||||
|
General-Policyprobe bestehen. Erst die folgende normale signierte Synchronisierung
|
||||||
|
kann die neue Baseline qualifizieren.
|
||||||
|
|||||||
Reference in New Issue
Block a user