Bound helper initialization lock contention and record native Linux proof
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -66,8 +66,7 @@ pub fn private_path(path: &Path, uid: u32, directory: bool) -> Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
use fs2::FileExt;
|
||||
fn open_state_lock(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
private_path(dir, uid, true)?;
|
||||
let f = fs::OpenOptions::new()
|
||||
@@ -79,9 +78,69 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(dir.join("executor.lock"))?;
|
||||
private_path(&dir.join("executor.lock"), uid, false)?;
|
||||
f.try_lock_exclusive()?;
|
||||
Ok(f)
|
||||
}
|
||||
pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
let file = open_state_lock(dir, uid)?;
|
||||
fs2::FileExt::try_lock_exclusive(&file)?;
|
||||
Ok(file)
|
||||
}
|
||||
/// Daemon initialization only. General and Full may start concurrently against
|
||||
/// the same ledger. Wait for that existing exclusive lock, never reopen its
|
||||
/// pathname or retry a job, grant, ledger transaction or other I/O failure.
|
||||
pub fn startup_lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
use std::time::{Duration, Instant};
|
||||
let directory = fs::File::open(dir)?;
|
||||
let file = open_state_lock(dir, uid)?;
|
||||
let identity = |metadata: &fs::Metadata| {
|
||||
(
|
||||
metadata.dev(),
|
||||
metadata.ino(),
|
||||
metadata.uid(),
|
||||
metadata.gid(),
|
||||
metadata.mode(),
|
||||
metadata.nlink(),
|
||||
)
|
||||
};
|
||||
let parent = identity(&directory.metadata()?);
|
||||
let original = identity(&file.metadata()?);
|
||||
let started = Instant::now();
|
||||
let deadline = Duration::from_secs(5);
|
||||
loop {
|
||||
ensure!(
|
||||
started.elapsed() < deadline,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
private_path(dir, uid, true)?;
|
||||
private_path(&dir.join("executor.lock"), uid, false)?;
|
||||
ensure!(
|
||||
identity(&fs::symlink_metadata(dir)?) == parent
|
||||
&& identity(&directory.metadata()?) == parent
|
||||
&& identity(&fs::symlink_metadata(dir.join("executor.lock"))?) == original
|
||||
&& identity(&file.metadata()?) == original,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_REPLACED"
|
||||
);
|
||||
match fs2::FileExt::try_lock_exclusive(&file) {
|
||||
Ok(()) => {
|
||||
ensure!(
|
||||
started.elapsed() < deadline,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
return Ok(file);
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
ensure!(
|
||||
started.elapsed() < deadline,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
std::thread::sleep(
|
||||
Duration::from_millis(20).min(deadline.saturating_sub(started.elapsed())),
|
||||
);
|
||||
}
|
||||
Err(error) => return Err(error).context("HELPER_STARTUP_EXECUTOR_LOCK_IO"),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish, persist_outcome};
|
||||
|
||||
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
|
||||
@@ -2,7 +2,7 @@ mod image;
|
||||
use anyhow::{Result, ensure};
|
||||
use rmm_domain::jobs::Action;
|
||||
use rmm_privileged_helper::{
|
||||
Binding, Outcome, Request, control, finish, lock_state, open_ledger, private_path, reconcile,
|
||||
Binding, Outcome, Request, control, finish, open_ledger, private_path, reconcile,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::{
|
||||
@@ -891,7 +891,7 @@ fn run() -> Result<()> {
|
||||
"CONFIG_BINDING"
|
||||
);
|
||||
let mut db = {
|
||||
let _initialization_lock = lock_state(&config.state, 0)?;
|
||||
let _initialization_lock = rmm_privileged_helper::startup_lock_state(&config.state, 0)?;
|
||||
open_ledger(&config.state, 0)?
|
||||
};
|
||||
let update_clock = if config.profile == HelperProfile::FullMaintenanceV1
|
||||
|
||||
@@ -2,6 +2,88 @@ use super::*;
|
||||
use openssl::sign::Signer;
|
||||
use rmm_domain::jobs::{Action, encode};
|
||||
|
||||
#[test]
|
||||
fn startup_lock_waits_only_for_original_executor_and_times_out_without_effects() {
|
||||
use std::io::{BufRead, Write};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
const TEST: &str =
|
||||
"tests::startup_lock_waits_only_for_original_executor_and_times_out_without_effects";
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
if let Ok(path) = std::env::var("RMM_STARTUP_LOCK_CONTRACT_PATH") {
|
||||
let mode = std::env::var("RMM_STARTUP_LOCK_CONTRACT_MODE").unwrap();
|
||||
println!("STARTUP_LOCK_CHILD_ATTEMPT");
|
||||
std::io::stdout().flush().unwrap();
|
||||
let started = Instant::now();
|
||||
let result = startup_lock_state(Path::new(&path), uid);
|
||||
match mode.as_str() {
|
||||
"release" => {
|
||||
let held = result.unwrap();
|
||||
assert!(started.elapsed() >= Duration::from_millis(150));
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
assert_eq!(
|
||||
held.metadata().unwrap().ino(),
|
||||
fs::symlink_metadata(Path::new(&path).join("executor.lock"))
|
||||
.unwrap()
|
||||
.ino()
|
||||
);
|
||||
}
|
||||
"timeout" => {
|
||||
assert_eq!(
|
||||
result.unwrap_err().to_string(),
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
assert!(started.elapsed() >= Duration::from_secs(5));
|
||||
assert!(started.elapsed() < Duration::from_secs(7));
|
||||
}
|
||||
_ => panic!("closed contract mode"),
|
||||
}
|
||||
return;
|
||||
}
|
||||
let path = std::env::temp_dir().join(format!("rmm-startup-lock-contract-{}", Uuid::new_v4()));
|
||||
fs::create_dir(&path).unwrap();
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
for mode in ["release", "timeout"] {
|
||||
let original = lock_state(&path, uid).unwrap();
|
||||
let metadata = original.metadata().unwrap();
|
||||
let mut child = Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", TEST, "--nocapture", "--test-threads=1"])
|
||||
.env("RMM_STARTUP_LOCK_CONTRACT_PATH", &path)
|
||||
.env("RMM_STARTUP_LOCK_CONTRACT_MODE", mode)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let mut reader = std::io::BufReader::new(child.stdout.take().unwrap());
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
assert!(reader.read_line(&mut line).unwrap() > 0);
|
||||
if line.contains("STARTUP_LOCK_CHILD_ATTEMPT") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
if mode == "release" {
|
||||
drop(original);
|
||||
}
|
||||
let result = child.wait_with_output().unwrap();
|
||||
assert!(
|
||||
result.status.success(),
|
||||
"native child contract failed: {}",
|
||||
String::from_utf8_lossy(&result.stderr)
|
||||
);
|
||||
let after = fs::symlink_metadata(path.join("executor.lock")).unwrap();
|
||||
assert_eq!(
|
||||
(after.dev(), after.ino(), after.len()),
|
||||
(metadata.dev(), metadata.ino(), 0)
|
||||
);
|
||||
assert_eq!(fs::read_dir(&path).unwrap().count(), 1);
|
||||
}
|
||||
fs::remove_file(path.join("executor.lock")).unwrap();
|
||||
fs::remove_dir(path).unwrap();
|
||||
}
|
||||
|
||||
fn fixture() -> (
|
||||
Binding,
|
||||
Request,
|
||||
|
||||
Reference in New Issue
Block a user