Bound helper initialization lock contention and record native Linux proof
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -66,8 +66,7 @@ pub fn private_path(path: &Path, uid: u32, directory: bool) -> Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
use fs2::FileExt;
|
||||
fn open_state_lock(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
private_path(dir, uid, true)?;
|
||||
let f = fs::OpenOptions::new()
|
||||
@@ -79,9 +78,69 @@ pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(dir.join("executor.lock"))?;
|
||||
private_path(&dir.join("executor.lock"), uid, false)?;
|
||||
f.try_lock_exclusive()?;
|
||||
Ok(f)
|
||||
}
|
||||
pub fn lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
let file = open_state_lock(dir, uid)?;
|
||||
fs2::FileExt::try_lock_exclusive(&file)?;
|
||||
Ok(file)
|
||||
}
|
||||
/// Daemon initialization only. General and Full may start concurrently against
|
||||
/// the same ledger. Wait for that existing exclusive lock, never reopen its
|
||||
/// pathname or retry a job, grant, ledger transaction or other I/O failure.
|
||||
pub fn startup_lock_state(dir: &Path, uid: u32) -> Result<fs::File> {
|
||||
use std::time::{Duration, Instant};
|
||||
let directory = fs::File::open(dir)?;
|
||||
let file = open_state_lock(dir, uid)?;
|
||||
let identity = |metadata: &fs::Metadata| {
|
||||
(
|
||||
metadata.dev(),
|
||||
metadata.ino(),
|
||||
metadata.uid(),
|
||||
metadata.gid(),
|
||||
metadata.mode(),
|
||||
metadata.nlink(),
|
||||
)
|
||||
};
|
||||
let parent = identity(&directory.metadata()?);
|
||||
let original = identity(&file.metadata()?);
|
||||
let started = Instant::now();
|
||||
let deadline = Duration::from_secs(5);
|
||||
loop {
|
||||
ensure!(
|
||||
started.elapsed() < deadline,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
private_path(dir, uid, true)?;
|
||||
private_path(&dir.join("executor.lock"), uid, false)?;
|
||||
ensure!(
|
||||
identity(&fs::symlink_metadata(dir)?) == parent
|
||||
&& identity(&directory.metadata()?) == parent
|
||||
&& identity(&fs::symlink_metadata(dir.join("executor.lock"))?) == original
|
||||
&& identity(&file.metadata()?) == original,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_REPLACED"
|
||||
);
|
||||
match fs2::FileExt::try_lock_exclusive(&file) {
|
||||
Ok(()) => {
|
||||
ensure!(
|
||||
started.elapsed() < deadline,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
return Ok(file);
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
ensure!(
|
||||
started.elapsed() < deadline,
|
||||
"HELPER_STARTUP_EXECUTOR_LOCK_TIMEOUT"
|
||||
);
|
||||
std::thread::sleep(
|
||||
Duration::from_millis(20).min(deadline.saturating_sub(started.elapsed())),
|
||||
);
|
||||
}
|
||||
Err(error) => return Err(error).context("HELPER_STARTUP_EXECUTOR_LOCK_IO"),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub use rmm_agent_core::ledger::{bounded_journal_capacity, consume, finish, persist_outcome};
|
||||
|
||||
pub fn open_ledger(dir: &Path, uid: u32) -> Result<rusqlite::Connection> {
|
||||
|
||||
Reference in New Issue
Block a user