fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -92,6 +92,92 @@ fn hex_hash(s: &str) -> bool {
|
||||
s.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}
|
||||
|
||||
/// Inspect ONLY at a new process startup, never on a transport reconnect: no
|
||||
/// executor thread from the previous process can still deliver its outcome.
|
||||
/// This does not execute, release consumption or invent a successful outcome.
|
||||
pub fn interrupted(
|
||||
db: &Connection,
|
||||
ledger: &Connection,
|
||||
binding: &Binding,
|
||||
) -> Result<Vec<(JobOffer, String)>> {
|
||||
let mut statement = db.prepare(
|
||||
"SELECT offer,grant_id FROM job_attempts WHERE state IN ('requesting','running','verifying') ORDER BY job,attempt",
|
||||
)?;
|
||||
let rows = statement.query_map([], |r| {
|
||||
Ok((r.get::<_, Vec<u8>>(0)?, r.get::<_, String>(1)?))
|
||||
})?;
|
||||
let mut interrupted = Vec::new();
|
||||
for row in rows {
|
||||
let (bytes, grant) = row?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
let job = binding.job(&offer.payload_bytes)?;
|
||||
ensure!(
|
||||
offer.job_id == job.job_id.to_string()
|
||||
&& offer.attempt_id == job.attempt_id.to_string()
|
||||
&& offer.job_payload_hash == hash(&offer.payload_bytes)
|
||||
&& Uuid::parse_str(&offer.authorization_id).is_ok_and(|id| !id.is_nil())
|
||||
&& offer.plan_hash.len() == 64
|
||||
&& hex_hash(&offer.plan_hash),
|
||||
"INTERRUPTED_OFFER_BINDING"
|
||||
);
|
||||
if !grant.is_empty() {
|
||||
let row: Option<(String, u64, String, String, Vec<u8>)> = ledger
|
||||
.query_row(
|
||||
"SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",
|
||||
[&grant],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (agent, generation, job_id, attempt, start) =
|
||||
row.ok_or_else(|| anyhow::anyhow!("INTERRUPTED_START_EVIDENCE_MISSING"))?;
|
||||
ensure!(
|
||||
agent == binding.agent_id.to_string()
|
||||
&& generation == binding.enrollment_generation
|
||||
&& job_id == offer.job_id
|
||||
&& attempt == offer.attempt_id,
|
||||
"INTERRUPTED_CONSUMPTION_BINDING"
|
||||
);
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
let permit: rmm_domain::jobs::ExecutionPermitPayload =
|
||||
serde_json::from_slice(&request.permit_bytes)?;
|
||||
// Verify the historical signed request at its original valid time;
|
||||
// this is evidence only, never permission to start it again.
|
||||
let (_, verified) = binding.verify(&request, permit.not_before)?;
|
||||
ensure!(
|
||||
verified == permit
|
||||
&& permit.grant_id.to_string() == grant
|
||||
&& request.job_bytes == offer.payload_bytes
|
||||
&& request.plan_hash == offer.plan_hash
|
||||
&& request.authorization_id.to_string() == offer.authorization_id,
|
||||
"INTERRUPTED_START_BINDING"
|
||||
);
|
||||
} else {
|
||||
// A request may have committed remotely without its reply arriving.
|
||||
// It can only become local unknown, never a fabricated grant/result.
|
||||
let state: String = db.query_row(
|
||||
"SELECT state FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(state == "requesting", "INTERRUPTED_RUNNING_GRANT_MISSING");
|
||||
}
|
||||
interrupted.push((offer, grant));
|
||||
}
|
||||
Ok(interrupted)
|
||||
}
|
||||
|
||||
pub fn interrupted_request(db: &Connection, offer: &JobOffer) -> Result<()> {
|
||||
ensure!(
|
||||
db.execute(
|
||||
"UPDATE job_attempts SET state='unknown',revision=revision+1 WHERE job=?1 AND attempt=?2 AND state='requesting' AND grant_id=''",
|
||||
params![offer.job_id, offer.attempt_id],
|
||||
)? == 1,
|
||||
"INTERRUPTED_REQUEST_STATE"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn event(db: &Connection, envelope: &Envelope) -> Result<()> {
|
||||
let count: i64 = db.query_row("SELECT count(*) FROM job_events", [], |r| r.get(0))?;
|
||||
ensure!(count < 1000, "UNACKNOWLEDGED_EVENT_LIMIT");
|
||||
|
||||
@@ -209,6 +209,29 @@ impl<C: reconcile::SyncClock> Agent<C> {
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
/// New process only. Persist honest uncertainty before any helper snapshot
|
||||
/// can block; the ordinary first-channel replay publishes it before Sync.
|
||||
fn recover_interrupted(&mut self) -> Result<()> {
|
||||
let Some(binding) = &self.binding else {
|
||||
return Ok(());
|
||||
};
|
||||
let interrupted = jobs::interrupted(&self.db, &self.ledger, binding)?;
|
||||
for (offer, grant) in interrupted {
|
||||
if grant.is_empty() {
|
||||
jobs::interrupted_request(&self.db, &offer)?;
|
||||
} else {
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&grant,
|
||||
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED"),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
// No finish/unlock here. Only normal bound reconciliation can release
|
||||
// a consumed attempt once independent definitive evidence is available.
|
||||
Ok(())
|
||||
}
|
||||
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
|
||||
// A large replay backlog must not monopolize heartbeat/cancel handling.
|
||||
// On backpressure timeout, discard this channel rather than continuing
|
||||
@@ -789,6 +812,7 @@ async fn main() -> Result<()> {
|
||||
certificates,
|
||||
rotation_policy,
|
||||
};
|
||||
agent.recover_interrupted()?;
|
||||
let mut backoff = 1_u64;
|
||||
loop {
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -18,6 +18,181 @@ use std::{
|
||||
|
||||
struct Clock(bool);
|
||||
#[test]
|
||||
fn new_process_persists_interruption_before_helper_io_without_unlock_or_repeat() {
|
||||
use openssl::sign::Signer;
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, encode, hash};
|
||||
use rmm_privileged_helper::{Binding, Request};
|
||||
use rusqlite::params;
|
||||
let dir = PathBuf::from("/tmp/opencode").join(format!("interrupted-{}", Uuid::new_v4()));
|
||||
storage::prepare_dir(&dir).unwrap();
|
||||
let key = PKey::generate_ed25519().unwrap();
|
||||
let public = key.raw_public_key().unwrap();
|
||||
let binding = Binding {
|
||||
tenant_id: Uuid::new_v4(),
|
||||
asset_id: Uuid::new_v4(),
|
||||
agent_id: Uuid::new_v4(),
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
public_key_hex: public.iter().map(|b| format!("{b:02x}")).collect(),
|
||||
};
|
||||
let now = Utc::now();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::RunScript {
|
||||
interpreter: "posix_sh".into(),
|
||||
script_utf8: "printf 'fixture only; never executed\\n'".into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: Some("fixture only; never executed\n".into()),
|
||||
},
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::minutes(5),
|
||||
max_runtime_seconds: 120,
|
||||
max_output_bytes: 4096,
|
||||
};
|
||||
let permit = ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: Uuid::new_v4(),
|
||||
authorization_id: Uuid::new_v4(),
|
||||
tenant_id: job.tenant_id,
|
||||
asset_id: job.asset_id,
|
||||
agent_id: job.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
plan_hash: hash(b"fixed test plan; no native approval"),
|
||||
job_payload_hash: hash(&encode(&job).unwrap()),
|
||||
action: "run_script".into(),
|
||||
execution_identity: "root".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(59),
|
||||
max_runtime_seconds: 120,
|
||||
key_id: hash(&public),
|
||||
};
|
||||
let mut request = Request {
|
||||
job_bytes: encode(&job).unwrap(),
|
||||
permit_bytes: encode(&permit).unwrap(),
|
||||
signature: vec![],
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
authorization_id: permit.authorization_id,
|
||||
};
|
||||
request.signature = Signer::new_without_digest(&key)
|
||||
.unwrap()
|
||||
.sign_oneshot_to_vec(&request.permit_bytes)
|
||||
.unwrap();
|
||||
let offer = JobOffer {
|
||||
job_id: job.job_id.to_string(),
|
||||
attempt_id: job.attempt_id.to_string(),
|
||||
authorization_id: permit.authorization_id.to_string(),
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
job_payload_hash: permit.job_payload_hash.clone(),
|
||||
payload_bytes: request.job_bytes.clone(),
|
||||
};
|
||||
let db = storage::open(&dir).unwrap();
|
||||
let mut ledger = rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() }).unwrap();
|
||||
rmm_privileged_helper::consume(&mut ledger, &permit).unwrap();
|
||||
rmm_privileged_helper::save_start(&ledger, &permit, &request).unwrap();
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'running',1,?4)",
|
||||
params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),permit.grant_id.to_string()]).unwrap();
|
||||
let mut queued_job = job.clone();
|
||||
queued_job.job_id = Uuid::new_v4();
|
||||
queued_job.attempt_id = Uuid::new_v4();
|
||||
let mut queued = offer.clone();
|
||||
queued.job_id = queued_job.job_id.to_string();
|
||||
queued.attempt_id = queued_job.attempt_id.to_string();
|
||||
queued.payload_bytes = encode(&queued_job).unwrap();
|
||||
queued.job_payload_hash = hash(&queued.payload_bytes);
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'requesting',0,'')",
|
||||
params![queued.job_id,queued.attempt_id,queued.encode_to_vec()]).unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
let (itx, irx) = tokio::sync::mpsc::channel(1);
|
||||
let certificates = certificate_rotation::RotationStore::open(
|
||||
&dir.join("certificates"),
|
||||
&[],
|
||||
binding.agent_id,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
let mut agent = Agent {
|
||||
address: "127.0.0.1:1".parse().unwrap(),
|
||||
ca: vec![],
|
||||
id: binding.agent_id.to_string(),
|
||||
boot: rmm_agent_linux::boot_id().unwrap(),
|
||||
instance: Uuid::new_v4().to_string(),
|
||||
db,
|
||||
started: Instant::now(),
|
||||
clock: rmm_privileged_helper::TrustedClock::new(),
|
||||
binding: Some(binding.clone()),
|
||||
ledger,
|
||||
completed_tx: tx,
|
||||
completed_rx: rx,
|
||||
inventory_tx: itx,
|
||||
inventory_rx: irx,
|
||||
inventory_inflight: false,
|
||||
sync: Some(reconcile::ClientSync::new(binding.clone())),
|
||||
sync_clock: Clock(true),
|
||||
helpers: None,
|
||||
prepared: None,
|
||||
fence: Arc::new(EffectFence::default()),
|
||||
mode: wire_channel::Mode::Wss,
|
||||
certificates,
|
||||
rotation_policy: certificate_rotation::RotationPolicy::default(),
|
||||
};
|
||||
// A forged local consumption/start cannot become a purported grant result.
|
||||
agent
|
||||
.ledger
|
||||
.execute("UPDATE consumed SET agent=?1", [Uuid::new_v4().to_string()])
|
||||
.unwrap();
|
||||
assert!(agent.recover_interrupted().is_err());
|
||||
assert!(jobs::events(&agent.db).unwrap().is_empty());
|
||||
agent
|
||||
.ledger
|
||||
.execute(
|
||||
"UPDATE consumed SET agent=?1",
|
||||
[binding.agent_id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
agent.recover_interrupted().unwrap();
|
||||
let original = jobs::events(&agent.db).unwrap();
|
||||
assert_eq!(original.len(), 1);
|
||||
let Some(Payload::JobResult(result)) = &original[0].payload else {
|
||||
panic!("real durable result required")
|
||||
};
|
||||
assert_eq!(result.state, "unknown");
|
||||
assert_eq!(result.error_code, "AGENT_PROCESS_INTERRUPTED");
|
||||
assert_eq!(result.grant_id, permit.grant_id.to_string());
|
||||
assert_eq!(result.revision, 2);
|
||||
assert!(result.stdout.is_empty() && result.exit_code.is_none());
|
||||
assert!(!agent.fence.allows(1, true));
|
||||
let reopened = rusqlite::Connection::open(dir.join("journal.sqlite")).unwrap();
|
||||
assert_eq!(jobs::events(&reopened).unwrap(), original);
|
||||
let requesting: (String, u64) = reopened
|
||||
.query_row(
|
||||
"SELECT state,revision FROM job_attempts WHERE job=?1",
|
||||
[&queued.job_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(requesting, ("unknown".into(), 1));
|
||||
let retained: (i64, i64, i64) = agent.ledger.query_row("SELECT (SELECT count(*) FROM consumed),(SELECT count(*) FROM consumed WHERE result IS NULL),(SELECT count(*) FROM execution_evidence WHERE outcome IS NULL)",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(retained, (1, 1, 1));
|
||||
// The same process/restart cannot invent another revision or message ID.
|
||||
agent.recover_interrupted().unwrap();
|
||||
assert_eq!(jobs::events(&agent.db).unwrap(), original);
|
||||
drop(reopened);
|
||||
drop(agent);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn last_effect_fence_rejects_closed_old_generation_root_and_expired_admission() {
|
||||
let gate = EffectFence::default();
|
||||
assert!(!gate.allows(1, false));
|
||||
|
||||
@@ -65,8 +65,9 @@ pub fn open(dir: &Path) -> Result<rusqlite::Connection> {
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
rmm_privileged_helper::private_path(&path, uid, false)?;
|
||||
let db = rusqlite::Connection::open(path)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; PRAGMA max_page_count=1024; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("PRAGMA max_page_count=16384;
|
||||
rmm_privileged_helper::bounded_journal_capacity(&db)?;
|
||||
db.execute_batch("PRAGMA synchronous=FULL; PRAGMA journal_mode=DELETE; CREATE TABLE IF NOT EXISTS counter(id INTEGER PRIMARY KEY CHECK(id=1),seq INTEGER NOT NULL); INSERT OR IGNORE INTO counter VALUES(1,0); CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY CHECK(id=1),payload BLOB NOT NULL CHECK(length(payload)<=1048576));")?;
|
||||
db.execute_batch("
|
||||
CREATE TABLE IF NOT EXISTS job_attempts(job TEXT NOT NULL, attempt TEXT NOT NULL, offer BLOB NOT NULL,
|
||||
state TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0, grant_id TEXT NOT NULL DEFAULT '',
|
||||
cancel_requested INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(job,attempt));
|
||||
@@ -106,6 +107,29 @@ pub fn store_pending(db: &rusqlite::Connection, e: &Envelope) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod passive_pressure_tests {
|
||||
#[test]
|
||||
fn agent_journal_capacity_is_finite_and_preserves_sequence_on_reopen() {
|
||||
let dir = std::env::temp_dir().join(format!("rmm-capacity-{}", uuid::Uuid::new_v4()));
|
||||
super::prepare_dir(&dir).unwrap();
|
||||
let mut db = super::open(&dir).unwrap();
|
||||
assert_eq!(super::next_sequence(&mut db).unwrap(), 1);
|
||||
db.execute_batch("PRAGMA max_page_count=16384").unwrap();
|
||||
drop(db);
|
||||
let mut db = super::open(&dir).unwrap();
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
4096
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("PRAGMA max_page_count", [], |r| r.get::<_, u32>(0))
|
||||
.unwrap(),
|
||||
65536
|
||||
);
|
||||
assert_eq!(super::next_sequence(&mut db).unwrap(), 2);
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn pending_durable_traffic_backpressures_collection_without_deletion() {
|
||||
let db = rusqlite::Connection::open_in_memory().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user