Qualify Windows system delegation and crash-safe mirror adapter on native VM
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -135,14 +135,20 @@ impl SyncClock for JobClock {
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
generation: AtomicU64,
|
||||
helper_generation: AtomicU64,
|
||||
deadline: AtomicI64,
|
||||
}
|
||||
impl EffectFence {
|
||||
fn close(&self) {
|
||||
self.generation.store(0, Ordering::SeqCst);
|
||||
self.helper_generation.store(0, Ordering::SeqCst);
|
||||
}
|
||||
fn allows(&self, generation: u64, clock: &impl SyncClock) -> bool {
|
||||
self.generation.load(Ordering::SeqCst) == generation
|
||||
self.allows_identity(generation, false, clock)
|
||||
}
|
||||
fn allows_identity(&self, generation: u64, helper: bool, clock: &impl SyncClock) -> bool {
|
||||
(!helper || self.helper_generation.load(Ordering::SeqCst) == generation)
|
||||
&& self.generation.load(Ordering::SeqCst) == generation
|
||||
&& generation > 0
|
||||
&& clock.now().is_ok_and(|now| {
|
||||
(now + chrono::Duration::seconds(2)).timestamp_millis()
|
||||
@@ -151,6 +157,48 @@ impl EffectFence {
|
||||
}
|
||||
}
|
||||
type Completion = (JobOffer, ExecutionPermitPayload, Outcome);
|
||||
#[cfg(feature = "system-helper")]
|
||||
fn needs_bound_outcome(
|
||||
db: &Connection,
|
||||
bound: &rmm_agent_windows::helper_runtime::BoundOutcome,
|
||||
) -> Result<bool> {
|
||||
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
|
||||
// Archiving is durable retention only: no ACK, event deletion or unlock.
|
||||
reconcile::archive_events(db)?;
|
||||
let (state, revision, grant): (String, u64, String) = db.query_row(
|
||||
"SELECT state,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![bound.job_id.to_string(), bound.attempt_id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)?;
|
||||
let bytes:Option<Vec<u8>>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",rusqlite::params![bound.job_id.to_string(),bound.attempt_id.to_string()],|r|r.get(0)).optional()?;
|
||||
if state == bound.outcome.state
|
||||
&& grant == bound.grant_id.to_string()
|
||||
&& let Some(bytes) = bytes
|
||||
{
|
||||
let envelope = Envelope::decode(bytes.as_slice())?;
|
||||
if let Some(Payload::JobResult(r)) = envelope.payload
|
||||
&& r.revision == revision
|
||||
&& r.grant_id == grant
|
||||
{
|
||||
let saved = ReconciledOutcome {
|
||||
state: r.state,
|
||||
stdout: r.stdout,
|
||||
stderr: r.stderr,
|
||||
exit_code: r.exit_code,
|
||||
output_truncated: r.output_truncated,
|
||||
reboot_required: r.reboot_required,
|
||||
evidence: serde_json::from_str(&r.evidence_json)?,
|
||||
error_code: r.error_code,
|
||||
};
|
||||
let actual: ReconciledOutcome =
|
||||
serde_json::from_value(serde_json::to_value(&bound.outcome)?)?;
|
||||
if canonical_outcome_bytes(&saved)? == canonical_outcome_bytes(&actual)? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
struct Running {
|
||||
offer: JobOffer,
|
||||
grant: ExecutionPermitPayload,
|
||||
@@ -338,26 +386,40 @@ fn open_ledger(directory: &Path, binding: &Binding) -> Result<(Connection, fs::F
|
||||
Ok((db, file))
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
type MirrorCompletion = (
|
||||
Connection,
|
||||
Result<(
|
||||
rmm_agent_windows::helper_runtime::ControlSnapshot,
|
||||
Vec<rmm_agent_windows::helper_runtime::BoundOutcome>,
|
||||
)>,
|
||||
);
|
||||
#[cfg(feature = "system-helper")]
|
||||
enum HelperControl {
|
||||
Snapshot(
|
||||
u64,
|
||||
tokio::task::JoinHandle<Result<reconcile::HelperSnapshot>>,
|
||||
tokio::task::JoinHandle<Result<rmm_agent_windows::helper_runtime::ControlSnapshot>>,
|
||||
),
|
||||
Apply(
|
||||
u64,
|
||||
tokio::task::JoinHandle<Result<reconcile::HelperSnapshot>>,
|
||||
tokio::task::JoinHandle<Result<rmm_agent_windows::helper_runtime::ControlSnapshot>>,
|
||||
),
|
||||
Mirror(u64, tokio::task::JoinHandle<MirrorCompletion>),
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
impl HelperControl {
|
||||
fn finished(&self) -> bool {
|
||||
match self {
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => t.is_finished(),
|
||||
Self::Mirror(_, t) => t.is_finished(),
|
||||
}
|
||||
}
|
||||
fn abort(self) {
|
||||
fn abort(self) -> Option<Self> {
|
||||
match self {
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => t.abort(),
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => {
|
||||
t.abort();
|
||||
None
|
||||
}
|
||||
mirror @ Self::Mirror(..) => Some(mirror),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -389,6 +451,8 @@ struct Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_control: Option<HelperControl>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_anchor: Option<rmm_agent_core::helper_reconcile::NoStartAnchor>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_checked: bool,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_refused: bool,
|
||||
@@ -408,8 +472,9 @@ impl Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.prepared_request = None;
|
||||
self.helper_anchor = None;
|
||||
if let Some(control) = self.helper_control.take() {
|
||||
control.abort();
|
||||
self.helper_control = control.abort();
|
||||
}
|
||||
}
|
||||
self.deferred_permit = None;
|
||||
@@ -446,15 +511,69 @@ impl Agent {
|
||||
HelperControl::Snapshot(expected, task) => {
|
||||
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
||||
let snapshot = task.await??;
|
||||
self.send_sync(channel, generation, Some(&snapshot)).await?;
|
||||
let helper = self
|
||||
.helper
|
||||
.clone()
|
||||
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
||||
let observed_at = self.clock.now()?;
|
||||
let mut ledger = self.ledger.take().context("LEDGER_REQUIRED")?;
|
||||
self.helper_control = Some(HelperControl::Mirror(
|
||||
generation,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = helper
|
||||
.mirror(&snapshot.raw, &mut ledger, observed_at)
|
||||
.map(|unknowns| (snapshot, unknowns));
|
||||
(ledger, result)
|
||||
}),
|
||||
));
|
||||
}
|
||||
HelperControl::Mirror(expected, task) => {
|
||||
// Always restore ownership, including after reconnect. A
|
||||
// stale generation may discard its snapshot, never its DB.
|
||||
let (ledger, result) = task.await?;
|
||||
self.ledger = Some(ledger);
|
||||
let (snapshot, unknowns) = result?;
|
||||
for bound in unknowns {
|
||||
if needs_bound_outcome(self.journal.connection(), &bound)? {
|
||||
let bytes: Vec<u8> = self.journal.connection().query_row(
|
||||
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![
|
||||
bound.job_id.to_string(),
|
||||
bound.attempt_id.to_string()
|
||||
],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
self.record(
|
||||
generation,
|
||||
&JobOffer::decode(bytes.as_slice())?,
|
||||
&bound.grant_id.to_string(),
|
||||
bound.outcome,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
if expected != generation {
|
||||
self.close_gate();
|
||||
return Ok(());
|
||||
}
|
||||
self.send_sync(
|
||||
channel,
|
||||
generation,
|
||||
Some(&rmm_agent_windows::helper_runtime::map_snapshot(
|
||||
&snapshot.raw,
|
||||
)),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
HelperControl::Apply(expected, task) => {
|
||||
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
||||
let snapshot = task.await??;
|
||||
self.helper_anchor = snapshot.no_start_anchor;
|
||||
self.sync
|
||||
.as_mut()
|
||||
.context("SYNC_NOT_CONFIGURED")?
|
||||
.confirm_helper(&snapshot)?;
|
||||
.confirm_helper(&rmm_agent_windows::helper_runtime::map_snapshot(
|
||||
&snapshot.raw,
|
||||
))?;
|
||||
self.open_fence(generation)?;
|
||||
}
|
||||
}
|
||||
@@ -493,6 +612,7 @@ impl Agent {
|
||||
&& self.helper_control.is_none()
|
||||
&& !self.prepared
|
||||
&& self.running.is_none()
|
||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||
&& self.helper_probe_at.elapsed() >= Duration::from_secs(30)
|
||||
{
|
||||
let agent = self.identity.agent_id.clone();
|
||||
@@ -516,14 +636,45 @@ impl Agent {
|
||||
})
|
||||
}
|
||||
fn ready(&self, generation: u64) -> bool {
|
||||
self.ready_identity(generation, false)
|
||||
}
|
||||
fn ready_identity(&self, generation: u64, helper: bool) -> bool {
|
||||
#[cfg(feature = "system-helper")]
|
||||
if helper
|
||||
&& !self.helper_anchor.as_ref().is_some_and(|a| {
|
||||
a.connection_generation() == generation && a.boot_id().to_string() == self.boot
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.config.enable_jobs
|
||||
&& !self.helper_control_pending()
|
||||
&& self.channel(generation).is_ok_and(|channel| {
|
||||
self.sync
|
||||
.as_ref()
|
||||
.is_some_and(|sync| sync.can_start(&channel, false, &self.clock))
|
||||
.is_some_and(|sync| sync.can_start(&channel, helper, &self.clock))
|
||||
})
|
||||
}
|
||||
fn supported_job(&self, job: &rmm_domain::jobs::JobPayload) -> bool {
|
||||
if native::require_local_service().is_err() {
|
||||
return false;
|
||||
}
|
||||
if rmm_agent_windows::actions::supported(job) {
|
||||
return true;
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file()
|
||||
&& self
|
||||
.helper
|
||||
.as_ref()
|
||||
.is_some_and(|helper| helper.supports_system(job))
|
||||
}
|
||||
#[cfg(not(feature = "system-helper"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
fn capabilities(&self) -> Vec<String> {
|
||||
let mut caps: Vec<String> = CAPABILITIES.iter().map(|c| c.to_string()).collect();
|
||||
if self.config.enable_jobs
|
||||
@@ -559,6 +710,72 @@ impl Agent {
|
||||
let Some(binding) = self.binding.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
#[cfg(feature = "system-helper")]
|
||||
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
||||
let mut intents = rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
||||
let pending = intents.pending()?;
|
||||
if let Some(intent) = pending.first() {
|
||||
let p = intent.permit()?;
|
||||
let incompatible:bool=self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown','awaiting_reboot') AND NOT(job=?1 AND attempt=?2))",rusqlite::params![p.job_id.to_string(),p.attempt_id.to_string()],|r|r.get(0))?;
|
||||
ensure!(!incompatible, "SYSTEM_PENDING_INTENT_CONFLICT");
|
||||
binding.verify_historical_for_denial_with(
|
||||
&intent.request,
|
||||
reconcile::HelperIdentity::System,
|
||||
&rmm_agent_core::crypto::DalekVerifier,
|
||||
)?;
|
||||
let bytes: Vec<u8> = self.journal.connection().query_row(
|
||||
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![p.job_id.to_string(), p.attempt_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
ensure!(
|
||||
offer.payload_bytes == intent.request.job_bytes
|
||||
&& offer.plan_hash == intent.request.plan_hash
|
||||
&& offer.authorization_id == intent.request.authorization_id.to_string(),
|
||||
"SYSTEM_PENDING_INTENT_BINDING"
|
||||
);
|
||||
// Even prepared-only intents are never executed after restart.
|
||||
// Snapshot/explicit bound denial recovery is the only route.
|
||||
intents.check()?;
|
||||
let ledger = self.ledger.as_ref().context("LEDGER_REQUIRED")?;
|
||||
let consumed: bool = ledger.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM consumed WHERE grant_id=?1)",
|
||||
[p.grant_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let actual = if consumed {
|
||||
jobs::recover_interrupted_outcome(
|
||||
ledger,
|
||||
&binding,
|
||||
&offer,
|
||||
&p.grant_id.to_string(),
|
||||
)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(actual) = &actual {
|
||||
let bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
||||
job_id: p.job_id,
|
||||
attempt_id: p.attempt_id,
|
||||
grant_id: p.grant_id,
|
||||
outcome: actual.clone(),
|
||||
};
|
||||
if !needs_bound_outcome(self.journal.connection(), &bound)? {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&p.grant_id.to_string(),
|
||||
actual.unwrap_or_else(|| {
|
||||
Outcome::unknown("SYSTEM_DELEGATION_INTERRUPTED_REQUIRES_HELPER_EVIDENCE")
|
||||
}),
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let interrupted = jobs::interrupted(
|
||||
self.journal.connection(),
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
@@ -610,9 +827,38 @@ impl Agent {
|
||||
if let Some(helper) = self.helper.clone() {
|
||||
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
||||
let boot = self.channel(generation)?.boot_id;
|
||||
let instance = Uuid::parse_str(&self.instance)?;
|
||||
let clock = self.clock.clone();
|
||||
self.helper_control = Some(HelperControl::Snapshot(
|
||||
generation,
|
||||
tokio::spawn(async move { helper.snapshot(generation, boot).await }),
|
||||
tokio::spawn(async move {
|
||||
let denial_resume = tokio::task::spawn_blocking(move || -> Result<_> {
|
||||
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
||||
use rmm_agent_windows::helper_runtime::intent::{Phase, Store};
|
||||
let mut store = Store::open(true)?;
|
||||
if let Some(intent) = store.pending()?.first() {
|
||||
let (_, phase) = store.load(intent.id)?;
|
||||
if phase == Phase::KnownNoDispatch
|
||||
&& intent.agent_instance != instance
|
||||
&& store.claim_denial_resume(
|
||||
intent.id,
|
||||
instance,
|
||||
boot,
|
||||
clock.now()?,
|
||||
)?
|
||||
{
|
||||
return Ok(Some(intent.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
})
|
||||
.await??;
|
||||
if let Some(intent) = denial_resume {
|
||||
let _denial = helper.resume_denial(intent).await;
|
||||
}
|
||||
helper.snapshot(generation, boot).await
|
||||
}),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
@@ -694,6 +940,14 @@ impl Agent {
|
||||
.deadline
|
||||
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
||||
self.fence.generation.store(generation, Ordering::SeqCst);
|
||||
self.fence.helper_generation.store(
|
||||
if self.ready_identity(generation, true) {
|
||||
generation
|
||||
} else {
|
||||
0
|
||||
},
|
||||
Ordering::SeqCst,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -708,6 +962,19 @@ impl Agent {
|
||||
&& running.grant == grant,
|
||||
"EXECUTOR_COMPLETION_BINDING"
|
||||
);
|
||||
if grant.execution_identity == "system" {
|
||||
// No local consumption or definitive result from an IPC outcome.
|
||||
// Import only a fresh raw helper snapshot before the next Sync.
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
&grant.grant_id.to_string(),
|
||||
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"),
|
||||
)?;
|
||||
self.running = None;
|
||||
self.close_gate();
|
||||
return Ok(());
|
||||
}
|
||||
ledger::persist_outcome(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&grant,
|
||||
@@ -725,6 +992,9 @@ impl Agent {
|
||||
&outcome,
|
||||
)?;
|
||||
self.running = None;
|
||||
if grant.execution_identity == "system" {
|
||||
self.close_gate();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn permit(&mut self, generation: u64, permit: ExecutionPermit) -> Result<()> {
|
||||
@@ -764,10 +1034,82 @@ impl Agent {
|
||||
return Ok(());
|
||||
}
|
||||
let (job, grant) = verified_window(&binding, &request, &self.clock)?;
|
||||
ensure!(self.supported_job(&job), "UNSUPPORTED_CAPABILITY");
|
||||
let system = job.execution_identity == "system";
|
||||
ensure!(
|
||||
rmm_agent_windows::actions::supported(&job),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
self.ready_identity(generation, system),
|
||||
"SYSTEM_HELPER_SYNC_REQUIRED"
|
||||
);
|
||||
let cancelled: bool = self.journal.connection().query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
#[cfg(feature = "system-helper")]
|
||||
if system {
|
||||
use rmm_agent_windows::helper_runtime::intent::{Intent, Store};
|
||||
let helper = self
|
||||
.helper
|
||||
.clone()
|
||||
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
||||
let intent = Intent {
|
||||
schema_version: 1,
|
||||
id: Uuid::new_v4(),
|
||||
request: request.clone(),
|
||||
anchor: self
|
||||
.helper_anchor
|
||||
.clone()
|
||||
.context("SYSTEM_READY_ANCHOR_REQUIRED")?,
|
||||
original_boot_id: Uuid::parse_str(&self.boot)?,
|
||||
agent_instance: Uuid::parse_str(&self.instance)?,
|
||||
connection_generation: generation,
|
||||
recorded_at: self.clock.now()?,
|
||||
policy_sha256: helper.pin_hash().into(),
|
||||
helper_binary_sha256: helper.helper_binary_hash().into(),
|
||||
};
|
||||
let mut store = Store::open(true)?;
|
||||
store.prepare(&intent)?;
|
||||
if cancelled {
|
||||
store.decide(intent.id, false)?;
|
||||
}
|
||||
let cancelled = Arc::new(AtomicBool::new(cancelled));
|
||||
let worker_cancel = cancelled.clone();
|
||||
let clock = self.clock.clone();
|
||||
let fence = self.fence.clone();
|
||||
let completed = self.completed_tx.clone();
|
||||
let work_offer = offer.clone();
|
||||
let work_grant = grant.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
let _result = helper
|
||||
.delegate(intent, move || {
|
||||
ensure!(
|
||||
!worker_cancel.load(Ordering::SeqCst),
|
||||
"CANCELLED_BEFORE_DISPATCH"
|
||||
);
|
||||
ensure!(
|
||||
fence.allows_identity(generation, true, &clock),
|
||||
"FENCED_BEFORE_DISPATCH"
|
||||
);
|
||||
verified_window(&binding, &request, &clock)?;
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
let _ = completed
|
||||
.send((
|
||||
work_offer,
|
||||
work_grant,
|
||||
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"),
|
||||
))
|
||||
.await;
|
||||
});
|
||||
self.running = Some(Running {
|
||||
offer,
|
||||
grant,
|
||||
cancel: cancelled,
|
||||
worker,
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
let started_at = self.clock.now()?;
|
||||
let ledger = self.ledger.as_mut().context("LEDGER_REQUIRED")?;
|
||||
ledger::begin_execution(
|
||||
@@ -777,11 +1119,6 @@ impl Agent {
|
||||
Uuid::parse_str(&self.boot)?,
|
||||
started_at,
|
||||
)?;
|
||||
let cancelled: bool = self.journal.connection().query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if cancelled {
|
||||
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
||||
ledger::persist_outcome(
|
||||
@@ -1016,7 +1353,7 @@ impl Agent {
|
||||
if let Some(offer)=jobs::next_at(self.journal.connection(),now)? {
|
||||
let job=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.job(&offer.payload_bytes)?;
|
||||
if now>=job.start_before {self.record(generation,&offer,"",jobs::no_start("expired","START_DEADLINE_EXPIRED"))?;needs_sync=true;}
|
||||
else if now-chrono::Duration::seconds(2)>=job.not_before {
|
||||
else if now-chrono::Duration::seconds(2)>=job.not_before && self.ready_identity(generation,job.execution_identity=="system") {
|
||||
jobs::reserve_request(self.journal.connection(),&offer)?;
|
||||
let request=self.envelope(generation,Payload::ExecutionPermitRequest(ExecutionPermitRequest{job_id:offer.job_id,attempt_id:offer.attempt_id,job_payload_hash:offer.job_payload_hash,assignment_version:job.assignment_version}))?;
|
||||
channel.send(&request).await?;
|
||||
@@ -1039,8 +1376,9 @@ impl Agent {
|
||||
ensure!(self.config.enable_jobs,"UNSUPPORTED_EXECUTION_CAPABILITY");
|
||||
let binding=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.clone();
|
||||
let accepted=self.envelope(generation,Payload::JobAccepted(JobAccepted{job_id:offer.job_id.clone(),attempt_id:offer.attempt_id.clone(),job_payload_hash:offer.job_payload_hash.clone(),journal_proof:rmm_domain::jobs::hash(&offer.encode_to_vec())}))?;
|
||||
let supported=self.supported_job(&binding.job(&offer.payload_bytes)?);
|
||||
let tx=self.journal.connection_mut().transaction()?;
|
||||
let fresh=jobs::accept(&tx,&binding,&offer,rmm_agent_windows::actions::supported)?;
|
||||
let fresh=jobs::accept(&tx,&binding,&offer,|_|supported)?;
|
||||
jobs::event(&tx,&accepted)?;tx.commit()?;needs_sync|=fresh;
|
||||
},
|
||||
Some(Payload::ExecutionPermit(permit))=>self.permit(generation,permit)?,
|
||||
@@ -1049,6 +1387,11 @@ impl Agent {
|
||||
if let Some((bytes,state,grant))=row {
|
||||
self.journal.connection().execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2",rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
||||
if let Some(running)=&self.running && running.offer.job_id==cancel.job_id && running.offer.attempt_id==cancel.attempt_id {running.cancel.store(true,Ordering::SeqCst);}
|
||||
#[cfg(feature="system-helper")]
|
||||
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
||||
let mut store=rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
||||
if let Some(intent)=store.pending()?.first() && intent.permit()?.job_id.to_string()==cancel.job_id && intent.permit()?.attempt_id.to_string()==cancel.attempt_id {store.cancel_before_dispatch(intent.id)?;}
|
||||
}
|
||||
if state=="accepted" {self.record(generation,&JobOffer::decode(bytes.as_slice())?,&grant,jobs::no_start("cancelled","CANCELLED_BEFORE_START"))?;needs_sync=true;}
|
||||
}
|
||||
let ack=self.envelope(generation,Payload::Ack(Ack{message_id:message.message_id}))?;channel.send(&ack).await?;
|
||||
@@ -1158,6 +1501,8 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_control:None,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_anchor:None,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_checked:false,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_refused:false,
|
||||
@@ -1242,6 +1587,90 @@ pub fn main() -> Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[cfg(feature = "system-helper")]
|
||||
#[test]
|
||||
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
|
||||
let path = std::env::temp_dir().join(format!("windows-bound-unknown-{}", Uuid::new_v4()));
|
||||
fs::create_dir(&path).unwrap();
|
||||
let journal = Journal::open(&path).unwrap();
|
||||
let db = journal.connection();
|
||||
let job = Uuid::new_v4();
|
||||
let attempt = Uuid::new_v4();
|
||||
let grant = Uuid::new_v4();
|
||||
let offer = JobOffer {
|
||||
job_id: job.to_string(),
|
||||
attempt_id: attempt.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'unknown',1,?4)",rusqlite::params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),grant.to_string()]).unwrap();
|
||||
let original = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(jobs::result(
|
||||
&offer,
|
||||
1,
|
||||
&grant.to_string(),
|
||||
Outcome::unknown("GENERIC_REPLY_LOST"),
|
||||
))),
|
||||
..Default::default()
|
||||
};
|
||||
jobs::event(db, &original).unwrap();
|
||||
let mut actual = Outcome::unknown("SYSTEM_PREVIOUS_EFFECT_UNRESOLVED");
|
||||
actual.evidence = serde_json::json!({"retained_helper_evidence":true});
|
||||
let bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
||||
job_id: job,
|
||||
attempt_id: attempt,
|
||||
grant_id: grant,
|
||||
outcome: actual.clone(),
|
||||
};
|
||||
// Represents a crash after actual raw-helper outcome persistence but
|
||||
// before the corresponding journal write, even if mirror receipt exists.
|
||||
assert!(needs_bound_outcome(db, &bound).unwrap());
|
||||
assert_eq!(jobs::events(db).unwrap().len(), 1);
|
||||
let revision = jobs::transition(db, &offer, "unknown", &grant.to_string()).unwrap();
|
||||
let repaired = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(jobs::result(
|
||||
&offer,
|
||||
revision,
|
||||
&grant.to_string(),
|
||||
actual,
|
||||
))),
|
||||
..Default::default()
|
||||
};
|
||||
jobs::event(db, &repaired).unwrap();
|
||||
assert!(!needs_bound_outcome(db, &bound).unwrap());
|
||||
assert_eq!(jobs::events(db).unwrap().len(), 2); // retention is never ACK
|
||||
let mut terminal = bound.outcome.clone();
|
||||
terminal.state = "failed".into();
|
||||
let terminal_bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
||||
job_id: job,
|
||||
attempt_id: attempt,
|
||||
grant_id: grant,
|
||||
outcome: terminal.clone(),
|
||||
};
|
||||
let revision = jobs::transition(db, &offer, "failed", &grant.to_string()).unwrap();
|
||||
jobs::event(
|
||||
db,
|
||||
&Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(jobs::result(
|
||||
&offer,
|
||||
revision,
|
||||
&grant.to_string(),
|
||||
terminal,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
// Two restarts after the terminal journal committed but before a
|
||||
// mirror receipt may never request terminal -> terminal transitions.
|
||||
assert!(!needs_bound_outcome(db, &terminal_bound).unwrap());
|
||||
assert!(!needs_bound_outcome(db, &terminal_bound).unwrap());
|
||||
assert_eq!(jobs::events(db).unwrap().len(), 3);
|
||||
drop(journal);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn old_passive_configuration_cannot_enable_jobs_or_read_the_clock() {
|
||||
let config:Config=serde_json::from_value(serde_json::json!({"server_name":"example.invalid","connect_address":"127.0.0.1:443","ca_file":"C:/fixture/ca.pem","state_directory":"C:/fixture/state","enrollment_token_file":"C:/fixture/token","transport":"auto"})).unwrap();
|
||||
@@ -1319,9 +1748,13 @@ mod tests {
|
||||
assert!(!fence.allows(1, &Clock(now)));
|
||||
fence.generation.store(1, Ordering::SeqCst);
|
||||
assert!(fence.allows(1, &Clock(now)));
|
||||
assert!(!fence.allows_identity(1, true, &Clock(now)));
|
||||
fence.helper_generation.store(1, Ordering::SeqCst);
|
||||
assert!(fence.allows_identity(1, true, &Clock(now)));
|
||||
assert!(!fence.allows(2, &Clock(now)));
|
||||
assert!(!fence.allows(1, &Clock(now + chrono::Duration::seconds(1))));
|
||||
fence.close();
|
||||
assert!(!fence.allows_identity(1, true, &Clock(now)));
|
||||
assert!(!fence.allows(1, &Clock(now)));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user